from scrapers.proxy_manager import get_privado_proxy_list, get_random_privado_proxy import os import time import pandas as pd from typing import List, Dict, Any from scrapers.ats_ingestion import determine_experience_level, determine_department def get_jobspy_proxies(): proxies = get_privado_proxy_list() if proxies: return proxies return None def run_jobspy_scrapes() -> List[Dict[Any, Any]]: """ Executes JobSpy searches across nationwide US hubs and remote roles covering major industries: Tech, Finance, Healthcare, Retail, Trades, etc. Gracefully handles Cloudflare/datacenter blocks and proxy rotation. """ collected_jobs = [] try: from jobspy import scrape_jobs except ImportError: print("[JobSpy] python-jobspy is not installed. Skipping JobSpy runner.") return [] proxies = get_jobspy_proxies() if proxies: print(f"[JobSpy] Using configured proxy for JobSpy requests: {proxies[0].split('@')[-1]}") # Prioritize Indeed (reliable without Cloudflare Captcha compared to ZipRecruiter on server IPs) # If proxies are configured, enable zip_recruiter and glassdoor sites = ["indeed"] if proxies: sites.extend(["zip_recruiter", "glassdoor"]) # Target key employment hubs across US states covering all industries us_regions = [ ("New York, NY", ["Finance", "Accountant", "Marketing", "Data Analyst", "Nurse", "Paralegal", "Sales"]), ("Austin, TX", ["Software Engineer", "Project Manager", "Customer Support", "Operations", "Electrician"]), ("San Francisco, CA", ["AI Engineer", "Product Manager", "Graphic Designer", "Recruiter"]), ("Chicago, IL", ["Operations", "Healthcare", "Logistics", "Accountant", "Warehouse", "HR Specialist"]), ("Atlanta, GA", ["IT Support", "Sales", "Supply Chain", "Administrative", "Medical Assistant", "Customer Service"]), ("Seattle, WA", ["Software Developer", "Data Scientist", "Procurement", "Compliance Officer"]), ("Boston, MA", ["Biotech", "Clinical Research", "Healthcare", "Financial Analyst", "Teacher"]), ("Denver, CO", ["Customer Success", "Cybersecurity", "Construction Manager", "Account Executive"]), ("Connecticut", ["Healthcare", "Nurse", "Finance", "Insurance Underwriter", "Manufacturing", "Electrician", "Administrative"]) ] # Nationwide Remote queries across ALL professional disciplines remote_queries = [ # Healthcare & Medical "Medical Biller", "Telehealth Nurse", "Clinical Research Coordinator", "Healthcare Recruiter", # Finance, Accounting & Legal "Staff Accountant", "Financial Analyst", "Bookkeeper", "Paralegal", "Compliance Specialist", "Underwriter", # Sales, Marketing & Customer Support "Customer Support Representative", "Customer Success Manager", "Account Executive", "Digital Marketing Specialist", "Content Writer", # Human Resources & Operations "HR Generalist", "Technical Recruiter", "Executive Assistant", "Operations Coordinator", "Project Coordinator", # Art, Design & Creative "Graphic Designer", "UX Designer", "Video Editor", "Instructional Designer", # Logistics, Supply Chain & Purchasing "Logistics Coordinator", "Supply Chain Analyst", "Procurement Specialist", # Tech & Engineering "Software Engineer", "IT Support Specialist", "Data Analyst", "Systems Administrator", "DevOps Engineer" ] def _execute_scrape(site_list, term, loc, is_rem): # 1. Try with configured proxies if proxies: try: return scrape_jobs( site_name=site_list, search_term=term, location=loc if not is_rem else None, results_wanted=35, hours_old=72, country_indeed="USA", is_remote=is_rem, proxies=proxies ) except Exception as e: # Proxy or site auth error, drop proxy and fall back pass # 2. Fall back to direct Indeed scrape (no proxy, highly reliable) try: return scrape_jobs( site_name=["indeed"], search_term=term, location=loc if not is_rem else None, results_wanted=35, hours_old=72, country_indeed="USA", is_remote=is_rem, proxies=None ) except Exception as e: print(f"[JobSpy Warning] Scrape failed for '{term}' ({loc}): {e}") return None print(f"[JobSpy] Starting Nationwide Regional Scrapes using sites: {sites}...") for loc, queries in us_regions: for query in queries: try: print(f"[JobSpy] Searching {loc}: '{query}'") jobs_df = _execute_scrape(sites, query, loc, False) if isinstance(jobs_df, pd.DataFrame) and not jobs_df.empty: for idx, row in jobs_df.iterrows(): job_url = str(row.get("job_url", "") or row.get("job_url_direct", "")) if not job_url or job_url == "nan": continue loc_val = str(row.get("location", loc) or loc) if loc_val == "nan": loc_val = loc title_str = str(row.get("title", "Untitled")) collected_jobs.append({ "title": title_str, "company": str(row.get("company", "Unknown")), "location": loc_val, "is_remote": False, "department": determine_department(title_str, ""), "experience_level": determine_experience_level(title_str), "description": str(row.get("description", "") or "No description provided."), "salary_min": float(row.get("min_amount")) if pd.notnull(row.get("min_amount")) else None, "salary_max": float(row.get("max_amount")) if pd.notnull(row.get("max_amount")) else None, "job_url": job_url, "source": str(row.get("site", "jobspy")), "date_posted": str(row.get("date_posted")) if pd.notnull(row.get("date_posted")) else None }) time.sleep(1.5) except Exception as e: print(f"[JobSpy Warning] Search '{query}' in '{loc}' failed: {e}") print("[JobSpy] Starting US Nationwide Remote Scrapes...") for query in remote_queries: try: print(f"[JobSpy] Searching US Remote: '{query}'") jobs_df = _execute_scrape(sites, query, "USA", True) if isinstance(jobs_df, pd.DataFrame) and not jobs_df.empty: for idx, row in jobs_df.iterrows(): job_url = str(row.get("job_url", "") or row.get("job_url_direct", "")) if not job_url or job_url == "nan": continue title_str = str(row.get("title", "Untitled")) collected_jobs.append({ "title": title_str, "company": str(row.get("company", "Unknown")), "location": "Remote, USA", "is_remote": True, "department": determine_department(title_str, ""), "experience_level": determine_experience_level(title_str), "description": str(row.get("description", "") or "No description provided."), "salary_min": float(row.get("min_amount")) if pd.notnull(row.get("min_amount")) else None, "salary_max": float(row.get("max_amount")) if pd.notnull(row.get("max_amount")) else None, "job_url": job_url, "source": str(row.get("site", "jobspy")), "date_posted": str(row.get("date_posted")) if pd.notnull(row.get("date_posted")) else None }) time.sleep(1.5) except Exception as e: print(f"[JobSpy Warning] Remote search '{query}' failed: {e}") print(f"[JobSpy] Finished. Total jobs parsed: {len(collected_jobs)}") return collected_jobs