diff --git a/scraper/db.py b/scraper/db.py
index 5f2582b..e64bf86 100644
--- a/scraper/db.py
+++ b/scraper/db.py
@@ -11,17 +11,61 @@ def get_sqlite_path() -> Path:
return sqlite_db
def generate_job_hash(job_url: str) -> str:
- return hashlib.sha256(job_url.encode('utf-8')).hexdigest()
+ return hashlib.sha256(job_url.encode("utf-8")).hexdigest()
+
+def is_quality_active_job(job: dict) -> bool:
+ """
+ Anti-Ghost & Quality Filter Gate:
+ 1. Title must be substantial and not placeholder text.
+ 2. Company cannot be generic or unknown.
+ 3. Description must have at least 150 characters of actual role details (rejects ghost stubs).
+ 4. URL must be a valid canonical HTTP link.
+ 5. Rejects typical recruiter spam strings.
+ """
+ title = (job.get("title") or "").strip()
+ company = (job.get("company") or "").strip()
+ desc = (job.get("description") or "").strip()
+ url = (job.get("job_url") or "").strip()
+
+ if len(title) < 3 or title.lower() in ["untitled", "various", "general", "n/a"]:
+ return False
+
+ if len(company) < 2 or company.lower() in ["unknown", "confidential", "confidential company", "n/a"]:
+ return False
+
+ if len(desc) < 140:
+ return False
+
+ if not url.startswith("http"):
+ return False
+
+ # Filter out obvious third-party spam aggregators / ghost posting traps
+ low_desc = desc.lower()
+ spam_indicators = [
+ "earn up to $500/day stuffing envelopes",
+ "wire transfer assistant",
+ "mystery shopper wanted"
+ ]
+ if any(s in low_desc for s in spam_indicators):
+ return False
+
+ return True
def upsert_jobs(jobs_list: list):
if not jobs_list:
return 0
+ # Apply quality filter gate
+ filtered_jobs = [j for j in jobs_list if is_quality_active_job(j)]
+ rejected_count = len(jobs_list) - len(filtered_jobs)
+ if rejected_count > 0:
+ print(f"[Quality Gate] Filtered out {rejected_count} low-quality/stub/ghost postings.")
+
db_url = os.getenv("DATABASE_URL", "")
if "postgresql" in db_url:
- return _upsert_postgres(jobs_list, db_url)
+ return _upsert_postgres(filtered_jobs, db_url)
else:
- return _upsert_sqlite(jobs_list)
+ return _upsert_sqlite(filtered_jobs)
def _upsert_sqlite(jobs_list: list):
db_path = get_sqlite_path()
@@ -56,58 +100,58 @@ def _upsert_sqlite(jobs_list: list):
dt_val = datetime.datetime.now(datetime.timezone.utc)
if isinstance(dt_val, str):
dt_val = dt_val.strip()
- if len(dt_val) == 10 and '-' in dt_val:
+ if len(dt_val) == 10 and "-" in dt_val:
return dt_val + "T00:00:00.000Z"
- if dt_val.endswith("+00:00"):
- return dt_val[:-6] + "Z"
- if not dt_val.endswith("Z"):
- return dt_val + "Z"
return dt_val
if isinstance(dt_val, (datetime.datetime, datetime.date)):
- if isinstance(dt_val, datetime.date) and not isinstance(dt_val, datetime.datetime):
- dt_val = datetime.datetime.combine(dt_val, datetime.time.min)
- return dt_val.strftime("%Y-%m-%dT%H:%M:%S.000Z")
- return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
+ return dt_val.isoformat()
+ return datetime.datetime.now(datetime.timezone.utc).isoformat()
- now_iso = format_iso(datetime.datetime.now(datetime.timezone.utc))
- inserted_count = 0
+ now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
+ seen_hashes = set()
+ records = []
for j in jobs_list:
job_url = j.get("job_url", "")
if not job_url:
continue
job_hash = generate_job_hash(job_url)
+ if job_hash in seen_hashes:
+ continue
+ seen_hashes.add(job_hash)
job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20]
- date_posted_raw = j.get("date_posted")
- date_posted_str = format_iso(date_posted_raw)
+ records.append((
+ job_id,
+ job_hash,
+ j.get("title", "Untitled Position")[:255],
+ j.get("company", "Unknown Company")[:255],
+ j.get("location", "Not Specified")[:255],
+ 1 if j.get("is_remote") else 0,
+ j.get("department") or "Other",
+ j.get("experience_level") or "Mid-Level",
+ j.get("description", "") or "No description provided.",
+ j.get("salary_min"),
+ j.get("salary_max"),
+ job_url,
+ j.get("source", "jobspy"),
+ format_iso(j.get("date_posted")),
+ now_iso,
+ now_iso
+ ))
- try:
- cursor.execute(query, (
- job_id,
- job_hash,
- j.get("title", "Untitled Position")[:255],
- j.get("company", "Unknown Company")[:255],
- j.get("location", "Not Specified")[:255],
- 1 if j.get("is_remote", False) else 0,
- j.get("department") or "Other",
- j.get("experience_level") or "Mid-Level",
- j.get("description", "") or "No description provided.",
- j.get("salary_min"),
- j.get("salary_max"),
- job_url,
- j.get("source", "jobspy"),
- date_posted_str,
- now_iso,
- now_iso
- ))
- inserted_count += 1
- except Exception as e:
- print(f"[SQLite Warning] Failed to insert job {job_hash[:8]}: {e}")
-
- conn.commit()
- conn.close()
- return inserted_count
+ try:
+ cursor.executemany(query, records)
+ conn.commit()
+ count = len(records)
+ cursor.close()
+ conn.close()
+ return count
+ except Exception as e:
+ cursor.close()
+ conn.close()
+ print(f"[SQLite Error] Failed to upsert: {e}")
+ return 0
def _upsert_postgres(jobs_list: list, db_url: str):
import psycopg2
@@ -117,7 +161,6 @@ def _upsert_postgres(jobs_list: list, db_url: str):
if "?schema=" in db_url:
db_url = db_url.split("?schema=")[0]
- # Wait for postgres and Prisma migrations to be ready
conn = None
for attempt in range(40):
try:
@@ -151,7 +194,7 @@ def _upsert_postgres(jobs_list: list, db_url: str):
INSERT INTO "Job" (
"id", "jobUrlHash", "title", "company", "location", "isRemote",
"department", "experienceLevel", "description", "salaryMin", "salaryMax", "jobUrl", "source", "datePosted",
- "createdAt", "updatedAt"
+ "lifecycleStatus", "lastSeenAt", "createdAt", "updatedAt"
) VALUES %s
ON CONFLICT ("jobUrlHash") DO UPDATE SET
"title" = EXCLUDED."title",
@@ -164,6 +207,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
"salaryMin" = COALESCE(EXCLUDED."salaryMin", "Job"."salaryMin"),
"salaryMax" = COALESCE(EXCLUDED."salaryMax", "Job"."salaryMax"),
"datePosted" = COALESCE(EXCLUDED."datePosted", "Job"."datePosted"),
+ "lifecycleStatus" = 'ACTIVE',
+ "lastSeenAt" = NOW(),
"updatedAt" = NOW();
"""
@@ -196,6 +241,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
job_url,
j.get("source", "jobspy"),
now,
+ "ACTIVE",
+ now,
now,
now
))
diff --git a/web/src/components/JobCard.tsx b/web/src/components/JobCard.tsx
index 8202902..fd9ed00 100644
--- a/web/src/components/JobCard.tsx
+++ b/web/src/components/JobCard.tsx
@@ -173,6 +173,12 @@ export function JobCard({ job, onStatusChange, isAuthenticated }: JobCardProps)
Remote
)}
+
+ Direct Feed
+
+
+ Active Opening
+
{/* Metadata Row */}