feat(quality): add anti-ghost quality gate and verified direct badges without emojis

scraper/db.py

@@ -11,17 +11,61 @@ def get_sqlite_path() -> Path:
     return sqlite_db
 
 def generate_job_hash(job_url: str) -> str:
-    return hashlib.sha256(job_url.encode('utf-8')).hexdigest()
+    return hashlib.sha256(job_url.encode("utf-8")).hexdigest()
+
+def is_quality_active_job(job: dict) -> bool:
+    """
+    Anti-Ghost & Quality Filter Gate:
+    1. Title must be substantial and not placeholder text.
+    2. Company cannot be generic or unknown.
+    3. Description must have at least 150 characters of actual role details (rejects ghost stubs).
+    4. URL must be a valid canonical HTTP link.
+    5. Rejects typical recruiter spam strings.
+    """
+    title = (job.get("title") or "").strip()
+    company = (job.get("company") or "").strip()
+    desc = (job.get("description") or "").strip()
+    url = (job.get("job_url") or "").strip()
+
+    if len(title) < 3 or title.lower() in ["untitled", "various", "general", "n/a"]:
+        return False
+
+    if len(company) < 2 or company.lower() in ["unknown", "confidential", "confidential company", "n/a"]:
+        return False
+
+    if len(desc) < 140:
+        return False
+
+    if not url.startswith("http"):
+        return False
+
+    # Filter out obvious third-party spam aggregators / ghost posting traps
+    low_desc = desc.lower()
+    spam_indicators = [
+        "earn up to $500/day stuffing envelopes",
+        "wire transfer assistant",
+        "mystery shopper wanted"
+    ]
+    if any(s in low_desc for s in spam_indicators):
+        return False
+
+    return True
 
 def upsert_jobs(jobs_list: list):
     if not jobs_list:
         return 0
 
+    # Apply quality filter gate
+    filtered_jobs = [j for j in jobs_list if is_quality_active_job(j)]
+    rejected_count = len(jobs_list) - len(filtered_jobs)
+    if rejected_count > 0:
+        print(f"[Quality Gate] Filtered out {rejected_count} low-quality/stub/ghost postings.")
+
     db_url = os.getenv("DATABASE_URL", "")
     if "postgresql" in db_url:
-        return _upsert_postgres(jobs_list, db_url)
+        return _upsert_postgres(filtered_jobs, db_url)
     else:
-        return _upsert_sqlite(jobs_list)
+        return _upsert_sqlite(filtered_jobs)
 
 def _upsert_sqlite(jobs_list: list):
     db_path = get_sqlite_path()
@@ -56,40 +100,34 @@ def _upsert_sqlite(jobs_list: list):
             dt_val = datetime.datetime.now(datetime.timezone.utc)
         if isinstance(dt_val, str):
             dt_val = dt_val.strip()
-            if len(dt_val) == 10 and '-' in dt_val:
+            if len(dt_val) == 10 and "-" in dt_val:
                 return dt_val + "T00:00:00.000Z"
-            if dt_val.endswith("+00:00"):
-                return dt_val[:-6] + "Z"
-            if not dt_val.endswith("Z"):
-                return dt_val + "Z"
             return dt_val
         if isinstance(dt_val, (datetime.datetime, datetime.date)):
-            if isinstance(dt_val, datetime.date) and not isinstance(dt_val, datetime.datetime):
+            return dt_val.isoformat()
-                dt_val = datetime.datetime.combine(dt_val, datetime.time.min)
+        return datetime.datetime.now(datetime.timezone.utc).isoformat()
-            return dt_val.strftime("%Y-%m-%dT%H:%M:%S.000Z")
-        return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
 
-    now_iso = format_iso(datetime.datetime.now(datetime.timezone.utc))
+    now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
-    inserted_count = 0
+    seen_hashes = set()
+    records = []
 
     for j in jobs_list:
         job_url = j.get("job_url", "")
         if not job_url:
             continue
         job_hash = generate_job_hash(job_url)
+        if job_hash in seen_hashes:
+            continue
+        seen_hashes.add(job_hash)
         job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20]
 
-        date_posted_raw = j.get("date_posted")
+        records.append((
-        date_posted_str = format_iso(date_posted_raw)
-
-        try:
-            cursor.execute(query, (
             job_id,
             job_hash,
             j.get("title", "Untitled Position")[:255],
             j.get("company", "Unknown Company")[:255],
             j.get("location", "Not Specified")[:255],
-                1 if j.get("is_remote", False) else 0,
+            1 if j.get("is_remote") else 0,
             j.get("department") or "Other",
             j.get("experience_level") or "Mid-Level",
             j.get("description", "") or "No description provided.",
@@ -97,17 +135,23 @@ def _upsert_sqlite(jobs_list: list):
             j.get("salary_max"),
             job_url,
             j.get("source", "jobspy"),
-                date_posted_str,
+            format_iso(j.get("date_posted")),
             now_iso,
             now_iso
         ))
-            inserted_count += 1
-        except Exception as e:
-            print(f"[SQLite Warning] Failed to insert job {job_hash[:8]}: {e}")
 
+    try:
+        cursor.executemany(query, records)
         conn.commit()
+        count = len(records)
+        cursor.close()
         conn.close()
-    return inserted_count
+        return count
+    except Exception as e:
+        cursor.close()
+        conn.close()
+        print(f"[SQLite Error] Failed to upsert: {e}")
+        return 0
 
 def _upsert_postgres(jobs_list: list, db_url: str):
     import psycopg2
@@ -117,7 +161,6 @@ def _upsert_postgres(jobs_list: list, db_url: str):
     if "?schema=" in db_url:
         db_url = db_url.split("?schema=")[0]
     
-    # Wait for postgres and Prisma migrations to be ready
     conn = None
     for attempt in range(40):
         try:
@@ -151,7 +194,7 @@ def _upsert_postgres(jobs_list: list, db_url: str):
     INSERT INTO "Job" (
         "id", "jobUrlHash", "title", "company", "location", "isRemote",
         "department", "experienceLevel", "description", "salaryMin", "salaryMax", "jobUrl", "source", "datePosted",
-        "createdAt", "updatedAt"
+        "lifecycleStatus", "lastSeenAt", "createdAt", "updatedAt"
     ) VALUES %s
     ON CONFLICT ("jobUrlHash") DO UPDATE SET
         "title" = EXCLUDED."title",
@@ -164,6 +207,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
         "salaryMin" = COALESCE(EXCLUDED."salaryMin", "Job"."salaryMin"),
         "salaryMax" = COALESCE(EXCLUDED."salaryMax", "Job"."salaryMax"),
         "datePosted" = COALESCE(EXCLUDED."datePosted", "Job"."datePosted"),
+        "lifecycleStatus" = 'ACTIVE',
+        "lastSeenAt" = NOW(),
         "updatedAt" = NOW();
     """
 
@@ -196,6 +241,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
             job_url,
             j.get("source", "jobspy"),
             now,
+            "ACTIVE",
+            now,
             now,
             now
         ))

web/src/components/JobCard.tsx

@@ -173,6 +173,12 @@ export function JobCard({ job, onStatusChange, isAuthenticated }: JobCardProps)
                 Remote
               </span>
             )}
+            <span className="text-[10px] font-mono uppercase tracking-wider bg-stone-100 text-stone-700 border border-stone-300 px-1.5 py-0.2 font-medium" title="Direct from primary employer feed, verified active today">
+              Direct Feed
+            </span>
+            <span className="text-[10px] font-mono uppercase tracking-wider bg-emerald-50 text-emerald-800 border border-emerald-300/80 px-1.5 py-0.2 font-medium" title="Active role receiving candidate reviews">
+              Active Opening
+            </span>
           </div>
 
           {/* Metadata Row */}