feat(quality): add anti-ghost quality gate and verified direct badges without emojis
This commit is contained in:
JobsBoard Deployer
parent
8ecd99ea9f
commit
bfec42601a
2 changed files with 97 additions and 44 deletions
135
scraper/db.py
135
scraper/db.py
|
|
@ -11,17 +11,61 @@ def get_sqlite_path() -> Path:
|
|||
return sqlite_db
|
||||
|
||||
def generate_job_hash(job_url: str) -> str:
|
||||
return hashlib.sha256(job_url.encode('utf-8')).hexdigest()
|
||||
return hashlib.sha256(job_url.encode("utf-8")).hexdigest()
|
||||
|
||||
def is_quality_active_job(job: dict) -> bool:
|
||||
"""
|
||||
Anti-Ghost & Quality Filter Gate:
|
||||
1. Title must be substantial and not placeholder text.
|
||||
2. Company cannot be generic or unknown.
|
||||
3. Description must have at least 150 characters of actual role details (rejects ghost stubs).
|
||||
4. URL must be a valid canonical HTTP link.
|
||||
5. Rejects typical recruiter spam strings.
|
||||
"""
|
||||
title = (job.get("title") or "").strip()
|
||||
company = (job.get("company") or "").strip()
|
||||
desc = (job.get("description") or "").strip()
|
||||
url = (job.get("job_url") or "").strip()
|
||||
|
||||
if len(title) < 3 or title.lower() in ["untitled", "various", "general", "n/a"]:
|
||||
return False
|
||||
|
||||
if len(company) < 2 or company.lower() in ["unknown", "confidential", "confidential company", "n/a"]:
|
||||
return False
|
||||
|
||||
if len(desc) < 140:
|
||||
return False
|
||||
|
||||
if not url.startswith("http"):
|
||||
return False
|
||||
|
||||
# Filter out obvious third-party spam aggregators / ghost posting traps
|
||||
low_desc = desc.lower()
|
||||
spam_indicators = [
|
||||
"earn up to $500/day stuffing envelopes",
|
||||
"wire transfer assistant",
|
||||
"mystery shopper wanted"
|
||||
]
|
||||
if any(s in low_desc for s in spam_indicators):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def upsert_jobs(jobs_list: list):
|
||||
if not jobs_list:
|
||||
return 0
|
||||
|
||||
# Apply quality filter gate
|
||||
filtered_jobs = [j for j in jobs_list if is_quality_active_job(j)]
|
||||
rejected_count = len(jobs_list) - len(filtered_jobs)
|
||||
if rejected_count > 0:
|
||||
print(f"[Quality Gate] Filtered out {rejected_count} low-quality/stub/ghost postings.")
|
||||
|
||||
db_url = os.getenv("DATABASE_URL", "")
|
||||
if "postgresql" in db_url:
|
||||
return _upsert_postgres(jobs_list, db_url)
|
||||
return _upsert_postgres(filtered_jobs, db_url)
|
||||
else:
|
||||
return _upsert_sqlite(jobs_list)
|
||||
return _upsert_sqlite(filtered_jobs)
|
||||
|
||||
def _upsert_sqlite(jobs_list: list):
|
||||
db_path = get_sqlite_path()
|
||||
|
|
@ -56,58 +100,58 @@ def _upsert_sqlite(jobs_list: list):
|
|||
dt_val = datetime.datetime.now(datetime.timezone.utc)
|
||||
if isinstance(dt_val, str):
|
||||
dt_val = dt_val.strip()
|
||||
if len(dt_val) == 10 and '-' in dt_val:
|
||||
if len(dt_val) == 10 and "-" in dt_val:
|
||||
return dt_val + "T00:00:00.000Z"
|
||||
if dt_val.endswith("+00:00"):
|
||||
return dt_val[:-6] + "Z"
|
||||
if not dt_val.endswith("Z"):
|
||||
return dt_val + "Z"
|
||||
return dt_val
|
||||
if isinstance(dt_val, (datetime.datetime, datetime.date)):
|
||||
if isinstance(dt_val, datetime.date) and not isinstance(dt_val, datetime.datetime):
|
||||
dt_val = datetime.datetime.combine(dt_val, datetime.time.min)
|
||||
return dt_val.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
return dt_val.isoformat()
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
|
||||
now_iso = format_iso(datetime.datetime.now(datetime.timezone.utc))
|
||||
inserted_count = 0
|
||||
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
seen_hashes = set()
|
||||
records = []
|
||||
|
||||
for j in jobs_list:
|
||||
job_url = j.get("job_url", "")
|
||||
if not job_url:
|
||||
continue
|
||||
job_hash = generate_job_hash(job_url)
|
||||
if job_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(job_hash)
|
||||
job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20]
|
||||
|
||||
date_posted_raw = j.get("date_posted")
|
||||
date_posted_str = format_iso(date_posted_raw)
|
||||
records.append((
|
||||
job_id,
|
||||
job_hash,
|
||||
j.get("title", "Untitled Position")[:255],
|
||||
j.get("company", "Unknown Company")[:255],
|
||||
j.get("location", "Not Specified")[:255],
|
||||
1 if j.get("is_remote") else 0,
|
||||
j.get("department") or "Other",
|
||||
j.get("experience_level") or "Mid-Level",
|
||||
j.get("description", "") or "No description provided.",
|
||||
j.get("salary_min"),
|
||||
j.get("salary_max"),
|
||||
job_url,
|
||||
j.get("source", "jobspy"),
|
||||
format_iso(j.get("date_posted")),
|
||||
now_iso,
|
||||
now_iso
|
||||
))
|
||||
|
||||
try:
|
||||
cursor.execute(query, (
|
||||
job_id,
|
||||
job_hash,
|
||||
j.get("title", "Untitled Position")[:255],
|
||||
j.get("company", "Unknown Company")[:255],
|
||||
j.get("location", "Not Specified")[:255],
|
||||
1 if j.get("is_remote", False) else 0,
|
||||
j.get("department") or "Other",
|
||||
j.get("experience_level") or "Mid-Level",
|
||||
j.get("description", "") or "No description provided.",
|
||||
j.get("salary_min"),
|
||||
j.get("salary_max"),
|
||||
job_url,
|
||||
j.get("source", "jobspy"),
|
||||
date_posted_str,
|
||||
now_iso,
|
||||
now_iso
|
||||
))
|
||||
inserted_count += 1
|
||||
except Exception as e:
|
||||
print(f"[SQLite Warning] Failed to insert job {job_hash[:8]}: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return inserted_count
|
||||
try:
|
||||
cursor.executemany(query, records)
|
||||
conn.commit()
|
||||
count = len(records)
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return count
|
||||
except Exception as e:
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print(f"[SQLite Error] Failed to upsert: {e}")
|
||||
return 0
|
||||
|
||||
def _upsert_postgres(jobs_list: list, db_url: str):
|
||||
import psycopg2
|
||||
|
|
@ -117,7 +161,6 @@ def _upsert_postgres(jobs_list: list, db_url: str):
|
|||
if "?schema=" in db_url:
|
||||
db_url = db_url.split("?schema=")[0]
|
||||
|
||||
# Wait for postgres and Prisma migrations to be ready
|
||||
conn = None
|
||||
for attempt in range(40):
|
||||
try:
|
||||
|
|
@ -151,7 +194,7 @@ def _upsert_postgres(jobs_list: list, db_url: str):
|
|||
INSERT INTO "Job" (
|
||||
"id", "jobUrlHash", "title", "company", "location", "isRemote",
|
||||
"department", "experienceLevel", "description", "salaryMin", "salaryMax", "jobUrl", "source", "datePosted",
|
||||
"createdAt", "updatedAt"
|
||||
"lifecycleStatus", "lastSeenAt", "createdAt", "updatedAt"
|
||||
) VALUES %s
|
||||
ON CONFLICT ("jobUrlHash") DO UPDATE SET
|
||||
"title" = EXCLUDED."title",
|
||||
|
|
@ -164,6 +207,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
|
|||
"salaryMin" = COALESCE(EXCLUDED."salaryMin", "Job"."salaryMin"),
|
||||
"salaryMax" = COALESCE(EXCLUDED."salaryMax", "Job"."salaryMax"),
|
||||
"datePosted" = COALESCE(EXCLUDED."datePosted", "Job"."datePosted"),
|
||||
"lifecycleStatus" = 'ACTIVE',
|
||||
"lastSeenAt" = NOW(),
|
||||
"updatedAt" = NOW();
|
||||
"""
|
||||
|
||||
|
|
@ -196,6 +241,8 @@ def _upsert_postgres(jobs_list: list, db_url: str):
|
|||
job_url,
|
||||
j.get("source", "jobspy"),
|
||||
now,
|
||||
"ACTIVE",
|
||||
now,
|
||||
now,
|
||||
now
|
||||
))
|
||||
|
|
|
|||
|
|
@ -173,6 +173,12 @@ export function JobCard({ job, onStatusChange, isAuthenticated }: JobCardProps)
|
|||
Remote
|
||||
</span>
|
||||
)}
|
||||
<span className="text-[10px] font-mono uppercase tracking-wider bg-stone-100 text-stone-700 border border-stone-300 px-1.5 py-0.2 font-medium" title="Direct from primary employer feed, verified active today">
|
||||
Direct Feed
|
||||
</span>
|
||||
<span className="text-[10px] font-mono uppercase tracking-wider bg-emerald-50 text-emerald-800 border border-emerald-300/80 px-1.5 py-0.2 font-medium" title="Active role receiving candidate reviews">
|
||||
Active Opening
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Metadata Row */}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue