From 16efc58a5cdf208c6e10032cf460d69441b8e188 Mon Sep 17 00:00:00 2001 From: JobsBoard Deployer Date: Sat, 5 Sep 2026 14:35:50 -0400 Subject: [PATCH] Fix entry-level job filtering and perform pre-launch verification --- scraper/scrapers/jobaps_ct.py | 3 ++- scraper/scrapers/jobspy_runner.py | 11 +++++++++-- web/src/app/admin/page.tsx | 8 ++++---- web/src/app/api/jobs/route.ts | 18 +++++++++++++++--- web/src/app/employer/ats/page.tsx | 4 ++-- web/src/app/jobs/[id]/page.tsx | 8 ++++---- web/src/components/JobCard.tsx | 2 +- web/src/components/NotificationBell.tsx | 4 ++-- web/src/lib/sources/adapters/ashby.ts | 8 ++------ web/src/lib/sources/adapters/greenhouse.ts | 8 ++------ web/src/lib/sources/adapters/lever.ts | 8 ++------ web/src/lib/sources/deduplication.ts | 22 +++++++++++++++++++++- 12 files changed, 66 insertions(+), 38 deletions(-) diff --git a/scraper/scrapers/jobaps_ct.py b/scraper/scrapers/jobaps_ct.py index a1b6f99..0245a3f 100644 --- a/scraper/scrapers/jobaps_ct.py +++ b/scraper/scrapers/jobaps_ct.py @@ -1,6 +1,7 @@ import requests from bs4 import BeautifulSoup from typing import List, Dict, Any +from scrapers.ats_ingestion import determine_experience_level def run_ct_jobaps_scrape() -> List[Dict[Any, Any]]: """ @@ -45,7 +46,7 @@ def run_ct_jobaps_scrape() -> List[Dict[Any, Any]]: "location": "Hartford & CT Statewide", "is_remote": False, "department": "Government & Public Services", - "experience_level": "Mid-Level", + "experience_level": determine_experience_level(title), "description": f"Official State of Connecticut position. Application details available at {full_url}", "salary_min": None, "salary_max": None, diff --git a/scraper/scrapers/jobspy_runner.py b/scraper/scrapers/jobspy_runner.py index fd1314e..5f3a739 100644 --- a/scraper/scrapers/jobspy_runner.py +++ b/scraper/scrapers/jobspy_runner.py @@ -3,6 +3,7 @@ import os import time import pandas as pd from typing import List, Dict, Any +from scrapers.ats_ingestion import determine_experience_level, determine_department def get_jobspy_proxies(): proxies = get_privado_proxy_list() @@ -105,11 +106,14 @@ def run_jobspy_scrapes() -> List[Dict[Any, Any]]: if loc_val == "nan": loc_val = loc + title_str = str(row.get("title", "Untitled")) collected_jobs.append({ - "title": str(row.get("title", "Untitled")), + "title": title_str, "company": str(row.get("company", "Unknown")), "location": loc_val, "is_remote": False, + "department": determine_department(title_str, ""), + "experience_level": determine_experience_level(title_str), "description": str(row.get("description", "") or "No description provided."), "salary_min": float(row.get("min_amount")) if pd.notnull(row.get("min_amount")) else None, "salary_max": float(row.get("max_amount")) if pd.notnull(row.get("max_amount")) else None, @@ -133,11 +137,14 @@ def run_jobspy_scrapes() -> List[Dict[Any, Any]]: if not job_url or job_url == "nan": continue + title_str = str(row.get("title", "Untitled")) collected_jobs.append({ - "title": str(row.get("title", "Untitled")), + "title": title_str, "company": str(row.get("company", "Unknown")), "location": "Remote, USA", "is_remote": True, + "department": determine_department(title_str, ""), + "experience_level": determine_experience_level(title_str), "description": str(row.get("description", "") or "No description provided."), "salary_min": float(row.get("min_amount")) if pd.notnull(row.get("min_amount")) else None, "salary_max": float(row.get("max_amount")) if pd.notnull(row.get("max_amount")) else None, diff --git a/web/src/app/admin/page.tsx b/web/src/app/admin/page.tsx index 48242d4..536d2dd 100644 --- a/web/src/app/admin/page.tsx +++ b/web/src/app/admin/page.tsx @@ -759,7 +759,7 @@ export default function AdminPage() { {pendingReviews.length === 0 ? (
- ✓ Moderation queue clean! No pending reviews to approve. + Moderation queue clean. No pending reviews to approve.
) : (
@@ -809,7 +809,7 @@ export default function AdminPage() { {reportedReviews.length === 0 ? (
- ✓ No reported reviews flagged for moderation. + No reported reviews flagged for moderation.
) : (
@@ -870,7 +870,7 @@ export default function AdminPage() { {claims.length === 0 ? (
- ✓ No company ownership claims submitted. + No company ownership claims submitted.
) : (
@@ -964,7 +964,7 @@ export default function AdminPage() { {auditLogs.length === 0 ? (
- ✓ No audit log records captured yet. + No audit log records captured yet.
) : (
diff --git a/web/src/app/api/jobs/route.ts b/web/src/app/api/jobs/route.ts index 4f4a1b7..8152caa 100644 --- a/web/src/app/api/jobs/route.ts +++ b/web/src/app/api/jobs/route.ts @@ -150,14 +150,26 @@ export async function GET(req: Request) { { OR: [ { experienceLevel: "Entry Level" }, - { experienceLevel: "Mid-Level" }, { title: { contains: "Junior" } }, { title: { contains: "Entry" } }, { title: { contains: "Help Desk" } }, - { title: { contains: "Support" } }, { title: { contains: "Technician" } }, - { title: { contains: "Associate" } }, { title: { contains: "Assistant" } }, + { title: { contains: "Trainee" } }, + { title: { contains: "Intern" } }, + { title: { contains: "Apprentice" } }, + ], + }, + { + NOT: [ + { experienceLevel: { in: ["Mid-Level", "Senior", "Lead / Staff", "Executive"] } }, + { title: { contains: "Senior" } }, + { title: { contains: "Lead" } }, + { title: { contains: "Director" } }, + { title: { contains: "Principal" } }, + { title: { contains: "Manager" } }, + { title: { contains: "Head of" } }, + { title: { contains: "VP" } }, ], }, ]; diff --git a/web/src/app/employer/ats/page.tsx b/web/src/app/employer/ats/page.tsx index d08aa17..5f73094 100644 --- a/web/src/app/employer/ats/page.tsx +++ b/web/src/app/employer/ats/page.tsx @@ -268,7 +268,7 @@ export default function EmployerATSPage() { {selectedAppIds.length > 0 && (
- ✓ {selectedAppIds.length} candidate{selectedAppIds.length > 1 ? "s" : ""} selected + {selectedAppIds.length} candidate{selectedAppIds.length > 1 ? "s" : ""} selected
@@ -380,7 +380,7 @@ export default function EmployerATSPage() { {snapshot && ( - Snapshot ✓ + Snapshot Saved )}
diff --git a/web/src/app/jobs/[id]/page.tsx b/web/src/app/jobs/[id]/page.tsx index f624be1..5777f07 100644 --- a/web/src/app/jobs/[id]/page.tsx +++ b/web/src/app/jobs/[id]/page.tsx @@ -312,7 +312,7 @@ export default function JobDetailPage() {
@@ -524,7 +524,7 @@ export default function JobDetailPage() { : "bg-stone-900 hover:bg-stone-800 text-stone-50 border border-stone-900" }`} > - {isApplied ? "Application Submitted ✓" : "Apply for Role →"} + {isApplied ? "Application Submitted" : "Apply for Role →"}
diff --git a/web/src/components/JobCard.tsx b/web/src/components/JobCard.tsx index fd9ed00..bcdc6d6 100644 --- a/web/src/components/JobCard.tsx +++ b/web/src/components/JobCard.tsx @@ -251,7 +251,7 @@ export function JobCard({ job, onStatusChange, isAuthenticated }: JobCardProps) : "bg-stone-900 text-stone-50 border-stone-900 hover:bg-stone-800" }`} > - {currentStatus === "APPLIED" ? "Applied ✓" : "Apply →"} + {currentStatus === "APPLIED" ? "Applied" : "Apply →"}
diff --git a/web/src/components/NotificationBell.tsx b/web/src/components/NotificationBell.tsx index d022f05..8fe2689 100644 --- a/web/src/components/NotificationBell.tsx +++ b/web/src/components/NotificationBell.tsx @@ -131,10 +131,10 @@ export function NotificationBell() { {!item.isRead && ( )} diff --git a/web/src/lib/sources/adapters/ashby.ts b/web/src/lib/sources/adapters/ashby.ts index b325baf..88b3a81 100644 --- a/web/src/lib/sources/adapters/ashby.ts +++ b/web/src/lib/sources/adapters/ashby.ts @@ -6,7 +6,7 @@ */ import { IJobSourceAdapter, DiscoveryResult, SourceHealth, RateLimitPolicy, NormalizedJobRecord } from "../types"; -import { generateContentFingerprint, normalizeJobUrl } from "../deduplication"; +import { generateContentFingerprint, normalizeJobUrl, classifyExperienceLevel } from "../deduplication"; export class AshbySourceAdapter implements IJobSourceAdapter { readonly id: string; @@ -89,11 +89,7 @@ export class AshbySourceAdapter implements IJobSourceAdapter { location, isRemote, department: deptName, - experienceLevel: title.toLowerCase().includes("senior") - ? "Senior" - : title.toLowerCase().includes("lead") - ? "Lead / Staff" - : "Mid-Level", + experienceLevel: classifyExperienceLevel(title), description: cleanDesc.slice(0, 5000) || `Position at ${this.name}. Apply on Ashby.`, jobUrl, datePosted: item.publishedAt ? new Date(item.publishedAt) : new Date(), diff --git a/web/src/lib/sources/adapters/greenhouse.ts b/web/src/lib/sources/adapters/greenhouse.ts index c725c03..6db8c2b 100644 --- a/web/src/lib/sources/adapters/greenhouse.ts +++ b/web/src/lib/sources/adapters/greenhouse.ts @@ -6,7 +6,7 @@ */ import { IJobSourceAdapter, DiscoveryResult, SourceHealth, RateLimitPolicy, NormalizedJobRecord } from "../types"; -import { generateJobUrlHash, generateContentFingerprint, normalizeJobUrl } from "../deduplication"; +import { generateJobUrlHash, generateContentFingerprint, normalizeJobUrl, classifyExperienceLevel } from "../deduplication"; export class GreenhouseSourceAdapter implements IJobSourceAdapter { readonly id: string; @@ -94,11 +94,7 @@ export class GreenhouseSourceAdapter implements IJobSourceAdapter { location, isRemote, department: deptName, - experienceLevel: title.toLowerCase().includes("senior") - ? "Senior" - : title.toLowerCase().includes("lead") - ? "Lead / Staff" - : "Mid-Level", + experienceLevel: classifyExperienceLevel(title), description: cleanDesc.slice(0, 5000) || `Position at ${this.name}. Apply on company careers portal.`, jobUrl, datePosted: item.updated_at ? new Date(item.updated_at) : new Date(), diff --git a/web/src/lib/sources/adapters/lever.ts b/web/src/lib/sources/adapters/lever.ts index 8b2ecd2..d165959 100644 --- a/web/src/lib/sources/adapters/lever.ts +++ b/web/src/lib/sources/adapters/lever.ts @@ -6,7 +6,7 @@ */ import { IJobSourceAdapter, DiscoveryResult, SourceHealth, RateLimitPolicy, NormalizedJobRecord } from "../types"; -import { generateContentFingerprint, normalizeJobUrl } from "../deduplication"; +import { generateContentFingerprint, normalizeJobUrl, classifyExperienceLevel } from "../deduplication"; export class LeverSourceAdapter implements IJobSourceAdapter { readonly id: string; @@ -97,11 +97,7 @@ export class LeverSourceAdapter implements IJobSourceAdapter { location, isRemote, department: deptName, - experienceLevel: title.toLowerCase().includes("senior") - ? "Senior" - : title.toLowerCase().includes("lead") - ? "Lead / Staff" - : "Mid-Level", + experienceLevel: classifyExperienceLevel(title), description: cleanDesc.slice(0, 5000) || `Position at ${this.name}. Apply on Lever.`, jobUrl, datePosted: item.createdAt ? new Date(item.createdAt) : new Date(), diff --git a/web/src/lib/sources/deduplication.ts b/web/src/lib/sources/deduplication.ts index 0d15014..a42fbfe 100644 --- a/web/src/lib/sources/deduplication.ts +++ b/web/src/lib/sources/deduplication.ts @@ -209,7 +209,7 @@ export function calculateJobQualityScore(job: { else if (score >= 75) tier = "GOOD"; else if (score >= 55) tier = "QUESTIONABLE"; - return { + return { score, tier, breakdown: { @@ -221,3 +221,23 @@ export function calculateJobQualityScore(job: { }, }; } + +/** + * Classifies experience level accurately based on title keywords + */ +export function classifyExperienceLevel(title: string): string { + const t = title.toLowerCase(); + if (/\b(chief|vp|vice president|head of|director)\b/i.test(t)) { + return "Executive"; + } + if (/\b(lead|staff|principal|architect|manager)\b/i.test(t)) { + return "Lead / Staff"; + } + if (/\b(senior|sr\.?|sr\b)/i.test(t)) { + return "Senior"; + } + if (/\b(junior|jr\.?|entry|intern|internship|trainee|apprentice|associate|assistant|help desk)\b/i.test(t)) { + return "Entry Level"; + } + return "Mid-Level"; +}