439 lines
15 KiB
JavaScript
439 lines
15 KiB
JavaScript
/**
|
|
* Real Job Acquisition Live Ingestion Script
|
|
*
|
|
* Runs genuine ATS adapters against verified public job board APIs:
|
|
* - Greenhouse: Stripe, Cloudflare, Figma, Datadog
|
|
* - Lever: Palantir
|
|
* - Ashby: Ramp, Notion
|
|
*
|
|
* Enforces deduplication, canonical normalization, and execution audit logging.
|
|
*/
|
|
|
|
const { PrismaClient } = require("@prisma/client");
|
|
const prisma = new PrismaClient();
|
|
const crypto = require("crypto");
|
|
|
|
function normalizeJobUrl(rawUrl) {
|
|
try {
|
|
const parsed = new URL(rawUrl.trim());
|
|
const trackingParams = [
|
|
"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
|
|
"ref", "source", "gh_src", "lever-source", "ashby_jid", "referrer"
|
|
];
|
|
for (const p of trackingParams) {
|
|
parsed.searchParams.delete(p);
|
|
}
|
|
let clean = parsed.toString();
|
|
if (clean.endsWith("/") && parsed.pathname !== "/") {
|
|
clean = clean.slice(0, -1);
|
|
}
|
|
return clean;
|
|
} catch {
|
|
return rawUrl.trim();
|
|
}
|
|
}
|
|
|
|
function generateJobUrlHash(url) {
|
|
return crypto.createHash("sha256").update(normalizeJobUrl(url)).digest("hex");
|
|
}
|
|
|
|
function generateContentFingerprint(company, title, location, isRemote) {
|
|
const normComp = (company || "").toLowerCase().replace(/[^a-z0-9]/g, "").trim();
|
|
const normTitle = (title || "")
|
|
.toLowerCase()
|
|
.replace(/\b(senior|sr\.?|junior|jr\.?|lead|staff|principal)\b/g, "")
|
|
.replace(/[^a-z0-9]/g, "")
|
|
.trim();
|
|
const normLoc = (location || "").toLowerCase().replace(/[^a-z0-9]/g, "").trim();
|
|
const remoteFlag = isRemote ? "remote" : "onsite";
|
|
return crypto.createHash("sha256").update(`${normComp}:${normTitle}:${normLoc}:${remoteFlag}`).digest("hex");
|
|
}
|
|
|
|
async function ingestGreenhouse(boardToken, companyName, limit = 50) {
|
|
console.log(`\n[ACQUIRE] Greenhouse: ${companyName} (${boardToken})...`);
|
|
const start = Date.now();
|
|
const url = `https://boards-api.greenhouse.io/v1/boards/${boardToken}/jobs?content=true`;
|
|
const res = await fetch(url, { headers: { "User-Agent": "JobsBoard-Acquisition/1.0" } });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
|
|
|
|
const data = await res.json();
|
|
const rawJobs = (data.jobs || []).slice(0, limit);
|
|
let created = 0;
|
|
let updated = 0;
|
|
let duplicates = 0;
|
|
|
|
for (const item of rawJobs) {
|
|
const title = (item.title || "").trim();
|
|
const rawUrl = item.absolute_url || "";
|
|
if (!title || !rawUrl) continue;
|
|
|
|
const jobUrl = normalizeJobUrl(rawUrl);
|
|
const urlHash = generateJobUrlHash(jobUrl);
|
|
const location = item.location?.name || "Remote, USA";
|
|
const isRemote =
|
|
location.toLowerCase().includes("remote") ||
|
|
title.toLowerCase().includes("remote") ||
|
|
location.toLowerCase().includes("anywhere");
|
|
const dept = item.departments?.[0]?.name || "Engineering";
|
|
const cleanDesc = (item.content || "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
const fingerprint = generateContentFingerprint(companyName, title, location, isRemote);
|
|
|
|
const existing = await prisma.job.findUnique({ where: { jobUrlHash: urlHash } });
|
|
if (existing) {
|
|
await prisma.job.update({
|
|
where: { id: existing.id },
|
|
data: {
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
lifecycleStatus: "ACTIVE",
|
|
title,
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
},
|
|
});
|
|
updated++;
|
|
} else {
|
|
const fpMatch = await prisma.job.findFirst({ where: { fingerprintHash: fingerprint } });
|
|
if (fpMatch) duplicates++;
|
|
|
|
await prisma.job.create({
|
|
data: {
|
|
jobUrlHash: urlHash,
|
|
fingerprintHash: fingerprint,
|
|
title,
|
|
company: companyName,
|
|
location,
|
|
isRemote,
|
|
department: dept,
|
|
experienceLevel: title.toLowerCase().includes("senior") ? "Senior" : "Mid-Level",
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
jobUrl,
|
|
source: "greenhouse",
|
|
sourceId: `gh_${boardToken}`,
|
|
moderationStatus: "APPROVED",
|
|
lifecycleStatus: "ACTIVE",
|
|
firstSeenAt: new Date(),
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
datePosted: item.updated_at ? new Date(item.updated_at) : new Date(),
|
|
},
|
|
});
|
|
created++;
|
|
}
|
|
}
|
|
|
|
const durationMs = Date.now() - start;
|
|
// Upsert Source Registry Record
|
|
const dbSource = await prisma.jobSource.upsert({
|
|
where: { provider_identifier: { provider: "greenhouse", identifier: `gh_${boardToken}` } },
|
|
update: {
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
status: "ACTIVE",
|
|
averageDurationMs: durationMs,
|
|
},
|
|
create: {
|
|
name: companyName,
|
|
type: "ATS",
|
|
provider: "greenhouse",
|
|
baseUrl: `https://boards.greenhouse.io/${boardToken}`,
|
|
identifier: `gh_${boardToken}`,
|
|
status: "ACTIVE",
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
},
|
|
});
|
|
|
|
// Log Execution
|
|
await prisma.sourceExecutionLog.create({
|
|
data: {
|
|
sourceId: dbSource.id,
|
|
status: "SUCCESS",
|
|
durationMs,
|
|
discoveredCount: rawJobs.length,
|
|
createdCount: created,
|
|
updatedCount: updated,
|
|
duplicateCount: duplicates,
|
|
},
|
|
});
|
|
|
|
console.log(` -> Finished ${companyName}: Discovered=${rawJobs.length}, Created=${created}, Updated=${updated}, Dupes=${duplicates} (${durationMs}ms)`);
|
|
return { discovered: rawJobs.length, created, updated, duplicates };
|
|
}
|
|
|
|
async function ingestLever(boardToken, companyName, limit = 50) {
|
|
console.log(`\n[ACQUIRE] Lever: ${companyName} (${boardToken})...`);
|
|
const start = Date.now();
|
|
const url = `https://api.lever.co/v0/postings/${boardToken}?mode=json`;
|
|
const res = await fetch(url, { headers: { "User-Agent": "JobsBoard-Acquisition/1.0" } });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
|
|
|
|
const rawList = await res.json();
|
|
const rawJobs = Array.isArray(rawList) ? rawList.slice(0, limit) : [];
|
|
let created = 0;
|
|
let updated = 0;
|
|
let duplicates = 0;
|
|
|
|
for (const item of rawJobs) {
|
|
const title = (item.text || "").trim();
|
|
const rawUrl = item.hostedUrl || "";
|
|
if (!title || !rawUrl) continue;
|
|
|
|
const jobUrl = normalizeJobUrl(rawUrl);
|
|
const urlHash = generateJobUrlHash(jobUrl);
|
|
const categories = item.categories || {};
|
|
const location = categories.location || "Remote, USA";
|
|
const isRemote =
|
|
location.toLowerCase().includes("remote") ||
|
|
title.toLowerCase().includes("remote");
|
|
const dept = categories.department || categories.team || "Engineering";
|
|
const cleanDesc = (item.descriptionPlain || item.description || "")
|
|
.replace(/<[^>]+>/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
const fingerprint = generateContentFingerprint(companyName, title, location, isRemote);
|
|
|
|
const existing = await prisma.job.findUnique({ where: { jobUrlHash: urlHash } });
|
|
if (existing) {
|
|
await prisma.job.update({
|
|
where: { id: existing.id },
|
|
data: {
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
lifecycleStatus: "ACTIVE",
|
|
title,
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
},
|
|
});
|
|
updated++;
|
|
} else {
|
|
const fpMatch = await prisma.job.findFirst({ where: { fingerprintHash: fingerprint } });
|
|
if (fpMatch) duplicates++;
|
|
|
|
await prisma.job.create({
|
|
data: {
|
|
jobUrlHash: urlHash,
|
|
fingerprintHash: fingerprint,
|
|
title,
|
|
company: companyName,
|
|
location,
|
|
isRemote,
|
|
department: dept,
|
|
experienceLevel: title.toLowerCase().includes("senior") ? "Senior" : "Mid-Level",
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
jobUrl,
|
|
source: "lever",
|
|
sourceId: `lever_${boardToken}`,
|
|
moderationStatus: "APPROVED",
|
|
lifecycleStatus: "ACTIVE",
|
|
firstSeenAt: new Date(),
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
datePosted: item.createdAt ? new Date(item.createdAt) : new Date(),
|
|
},
|
|
});
|
|
created++;
|
|
}
|
|
}
|
|
|
|
const durationMs = Date.now() - start;
|
|
const dbSource = await prisma.jobSource.upsert({
|
|
where: { provider_identifier: { provider: "lever", identifier: `lever_${boardToken}` } },
|
|
update: {
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
status: "ACTIVE",
|
|
averageDurationMs: durationMs,
|
|
},
|
|
create: {
|
|
name: companyName,
|
|
type: "ATS",
|
|
provider: "lever",
|
|
baseUrl: `https://jobs.lever.co/${boardToken}`,
|
|
identifier: `lever_${boardToken}`,
|
|
status: "ACTIVE",
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
},
|
|
});
|
|
|
|
await prisma.sourceExecutionLog.create({
|
|
data: {
|
|
sourceId: dbSource.id,
|
|
status: "SUCCESS",
|
|
durationMs,
|
|
discoveredCount: rawJobs.length,
|
|
createdCount: created,
|
|
updatedCount: updated,
|
|
duplicateCount: duplicates,
|
|
},
|
|
});
|
|
|
|
console.log(` -> Finished ${companyName}: Discovered=${rawJobs.length}, Created=${created}, Updated=${updated}, Dupes=${duplicates} (${durationMs}ms)`);
|
|
return { discovered: rawJobs.length, created, updated, duplicates };
|
|
}
|
|
|
|
async function ingestAshby(boardToken, companyName, limit = 50) {
|
|
console.log(`\n[ACQUIRE] Ashby: ${companyName} (${boardToken})...`);
|
|
const start = Date.now();
|
|
const url = `https://api.ashbyhq.com/posting-api/job-board/${boardToken}`;
|
|
const res = await fetch(url, { headers: { "User-Agent": "JobsBoard-Acquisition/1.0" } });
|
|
if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
|
|
|
|
const data = await res.json();
|
|
const rawJobs = (data.jobs || []).slice(0, limit);
|
|
let created = 0;
|
|
let updated = 0;
|
|
let duplicates = 0;
|
|
|
|
for (const item of rawJobs) {
|
|
const title = (item.title || "").trim();
|
|
const rawUrl = item.applyUrl || item.jobUrl || "";
|
|
if (!title || !rawUrl) continue;
|
|
|
|
const jobUrl = normalizeJobUrl(rawUrl);
|
|
const urlHash = generateJobUrlHash(jobUrl);
|
|
const location = item.location || "Remote, USA";
|
|
const isRemote = item.isRemote || location.toLowerCase().includes("remote") || title.toLowerCase().includes("remote");
|
|
const dept = item.department || "Engineering";
|
|
const cleanDesc = (item.descriptionPlain || item.descriptionHtml || "")
|
|
.replace(/<[^>]+>/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
const fingerprint = generateContentFingerprint(companyName, title, location, isRemote);
|
|
|
|
const existing = await prisma.job.findUnique({ where: { jobUrlHash: urlHash } });
|
|
if (existing) {
|
|
await prisma.job.update({
|
|
where: { id: existing.id },
|
|
data: {
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
lifecycleStatus: "ACTIVE",
|
|
title,
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
},
|
|
});
|
|
updated++;
|
|
} else {
|
|
const fpMatch = await prisma.job.findFirst({ where: { fingerprintHash: fingerprint } });
|
|
if (fpMatch) duplicates++;
|
|
|
|
await prisma.job.create({
|
|
data: {
|
|
jobUrlHash: urlHash,
|
|
fingerprintHash: fingerprint,
|
|
title,
|
|
company: companyName,
|
|
location,
|
|
isRemote,
|
|
department: dept,
|
|
experienceLevel: title.toLowerCase().includes("senior") ? "Senior" : "Mid-Level",
|
|
description: cleanDesc.slice(0, 5000) || `Position at ${companyName}.`,
|
|
jobUrl,
|
|
source: "ashby",
|
|
sourceId: `ashby_${boardToken}`,
|
|
moderationStatus: "APPROVED",
|
|
lifecycleStatus: "ACTIVE",
|
|
firstSeenAt: new Date(),
|
|
lastSeenAt: new Date(),
|
|
missingScanCount: 0,
|
|
datePosted: item.publishedAt ? new Date(item.publishedAt) : new Date(),
|
|
},
|
|
});
|
|
created++;
|
|
}
|
|
}
|
|
|
|
const durationMs = Date.now() - start;
|
|
const dbSource = await prisma.jobSource.upsert({
|
|
where: { provider_identifier: { provider: "ashby", identifier: `ashby_${boardToken}` } },
|
|
update: {
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
status: "ACTIVE",
|
|
averageDurationMs: durationMs,
|
|
},
|
|
create: {
|
|
name: companyName,
|
|
type: "ATS",
|
|
provider: "ashby",
|
|
baseUrl: `https://jobs.ashbyhq.com/${boardToken}`,
|
|
identifier: `ashby_${boardToken}`,
|
|
status: "ACTIVE",
|
|
lastRunAt: new Date(),
|
|
lastSuccessAt: new Date(),
|
|
jobsDiscoveredLastRun: rawJobs.length,
|
|
jobsCreatedLastRun: created,
|
|
jobsUpdatedLastRun: updated,
|
|
},
|
|
});
|
|
|
|
await prisma.sourceExecutionLog.create({
|
|
data: {
|
|
sourceId: dbSource.id,
|
|
status: "SUCCESS",
|
|
durationMs,
|
|
discoveredCount: rawJobs.length,
|
|
createdCount: created,
|
|
updatedCount: updated,
|
|
duplicateCount: duplicates,
|
|
},
|
|
});
|
|
|
|
console.log(` -> Finished ${companyName}: Discovered=${rawJobs.length}, Created=${created}, Updated=${updated}, Dupes=${duplicates} (${durationMs}ms)`);
|
|
return { discovered: rawJobs.length, created, updated, duplicates };
|
|
}
|
|
|
|
async function main() {
|
|
console.log("=================================================");
|
|
console.log(" JOBSBOARD LIVE ACQUISITION CAMPAIGN EXECUTION ");
|
|
console.log("=================================================");
|
|
|
|
const initialJobCount = await prisma.job.count();
|
|
console.log(`Current Total Jobs in Database: ${initialJobCount}`);
|
|
|
|
// Ingest from verified permitted sources
|
|
await ingestGreenhouse("stripe", "Stripe", 40);
|
|
await ingestGreenhouse("cloudflare", "Cloudflare", 40);
|
|
await ingestGreenhouse("datadog", "Datadog", 40);
|
|
await ingestGreenhouse("figma", "Figma", 40);
|
|
await ingestLever("palantir", "Palantir Technologies", 40);
|
|
await ingestAshby("ramp", "Ramp", 40);
|
|
await ingestAshby("notion", "Notion", 40);
|
|
|
|
const finalJobCount = await prisma.job.count();
|
|
const newJobsCreated = finalJobCount - initialJobCount;
|
|
const totalSources = await prisma.jobSource.count();
|
|
const totalLogs = await prisma.sourceExecutionLog.count();
|
|
|
|
console.log("\n=================================================");
|
|
console.log(" ACQUISITION CAMPAIGN RESULTS ");
|
|
console.log("=================================================");
|
|
console.log(` Initial Jobs: ${initialJobCount}`);
|
|
console.log(` Final Jobs: ${finalJobCount} (+${newJobsCreated} genuine jobs added)`);
|
|
console.log(` Active Sources: ${totalSources}`);
|
|
console.log(` Audit Logs Recorded: ${totalLogs}`);
|
|
console.log("=================================================\n");
|
|
|
|
await prisma.$disconnect();
|
|
}
|
|
|
|
main().catch(err => {
|
|
console.error("Acquisition campaign failed:", err);
|
|
process.exit(1);
|
|
});
|