ad3laid3/JB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(scraper): deduplicate job_hash in Python before PostgreSQL batch upsert

Browse source
This commit is contained in:
JobsBoard Deployer 2026-09-05 11:49:16 -04:00
parent 3aad6705d0
commit 83ee7378da
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
scraper/db.py
View file

@ -168,6 +168,7 @@ def _upsert_postgres(jobs_list: list, db_url: str):
""" """
records = [] records = []
seen_hashes = set()
now = datetime.datetime.now(datetime.timezone.utc) now = datetime.datetime.now(datetime.timezone.utc)
for j in jobs_list: for j in jobs_list:
@ -175,6 +176,9 @@ def _upsert_postgres(jobs_list: list, db_url: str):
if not job_url: if not job_url:
continue continue
job_hash = generate_job_hash(job_url) job_hash = generate_job_hash(job_url)
if job_hash in seen_hashes:
continue
seen_hashes.add(job_hash)
job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20] job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20]
records.append(( records.append((