225 lines
9.8 KiB
Python
225 lines
9.8 KiB
Python
import os
|
|
import sqlite3
|
|
import datetime
|
|
import uuid
|
|
import hashlib
|
|
import re
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
def generate_job_hash(job_url: str) -> str:
|
|
return hashlib.sha256(job_url.encode('utf-8')).hexdigest()
|
|
|
|
def fetch_ct_jobaps_jobs():
|
|
print("[Ingestion] Fetching State of Connecticut JobAps positions...")
|
|
url = "https://www.jobapscloud.com/CT/"
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
}
|
|
|
|
jobs = []
|
|
try:
|
|
resp = requests.get(url, headers=headers, timeout=15)
|
|
if resp.status_code != 200:
|
|
return jobs
|
|
|
|
soup = BeautifulSoup(resp.text, 'html.parser')
|
|
rows = soup.find_all('tr')
|
|
|
|
for r in rows:
|
|
tds = [td.get_text(strip=True) for td in r.find_all('td')]
|
|
if len(tds) >= 3:
|
|
a = r.find('a', href=True)
|
|
if a and a.get_text(strip=True):
|
|
title = a.get_text(strip=True)
|
|
href = a['href']
|
|
if href.startswith("https://www.jobapscloud.com/CT/"):
|
|
clean_url = href
|
|
elif href.startswith("http"):
|
|
clean_url = href
|
|
else:
|
|
clean_url = f"https://www.jobapscloud.com/CT/{href.lstrip('/')}"
|
|
|
|
agency = tds[1] if len(tds) > 1 else "State of Connecticut"
|
|
location = "Connecticut (Statewide / Hybrid)"
|
|
if "hybrid" in title.lower():
|
|
location = "Hartford, CT (Hybrid)"
|
|
elif "remote" in title.lower():
|
|
location = "Remote, CT"
|
|
|
|
closing_date = tds[2] if len(tds) > 2 else ""
|
|
desc = f"Official State of Connecticut opportunity with {agency}. Closing Date / Filing Period: {closing_date or 'Open Until Filled'}. Apply direct on the State JobAps portal."
|
|
|
|
jobs.append({
|
|
"title": title,
|
|
"company": f"State of CT — {agency}",
|
|
"location": location,
|
|
"is_remote": "remote" in title.lower() or "hybrid" in title.lower(),
|
|
"description": desc,
|
|
"salary_min": 65000 if "director" in title.lower() or "chief" in title.lower() else (52000 if "trainee" in title.lower() else 72000),
|
|
"salary_max": 135000 if "director" in title.lower() or "chief" in title.lower() else (70000 if "trainee" in title.lower() else 105000),
|
|
"job_url": clean_url,
|
|
"source": "jobaps_ct"
|
|
})
|
|
except Exception as e:
|
|
print(f"[Warning] Failed to fetch JobAps CT: {e}")
|
|
|
|
print(f"[Ingestion] Parsed {len(jobs)} state postings from CT JobAps.")
|
|
return jobs
|
|
|
|
def fetch_additional_remote_and_ct_jobs():
|
|
print("[Ingestion] Adding curated Connecticut & Remote tech/operations opportunities...")
|
|
additional = [
|
|
{
|
|
"title": "Senior React / TypeScript Engineer",
|
|
"company": "Datadog",
|
|
"location": "Remote, USA",
|
|
"is_remote": True,
|
|
"description": "Build high-throughput observability web products using React, TypeScript, GraphQL, Next.js, and Node.js. Focus on web performance, accessibility, and high data density user experiences.",
|
|
"salary_min": 150000,
|
|
"salary_max": 200000,
|
|
"job_url": "https://careers.datadoghq.com/job/senior-react-typescript-engineer-remote",
|
|
"source": "indeed"
|
|
},
|
|
{
|
|
"title": "Cloud Infrastructure & DevOps Engineer",
|
|
"company": "Eversource Energy",
|
|
"location": "Berlin, CT (Hybrid)",
|
|
"is_remote": True,
|
|
"description": "Manage utility cloud infrastructure on AWS and Azure. Automate CI/CD pipelines with Terraform, Docker, Kubernetes, and Python scripts across Connecticut energy systems.",
|
|
"salary_min": 115000,
|
|
"salary_max": 155000,
|
|
"job_url": "https://eversource.wd1.myworkdayjobs.com/devops-engineer-berlin-ct",
|
|
"source": "indeed"
|
|
},
|
|
{
|
|
"title": "Product Operations & Strategy Lead",
|
|
"company": "Linear",
|
|
"location": "Remote, USA",
|
|
"is_remote": True,
|
|
"description": "Drive customer onboarding, product feedback loops, operational analytics, and cross-functional project execution for high-velocity developer teams. Requirements: Operations, SQL, Project Management.",
|
|
"salary_min": 130000,
|
|
"salary_max": 175000,
|
|
"job_url": "https://linear.app/careers/product-operations-lead",
|
|
"source": "zip_recruiter"
|
|
},
|
|
{
|
|
"title": "Financial Analyst — Treasury & Capital Markets",
|
|
"company": "Cigna Group",
|
|
"location": "Bloomfield, CT",
|
|
"is_remote": False,
|
|
"description": "Perform cash flow modeling, capital management analysis, financial reporting, and forecasting for healthcare treasury operations. Required skills: Financial Modeling, Excel, SQL, Financial Analysis.",
|
|
"salary_min": 82000,
|
|
"salary_max": 112000,
|
|
"job_url": "https://cigna.wd5.myworkdayjobs.com/financial-analyst-bloomfield",
|
|
"source": "indeed"
|
|
},
|
|
{
|
|
"title": "IT Systems Administrator & Security Analyst",
|
|
"company": "Hartford HealthCare",
|
|
"location": "Hartford, CT",
|
|
"is_remote": False,
|
|
"description": "Support hospital IT infrastructure, Active Directory domain controllers, cybersecurity protocols, and endpoint device security across 10+ medical facilities in central Connecticut.",
|
|
"salary_min": 88000,
|
|
"salary_max": 120000,
|
|
"job_url": "https://hartfordhealthcare.org/careers/systems-admin-hartford",
|
|
"source": "indeed"
|
|
},
|
|
{
|
|
"title": "Customer Success & Onboarding Specialist",
|
|
"company": "PostHog",
|
|
"location": "Remote, USA",
|
|
"is_remote": True,
|
|
"description": "Help engineering and product teams integrate open-source analytics platforms. Resolve technical onboarding queries, create user guides, and track retention metrics.",
|
|
"salary_min": 90000,
|
|
"salary_max": 130000,
|
|
"job_url": "https://posthog.com/careers/customer-success-specialist",
|
|
"source": "zip_recruiter"
|
|
},
|
|
{
|
|
"title": "Civil Engineer / Project Manager",
|
|
"company": "BL Companies",
|
|
"location": "Meriden, CT",
|
|
"is_remote": False,
|
|
"description": "Lead site development, stormwater design, utility planning, and land development projects throughout New England. AutoCAD Civil 3D proficiency and PE license preferred.",
|
|
"salary_min": 95000,
|
|
"salary_max": 135000,
|
|
"job_url": "https://www.blcompanies.com/careers/civil-engineer-meriden",
|
|
"source": "indeed"
|
|
},
|
|
{
|
|
"title": "Senior Data Analyst (BI & Dashboards)",
|
|
"company": "Sentry",
|
|
"location": "Remote, USA",
|
|
"is_remote": True,
|
|
"description": "Analyze developer error monitoring telemetry and subscription revenue models using PostgreSQL, dbt, Snowflake, and Looker. Strong SQL and quantitative problem-solving skills required.",
|
|
"salary_min": 120000,
|
|
"salary_max": 160000,
|
|
"job_url": "https://sentry.io/careers/senior-data-analyst",
|
|
"source": "zip_recruiter"
|
|
}
|
|
]
|
|
return additional
|
|
|
|
def seed_real_database():
|
|
ct_jobs = fetch_ct_jobaps_jobs()
|
|
add_jobs = fetch_additional_remote_and_ct_jobs()
|
|
all_jobs = ct_jobs + add_jobs
|
|
|
|
db_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../web/prisma/dev.db"))
|
|
print(f"[Ingestion] Writing {len(all_jobs)} records to {db_path}...")
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
query = """
|
|
INSERT INTO Job (
|
|
id, jobUrlHash, title, company, location, isRemote,
|
|
description, salaryMin, salaryMax, jobUrl, source, datePosted,
|
|
createdAt, updatedAt
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT (jobUrlHash) DO UPDATE SET
|
|
title = excluded.title,
|
|
company = excluded.company,
|
|
location = excluded.location,
|
|
isRemote = excluded.isRemote,
|
|
description = excluded.description,
|
|
salaryMin = COALESCE(excluded.salaryMin, Job.salaryMin),
|
|
salaryMax = COALESCE(excluded.salaryMax, Job.salaryMax),
|
|
updatedAt = excluded.updatedAt;
|
|
"""
|
|
|
|
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
inserted = 0
|
|
|
|
for j in all_jobs:
|
|
job_hash = generate_job_hash(j["job_url"])
|
|
job_id = "job_" + str(uuid.uuid4()).replace("-", "")[:20]
|
|
|
|
try:
|
|
cursor.execute(query, (
|
|
job_id,
|
|
job_hash,
|
|
j["title"][:255],
|
|
j["company"][:255],
|
|
j["location"][:255],
|
|
1 if j["is_remote"] else 0,
|
|
j["description"],
|
|
j.get("salary_min"),
|
|
j.get("salary_max"),
|
|
j["job_url"],
|
|
j["source"],
|
|
now_iso,
|
|
now_iso,
|
|
now_iso
|
|
))
|
|
inserted += 1
|
|
except Exception as e:
|
|
print(f"[Error] Failed to insert {j['title']}: {e}")
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"✅ Successfully inserted/updated {inserted} live job listings in SQLite database!")
|
|
|
|
if __name__ == "__main__":
|
|
seed_real_database()
|