fix(docker): install openssl in alpine and refine scraper readiness loop
This commit is contained in:
parent
00775c48ed
commit
3aad6705d0
3 changed files with 14 additions and 8 deletions
|
|
@ -119,21 +119,23 @@ def _upsert_postgres(jobs_list: list, db_url: str):
|
|||
|
||||
# Wait for postgres and Prisma migrations to be ready
|
||||
conn = None
|
||||
for attempt in range(15):
|
||||
for attempt in range(40):
|
||||
try:
|
||||
conn = psycopg2.connect(db_url)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT to_regclass('\"Job\"');")
|
||||
table_exists = cursor.fetchone()[0]
|
||||
result = cursor.fetchone()
|
||||
table_exists = result[0] if result else None
|
||||
if table_exists:
|
||||
break
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print(f"[Scraper] Waiting for database tables to initialize (attempt {attempt + 1}/15)...")
|
||||
time.sleep(3)
|
||||
conn = None
|
||||
print(f"[Scraper] Waiting for database tables to initialize (attempt {attempt + 1}/40)...")
|
||||
time.sleep(5)
|
||||
except Exception as conn_err:
|
||||
print(f"[Scraper] Waiting for PostgreSQL readiness ({conn_err}) (attempt {attempt + 1}/15)...")
|
||||
time.sleep(3)
|
||||
print(f"[Scraper] Waiting for PostgreSQL readiness ({conn_err}) (attempt {attempt + 1}/40)...")
|
||||
time.sleep(5)
|
||||
|
||||
if not conn or conn.closed:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def run_jobspy_scrapes() -> List[Dict[Any, Any]]:
|
|||
try:
|
||||
print(f"[JobSpy] Searching CT Local: '{query}'")
|
||||
jobs_df = scrape_jobs(
|
||||
site_name=["indeed", "zip_recruiter"],
|
||||
site_name=["indeed"],
|
||||
search_term=query,
|
||||
location="Connecticut",
|
||||
results_wanted=15,
|
||||
|
|
@ -68,7 +68,7 @@ def run_jobspy_scrapes() -> List[Dict[Any, Any]]:
|
|||
try:
|
||||
print(f"[JobSpy] Searching Remote: '{query}'")
|
||||
jobs_df = scrape_jobs(
|
||||
site_name=["indeed", "zip_recruiter"],
|
||||
site_name=["indeed"],
|
||||
search_term=query,
|
||||
results_wanted=15,
|
||||
hours_old=72,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
FROM node:20-alpine AS builder
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk add --no-cache openssl libc6-compat
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
|
||||
|
|
@ -12,6 +14,8 @@ RUN npm run build
|
|||
FROM node:20-alpine AS runner
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk add --no-cache openssl libc6-compat
|
||||
|
||||
ENV NODE_ENV=production
|
||||
|
||||
COPY package*.json ./
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue