// Package jobs is a table-backed work queue. // // It lives in the same SQLite file as everything else: no Redis, no second // process, and the queue is included in the same backup. SQLite's single // writer makes the claim below atomic without any extra locking. package jobs import ( "context" "database/sql" "encoding/json" "errors" "fmt" "log/slog" "time" ) // States a job can be in. const ( StateQueued = "queued" StateRunning = "running" StateDone = "done" StateFailed = "failed" ) // Handler processes one job payload. type Handler func(ctx context.Context, payload json.RawMessage) error // Queue claims and runs jobs. type Queue struct { db *sql.DB log *slog.Logger handlers map[string]Handler maxAttempts int pollEvery time.Duration baseBackoff time.Duration } // New builds a Queue with sensible defaults. func New(db *sql.DB, log *slog.Logger) *Queue { return &Queue{ db: db, log: log, handlers: map[string]Handler{}, maxAttempts: 5, pollEvery: time.Second, baseBackoff: 30 * time.Second, } } // Register installs the handler for a job kind. func (q *Queue) Register(kind string, h Handler) { q.handlers[kind] = h } // Enqueue adds a job to run as soon as possible. func (q *Queue) Enqueue(ctx context.Context, kind string, payload any) error { return q.EnqueueAfter(ctx, kind, payload, 0) } // EnqueueAfter adds a job that will not run before delay has elapsed. func (q *Queue) EnqueueAfter(ctx context.Context, kind string, payload any, delay time.Duration) error { body := "{}" if payload != nil { raw, err := json.Marshal(payload) if err != nil { return fmt.Errorf("jobs: marshal payload: %w", err) } body = string(raw) } _, err := q.db.ExecContext(ctx, ` INSERT INTO jobs (kind, payload, state, run_after) VALUES (?, ?, ?, ?)`, kind, body, StateQueued, time.Now().UTC().Add(delay).Format(time.RFC3339Nano)) if err != nil { return fmt.Errorf("jobs: enqueue %s: %w", kind, err) } return nil } // Run processes jobs until the context is cancelled. func (q *Queue) Run(ctx context.Context) { ticker := time.NewTicker(q.pollEvery) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: // Drain everything currently due before waiting again. for { ran, err := q.RunOnce(ctx) if err != nil { if ctx.Err() != nil { return } q.log.Error("job queue", "err", err) break } if !ran { break } } } } } // RunOnce claims and runs at most one job. It reports whether it ran anything, // which is what makes the queue testable without a ticker. func (q *Queue) RunOnce(ctx context.Context) (bool, error) { job, ok, err := q.claim(ctx) if err != nil || !ok { return false, err } handler, exists := q.handlers[job.Kind] if !exists { // A missing handler is a programming error, not a transient fault. // Retrying it just delays the inevitable and clutters the queue. q.failPermanently(ctx, job, fmt.Errorf("no handler registered for %q", job.Kind)) return true, nil } if err := handler(ctx, job.Payload); err != nil { q.log.Warn("job failed", "kind", job.Kind, "id", job.ID, "attempt", job.Attempts, "err", err) q.finish(ctx, job, err) return true, nil } q.finish(ctx, job, nil) return true, nil } type claimedJob struct { ID int64 Kind string Payload json.RawMessage Attempts int } // claim atomically takes the oldest due job. // // The UPDATE ... RETURNING form means a job cannot be claimed twice even with // several workers: the subselect and the state change are one statement. func (q *Queue) claim(ctx context.Context) (claimedJob, bool, error) { var ( job claimedJob payload string ) err := q.db.QueryRowContext(ctx, ` UPDATE jobs SET state = ?, attempts = attempts + 1 WHERE id = ( SELECT id FROM jobs WHERE state = ? AND run_after <= ? ORDER BY run_after, id LIMIT 1 ) RETURNING id, kind, payload, attempts`, StateRunning, StateQueued, time.Now().UTC().Format(time.RFC3339Nano), ).Scan(&job.ID, &job.Kind, &payload, &job.Attempts) if errors.Is(err, sql.ErrNoRows) { return claimedJob{}, false, nil } if err != nil { return claimedJob{}, false, fmt.Errorf("jobs: claim: %w", err) } job.Payload = json.RawMessage(payload) return job, true, nil } // finish marks a job done, or reschedules it with exponential backoff until it // exhausts its attempts. func (q *Queue) finish(ctx context.Context, job claimedJob, runErr error) { if runErr == nil { if _, err := q.db.ExecContext(ctx, `UPDATE jobs SET state = ?, error = '' WHERE id = ?`, StateDone, job.ID); err != nil { q.log.Error("jobs: mark done", "id", job.ID, "err", err) } return } if job.Attempts >= q.maxAttempts { if _, err := q.db.ExecContext(ctx, `UPDATE jobs SET state = ?, error = ? WHERE id = ?`, StateFailed, runErr.Error(), job.ID); err != nil { q.log.Error("jobs: mark failed", "id", job.ID, "err", err) } return } backoff := q.baseBackoff * time.Duration(1<<(job.Attempts-1)) runAfter := time.Now().UTC().Add(backoff).Format(time.RFC3339Nano) if _, err := q.db.ExecContext(ctx, `UPDATE jobs SET state = ?, error = ?, run_after = ? WHERE id = ?`, StateQueued, runErr.Error(), runAfter, job.ID); err != nil { q.log.Error("jobs: reschedule", "id", job.ID, "err", err) } } // failPermanently marks a job failed with no further attempts. func (q *Queue) failPermanently(ctx context.Context, job claimedJob, err error) { if _, err := q.db.ExecContext(ctx, `UPDATE jobs SET state = ?, error = ? WHERE id = ?`, StateFailed, err.Error(), job.ID); err != nil { q.log.Error("jobs: mark failed", "id", job.ID, "err", err) } } // Stats reports job counts by state, for the health endpoint and debugging. func (q *Queue) Stats(ctx context.Context) (map[string]int, error) { rows, err := q.db.QueryContext(ctx, `SELECT state, count(*) FROM jobs GROUP BY state`) if err != nil { return nil, err } defer rows.Close() out := map[string]int{} for rows.Next() { var state string var n int if err := rows.Scan(&state, &n); err != nil { return nil, err } out[state] = n } return out, rows.Err() }