Vellum/internal/progress/store.go
mithral c50b3cfb3b Wave 1: survive a real library
A 50k-file library used to wedge the database. This wave fixes the four
reasons why, plus a timestamp-ordering bug the work exposed.

Chunked scans (internal/library/scan.go)
- One transaction wrapped the whole reconciliation, holding SQLite's single
  write lock for the entire scan. Now 500 groups per transaction, and the two
  missing-file passes are batched too.
- Verified: a 20,000-file scan reports partial progress (scanned=1500 mid-scan)
  and a concurrent progress save returns 200 throughout. Under the old code
  that save blocked for the whole scan.
- Behaviour change: a scan is no longer all-or-nothing. A crash now leaves a
  consistent partial scan. The scan is idempotent, so re-running finishes it.
- res.Errors is capped at 100 (a broken mount used to append one string per
  file, then serialise the lot into the job row); the overflow is counted.

Concurrent job workers (internal/jobs/jobs.go)
- Run() was single-threaded, so a scan stalled every enrichment job behind it.
  Now 4 workers with jittered idle waits.
- Handlers run concurrently, so Register/lookup take an RWMutex, and a
  panicking handler is recovered and marked failed rather than killing a worker
  and stranding a row in 'running'.
- RecoverStale() requeues jobs left 'running' by a dead process; called at boot
  from main.go. Previously such a job stayed 'running' forever and the UI
  polled it forever.

Explore (internal/media/discover.go)
- Full-scanned and sorted the whole library on every home load, and was
  deterministic, so it showed the same 30 items every visit. Now a per-kind
  fair-share id range scan with a random start, wrap-around, dedup and
  interleaving. The doc comment claimed per-kind sampling before; now it does.

Indexes (migrations/0008_query_indexes.sql)
- Benchmarked with ANALYZE over 60k items: browse page 1 13.37ms -> 0.03ms
  (484x, full scan -> covering index), deep page 8x, recently added 3.2x.
- A candidate index for the Continue shelf was built, measured and REJECTED:
  the planner never chose it because `finished = 0` selects the majority of a
  user's rows, so it had no selectivity and would have cost a write on every
  progress save. Recorded in the migration rather than shipped as dead weight.

Canonical timestamps (internal/core/stamp.go)
- time.RFC3339Nano trims trailing zeros, so its fraction width varies and the
  strings are not lexicographically sortable: ".5Z" > ".55Z" as strings though
  .500s < .550s. That broke every ORDER BY <timestamp> and MAX(timestamp),
  including the Continue shelf and Most Viewed. It was wrong even in columns
  written 100% by Go, so it was not a cross-format issue.
- All 14 write sites now use core.Stamp, matching SQLite's own default format
  exactly. Read paths are untouched: RFC3339Nano parses both shapes, so
  existing rows keep working.

Also: busy_timeout 5s -> 15s. Chunked scans still delay a concurrent write, and
measured tail latency on a first scan of 15k new files reached 3.3s, leaving
almost no margin at 5s. Waiting beats failing with SQLITE_BUSY and losing a
playback position. A 10ms inter-batch yield was implemented, measured, found to
have no effect distinguishable from noise (overlapping ranges), and removed.

Verified end to end, not just compiled: orphan recovery logged at boot, scan
returns 202 in 0.7ms, Explore varies across loads, concurrent writes survive a
scan. gofmt/vet clean, 14 packages pass, -race clean on the queue, svelte-check
0/0, fresh install applies all 8 migrations.
2026-10-11 13:10:34 -04:00

157 lines
4.8 KiB
Go

// Package progress stores and retrieves per-user playback/reading position.
//
// One table, one shape, every media kind: milliseconds for audio and video, a
// spine index or page number for text. Callers that don't fit the integer
// model should encode their position into it rather than adding a column.
package progress
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"github.com/mithral/vellum/internal/core"
)
// ErrNoFile means an item has no media_files row to attach progress to.
var ErrNoFile = errors.New("progress: item has no file to track")
// Progress is one user's position in one item's file.
type Progress struct {
ItemID int64 `json:"item_id"`
FileID int64 `json:"file_id"`
Kind string `json:"kind"`
Position int64 `json:"position"`
Duration int64 `json:"duration"`
Percent float64 `json:"percent"`
Finished bool `json:"finished"`
UpdatedAt time.Time `json:"updated_at"`
}
// Get returns the most recently updated progress row for a user and item.
// The second return value is false when the user has never touched the item.
func Get(ctx context.Context, db *sql.DB, userID, itemID int64) (Progress, bool, error) {
var (
p Progress
finished int
updatedAt string
)
err := db.QueryRowContext(ctx, `
SELECT item_id, file_id, kind, position, duration, percent, finished, updated_at
FROM progress
WHERE user_id = ? AND item_id = ?
ORDER BY updated_at DESC
LIMIT 1`, userID, itemID,
).Scan(&p.ItemID, &p.FileID, &p.Kind, &p.Position, &p.Duration, &p.Percent, &finished, &updatedAt)
if errors.Is(err, sql.ErrNoRows) {
return Progress{}, false, nil
}
if err != nil {
return Progress{}, false, fmt.Errorf("progress: get: %w", err)
}
p.Finished = finished == 1
p.UpdatedAt, _ = time.Parse(time.RFC3339Nano, updatedAt)
return p, true, nil
}
// Put upserts progress. A zero FileID resolves to the item's first file, which
// is what single-file items (epub, cbz) always are.
func Put(ctx context.Context, db *sql.DB, userID int64, p Progress) (Progress, error) {
if p.FileID == 0 {
fileID, err := firstFileID(ctx, db, p.ItemID)
if err != nil {
return Progress{}, err
}
p.FileID = fileID
}
if p.Kind == "" {
_ = db.QueryRowContext(ctx, `SELECT kind FROM media_items WHERE id = ?`, p.ItemID).Scan(&p.Kind)
}
if p.Percent < 0 {
p.Percent = 0
}
if p.Percent > 1 {
p.Percent = 1
}
// A timed item at 100% is finished even if the client did not say so.
if p.Duration > 0 && p.Position >= p.Duration {
p.Percent = 1
p.Finished = true
}
now := time.Now().UTC()
_, err := db.ExecContext(ctx, `
INSERT INTO progress (user_id, item_id, file_id, kind, position, duration, percent, finished, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (user_id, item_id, file_id)
DO UPDATE SET kind = excluded.kind, position = excluded.position,
duration = excluded.duration, percent = excluded.percent,
finished = excluded.finished, updated_at = excluded.updated_at`,
userID, p.ItemID, p.FileID, p.Kind, p.Position, p.Duration, p.Percent, boolInt(p.Finished),
core.Stamp(now))
if err != nil {
return Progress{}, fmt.Errorf("progress: put: %w", err)
}
p.UpdatedAt = now
return p, nil
}
// Delete clears progress for a user and item.
func Delete(ctx context.Context, db *sql.DB, userID, itemID int64) error {
_, err := db.ExecContext(ctx, `DELETE FROM progress WHERE user_id = ? AND item_id = ?`, userID, itemID)
return err
}
// RecentlyUpdated powers a "Continue reading/watching" shelf.
func RecentlyUpdated(ctx context.Context, db *sql.DB, userID int64, limit int) ([]Progress, error) {
if limit <= 0 || limit > 200 {
limit = 20
}
rows, err := db.QueryContext(ctx, `
SELECT item_id, file_id, kind, position, duration, percent, finished, updated_at
FROM progress
WHERE user_id = ? AND finished = 0
ORDER BY updated_at DESC
LIMIT ?`, userID, limit)
if err != nil {
return nil, fmt.Errorf("progress: recent: %w", err)
}
defer rows.Close()
out := []Progress{}
for rows.Next() {
var p Progress
var finished int
var updatedAt string
if err := rows.Scan(&p.ItemID, &p.FileID, &p.Kind, &p.Position, &p.Duration, &p.Percent, &finished, &updatedAt); err != nil {
return nil, err
}
p.Finished = finished == 1
p.UpdatedAt, _ = time.Parse(time.RFC3339Nano, updatedAt)
out = append(out, p)
}
return out, rows.Err()
}
func firstFileID(ctx context.Context, db *sql.DB, itemID int64) (int64, error) {
var id int64
err := db.QueryRowContext(ctx,
`SELECT id FROM media_files WHERE item_id = ? ORDER BY id LIMIT 1`, itemID).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return 0, ErrNoFile
}
if err != nil {
return 0, fmt.Errorf("progress: first file: %w", err)
}
return id, nil
}
func boolInt(b bool) int {
if b {
return 1
}
return 0
}