A 50k-file library used to wedge the database. This wave fixes the four reasons why, plus a timestamp-ordering bug the work exposed. Chunked scans (internal/library/scan.go) - One transaction wrapped the whole reconciliation, holding SQLite's single write lock for the entire scan. Now 500 groups per transaction, and the two missing-file passes are batched too. - Verified: a 20,000-file scan reports partial progress (scanned=1500 mid-scan) and a concurrent progress save returns 200 throughout. Under the old code that save blocked for the whole scan. - Behaviour change: a scan is no longer all-or-nothing. A crash now leaves a consistent partial scan. The scan is idempotent, so re-running finishes it. - res.Errors is capped at 100 (a broken mount used to append one string per file, then serialise the lot into the job row); the overflow is counted. Concurrent job workers (internal/jobs/jobs.go) - Run() was single-threaded, so a scan stalled every enrichment job behind it. Now 4 workers with jittered idle waits. - Handlers run concurrently, so Register/lookup take an RWMutex, and a panicking handler is recovered and marked failed rather than killing a worker and stranding a row in 'running'. - RecoverStale() requeues jobs left 'running' by a dead process; called at boot from main.go. Previously such a job stayed 'running' forever and the UI polled it forever. Explore (internal/media/discover.go) - Full-scanned and sorted the whole library on every home load, and was deterministic, so it showed the same 30 items every visit. Now a per-kind fair-share id range scan with a random start, wrap-around, dedup and interleaving. The doc comment claimed per-kind sampling before; now it does. Indexes (migrations/0008_query_indexes.sql) - Benchmarked with ANALYZE over 60k items: browse page 1 13.37ms -> 0.03ms (484x, full scan -> covering index), deep page 8x, recently added 3.2x. - A candidate index for the Continue shelf was built, measured and REJECTED: the planner never chose it because `finished = 0` selects the majority of a user's rows, so it had no selectivity and would have cost a write on every progress save. Recorded in the migration rather than shipped as dead weight. Canonical timestamps (internal/core/stamp.go) - time.RFC3339Nano trims trailing zeros, so its fraction width varies and the strings are not lexicographically sortable: ".5Z" > ".55Z" as strings though .500s < .550s. That broke every ORDER BY <timestamp> and MAX(timestamp), including the Continue shelf and Most Viewed. It was wrong even in columns written 100% by Go, so it was not a cross-format issue. - All 14 write sites now use core.Stamp, matching SQLite's own default format exactly. Read paths are untouched: RFC3339Nano parses both shapes, so existing rows keep working. Also: busy_timeout 5s -> 15s. Chunked scans still delay a concurrent write, and measured tail latency on a first scan of 15k new files reached 3.3s, leaving almost no margin at 5s. Waiting beats failing with SQLITE_BUSY and losing a playback position. A 10ms inter-batch yield was implemented, measured, found to have no effect distinguishable from noise (overlapping ranges), and removed. Verified end to end, not just compiled: orphan recovery logged at boot, scan returns 202 in 0.7ms, Explore varies across loads, concurrent writes survive a scan. gofmt/vet clean, 14 packages pass, -race clean on the queue, svelte-check 0/0, fresh install applies all 8 migrations.
217 lines
6.4 KiB
Go
217 lines
6.4 KiB
Go
package media
|
|
|
|
import (
|
|
"database/sql"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/mithral/vellum/internal/auth"
|
|
"github.com/mithral/vellum/internal/core"
|
|
"github.com/mithral/vellum/internal/httpx"
|
|
)
|
|
|
|
// shelfItem is one card on the home screen.
|
|
type shelfItem struct {
|
|
ID int64 `json:"id"`
|
|
LibraryID int64 `json:"library_id"`
|
|
Kind string `json:"kind"`
|
|
Title string `json:"title"`
|
|
Percent float64 `json:"percent"`
|
|
FileID int64 `json:"file_id"`
|
|
Views int `json:"views"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
}
|
|
|
|
// suggestion is a recommendation with the reason it was made.
|
|
type suggestion struct {
|
|
ID int64 `json:"id"`
|
|
LibraryID int64 `json:"library_id"`
|
|
Kind string `json:"kind"`
|
|
Title string `json:"title"`
|
|
Year int `json:"year"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
// homeResponse is everything the landing screen needs in one request.
|
|
type homeResponse struct {
|
|
Continue []shelfItem `json:"continue"`
|
|
MostViewed []shelfItem `json:"most_viewed"`
|
|
Suggestions []suggestion `json:"suggestions"`
|
|
Recent []shelfItem `json:"recent"`
|
|
Explore []shelfItem `json:"explore"`
|
|
CountsByKind map[string]int `json:"counts_by_kind"`
|
|
LibraryCount int `json:"library_count"`
|
|
TotalItems int `json:"total_items"`
|
|
}
|
|
|
|
// handleHome powers the landing screen: five shelves plus the counts the
|
|
// sidebar needs.
|
|
//
|
|
// One request rather than six round trips, because the landing screen is the
|
|
// first thing a user sees and should not pop in piece by piece.
|
|
func (h *Handlers) handleHome(w http.ResponseWriter, r *http.Request, u auth.User) {
|
|
ctx := r.Context()
|
|
|
|
libs, err := auth.VisibleLibraryIDs(ctx, h.db, u)
|
|
if err != nil {
|
|
h.fail(w, "visible libraries", err)
|
|
return
|
|
}
|
|
|
|
resp := homeResponse{
|
|
Continue: []shelfItem{},
|
|
MostViewed: []shelfItem{},
|
|
Suggestions: []suggestion{},
|
|
Recent: []shelfItem{},
|
|
Explore: []shelfItem{},
|
|
CountsByKind: map[string]int{},
|
|
}
|
|
if len(libs) == 0 {
|
|
httpx.WriteJSON(w, http.StatusOK, resp)
|
|
return
|
|
}
|
|
resp.LibraryCount = len(libs)
|
|
in, args := inClause(libs)
|
|
|
|
// ── Continue: started but unfinished, one row per item ───────────────
|
|
//
|
|
// GROUP BY item_id with MAX(updated_at): SQLite takes the bare column
|
|
// values from the row that produced the max, so a multi-track audiobook
|
|
// contributes its most recent track rather than one card per chapter.
|
|
rows, err := h.db.QueryContext(ctx, `
|
|
SELECT i.id, i.library_id, i.kind, i.title, p.percent, p.file_id, 0,
|
|
MAX(p.updated_at) AS updated_at
|
|
FROM progress p
|
|
JOIN media_items i ON i.id = p.item_id
|
|
WHERE p.user_id = ? AND p.finished = 0 AND i.missing = 0
|
|
AND i.library_id IN (`+in+`)
|
|
GROUP BY p.item_id
|
|
ORDER BY updated_at DESC
|
|
LIMIT 20`, append([]any{u.ID}, args...)...)
|
|
if err != nil {
|
|
h.fail(w, "home: continue", err)
|
|
return
|
|
}
|
|
if resp.Continue, err = scanShelf(rows); err != nil {
|
|
h.fail(w, "home: scan continue", err)
|
|
return
|
|
}
|
|
|
|
// ── Most viewed: what this user actually opens ───────────────────────
|
|
if resp.MostViewed, err = MostViewed(ctx, h.db, u.ID, libs, 20); err != nil {
|
|
h.fail(w, "home: most viewed", err)
|
|
return
|
|
}
|
|
|
|
// ── Suggestions: related to what they have been consuming ────────────
|
|
if resp.Suggestions, err = Suggestions(ctx, h.db, u.ID, libs, 20); err != nil {
|
|
h.fail(w, "home: suggestions", err)
|
|
return
|
|
}
|
|
|
|
// ── Recently added ───────────────────────────────────────────────────
|
|
rows, err = h.db.QueryContext(ctx, `
|
|
SELECT i.id, i.library_id, i.kind, i.title, 0, 0, 0, i.added_at
|
|
FROM media_items i
|
|
WHERE i.missing = 0 AND i.library_id IN (`+in+`)
|
|
ORDER BY i.added_at DESC, i.id DESC
|
|
LIMIT 40`, args...)
|
|
if err != nil {
|
|
h.fail(w, "home: recent", err)
|
|
return
|
|
}
|
|
if resp.Recent, err = scanShelf(rows); err != nil {
|
|
h.fail(w, "home: scan recent", err)
|
|
return
|
|
}
|
|
|
|
// ── Explore: a mixed sample of everything else ───────────────────────
|
|
if resp.Explore, err = Explore(ctx, h.db, u.ID, libs, 30); err != nil {
|
|
h.fail(w, "home: explore", err)
|
|
return
|
|
}
|
|
|
|
// ── Counts per kind, for the sidebar and section headers ─────────────
|
|
rows, err = h.db.QueryContext(ctx, `
|
|
SELECT i.kind, count(*) FROM media_items i
|
|
WHERE i.missing = 0 AND i.library_id IN (`+in+`)
|
|
GROUP BY i.kind`, args...)
|
|
if err != nil {
|
|
h.fail(w, "home: counts", err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var kind string
|
|
var n int
|
|
if err := rows.Scan(&kind, &n); err != nil {
|
|
h.fail(w, "home: scan counts", err)
|
|
return
|
|
}
|
|
resp.CountsByKind[kind] = n
|
|
resp.TotalItems += n
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
h.fail(w, "home: iterate counts", err)
|
|
return
|
|
}
|
|
|
|
httpx.WriteJSON(w, http.StatusOK, resp)
|
|
}
|
|
|
|
// handleRecordView increments the user's view count for an item.
|
|
func (h *Handlers) handleRecordView(w http.ResponseWriter, r *http.Request, u auth.User) {
|
|
it, ok := h.authorized(w, r, u)
|
|
if !ok {
|
|
return
|
|
}
|
|
views, err := RecordView(r.Context(), h.db, u.ID, it.ID)
|
|
if err != nil {
|
|
h.fail(w, "record view", err)
|
|
return
|
|
}
|
|
httpx.WriteJSON(w, http.StatusOK, map[string]any{"item_id": it.ID, "views": views})
|
|
}
|
|
|
|
// handleAuthors returns the people credited on an item.
|
|
func (h *Handlers) handleAuthors(w http.ResponseWriter, r *http.Request, u auth.User) {
|
|
it, ok := h.authorized(w, r, u)
|
|
if !ok {
|
|
return
|
|
}
|
|
authors, err := h.authors(r, it.ID)
|
|
if err != nil {
|
|
h.fail(w, "authors", err)
|
|
return
|
|
}
|
|
httpx.WriteJSON(w, http.StatusOK, map[string]any{"authors": authors})
|
|
}
|
|
|
|
func scanShelf(rows *sql.Rows) ([]shelfItem, error) {
|
|
defer rows.Close()
|
|
out := []shelfItem{}
|
|
for rows.Next() {
|
|
var s shelfItem
|
|
if err := rows.Scan(&s.ID, &s.LibraryID, &s.Kind, &s.Title, &s.Percent, &s.FileID, &s.Views, &s.UpdatedAt); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, s)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// inClause builds "?,?,?" plus the matching args for an IN list.
|
|
func inClause(ids []int64) (string, []any) {
|
|
placeholders := make([]string, len(ids))
|
|
args := make([]any, len(ids))
|
|
for i, id := range ids {
|
|
placeholders[i] = "?"
|
|
args[i] = id
|
|
}
|
|
return strings.Join(placeholders, ","), args
|
|
}
|
|
|
|
// nowRFC3339 returns the canonical database timestamp for a write.
|
|
func nowRFC3339() string {
|
|
return core.Now()
|
|
}
|