// Package metadata enriches catalogue items from external providers. // // It owns the policy: which provider to try, whether a fuzzy candidate is // actually the item being asked about, and what gets written back. The HTTP // clients live in internal/providers, which knows nothing about the database. package metadata import ( "context" "database/sql" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" "os" "path" "path/filepath" "strings" "time" "github.com/mithral/vellum/internal/catalog" "github.com/mithral/vellum/internal/core" "github.com/mithral/vellum/internal/library" "github.com/mithral/vellum/internal/media" "github.com/mithral/vellum/internal/providers" ) // JobKind is the queue job that enriches a single item. const JobKind = "enrich_item" // retryAfter is how long a failed or empty lookup is trusted before the // provider is asked again. It stops a rescan from re-hammering a rate-limited // API for an item that will never match. const retryAfter = 30 * 24 * time.Hour // maxCoverBytes caps a downloaded cover. const maxCoverBytes = 8 << 20 // Enricher matches items against providers and writes the results back. type Enricher struct { db *sql.DB log *slog.Logger cacheDir string client *http.Client // providers resolves the provider list for a media kind. It defaults to // the global registry; tests override it to avoid network and global state. providers func(kind string) []providers.Provider } // NewEnricher builds the enricher. func NewEnricher(db *sql.DB, log *slog.Logger, cacheDir string) *Enricher { return &Enricher{ db: db, log: log, cacheDir: cacheDir, client: &http.Client{Timeout: 20 * time.Second}, providers: providers.For, } } type enrichPayload struct { ItemID int64 `json:"item_id"` } // Handle is the queue handler for JobKind. func (e *Enricher) Handle(ctx context.Context, payload json.RawMessage) error { var p enrichPayload if err := json.Unmarshal(payload, &p); err != nil { return fmt.Errorf("metadata: bad payload: %w", err) } if p.ItemID <= 0 { return errors.New("metadata: payload has no item_id") } return e.EnrichItem(ctx, p.ItemID) } // EnrichItem tries each provider for the item's kind until one matches. // // A provider error is recorded and the next provider is tried. The function // returns an error only for a database failure, because a missing match is a // normal outcome and must not fail the job. func (e *Enricher) EnrichItem(ctx context.Context, itemID int64) error { // This runs as a background job, not a request, so there is no user and no // grant to check. catalog.Load is the unchecked read, which is correct // here: the job acts on an item the scanner already admitted to the // catalogue, and a job that silently skipped its work because of a grant // check it cannot perform would be a bug. it, err := catalog.LoadUnchecked(ctx, e.db, itemID) if err != nil { if errors.Is(err, catalog.ErrNotFound) { return nil // deleted since the job was queued } return err } // Already matched by a previous run. var matched int if err := e.db.QueryRowContext(ctx, `SELECT count(*) FROM provider_ids WHERE item_id = ?`, itemID).Scan(&matched); err != nil { return err } if matched > 0 { return nil } provs := e.providers(it.Kind) if len(provs) == 0 { return nil } q := providers.Query{Title: searchTitle(it.Title), Author: artistHint(it), Kind: it.Kind} for _, p := range provs { if e.triedRecently(ctx, itemID, p.Name()) { continue } candidates, err := p.Search(ctx, q) if err != nil { e.log.Warn("metadata provider failed", "provider", p.Name(), "item", itemID, "err", err) if recErr := e.record(ctx, itemID, p.Name(), "error", err.Error()); recErr != nil { return recErr } continue } if len(candidates) == 0 { if recErr := e.record(ctx, itemID, p.Name(), "notfound", ""); recErr != nil { return recErr } continue } // Providers return fuzzy matches. Taking the first blindly is how you // end up labelling Daft Punk's album as Franz Kirmann's. Require the // title to actually correspond before accepting. best, ok := pickBest(candidates, it.Title) if !ok { e.log.Info("metadata: no close match", "item", itemID, "provider", p.Name(), "title", it.Title) if recErr := e.record(ctx, itemID, p.Name(), "notfound", "no close title match"); recErr != nil { return recErr } continue } if err := e.apply(ctx, it, best); err != nil { return err } if err := e.record(ctx, itemID, p.Name(), "ok", ""); err != nil { return err } e.log.Info("metadata matched", "item", itemID, "provider", p.Name(), "title", best.Title) return nil } return nil } // artistHint recovers the artist or author from the directory structure. // // Directory-grouped kinds store the item as "Artist/Album" or "Author/Book", // so the parent directory is a strong signal that the provider search should // use to disambiguate. Without it, "Random Access Memories" matches a release // by an unrelated artist. func artistHint(it catalog.Item) string { if library.GroupModeFor(it.Kind) != library.GroupByDirectory { return "" } dir := path.Dir(it.Path) if dir == "." || dir == "/" || dir == "" { return "" } return path.Base(dir) } // searchTitle strips the region, language, and version tags that ROM and // release filenames carry — "(USA)", "[!]", "(Rev 1)", "(En,Fr,De)" — because // they are noise to a metadata search. "Sonic the Hedgehog (USA)" matches // nothing; "Sonic the Hedgehog" matches immediately. func searchTitle(s string) string { s = strings.TrimSpace(s) for { trimmed := strings.TrimSpace(s) var cut int switch { case strings.HasSuffix(trimmed, ")"): cut = strings.LastIndexByte(trimmed, '(') case strings.HasSuffix(trimmed, "]"): cut = strings.LastIndexByte(trimmed, '[') default: return trimmed } // cut <= 0 would mean the whole title is a bracketed group; leave it. if cut <= 0 { return trimmed } s = strings.TrimSpace(trimmed[:cut]) if s == "" { return trimmed } } } // pickBest returns the first candidate whose title corresponds to want. // // The bar is deliberately loose (normalised equality or containment in either // direction) but not absent: it accepts "The Hobbit" against "The Hobbit, or // There and Back Again" and rejects "Random Access Memories" against an // unrelated album that merely shares a word. func pickBest(candidates []providers.Candidate, want string) (providers.Candidate, bool) { w := normalizeTitle(want) if len(w) < 3 { return providers.Candidate{}, false } for _, c := range candidates { got := normalizeTitle(c.Title) if len(got) < 3 { continue } if got == w || strings.Contains(got, w) || strings.Contains(w, got) { return c, true } } return providers.Candidate{}, false } // normalizeTitle lowercases and strips punctuation so titles can be compared // regardless of formatting differences. func normalizeTitle(s string) string { var b strings.Builder lastSpace := true for _, r := range strings.ToLower(s) { switch { case r >= 'a' && r <= 'z', r >= '0' && r <= '9': b.WriteRune(r) lastSpace = false case r >= 'à' && r <= 'ÿ': // Keep accented letters as-is; providers and files usually agree. b.WriteRune(r) lastSpace = false default: if !lastSpace { b.WriteByte(' ') lastSpace = true } } } return strings.TrimSpace(b.String()) } // apply writes a chosen candidate back to the database. func (e *Enricher) apply(ctx context.Context, it catalog.Item, c providers.Candidate) error { tx, err := e.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck if _, err := tx.ExecContext(ctx, ` INSERT INTO provider_ids (item_id, provider, external_id) VALUES (?, ?, ?) ON CONFLICT (item_id, provider) DO UPDATE SET external_id = excluded.external_id`, it.ID, c.Provider, c.ExternalID); err != nil { return fmt.Errorf("metadata: store provider id: %w", err) } // Only overwrite the scanner's title when the catalogue gave us one. title := it.Title if strings.TrimSpace(c.Title) != "" { title = strings.TrimSpace(c.Title) } if _, err := tx.ExecContext(ctx, ` UPDATE media_items SET title = ?, sort_title = ?, year = ? WHERE id = ?`, title, strings.ToLower(title), nullableYear(c.Year), it.ID); err != nil { return fmt.Errorf("metadata: update item: %w", err) } if _, err := tx.ExecContext(ctx, `DELETE FROM item_people WHERE item_id = ? AND role = 'author'`, it.ID); err != nil { return err } for i, name := range c.Authors { name = strings.TrimSpace(name) if name == "" { continue } var personID int64 if err := tx.QueryRowContext(ctx, ` INSERT INTO people (name) VALUES (?) ON CONFLICT (name) DO UPDATE SET name = excluded.name RETURNING id`, name).Scan(&personID); err != nil { return fmt.Errorf("metadata: upsert person: %w", err) } if _, err := tx.ExecContext(ctx, `INSERT INTO item_people (item_id, person_id, role, ord) VALUES (?, ?, 'author', ?) ON CONFLICT (item_id, person_id, role) DO UPDATE SET ord = excluded.ord`, it.ID, personID, i); err != nil { return fmt.Errorf("metadata: link author: %w", err) } } if err := tx.Commit(); err != nil { return err } // Cover download is best-effort and outside the transaction: a slow image // host must not hold a write lock. if c.CoverURL != "" { if err := e.downloadCover(ctx, it, c); err != nil { e.log.Warn("cover download failed", "item", it.ID, "err", err) } } return nil } // downloadCover fetches a cover into the cache and records it as an asset. func (e *Enricher) downloadCover(ctx context.Context, it catalog.Item, c providers.Candidate) error { var existing int if err := e.db.QueryRowContext(ctx, `SELECT count(*) FROM assets WHERE item_id = ? AND role = 'cover'`, it.ID).Scan(&existing); err != nil { return err } if existing > 0 { return nil } req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.CoverURL, nil) if err != nil { return err } req.Header.Set("User-Agent", "Vellum/0.1 (self-hosted media server)") resp, err := e.client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("cover status %d", resp.StatusCode) } data, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverBytes)) if err != nil { return err } if len(data) == 0 { return errors.New("empty cover") } dir, err := media.CoverCacheDir(e.cacheDir, it.Kind) if err != nil { return err } ext := coverExt(resp.Header.Get("Content-Type"), c.CoverURL) outPath := filepath.Join(dir, fmt.Sprintf("%d%s", it.ID, ext)) tmp := outPath + ".tmp" if err := os.WriteFile(tmp, data, 0o640); err != nil { return err } if err := os.Rename(tmp, outPath); err != nil { _ = os.Remove(tmp) return err } _, err = e.db.ExecContext(ctx, ` INSERT INTO assets (item_id, role, path, bytes) VALUES (?, 'cover', ?, ?) ON CONFLICT (item_id, role, path) DO NOTHING`, it.ID, outPath, len(data)) return err } // triedRecently reports whether a provider was already consulted for an item. func (e *Enricher) triedRecently(ctx context.Context, itemID int64, provider string) bool { var status, fetchedAt string err := e.db.QueryRowContext(ctx, `SELECT status, fetched_at FROM metadata_fetch WHERE item_id = ? AND provider = ?`, itemID, provider).Scan(&status, &fetchedAt) if err != nil { return false } if status != "ok" && status != "notfound" { return false // errors are always worth retrying } when, err := time.Parse(time.RFC3339Nano, fetchedAt) if err != nil { return false } return time.Since(when) < retryAfter } func (e *Enricher) record(ctx context.Context, itemID int64, provider, status, detail string) error { _, err := e.db.ExecContext(ctx, ` INSERT INTO metadata_fetch (item_id, provider, status, detail, fetched_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT (item_id, provider) DO UPDATE SET status = excluded.status, detail = excluded.detail, fetched_at = excluded.fetched_at`, itemID, provider, status, detail, core.Now()) return err } func coverExt(contentType, url string) string { switch { case strings.Contains(contentType, "png"): return ".png" case strings.Contains(contentType, "webp"): return ".webp" case strings.Contains(contentType, "gif"): return ".gif" case strings.Contains(contentType, "jpeg"), strings.Contains(contentType, "jpg"): return ".jpg" } lower := strings.ToLower(url) for _, ext := range []string{".png", ".webp", ".gif", ".jpeg", ".jpg"} { if strings.Contains(lower, ext) { return ext } } return ".jpg" } // nullableYear stores 0 as NULL rather than a literal year zero. func nullableYear(year int) any { if year <= 0 { return nil } return year }