Vellum/internal/metadata/enrich.go

// Package metadata enriches catalogue items from external providers.
//
// It owns the policy: which provider to try, whether a fuzzy candidate is
// actually the item being asked about, and what gets written back. The HTTP
// clients live in internal/providers, which knows nothing about the database.
package metadata

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/mithral/vellum/internal/catalog"
	"github.com/mithral/vellum/internal/core"
	"github.com/mithral/vellum/internal/library"
	"github.com/mithral/vellum/internal/media"
	"github.com/mithral/vellum/internal/providers"
)

// JobKind is the queue job that enriches a single item.
const JobKind = "enrich_item"

// retryAfter is how long a failed or empty lookup is trusted before the
// provider is asked again. It stops a rescan from re-hammering a rate-limited
// API for an item that will never match.
const retryAfter = 30 * 24 * time.Hour

// maxCoverBytes caps a downloaded cover.
const maxCoverBytes = 8 << 20

// Enricher matches items against providers and writes the results back.
type Enricher struct {
	db       *sql.DB
	log      *slog.Logger
	cacheDir string
	client   *http.Client

	// providers resolves the provider list for a media kind. It defaults to
	// the global registry; tests override it to avoid network and global state.
	providers func(kind string) []providers.Provider
}

// NewEnricher builds the enricher.
func NewEnricher(db *sql.DB, log *slog.Logger, cacheDir string) *Enricher {
	return &Enricher{
		db:        db,
		log:       log,
		cacheDir:  cacheDir,
		client:    &http.Client{Timeout: 20 * time.Second},
		providers: providers.For,
	}
}

type enrichPayload struct {
	ItemID int64 `json:"item_id"`
}

// Handle is the queue handler for JobKind.
func (e *Enricher) Handle(ctx context.Context, payload json.RawMessage) error {
	var p enrichPayload
	if err := json.Unmarshal(payload, &p); err != nil {
		return fmt.Errorf("metadata: bad payload: %w", err)
	}
	if p.ItemID <= 0 {
		return errors.New("metadata: payload has no item_id")
	}
	return e.EnrichItem(ctx, p.ItemID)
}

// EnrichItem tries each provider for the item's kind until one matches.
//
// A provider error is recorded and the next provider is tried. The function
// returns an error only for a database failure, because a missing match is a
// normal outcome and must not fail the job.
func (e *Enricher) EnrichItem(ctx context.Context, itemID int64) error {
	// This runs as a background job, not a request, so there is no user and no
	// grant to check. catalog.Load is the unchecked read, which is correct
	// here: the job acts on an item the scanner already admitted to the
	// catalogue, and a job that silently skipped its work because of a grant
	// check it cannot perform would be a bug.
	it, err := catalog.LoadUnchecked(ctx, e.db, itemID)
	if err != nil {
		if errors.Is(err, catalog.ErrNotFound) {
			return nil // deleted since the job was queued
		}
		return err
	}

	// Already matched by a previous run.
	var matched int
	if err := e.db.QueryRowContext(ctx,
		`SELECT count(*) FROM provider_ids WHERE item_id = ?`, itemID).Scan(&matched); err != nil {
		return err
	}
	if matched > 0 {
		return nil
	}

	provs := e.providers(it.Kind)
	if len(provs) == 0 {
		return nil
	}

	q := providers.Query{Title: searchTitle(it.Title), Author: artistHint(it), Kind: it.Kind}
	for _, p := range provs {
		if e.triedRecently(ctx, itemID, p.Name()) {
			continue
		}

		candidates, err := p.Search(ctx, q)
		if err != nil {
			e.log.Warn("metadata provider failed", "provider", p.Name(), "item", itemID, "err", err)
			if recErr := e.record(ctx, itemID, p.Name(), "error", err.Error()); recErr != nil {
				return recErr
			}
			continue
		}
		if len(candidates) == 0 {
			if recErr := e.record(ctx, itemID, p.Name(), "notfound", ""); recErr != nil {
				return recErr
			}
			continue
		}

		// Providers return fuzzy matches. Taking the first blindly is how you
		// end up labelling Daft Punk's album as Franz Kirmann's. Require the
		// title to actually correspond before accepting.
		best, ok := pickBest(candidates, it.Title)
		if !ok {
			e.log.Info("metadata: no close match", "item", itemID, "provider", p.Name(), "title", it.Title)
			if recErr := e.record(ctx, itemID, p.Name(), "notfound", "no close title match"); recErr != nil {
				return recErr
			}
			continue
		}

		if err := e.apply(ctx, it, best); err != nil {
			return err
		}
		if err := e.record(ctx, itemID, p.Name(), "ok", ""); err != nil {
			return err
		}
		e.log.Info("metadata matched", "item", itemID, "provider", p.Name(), "title", best.Title)
		return nil
	}
	return nil
}

// artistHint recovers the artist or author from the directory structure.
//
// Directory-grouped kinds store the item as "Artist/Album" or "Author/Book",
// so the parent directory is a strong signal that the provider search should
// use to disambiguate. Without it, "Random Access Memories" matches a release
// by an unrelated artist.
func artistHint(it catalog.Item) string {
	if library.GroupModeFor(it.Kind) != library.GroupByDirectory {
		return ""
	}
	dir := path.Dir(it.Path)
	if dir == "." || dir == "/" || dir == "" {
		return ""
	}
	return path.Base(dir)
}

// searchTitle strips the region, language, and version tags that ROM and
// release filenames carry — "(USA)", "[!]", "(Rev 1)", "(En,Fr,De)" — because
// they are noise to a metadata search. "Sonic the Hedgehog (USA)" matches
// nothing; "Sonic the Hedgehog" matches immediately.
func searchTitle(s string) string {
	s = strings.TrimSpace(s)
	for {
		trimmed := strings.TrimSpace(s)
		var cut int
		switch {
		case strings.HasSuffix(trimmed, ")"):
			cut = strings.LastIndexByte(trimmed, '(')
		case strings.HasSuffix(trimmed, "]"):
			cut = strings.LastIndexByte(trimmed, '[')
		default:
			return trimmed
		}
		// cut <= 0 would mean the whole title is a bracketed group; leave it.
		if cut <= 0 {
			return trimmed
		}
		s = strings.TrimSpace(trimmed[:cut])
		if s == "" {
			return trimmed
		}
	}
}

// pickBest returns the first candidate whose title corresponds to want.
//
// The bar is deliberately loose (normalised equality or containment in either
// direction) but not absent: it accepts "The Hobbit" against "The Hobbit, or
// There and Back Again" and rejects "Random Access Memories" against an
// unrelated album that merely shares a word.
func pickBest(candidates []providers.Candidate, want string) (providers.Candidate, bool) {
	w := normalizeTitle(want)
	if len(w) < 3 {
		return providers.Candidate{}, false
	}
	for _, c := range candidates {
		got := normalizeTitle(c.Title)
		if len(got) < 3 {
			continue
		}
		if got == w || strings.Contains(got, w) || strings.Contains(w, got) {
			return c, true
		}
	}
	return providers.Candidate{}, false
}

// normalizeTitle lowercases and strips punctuation so titles can be compared
// regardless of formatting differences.
func normalizeTitle(s string) string {
	var b strings.Builder
	lastSpace := true
	for _, r := range strings.ToLower(s) {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
			lastSpace = false
		case r >= 'à' && r <= 'ÿ':
			// Keep accented letters as-is; providers and files usually agree.
			b.WriteRune(r)
			lastSpace = false
		default:
			if !lastSpace {
				b.WriteByte(' ')
				lastSpace = true
			}
		}
	}
	return strings.TrimSpace(b.String())
}

// apply writes a chosen candidate back to the database.
func (e *Enricher) apply(ctx context.Context, it catalog.Item, c providers.Candidate) error {
	tx, err := e.db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	if _, err := tx.ExecContext(ctx, `
		INSERT INTO provider_ids (item_id, provider, external_id) VALUES (?, ?, ?)
		ON CONFLICT (item_id, provider) DO UPDATE SET external_id = excluded.external_id`,
		it.ID, c.Provider, c.ExternalID); err != nil {
		return fmt.Errorf("metadata: store provider id: %w", err)
	}

	// Only overwrite the scanner's title when the catalogue gave us one.
	title := it.Title
	if strings.TrimSpace(c.Title) != "" {
		title = strings.TrimSpace(c.Title)
	}
	if _, err := tx.ExecContext(ctx, `
		UPDATE media_items SET title = ?, sort_title = ?, year = ? WHERE id = ?`,
		title, strings.ToLower(title), nullableYear(c.Year), it.ID); err != nil {
		return fmt.Errorf("metadata: update item: %w", err)
	}

	if _, err := tx.ExecContext(ctx,
		`DELETE FROM item_people WHERE item_id = ? AND role = 'author'`, it.ID); err != nil {
		return err
	}
	for i, name := range c.Authors {
		name = strings.TrimSpace(name)
		if name == "" {
			continue
		}
		var personID int64
		if err := tx.QueryRowContext(ctx, `
			INSERT INTO people (name) VALUES (?)
			ON CONFLICT (name) DO UPDATE SET name = excluded.name
			RETURNING id`, name).Scan(&personID); err != nil {
			return fmt.Errorf("metadata: upsert person: %w", err)
		}
		if _, err := tx.ExecContext(ctx,
			`INSERT INTO item_people (item_id, person_id, role, ord) VALUES (?, ?, 'author', ?)
			 ON CONFLICT (item_id, person_id, role) DO UPDATE SET ord = excluded.ord`,
			it.ID, personID, i); err != nil {
			return fmt.Errorf("metadata: link author: %w", err)
		}
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	// Cover download is best-effort and outside the transaction: a slow image
	// host must not hold a write lock.
	if c.CoverURL != "" {
		if err := e.downloadCover(ctx, it, c); err != nil {
			e.log.Warn("cover download failed", "item", it.ID, "err", err)
		}
	}
	return nil
}

// downloadCover fetches a cover into the cache and records it as an asset.
func (e *Enricher) downloadCover(ctx context.Context, it catalog.Item, c providers.Candidate) error {
	var existing int
	if err := e.db.QueryRowContext(ctx,
		`SELECT count(*) FROM assets WHERE item_id = ? AND role = 'cover'`, it.ID).Scan(&existing); err != nil {
		return err
	}
	if existing > 0 {
		return nil
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.CoverURL, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "Vellum/0.1 (self-hosted media server)")

	resp, err := e.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("cover status %d", resp.StatusCode)
	}

	data, err := io.ReadAll(io.LimitReader(resp.Body, maxCoverBytes))
	if err != nil {
		return err
	}
	if len(data) == 0 {
		return errors.New("empty cover")
	}

	dir, err := media.CoverCacheDir(e.cacheDir, it.Kind)
	if err != nil {
		return err
	}
	ext := coverExt(resp.Header.Get("Content-Type"), c.CoverURL)
	outPath := filepath.Join(dir, fmt.Sprintf("%d%s", it.ID, ext))

	tmp := outPath + ".tmp"
	if err := os.WriteFile(tmp, data, 0o640); err != nil {
		return err
	}
	if err := os.Rename(tmp, outPath); err != nil {
		_ = os.Remove(tmp)
		return err
	}

	_, err = e.db.ExecContext(ctx, `
		INSERT INTO assets (item_id, role, path, bytes) VALUES (?, 'cover', ?, ?)
		ON CONFLICT (item_id, role, path) DO NOTHING`, it.ID, outPath, len(data))
	return err
}

// triedRecently reports whether a provider was already consulted for an item.
func (e *Enricher) triedRecently(ctx context.Context, itemID int64, provider string) bool {
	var status, fetchedAt string
	err := e.db.QueryRowContext(ctx,
		`SELECT status, fetched_at FROM metadata_fetch WHERE item_id = ? AND provider = ?`,
		itemID, provider).Scan(&status, &fetchedAt)
	if err != nil {
		return false
	}
	if status != "ok" && status != "notfound" {
		return false // errors are always worth retrying
	}
	when, err := time.Parse(time.RFC3339Nano, fetchedAt)
	if err != nil {
		return false
	}
	return time.Since(when) < retryAfter
}

func (e *Enricher) record(ctx context.Context, itemID int64, provider, status, detail string) error {
	_, err := e.db.ExecContext(ctx, `
		INSERT INTO metadata_fetch (item_id, provider, status, detail, fetched_at)
		VALUES (?, ?, ?, ?, ?)
		ON CONFLICT (item_id, provider)
		DO UPDATE SET status = excluded.status, detail = excluded.detail, fetched_at = excluded.fetched_at`,
		itemID, provider, status, detail, core.Now())
	return err
}

func coverExt(contentType, url string) string {
	switch {
	case strings.Contains(contentType, "png"):
		return ".png"
	case strings.Contains(contentType, "webp"):
		return ".webp"
	case strings.Contains(contentType, "gif"):
		return ".gif"
	case strings.Contains(contentType, "jpeg"), strings.Contains(contentType, "jpg"):
		return ".jpg"
	}
	lower := strings.ToLower(url)
	for _, ext := range []string{".png", ".webp", ".gif", ".jpeg", ".jpg"} {
		if strings.Contains(lower, ext) {
			return ext
		}
	}
	return ".jpg"
}

// nullableYear stores 0 as NULL rather than a literal year zero.
func nullableYear(year int) any {
	if year <= 0 {
		return nil
	}
	return year
}