diff --git a/README.md b/README.md index c6bc3cb..d472675 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The full plan lives in `~/.opencode/plan/vellum.md`. | **3** | EPUB reader + progress | **done** | | **4** | CBZ comic reader | **done** | | **PDF** | PDF reading (pdf.js) | **done** | -| **5** | Job queue, metadata (Open Library, Google Books, MusicBrainz, ComicVine, IGDB) | **done** | +| **5** | Job queue, metadata (Open Library, Google Books, MusicBrainz, ComicVine, IGDB, TMDB) | **done** | | **6a** | Audiobooks + music playback | **done** | | **6c** | Photos + thumbnails | **done** | | **7** | Games: ROM scanning, platforms, cloud saves, EmulatorJS | **done** | @@ -72,6 +72,7 @@ binary runs with no configuration at all. | `VELLUM_DISCOVERY` | `on` | Set `off` to stop advertising over mDNS on the LAN | | `VELLUM_IGDB_CLIENT_ID` | *(none)* | Enables game metadata via IGDB | | `VELLUM_IGDB_CLIENT_SECRET` | *(none)* | IGDB client secret (both required) | +| `VELLUM_TMDB_API_KEY` | *(none)* | Enables movie and TV metadata via TMDB | | `VELLUM_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` | | `VELLUM_LOG_FORMAT` | `text` | `text` or `json` | | `VELLUM_SESSION_TTL` | `720h` | Login session lifetime | @@ -99,7 +100,7 @@ internal/ home/ the five home shelves search/ FTS5 queries metadata/ the enrichment job and its matching policy - providers/ Open Library, Google Books, MusicBrainz, ComicVine, IGDB + providers/ Open Library, Google Books, MusicBrainz, ComicVine, IGDB, TMDB playback/ playback engine selection endpoint engine/ playback engine registry (html5, mpv, vlc, mdk) progress/ the polymorphic per-user progress store diff --git a/cmd/vellum/main.go b/cmd/vellum/main.go index d81c267..b04da0d 100644 --- a/cmd/vellum/main.go +++ b/cmd/vellum/main.go @@ -167,6 +167,11 @@ func run(cfg core.Config, log *slog.Logger) error { } else { log.Info("game metadata disabled: set VELLUM_IGDB_CLIENT_ID and VELLUM_IGDB_CLIENT_SECRET to enable IGDB lookups") } + if cfg.TMDBAPIKey != "" { + providers.Register(providers.NewTMDB(cfg.TMDBAPIKey)) + } else { + log.Info("video metadata disabled: set VELLUM_TMDB_API_KEY to enable TMDB movie and TV lookups") + } enricher := metadata.NewEnricher(db, log, cfg.ConfigDir) queue.Register(metadata.JobKind, enricher.Handle) metadata.NewHandlers(db, log, authenticator, queue).Register(api.Mux()) diff --git a/internal/core/config.go b/internal/core/config.go index f7ce47f..2999020 100644 --- a/internal/core/config.go +++ b/internal/core/config.go @@ -37,6 +37,10 @@ type Config struct { // the provider is simply absent when this is empty. ComicVineAPIKey string + // TMDBAPIKey enables movie and TV metadata. TMDB has no keyless tier, so the + // provider is simply absent when this is empty. + TMDBAPIKey string + // IGDBClientID / IGDBClientSecret enable game metadata via Twitch's // client-credentials flow. Both must be set for the provider to appear. IGDBClientID string @@ -81,6 +85,8 @@ func LoadConfig() (Config, error) { ComicVineAPIKey: env("VELLUM_COMICVINE_API_KEY", ""), + TMDBAPIKey: env("VELLUM_TMDB_API_KEY", ""), + IGDBClientID: env("VELLUM_IGDB_CLIENT_ID", ""), IGDBClientSecret: env("VELLUM_IGDB_CLIENT_SECRET", ""), CookieName: env("VELLUM_COOKIE_NAME", "vellum_session"), diff --git a/internal/metadata/enrich.go b/internal/metadata/enrich.go index 81c7514..f79cfd4 100644 --- a/internal/metadata/enrich.go +++ b/internal/metadata/enrich.go @@ -111,7 +111,10 @@ func (e *Enricher) EnrichItem(ctx context.Context, itemID int64) error { return nil } - q := providers.Query{Title: searchTitle(it.Title), Author: artistHint(it), Kind: it.Kind} + // Year is what disambiguates a remake from the original, and it is the one + // hint every provider that matches titles can use. Providers that cannot + // ignore it. + q := providers.Query{Title: searchTitle(it.Title), Year: it.Year, Author: artistHint(it), Kind: it.Kind} for _, p := range provs { if e.triedRecently(ctx, itemID, p.Name()) { continue diff --git a/internal/providers/provider.go b/internal/providers/provider.go index e18ac9b..45bed50 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -18,7 +18,10 @@ import ( // Query is what a provider is asked to match. type Query struct { - Title string + Title string + // Year narrows a search when the caller knows it. Providers that cannot use + // it ignore it. + Year int Author string ISBN string Kind string diff --git a/internal/providers/tmdb.go b/internal/providers/tmdb.go new file mode 100644 index 0000000..ff56d3e --- /dev/null +++ b/internal/providers/tmdb.go @@ -0,0 +1,185 @@ +package providers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// TMDB matches video — films and television — against The Movie Database. +// +// A Query does not say which of the two it is, and TMDB keeps them in separate +// namespaces behind separate endpoints, so Search asks both at once rather than +// guessing from the title. The results are merged by TMDB's own popularity, +// which is the only ranking signal available without a second request per item. +type TMDB struct { + APIKey string + BaseURL string + Client *http.Client +} + +// NewTMDB builds the provider. An empty key yields a provider that finds +// nothing, so a misconfiguration degrades rather than erroring. +func NewTMDB(apiKey string) *TMDB { + return &TMDB{ + APIKey: apiKey, + BaseURL: "https://api.themoviedb.org/3", + // One client for both requests: connection reuse across the pair is + // most of what makes the concurrent form cheap. + Client: &http.Client{Timeout: 15 * time.Second}, + } +} + +func (p *TMDB) Name() string { return "tmdb" } + +func (p *TMDB) Kinds() []string { return []string{"video"} } + +// tmdbResponse is the subset of a TMDB search payload we read. Film and +// television share the envelope but not the field names, so the few that differ +// are read together and picked between below. +type tmdbResponse struct { + Results []struct { + ID int `json:"id"` + Title string `json:"title"` + Name string `json:"name"` + Overview string `json:"overview"` + ReleaseDate string `json:"release_date"` + FirstAirDate string `json:"first_air_date"` + PosterPath string `json:"poster_path"` + Popularity float64 `json:"popularity"` + } `json:"results"` +} + +// tmdbHit carries a candidate with the popularity it was ranked by, which +// Candidate has no field for. +type tmdbHit struct { + cand Candidate + popularity float64 +} + +func (p *TMDB) Search(ctx context.Context, q Query) ([]Candidate, error) { + title := strings.TrimSpace(q.Title) + if p.APIKey == "" || title == "" { + return nil, nil + } + + // The two endpoints run at once: the call costs one response time, not two. + // Whichever fails first cancels the other, so one dead endpoint cannot hold + // the result hostage while the healthy one is waited out. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + wg sync.WaitGroup + hits [2][]tmdbHit + errs [2]error + fetch = func(i int, endpoint, kind string) { + defer wg.Done() + hits[i], errs[i] = p.search(ctx, endpoint, kind, title, q.Year) + if errs[i] != nil { + cancel() + } + } + ) + wg.Add(2) + go fetch(0, "/search/movie", "movie") + go fetch(1, "/search/tv", "tv") + wg.Wait() + + merged := append(hits[0], hits[1]...) + if len(merged) == 0 && (errs[0] != nil || errs[1] != nil) { + // Only a total failure is an error. One endpoint answering is a result. + return nil, errors.Join(errs[0], errs[1]) + } + sort.SliceStable(merged, func(i, j int) bool { + return merged[i].popularity > merged[j].popularity + }) + + out := make([]Candidate, 0, len(merged)) + for _, h := range merged { + out = append(out, h.cand) + } + return out, nil +} + +// search queries one TMDB endpoint and maps its results. +func (p *TMDB) search(ctx context.Context, endpoint, kind, title string, year int) ([]tmdbHit, error) { + params := url.Values{} + params.Set("api_key", p.APIKey) + params.Set("query", title) + params.Set("limit", "5") + if year > 0 { + // The two namespaces spell the year filter differently. + if kind == "movie" { + params.Set("year", strconv.Itoa(year)) + } else { + params.Set("first_air_date_year", strconv.Itoa(year)) + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + strings.TrimRight(p.BaseURL, "/")+endpoint+"?"+params.Encode(), nil) + if err != nil { + return nil, err + } + resp, err := p.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("tmdb: %s: %w", kind, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("tmdb: %s: status %d", kind, resp.StatusCode) + } + + var parsed tmdbResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, fmt.Errorf("tmdb: %s: decode: %w", kind, err) + } + + out := make([]tmdbHit, 0, len(parsed.Results)) + for _, r := range parsed.Results { + name := r.Title + if name == "" { + name = r.Name // television has no title, only a name + } + if name == "" { + continue + } + date := r.ReleaseDate + if date == "" { + date = r.FirstAirDate + } + out = append(out, tmdbHit{ + cand: Candidate{ + Provider: p.Name(), + // Namespaced: TMDB keeps films and television in separate id + // spaces, so film 603 and show 603 are different works. A bare + // id would store them identically and collide in provider_ids. + ExternalID: kind + "/" + strconv.Itoa(r.ID), + Title: name, + Year: parseYear(date), + Description: r.Overview, + CoverURL: tmdbPoster(r.PosterPath), + }, + popularity: r.Popularity, + }) + } + return out, nil +} + +// tmdbPoster expands TMDB's relative poster path into a full image URL. The +// bytes are never fetched here: the enricher caches covers it actually uses. +func tmdbPoster(path string) string { + if path == "" { + return "" + } + return "https://image.tmdb.org/t/p/w500" + path +} diff --git a/internal/providers/tmdb_test.go b/internal/providers/tmdb_test.go new file mode 100644 index 0000000..9f70ef1 --- /dev/null +++ b/internal/providers/tmdb_test.go @@ -0,0 +1,244 @@ +package providers + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +// tmdbSrv answers both search endpoints from one handler, so a test can watch +// what each side did without two servers to keep in sync. +func tmdbSrv(t *testing.T, fn func(endpoint string, r *http.Request) (string, int)) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, status := fn(r.URL.Path, r) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestTMDBMapsMovieAndTV(t *testing.T) { + // Each endpoint records only its own request: the two handlers run on + // different goroutines, so a shared capture would be a race. + var gotMovieKey, gotMovieQuery, gotMovieYear, gotTVKey, gotTVYear string + srv := tmdbSrv(t, func(endpoint string, r *http.Request) (string, int) { + q := r.URL.Query() + switch endpoint { + case "/search/movie": + gotMovieKey, gotMovieQuery, gotMovieYear = q.Get("api_key"), q.Get("query"), q.Get("year") + return `{"results":[ + {"id": 603, "title": "The Matrix", "overview": "A hacker learns the truth.", + "release_date": "1999-03-31", "poster_path": "/matrix.jpg", "popularity": 88.1} + ]}`, http.StatusOK + case "/search/tv": + gotTVKey, gotTVYear = q.Get("api_key"), q.Get("first_air_date_year") + return `{"results":[ + {"id": 1399, "name": "Game of Thrones", "overview": "Noble houses fight for a throne.", + "first_air_date": "2011-04-17", "poster_path": "/got.jpg", "popularity": 120.5} + ]}`, http.StatusOK + } + return `{"results":[]}`, http.StatusOK + }) + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + cands, err := p.Search(context.Background(), Query{Title: "The Matrix", Year: 1999}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(cands) != 2 { + t.Fatalf("got %d candidates, want 2", len(cands)) + } + + // Merged by TMDB popularity: the show (120.5) outranks the film (88.1). + show, movie := cands[0], cands[1] + if show.Title != "Game of Thrones" || movie.Title != "The Matrix" { + t.Fatalf("merge order wrong: %q then %q", show.Title, movie.Title) + } + + if movie.Provider != "tmdb" || movie.ExternalID != "movie/603" { + t.Errorf("movie id = %q (provider %q), want movie/603", movie.ExternalID, movie.Provider) + } + if movie.Year != 1999 { + t.Errorf("movie Year = %d, want 1999", movie.Year) + } + if movie.Description != "A hacker learns the truth." { + t.Errorf("movie Description = %q", movie.Description) + } + if movie.CoverURL != "https://image.tmdb.org/t/p/w500/matrix.jpg" { + t.Errorf("movie CoverURL = %q", movie.CoverURL) + } + if show.ExternalID != "tv/1399" || show.Year != 2011 { + t.Errorf("show id/year = %q/%d, want tv/1399/2011", show.ExternalID, show.Year) + } + if show.CoverURL != "https://image.tmdb.org/t/p/w500/got.jpg" { + t.Errorf("show CoverURL = %q", show.CoverURL) + } + + if gotMovieKey != "secret-key" || gotTVKey != "secret-key" { + t.Errorf("api_key = movie %q, tv %q, want secret-key", gotMovieKey, gotTVKey) + } + if gotMovieQuery != "The Matrix" { + t.Errorf("movie query = %q", gotMovieQuery) + } + // The two namespaces spell the year filter differently; both must get it. + if gotMovieYear != "1999" || gotTVYear != "1999" { + t.Errorf("year filter = movie %q, tv %q, want 1999", gotMovieYear, gotTVYear) + } +} + +func TestTMDBSearchesConcurrently(t *testing.T) { + var arrived int32 + var overlapped atomic.Bool + both := make(chan struct{}) + + srv := tmdbSrv(t, func(_ string, _ *http.Request) (string, int) { + // Neither endpoint answers until both requests are in flight, so a + // sequential implementation deadlocks here rather than passing. + if atomic.AddInt32(&arrived, 1) == 2 { + overlapped.Store(true) + close(both) + } + select { + case <-both: + case <-time.After(2 * time.Second): + } + return `{"results":[{"id": 1, "title": "Anything", "popularity": 1}]}`, http.StatusOK + }) + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + cands, err := p.Search(context.Background(), Query{Title: "Anything"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(cands) != 2 { + t.Errorf("got %d candidates, want one per endpoint", len(cands)) + } + if n := atomic.LoadInt32(&arrived); n != 2 { + t.Errorf("endpoints hit %d times, want 2", n) + } + if !overlapped.Load() { + t.Error("the two searches did not overlap: they ran one after the other") + } +} + +// TMDB keeps films and television in one numeric id space: film 603 and show +// 603 are different works. A bare id would write them identically into +// provider_ids, so the namespace has to be part of what gets stored. +func TestTMDBExternalIDIsNamespaced(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + // The same id from both endpoints is the collision under test. + if strings.Contains(r.URL.Path, "/movie") { + _, _ = w.Write([]byte(`{"results":[{"id":603,"title":"Film 603","release_date":"1999-03-31","popularity":10}]}`)) + return + } + _, _ = w.Write([]byte(`{"results":[{"id":603,"name":"Show 603","first_air_date":"2011-04-17","popularity":20}]}`)) + })) + defer srv.Close() + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + cands, err := p.Search(context.Background(), Query{Title: "603"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(cands) != 2 { + t.Fatalf("got %d candidates, want 2", len(cands)) + } + if cands[0].ExternalID == cands[1].ExternalID { + t.Fatalf("a film and a show sharing id 603 both produced ExternalID %q", cands[0].ExternalID) + } +} + +func TestTMDBOneEndpointFailureKeepsOtherResults(t *testing.T) { // The movie endpoint only fails once the TV response has been written, so + // this is an ordering guarantee, not a race the test happens to win. + tvDone := make(chan struct{}) + srv := tmdbSrv(t, func(endpoint string, _ *http.Request) (string, int) { + switch endpoint { + case "/search/tv": + close(tvDone) + return `{"results":[{"id": 1396, "name": "Breaking Bad", "popularity": 200}]}`, http.StatusOK + default: + select { + case <-tvDone: + case <-time.After(2 * time.Second): + } + return `{"status_message":"boom"}`, http.StatusInternalServerError + } + }) + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + cands, err := p.Search(context.Background(), Query{Title: "Breaking Bad"}) + if err != nil { + t.Fatalf("Search: %v (one failing endpoint must not discard the other)", err) + } + if len(cands) != 1 || cands[0].Title != "Breaking Bad" { + t.Fatalf("got %+v, want just the TV match", cands) + } +} + +func TestTMDBEmptyResultsAreNotAnError(t *testing.T) { + srv := tmdbSrv(t, func(_ string, _ *http.Request) (string, int) { + return `{"page":1,"results":[],"total_pages":1,"total_results":0}`, http.StatusOK + }) + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + cands, err := p.Search(context.Background(), Query{Title: "Nothing At All Like This"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(cands) != 0 { + t.Errorf("got %d candidates, want 0", len(cands)) + } +} + +func TestTMDBWithoutUsableTitleDoesNotCallOut(t *testing.T) { + var hits int32 + srv := tmdbSrv(t, func(_ string, _ *http.Request) (string, int) { + atomic.AddInt32(&hits, 1) + return `{"results":[{"id": 1, "title": "Unwanted", "popularity": 1}]}`, http.StatusOK + }) + + p := NewTMDB("secret-key") + p.BaseURL = srv.URL + + for _, q := range []Query{{}, {Title: " "}} { + cands, err := p.Search(context.Background(), q) + if err != nil { + t.Fatalf("Search(%+v): %v", q, err) + } + if len(cands) != 0 { + t.Errorf("Search(%+v) returned %d candidates, want 0", q, len(cands)) + } + } + if n := atomic.LoadInt32(&hits); n != 0 { + t.Errorf("TMDB was called %d times with no usable title, want 0", n) + } +} + +func TestTMDBWithoutKeyDoesNothing(t *testing.T) { + p := NewTMDB("") + cands, err := p.Search(context.Background(), Query{Title: "The Matrix"}) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(cands) != 0 { + t.Errorf("got %d candidates with no key, want 0", len(cands)) + } +}