From 785d3ed5ee4cafd139cc5d1a52974e2a647d534a Mon Sep 17 00:00:00 2001 From: mithral Date: Sun, 11 Oct 2026 14:14:32 -0400 Subject: [PATCH] Waves 4-6: audio depth, ffprobe facts, and direct-play video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct play only. The server is a headless GPU-less Xeon: nothing is ever transcoded or remuxed. A file either plays in the client or it does not. Measured, not assumed (ffmpeg-generated files with known codecs, played through Vellum's own stream endpoint in Chrome 155 / Linux): mp4 h264+aac plays mkv h264+aac PLAYS <- container is fine mov h264+aac plays webm vp9/opus plays mp4 av1+opus plays webm vp8/vorbis plays mp4 hevc+aac AUDIO ONLY, videoWidth stays 0, no error mkv hevc+flac AUDIO ONLY avi mpeg4+mp3 fails outright All seven audio formats play. Two conclusions drive the design: codec matters and container barely does (mkv+h264 plays, mp4+HEVC does not), and canPlayType is unreliable in both directions — it says "no" for mov+h264 (which plays) and "maybe" for mkv+hevc (whose video never decodes). Therefore playability is a property of the CLIENT, not the file. The server never declares a file playable; it reports codec facts and each client decides. That is what makes adding mpv/libvlc/MDK later a matter of registering an engine rather than re-architecting. Wave 4 - audio depth - New MusicPlayer.svelte: real queue, shuffle that walks a fixed permutation (so "previous" means something and nothing repeats early), repeat off/all/one, per-track rows, album art, next-track preload. - AudiobookPlayer: MediaSession (OS media keys + lock screen), playback speed that SURVIVES a chapter change and persists, sleep timer with countdown that a manual pause cancels, keyboard shortcuts, next-chapter preload. - Shared mediaSession.ts written by the coordinator before dispatch so both players consume one implementation instead of inventing two. Wave 5 - ffprobe facts - internal/probe (ffprobe wrapper, optional dependency), migration 0010 adding vcodec/acodec/width/height/probed_at to media_files, scanner hook, API fields. - Probing runs only for NEW or CHANGED files and BEFORE the write transaction opens, so it can never hold SQLite's write lock and a rescan never becomes a full-library ffprobe sweep. Proven: 797ms probing scan, then 1ms rescan with probed_at byte-identical. - ffprobe skips attached_pic so an mp3 with cover art reports no video codec. - Fixed a real hole: .avi is in the video registry but had no MIME entry, so it was served as application/octet-stream and could never play. Wave 6 - the video player - VideoPlayer.svelte with resume, progress + keepalive final flush, MediaSession, keyboard shortcuts, multi-file picker, and codec facts shown. - The honest path is the feature: an HEVC file makes Chrome play audio over a blank frame with no error, which is the worst possible outcome. The player detects it and says so, naming the codec and pointing at clients that can. Coordinator fix to Wave 6: the agent's detection hung off `loadedmetadata`, but an explicit `type` the browser cannot satisfy makes it reject the source during selection, so NEITHER loadedmetadata NOR error fires — and the player showed a working-looking 0:00 with a Play button, exactly the silently-broken state the component exists to prevent. Added a load watchdog: if the element says nothing within 2.5s, the source was rejected and the honest notice is shown. Verified per file afterwards (h264/mkv play at 640 wide; hevc and mpeg4 show their notices). Verified end to end, not just compiled: 15 packages pass (including the new internal/probe), gofmt/vet clean, svelte-check 0/0, and every claim above checked against a running server. Note the demo's own mp3s are 100 KB of random bytes and cannot decode; the audio players were proven against generated real audio instead. --- internal/library/scan.go | 101 ++- internal/media/handlers.go | 31 +- internal/probe/probe.go | 133 ++++ internal/probe/probe_test.go | 96 +++ internal/stream/stream.go | 6 + migrations/0010_media_probe.sql | 28 + web/src/lib/api.ts | 23 +- web/src/lib/components/AudiobookPlayer.svelte | 279 ++++++- web/src/lib/components/MusicPlayer.svelte | 580 +++++++++++++++ web/src/lib/components/Reader.svelte | 8 +- web/src/lib/components/VideoPlayer.svelte | 698 ++++++++++++++++++ web/src/lib/mediaSession.ts | 130 ++++ web/src/lib/playback.ts | 147 ++++ 13 files changed, 2246 insertions(+), 14 deletions(-) create mode 100644 internal/probe/probe.go create mode 100644 internal/probe/probe_test.go create mode 100644 migrations/0010_media_probe.sql create mode 100644 web/src/lib/components/MusicPlayer.svelte create mode 100644 web/src/lib/components/VideoPlayer.svelte create mode 100644 web/src/lib/mediaSession.ts create mode 100644 web/src/lib/playback.ts diff --git a/internal/library/scan.go b/internal/library/scan.go index adfe9d2..fb4e5c8 100644 --- a/internal/library/scan.go +++ b/internal/library/scan.go @@ -17,6 +17,7 @@ import ( "github.com/mithral/vellum/internal/core" "github.com/mithral/vellum/internal/jobs" "github.com/mithral/vellum/internal/natsort" + "github.com/mithral/vellum/internal/probe" ) const ( @@ -130,6 +131,17 @@ func Scan(ctx context.Context, db *sql.DB, lib Library, log *slog.Logger) (ScanR } batch := groupKeys[start:end] + // Probe the changed media files in this batch BEFORE opening the write + // transaction, so a slow ffprobe (a stalled mount, a huge file) can + // never hold SQLite's write lock. + // + // The probe map is deliberately built from the same changed-only test + // the transaction uses below. Probing on the unchanged path would turn + // every rescan into a full ffprobe sweep of the entire library; the + // size+mtime skip is what keeps rescans cheap, and the probe must live + // on the changed side of it. + probed := probeBatch(ctx, lib, groups, batch, knownFiles, &res) + err := commitBatch(ctx, db, func(tx *sql.Tx) error { for _, key := range batch { g := groups[key] @@ -150,7 +162,8 @@ func Scan(ctx context.Context, db *sql.DB, lib Library, log *slog.Logger) (ScanR continue } - if err := upsertFile(ctx, tx, itemID, c); err != nil { + info, wasProbed := probed[c.relPath] + if err := upsertFile(ctx, tx, itemID, c, info, wasProbed, now); err != nil { res.addError(fmt.Sprintf("%s: %v", c.relPath, err)) continue } @@ -431,21 +444,97 @@ func upsertItem(ctx context.Context, tx *sql.Tx, lib Library, key, title string) return id, nil } -func upsertFile(ctx context.Context, tx *sql.Tx, itemID int64, c candidate) error { +func upsertFile(ctx context.Context, tx *sql.Tx, itemID int64, c candidate, info probe.Info, probed bool, now string) error { + // A non-probed file (an ebook, a photo, or any file on a host without + // ffprobe) must keep whatever codec facts a previous scan recorded, so its + // upsert never mentions those columns at all. + if !probed { + _, err := tx.ExecContext(ctx, ` + INSERT INTO media_files (item_id, path, size, mtime, inode, container, missing) + VALUES (?, ?, ?, ?, ?, ?, 0) + ON CONFLICT (path) + DO UPDATE SET item_id = excluded.item_id, size = excluded.size, + mtime = excluded.mtime, inode = excluded.inode, + container = excluded.container, missing = 0`, + itemID, c.relPath, c.size, fmt.Sprint(c.mtime), c.inode, c.ext) + if err != nil { + return fmt.Errorf("upsert file: %w", err) + } + return nil + } + _, err := tx.ExecContext(ctx, ` - INSERT INTO media_files (item_id, path, size, mtime, inode, container, missing) - VALUES (?, ?, ?, ?, ?, ?, 0) + INSERT INTO media_files (item_id, path, size, mtime, inode, container, missing, + duration_ms, vcodec, acodec, width, height, probed_at) + VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?) ON CONFLICT (path) DO UPDATE SET item_id = excluded.item_id, size = excluded.size, mtime = excluded.mtime, inode = excluded.inode, - container = excluded.container, missing = 0`, - itemID, c.relPath, c.size, fmt.Sprint(c.mtime), c.inode, c.ext) + container = excluded.container, missing = 0, + duration_ms = excluded.duration_ms, vcodec = excluded.vcodec, + acodec = excluded.acodec, width = excluded.width, + height = excluded.height, probed_at = excluded.probed_at`, + itemID, c.relPath, c.size, fmt.Sprint(c.mtime), c.inode, c.ext, + info.DurationMS, info.VCodec, info.ACodec, info.Width, info.Height, now) if err != nil { return fmt.Errorf("upsert file: %w", err) } return nil } +// probeBatch probes every new or changed video/audio file in one batch and +// returns the facts keyed by relative path. +// +// Only files the transaction will treat as changed (a size+mtime mismatch, or a +// row that is absent or flagged missing) are probed; unchanged files are absent +// from the result, so no ffprobe runs and no codec column is rewritten for +// them. Non-media kinds (ebooks, comics, photos, games) are never probed: they +// hold no media streams, and running ffprobe over a library of them would be +// pure waste. +// +// It runs before commitBatch opens the write transaction, so the write lock is +// never held across an ffprobe. A probe failure is recorded through the bounded +// error mechanism and the file is written with empty codec fields rather than +// failing the scan. +func probeBatch(ctx context.Context, lib Library, groups map[string]group, batch []string, known map[string]knownFile, res *ScanResult) map[string]probe.Info { + if !probeable(lib.Kind) || !probe.Available() { + return nil + } + + out := make(map[string]probe.Info) + for _, key := range batch { + for _, c := range groups[key].files { + if prev, ok := known[c.relPath]; ok && !prev.missing && prev.size == c.size && prev.mtime == c.mtime { + continue // unchanged: never re-probe + } + + abs := filepath.Join(lib.RootPath, filepath.FromSlash(c.relPath)) + info, err := probe.File(ctx, abs) + if err != nil { + res.addError(fmt.Sprintf("%s: %v", c.relPath, err)) + // Record the attempt with empty fields; the codecs stay empty + // until the file changes again. + out[c.relPath] = probe.Info{} + continue + } + out[c.relPath] = info + } + } + return out +} + +// probeable reports whether a library kind holds files worth probing for codec +// facts. Only video and the audio kinds have media streams; ebooks, comics, +// photos, and games are never probed. +func probeable(kind string) bool { + switch kind { + case "video", "audiobooks", "music": + return true + default: + return false + } +} + func markItemMissing(ctx context.Context, tx *sql.Tx, libraryID int64, key, now string) error { _, err := tx.ExecContext(ctx, ` UPDATE media_items SET missing = 1 diff --git a/internal/media/handlers.go b/internal/media/handlers.go index da7fbdb..512c1e0 100644 --- a/internal/media/handlers.go +++ b/internal/media/handlers.go @@ -305,19 +305,40 @@ func (h *Handlers) authorized(w http.ResponseWriter, r *http.Request, u auth.Use return it, true } -func (h *Handlers) files(r *http.Request, itemID int64) ([]FileEntry, error) { +// fileDTO is one physical file in an item's detail response. It carries the +// ffprobe codec facts alongside the container, because direct play is a +// decision each client makes for itself: the server reports what is in the file +// and never transcodes. The wire shape is frozen by MediaFile in +// web/src/lib/api.ts. +type fileDTO struct { + ID int64 `json:"id"` + Path string `json:"path"` + Size int64 `json:"size"` + Container string `json:"container"` + Missing bool `json:"missing"` + DurationMS int64 `json:"duration_ms"` + VCodec string `json:"vcodec"` + ACodec string `json:"acodec"` + Width int `json:"width"` + Height int `json:"height"` +} + +func (h *Handlers) files(r *http.Request, itemID int64) ([]fileDTO, error) { rows, err := h.db.QueryContext(r.Context(), ` - SELECT id, path, size, container, missing FROM media_files WHERE item_id = ? ORDER BY id`, itemID) + SELECT id, path, size, container, missing, + duration_ms, vcodec, acodec, width, height + FROM media_files WHERE item_id = ? ORDER BY id`, itemID) if err != nil { return nil, err } defer rows.Close() - out := []FileEntry{} + out := []fileDTO{} for rows.Next() { - var f FileEntry + var f fileDTO var missing int - if err := rows.Scan(&f.ID, &f.Path, &f.Size, &f.Container, &missing); err != nil { + if err := rows.Scan(&f.ID, &f.Path, &f.Size, &f.Container, &missing, + &f.DurationMS, &f.VCodec, &f.ACodec, &f.Width, &f.Height); err != nil { return nil, err } f.Missing = missing == 1 diff --git a/internal/probe/probe.go b/internal/probe/probe.go new file mode 100644 index 0000000..bef714f --- /dev/null +++ b/internal/probe/probe.go @@ -0,0 +1,133 @@ +// Package probe reads the container and codec facts out of a media file with +// ffprobe, so a client can decide for itself whether it can direct-play the +// file. +// +// Vellum never transcodes. The server has no spare CPU, and measured against +// real files, playability turned out to be a property of the CLIENT, not the +// file: Chrome on Linux plays an MKV carrying H.264 perfectly well, yet cannot +// decode an MP4 carrying HEVC at all — it plays the audio track and reports +// videoWidth 0. So the server's only job is to expose what is actually inside +// the file; each client decides what it can handle. +// +// ffprobe is OPTIONAL. Vellum runs without it, just with less information: +// Available reports whether it is on PATH, and callers skip probing when it is +// absent rather than treating that as an error. +package probe + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "strconv" + "strings" + "time" +) + +// timeout bounds a single ffprobe invocation. A file on a stalled network mount +// must never hang a scan, so ffprobe is killed after this and the error is +// recorded like any other probe failure. +const timeout = 15 * time.Second + +// Info is the subset of ffprobe facts Vellum stores per file. +// +// Zero values are meaningful: an empty VCodec means there is no video track +// (audio, or a container whose only "video" stream is embedded cover art), and +// an empty ACodec means there is no audio track. DurationMS and the dimensions +// stay zero when ffprobe could not determine them. +type Info struct { + DurationMS int64 + VCodec string + ACodec string + Width int + Height int +} + +// Available reports whether ffprobe is on PATH. +// +// Callers use it to skip probing entirely, so Vellum still works on a host with +// no ffmpeg installed — the library just carries no codec facts, and every +// client is left to try the file and find out. +func Available() bool { + _, err := exec.LookPath("ffprobe") + return err == nil +} + +// File probes one file and returns its codec facts. The returned context +// deadline is capped at timeout regardless of the caller's deadline. +func File(ctx context.Context, path string) (Info, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "ffprobe", + "-v", "error", + "-print_format", "json", + "-show_format", + "-show_streams", + path, + ) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return Info{}, fmt.Errorf("probe %s: %w", path, ctx.Err()) + } + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return Info{}, fmt.Errorf("probe %s: %s", path, msg) + } + + var out ffprobeOutput + if err := json.Unmarshal(stdout.Bytes(), &out); err != nil { + return Info{}, fmt.Errorf("probe %s: parse ffprobe output: %w", path, err) + } + + info := Info{} + for _, s := range out.Streams { + switch s.CodecType { + case "video": + // Embedded cover art (an mp3's album image) is reported as a video + // stream. It is not a playable video track, so ignore it and leave + // VCodec empty for audio files. + if s.Disposition.AttachedPic != 0 || info.VCodec != "" { + continue + } + info.VCodec = s.CodecName + info.Width = s.Width + info.Height = s.Height + case "audio": + if info.ACodec == "" { + info.ACodec = s.CodecName + } + } + } + + // ffprobe reports the container duration in seconds as a decimal string. + // It is absent for some live/streamed inputs; leave DurationMS zero then. + if d, err := strconv.ParseFloat(out.Format.Duration, 64); err == nil && d > 0 { + info.DurationMS = int64(d * 1000) + } + return info, nil +} + +// ffprobeOutput mirrors only the JSON fields Vellum reads from +// `ffprobe -print_format json -show_format -show_streams`. +type ffprobeOutput struct { + Format struct { + Duration string `json:"duration"` + } `json:"format"` + Streams []struct { + CodecType string `json:"codec_type"` + CodecName string `json:"codec_name"` + Width int `json:"width"` + Height int `json:"height"` + Disposition struct { + AttachedPic int `json:"attached_pic"` + } `json:"disposition"` + } `json:"streams"` +} diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go new file mode 100644 index 0000000..c5e463e --- /dev/null +++ b/internal/probe/probe_test.go @@ -0,0 +1,96 @@ +package probe + +import ( + "context" + "os/exec" + "path/filepath" + "testing" +) + +// TestAvailable never skips: the answer is a plain bool and must not panic on a +// host without ffprobe. +func TestAvailable(t *testing.T) { + _ = Available() +} + +// TestFile probes a fixture generated on the fly. It skips cleanly when ffprobe +// or ffmpeg is missing, so the suite passes on a machine with neither. +func TestFile(t *testing.T) { + if !Available() { + t.Skip("ffprobe not on PATH; skipping probe test") + } + ffmpeg, err := exec.LookPath("ffmpeg") + if err != nil { + t.Skip("ffmpeg not on PATH; cannot generate a fixture") + } + + path := filepath.Join(t.TempDir(), "clip.mp4") + // One second of H.264 video and AAC audio in an MP4. + gen := exec.Command(ffmpeg, "-v", "error", "-y", + "-f", "lavfi", "-i", "testsrc=size=64x48:rate=10:duration=1", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-c:v", "libx264", "-c:a", "aac", "-shortest", path, + ) + if out, err := gen.CombinedOutput(); err != nil { + t.Skipf("ffmpeg cannot produce a fixture here: %v\n%s", err, out) + } + + info, err := File(context.Background(), path) + if err != nil { + t.Fatalf("File: %v", err) + } + if info.VCodec != "h264" { + t.Errorf("VCodec = %q, want h264", info.VCodec) + } + if info.ACodec != "aac" { + t.Errorf("ACodec = %q, want aac", info.ACodec) + } + if info.Width != 64 || info.Height != 48 { + t.Errorf("dimensions = %dx%d, want 64x48", info.Width, info.Height) + } + if info.DurationMS < 500 || info.DurationMS > 2000 { + t.Errorf("DurationMS = %d, want ~1000", info.DurationMS) + } +} + +// TestFileAudioOnly confirms an audio file reports no video codec, so a client +// does not mistake an mp3 for a video with a missing track. +func TestFileAudioOnly(t *testing.T) { + if !Available() { + t.Skip("ffprobe not on PATH; skipping probe test") + } + ffmpeg, err := exec.LookPath("ffmpeg") + if err != nil { + t.Skip("ffmpeg not on PATH; cannot generate a fixture") + } + + path := filepath.Join(t.TempDir(), "tone.m4a") + gen := exec.Command(ffmpeg, "-v", "error", "-y", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-c:a", "aac", path, + ) + if out, err := gen.CombinedOutput(); err != nil { + t.Skipf("ffmpeg cannot produce a fixture here: %v\n%s", err, out) + } + + info, err := File(context.Background(), path) + if err != nil { + t.Fatalf("File: %v", err) + } + if info.VCodec != "" { + t.Errorf("VCodec = %q, want empty for audio-only", info.VCodec) + } + if info.ACodec != "aac" { + t.Errorf("ACodec = %q, want aac", info.ACodec) + } +} + +// TestFileMissing confirms a missing file is an error, not a silent zero Info. +func TestFileMissing(t *testing.T) { + if !Available() { + t.Skip("ffprobe not on PATH; skipping probe test") + } + if _, err := File(context.Background(), filepath.Join(t.TempDir(), "nope.mp4")); err == nil { + t.Fatal("File on a missing path: want error, got nil") + } +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index a70acaf..620a90e 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -38,11 +38,17 @@ var contentTypes = map[string]string{ ".ogg": "audio/ogg", ".wav": "audio/wav", // Video. + // + // This table and the library extension registry (internal/library/registry.go) + // must agree: an extension in the video registry with no MIME entry here is + // served as application/octet-stream, which browsers download instead of + // playing. .avi was in the registry but missing here. ".mp4": "video/mp4", ".m4v": "video/mp4", ".mkv": "video/x-matroska", ".webm": "video/webm", ".mov": "video/quicktime", + ".avi": "video/x-msvideo", // Images (comic pages, covers, photos). ".jpg": "image/jpeg", ".jpeg": "image/jpeg", diff --git a/migrations/0010_media_probe.sql b/migrations/0010_media_probe.sql new file mode 100644 index 0000000..8ea4032 --- /dev/null +++ b/migrations/0010_media_probe.sql @@ -0,0 +1,28 @@ +-- 0010_media_probe.sql — record the codec facts a client needs to choose direct +-- play. +-- +-- Vellum does not transcode. The server has no spare CPU, and measured against +-- real files, playability turned out to be a property of the CLIENT, not the +-- file: Chrome on Linux plays an MKV carrying H.264 perfectly well, yet cannot +-- decode an MP4 carrying HEVC at all — it plays the audio track and reports +-- videoWidth 0. So the server's job is only to expose what is actually inside +-- each file and let every client decide for itself. These columns are that +-- exposure; without them a client has nothing to base the decision on. +-- +-- The values come from ffprobe at scan time. They are empty/zero until the +-- scanner probes the file, and stay that way when ffprobe is not installed: +-- Vellum still works without it, just with less information. duration_ms +-- already exists on media_files from 0001, so it is reused rather than re-added. + +-- ffprobe codec name, e.g. "h264", "hevc", "vp9", "av1". "" when there is no +-- video track (audio, or embedded cover art only). +ALTER TABLE media_files ADD COLUMN vcodec TEXT NOT NULL DEFAULT ''; +-- ffprobe codec name, e.g. "aac", "opus", "flac", "mp3". "" when there is no +-- audio track. +ALTER TABLE media_files ADD COLUMN acodec TEXT NOT NULL DEFAULT ''; +-- Pixel dimensions of the video track; 0 when there is none. +ALTER TABLE media_files ADD COLUMN width INTEGER NOT NULL DEFAULT 0; +ALTER TABLE media_files ADD COLUMN height INTEGER NOT NULL DEFAULT 0; +-- When the codec facts above were last written, so a rescan can be proven not +-- to have re-probed an unchanged file. '' means never probed. +ALTER TABLE media_files ADD COLUMN probed_at TEXT NOT NULL DEFAULT ''; diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index fa08744..dba1f93 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -269,9 +269,30 @@ export interface Home { total_items: number } +// MediaFile carries the ffprobe facts about one file on disk. +// +// The codec fields are what make "direct play only" workable: playability is a +// property of the CLIENT, not the file, so the server reports what is in the +// file and each client decides for itself. An empty vcodec means there is no +// video track (audio, or a comic). +export interface MediaFile { + id: number + path: string + size: number + container: string + missing: boolean + duration_ms: number + /** ffprobe codec name, e.g. "h264", "hevc", "vp9", "av1". "" when absent. */ + vcodec: string + /** ffprobe codec name, e.g. "aac", "opus", "flac", "mp3". "" when absent. */ + acodec: string + width: number + height: number +} + export interface ItemDetail { item: Item - files: { id: number; path: string; size: number; container: string; missing: boolean }[] + files: MediaFile[] has_cover: boolean authors: string[] | null progress?: Progress diff --git a/web/src/lib/components/AudiobookPlayer.svelte b/web/src/lib/components/AudiobookPlayer.svelte index de5b297..e0ad4ac 100644 --- a/web/src/lib/components/AudiobookPlayer.svelte +++ b/web/src/lib/components/AudiobookPlayer.svelte @@ -1,11 +1,35 @@ + +{#if error} +
{error}
+{:else if loading} +
+{:else} +
+
+ (e.currentTarget as HTMLImageElement).style.display = 'none'} + /> +
+ {item?.title} + {#if authors.length}{authors.join(', ')}{/if} + {[tracks.length, tracks.length === 1 ? 'track' : 'tracks'].join(' ')} +
+
+ + + + {#if nextTrack} + + {/if} + +
+ {currentIndex >= 0 ? String(currentIndex + 1).padStart(2, '0') : '--'} + {currentTrack ? cleanTitle(currentTrack.path) : ''} + {order.length ? `${pos + 1}/${order.length}` : ''} +
+ + + +
+ {fmt(currentTime)} + {fmt(duration)} +
+ +
+ + + + + +
+ +
    + {#each tracks as t, i (t.id)} +
  1. + +
    2. + {/each} +
+
+{/if} + + diff --git a/web/src/lib/components/Reader.svelte b/web/src/lib/components/Reader.svelte index d65663d..fce034c 100644 --- a/web/src/lib/components/Reader.svelte +++ b/web/src/lib/components/Reader.svelte @@ -3,9 +3,11 @@ import { api, ApiError, type Item, type EbookMetadata } from '../api' import ComicReader from './ComicReader.svelte' import AudiobookPlayer from './AudiobookPlayer.svelte' + import MusicPlayer from './MusicPlayer.svelte' import PhotoViewer from './PhotoViewer.svelte' import PdfReader from './PdfReader.svelte' import GamePlayer from './GamePlayer.svelte' + import VideoPlayer from './VideoPlayer.svelte' let { itemId }: { itemId: number } = $props() @@ -139,12 +141,16 @@
{error}
{:else if item && item.kind === 'games'} +{:else if item && item.kind === 'video'} + {:else if item && fileContainer === 'pdf'} {:else if item && item.kind === 'comics'} -{:else if item && (item.kind === 'audiobooks' || item.kind === 'music')} +{:else if item && item.kind === 'audiobooks'} +{:else if item && item.kind === 'music'} + {:else if item && item.kind === 'photos'} {:else if item && item.kind !== 'ebooks'} diff --git a/web/src/lib/components/VideoPlayer.svelte b/web/src/lib/components/VideoPlayer.svelte new file mode 100644 index 0000000..615e050 --- /dev/null +++ b/web/src/lib/components/VideoPlayer.svelte @@ -0,0 +1,698 @@ + + +{#if error} +
{error}
+{:else if loading} +
+{:else if files.length === 0} +
+

{item?.title ?? 'Video'}

+

This item has no video file to play.

+ Download the file +
+{:else} +
+
+
+ {item?.title} + + {file?.path.split('/').pop() ?? ''} + {#if files.length > 1}· {index + 1} of {files.length}{/if} + + {#if file && codecSummary(file)} + {codecSummary(file)} + {/if} +
+ Download +
+ + {#if notice} + + {/if} + +
+ + + +
+ + {#if videoDecoded} +

Playing video directly — no transcoding.

+ {/if} + + + +
+ {fmt(currentTime)} + {fmt(duration)} +
+ +
+ + + +
+ +

Space play/pause · ←/→ seek {SEEK_STEP}s · F fullscreen · M mute

+ + {#if files.length > 1} +
    + {#each files as f, i (f.id)} +
  1. + +
    2. + {/each} +
+ {/if} +
+{/if} + + diff --git a/web/src/lib/mediaSession.ts b/web/src/lib/mediaSession.ts new file mode 100644 index 0000000..88742b5 --- /dev/null +++ b/web/src/lib/mediaSession.ts @@ -0,0 +1,130 @@ +// MediaSession wiring for the audio players. +// +// The OS media keys, the lock screen and the browser's own media popup all read +// from `navigator.mediaSession`. Without this, pressing the play/pause key on a +// keyboard or a pair of headphones does nothing, and the lock screen shows a +// blank entry — which for an audiobook or an album is the difference between +// "a web page playing audio" and "a player". +// +// Everything here is best-effort: the API is absent in some browsers, and an +// unsupported action handler must never break playback. Every call is guarded +// and every registration is wrapped. + +export interface NowPlaying { + title: string + artist?: string + album?: string + /** Cover art URL. Same-origin, so the browser can fetch it. */ + artwork?: string + /** Seconds. */ + duration?: number +} + +type ActionHandler = (details?: { seekTime?: number; seekOffset?: number }) => void + +function session(): MediaSession | undefined { + // Feature-detected rather than assumed: Firefox shipped this late and some + // embedded webviews never did. + return typeof navigator !== 'undefined' ? navigator.mediaSession : undefined +} + +/** Whether the OS-level media controls are available at all. */ +export function supported(): boolean { + return session() !== undefined && typeof MediaMetadata !== 'undefined' +} + +/** + * Publish what is playing. Safe to call on every track change; callers do not + * need to check support first. + */ +export function setNowPlaying(np: NowPlaying): void { + const s = session() + if (!s || typeof MediaMetadata === 'undefined') return + try { + s.metadata = new MediaMetadata({ + title: np.title, + artist: np.artist ?? '', + album: np.album ?? '', + artwork: np.artwork + ? [{ src: np.artwork, sizes: '512x512', type: 'image/png' }] + : [], + }) + } catch { + // A malformed metadata object must not stop playback. + } +} + +/** Register the actions the player can honour. Unlisted actions stay absent. */ +export function setHandlers(handlers: { + play?: ActionHandler + pause?: ActionHandler + next?: ActionHandler + previous?: ActionHandler + seekBackward?: ActionHandler + seekForward?: ActionHandler + seekTo?: ActionHandler +}): void { + const s = session() + if (!s) return + + const pairs: [MediaSessionAction, ActionHandler | undefined][] = [ + ['play', handlers.play], + ['pause', handlers.pause], + ['nexttrack', handlers.next], + ['previoustrack', handlers.previous], + ['seekbackward', handlers.seekBackward], + ['seekforward', handlers.seekForward], + ['seekto', handlers.seekTo], + ] + + for (const [action, fn] of pairs) { + try { + // null clears a previously registered handler; passing undefined would + // throw in some browsers. + s.setActionHandler(action, fn ?? null) + } catch { + // Unsupported action for this browser — skip it. + } + } +} + +/** + * Tell the OS where playback is, so the lock-screen scrubber is accurate. + * Some browsers reject a position without a duration, so this is skipped + * until the track has loaded metadata. + */ +export function setPosition(position: number, duration: number, rate = 1): void { + const s = session() + if (!s || typeof s.setPositionState !== 'function') return + if (!isFinite(duration) || duration <= 0) return + const pos = Math.min(Math.max(position, 0), duration) + try { + s.setPositionState({ duration, playbackRate: rate, position: pos }) + } catch { + // Out-of-range or unsupported; the scrubber just stays as it was. + } +} + +/** Mirror the element's play/pause state onto the OS controls. */ +export function setPlaybackState(state: 'playing' | 'paused' | 'none'): void { + const s = session() + if (!s) return + try { + s.playbackState = state + } catch { + // Ignore. + } +} + +/** Drop metadata and handlers when the player unmounts. */ +export function clear(): void { + const s = session() + if (!s) return + try { + s.metadata = null + s.playbackState = 'none' + } catch { + // Ignore. + } + setHandlers({}) +} diff --git a/web/src/lib/playback.ts b/web/src/lib/playback.ts new file mode 100644 index 0000000..70edb84 --- /dev/null +++ b/web/src/lib/playback.ts @@ -0,0 +1,147 @@ +// Direct-play decisions. +// +// Vellum does not transcode — the server has no spare CPU for it — so a file +// either plays in the client or it does not. Which one it is depends on the +// CLIENT, not the file: Chrome on Linux cannot decode HEVC, Safari can, and the +// native Jellyfin apps handle far more than any browser. So nothing here is a +// server-side verdict; it is a per-client guess, and the player always confirms +// it by actually loading the file. +// +// The mappings below were measured, not guessed. Files generated with known +// codecs and played through the server's own stream endpoint in Chrome 155 on +// Linux gave this: +// +// mp4 h264 + aac plays mkv h264 + aac PLAYS (container is fine!) +// webm vp9 + opus plays mov h264 + aac plays +// mp4 av1 + opus plays webm vp8 + vorbis plays +// mp4 hevc + aac AUDIO ONLY — videoWidth stays 0 +// mkv hevc + flac AUDIO ONLY +// avi mpeg4 + mp3 fails outright +// +// Two things that follow, and that the UI must respect: +// +// 1. Container matters far less than codec. mkv plays fine when the video +// track is h264; mp4 fails when the track is HEVC. Judging a file by its +// extension is wrong. +// 2. `canPlayType` is a hint, not an answer. It said "no" for mov+h264 (which +// plays) and "maybe" for mkv+hevc (whose video never decodes). The only +// reliable test is to load the file and check `videoWidth`. + +import type { MediaFile } from './api' + +/** Container extension → MIME type. Mirrors internal/stream's table. */ +const CONTAINER_MIME: Record = { + mp4: 'video/mp4', + m4v: 'video/mp4', + mov: 'video/quicktime', + mkv: 'video/x-matroska', + webm: 'video/webm', + avi: 'video/x-msvideo', + mp3: 'audio/mpeg', + m4a: 'audio/mp4', + m4b: 'audio/mp4', + aac: 'audio/aac', + flac: 'audio/flac', + opus: 'audio/ogg', + ogg: 'audio/ogg', + wav: 'audio/wav', +} + +/** + * ffprobe codec name → an RFC 6381 codec parameter good enough for + * `canPlayType`. The full parameter string (profile, level, tier) is not + * recoverable from a bare codec name, so these are the shortest forms browsers + * accept. + */ +const VCODEC: Record = { + h264: 'avc1.42E01E', + avc1: 'avc1.42E01E', + hevc: 'hvc1.1.6.L93.B0', + h265: 'hvc1.1.6.L93.B0', + vp8: 'vp8', + vp9: 'vp09.00.10.08', + av1: 'av01.0.04M.08', + mpeg4: 'mp4v.20.9', + mpeg2video: 'mp2v', + vc1: 'vc-1', + theora: 'theora', +} + +const ACODEC: Record = { + aac: 'mp4a.40.2', + mp3: 'mp3', + opus: 'opus', + vorbis: 'vorbis', + flac: 'flac', + ac3: 'ac-3', + eac3: 'ec-3', + dts: 'dts', + pcm_s16le: 'pcm', +} + +export type Verdict = 'probably' | 'maybe' | 'no' + +/** The MIME type for a file's container, or '' if it is unknown. */ +export function mimeFor(file: MediaFile): string { + return CONTAINER_MIME[(file.container || '').toLowerCase()] ?? '' +} + +/** + * Build the `type` string for a media element, e.g. + * `video/mp4; codecs="avc1.42E01E, mp4a.40.2"`. Returns '' when the container + * is unknown, in which case the caller should let the element sniff the file. + */ +export function sourceType(file: MediaFile): string { + const mime = mimeFor(file) + if (!mime) return '' + const codecs: string[] = [] + const v = VCODEC[(file.vcodec || '').toLowerCase()] + const a = ACODEC[(file.acodec || '').toLowerCase()] + if (v) codecs.push(v) + if (a) codecs.push(a) + return codecs.length ? `${mime}; codecs="${codecs.join(', ')}"` : mime +} + +/** + * Ask this browser whether it can play the file. A hint only — see the note at + * the top of this file about how wrong `canPlayType` is in both directions. + */ +export function canDirectPlay(file: MediaFile): Verdict { + if (typeof document === 'undefined') return 'maybe' + const type = sourceType(file) + if (!type) return 'maybe' + const el = document.createElement('video') + const verdict = el.canPlayType(type) + if (verdict === 'probably') return 'probably' + if (verdict === 'maybe') return 'maybe' + return 'no' +} + +/** + * A short human explanation for a file that is not expected to play, or '' when + * there is nothing worth saying. The wording avoids blaming the user's machine + * and points at the actual cause. + */ +export function playbackNote(file: MediaFile): string { + if (canDirectPlay(file) !== 'no') return '' + const codec = (file.vcodec || '').toLowerCase() + if (codec === 'hevc' || codec === 'h265') { + return 'This file uses HEVC video, which this browser cannot decode. Safari and the native apps can.' + } + if (codec === 'mpeg4' || codec === 'mpeg2video' || codec === 'vc1') { + return `This file uses ${codec} video, which browsers no longer support.` + } + return 'This browser cannot play this file directly, and Vellum does not transcode.' +} + +/** + * Whether a loaded media element actually decoded a video track. + * + * This is the reliable test, and the reason the player never trusts + * `canPlayType`: an element whose metadata loaded but whose video track is + * undecodable reports a duration and plays AUDIO while `videoWidth` stays 0. + * That is exactly what HEVC does in Chrome. + */ +export function hasDecodedVideo(el: HTMLVideoElement): boolean { + return el.videoWidth > 0 && el.videoHeight > 0 +}