diff --git a/cmd/vellum/main.go b/cmd/vellum/main.go index 62a5834..1ed253a 100644 --- a/cmd/vellum/main.go +++ b/cmd/vellum/main.go @@ -15,6 +15,7 @@ import ( "github.com/mithral/vellum/internal/auth" "github.com/mithral/vellum/internal/core" "github.com/mithral/vellum/internal/httpapi" + "github.com/mithral/vellum/internal/library" "github.com/mithral/vellum/migrations" ) @@ -65,9 +66,12 @@ func run() error { return fmt.Errorf("bootstrap admin: %w", err) } - auth.NewHandlers(authenticator, log).Register(api.Mux()) + authHandlers := auth.NewHandlers(authenticator, log) + authHandlers.Register(api.Mux()) api.SetAuth(authenticator) + library.NewHandlers(db, log, authenticator).Register(api.Mux()) + srv := &http.Server{ Addr: cfg.Listen, Handler: api.Handler(), diff --git a/internal/core/migrate_test.go b/internal/core/migrate_test.go index e05911d..7045d32 100644 --- a/internal/core/migrate_test.go +++ b/internal/core/migrate_test.go @@ -43,8 +43,8 @@ func TestMigrateAppliesAndIsIdempotent(t *testing.T) { if err := db.QueryRowContext(ctx, "PRAGMA user_version").Scan(&version); err != nil { t.Fatalf("read user_version: %v", err) } - if version != 1 { - t.Errorf("user_version = %d, want 1", version) + if version != 2 { + t.Errorf("user_version = %d, want 2", version) } var tables int diff --git a/internal/library/handlers.go b/internal/library/handlers.go new file mode 100644 index 0000000..267333a --- /dev/null +++ b/internal/library/handlers.go @@ -0,0 +1,230 @@ +package library + +import ( + "database/sql" + "errors" + "log/slog" + "net/http" + "strconv" + + "github.com/mithral/vellum/internal/auth" + "github.com/mithral/vellum/internal/httpx" +) + +// Handlers serves library management and scanning endpoints. +type Handlers struct { + db *sql.DB + log *slog.Logger + auth *auth.Authenticator +} + +// NewHandlers builds the library HTTP layer. +func NewHandlers(db *sql.DB, log *slog.Logger, a *auth.Authenticator) *Handlers { + return &Handlers{db: db, log: log, auth: a} +} + +// Register mounts the routes. +func (h *Handlers) Register(mux *http.ServeMux) { + mux.HandleFunc("GET /api/v1/media-types", h.handleMediaTypes) + + mux.Handle("GET /api/v1/libraries", h.auth.Require(h.handleList)) + mux.Handle("GET /api/v1/libraries/{id}", h.auth.Require(h.handleGet)) + mux.Handle("GET /api/v1/libraries/{id}/items", h.auth.Require(h.handleListItems)) + + mux.Handle("POST /api/v1/libraries", h.auth.RequireAdmin(http.HandlerFunc(h.handleCreate))) + mux.Handle("DELETE /api/v1/libraries/{id}", h.auth.RequireAdmin(http.HandlerFunc(h.handleDelete))) + mux.Handle("POST /api/v1/libraries/{id}/scan", h.auth.RequireAdmin(http.HandlerFunc(h.handleScan))) +} + +// handleMediaTypes is public: the UI needs the list before login to render. +func (h *Handlers) handleMediaTypes(w http.ResponseWriter, r *http.Request) { + httpx.WriteJSON(w, http.StatusOK, map[string]any{"media_types": SupportedTypes()}) +} + +func (h *Handlers) handleList(w http.ResponseWriter, r *http.Request, u auth.User) { + ids, err := auth.VisibleLibraryIDs(r.Context(), h.db, u) + if err != nil { + h.fail(w, "visible libraries", err) + return + } + libs, err := ListVisible(r.Context(), h.db, ids) + if err != nil { + h.fail(w, "list libraries", err) + return + } + httpx.WriteJSON(w, http.StatusOK, map[string]any{"libraries": libs}) +} + +func (h *Handlers) handleGet(w http.ResponseWriter, r *http.Request, u auth.User) { + id, ok := pathInt(w, r, "id") + if !ok { + return + } + allowed, err := auth.CanAccessLibrary(r.Context(), h.db, u, id) + if err != nil { + h.fail(w, "check access", err) + return + } + if !allowed { + // 404 rather than 403: an unauthorised user should not learn that a + // library exists. + httpx.WriteError(w, http.StatusNotFound, "library not found") + return + } + lib, err := Get(r.Context(), h.db, id) + if errors.Is(err, ErrNotFound) { + httpx.WriteError(w, http.StatusNotFound, "library not found") + return + } + if err != nil { + h.fail(w, "get library", err) + return + } + count, _ := CountItems(r.Context(), h.db, id) + httpx.WriteJSON(w, http.StatusOK, map[string]any{ + "library": lib, + "item_count": count, + "reachable": RootExists(lib), + }) +} + +func (h *Handlers) handleCreate(w http.ResponseWriter, r *http.Request) { + var req struct { + Name string `json:"name"` + Kind string `json:"kind"` + RootPath string `json:"root_path"` + } + if err := httpx.DecodeJSON(r, &req); err != nil { + httpx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + + lib, err := Create(r.Context(), h.db, req.Name, req.Kind, req.RootPath) + switch { + case errors.Is(err, ErrNameExists): + httpx.WriteError(w, http.StatusConflict, "a library with that name already exists") + return + case errors.Is(err, ErrInvalidRoot), errors.Is(err, ErrUnknownKind): + httpx.WriteError(w, http.StatusBadRequest, err.Error()) + return + case err != nil: + h.fail(w, "create library", err) + return + } + + h.log.Info("library created", "name", lib.Name, "kind", lib.Kind, "path", lib.RootPath) + httpx.WriteJSON(w, http.StatusCreated, map[string]any{"library": lib}) +} + +func (h *Handlers) handleDelete(w http.ResponseWriter, r *http.Request) { + id, ok := pathInt(w, r, "id") + if !ok { + return + } + if err := Delete(r.Context(), h.db, id); errors.Is(err, ErrNotFound) { + httpx.WriteError(w, http.StatusNotFound, "library not found") + return + } else if err != nil { + h.fail(w, "delete library", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handlers) handleScan(w http.ResponseWriter, r *http.Request) { + id, ok := pathInt(w, r, "id") + if !ok { + return + } + lib, err := Get(r.Context(), h.db, id) + if errors.Is(err, ErrNotFound) { + httpx.WriteError(w, http.StatusNotFound, "library not found") + return + } + if err != nil { + h.fail(w, "get library", err) + return + } + + res, err := Scan(r.Context(), h.db, lib, h.log) + if err != nil { + h.fail(w, "scan library", err) + return + } + httpx.WriteJSON(w, http.StatusOK, res) +} + +// handleListItems returns items in a library the caller may read. +func (h *Handlers) handleListItems(w http.ResponseWriter, r *http.Request, u auth.User) { + id, ok := pathInt(w, r, "id") + if !ok { + return + } + allowed, err := auth.CanAccessLibrary(r.Context(), h.db, u, id) + if err != nil { + h.fail(w, "check access", err) + return + } + if !allowed { + httpx.WriteError(w, http.StatusNotFound, "library not found") + return + } + + limit := 100 + if v := r.URL.Query().Get("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 { + limit = n + } + } + + rows, err := h.db.QueryContext(r.Context(), ` + SELECT id, kind, path, title, COALESCE(year, 0), added_at + FROM media_items + WHERE library_id = ? AND missing = 0 + ORDER BY sort_title + LIMIT ?`, id, limit) + if err != nil { + h.fail(w, "list items", err) + return + } + defer rows.Close() + + type itemDTO struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + Path string `json:"path"` + Title string `json:"title"` + Year int `json:"year"` + AddedAt string `json:"added_at"` + } + items := []itemDTO{} + for rows.Next() { + var it itemDTO + if err := rows.Scan(&it.ID, &it.Kind, &it.Path, &it.Title, &it.Year, &it.AddedAt); err != nil { + h.fail(w, "scan items", err) + return + } + items = append(items, it) + } + if err := rows.Err(); err != nil { + h.fail(w, "iterate items", err) + return + } + httpx.WriteJSON(w, http.StatusOK, map[string]any{"items": items, "limit": limit}) +} + +// ── helpers ──────────────────────────────────────────────────────────── + +func (h *Handlers) fail(w http.ResponseWriter, what string, err error) { + h.log.Error(what, "err", err) + httpx.WriteError(w, http.StatusInternalServerError, "internal server error") +} + +func pathInt(w http.ResponseWriter, r *http.Request, name string) (int64, bool) { + n, err := strconv.ParseInt(r.PathValue(name), 10, 64) + if err != nil || n <= 0 { + httpx.WriteError(w, http.StatusBadRequest, "invalid "+name) + return 0, false + } + return n, true +} diff --git a/internal/library/library.go b/internal/library/library.go new file mode 100644 index 0000000..298268c --- /dev/null +++ b/internal/library/library.go @@ -0,0 +1,192 @@ +package library + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +var ( + // ErrNotFound means no library matched. + ErrNotFound = errors.New("library: not found") + // ErrNameExists means the library name is taken. + ErrNameExists = errors.New("library: name already in use") + // ErrInvalidRoot means the root path is missing, not a directory, or not + // readable. + ErrInvalidRoot = errors.New("library: invalid root path") +) + +// Library is a media root of a single kind. +type Library struct { + ID int64 `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + RootPath string `json:"root_path"` + CreatedAt time.Time `json:"created_at"` +} + +const librarySelect = `SELECT id, name, kind, root_path, created_at FROM libraries` + +// Create validates and inserts a library. +// +// The root path is resolved to an absolute, symlink-free path and must exist. +// Storing the resolved form matters: the scanner compares filesystem paths +// against this value, and "/media/../media" would silently match nothing. +func Create(ctx context.Context, db *sql.DB, name, kind, rootPath string) (Library, error) { + name = strings.TrimSpace(name) + if name == "" { + return Library{}, errors.New("library: name must not be empty") + } + if err := ValidateKind(kind); err != nil { + return Library{}, err + } + + resolved, err := resolveRoot(rootPath) + if err != nil { + return Library{}, err + } + + var lib Library + var created string + err = db.QueryRowContext(ctx, ` + INSERT INTO libraries (name, kind, root_path) + VALUES (?, ?, ?) + RETURNING id, created_at`, + name, strings.ToLower(kind), resolved, + ).Scan(&lib.ID, &created) + if err != nil { + if isUnique(err) { + return Library{}, ErrNameExists + } + return Library{}, fmt.Errorf("library: create: %w", err) + } + lib.Name, lib.Kind, lib.RootPath = name, strings.ToLower(kind), resolved + lib.CreatedAt, _ = time.Parse(time.RFC3339Nano, created) + return lib, nil +} + +// Get loads one library by id. +func Get(ctx context.Context, db *sql.DB, id int64) (Library, error) { + return scanLibrary(db.QueryRowContext(ctx, librarySelect+` WHERE id = ?`, id)) +} + +// List returns all libraries. +func List(ctx context.Context, db *sql.DB) ([]Library, error) { + return queryLibraries(ctx, db, librarySelect+` ORDER BY name`) +} + +// ListVisible returns the libraries in the given id set. An empty set yields an +// empty slice, never "all" — that distinction is the whole point of the +// access-control model. +func ListVisible(ctx context.Context, db *sql.DB, ids []int64) ([]Library, error) { + if len(ids) == 0 { + return []Library{}, nil + } + placeholders := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + q := librarySelect + ` WHERE id IN (` + strings.Join(placeholders, ",") + `) ORDER BY name` + return queryLibraries(ctx, db, q, args...) +} + +// Delete removes a library. Items, files, and grants cascade. +func Delete(ctx context.Context, db *sql.DB, id int64) error { + res, err := db.ExecContext(ctx, `DELETE FROM libraries WHERE id = ?`, id) + if err != nil { + return fmt.Errorf("library: delete: %w", err) + } + n, _ := res.RowsAffected() + if n == 0 { + return ErrNotFound + } + return nil +} + +// CountItems returns how many items a library holds (excluding missing). +func CountItems(ctx context.Context, db *sql.DB, id int64) (int, error) { + var n int + err := db.QueryRowContext(ctx, + `SELECT count(*) FROM media_items WHERE library_id = ? AND missing = 0`, id).Scan(&n) + return n, err +} + +// ── helpers ──────────────────────────────────────────────────────────── + +func resolveRoot(rootPath string) (string, error) { + if strings.TrimSpace(rootPath) == "" { + return "", fmt.Errorf("%w: path is empty", ErrInvalidRoot) + } + abs, err := filepath.Abs(rootPath) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrInvalidRoot, err) + } + if resolved, err := filepath.EvalSymlinks(abs); err == nil { + abs = resolved + } else { + return "", fmt.Errorf("%w: %v", ErrInvalidRoot, err) + } + + info, err := os.Stat(abs) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrInvalidRoot, err) + } + if !info.IsDir() { + return "", fmt.Errorf("%w: %s is not a directory", ErrInvalidRoot, abs) + } + // Confirm readability; a directory can exist and still deny access. + f, err := os.Open(abs) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrInvalidRoot, err) + } + _ = f.Close() + return abs, nil +} + +func queryLibraries(ctx context.Context, db *sql.DB, q string, args ...any) ([]Library, error) { + rows, err := db.QueryContext(ctx, q, args...) + if err != nil { + return nil, fmt.Errorf("library: query: %w", err) + } + defer rows.Close() + + out := []Library{} + for rows.Next() { + lib, err := scanLibrary(rows) + if err != nil { + return nil, err + } + out = append(out, lib) + } + return out, rows.Err() +} + +type scanner interface{ Scan(dest ...any) error } + +func scanLibrary(row scanner) (Library, error) { + var lib Library + var created string + if err := row.Scan(&lib.ID, &lib.Name, &lib.Kind, &lib.RootPath, &created); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Library{}, ErrNotFound + } + return Library{}, err + } + lib.CreatedAt, _ = time.Parse(time.RFC3339Nano, created) + return lib, nil +} + +func isUnique(err error) bool { + if err == nil { + return false + } + msg := strings.ToUpper(err.Error()) + return strings.Contains(msg, "UNIQUE CONSTRAINT FAILED") || strings.Contains(msg, "CONSTRAINT FAILED: UNIQUE") +} diff --git a/internal/library/library_test.go b/internal/library/library_test.go new file mode 100644 index 0000000..2ab7f99 --- /dev/null +++ b/internal/library/library_test.go @@ -0,0 +1,111 @@ +package library + +import ( + "context" + "database/sql" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/mithral/vellum/internal/core" + "github.com/mithral/vellum/migrations" +) + +func testDB(t *testing.T) *sql.DB { + t.Helper() + db, err := core.OpenDB(filepath.Join(t.TempDir(), "lib.db")) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + if err := core.Migrate(context.Background(), db, migrations.FS, log); err != nil { + t.Fatalf("Migrate: %v", err) + } + return db +} + +func testLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestRegistryBasics(t *testing.T) { + if err := ValidateKind("ebooks"); err != nil { + t.Errorf("ebooks should be valid: %v", err) + } + if err := ValidateKind("nonsense"); !errors.Is(err, ErrUnknownKind) { + t.Errorf("nonsense kind error = %v, want ErrUnknownKind", err) + } + if k, ok := KindForExtension(".epub"); !ok || k != "ebooks" { + t.Errorf("KindForExtension(.epub) = %q,%v", k, ok) + } + if k, ok := KindForExtension("CBZ"); !ok || k != "comics" { + t.Errorf("KindForExtension(CBZ) = %q,%v (should be case-insensitive)", k, ok) + } + if len(SupportedTypes()) == 0 { + t.Error("no media types registered") + } +} + +func TestCleanTitle(t *testing.T) { + cases := map[string]string{ + "Project Hail Mary.epub": "Project Hail Mary", + "the_hobbit.mobi": "the hobbit", + "Book.Title.2020.epub": "Book Title 2020", + "/deep/path/Saga 001.cbz": "Saga 001", + } + for in, want := range cases { + if got := CleanTitle(in); got != want { + t.Errorf("CleanTitle(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCreateLibraryValidation(t *testing.T) { + db := testDB(t) + ctx := context.Background() + root := t.TempDir() + + if _, err := Create(ctx, db, "Good", "ebooks", root); err != nil { + t.Fatalf("valid create failed: %v", err) + } + if _, err := Create(ctx, db, "Good", "ebooks", root); !errors.Is(err, ErrNameExists) { + t.Errorf("duplicate name error = %v, want ErrNameExists", err) + } + if _, err := Create(ctx, db, "Bad", "nonsense", root); !errors.Is(err, ErrUnknownKind) { + t.Errorf("bad kind error = %v, want ErrUnknownKind", err) + } + if _, err := Create(ctx, db, "Bad", "ebooks", "/nope/not/here"); !errors.Is(err, ErrInvalidRoot) { + t.Errorf("bad root error = %v, want ErrInvalidRoot", err) + } +} + +func TestListVisibleNeverMeansAll(t *testing.T) { + db := testDB(t) + ctx := context.Background() + root := t.TempDir() + if _, err := Create(ctx, db, "Ebooks", "ebooks", root); err != nil { + t.Fatal(err) + } + + libs, err := ListVisible(ctx, db, nil) + if err != nil { + t.Fatal(err) + } + if len(libs) != 0 { + t.Errorf("ListVisible(nil) returned %d libraries; empty set must mean none", len(libs)) + } +} diff --git a/internal/library/registry.go b/internal/library/registry.go new file mode 100644 index 0000000..114bc41 --- /dev/null +++ b/internal/library/registry.go @@ -0,0 +1,138 @@ +// Package library owns library definitions and the scanner that turns files on +// disk into rows in media_items / media_files. +// +// Modules declare what file extensions they care about via RegisterType; this +// package never imports a module. +package library + +import ( + "errors" + "fmt" + "path/filepath" + "sort" + "strings" + "sync" +) + +// MediaType declares one kind of library and the file extensions it owns. +type MediaType struct { + // Kind is the value stored in libraries.kind and media_items.kind. + Kind string + // Name is the human label shown in the UI. + Name string + // Extensions are lowercase, with a leading dot (".epub"). + Extensions []string +} + +// builtinTypes are the media types Vellum understands out of the box. Modules +// may add more at runtime with RegisterType; these exist so the core is +// useful before any module is wired up. +var builtinTypes = []MediaType{ + {Kind: "ebooks", Name: "Ebooks", Extensions: []string{".epub", ".mobi", ".azw3", ".fb2"}}, + {Kind: "comics", Name: "Comics", Extensions: []string{".cbz", ".cbr", ".pdf"}}, + {Kind: "audiobooks", Name: "Audiobooks", Extensions: []string{".m4b", ".mp3", ".m4a", ".opus", ".flac", ".ogg"}}, + {Kind: "music", Name: "Music", Extensions: []string{".mp3", ".flac", ".m4a", ".opus", ".ogg", ".wav"}}, + {Kind: "photos", Name: "Photos", Extensions: []string{".jpg", ".jpeg", ".png", ".webp", ".heic", ".gif", ".avif"}}, + {Kind: "video", Name: "Video", Extensions: []string{".mkv", ".mp4", ".webm", ".avi", ".mov"}}, +} + +var ( + typesMu sync.RWMutex + types = map[string]MediaType{} + extToKind = map[string]string{} +) + +func init() { + for _, t := range builtinTypes { + registerTypeLocked(t) + } +} + +// RegisterType adds or replaces a media type. Intended to be called from a +// module's init or wiring function. +func RegisterType(t MediaType) { + typesMu.Lock() + defer typesMu.Unlock() + registerTypeLocked(t) +} + +func registerTypeLocked(t MediaType) { + t.Kind = strings.ToLower(strings.TrimSpace(t.Kind)) + for i, e := range t.Extensions { + t.Extensions[i] = normalizeExt(e) + } + types[t.Kind] = t + for _, e := range t.Extensions { + // Last registration wins for a shared extension (e.g. .pdf is comics). + extToKind[e] = t.Kind + } +} + +// SupportedTypes returns every registered media type, sorted by kind. +func SupportedTypes() []MediaType { + typesMu.RLock() + defer typesMu.RUnlock() + out := make([]MediaType, 0, len(types)) + for _, t := range types { + out = append(out, t) + } + sort.Slice(out, func(i, j int) bool { return out[i].Kind < out[j].Kind }) + return out +} + +// KindForExtension maps a file extension to the media kind that owns it. +func KindForExtension(ext string) (string, bool) { + typesMu.RLock() + defer typesMu.RUnlock() + k, ok := extToKind[normalizeExt(ext)] + return k, ok +} + +var ( + // ErrUnknownKind means the requested media kind is not registered. + ErrUnknownKind = errors.New("library: unknown media kind") +) + +// ValidateKind reports whether kind is a registered media type. +func ValidateKind(kind string) error { + typesMu.RLock() + defer typesMu.RUnlock() + k := strings.ToLower(strings.TrimSpace(kind)) + if _, ok := types[k]; !ok { + return fmt.Errorf("%w %q", ErrUnknownKind, kind) + } + return nil +} + +// extensionsFor returns the extension set owned by a library kind. +func extensionsFor(kind string) map[string]bool { + typesMu.RLock() + defer typesMu.RUnlock() + t, ok := types[kind] + if !ok { + return nil + } + set := make(map[string]bool, len(t.Extensions)) + for _, e := range t.Extensions { + set[e] = true + } + return set +} + +func normalizeExt(ext string) string { + ext = strings.ToLower(strings.TrimSpace(ext)) + if ext != "" && !strings.HasPrefix(ext, ".") { + ext = "." + ext + } + return ext +} + +// CleanTitle turns a filename into a human title: strips the extension, +// collapses separators, and trims noise. +func CleanTitle(filename string) string { + base := strings.TrimSuffix(filepath.Base(filename), filepath.Ext(filename)) + base = strings.ReplaceAll(base, "_", " ") + base = strings.ReplaceAll(base, ".", " ") + base = strings.Join(strings.Fields(base), " ") + return strings.TrimSpace(base) +} diff --git a/internal/library/scan.go b/internal/library/scan.go new file mode 100644 index 0000000..3542814 --- /dev/null +++ b/internal/library/scan.go @@ -0,0 +1,285 @@ +package library + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +// ScanResult summarises one scan pass. +type ScanResult struct { + LibraryID int64 `json:"library_id"` + Scanned int `json:"scanned"` + Added int `json:"added"` + Updated int `json:"updated"` + Restored int `json:"restored"` + Missing int `json:"missing"` + Unchanged int `json:"unchanged"` + Duration int64 `json:"duration_ms"` + Errors []string `json:"errors,omitempty"` +} + +// candidate is a supported file found on disk. +type candidate struct { + relPath string + size int64 + mtime int64 + inode int64 + ext string +} + +// knownFile is an existing row for this library. +type knownFile struct { + itemID int64 + fileID int64 + size int64 + mtime int64 + missing bool +} + +// Scan walks a library's root and reconciles it with the database. +// +// Files that changed are re-upserted; files that vanished are marked missing +// rather than deleted, so their progress survives a temporarily unavailable +// mount. A scan never fails because of a single unreadable file — it records +// the error and continues, because a partial library is better than none. +func Scan(ctx context.Context, db *sql.DB, lib Library, log *slog.Logger) (ScanResult, error) { + start := time.Now() + res := ScanResult{LibraryID: lib.ID} + + exts := extensionsFor(lib.Kind) + if len(exts) == 0 { + return res, fmt.Errorf("library %d: no extensions registered for kind %q", lib.ID, lib.Kind) + } + + // 1. What do we already know? + known, err := loadKnownFiles(ctx, db, lib.ID) + if err != nil { + return res, err + } + + // 2. What is on disk? + found, errs := walkRoot(lib.RootPath, exts) + res.Errors = errs + + // 3. Reconcile in one transaction. + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return res, err + } + defer tx.Rollback() //nolint:errcheck + + now := time.Now().UTC().Format(time.RFC3339Nano) + for rel, c := range found { + res.Scanned++ + prev, exists := known[rel] + + switch { + case exists && !prev.missing && prev.size == c.size && prev.mtime == c.mtime: + res.Unchanged++ + continue + + case exists && prev.missing: + res.Restored++ + case exists: + res.Updated++ + default: + res.Added++ + } + + itemID, err := upsertItem(ctx, tx, lib, rel, c) + if err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("%s: %v", rel, err)) + continue + } + if err := upsertFile(ctx, tx, itemID, rel, c); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("%s: %v", rel, err)) + continue + } + } + + // 4. Anything we knew about that is gone. + for rel := range known { + if _, ok := found[rel]; ok { + continue + } + if err := markMissing(ctx, tx, lib.ID, rel, now); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("%s: %v", rel, err)) + continue + } + res.Missing++ + } + + if err := tx.Commit(); err != nil { + return res, fmt.Errorf("library: commit scan: %w", err) + } + + res.Duration = time.Since(start).Milliseconds() + log.Info("scan complete", + "library", lib.Name, "kind", lib.Kind, + "scanned", res.Scanned, "added", res.Added, "updated", res.Updated, + "restored", res.Restored, "missing", res.Missing, "unchanged", res.Unchanged, + "duration_ms", res.Duration, "errors", len(res.Errors)) + return res, nil +} + +// walkRoot collects supported files under root, keyed by slash-separated +// path relative to root. +func walkRoot(root string, exts map[string]bool) (map[string]candidate, []string) { + out := make(map[string]candidate) + var errs []string + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", path, err)) + // An unreadable directory should not abort the walk. + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + + name := d.Name() + if d.IsDir() { + if path != root && strings.HasPrefix(name, ".") { + return fs.SkipDir + } + return nil + } + if strings.HasPrefix(name, ".") { + return nil + } + + ext := strings.ToLower(filepath.Ext(name)) + if !exts[ext] { + return nil + } + + info, err := d.Info() + if err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", path, err)) + return nil + } + if !info.Mode().IsRegular() { + return nil + } + + rel, err := filepath.Rel(root, path) + if err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", path, err)) + return nil + } + + c := candidate{ + relPath: filepath.ToSlash(rel), + size: info.Size(), + mtime: info.ModTime().UnixNano(), + ext: strings.TrimPrefix(ext, "."), + } + if st, ok := info.Sys().(*syscall.Stat_t); ok { + c.inode = int64(st.Ino) + } + out[c.relPath] = c + return nil + }) + if err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", root, err)) + } + return out, errs +} + +func loadKnownFiles(ctx context.Context, db *sql.DB, libraryID int64) (map[string]knownFile, error) { + rows, err := db.QueryContext(ctx, ` + SELECT f.path, f.item_id, f.id, f.size, f.mtime, f.missing + FROM media_files f + JOIN media_items i ON i.id = f.item_id + WHERE i.library_id = ?`, libraryID) + if err != nil { + return nil, fmt.Errorf("library: load known files: %w", err) + } + defer rows.Close() + + out := make(map[string]knownFile) + for rows.Next() { + var ( + absPath string + kf knownFile + mtime string + missing int + ) + if err := rows.Scan(&absPath, &kf.itemID, &kf.fileID, &kf.size, &mtime, &missing); err != nil { + return nil, err + } + kf.mtime = parseUnixNano(mtime) + kf.missing = missing == 1 + out[absPath] = kf + } + return out, rows.Err() +} + +func upsertItem(ctx context.Context, tx *sql.Tx, lib Library, rel string, c candidate) (int64, error) { + title := CleanTitle(rel) + var id int64 + err := tx.QueryRowContext(ctx, ` + INSERT INTO media_items (library_id, kind, path, title, sort_title, missing) + VALUES (?, ?, ?, ?, ?, 0) + ON CONFLICT (library_id, path) + DO UPDATE SET missing = 0 + RETURNING id`, + lib.ID, lib.Kind, rel, title, strings.ToLower(title), + ).Scan(&id) + if err != nil { + return 0, fmt.Errorf("upsert item: %w", err) + } + return id, nil +} + +func upsertFile(ctx context.Context, tx *sql.Tx, itemID int64, rel string, c candidate) error { + _, err := tx.ExecContext(ctx, ` + INSERT INTO media_files (item_id, path, size, mtime, inode, container, missing) + VALUES (?, ?, ?, ?, ?, ?, 0) + ON CONFLICT (path) + DO UPDATE SET item_id = excluded.item_id, size = excluded.size, + mtime = excluded.mtime, inode = excluded.inode, + container = excluded.container, missing = 0`, + itemID, rel, c.size, fmt.Sprint(c.mtime), c.inode, c.ext) + if err != nil { + return fmt.Errorf("upsert file: %w", err) + } + return nil +} + +func markMissing(ctx context.Context, tx *sql.Tx, libraryID int64, rel, now string) error { + if _, err := tx.ExecContext(ctx, + `UPDATE media_files SET missing = 1 WHERE path = ?`, rel); err != nil { + return err + } + _, err := tx.ExecContext(ctx, ` + UPDATE media_items SET missing = 1 + WHERE library_id = ? AND path = ?`, libraryID, rel) + return err +} + +func parseUnixNano(s string) int64 { + var n int64 + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return 0 + } + n = n*10 + int64(s[i]-'0') + } + return n +} + +// RootExists reports whether a library's root is currently reachable. +func RootExists(lib Library) bool { + info, err := os.Stat(lib.RootPath) + return err == nil && info.IsDir() +} diff --git a/internal/library/scan_test.go b/internal/library/scan_test.go new file mode 100644 index 0000000..ffa7a05 --- /dev/null +++ b/internal/library/scan_test.go @@ -0,0 +1,130 @@ +package library + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestScanLifecycle(t *testing.T) { + db := testDB(t) + ctx := context.Background() + root := t.TempDir() + + writeFile(t, filepath.Join(root, "a.epub"), "aaaaaaaaaa") + writeFile(t, filepath.Join(root, "b.epub"), "bbbbbbbbbbbbbbbbbbbb") + writeFile(t, filepath.Join(root, "sub", "c.epub"), "cccccccccccccccccccccccccccccc") + writeFile(t, filepath.Join(root, "notes.txt"), "not media") + writeFile(t, filepath.Join(root, ".hidden.epub"), "hidden") + writeFile(t, filepath.Join(root, ".git", "d.epub"), "in a hidden dir") + + lib, err := Create(ctx, db, "Ebooks", "ebooks", root) + if err != nil { + t.Fatalf("Create: %v", err) + } + + // First scan: three supported files, nothing else. + res, err := Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if res.Added != 3 || res.Scanned != 3 { + t.Fatalf("first scan added=%d scanned=%d, want 3/3 (%+v)", res.Added, res.Scanned, res) + } + if n, _ := CountItems(ctx, db, lib.ID); n != 3 { + t.Errorf("item count = %d, want 3", n) + } + + // Second scan: nothing changed. + res, err = Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatal(err) + } + if res.Unchanged != 3 || res.Added != 0 || res.Updated != 0 { + t.Errorf("second scan unchanged=%d added=%d updated=%d, want 3/0/0", res.Unchanged, res.Added, res.Updated) + } + + // Modify a.epub (different size). + writeFile(t, filepath.Join(root, "a.epub"), "aaaaaaaaaaaaaaaaaaaaaaaaaaaa") + res, err = Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatal(err) + } + if res.Updated != 1 || res.Unchanged != 2 { + t.Errorf("after modify updated=%d unchanged=%d, want 1/2", res.Updated, res.Unchanged) + } + + // Delete b.epub: it goes missing, it is not deleted. + if err := os.Remove(filepath.Join(root, "b.epub")); err != nil { + t.Fatal(err) + } + res, err = Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatal(err) + } + if res.Missing != 1 { + t.Errorf("after delete missing=%d, want 1", res.Missing) + } + if n, _ := CountItems(ctx, db, lib.ID); n != 2 { + t.Errorf("item count (excluding missing) = %d, want 2", n) + } + + // The row still exists, flagged missing. + var missing int + if err := db.QueryRowContext(ctx, + `SELECT missing FROM media_items WHERE library_id = ? AND path = ?`, lib.ID, "b.epub").Scan(&missing); err != nil { + t.Fatalf("missing item row was deleted: %v", err) + } + if missing != 1 { + t.Errorf("missing flag = %d, want 1", missing) + } + + // Restore b.epub. + writeFile(t, filepath.Join(root, "b.epub"), "bbbbbbbbbbbbbbbbbbbb") + res, err = Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatal(err) + } + if res.Restored != 1 { + t.Errorf("after restore restored=%d, want 1 (%+v)", res.Restored, res) + } + if n, _ := CountItems(ctx, db, lib.ID); n != 3 { + t.Errorf("item count after restore = %d, want 3", n) + } +} + +func TestScanUnreadableRoot(t *testing.T) { + db := testDB(t) + ctx := context.Background() + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.epub"), "x") + + lib, err := Create(ctx, db, "Ebooks", "ebooks", root) + if err != nil { + t.Fatal(err) + } + // Index it once so there is a known file to go missing. + if _, err := Scan(ctx, db, lib, testLogger()); err != nil { + t.Fatal(err) + } + + // Remove the root out from under the library. + if err := os.RemoveAll(root); err != nil { + t.Fatal(err) + } + + res, err := Scan(ctx, db, lib, testLogger()) + if err != nil { + t.Fatalf("scan should not fail on a missing root: %v", err) + } + if res.Added != 0 { + t.Errorf("added=%d, want 0", res.Added) + } + if res.Missing != 1 { + t.Errorf("missing=%d, want 1 (existing item now vanished)", res.Missing) + } + if len(res.Errors) == 0 { + t.Error("expected the walk error to be recorded") + } +} diff --git a/migrations/0002_missing.sql b/migrations/0002_missing.sql new file mode 100644 index 0000000..8fd77d9 --- /dev/null +++ b/migrations/0002_missing.sql @@ -0,0 +1,13 @@ +-- 0002_missing.sql — track files that disappear from disk. +-- +-- A rescan must not delete an item whose file is temporarily unavailable +-- (unmounted drive, NFS hiccup, user mid-reorganise). Deleting cascades into +-- progress, which is the one thing in this schema that cannot be regenerated. +-- Mark instead, and filter on read. A later pass can prune rows that have been +-- missing for a long time. + +ALTER TABLE media_items ADD COLUMN missing INTEGER NOT NULL DEFAULT 0; +ALTER TABLE media_files ADD COLUMN missing INTEGER NOT NULL DEFAULT 0; + +CREATE INDEX idx_items_library_missing ON media_items(library_id, missing); +CREATE INDEX idx_files_missing ON media_files(missing);