diff --git a/README.md b/README.md index da66a17..c4d98e5 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ codec-weighted bitrate (Opus and AAC count for more per kbps than MP3). You then libraries share the better file. - **Ignore.** The pair is hidden from review until you un-ignore it. +**Export.** The Review tab's **Export CSV** button downloads either the current view (filter and search applied) or the entire scan. +Each row has the match status and type, a recommendation, both paths, where linking would place the file, codec, bitrate, sample rate, +bit depth and duration for both sides, the quality verdict and any notes. The file opens cleanly in Excel, Numbers and LibreOffice +(UTF-8, and cells a spreadsheet would treat as formulas are escaped). + **Build mode** covers the "start fresh" workflow. Point the slave at an empty folder, browse the master library, tick the folders or songs you want, and playlink hardlinks them in at the same relative path. **Remove** deletes slave files, but only ones that are hardlinks of the master file. diff --git a/internal/api/export.go b/internal/api/export.go new file mode 100644 index 0000000..d5fc324 --- /dev/null +++ b/internal/api/export.go @@ -0,0 +1,179 @@ +package api + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/cdnutter/playlink/internal/engine" + "github.com/cdnutter/playlink/internal/probe" + "github.com/cdnutter/playlink/internal/store" +) + +var kindLabels = map[string]string{ + store.KindPath: "same path", + store.KindPathCase: "path differs in case", + store.KindExtension: "other format", + store.KindFuzzy: "similar title", + store.KindElsewhere: "same top folder, other subfolder", + store.KindApprox: "approximate match", + store.KindNone: "no match", +} + +var exportHeader = []string{ + "status", "match_type", "ignored", "recommendation", + "slave_path", "master_path", "link_target", "reason", + "slave_size_bytes", "master_size_bytes", + "slave_codec", "slave_lossless", "slave_bitrate_kbps", "slave_sample_rate_hz", "slave_bit_depth", "slave_duration_s", + "master_codec", "master_lossless", "master_bitrate_kbps", "master_sample_rate_hz", "master_bit_depth", "master_duration_s", + "better_copy", "duration_difference_s", "notes", "error", + "slave_modified", "master_modified", "checked_at", +} + +// csvSafe neutralises cells a spreadsheet would evaluate as a formula (a file +// named "=HYPERLINK(...).mp3" must stay text). +func csvSafe(s string) string { + if s != "" && strings.ContainsRune("=+-@\t\r", rune(s[0])) { + return "'" + s + } + return s +} + +func recommendation(m store.Match, v probe.Verdict) string { + switch { + case m.Ignored: + return "ignored" + case m.Status == store.StatusLinked: + return "done" + case m.Status == store.StatusIdentical: + return "link (identical)" + case m.Status == store.StatusDifferent || m.Status == store.StatusSimilar: + switch v.Better { + case "master": + return "upgrade to master copy" + case "slave": + return "review: master is lower quality" + } + return "review" + case m.Status == store.StatusError: + return "fix error" + } + return "" +} + +func probeCols(raw json.RawMessage) []string { + if len(raw) == 0 { + return make([]string, 6) + } + var p probe.Info + if json.Unmarshal(raw, &p) != nil || p.Error != "" { + return make([]string, 6) + } + num := func(n int) string { + if n == 0 { + return "" + } + return strconv.Itoa(n) + } + bitrate := "" + if p.Bitrate > 0 { + bitrate = strconv.FormatInt((p.Bitrate+500)/1000, 10) + } + dur := "" + if p.Duration > 0 { + dur = strconv.FormatFloat(p.Duration, 'f', 1, 64) + } + return []string{p.Codec, strconv.FormatBool(p.Lossless), bitrate, num(p.SampleRate), num(p.BitDepth), dur} +} + +func nsTime(ns int64) string { + if ns == 0 { + return "" + } + return time.Unix(0, ns).UTC().Format(time.RFC3339) +} + +var unsafeName = regexp.MustCompile(`[^A-Za-z0-9._-]+`) + +// handleExport streams the pair's scan results as CSV. It honours the same +// status and search filters as the Review list. +func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) { + id, _ := idParam(r) + p, err := s.st.Pair(r.Context(), id) + if err != nil { + opErr(w, err) + return + } + q := r.URL.Query() + f := store.MatchFilter{Status: q.Get("status"), Query: q.Get("q")} + + scope := f.Status + if scope == "" { + scope = "all" + } + slug := strings.Trim(unsafeName.ReplaceAllString(p.Name, "-"), "-") + if slug == "" { + slug = fmt.Sprintf("pair%d", p.ID) + } + name := fmt.Sprintf("playlink-%s-%s-%s.csv", slug, scope, time.Now().Format("20060102-1504")) + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, name, url.PathEscape(name))) + + // A UTF-8 byte order mark makes Excel read non-ASCII names correctly. + _, _ = w.Write([]byte{0xEF, 0xBB, 0xBF}) + cw := csv.NewWriter(w) + _ = cw.Write(exportHeader) + rows := 0 + err = s.st.ExportMatches(r.Context(), id, f, func(m store.Match) error { + var v probe.Verdict + _ = json.Unmarshal(m.Verdict, &v) + target := "" + if m.RelMaster != "" && m.Status != store.StatusLinked { + target = engine.LinkTarget(m.RelSlave, m.RelMaster, false) + } + size := func(n int64) string { + if n == 0 { + return "" + } + return strconv.FormatInt(n, 10) + } + delta := "" + if v.DurationDelta > 0 { + delta = strconv.FormatFloat(v.DurationDelta, 'f', 1, 64) + } + rec := []string{ + m.Status, kindLabels[m.Kind], strconv.FormatBool(m.Ignored), recommendation(m, v), + csvSafe(m.RelSlave), csvSafe(m.RelMaster), csvSafe(target), csvSafe(m.Reason), + size(m.SlaveSize), size(m.MasterSize), + } + rec = append(rec, probeCols(m.SlaveProbe)...) + rec = append(rec, probeCols(m.MasterProbe)...) + better := v.Better + if better == "unknown" { + better = "" + } + rec = append(rec, better, delta, csvSafe(strings.Join(v.Notes, "; ")), csvSafe(m.Note), + nsTime(m.SlaveMtime), nsTime(m.MasterMtime), time.Unix(m.UpdatedAt, 0).UTC().Format(time.RFC3339)) + rows++ + if err := cw.Write(rec); err != nil { + return err + } + if rows%500 == 0 { + cw.Flush() + } + return cw.Error() + }) + cw.Flush() + if err != nil { + // Headers are already sent; the truncated file is the only signal left. + s.log.Warn("csv export interrupted", "pair", id, "rows", rows, "err", err) + return + } + s.log.Info("csv export", "pair", p.Name, "filter", scope, "rows", rows) +} diff --git a/internal/api/export_test.go b/internal/api/export_test.go new file mode 100644 index 0000000..6e98493 --- /dev/null +++ b/internal/api/export_test.go @@ -0,0 +1,162 @@ +package api + +import ( + "bytes" + "context" + "encoding/csv" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + "time" + + "github.com/cdnutter/playlink/internal/engine" + "github.com/cdnutter/playlink/internal/probe" + "github.com/cdnutter/playlink/internal/store" +) + +func TestExportCSV(t *testing.T) { + ctx := context.Background() + root, _ := filepath.EvalSymlinks(t.TempDir()) + master, slave := filepath.Join(root, "Albums"), filepath.Join(root, "Singles") + write := func(base, rel, content string) { + p := filepath.Join(base, rel) + os.MkdirAll(filepath.Dir(p), 0o755) + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + // Enough rows to need more than one export page. + for i := 0; i < 1005; i++ { + rel := fmt.Sprintf("Bulk/Album/%04d.flac", i) + write(master, rel, "same") + write(slave, rel, "same") + } + write(master, "Beyoncé/Lemonade/01 Pray You Catch Me.flac", "a") + write(slave, "Beyoncé/Lemonade/01 Pray You Catch Me.flac", "b") + write(slave, "=HYPERLINK(evil)/song.flac", "x") // a folder name a spreadsheet would run + + st, err := store.Open(filepath.Join(root)) + if err != nil { + t.Fatal(err) + } + defer st.DB.Close() + eng := engine.New(st, probe.Prober{}, []string{root}, slog.New(slog.NewTextHandler(io.Discard, nil))) + p := store.Pair{Name: "Chris' Music", MasterRoot: master, SlaveRoot: slave} + if _, err := eng.NormalizePair(&p); err != nil { + t.Fatal(err) + } + if err := st.SavePair(ctx, &p); err != nil { + t.Fatal(err) + } + if _, err := eng.StartScan(p.ID, "test"); err != nil { + t.Fatal(err) + } + for deadline := time.Now().Add(20 * time.Second); ; { + if j := eng.Job(p.ID); j != nil && j.State != "running" { + break + } + if time.Now().After(deadline) { + t.Fatal("scan did not finish") + } + time.Sleep(20 * time.Millisecond) + } + + h := New(Config{}, st, eng, slog.New(slog.NewTextHandler(io.Discard, nil)), fstest.MapFS{}).Handler() + get := func(url string, auth bool) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, url, nil) + if auth { + req.AddCookie(&http.Cookie{Name: cookieName, Value: "tok"}) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec + } + url := fmt.Sprintf("/api/pairs/%d/export.csv", p.ID) + if rec := get(url, false); rec.Code != http.StatusUnauthorized { + t.Fatalf("export without a session must be refused, got %d", rec.Code) + } + if err := st.CreateFirstUser(ctx, "chris", "x"); err != nil { + t.Fatal(err) + } + u, _ := st.UserByName(ctx, "chris") + if err := st.CreateSession(ctx, hashToken("tok"), u.ID, time.Now().Add(time.Hour).Unix()); err != nil { + t.Fatal(err) + } + + rec := get(url, true) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/csv") { + t.Fatalf("content type %q", ct) + } + if cd := rec.Header().Get("Content-Disposition"); !strings.Contains(cd, `filename="playlink-Chris-Music-all-`) { + t.Fatalf("content disposition %q", cd) + } + body := rec.Body.Bytes() + if !bytes.HasPrefix(body, []byte{0xEF, 0xBB, 0xBF}) { + t.Fatal("missing UTF-8 byte order mark") + } + rows, err := csv.NewReader(bytes.NewReader(body[3:])).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(rows) != 1+1005+2 { + t.Fatalf("want header + 1007 rows (paging), got %d", len(rows)) + } + if strings.Join(rows[0], ",") != strings.Join(exportHeader, ",") { + t.Fatal("unexpected header") + } + col := map[string]int{} + for i, h := range rows[0] { + col[h] = i + } + seen := map[string][]string{} + for _, r := range rows[1:] { + seen[strings.TrimPrefix(r[col["slave_path"]], "'")] = r + } + if r := seen["Beyoncé/Lemonade/01 Pray You Catch Me.flac"]; r == nil || r[col["status"]] != "different" || r[col["recommendation"]] != "review" { + t.Fatalf("non-ASCII row missing or wrong: %v", r) + } + if r := seen["=HYPERLINK(evil)/song.flac"]; r == nil || r[col["slave_path"]] != "'=HYPERLINK(evil)/song.flac" { + t.Fatalf("a path starting with = must be neutralised, got %v", r) + } + if r := seen["Bulk/Album/0000.flac"]; r[col["recommendation"]] != "link (identical)" || r[col["match_type"]] != "same path" { + t.Fatalf("identical row: %v", r) + } + + // Filters narrow the export exactly like the Review list. + rec = get(url+"?status=different", true) + rows, _ = csv.NewReader(bytes.NewReader(rec.Body.Bytes()[3:])).ReadAll() + if len(rows) != 2 { + t.Fatalf("status filter: want header + 1 row, got %d", len(rows)) + } + rec = get(url+"?q=0100", true) + rows, _ = csv.NewReader(bytes.NewReader(rec.Body.Bytes()[3:])).ReadAll() + if len(rows) != 2 { + t.Fatalf("search filter: want header + 1 row, got %d", len(rows)) + } +} + +func TestCSVSafe(t *testing.T) { + for in, want := range map[string]string{ + "=SUM(A1)": "'=SUM(A1)", + "+1": "'+1", + "-x": "'-x", + "@cmd": "'@cmd", + "Artist/Song": "Artist/Song", + "": "", + "Beyoncé/Halo": "Beyoncé/Halo", + } { + if got := csvSafe(in); got != want { + t.Errorf("csvSafe(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index e3681bc..a62126f 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -83,6 +83,7 @@ func (s *Server) Handler() http.Handler { r.Get("/job", s.handleJob) r.Post("/job/cancel", s.handleCancelJob) r.Get("/matches", s.handleMatches) + r.Get("/export.csv", s.handleExport) r.Post("/link-identical", s.handleLinkIdentical) r.Get("/tree", s.handleTree) r.Post("/build", s.handleBuild) diff --git a/internal/store/models.go b/internal/store/models.go index 27350c5..53e77fb 100644 --- a/internal/store/models.go +++ b/internal/store/models.go @@ -314,7 +314,8 @@ type MatchFilter struct { Offset int } -func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Match, int, error) { +// matchWhere builds the WHERE clause shared by listing and export. +func matchWhere(pairID int64, f MatchFilter) (string, []any) { where := []string{"pair_id = ?"} args := []any{pairID} switch f.Status { @@ -334,7 +335,11 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat q := "%" + strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(f.Query) + "%" args = append(args, q, q) } - w := strings.Join(where, " AND ") + return strings.Join(where, " AND "), args +} + +func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Match, int, error) { + w, args := matchWhere(pairID, f) var total int if err := s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE `+w, args...).Scan(&total); err != nil { return nil, 0, err @@ -359,6 +364,45 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat return out, total, rows.Err() } +// ExportMatches calls fn for every match of the filter, ordered by slave +// path. Rows are read in pages so a slow download never holds the database +// connection (scans need it) for longer than one page. +func (s *Store) ExportMatches(ctx context.Context, pairID int64, f MatchFilter, fn func(Match) error) error { + const page = 1000 + w, args := matchWhere(pairID, f) + after := "" + for { + rows, err := s.DB.QueryContext(ctx, `SELECT `+matchCols+` FROM matches WHERE `+w+ + ` AND rel_slave > ? ORDER BY rel_slave LIMIT ?`, append(append([]any{}, args...), after, page)...) + if err != nil { + return err + } + batch := make([]Match, 0, page) + for rows.Next() { + m, err := scanMatch(rows) + if err != nil { + rows.Close() + return err + } + batch = append(batch, m) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, m := range batch { + if err := fn(m); err != nil { + return err + } + } + if len(batch) < page { + return nil + } + after = batch[len(batch)-1].RelSlave + } +} + // IdenticalIDs lists linkable identical matches (not ignored). func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error) { rows, err := s.DB.QueryContext(ctx, `SELECT id FROM matches WHERE pair_id = ? AND status = 'identical' AND ignored = 0 ORDER BY rel_slave`, pairID) diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 03dfe4f..c4a73d1 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1,6 +1,6 @@ import { useEffect, useState } from "react"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { AlertTriangle, ArrowRight, ArrowUpCircle, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; +import { AlertTriangle, ArrowRight, ArrowUpCircle, Download, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; import { ApiError, api, type Match, type Page, type Pair, type ProbeInfo } from "../api"; import { Alert, Badge, Button, Card, Empty, Modal, Spinner, cx, formatBytes, formatDuration, inputCls, statusTone } from "../ui"; import { useToast } from "../toast"; @@ -86,9 +86,12 @@ export function Review({ pair }: { pair: Pair }) { {count(f.id) !== undefined && {count(f.id)}} ))} -