From 206b2ab5db0b822ead09bb68c2946c42658ad94f Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 30 Sep 2026 19:44:03 -0700 Subject: [PATCH] Find and remove duplicate songs within the slave library A new Duplicates filter in Review finds songs filed more than once in the slave library, such as a single next to its album version, and recommends which copy to keep. Removed copies go to the quarantine and can be restored. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 + internal/api/server.go | 35 +++ internal/engine/dupes.go | 519 ++++++++++++++++++++++++++++++++++ internal/engine/dupes_test.go | 178 ++++++++++++ internal/engine/engine.go | 3 + internal/engine/ops.go | 12 +- internal/store/models.go | 3 +- web/src/api.ts | 2 +- web/src/pages/Activity.tsx | 5 + web/src/pages/Duplicates.tsx | 327 +++++++++++++++++++++ web/src/pages/PairPage.tsx | 1 + web/src/pages/Player.tsx | 14 +- web/src/pages/Review.tsx | 25 +- 13 files changed, 1123 insertions(+), 11 deletions(-) create mode 100644 internal/engine/dupes.go create mode 100644 internal/engine/dupes_test.go create mode 100644 web/src/pages/Duplicates.tsx diff --git a/README.md b/README.md index 8b7116e..e8fd3ad 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,16 @@ Each row has the match status and type, a recommendation, both paths, where link bit depth and duration for both sides, the quality verdict and any notes. The file opens cleanly in Excel, Numbers and LibreOffice (UTF-8, and cells a spreadsheet would treat as formulas are escaped). +**Duplicates within the slave library.** The Review tab's **Duplicates** filter is separate from master/slave matching: it looks +only inside the slave library for songs filed more than once, such as a single next to its album version, a compilation track, or one +song in two formats. **Find duplicates** groups songs in the same top-level folder by title, using the same rules as approximate matching +(track numbers, disc folders and release labels such as *Single Version* or *Remastered* are ignored; *Live*, *Remix*, *Acoustic* and +similar versions are kept apart). With ffprobe, copies whose artist tags disagree are kept apart too, and each copy shows its quality and +length, with a warning when lengths differ by more than 10 seconds. Each song has a play button to compare the copies. One copy is +recommended to keep (the album version, then one already linked to the master, then the better quality) and the others start selected. +**Remove** moves the selected copies into the quarantine after re-checking that each is unchanged and that another copy is still there; +one copy of every song is always kept, emptied folders are removed, dry run applies, and every removal can be restored from History. + **Listening in build mode.** Every song in the Build tab has a play button. A mini-player along the bottom of the page plays the folder in order, with seeking, previous/next, volume and space to pause. Songs are streamed from the master library as they are (nothing is converted), so they play when the browser supports the format: FLAC, MP3, AAC/M4A, Ogg/Opus and WAV in current browsers, diff --git a/internal/api/server.go b/internal/api/server.go index e6a7ca8..440bbc9 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -105,6 +105,9 @@ func (s *Server) Handler() http.Handler { r.Get("/empty-folders", s.handleEmptyFolders) r.Post("/empty-folders/scan", s.handleEmptyFolderScan) r.Post("/empty-folders/remove", s.handleRemoveEmptyFolders) + r.Get("/duplicates", s.handleDupes) + r.Post("/duplicates/scan", s.handleDupeScan) + r.Post("/duplicates/remove", s.handleRemoveDupes) r.Post("/build", s.handleBuild) r.Post("/remove", s.handleRemove) }) @@ -797,6 +800,38 @@ var audioTypes = map[string]string{ } // handleListen streams a song with range support, so the browser can seek. +func (s *Server) handleDupes(w http.ResponseWriter, r *http.Request) { + id, _ := idParam(r) + writeJSON(w, 200, map[string]any{"scan": s.eng.DupeResult(id), "dry_run": s.st.DryRun(r.Context())}) +} +func (s *Server) handleDupeScan(w http.ResponseWriter, r *http.Request) { + id, _ := idParam(r) + j, err := s.eng.StartDupeScan(id) + if err != nil { + opErr(w, err) + return + } + writeJSON(w, 202, j) +} +func (s *Server) handleRemoveDupes(w http.ResponseWriter, r *http.Request) { + id, _ := idParam(r) + var body struct { + Files []string `json:"files"` // must come from the latest search + } + if !readJSON(w, r, &body) { + return + } + if !s.preAction(w, r, id, "remove duplicate songs") { + return + } + j, err := s.eng.StartRemoveDupes(id, body.Files) + if err != nil { + opErr(w, err) + return + } + s.log.Info("removing duplicate songs", "pair", id, "files", len(body.Files)) + writeJSON(w, 202, j) +} func (s *Server) handleListen(w http.ResponseWriter, r *http.Request) { id, _ := idParam(r) q := r.URL.Query() diff --git a/internal/engine/dupes.go b/internal/engine/dupes.go new file mode 100644 index 0000000..e519f41 --- /dev/null +++ b/internal/engine/dupes.go @@ -0,0 +1,519 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "io/fs" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/cdnutter/playlink/internal/fsops" + "github.com/cdnutter/playlink/internal/probe" + "github.com/cdnutter/playlink/internal/store" +) + +// Duplicates are the same song filed more than once in the slave library, +// typically a single next to the album version ("Song - Single/Song.mp3" and +// "Album/03 Song.flac"), or one song in two formats. They are found by the +// same title rules as approximate matching (track numbers and release labels +// such as "Single Version" ignored, live/remix/acoustic versions kept apart) +// within one top-level folder. One copy of each group is recommended to keep; +// removing the others needs the user's confirmation and moves them into the +// quarantine, so they can be restored from History. + +// DupeFile is one copy of a duplicated song. +type DupeFile struct { + Rel string `json:"rel"` + Size int64 `json:"size"` + Release string `json:"release"` // album, single or compilation (from the folder name) + Linked bool `json:"linked"` // a hardlink of a master file: it takes no extra space + Probe *probe.Info `json:"probe,omitempty"` + Keep bool `json:"keep"` // the copy recommended to keep + + ino uint64 + mtimeNs int64 + tracks int // songs in its album folder +} + +// DupeGroup is one song found more than once. +type DupeGroup struct { + ID string `json:"id"` + Title string `json:"title"` + Files []DupeFile `json:"files"` + Notes []string `json:"notes,omitempty"` +} + +// DupeScan is the result of the latest duplicate search. +type DupeScan struct { + CreatedAt int64 `json:"created_at"` + Files int `json:"files"` // slave files checked + Groups []DupeGroup `json:"groups"` + Probed bool `json:"probed"` // codec details and artist tags were available +} + +// dupeKey groups songs: top-level folder (normally the artist), the cleaned +// title and its version words. +func dupeKey(rel string, ti titleInfo) string { + if ti.norm == "" { + return "" + } + vs := make([]string, 0, len(ti.versions)) + for v := range ti.versions { + vs = append(vs, v) + } + sort.Strings(vs) + return strings.ToLower(firstComponent(rel)) + "|" + ti.norm + "|" + strings.Join(vs, ",") +} + +func releaseOf(ti titleInfo) string { + switch { + case ti.single: + return "single" + case ti.compilation: + return "compilation" + } + return "album" +} + +// normArtist reduces an artist tag to letters and digits. +func normArtist(s string) string { + return strings.Join(words(s), "") +} + +// sameArtist treats "Artist" and "Artist feat. Someone" as one artist. +func sameArtist(a, b string) bool { + return a == b || strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + +// splitByArtist keeps files whose artist tags disagree apart, so a folder +// layout like Genre/Artist/Song doesn't group two artists' songs with the +// same title. Untagged files join the tagged ones only when there is no +// doubt which artist they belong to. +func splitByArtist(files []DupeFile) [][]DupeFile { + var clusters [][]DupeFile + var reps []string + var untagged []DupeFile + for _, f := range files { + a := "" + if f.Probe != nil && f.Probe.Error == "" { + a = normArtist(f.Probe.Artist) + } + if a == "" { + untagged = append(untagged, f) + continue + } + placed := false + for i, r := range reps { + if sameArtist(a, r) { + clusters[i] = append(clusters[i], f) + placed = true + break + } + } + if !placed { + reps = append(reps, a) + clusters = append(clusters, []DupeFile{f}) + } + } + switch len(clusters) { + case 0: + return [][]DupeFile{untagged} + case 1: + clusters[0] = append(clusters[0], untagged...) + } + return clusters +} + +// keepBetter reports whether a is the better copy to keep than b: an album +// release over a single or compilation, then one already linked to the +// master (it takes no extra space), then the better quality, then the fuller +// album, then the better format. +func keepBetter(a, b DupeFile) bool { + if ra, rb := a.Release == "album", b.Release == "album"; ra != rb { + return ra + } + if a.Release != b.Release { + return a.Release == "single" // a single over a compilation + } + if a.Linked != b.Linked { + return a.Linked + } + if a.Probe != nil && b.Probe != nil { + sa, sb := probe.Score(*a.Probe), probe.Score(*b.Probe) + if sa > 0 && sb > 0 && math.Abs(sa-sb) > 0.05*math.Max(sa, sb) { + return sa > sb + } + } + if a.tracks != b.tracks { + return a.tracks > b.tracks + } + if ra, rb := extRank(a.Rel), extRank(b.Rel); ra != rb { + return ra < rb + } + return a.Rel < b.Rel +} + +func minSec(sec float64) string { + s := int(sec + 0.5) + return fmt.Sprintf("%d:%02d", s/60, s%60) +} + +// findDupes groups the slave files that are the same song. linked holds the +// slave paths that are hardlinks of a master file; probeFn (optional) +// supplies codec details and tags for the files in a group, and only +// (optional) limits the search to those files. +func findDupes(ctx context.Context, files map[string]fsops.Info, linked, only map[string]bool, probeFn func(rel string) *probe.Info) ([]DupeGroup, error) { + byKey := map[string][]string{} + titles := map[string]titleInfo{} + tracks := map[string]int{} + for rel := range files { + tracks[albumKey(rel)]++ // album sizes always count every file + if only != nil && !only[rel] { + continue + } + ti := parseTitle(rel) + titles[rel] = ti + if k := dupeKey(rel, ti); k != "" { + byKey[k] = append(byKey[k], rel) + } + } + keys := make([]string, 0, len(byKey)) + for k, rels := range byKey { + if len(rels) > 1 { + keys = append(keys, k) + } + } + sort.Strings(keys) + + var groups []DupeGroup + for _, k := range keys { + if ctx.Err() != nil { + return nil, ctx.Err() + } + var copies []DupeFile + for _, rel := range byKey[k] { + in := files[rel] + f := DupeFile{Rel: rel, Size: in.Size, Release: releaseOf(titles[rel]), Linked: linked[rel], + ino: in.Ino, mtimeNs: in.MtimeNs, tracks: tracks[albumKey(rel)]} + if probeFn != nil { + f.Probe = probeFn(rel) + } + copies = append(copies, f) + } + for _, c := range splitByArtist(copies) { + if len(c) < 2 { + continue + } + sort.Slice(c, func(i, j int) bool { return keepBetter(c[i], c[j]) }) + c[0].Keep = true + g := DupeGroup{Files: c} + name := filepath.Base(c[0].Rel) + g.Title = strings.TrimSpace(trackNum.ReplaceAllString(strings.TrimSuffix(name, filepath.Ext(name)), "")) + if g.Title == "" { + g.Title = name + } + if top := firstComponent(c[0].Rel); top != "" { + g.Title = top + " – " + g.Title + } + lo, hi := math.Inf(1), 0.0 + for _, f := range c { + if f.Probe != nil && f.Probe.Duration > 0 { + lo, hi = math.Min(lo, f.Probe.Duration), math.Max(hi, f.Probe.Duration) + } + } + if hi-lo > 10 { + g.Notes = append(g.Notes, fmt.Sprintf("Lengths differ by %s: one may be a radio edit or a different version. Listen before removing.", minSec(hi-lo))) + } + same := true + for _, f := range c[1:] { + same = same && f.ino == c[0].ino + } + if same { + g.Notes = append(g.Notes, "All copies are the same file (hardlinks), so removing one frees no space, only the extra entry in your media server.") + } + groups = append(groups, g) + } + } + for i := range groups { + groups[i].ID = groups[i].Files[0].Rel + } + sort.Slice(groups, func(i, j int) bool { return strings.ToLower(groups[i].Title) < strings.ToLower(groups[j].Title) }) + if groups == nil { + groups = []DupeGroup{} + } + return groups, nil +} + +// StartDupeScan looks for songs that are in the slave library more than once. +func (e *Engine) StartDupeScan(pairID int64) (*Job, error) { + return e.startJob(pairID, "find-duplicates", func(ctx context.Context, j *Job) error { + l, err := e.loadPair(ctx, pairID) + if err != nil { + return err + } + j.phase("Listing the slave library", 0) + files := map[string]fsops.Info{} + if _, err := walk(ctx, l, l.slave, files); err != nil { + return err + } + linked := map[string]bool{} + if ms, err := e.st.AllMatches(ctx, pairID); err == nil { + for rel, m := range ms { + // Only trust the scan's verdict for the file it saw. + if in, ok := files[rel]; ok && m.Status == store.StatusLinked && m.SlaveIno == in.Ino { + linked[rel] = true + } + } + } + // Probe only files that share a title with another: a first pass + // without tags finds them. + cands, err := findDupes(ctx, files, linked, nil, nil) + if err != nil { + return err + } + var probeFn func(string) *probe.Info + if e.prober.Bin != "" { + n := 0 + for _, g := range cands { + n += len(g.Files) + } + j.phase("Reading tags and codec details", int64(n)) + probes := map[string]*probe.Info{} + for _, g := range cands { + for _, f := range g.Files { + if ctx.Err() != nil { + return ctx.Err() + } + p := e.prober.Probe(ctx, filepath.Join(l.slave.Path, f.Rel)) + probes[f.Rel] = &p + j.tick() + } + } + probeFn = func(rel string) *probe.Info { return probes[rel] } + only := map[string]bool{} + for rel := range probes { + only[rel] = true + } + // Again with tags: artists that differ are told apart. + if cands, err = findDupes(ctx, files, linked, only, probeFn); err != nil { + return err + } + } + e.dupeMu.Lock() + if e.dupes == nil { + e.dupes = map[int64]*DupeScan{} + } + e.dupes[pairID] = &DupeScan{CreatedAt: time.Now().Unix(), Files: len(files), Groups: cands, Probed: probeFn != nil} + e.dupeMu.Unlock() + extra := 0 + for _, g := range cands { + extra += len(g.Files) - 1 + } + j.update(func(j *Job) { + j.Message = fmt.Sprintf("%d songs are in the slave library more than once (%d extra copies)", len(cands), extra) + }) + return nil + }) +} + +// DupeResult returns the latest duplicate search. +func (e *Engine) DupeResult(pairID int64) *DupeScan { + e.dupeMu.Lock() + defer e.dupeMu.Unlock() + return e.dupes[pairID] +} + +// StartRemoveDupes moves the chosen copies of the latest duplicate search into +// the quarantine. Only files that search listed can be chosen, and at least +// one copy of every song must be kept. +func (e *Engine) StartRemoveDupes(pairID int64, rels []string) (*Job, error) { + e.dupeMu.Lock() + scan := e.dupes[pairID] + e.dupeMu.Unlock() + if scan == nil { + return nil, fmt.Errorf("%w: look for duplicates first", ErrInvalidBulk) + } + type entry struct { + f DupeFile + group int + } + listed := map[string]entry{} + for gi, g := range scan.Groups { + for _, f := range g.Files { + listed[f.Rel] = entry{f, gi} + } + } + chosen := map[string]bool{} + var order []entry + for _, r := range rels { + en, ok := listed[r] + if !ok { + return nil, fmt.Errorf("%w: %q is not in the latest search", ErrInvalidBulk, r) + } + if !chosen[r] { + chosen[r] = true + order = append(order, en) + } + } + if len(order) == 0 { + return nil, fmt.Errorf("%w: no files selected", ErrInvalidBulk) + } + keepers := map[int][]string{} + for gi, g := range scan.Groups { + for _, f := range g.Files { + if !chosen[f.Rel] { + keepers[gi] = append(keepers[gi], f.Rel) + } + } + } + for _, en := range order { + if len(keepers[en.group]) == 0 { + return nil, fmt.Errorf("%w: every copy of %q is selected; keep at least one", ErrInvalidBulk, scan.Groups[en.group].Title) + } + } + return e.startJob(pairID, "remove-duplicates", func(ctx context.Context, j *Job) error { + l, err := e.loadPair(ctx, pairID) + if err != nil { + return err + } + j.phase("Moving duplicates to the quarantine", int64(len(order))) + removed := map[string]bool{} + for _, en := range order { + if ctx.Err() != nil { + return ctx.Err() + } + if err := e.removeDupe(ctx, l, en.f, keepers[en.group]); err != nil { + j.skip(en.f.Rel + ": " + err.Error()) + } else if e.st.DryRun(ctx) { + j.count("planned") + } else { + j.count("removed") + removed[en.f.Rel] = true + } + j.tick() + } + e.dropFromDupeScan(pairID, scan, removed) + return nil + }) +} + +// dropFromDupeScan takes removed files out of the search result so the list +// stays current without searching again. +func (e *Engine) dropFromDupeScan(pairID int64, scan *DupeScan, removed map[string]bool) { + if len(removed) == 0 { + return + } + next := &DupeScan{CreatedAt: scan.CreatedAt, Files: scan.Files - len(removed), Probed: scan.Probed, Groups: []DupeGroup{}} + for _, g := range scan.Groups { + var left []DupeFile + for _, f := range g.Files { + if !removed[f.Rel] { + left = append(left, f) + } + } + if len(left) > 1 { + g.Files = left + next.Groups = append(next.Groups, g) + } + } + e.dupeMu.Lock() + if e.dupes[pairID] == scan { + e.dupes[pairID] = next + } + e.dupeMu.Unlock() +} + +// removeDupe moves one copy of a song into the quarantine, after checking it +// is still the file that was found and that a copy being kept still exists. +func (e *Engine) removeDupe(ctx context.Context, l *libs, f DupeFile, keepers []string) error { + e.mu.Lock() + defer e.mu.Unlock() + abs, err := l.slave.Resolve(f.Rel) + if err != nil { + return err + } + si, err := fsops.RegularFile(abs) + if err != nil { + return err + } + if si.Ino != f.ino || !si.Unchanged(f.Size, f.mtimeNs) { + return fsops.ErrChanged + } + kept := false + for _, k := range keepers { + if p, err := l.slave.Resolve(k); err == nil { + if _, err := fsops.RegularFile(p); err == nil { + kept = true + break + } + } + } + if !kept { + return errors.New("no other copy of this song is left in the slave library; skipped") + } + o := store.Operation{PairID: l.pair.ID, Action: store.ActRemoveDupe, RelSlave: f.Rel, QuarantineSide: SideSlave, Trigger: "manual"} + // Remember the master pairing so an undo restores it. + if m, err := e.st.MatchBySlave(ctx, l.pair.ID, f.Rel); err == nil && m.Kind != store.KindNone { + o.RelMaster, o.MatchKind, o.MatchReason = m.RelMaster, m.Kind, m.Reason + } + if err := e.begin(ctx, &o); err != nil { + return err + } + if o.DryRun { + return nil + } + q := quarantinePath(l, o.ID, SideSlave, f.Rel) + err = func() error { + // Recorded first, so the file is always findable from History. + if _, err := e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = ? WHERE id = ?`, q, o.ID); err != nil { + return err + } + if err := quarantine(abs, si, q); err != nil { + return err + } + if err := removeOriginal(abs, si, q); err != nil { + if stillQuarantined(abs, q) { + dropQuarantine(l, q, si) + } + return err + } + return nil + }() + if err = e.finish(ctx, &o, err); err != nil { + return err + } + pruneEmpty(filepath.Dir(abs), l.slave.Path) + e.refresh(ctx, l, f.Rel) + return nil +} + +// undoRemoveDupe puts a removed duplicate back from the quarantine. +func (e *Engine) undoRemoveDupe(ctx context.Context, l *libs, o store.Operation) error { + dst, err := l.slave.Resolve(o.RelSlave) + if err != nil { + return err + } + qi, err := e.quarantined(l, o.QuarantinePath) + if err != nil { + return err + } + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("%q exists again: %w", o.RelSlave, fsops.ErrExists) + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + if err := fsops.MkdirAllLike(filepath.Dir(dst)); err != nil { + return err + } + if err := fsops.LinkNew(o.QuarantinePath, qi, dst); err != nil { + return err + } + dropQuarantine(l, o.QuarantinePath, qi) + e.refresh(ctx, l, o.RelSlave, store.Match{RelMaster: o.RelMaster, Kind: o.MatchKind, Reason: o.MatchReason}) + return nil +} diff --git a/internal/engine/dupes_test.go b/internal/engine/dupes_test.go new file mode 100644 index 0000000..676897d --- /dev/null +++ b/internal/engine/dupes_test.go @@ -0,0 +1,178 @@ +package engine + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cdnutter/playlink/internal/fsops" + "github.com/cdnutter/playlink/internal/probe" + "github.com/cdnutter/playlink/internal/store" +) + +func (ev *env) dupes(t *testing.T) *DupeScan { + t.Helper() + if _, err := ev.e.StartDupeScan(ev.pair.ID); err != nil { + t.Fatal(err) + } + ev.wait() + s := ev.e.DupeResult(ev.pair.ID) + if s == nil { + t.Fatal("no search result") + } + return s +} + +func groupRels(g DupeGroup) string { + var out []string + for _, f := range g.Files { + out = append(out, f.Rel) + } + return strings.Join(out, ",") +} + +func TestFindDuplicates(t *testing.T) { + ev := setup(t, false) + s := ev.slave + // A single next to the album version: the album copy is kept. + ev.write(s, "A7X/Hail to the King - Single/Hail to the King.mp3", "single") + ev.write(s, "A7X/Hail to the King/01 Hail to the King.flac", "album") + ev.write(s, "A7X/Hail to the King/02 Shepherd of Fire.flac", "album") + // Release labels and disc folders don't matter; a live version is a different song. + ev.write(s, "A7X/City of Evil/Disc 1/03 Bat Country.flac", "album") + ev.write(s, "A7X/Greatest Hits/07 Bat Country (Remastered).mp3", "comp") + ev.write(s, "A7X/Live in the LBC/05 Bat Country.flac", "live") + ev.write(s, "A7X/Singles/Nightmare (Live).mp3", "live single") + ev.write(s, "A7X/Nightmare/01 Nightmare.flac", "album") + // The same title by another artist is not a duplicate. + ev.write(s, "Other/Album/Nightmare.flac", "other") + // One song in two formats in one folder. + ev.write(s, "Band/Album/Song.mp3", "mp3") + ev.write(s, "Band/Album/Song.flac", "flac") + + scan := ev.dupes(t) + var got []string + for _, g := range scan.Groups { + got = append(got, groupRels(g)) + if !g.Files[0].Keep || g.Files[1].Keep { + t.Errorf("%s: only the first copy should be kept", g.Title) + } + } + want := []string{ + "A7X/City of Evil/Disc 1/03 Bat Country.flac,A7X/Greatest Hits/07 Bat Country (Remastered).mp3", + "A7X/Hail to the King/01 Hail to the King.flac,A7X/Hail to the King - Single/Hail to the King.mp3", + "Band/Album/Song.flac,Band/Album/Song.mp3", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + if scan.Groups[1].Files[1].Release != "single" || scan.Groups[0].Files[1].Release != "compilation" { + t.Fatalf("releases: %+v", scan.Groups) + } + if scan.Groups[1].Title != "A7X – Hail to the King" { + t.Fatalf("title %q", scan.Groups[1].Title) + } +} + +func TestDuplicatesSplitByArtistTag(t *testing.T) { + files := map[string]fsops.Info{ + "Rock/Song.mp3": {Ino: 1}, + "Rock/Album/Song.mp3": {Ino: 2}, + "Rock/Other/Song.mp3": {Ino: 3}, + } + artist := map[string]string{"Rock/Song.mp3": "Band", "Rock/Album/Song.mp3": "Band feat. Guest", "Rock/Other/Song.mp3": "Someone Else"} + probeFn := func(rel string) *probe.Info { return &probe.Info{Codec: "mp3", Artist: artist[rel]} } + groups, err := findDupes(ctx, files, nil, nil, probeFn) + if err != nil { + t.Fatal(err) + } + if len(groups) != 1 || len(groups[0].Files) != 2 || strings.Contains(groupRels(groups[0]), "Other") { + t.Fatalf("got %+v", groups) + } +} + +func TestRemoveDuplicates(t *testing.T) { + ev := setup(t, false) + s := ev.slave + ev.write(ev.master, "A/Album/01 Song.flac", "album") + keep := ev.write(s, "A/Album/01 Song.flac", "album") + single := ev.write(s, "A/Song - Single/Song.mp3", "single") + ev.write(s, "B/X/Tune.mp3", "1") + ev.write(s, "B/Y/Tune.mp3", "2") + ev.scan() + scan := ev.dupes(t) + if len(scan.Groups) != 2 { + t.Fatalf("groups %+v", scan.Groups) + } + + if _, err := ev.e.StartRemoveDupes(ev.pair.ID, []string{"A/Album/02 Nope.flac"}); err == nil { + t.Fatal("a file that wasn't listed must be refused") + } + if _, err := ev.e.StartRemoveDupes(ev.pair.ID, []string{"B/X/Tune.mp3", "B/Y/Tune.mp3"}); err == nil { + t.Fatal("removing every copy must be refused") + } + // A file changed since the search is skipped. + ev.write(s, "B/Y/Tune.mp3", "changed!") + if _, err := ev.e.StartRemoveDupes(ev.pair.ID, []string{"A/Song - Single/Song.mp3", "B/Y/Tune.mp3"}); err != nil { + t.Fatal(err) + } + j := ev.wait() + if j.Results["removed"] != 1 || len(j.Skipped) != 1 || !strings.Contains(j.Skipped[0], "changed") { + t.Fatalf("results %+v skipped %v", j.Results, j.Skipped) + } + if exists(single) || exists(filepath.Dir(single)) || !exists(keep) { + t.Fatal("the single (and its emptied folder) should be gone, the album copy kept") + } + if g := ev.e.DupeResult(ev.pair.ID).Groups; len(g) != 1 || !strings.HasPrefix(g[0].Files[0].Rel, "B/") { + t.Fatalf("the removed song should leave the list: %+v", g) + } + if _, err := ev.st.MatchBySlave(ctx, ev.pair.ID, "A/Song - Single/Song.mp3"); err == nil { + t.Fatal("the scan result for the removed file should be gone") + } + + ops, _, _ := ev.st.Ops(ctx, store.OpFilter{PairID: ev.pair.ID}) + var op store.Operation + for _, o := range ops { + if o.Action == store.ActRemoveDupe { + op = o + } + } + if op.ID == 0 || ev.read(op.QuarantinePath) != "single" { + t.Fatalf("the single should be in the quarantine: %+v", op) + } + if err := ev.e.Undo(ctx, op.ID); err != nil { + t.Fatal(err) + } + if ev.read(single) != "single" || exists(op.QuarantinePath) { + t.Fatal("undo should move the file back") + } + if _, err := ev.st.MatchBySlave(ctx, ev.pair.ID, "A/Song - Single/Song.mp3"); err != nil { + t.Fatal("undo should restore the scan result") + } +} + +func TestRemoveDuplicatesKeepsACopy(t *testing.T) { + ev := setup(t, false) + ev.write(ev.slave, "A/X/Song.mp3", "1") + ev.write(ev.slave, "A/Y/Song.mp3", "2") + ev.dupes(t) + // The copy being kept disappeared after the search. + os.Remove(filepath.Join(ev.slave, "A/X/Song.mp3")) + ev.e.StartRemoveDupes(ev.pair.ID, []string{"A/Y/Song.mp3"}) + j := ev.wait() + if j.Results["removed"] != 0 || !exists(filepath.Join(ev.slave, "A/Y/Song.mp3")) { + t.Fatalf("the last copy must never be removed: %+v %v", j.Results, j.Skipped) + } +} + +func TestRemoveDuplicatesDryRun(t *testing.T) { + ev := setup(t, true) + ev.write(ev.slave, "A/X/Song.mp3", "1") + p := ev.write(ev.slave, "A/Y/Song.mp3", "2") + ev.dupes(t) + ev.e.StartRemoveDupes(ev.pair.ID, []string{"A/Y/Song.mp3"}) + if j := ev.wait(); j.Results["planned"] != 1 || !exists(p) { + t.Fatalf("dry run must only record: %+v", j.Results) + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 3c8e432..c95f7b6 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -46,6 +46,9 @@ type Engine struct { emptyMu sync.Mutex empty map[int64]*NoMusicScan // latest scan for folders without music, per pair + dupeMu sync.Mutex + dupes map[int64]*DupeScan // latest duplicate search, per pair + jobsMu sync.Mutex jobs map[int64]*Job nextID int64 diff --git a/internal/engine/ops.go b/internal/engine/ops.go index cc76f4e..19273b3 100644 --- a/internal/engine/ops.go +++ b/internal/engine/ops.go @@ -622,7 +622,7 @@ func (e *Engine) Undo(ctx context.Context, opID int64) error { return errors.New("the quarantined original was already purged") } switch o.Action { - case store.ActLink, store.ActPromote, store.ActBuild, store.ActRemove, store.ActRename, store.ActRemoveFolder: + case store.ActLink, store.ActPromote, store.ActBuild, store.ActRemove, store.ActRename, store.ActRemoveFolder, store.ActRemoveDupe: default: // History can hold operations of features that were removed (tag // edits, file moves); undoing them as a link would be wrong. @@ -643,6 +643,16 @@ func (e *Engine) Undo(ctx context.Context, opID int64) error { pruneEmpty(filepath.Dir(o.QuarantinePath), l.quarantine) return nil } + if o.Action == store.ActRemoveDupe { + if err := e.undoRemoveDupe(ctx, l, o); err != nil { + return err + } + if err := e.st.MarkUndone(ctx, o.ID); err != nil { + return err + } + e.log.Info("duplicate restored from quarantine", "op", o.ID, "slave", o.RelSlave) + return nil + } if o.Action == store.ActRename { if err := undoRename(l, o); err != nil { return err diff --git a/internal/store/models.go b/internal/store/models.go index dfc82ec..36700dc 100644 --- a/internal/store/models.go +++ b/internal/store/models.go @@ -654,7 +654,8 @@ const ( ActRemove = "remove" // slave hardlink removed (master copy kept) ActRename = "rename" // slave folder renamed to the master's spelling - ActRemoveFolder = "remove-folder" // folder without music moved to the quarantine + ActRemoveFolder = "remove-folder" // folder without music moved to the quarantine + ActRemoveDupe = "remove-duplicate" // extra copy of a song in the slave moved to the quarantine ) // Operation statuses. diff --git a/web/src/api.ts b/web/src/api.ts index 1bd757b..a549a15 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -139,7 +139,7 @@ export type Match = { export type Operation = { id: number; pair_id: number; - action: "link" | "promote" | "build" | "remove" | "rename" | "remove-folder"; + action: "link" | "promote" | "build" | "remove" | "rename" | "remove-folder" | "remove-duplicate"; rel_master: string; rel_slave: string; rel_new: string; diff --git a/web/src/pages/Activity.tsx b/web/src/pages/Activity.tsx index 23074a7..33abf20 100644 --- a/web/src/pages/Activity.tsx +++ b/web/src/pages/Activity.tsx @@ -6,6 +6,7 @@ import { ChevronLeft, ChevronRight, CircleAlert, + CopyMinus, Crown, FolderPen, FolderPlus, @@ -65,6 +66,7 @@ const actionLabel: Record = { remove: "Removed from slave", rename: "Renamed slave folder", "remove-folder": "Removed folder without music", + "remove-duplicate": "Removed duplicate song", }; const actionIcon: Record = { @@ -74,6 +76,7 @@ const actionIcon: Record = { remove: Unlink, rename: FolderPen, "remove-folder": FolderX, + "remove-duplicate": CopyMinus, }; // Only exceptions get a badge: "done" is the normal case. @@ -306,5 +309,7 @@ function undoText(o: Operation) { return `The slave folder is renamed back from "${o.rel_new}" to "${o.rel_slave}", and the library is rescanned.`; case "remove-folder": return `The folder "${o.rel_slave || o.rel_master}" is moved back from the quarantine into the ${o.quarantine_side} library, with everything that was in it.`; + case "remove-duplicate": + return `The song "${o.rel_slave}" is moved back from the quarantine into the slave library.`; } } diff --git a/web/src/pages/Duplicates.tsx b/web/src/pages/Duplicates.tsx new file mode 100644 index 0000000..8f0911f --- /dev/null +++ b/web/src/pages/Duplicates.tsx @@ -0,0 +1,327 @@ +import { useEffect, useMemo, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { AlertTriangle, AudioLines, CopyMinus, Link2, Play, Search, Star } from "lucide-react"; +import { api, type Pair, type ProbeInfo } from "../api"; +import { Alert, Badge, Button, Card, Modal, cx, formatAgo, formatBytes, formatDuration, inputCls } from "../ui"; +import { useToast } from "../toast"; +import { JobBar } from "./PairPage"; +import { MiniPlayer, type Track } from "./Player"; + +type DupeFile = { rel: string; size: number; release: "album" | "single" | "compilation"; linked: boolean; probe?: ProbeInfo; keep: boolean }; +type DupeGroup = { id: string; title: string; files: DupeFile[]; notes?: string[] | null }; +type DupeScan = { created_at: number; files: number; groups: DupeGroup[]; probed: boolean }; + +const SHOW = 100; + +function quality(p?: ProbeInfo) { + if (!p || p.error) return null; + if (p.lossless) return `${p.codec.toUpperCase()} ${p.bit_depth || "?"}/${(p.sample_rate / 1000).toFixed(1)}`; + return `${p.codec.toUpperCase()} ${Math.round(p.bitrate / 1000)}k`; +} + +function split(rel: string) { + const i = rel.lastIndexOf("/"); + return i >= 0 ? [rel.slice(0, i + 1), rel.slice(i + 1)] : ["", rel]; +} + +/** + * Songs that are in the slave library more than once, such as a single next + * to the album version. Separate from master/slave matching: this only looks + * inside the slave library. + */ +export function Duplicates({ pair }: { pair: Pair }) { + const qc = useQueryClient(); + const toast = useToast(); + const [sel, setSel] = useState>(new Set()); + const [q, setQ] = useState(""); + const [show, setShow] = useState(SHOW); + const [confirm, setConfirm] = useState(false); + const [player, setPlayer] = useState<{ queue: Track[]; index: number } | null>(null); + const nowPlaying = player ? player.queue[player.index].rel : null; + + const data = useQuery({ + queryKey: ["duplicates", pair.id], + queryFn: () => api.get<{ scan: DupeScan | null; dry_run: boolean }>(`/pairs/${pair.id}/duplicates`), + }); + const scan = data.data?.scan ?? null; + const running = pair.job?.state === "running"; + const ourJob = running && pair.job!.kind.includes("duplicates"); + const refresh = () => { + qc.invalidateQueries({ queryKey: ["pair", pair.id] }); + qc.invalidateQueries({ queryKey: ["duplicates", pair.id] }); + }; + // Every copy except the recommended one starts selected. + useEffect(() => { + setSel(new Set((scan?.groups ?? []).flatMap((g) => g.files.filter((f) => !f.keep).map((f) => f.rel)))); + }, [scan?.created_at]); // eslint-disable-line react-hooks/exhaustive-deps + // A removal shrinks the list: drop selections of files that are gone. + useEffect(() => { + if (!scan) return; + const listed = new Set(scan.groups.flatMap((g) => g.files.map((f) => f.rel))); + setSel((s) => new Set([...s].filter((r) => listed.has(r)))); + }, [scan]); + + const find = useMutation({ + mutationFn: () => api.post(`/pairs/${pair.id}/duplicates/scan`), + onSuccess: refresh, + onError: (e) => toast("err", e.message), + }); + + const groups = scan?.groups ?? []; + const needle = q.trim().toLowerCase(); + const shown = useMemo( + () => (needle ? groups.filter((g) => g.title.toLowerCase().includes(needle) || g.files.some((f) => f.rel.toLowerCase().includes(needle))) : groups), + [groups, needle], + ); + const chosen = groups.flatMap((g) => g.files.filter((f) => sel.has(f.rel))); + const extra = groups.reduce((n, g) => n + g.files.length - 1, 0); + + const toggle = (g: DupeGroup, rel: string) => + setSel((s) => { + const n = new Set(s); + if (n.has(rel)) n.delete(rel); + else if (g.files.filter((f) => f.rel !== rel).some((f) => !n.has(f.rel))) n.add(rel); // always keep one copy + return n; + }); + const play = (g: DupeGroup, rel: string) => { + const queue = g.files.map((f) => ({ rel: f.rel, name: split(f.rel)[1], side: "slave" as const })); + setPlayer({ queue, index: queue.findIndex((t) => t.rel === rel) }); + }; + + return ( +
+ +
+
+

+ Duplicate songs in the slave library +

+

+ Finds songs that are in the slave library more than once, such as a single next to its album version, a compilation track, or one + song in two formats. This only looks inside the slave library and is separate from matching against the master. Live, remix, + acoustic and other versions are kept apart. Removed copies go to the quarantine and can be restored from History. +

+
+ +
+ {ourJob && } + + + {scan && !ourJob && ( + +
+
+ {groups.length === 0 ? ( + "No duplicates" + ) : ( + <> + {groups.length} {groups.length === 1 ? "song is" : "songs are"} in the slave library more than + once ({extra} extra {extra === 1 ? "copy" : "copies"}) + + )} + + {" "} + · {scan.files} songs checked {formatAgo(scan.created_at)} + +
+ {groups.length > 0 && ( +
+
+ + setQ(e.target.value)} /> +
+ +
+ )} +
+ {!scan.probed && groups.length > 0 && ( +
+ ffprobe isn't available, so artist tags, lengths and quality weren't compared. Check each group before removing. +
+ )} + {groups.length === 0 ? ( +
Every song is in the slave library once.
+ ) : ( + <> +
+ marks the copy recommended to keep: the album version first, then one + already linked to the master, then the better quality. The others start selected; one copy of every song is always kept. +
+
+ {shown.slice(0, show).map((g) => ( +
+
+ {g.title} + {g.files.length} copies +
+ {g.notes?.map((n) => ( +
+ {n} +
+ ))} +
+ {g.files.map((f) => { + const [dir, base] = split(f.rel); + const on = sel.has(f.rel); + const lastKept = !on && g.files.every((o) => o.rel === f.rel || sel.has(o.rel)); + const playing = nowPlaying === f.rel; + return ( +
+ toggle(g, f.rel)} + /> + +
+ {dir} + {base} +
+
+ {f.keep && ( + + keep + + )} + {f.release !== "album" && {f.release}} + {f.linked && ( + + linked + + )} +
+
{quality(f.probe) ?? ""}
+
+ {f.probe?.duration ? formatDuration(f.probe.duration) : ""} +
+
{formatBytes(f.size)}
+
+ ); + })} +
+
+ ))} +
+ {shown.length === 0 &&
No duplicates match this filter.
} + {shown.length > show && ( +
+ +
+ )} + + )} + + )} + + {player && ( + setPlayer({ ...player, index })} onClose={() => setPlayer(null)} /> + )} + + {confirm && scan && ( + setConfirm(false)} + onDone={() => { + setConfirm(false); + setPlayer(null); + refresh(); + }} + /> + )} +
+ ); +} + +function ConfirmRemove({ + pair, + groups, + files, + dry, + onClose, + onDone, +}: { + pair: Pair; + groups: DupeGroup[]; + files: DupeFile[]; + dry: boolean; + onClose: () => void; + onDone: () => void; +}) { + const toast = useToast(); + const chosen = new Set(files.map((f) => f.rel)); + // Groups where the recommended copy is the one being removed, or whose lengths differ. + const overridden = groups.filter((g) => g.files.some((f) => f.keep && chosen.has(f.rel))).length; + const lengths = groups.filter((g) => g.files.some((f) => chosen.has(f.rel)) && (g.notes ?? []).some((n) => n.startsWith("Lengths"))).length; + const bytes = files.filter((f) => !f.linked).reduce((n, f) => n + f.size, 0); + const remove = useMutation({ + mutationFn: () => api.post(`/pairs/${pair.id}/duplicates/remove`, { files: files.map((f) => f.rel) }), + onSuccess: onDone, + onError: (e) => toast("err", e.message), + }); + const n = files.length; + return ( + +
+

+ These files are moved from the slave library into the quarantine. Each is checked right before it moves: it's skipped if it changed since + the search or if no other copy of the song is left. Emptied folders are removed. You can restore them from History + {pair.retention_days > 0 ? `; they are deleted for good after ${pair.retention_days} days` : "; they stay in the quarantine until you purge them"} + {bytes > 0 && <>, which frees up to {formatBytes(bytes)}}. +

+
+ {files.map((f) => ( +
+ {f.rel} +
+ ))} +
+ {(overridden > 0 || lengths > 0) && ( + }> + {overridden > 0 && ( +
+ In {overridden} {overridden === 1 ? "song" : "songs"}, you're removing the copy recommended to keep. +
+ )} + {lengths > 0 && ( +
+ {lengths} {lengths === 1 ? "song has" : "songs have"} copies of different lengths, which may be different edits. +
+ )} + + )} +

Playlists that point at the removed files will need updating.

+ {dry && Dry run is on: the removals are only recorded in Activity, and no file moves.} +
+ + +
+
+ + ); +} diff --git a/web/src/pages/PairPage.tsx b/web/src/pages/PairPage.tsx index b60cdef..4d09836 100644 --- a/web/src/pages/PairPage.tsx +++ b/web/src/pages/PairPage.tsx @@ -40,6 +40,7 @@ function useJobFinished(pair: Pair | undefined) { qc.invalidateQueries({ queryKey: ["ops"] }); qc.invalidateQueries({ queryKey: ["pairs"] }); qc.invalidateQueries({ queryKey: ["empty-folders"] }); + qc.invalidateQueries({ queryKey: ["duplicates"] }); const j = pair.job!; if (st === "done") toast("ok", `${j.kind} finished${j.message ? `: ${j.message}` : ""}`); else toast("err", `${j.kind} ${st}: ${j.error}`); diff --git a/web/src/pages/Player.tsx b/web/src/pages/Player.tsx index 4b8d4b8..6eb7ecf 100644 --- a/web/src/pages/Player.tsx +++ b/web/src/pages/Player.tsx @@ -2,11 +2,11 @@ import { useEffect, useRef, useState } from "react"; import { CircleAlert, Disc3, Pause, Play, SkipBack, SkipForward, Volume1, Volume2, VolumeX, X } from "lucide-react"; import { cx } from "../ui"; -/** A song the player can play: its path in the master library and a display name. */ -export type Track = { rel: string; name: string }; +/** A song the player can play: its path in a library (the master unless side says otherwise) and a display name. */ +export type Track = { rel: string; name: string; side?: "master" | "slave" }; -export function listenURL(pairId: number, rel: string) { - return `/api/pairs/${pairId}/listen?side=master&path=${encodeURIComponent(rel)}`; +export function listenURL(pairId: number, rel: string, side: "master" | "slave" = "master") { + return `/api/pairs/${pairId}/listen?side=${side}&path=${encodeURIComponent(rel)}`; } function clock(sec: number) { @@ -59,9 +59,9 @@ export function MiniPlayer({ setError(null); setTime(0); setDuration(0); - a.src = listenURL(pairId, track.rel); + a.src = listenURL(pairId, track.rel, track.side); a.play().catch(() => {}); // a refused autoplay just leaves it paused - }, [pairId, track.rel]); + }, [pairId, track.rel, track.side]); useEffect(() => { if (audio.current) audio.current.volume = volume; @@ -126,7 +126,7 @@ export function MiniPlayer({ ) : ( <> - {folder || "master"} · {index + 1} of {queue.length} + {folder || track.side || "master"} · {index + 1} of {queue.length} )} diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 1d1312e..7dd0913 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1,10 +1,11 @@ import { useEffect, useState } from "react"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { AlertTriangle, ArrowRight, ArrowUpCircle, Disc3, Download, ListMusic, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; +import { AlertTriangle, ArrowRight, ArrowUpCircle, Disc3, Download, ListMusic, ChevronDown, ChevronRight, CopyCheck, CopyMinus, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; import { ApiError, api, type Match, type Page, type Pair, type ProbeInfo } from "../api"; import { Alert, Badge, Button, Card, Empty, Modal, Spinner, cx, formatBytes, formatDuration, inputCls, statusTone } from "../ui"; import { useToast } from "../toast"; import { AlbumView, BulkBar, TriCheckbox, toggleIDs, type Selection } from "./Bulk"; +import { Duplicates } from "./Duplicates"; const filters = [ { id: "review", label: "Needs review" }, @@ -72,6 +73,7 @@ export function Review({ pair }: { pair: Pair }) { queryFn: () => api.get>(`/pairs/${pair.id}/matches?status=${status}&q=${encodeURIComponent(debounced)}&limit=${PAGE}&offset=${page * PAGE}`), placeholderData: keepPreviousData, + enabled: status !== "duplicates", }); const bulk = useMutation({ @@ -105,6 +107,19 @@ export function Review({ pair }: { pair: Pair }) { {count(f.id) !== undefined && {count(f.id)}} ))} + {/* Not a match status: duplicates within the slave library. */} + + + {status !== "duplicates" && (
@@ -126,8 +141,14 @@ export function Review({ pair }: { pair: Pair }) {
f.id === status)?.label ?? "All"} />
+ )} + {status === "duplicates" ? ( + + ) : ( + <> + {status === "identical" && identical > 0 && (
@@ -191,6 +212,8 @@ export function Review({ pair }: { pair: Pair }) { )} setSel(new Set())} /> + + )} setConfirmBulk(false)} title="Link all identical files?">