From 381b804742cf4e2435d9329bcadafb00056876d4 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 22 Sep 2026 16:58:53 -0700 Subject: [PATCH] Add opt-in approximate matching and upgrade recommendations A per-pair "Approximate matching" setting adds a second pass for slave files the strict rules could not place. It searches the same album with disc folders (Disc 02, CD2, ...) ignored, then the top-level folder, and scores titles by edit distance and word containment against a configurable threshold (default 85%), with a small track-number bonus. Guards against false matches: version words (live, remix, reprise, instrumental, ...) must agree, near-tied candidates are left unmatched, and approximate matches always go to manual review, never auto-link. Each match records a human-readable reason. Review now flags matches where the master copy is better as upgrades, with an Upgrades filter and count and an "Upgrade to master copy" action. The redundant "master is lossless" warning note is dropped. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +- internal/api/server.go | 3 + internal/engine/approx.go | 257 +++++++++++++++++++++++++++++++++ internal/engine/approx_test.go | 126 ++++++++++++++++ internal/engine/engine.go | 6 + internal/engine/scan.go | 45 ++++-- internal/probe/probe.go | 5 +- internal/store/models.go | 58 +++++--- internal/store/store.go | 5 + web/src/api.ts | 8 +- web/src/pages/PairForm.tsx | 35 +++++ web/src/pages/Review.tsx | 42 ++++-- 12 files changed, 552 insertions(+), 51 deletions(-) create mode 100644 internal/engine/approx.go create mode 100644 internal/engine/approx_test.go diff --git a/README.md b/README.md index 73f4934..1524008 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,21 @@ path on both sides. The fallbacks below catch small differences, and a scan warn | Same title, **different format** (`Song.mp3` vs `Song.flac`) | Status `similar` (other format). Manual review. | | Same title after ignoring track numbers/punctuation (`01 - Song!.flac` vs `Song.mp3`) | Status `similar` (similar title). Manual review. | | Same top-level folder and title, but a **different subfolder** | Status `similar` (same top folder, other subfolder). Manual review. | +| *(opt-in)* **Approximately** the same title, disc folders ignored (`Cats/Disc 02/07 Memory.flac` vs `Cats/07 Memory.mp3`) | Status `similar` (approximate match), with the reason shown. Manual review. | | Only exists in the slave library | Status `unmatched`. Never touched. | | Already the same inode | Status `linked`. | +**Approximate matching** is an opt-in setting on each pair. For slave files the strict rules could not place, it searches the same +album with disc folders (`Disc 02`, `CD2`, …) ignored, then the whole top-level folder, and compares titles by similarity against a +threshold you choose (85% by default; a matching track number adds a small bonus). Several guards keep it from linking the wrong song: +version words such as *Live*, *Remix*, *Reprise* or *Instrumental* must agree on both sides, a file is left unmatched when two +candidates are nearly tied, and approximate matches never auto-link. + **Manual review** shows both files side by side and ranks their quality: lossless beats lossy, then bit depth and sample rate, then codec-weighted bitrate (Opus and AAC count for more per kbps than MP3). You then have three choices: -- **Link to master copy.** The default. The master is kept because it belongs to a complete album. If the master is *lower* quality, playlink +- **Upgrade / link to master copy.** The default. When the master copy is better, the match is flagged as an **upgrade** and listed + under the *Upgrades* filter. The master is kept because it belongs to a complete album. If the master is *lower* quality, playlink warns you and asks for explicit confirmation. - **Use slave copy as master.** The override. The slave's better file replaces the master file, the old master file is quarantined, and both libraries share the better file. @@ -108,7 +116,8 @@ Run a scan, check the Review tab, and turn off dry run once the planned actions | `PLAYLINK_FFPROBE` | `ffprobe` | Path to ffprobe. Without it, quality comparison is disabled but everything else works. | | `PLAYLINK_LOG_LEVEL` | `info` | `debug` for verbose logs. | -Per-pair settings (in the UI) are file extensions, auto-link, scan interval, quarantine folder, retention and settle time. +Per-pair settings (in the UI) are file extensions, auto-link, approximate matching and its threshold, scan interval, quarantine folder, +retention and settle time. ## Development diff --git a/internal/api/server.go b/internal/api/server.go index 512dea0..d5f95b4 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -306,12 +306,15 @@ type pairInput struct { ScanIntervalMinutes int `json:"scan_interval_minutes"` RetentionDays int `json:"retention_days"` SettleMinutes int `json:"settle_minutes"` + ApproxMatching bool `json:"approx_matching"` + ApproxThreshold int `json:"approx_threshold"` } func (in pairInput) apply(p *store.Pair) { p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir = in.Name, in.MasterRoot, in.SlaveRoot, in.QuarantineDir p.Extensions, p.AutoLinkIdentical = in.Extensions, in.AutoLinkIdentical p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes = in.ScanIntervalMinutes, in.RetentionDays, in.SettleMinutes + p.ApproxMatching, p.ApproxThreshold = in.ApproxMatching, in.ApproxThreshold } func (s *Server) handleValidatePair(w http.ResponseWriter, r *http.Request) { diff --git a/internal/engine/approx.go b/internal/engine/approx.go new file mode 100644 index 0000000..8b5dae8 --- /dev/null +++ b/internal/engine/approx.go @@ -0,0 +1,257 @@ +package engine + +import ( + "fmt" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "unicode" +) + +// Approximate matching is an opt-in, per-pair second pass for slave files the +// strict rules could not place. It tolerates disc subfolders ("Disc 02/", +// "CD2/") and slightly different titles, but never auto-links: every result +// goes to manual review. + +// discFolder matches disc/side subfolders such as "Disc 02", "CD2", "Disk 1 - Bonus". +var discFolder = regexp.MustCompile(`(?i)^(disc|disk|cd|dvd|side|vol|volume)[\s._-]*\d+\b.*$`) + +// trackNum extracts the leading track number ("07 Memory", "2-07 Memory"). +var trackNum = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?(\d{1,3})(?:\s*[-._)]\s*|\s+)`) + +// versionWords mark a different recording or arrangement. Two titles only +// match approximately if they carry the same set of these words, so +// "Memory (Live)" never matches "Memory". +var versionWords = map[string]bool{ + "live": true, "remix": true, "mix": true, "acoustic": true, "demo": true, "instrumental": true, + "edit": true, "version": true, "reprise": true, "karaoke": true, "cover": true, "extended": true, + "radio": true, "unplugged": true, "orchestral": true, "piano": true, "clean": true, "acapella": true, + "cappella": true, "interlude": true, "intro": true, "outro": true, "skit": true, "vip": true, "dub": true, + "sped": true, "slowed": true, "nightcore": true, "reverb": true, "mashup": true, "bootleg": true, +} + +// albumKey is the lower-cased directory with disc folders removed. +func albumKey(rel string) string { + var parts []string + for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) { + if p != "." && !discFolder.MatchString(p) { + parts = append(parts, strings.ToLower(p)) + } + } + return strings.Join(parts, "/") +} + +func hasDiscFolder(rel string) bool { + for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) { + if discFolder.MatchString(p) { + return true + } + } + return false +} + +type titleInfo struct { + norm string // normTitle: letters+digits only + words []string // non-version words + versions map[string]bool // version words present + track int // 0 = unknown +} + +func parseTitle(name string) titleInfo { + stem := strings.TrimSuffix(name, filepath.Ext(name)) + ti := titleInfo{norm: normTitle(name), versions: map[string]bool{}} + if m := trackNum.FindStringSubmatch(stem); m != nil { + ti.track, _ = strconv.Atoi(m[1]) + if s := trackNum.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" { + stem = s + } + } + for _, w := range strings.FieldsFunc(strings.ToLower(stem), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if versionWords[w] { + ti.versions[w] = true + } else { + ti.words = append(ti.words, w) + } + } + return ti +} + +func sameVersions(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for k := range a { + if !b[k] { + return false + } + } + return true +} + +// levenshteinRatio is 1 - editDistance/maxLen over runes. +func levenshteinRatio(a, b string) float64 { + ra, rb := []rune(a), []rune(b) + if len(ra) == 0 || len(rb) == 0 { + return 0 + } + prev := make([]int, len(rb)+1) + cur := make([]int, len(rb)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ra); i++ { + cur[0] = i + for j := 1; j <= len(rb); j++ { + cost := 1 + if ra[i-1] == rb[j-1] { + cost = 0 + } + cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost) + } + prev, cur = cur, prev + } + return 1 - float64(prev[len(rb)])/float64(max(len(ra), len(rb))) +} + +// wordContainment scores titles where one adds words to the other +// ("Memory" vs "Memory from Cats"). It needs at least two shared words so a +// one-word title like "Love" cannot swallow "Love Story". +func wordContainment(a, b []string) float64 { + if len(a) > len(b) { + a, b = b, a + } + if len(a) < 2 { + return 0 + } + set := map[string]bool{} + for _, w := range b { + set[w] = true + } + hit := 0 + for _, w := range a { + if set[w] { + hit++ + } + } + if hit != len(a) { + return 0 + } + return 0.9 // a full subset is strong but never a perfect match +} + +// titleScore is the similarity of two titles in [0,1], or -1 if they are +// different versions of a song. +func titleScore(a, b titleInfo) float64 { + if a.norm == "" || b.norm == "" || !sameVersions(a.versions, b.versions) { + return -1 + } + s := max(levenshteinRatio(a.norm, b.norm), wordContainment(a.words, b.words)) + if a.track > 0 && b.track > 0 { + if a.track == b.track { + s += 0.05 + } else { + s -= 0.05 + } + } + return min(s, 1) +} + +type approxIndex struct { + byAlbum map[string][]string // albumKey -> master rels + byTop map[string][]string // lower(top folder) -> master rels + titles map[string]titleInfo +} + +func buildApproxIndex(files []string) *approxIndex { + ax := &approxIndex{byAlbum: map[string][]string{}, byTop: map[string][]string{}, titles: map[string]titleInfo{}} + for _, rel := range files { + ax.titles[rel] = parseTitle(filepath.Base(rel)) + k := albumKey(rel) + ax.byAlbum[k] = append(ax.byAlbum[k], rel) + if top := firstComponent(rel); top != "" { + t := strings.ToLower(top) + ax.byTop[t] = append(ax.byTop[t], rel) + } + } + return ax +} + +type scored struct { + rel string + score float64 +} + +// approxMargin is how far the best candidate must beat the runner-up. +const approxMargin = 0.05 + +// find returns the best approximate master match for a slave file, plus a +// human-readable reason. The album (disc folders ignored) is searched first, +// then the whole top-level folder. +func (ax *approxIndex) find(relSlave string, threshold float64) (string, string) { + st := parseTitle(filepath.Base(relSlave)) + scopes := []struct { + name string + cands []string + }{ + {"same album", ax.byAlbum[albumKey(relSlave)]}, + } + if top := firstComponent(relSlave); top != "" { + scopes = append(scopes, struct { + name string + cands []string + }{"same top-level folder, different album folder", ax.byTop[strings.ToLower(top)]}) + } + for _, sc := range scopes { + var best []scored + for _, rel := range sc.cands { + if s := titleScore(st, ax.titles[rel]); s >= threshold { + best = append(best, scored{rel, s}) + } + } + if len(best) == 0 { + continue + } + sort.Slice(best, func(i, j int) bool { + if best[i].score != best[j].score { + return best[i].score > best[j].score + } + return extRank(best[i].rel) < extRank(best[j].rel) + }) + // Several formats of the same title in one folder are one song. + top := best[0] + ambiguous := false + for _, o := range best[1:] { + sameSong := filepath.Dir(o.rel) == filepath.Dir(top.rel) && + strings.TrimSuffix(filepath.Base(o.rel), filepath.Ext(o.rel)) == strings.TrimSuffix(filepath.Base(top.rel), filepath.Ext(top.rel)) + if !sameSong && top.score-o.score < approxMargin { + ambiguous = true + break + } + } + if ambiguous { + return "", "" + } + return top.rel, approxReason(relSlave, top.rel, st, ax.titles[top.rel], top.score, sc.name) + } + return "", "" +} + +func approxReason(relSlave, relMaster string, st, mt titleInfo, score float64, scope string) string { + parts := []string{fmt.Sprintf("title %d%% similar", int(score*100+0.5))} + if st.track > 0 && mt.track > 0 { + if st.track == mt.track { + parts = append(parts, fmt.Sprintf("track %02d matches", st.track)) + } else { + parts = append(parts, fmt.Sprintf("track numbers differ (%02d vs %02d)", st.track, mt.track)) + } + } + if filepath.Dir(relSlave) != filepath.Dir(relMaster) && albumKey(relSlave) == albumKey(relMaster) { + parts = append(parts, "disc folder ignored") + } + parts = append(parts, scope) + return strings.Join(parts, " · ") +} diff --git a/internal/engine/approx_test.go b/internal/engine/approx_test.go new file mode 100644 index 0000000..b091eb0 --- /dev/null +++ b/internal/engine/approx_test.go @@ -0,0 +1,126 @@ +package engine + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/cdnutter/playlink/internal/store" +) + +func enableApprox(t *testing.T, ev *env) { + t.Helper() + ev.pair.ApproxMatching = true + ev.pair.ApproxThreshold = 85 + if err := ev.st.SavePair(ctx, &ev.pair); err != nil { + t.Fatal(err) + } +} + +// The user's example: the master album has disc folders, the slave does not. +func TestApproxDiscFolderUpgrade(t *testing.T) { + ev := setup(t, false) + master := ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac", "lossless") + ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 01/07 Macavity.flac", "other") + ev.write(ev.master, "Andrew Lloyd Webber/Evita/03 Memory.flac", "same title, other album") + ev.write(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.mp3", "lossy") + + // Off by default: the title exists on two albums, so nothing matches. + ev.scan() + if m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("approximate matching must be opt-in, got %s/%s", m.Status, m.Kind) + } + + enableApprox(t, ev) + ev.scan() + m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3") + if m.Kind != store.KindApprox || m.Status != store.StatusSimilar || m.RelMaster != "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac" { + t.Fatalf("unexpected match %s %s -> %s", m.Status, m.Kind, m.RelMaster) + } + if !strings.Contains(m.Reason, "disc folder ignored") || !strings.Contains(m.Reason, "track 07 matches") { + t.Fatalf("reason should explain the match, got %q", m.Reason) + } + + // Linking upgrades the slave: it gets the master's FLAC in its own folder. + if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); err != nil { + t.Fatal(err) + } + upgraded := filepath.Join(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.flac") + if !sameInode(t, master, upgraded) { + t.Fatal("slave should now hold a hardlink of the master FLAC") + } +} + +func TestApproxTitleVariations(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + ev.write(ev.master, "Band/Album/CD1/03 - Dont Stop Believin.flac", "a") + ev.write(ev.slave, "Band/Album/3 Don't Stop Believing.mp3", "b") // typo-level difference + ev.write(ev.master, "Band/Album/CD1/04 - Song.flac", "c") + ev.write(ev.slave, "Band/Album/04 Song (Live).mp3", "d") // different version: must not match + ev.write(ev.master, "Band/Album/CD2/05 Totally Different.flac", "e") + ev.write(ev.slave, "Band/Album/05 Something Else.mp3", "f") // too dissimilar + ev.scan() + + if m := ev.match("Band/Album/3 Don't Stop Believing.mp3"); m.Kind != store.KindApprox { + t.Fatalf("near-identical title should match approximately, got %s", m.Kind) + } + if m := ev.match("Band/Album/04 Song (Live).mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("live version must not match the studio track, got %s -> %s", m.Kind, m.RelMaster) + } + if m := ev.match("Band/Album/05 Something Else.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("dissimilar titles must not match, got %s -> %s", m.Kind, m.RelMaster) + } +} + +func TestApproxAmbiguousIsLeftAlone(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + // Two discs each carry an equally good candidate: refuse to guess. + ev.write(ev.master, "Band/Album/Disc 1/Anthem.flac", "a") + ev.write(ev.master, "Band/Album/Disc 2/Anthem.flac", "b") + ev.write(ev.slave, "Band/Album/Anthem.mp3", "c") + ev.scan() + if m := ev.match("Band/Album/Anthem.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("ambiguous candidates must not be matched, got %s -> %s", m.Kind, m.RelMaster) + } +} + +func TestApproxAlbumFolderRename(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + ev.write(ev.master, "Cast/Cats (Original London Cast)/Disc 2/07 Memory.flac", "a") + ev.write(ev.slave, "Cast/Cats/07. Memory.mp3", "b") + ev.scan() + m := ev.match("Cast/Cats/07. Memory.mp3") + if m.Kind != store.KindApprox || !strings.Contains(m.Reason, "different album folder") { + t.Fatalf("should match across album folder names, got %s %q", m.Kind, m.Reason) + } +} + +func TestTitleScore(t *testing.T) { + cases := []struct { + a, b string + min float64 + max float64 + }{ + {"07 Memory.flac", "07 Memory.mp3", 1, 1}, + {"07 Memory.flac", "08 Memory.mp3", 0.9, 0.96}, + {"Memory.flac", "Memory (Live).flac", -1, -1}, + {"Song (Remix).flac", "Song (Remix).mp3", 1, 1}, + {"Love.flac", "Love Story.flac", 0, 0.6}, + {"Memory from Cats.flac", "Memory from Cats Remastered.flac", 0.85, 1}, + } + for _, c := range cases { + got := titleScore(parseTitle(c.a), parseTitle(c.b)) + if got < c.min-1e-9 || got > c.max+1e-9 { + t.Errorf("titleScore(%q, %q) = %.3f, want [%.2f, %.2f]", c.a, c.b, got, c.min, c.max) + } + } + if k := albumKey("A/Cats/Disc 02/07 Memory.flac"); k != "a/cats" { + t.Errorf("albumKey = %q", k) + } + if k := albumKey("A/Cats/CD2 - Bonus/x.flac"); k != "a/cats" { + t.Errorf("albumKey = %q", k) + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 17e6b4d..a97edf0 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -280,6 +280,12 @@ func (e *Engine) NormalizePair(p *store.Pair) ([]string, error) { if p.RetentionDays < 0 || p.ScanIntervalMinutes < 0 || p.SettleMinutes < 0 { return nil, errors.New("intervals must not be negative") } + if p.ApproxThreshold == 0 { + p.ApproxThreshold = 85 + } + if p.ApproxThreshold < 60 || p.ApproxThreshold > 100 { + return nil, errors.New("approximate match threshold must be between 60 and 100 percent") + } if p.ScanIntervalMinutes > 0 && p.ScanIntervalMinutes < 5 { return nil, errors.New("scan interval must be 0 (manual) or at least 5 minutes") } diff --git a/internal/engine/scan.go b/internal/engine/scan.go index a0f4079..76737fb 100644 --- a/internal/engine/scan.go +++ b/internal/engine/scan.go @@ -104,11 +104,19 @@ type masterIndex struct { lower map[string]string // lower(rel) -> rel byDir map[string][]string // lower(dir) -> rels byTitle map[string][]string // lower(top folder)|normTitle -> rels + + approx *approxIndex // nil unless the pair enables approximate matching + threshold float64 } -func buildIndex(files map[string]fsops.Info) *masterIndex { +func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex { ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}} - for _, rel := range sortedKeys(files) { + rels := sortedKeys(files) + if p.ApproxMatching { + ix.approx = buildApproxIndex(rels) + ix.threshold = float64(p.ApproxThreshold) / 100 + } + for _, rel := range rels { ix.lower[strings.ToLower(rel)] = rel d := strings.ToLower(filepath.Dir(rel)) ix.byDir[d] = append(ix.byDir[d], rel) @@ -130,13 +138,14 @@ func extRank(rel string) int { return 2 } -// find locates the master counterpart of a slave file. -func (ix *masterIndex) find(relSlave string) (string, string) { +// find locates the master counterpart of a slave file. The reason is only +// set for approximate matches. +func (ix *masterIndex) find(relSlave string) (rel, kind, reason string) { if _, ok := ix.files[relSlave]; ok { - return relSlave, store.KindPath + return relSlave, store.KindPath, "" } if r, ok := ix.lower[strings.ToLower(relSlave)]; ok { - return r, store.KindPathCase + return r, store.KindPathCase, "" } dir := strings.ToLower(filepath.Dir(relSlave)) base := filepath.Base(relSlave) @@ -158,17 +167,25 @@ func (ix *masterIndex) find(relSlave string) (string, string) { best = r } } - return best, store.KindExtension + return best, store.KindExtension, "" } if len(fuzzyHits) == 1 { - return fuzzyHits[0], store.KindFuzzy + return fuzzyHits[0], store.KindFuzzy, "" + } + // Approximate matching runs before the top-folder fallback because it + // searches the album (disc folders ignored) first, which resolves titles + // that appear on several albums. + if ix.approx != nil { + if r, why := ix.approx.find(relSlave, ix.threshold); r != "" { + return r, store.KindApprox, why + } } if top := firstComponent(relSlave); top != "" && norm != "" { if hits := ix.byTitle[strings.ToLower(top)+"|"+norm]; len(hits) == 1 { - return hits[0], store.KindElsewhere + return hits[0], store.KindElsewhere, "" } } - return "", store.KindNone + return "", store.KindNone, "" } // classify fills in status, probes and verdict for a match, reusing cached @@ -269,7 +286,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) _ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns) return err } - ix := buildIndex(masterFiles) + ix := buildIndex(masterFiles, l.pair) prev, err := e.st.AllMatches(ctx, pairID) if err != nil { return err @@ -284,7 +301,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) } si := slaveFiles[rel] m := store.Match{PairID: pairID, RelSlave: rel} - m.RelMaster, m.Kind = ix.find(rel) + m.RelMaster, m.Kind, m.Reason = ix.find(rel) var pm *store.Match if p, ok := prev[rel]; ok { pm = &p @@ -304,7 +321,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) // Layout warnings based on what was actually found. if ss.files > 0 { - if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files { + if n := kinds[store.KindElsewhere] + kinds[store.KindApprox] + kinds[store.KindNone]; n*4 > ss.files { warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the master library. "+ "Check that both libraries use the same folder layout.", n, ss.files)) } @@ -426,7 +443,7 @@ func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) { if pm != nil && pm.RelMaster != "" { if in, err := fsops.RegularFile(filepath.Join(l.master.Path, pm.RelMaster)); err == nil { mi = in - m.RelMaster, m.Kind = pm.RelMaster, pm.Kind + m.RelMaster, m.Kind, m.Reason = pm.RelMaster, pm.Kind, pm.Reason } } } diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 4b37915..f8b9c5f 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -206,9 +206,8 @@ func Compare(master, slave Info) Verdict { v.Better = "slave" } if master.Lossless != slave.Lossless { - if master.Lossless { - v.Notes = append(v.Notes, "master is lossless, slave is lossy") - } else { + // Only the downgrade case is a warning; an upgrade is covered by Better. + if !master.Lossless { v.Notes = append(v.Notes, "slave is lossless but master is lossy: linking would downgrade the slave") } } diff --git a/internal/store/models.go b/internal/store/models.go index 3db89f5..aecc1b1 100644 --- a/internal/store/models.go +++ b/internal/store/models.go @@ -104,6 +104,8 @@ type Pair struct { ScanIntervalMinutes int `json:"scan_interval_minutes"` RetentionDays int `json:"retention_days"` SettleMinutes int `json:"settle_minutes"` + ApproxMatching bool `json:"approx_matching"` + ApproxThreshold int `json:"approx_threshold"` // percent LastScanAt *int64 `json:"last_scan_at"` LastScanError string `json:"last_scan_error"` LastScanWarnings []string `json:"last_scan_warnings"` @@ -111,7 +113,8 @@ type Pair struct { } const pairCols = `id, name, master_root, slave_root, quarantine_dir, extensions, auto_link_identical, - scan_interval_minutes, retention_days, settle_minutes, last_scan_at, last_scan_error, last_scan_warnings, created_at` + scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, last_scan_at, last_scan_error, + last_scan_warnings, created_at` type scanner interface{ Scan(...any) error } @@ -120,7 +123,8 @@ func scanPair(r scanner) (Pair, error) { var exts, warns string var last sql.NullInt64 err := r.Scan(&p.ID, &p.Name, &p.MasterRoot, &p.SlaveRoot, &p.QuarantineDir, &exts, &p.AutoLinkIdentical, - &p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &last, &p.LastScanError, &warns, &p.CreatedAt) + &p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &p.ApproxMatching, &p.ApproxThreshold, &last, &p.LastScanError, + &warns, &p.CreatedAt) if err != nil { return p, err } @@ -164,10 +168,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error { exts := strings.Join(p.Extensions, ",") if p.ID == 0 { res, err := s.DB.ExecContext(ctx, `INSERT INTO pairs (name, master_root, slave_root, quarantine_dir, extensions, - auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical, - p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, now()) + p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, now()) if err != nil { return err } @@ -175,10 +179,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error { return nil } _, err := s.DB.ExecContext(ctx, `UPDATE pairs SET name = ?, master_root = ?, slave_root = ?, quarantine_dir = ?, - extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ? - WHERE id = ?`, + extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?, + approx_matching = ?, approx_threshold = ? WHERE id = ?`, p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical, - p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ID) + p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, p.ID) return err } @@ -216,11 +220,12 @@ const ( // Match kinds (how the master candidate was found). const ( - KindPath = "path" // identical relative path - KindPathCase = "path_case" // relative path differs only by case - KindExtension = "extension" // same directory and title, different extension - KindFuzzy = "fuzzy" // same directory, normalized title matches - KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder + KindPath = "path" // identical relative path + KindPathCase = "path_case" // relative path differs only by case + KindExtension = "extension" // same directory and title, different extension + KindFuzzy = "fuzzy" // same directory, normalized title matches + KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder + KindApprox = "approximate" // opt-in: similar title, disc folders ignored KindNone = "none" ) @@ -241,18 +246,19 @@ type Match struct { SlaveProbe json.RawMessage `json:"slave_probe,omitempty"` Verdict json.RawMessage `json:"verdict,omitempty"` Note string `json:"note"` + Reason string `json:"reason"` // why an approximate match was chosen Ignored bool `json:"ignored"` UpdatedAt int64 `json:"updated_at"` } const matchCols = `id, pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, master_mtime, - slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at` + slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at` func scanMatch(r scanner) (Match, error) { var m Match var mp, sp, v string err := r.Scan(&m.ID, &m.PairID, &m.RelSlave, &m.RelMaster, &m.Kind, &m.Status, &m.MasterIno, &m.MasterSize, &m.MasterMtime, - &m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Ignored, &m.UpdatedAt) + &m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Ignored, &m.UpdatedAt) if mp != "" { m.MasterProbe = json.RawMessage(mp) } @@ -316,6 +322,8 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat where = append(where, "status IN ('different','similar') AND ignored = 0") case "ignored": where = append(where, "ignored = 1") + case "upgrade": + where = append(where, "status IN ('different','similar') AND ignored = 0 AND "+upgradeCond) default: where = append(where, "status = ? AND ignored = 0") args = append(args, f.Status) @@ -371,15 +379,15 @@ func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error) func (s *Store) UpsertMatch(ctx context.Context, m *Match) error { nullable := func(r json.RawMessage) string { return string(r) } _, err := s.DB.ExecContext(ctx, `INSERT INTO matches (pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, - master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(pair_id, rel_slave) DO UPDATE SET rel_master = excluded.rel_master, kind = excluded.kind, status = excluded.status, master_ino = excluded.master_ino, master_size = excluded.master_size, master_mtime = excluded.master_mtime, slave_ino = excluded.slave_ino, slave_size = excluded.slave_size, slave_mtime = excluded.slave_mtime, master_probe = excluded.master_probe, slave_probe = excluded.slave_probe, - verdict = excluded.verdict, note = excluded.note, ignored = excluded.ignored, updated_at = excluded.updated_at`, + verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, ignored = excluded.ignored, updated_at = excluded.updated_at`, m.PairID, m.RelSlave, m.RelMaster, m.Kind, m.Status, m.MasterIno, m.MasterSize, m.MasterMtime, m.SlaveIno, m.SlaveSize, - m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Ignored, now()) + m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Ignored, now()) return err } @@ -433,9 +441,14 @@ func (s *Store) LinkedMasterPaths(ctx context.Context, pairID int64, prefix stri return out, rows.Err() } +// upgradeCond selects review matches where the master copy scored better, so +// linking upgrades the slave. +const upgradeCond = `verdict LIKE '%"better":"master"%'` + type Summary struct { Counts map[string]int `json:"counts"` Review int `json:"review"` + Upgrades int `json:"upgrades"` Ignored int `json:"ignored"` ReclaimableSize int64 `json:"reclaimable_bytes"` LinkedSize int64 `json:"linked_bytes"` @@ -470,7 +483,12 @@ func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) { sum.Review += n } } - return sum, rows.Err() + if err := rows.Err(); err != nil { + return sum, err + } + err = s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0 + AND status IN ('different','similar') AND `+upgradeCond, pairID).Scan(&sum.Upgrades) + return sum, err } // ---- operations ---- diff --git a/internal/store/store.go b/internal/store/store.go index bf8d6ab..12ea060 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -104,6 +104,11 @@ var migrations = []string{ UPDATE operations SET quarantine_side = 'master' WHERE quarantine_side = 'host'; UPDATE matches SET verdict = replace(verdict, 'host', 'master'); UPDATE pairs SET last_scan_warnings = replace(last_scan_warnings, 'host', 'master');`, + + // 3: opt-in approximate matching, and the reason shown for such matches. + `ALTER TABLE pairs ADD COLUMN approx_matching INTEGER NOT NULL DEFAULT 0; + ALTER TABLE pairs ADD COLUMN approx_threshold INTEGER NOT NULL DEFAULT 85; + ALTER TABLE matches ADD COLUMN reason TEXT NOT NULL DEFAULT '';`, } // Open opens (and migrates) the database in dataDir. diff --git a/web/src/api.ts b/web/src/api.ts index ab6bfdc..c186e8a 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -54,6 +54,7 @@ export type Job = { export type Summary = { counts: Record; review: number; + upgrades: number; ignored: number; reclaimable_bytes: number; linked_bytes: number; @@ -70,6 +71,8 @@ export type Pair = { scan_interval_minutes: number; retention_days: number; settle_minutes: number; + approx_matching: boolean; + approx_threshold: number; last_scan_at: number | null; last_scan_error: string; last_scan_warnings: string[]; @@ -88,6 +91,8 @@ export type PairInput = Pick< | "scan_interval_minutes" | "retention_days" | "settle_minutes" + | "approx_matching" + | "approx_threshold" >; export type ProbeInfo = { @@ -115,7 +120,7 @@ export type Match = { pair_id: number; rel_slave: string; rel_master: string; - kind: "path" | "path_case" | "extension" | "fuzzy" | "elsewhere" | "none"; + kind: "path" | "path_case" | "extension" | "fuzzy" | "elsewhere" | "approximate" | "none"; status: Status; master_size: number; master_mtime: number; @@ -125,6 +130,7 @@ export type Match = { slave_probe?: ProbeInfo; verdict?: Verdict; note: string; + reason: string; ignored: boolean; }; diff --git a/web/src/pages/PairForm.tsx b/web/src/pages/PairForm.tsx index bd882fc..40ef0ed 100644 --- a/web/src/pages/PairForm.tsx +++ b/web/src/pages/PairForm.tsx @@ -22,6 +22,8 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; scan_interval_minutes: 0, retention_days: 30, settle_minutes: 10, + approx_matching: false, + approx_threshold: 85, }; const [f, setF] = useState(pair ?? blank); const [exts, setExts] = useState((pair?.extensions ?? []).join(", ")); @@ -47,6 +49,8 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; scan_interval_minutes: f.scan_interval_minutes, retention_days: f.retention_days, settle_minutes: f.settle_minutes, + approx_matching: f.approx_matching, + approx_threshold: f.approx_threshold, extensions: exts.split(",").map((s) => s.trim()).filter(Boolean), }); const set = (k: K, v: PairInput[K]) => { @@ -128,6 +132,37 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; Anything else always waits for manual review. Dry run still applies.

+
+ set("approx_matching", v)} + label={Approximate matching} + /> +

+ Also match songs whose file name is only approximately the same, and ignore disc folders. For example{" "} + Cats/Disc 02/07 Memory.flac matches Cats/07 Memory.mp3. Version words + must agree ("Live", "Remix", "Reprise"…), ambiguous candidates are skipped, and approximate matches always go to manual review. +

+ {f.approx_matching && ( +
+ + set("approx_threshold", Number(e.target.value))} + className="w-40 accent-violet-500" + /> + {f.approx_threshold}% + Lower finds more matches, but also more false positives. 85% is a good default. +
+ )} +
{validation && !validation.ok && {validation.error}} {validation?.ok && validation.warnings.length === 0 && Looks good. Both folders are valid and on the same filesystem.} diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 5888692..f450010 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1,12 +1,13 @@ import { useEffect, useState } from "react"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { AlertTriangle, ArrowRight, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; +import { AlertTriangle, ArrowRight, ArrowUpCircle, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; import { ApiError, api, type Match, type Page, type Pair, type ProbeInfo } from "../api"; import { Alert, Badge, Button, Card, Empty, Modal, Spinner, cx, formatBytes, formatDuration, inputCls, statusTone } from "../ui"; import { useToast } from "../toast"; const filters = [ { id: "review", label: "Needs review" }, + { id: "upgrade", label: "Upgrades" }, { id: "identical", label: "Identical" }, { id: "linked", label: "Linked" }, { id: "unmatched", label: "Slave only" }, @@ -21,6 +22,7 @@ const kindLabel: Record = { extension: "other format", fuzzy: "similar title", elsewhere: "same top folder, other subfolder", + approximate: "approximate match", none: "no match", }; @@ -64,7 +66,7 @@ export function Review({ pair }: { pair: Pair }) { const c = pair.summary.counts; const count = (id: string) => - id === "review" ? pair.summary.review : id === "ignored" ? pair.summary.ignored : id === "" ? undefined : c[id] ?? 0; + id === "review" ? pair.summary.review : id === "upgrade" ? pair.summary.upgrades : id === "ignored" ? pair.summary.ignored : id === "" ? undefined : c[id] ?? 0; const identical = c.identical ?? 0; const running = pair.job?.state === "running"; @@ -211,6 +213,7 @@ function MatchRow({ m }: { m: Match }) {
reviewable && setOpen(!open)}> + {m.reason &&
{m.reason}
} {m.note &&
{m.note}
}
{reviewable && ( @@ -223,7 +226,11 @@ function MatchRow({ m }: { m: Match }) { master lower )} - {better === "master" && master better} + {better === "master" && ( + + upgrade + + )} {better === "equal" && equal} )} @@ -266,7 +273,7 @@ const rows: [string, (p: ProbeInfo, size: number) => string][] = [ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { const toast = useToast(); const [confirm, setConfirm] = useState(null); - const hp = m.master_probe; + const mp = m.master_probe; const sp = m.slave_probe; const v = m.verdict; const renamed = m.rel_master.split("/").pop() !== m.rel_slave.split("/").pop(); @@ -295,8 +302,8 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { return (
- {(!hp || !sp) && No codec details available (ffprobe missing or failed). Compare the files yourself before linking.} - {hp && sp && ( + {(!mp || !sp) && No codec details available (ffprobe missing or failed). Compare the files yourself before linking.} + {mp && sp && (
@@ -312,7 +319,7 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { {rows.map(([label, f]) => { - const a = hp.error ? "–" : f(hp, m.master_size); + const a = mp.error ? "–" : f(mp, m.master_size); const b = sp.error ? "–" : f(sp, m.slave_size); return ( @@ -327,11 +334,23 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { )}
- {v?.notes?.map((n) => ( + {/* Scans before this change stored an upgrade note; the green alert below covers it. */} + {v?.notes?.filter((n) => !n.startsWith("master is lossless")).map((n) => ( }> {n} ))} + {m.kind === "approximate" && ( + + Approximate match: {m.reason}. Check that the titles and durations really are the same song before linking. + + )} + {v?.better === "master" && ( + }> + Recommended: the master copy is better ({qualityLabel(mp)} vs {qualityLabel(sp)}). Linking upgrades the slave to it and frees the + slave's copy. + + )} {v?.better === "slave" && ( }> The master copy is lower quality. The master is normally kept because it belongs to a complete album. You can still link to it (the @@ -347,7 +366,8 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) {