diff --git a/README.md b/README.md index 73f4934..1524008 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,21 @@ path on both sides. The fallbacks below catch small differences, and a scan warn | Same title, **different format** (`Song.mp3` vs `Song.flac`) | Status `similar` (other format). Manual review. | | Same title after ignoring track numbers/punctuation (`01 - Song!.flac` vs `Song.mp3`) | Status `similar` (similar title). Manual review. | | Same top-level folder and title, but a **different subfolder** | Status `similar` (same top folder, other subfolder). Manual review. | +| *(opt-in)* **Approximately** the same title, disc folders ignored (`Cats/Disc 02/07 Memory.flac` vs `Cats/07 Memory.mp3`) | Status `similar` (approximate match), with the reason shown. Manual review. | | Only exists in the slave library | Status `unmatched`. Never touched. | | Already the same inode | Status `linked`. | +**Approximate matching** is an opt-in setting on each pair. For slave files the strict rules could not place, it searches the same +album with disc folders (`Disc 02`, `CD2`, …) ignored, then the whole top-level folder, and compares titles by similarity against a +threshold you choose (85% by default; a matching track number adds a small bonus). Several guards keep it from linking the wrong song: +version words such as *Live*, *Remix*, *Reprise* or *Instrumental* must agree on both sides, a file is left unmatched when two +candidates are nearly tied, and approximate matches never auto-link. + **Manual review** shows both files side by side and ranks their quality: lossless beats lossy, then bit depth and sample rate, then codec-weighted bitrate (Opus and AAC count for more per kbps than MP3). You then have three choices: -- **Link to master copy.** The default. The master is kept because it belongs to a complete album. If the master is *lower* quality, playlink +- **Upgrade / link to master copy.** The default. When the master copy is better, the match is flagged as an **upgrade** and listed + under the *Upgrades* filter. The master is kept because it belongs to a complete album. If the master is *lower* quality, playlink warns you and asks for explicit confirmation. - **Use slave copy as master.** The override. The slave's better file replaces the master file, the old master file is quarantined, and both libraries share the better file. @@ -108,7 +116,8 @@ Run a scan, check the Review tab, and turn off dry run once the planned actions | `PLAYLINK_FFPROBE` | `ffprobe` | Path to ffprobe. Without it, quality comparison is disabled but everything else works. | | `PLAYLINK_LOG_LEVEL` | `info` | `debug` for verbose logs. | -Per-pair settings (in the UI) are file extensions, auto-link, scan interval, quarantine folder, retention and settle time. +Per-pair settings (in the UI) are file extensions, auto-link, approximate matching and its threshold, scan interval, quarantine folder, +retention and settle time. ## Development diff --git a/internal/api/server.go b/internal/api/server.go index 512dea0..d5f95b4 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -306,12 +306,15 @@ type pairInput struct { ScanIntervalMinutes int `json:"scan_interval_minutes"` RetentionDays int `json:"retention_days"` SettleMinutes int `json:"settle_minutes"` + ApproxMatching bool `json:"approx_matching"` + ApproxThreshold int `json:"approx_threshold"` } func (in pairInput) apply(p *store.Pair) { p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir = in.Name, in.MasterRoot, in.SlaveRoot, in.QuarantineDir p.Extensions, p.AutoLinkIdentical = in.Extensions, in.AutoLinkIdentical p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes = in.ScanIntervalMinutes, in.RetentionDays, in.SettleMinutes + p.ApproxMatching, p.ApproxThreshold = in.ApproxMatching, in.ApproxThreshold } func (s *Server) handleValidatePair(w http.ResponseWriter, r *http.Request) { diff --git a/internal/engine/approx.go b/internal/engine/approx.go new file mode 100644 index 0000000..8b5dae8 --- /dev/null +++ b/internal/engine/approx.go @@ -0,0 +1,257 @@ +package engine + +import ( + "fmt" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "unicode" +) + +// Approximate matching is an opt-in, per-pair second pass for slave files the +// strict rules could not place. It tolerates disc subfolders ("Disc 02/", +// "CD2/") and slightly different titles, but never auto-links: every result +// goes to manual review. + +// discFolder matches disc/side subfolders such as "Disc 02", "CD2", "Disk 1 - Bonus". +var discFolder = regexp.MustCompile(`(?i)^(disc|disk|cd|dvd|side|vol|volume)[\s._-]*\d+\b.*$`) + +// trackNum extracts the leading track number ("07 Memory", "2-07 Memory"). +var trackNum = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?(\d{1,3})(?:\s*[-._)]\s*|\s+)`) + +// versionWords mark a different recording or arrangement. Two titles only +// match approximately if they carry the same set of these words, so +// "Memory (Live)" never matches "Memory". +var versionWords = map[string]bool{ + "live": true, "remix": true, "mix": true, "acoustic": true, "demo": true, "instrumental": true, + "edit": true, "version": true, "reprise": true, "karaoke": true, "cover": true, "extended": true, + "radio": true, "unplugged": true, "orchestral": true, "piano": true, "clean": true, "acapella": true, + "cappella": true, "interlude": true, "intro": true, "outro": true, "skit": true, "vip": true, "dub": true, + "sped": true, "slowed": true, "nightcore": true, "reverb": true, "mashup": true, "bootleg": true, +} + +// albumKey is the lower-cased directory with disc folders removed. +func albumKey(rel string) string { + var parts []string + for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) { + if p != "." && !discFolder.MatchString(p) { + parts = append(parts, strings.ToLower(p)) + } + } + return strings.Join(parts, "/") +} + +func hasDiscFolder(rel string) bool { + for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) { + if discFolder.MatchString(p) { + return true + } + } + return false +} + +type titleInfo struct { + norm string // normTitle: letters+digits only + words []string // non-version words + versions map[string]bool // version words present + track int // 0 = unknown +} + +func parseTitle(name string) titleInfo { + stem := strings.TrimSuffix(name, filepath.Ext(name)) + ti := titleInfo{norm: normTitle(name), versions: map[string]bool{}} + if m := trackNum.FindStringSubmatch(stem); m != nil { + ti.track, _ = strconv.Atoi(m[1]) + if s := trackNum.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" { + stem = s + } + } + for _, w := range strings.FieldsFunc(strings.ToLower(stem), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) { + if versionWords[w] { + ti.versions[w] = true + } else { + ti.words = append(ti.words, w) + } + } + return ti +} + +func sameVersions(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for k := range a { + if !b[k] { + return false + } + } + return true +} + +// levenshteinRatio is 1 - editDistance/maxLen over runes. +func levenshteinRatio(a, b string) float64 { + ra, rb := []rune(a), []rune(b) + if len(ra) == 0 || len(rb) == 0 { + return 0 + } + prev := make([]int, len(rb)+1) + cur := make([]int, len(rb)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ra); i++ { + cur[0] = i + for j := 1; j <= len(rb); j++ { + cost := 1 + if ra[i-1] == rb[j-1] { + cost = 0 + } + cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost) + } + prev, cur = cur, prev + } + return 1 - float64(prev[len(rb)])/float64(max(len(ra), len(rb))) +} + +// wordContainment scores titles where one adds words to the other +// ("Memory" vs "Memory from Cats"). It needs at least two shared words so a +// one-word title like "Love" cannot swallow "Love Story". +func wordContainment(a, b []string) float64 { + if len(a) > len(b) { + a, b = b, a + } + if len(a) < 2 { + return 0 + } + set := map[string]bool{} + for _, w := range b { + set[w] = true + } + hit := 0 + for _, w := range a { + if set[w] { + hit++ + } + } + if hit != len(a) { + return 0 + } + return 0.9 // a full subset is strong but never a perfect match +} + +// titleScore is the similarity of two titles in [0,1], or -1 if they are +// different versions of a song. +func titleScore(a, b titleInfo) float64 { + if a.norm == "" || b.norm == "" || !sameVersions(a.versions, b.versions) { + return -1 + } + s := max(levenshteinRatio(a.norm, b.norm), wordContainment(a.words, b.words)) + if a.track > 0 && b.track > 0 { + if a.track == b.track { + s += 0.05 + } else { + s -= 0.05 + } + } + return min(s, 1) +} + +type approxIndex struct { + byAlbum map[string][]string // albumKey -> master rels + byTop map[string][]string // lower(top folder) -> master rels + titles map[string]titleInfo +} + +func buildApproxIndex(files []string) *approxIndex { + ax := &approxIndex{byAlbum: map[string][]string{}, byTop: map[string][]string{}, titles: map[string]titleInfo{}} + for _, rel := range files { + ax.titles[rel] = parseTitle(filepath.Base(rel)) + k := albumKey(rel) + ax.byAlbum[k] = append(ax.byAlbum[k], rel) + if top := firstComponent(rel); top != "" { + t := strings.ToLower(top) + ax.byTop[t] = append(ax.byTop[t], rel) + } + } + return ax +} + +type scored struct { + rel string + score float64 +} + +// approxMargin is how far the best candidate must beat the runner-up. +const approxMargin = 0.05 + +// find returns the best approximate master match for a slave file, plus a +// human-readable reason. The album (disc folders ignored) is searched first, +// then the whole top-level folder. +func (ax *approxIndex) find(relSlave string, threshold float64) (string, string) { + st := parseTitle(filepath.Base(relSlave)) + scopes := []struct { + name string + cands []string + }{ + {"same album", ax.byAlbum[albumKey(relSlave)]}, + } + if top := firstComponent(relSlave); top != "" { + scopes = append(scopes, struct { + name string + cands []string + }{"same top-level folder, different album folder", ax.byTop[strings.ToLower(top)]}) + } + for _, sc := range scopes { + var best []scored + for _, rel := range sc.cands { + if s := titleScore(st, ax.titles[rel]); s >= threshold { + best = append(best, scored{rel, s}) + } + } + if len(best) == 0 { + continue + } + sort.Slice(best, func(i, j int) bool { + if best[i].score != best[j].score { + return best[i].score > best[j].score + } + return extRank(best[i].rel) < extRank(best[j].rel) + }) + // Several formats of the same title in one folder are one song. + top := best[0] + ambiguous := false + for _, o := range best[1:] { + sameSong := filepath.Dir(o.rel) == filepath.Dir(top.rel) && + strings.TrimSuffix(filepath.Base(o.rel), filepath.Ext(o.rel)) == strings.TrimSuffix(filepath.Base(top.rel), filepath.Ext(top.rel)) + if !sameSong && top.score-o.score < approxMargin { + ambiguous = true + break + } + } + if ambiguous { + return "", "" + } + return top.rel, approxReason(relSlave, top.rel, st, ax.titles[top.rel], top.score, sc.name) + } + return "", "" +} + +func approxReason(relSlave, relMaster string, st, mt titleInfo, score float64, scope string) string { + parts := []string{fmt.Sprintf("title %d%% similar", int(score*100+0.5))} + if st.track > 0 && mt.track > 0 { + if st.track == mt.track { + parts = append(parts, fmt.Sprintf("track %02d matches", st.track)) + } else { + parts = append(parts, fmt.Sprintf("track numbers differ (%02d vs %02d)", st.track, mt.track)) + } + } + if filepath.Dir(relSlave) != filepath.Dir(relMaster) && albumKey(relSlave) == albumKey(relMaster) { + parts = append(parts, "disc folder ignored") + } + parts = append(parts, scope) + return strings.Join(parts, " · ") +} diff --git a/internal/engine/approx_test.go b/internal/engine/approx_test.go new file mode 100644 index 0000000..b091eb0 --- /dev/null +++ b/internal/engine/approx_test.go @@ -0,0 +1,126 @@ +package engine + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/cdnutter/playlink/internal/store" +) + +func enableApprox(t *testing.T, ev *env) { + t.Helper() + ev.pair.ApproxMatching = true + ev.pair.ApproxThreshold = 85 + if err := ev.st.SavePair(ctx, &ev.pair); err != nil { + t.Fatal(err) + } +} + +// The user's example: the master album has disc folders, the slave does not. +func TestApproxDiscFolderUpgrade(t *testing.T) { + ev := setup(t, false) + master := ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac", "lossless") + ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 01/07 Macavity.flac", "other") + ev.write(ev.master, "Andrew Lloyd Webber/Evita/03 Memory.flac", "same title, other album") + ev.write(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.mp3", "lossy") + + // Off by default: the title exists on two albums, so nothing matches. + ev.scan() + if m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("approximate matching must be opt-in, got %s/%s", m.Status, m.Kind) + } + + enableApprox(t, ev) + ev.scan() + m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3") + if m.Kind != store.KindApprox || m.Status != store.StatusSimilar || m.RelMaster != "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac" { + t.Fatalf("unexpected match %s %s -> %s", m.Status, m.Kind, m.RelMaster) + } + if !strings.Contains(m.Reason, "disc folder ignored") || !strings.Contains(m.Reason, "track 07 matches") { + t.Fatalf("reason should explain the match, got %q", m.Reason) + } + + // Linking upgrades the slave: it gets the master's FLAC in its own folder. + if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); err != nil { + t.Fatal(err) + } + upgraded := filepath.Join(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.flac") + if !sameInode(t, master, upgraded) { + t.Fatal("slave should now hold a hardlink of the master FLAC") + } +} + +func TestApproxTitleVariations(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + ev.write(ev.master, "Band/Album/CD1/03 - Dont Stop Believin.flac", "a") + ev.write(ev.slave, "Band/Album/3 Don't Stop Believing.mp3", "b") // typo-level difference + ev.write(ev.master, "Band/Album/CD1/04 - Song.flac", "c") + ev.write(ev.slave, "Band/Album/04 Song (Live).mp3", "d") // different version: must not match + ev.write(ev.master, "Band/Album/CD2/05 Totally Different.flac", "e") + ev.write(ev.slave, "Band/Album/05 Something Else.mp3", "f") // too dissimilar + ev.scan() + + if m := ev.match("Band/Album/3 Don't Stop Believing.mp3"); m.Kind != store.KindApprox { + t.Fatalf("near-identical title should match approximately, got %s", m.Kind) + } + if m := ev.match("Band/Album/04 Song (Live).mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("live version must not match the studio track, got %s -> %s", m.Kind, m.RelMaster) + } + if m := ev.match("Band/Album/05 Something Else.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("dissimilar titles must not match, got %s -> %s", m.Kind, m.RelMaster) + } +} + +func TestApproxAmbiguousIsLeftAlone(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + // Two discs each carry an equally good candidate: refuse to guess. + ev.write(ev.master, "Band/Album/Disc 1/Anthem.flac", "a") + ev.write(ev.master, "Band/Album/Disc 2/Anthem.flac", "b") + ev.write(ev.slave, "Band/Album/Anthem.mp3", "c") + ev.scan() + if m := ev.match("Band/Album/Anthem.mp3"); m.Status != store.StatusUnmatched { + t.Fatalf("ambiguous candidates must not be matched, got %s -> %s", m.Kind, m.RelMaster) + } +} + +func TestApproxAlbumFolderRename(t *testing.T) { + ev := setup(t, false) + enableApprox(t, ev) + ev.write(ev.master, "Cast/Cats (Original London Cast)/Disc 2/07 Memory.flac", "a") + ev.write(ev.slave, "Cast/Cats/07. Memory.mp3", "b") + ev.scan() + m := ev.match("Cast/Cats/07. Memory.mp3") + if m.Kind != store.KindApprox || !strings.Contains(m.Reason, "different album folder") { + t.Fatalf("should match across album folder names, got %s %q", m.Kind, m.Reason) + } +} + +func TestTitleScore(t *testing.T) { + cases := []struct { + a, b string + min float64 + max float64 + }{ + {"07 Memory.flac", "07 Memory.mp3", 1, 1}, + {"07 Memory.flac", "08 Memory.mp3", 0.9, 0.96}, + {"Memory.flac", "Memory (Live).flac", -1, -1}, + {"Song (Remix).flac", "Song (Remix).mp3", 1, 1}, + {"Love.flac", "Love Story.flac", 0, 0.6}, + {"Memory from Cats.flac", "Memory from Cats Remastered.flac", 0.85, 1}, + } + for _, c := range cases { + got := titleScore(parseTitle(c.a), parseTitle(c.b)) + if got < c.min-1e-9 || got > c.max+1e-9 { + t.Errorf("titleScore(%q, %q) = %.3f, want [%.2f, %.2f]", c.a, c.b, got, c.min, c.max) + } + } + if k := albumKey("A/Cats/Disc 02/07 Memory.flac"); k != "a/cats" { + t.Errorf("albumKey = %q", k) + } + if k := albumKey("A/Cats/CD2 - Bonus/x.flac"); k != "a/cats" { + t.Errorf("albumKey = %q", k) + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 17e6b4d..a97edf0 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -280,6 +280,12 @@ func (e *Engine) NormalizePair(p *store.Pair) ([]string, error) { if p.RetentionDays < 0 || p.ScanIntervalMinutes < 0 || p.SettleMinutes < 0 { return nil, errors.New("intervals must not be negative") } + if p.ApproxThreshold == 0 { + p.ApproxThreshold = 85 + } + if p.ApproxThreshold < 60 || p.ApproxThreshold > 100 { + return nil, errors.New("approximate match threshold must be between 60 and 100 percent") + } if p.ScanIntervalMinutes > 0 && p.ScanIntervalMinutes < 5 { return nil, errors.New("scan interval must be 0 (manual) or at least 5 minutes") } diff --git a/internal/engine/scan.go b/internal/engine/scan.go index a0f4079..76737fb 100644 --- a/internal/engine/scan.go +++ b/internal/engine/scan.go @@ -104,11 +104,19 @@ type masterIndex struct { lower map[string]string // lower(rel) -> rel byDir map[string][]string // lower(dir) -> rels byTitle map[string][]string // lower(top folder)|normTitle -> rels + + approx *approxIndex // nil unless the pair enables approximate matching + threshold float64 } -func buildIndex(files map[string]fsops.Info) *masterIndex { +func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex { ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}} - for _, rel := range sortedKeys(files) { + rels := sortedKeys(files) + if p.ApproxMatching { + ix.approx = buildApproxIndex(rels) + ix.threshold = float64(p.ApproxThreshold) / 100 + } + for _, rel := range rels { ix.lower[strings.ToLower(rel)] = rel d := strings.ToLower(filepath.Dir(rel)) ix.byDir[d] = append(ix.byDir[d], rel) @@ -130,13 +138,14 @@ func extRank(rel string) int { return 2 } -// find locates the master counterpart of a slave file. -func (ix *masterIndex) find(relSlave string) (string, string) { +// find locates the master counterpart of a slave file. The reason is only +// set for approximate matches. +func (ix *masterIndex) find(relSlave string) (rel, kind, reason string) { if _, ok := ix.files[relSlave]; ok { - return relSlave, store.KindPath + return relSlave, store.KindPath, "" } if r, ok := ix.lower[strings.ToLower(relSlave)]; ok { - return r, store.KindPathCase + return r, store.KindPathCase, "" } dir := strings.ToLower(filepath.Dir(relSlave)) base := filepath.Base(relSlave) @@ -158,17 +167,25 @@ func (ix *masterIndex) find(relSlave string) (string, string) { best = r } } - return best, store.KindExtension + return best, store.KindExtension, "" } if len(fuzzyHits) == 1 { - return fuzzyHits[0], store.KindFuzzy + return fuzzyHits[0], store.KindFuzzy, "" + } + // Approximate matching runs before the top-folder fallback because it + // searches the album (disc folders ignored) first, which resolves titles + // that appear on several albums. + if ix.approx != nil { + if r, why := ix.approx.find(relSlave, ix.threshold); r != "" { + return r, store.KindApprox, why + } } if top := firstComponent(relSlave); top != "" && norm != "" { if hits := ix.byTitle[strings.ToLower(top)+"|"+norm]; len(hits) == 1 { - return hits[0], store.KindElsewhere + return hits[0], store.KindElsewhere, "" } } - return "", store.KindNone + return "", store.KindNone, "" } // classify fills in status, probes and verdict for a match, reusing cached @@ -269,7 +286,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) _ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns) return err } - ix := buildIndex(masterFiles) + ix := buildIndex(masterFiles, l.pair) prev, err := e.st.AllMatches(ctx, pairID) if err != nil { return err @@ -284,7 +301,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) } si := slaveFiles[rel] m := store.Match{PairID: pairID, RelSlave: rel} - m.RelMaster, m.Kind = ix.find(rel) + m.RelMaster, m.Kind, m.Reason = ix.find(rel) var pm *store.Match if p, ok := prev[rel]; ok { pm = &p @@ -304,7 +321,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) // Layout warnings based on what was actually found. if ss.files > 0 { - if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files { + if n := kinds[store.KindElsewhere] + kinds[store.KindApprox] + kinds[store.KindNone]; n*4 > ss.files { warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the master library. "+ "Check that both libraries use the same folder layout.", n, ss.files)) } @@ -426,7 +443,7 @@ func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) { if pm != nil && pm.RelMaster != "" { if in, err := fsops.RegularFile(filepath.Join(l.master.Path, pm.RelMaster)); err == nil { mi = in - m.RelMaster, m.Kind = pm.RelMaster, pm.Kind + m.RelMaster, m.Kind, m.Reason = pm.RelMaster, pm.Kind, pm.Reason } } } diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 4b37915..f8b9c5f 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -206,9 +206,8 @@ func Compare(master, slave Info) Verdict { v.Better = "slave" } if master.Lossless != slave.Lossless { - if master.Lossless { - v.Notes = append(v.Notes, "master is lossless, slave is lossy") - } else { + // Only the downgrade case is a warning; an upgrade is covered by Better. + if !master.Lossless { v.Notes = append(v.Notes, "slave is lossless but master is lossy: linking would downgrade the slave") } } diff --git a/internal/store/models.go b/internal/store/models.go index 3db89f5..aecc1b1 100644 --- a/internal/store/models.go +++ b/internal/store/models.go @@ -104,6 +104,8 @@ type Pair struct { ScanIntervalMinutes int `json:"scan_interval_minutes"` RetentionDays int `json:"retention_days"` SettleMinutes int `json:"settle_minutes"` + ApproxMatching bool `json:"approx_matching"` + ApproxThreshold int `json:"approx_threshold"` // percent LastScanAt *int64 `json:"last_scan_at"` LastScanError string `json:"last_scan_error"` LastScanWarnings []string `json:"last_scan_warnings"` @@ -111,7 +113,8 @@ type Pair struct { } const pairCols = `id, name, master_root, slave_root, quarantine_dir, extensions, auto_link_identical, - scan_interval_minutes, retention_days, settle_minutes, last_scan_at, last_scan_error, last_scan_warnings, created_at` + scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, last_scan_at, last_scan_error, + last_scan_warnings, created_at` type scanner interface{ Scan(...any) error } @@ -120,7 +123,8 @@ func scanPair(r scanner) (Pair, error) { var exts, warns string var last sql.NullInt64 err := r.Scan(&p.ID, &p.Name, &p.MasterRoot, &p.SlaveRoot, &p.QuarantineDir, &exts, &p.AutoLinkIdentical, - &p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &last, &p.LastScanError, &warns, &p.CreatedAt) + &p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &p.ApproxMatching, &p.ApproxThreshold, &last, &p.LastScanError, + &warns, &p.CreatedAt) if err != nil { return p, err } @@ -164,10 +168,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error { exts := strings.Join(p.Extensions, ",") if p.ID == 0 { res, err := s.DB.ExecContext(ctx, `INSERT INTO pairs (name, master_root, slave_root, quarantine_dir, extensions, - auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical, - p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, now()) + p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, now()) if err != nil { return err } @@ -175,10 +179,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error { return nil } _, err := s.DB.ExecContext(ctx, `UPDATE pairs SET name = ?, master_root = ?, slave_root = ?, quarantine_dir = ?, - extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ? - WHERE id = ?`, + extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?, + approx_matching = ?, approx_threshold = ? WHERE id = ?`, p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical, - p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ID) + p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, p.ID) return err } @@ -216,11 +220,12 @@ const ( // Match kinds (how the master candidate was found). const ( - KindPath = "path" // identical relative path - KindPathCase = "path_case" // relative path differs only by case - KindExtension = "extension" // same directory and title, different extension - KindFuzzy = "fuzzy" // same directory, normalized title matches - KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder + KindPath = "path" // identical relative path + KindPathCase = "path_case" // relative path differs only by case + KindExtension = "extension" // same directory and title, different extension + KindFuzzy = "fuzzy" // same directory, normalized title matches + KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder + KindApprox = "approximate" // opt-in: similar title, disc folders ignored KindNone = "none" ) @@ -241,18 +246,19 @@ type Match struct { SlaveProbe json.RawMessage `json:"slave_probe,omitempty"` Verdict json.RawMessage `json:"verdict,omitempty"` Note string `json:"note"` + Reason string `json:"reason"` // why an approximate match was chosen Ignored bool `json:"ignored"` UpdatedAt int64 `json:"updated_at"` } const matchCols = `id, pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, master_mtime, - slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at` + slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at` func scanMatch(r scanner) (Match, error) { var m Match var mp, sp, v string err := r.Scan(&m.ID, &m.PairID, &m.RelSlave, &m.RelMaster, &m.Kind, &m.Status, &m.MasterIno, &m.MasterSize, &m.MasterMtime, - &m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Ignored, &m.UpdatedAt) + &m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Ignored, &m.UpdatedAt) if mp != "" { m.MasterProbe = json.RawMessage(mp) } @@ -316,6 +322,8 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat where = append(where, "status IN ('different','similar') AND ignored = 0") case "ignored": where = append(where, "ignored = 1") + case "upgrade": + where = append(where, "status IN ('different','similar') AND ignored = 0 AND "+upgradeCond) default: where = append(where, "status = ? AND ignored = 0") args = append(args, f.Status) @@ -371,15 +379,15 @@ func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error) func (s *Store) UpsertMatch(ctx context.Context, m *Match) error { nullable := func(r json.RawMessage) string { return string(r) } _, err := s.DB.ExecContext(ctx, `INSERT INTO matches (pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, - master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(pair_id, rel_slave) DO UPDATE SET rel_master = excluded.rel_master, kind = excluded.kind, status = excluded.status, master_ino = excluded.master_ino, master_size = excluded.master_size, master_mtime = excluded.master_mtime, slave_ino = excluded.slave_ino, slave_size = excluded.slave_size, slave_mtime = excluded.slave_mtime, master_probe = excluded.master_probe, slave_probe = excluded.slave_probe, - verdict = excluded.verdict, note = excluded.note, ignored = excluded.ignored, updated_at = excluded.updated_at`, + verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, ignored = excluded.ignored, updated_at = excluded.updated_at`, m.PairID, m.RelSlave, m.RelMaster, m.Kind, m.Status, m.MasterIno, m.MasterSize, m.MasterMtime, m.SlaveIno, m.SlaveSize, - m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Ignored, now()) + m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Ignored, now()) return err } @@ -433,9 +441,14 @@ func (s *Store) LinkedMasterPaths(ctx context.Context, pairID int64, prefix stri return out, rows.Err() } +// upgradeCond selects review matches where the master copy scored better, so +// linking upgrades the slave. +const upgradeCond = `verdict LIKE '%"better":"master"%'` + type Summary struct { Counts map[string]int `json:"counts"` Review int `json:"review"` + Upgrades int `json:"upgrades"` Ignored int `json:"ignored"` ReclaimableSize int64 `json:"reclaimable_bytes"` LinkedSize int64 `json:"linked_bytes"` @@ -470,7 +483,12 @@ func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) { sum.Review += n } } - return sum, rows.Err() + if err := rows.Err(); err != nil { + return sum, err + } + err = s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0 + AND status IN ('different','similar') AND `+upgradeCond, pairID).Scan(&sum.Upgrades) + return sum, err } // ---- operations ---- diff --git a/internal/store/store.go b/internal/store/store.go index bf8d6ab..12ea060 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -104,6 +104,11 @@ var migrations = []string{ UPDATE operations SET quarantine_side = 'master' WHERE quarantine_side = 'host'; UPDATE matches SET verdict = replace(verdict, 'host', 'master'); UPDATE pairs SET last_scan_warnings = replace(last_scan_warnings, 'host', 'master');`, + + // 3: opt-in approximate matching, and the reason shown for such matches. + `ALTER TABLE pairs ADD COLUMN approx_matching INTEGER NOT NULL DEFAULT 0; + ALTER TABLE pairs ADD COLUMN approx_threshold INTEGER NOT NULL DEFAULT 85; + ALTER TABLE matches ADD COLUMN reason TEXT NOT NULL DEFAULT '';`, } // Open opens (and migrates) the database in dataDir. diff --git a/web/src/api.ts b/web/src/api.ts index ab6bfdc..c186e8a 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -54,6 +54,7 @@ export type Job = { export type Summary = { counts: Record; review: number; + upgrades: number; ignored: number; reclaimable_bytes: number; linked_bytes: number; @@ -70,6 +71,8 @@ export type Pair = { scan_interval_minutes: number; retention_days: number; settle_minutes: number; + approx_matching: boolean; + approx_threshold: number; last_scan_at: number | null; last_scan_error: string; last_scan_warnings: string[]; @@ -88,6 +91,8 @@ export type PairInput = Pick< | "scan_interval_minutes" | "retention_days" | "settle_minutes" + | "approx_matching" + | "approx_threshold" >; export type ProbeInfo = { @@ -115,7 +120,7 @@ export type Match = { pair_id: number; rel_slave: string; rel_master: string; - kind: "path" | "path_case" | "extension" | "fuzzy" | "elsewhere" | "none"; + kind: "path" | "path_case" | "extension" | "fuzzy" | "elsewhere" | "approximate" | "none"; status: Status; master_size: number; master_mtime: number; @@ -125,6 +130,7 @@ export type Match = { slave_probe?: ProbeInfo; verdict?: Verdict; note: string; + reason: string; ignored: boolean; }; diff --git a/web/src/pages/PairForm.tsx b/web/src/pages/PairForm.tsx index bd882fc..40ef0ed 100644 --- a/web/src/pages/PairForm.tsx +++ b/web/src/pages/PairForm.tsx @@ -22,6 +22,8 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; scan_interval_minutes: 0, retention_days: 30, settle_minutes: 10, + approx_matching: false, + approx_threshold: 85, }; const [f, setF] = useState(pair ?? blank); const [exts, setExts] = useState((pair?.extensions ?? []).join(", ")); @@ -47,6 +49,8 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; scan_interval_minutes: f.scan_interval_minutes, retention_days: f.retention_days, settle_minutes: f.settle_minutes, + approx_matching: f.approx_matching, + approx_threshold: f.approx_threshold, extensions: exts.split(",").map((s) => s.trim()).filter(Boolean), }); const set = (k: K, v: PairInput[K]) => { @@ -128,6 +132,37 @@ export function PairForm({ pair, open, onClose }: { pair?: Pair; open: boolean; Anything else always waits for manual review. Dry run still applies.

+
+ set("approx_matching", v)} + label={Approximate matching} + /> +

+ Also match songs whose file name is only approximately the same, and ignore disc folders. For example{" "} + Cats/Disc 02/07 Memory.flac matches Cats/07 Memory.mp3. Version words + must agree ("Live", "Remix", "Reprise"…), ambiguous candidates are skipped, and approximate matches always go to manual review. +

+ {f.approx_matching && ( +
+ + set("approx_threshold", Number(e.target.value))} + className="w-40 accent-violet-500" + /> + {f.approx_threshold}% + Lower finds more matches, but also more false positives. 85% is a good default. +
+ )} +
{validation && !validation.ok && {validation.error}} {validation?.ok && validation.warnings.length === 0 && Looks good. Both folders are valid and on the same filesystem.} diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 5888692..f450010 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1,12 +1,13 @@ import { useEffect, useState } from "react"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { AlertTriangle, ArrowRight, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; +import { AlertTriangle, ArrowRight, ArrowUpCircle, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react"; import { ApiError, api, type Match, type Page, type Pair, type ProbeInfo } from "../api"; import { Alert, Badge, Button, Card, Empty, Modal, Spinner, cx, formatBytes, formatDuration, inputCls, statusTone } from "../ui"; import { useToast } from "../toast"; const filters = [ { id: "review", label: "Needs review" }, + { id: "upgrade", label: "Upgrades" }, { id: "identical", label: "Identical" }, { id: "linked", label: "Linked" }, { id: "unmatched", label: "Slave only" }, @@ -21,6 +22,7 @@ const kindLabel: Record = { extension: "other format", fuzzy: "similar title", elsewhere: "same top folder, other subfolder", + approximate: "approximate match", none: "no match", }; @@ -64,7 +66,7 @@ export function Review({ pair }: { pair: Pair }) { const c = pair.summary.counts; const count = (id: string) => - id === "review" ? pair.summary.review : id === "ignored" ? pair.summary.ignored : id === "" ? undefined : c[id] ?? 0; + id === "review" ? pair.summary.review : id === "upgrade" ? pair.summary.upgrades : id === "ignored" ? pair.summary.ignored : id === "" ? undefined : c[id] ?? 0; const identical = c.identical ?? 0; const running = pair.job?.state === "running"; @@ -211,6 +213,7 @@ function MatchRow({ m }: { m: Match }) {
reviewable && setOpen(!open)}> + {m.reason &&
{m.reason}
} {m.note &&
{m.note}
}
{reviewable && ( @@ -223,7 +226,11 @@ function MatchRow({ m }: { m: Match }) { master lower )} - {better === "master" && master better} + {better === "master" && ( + + upgrade + + )} {better === "equal" && equal} )} @@ -266,7 +273,7 @@ const rows: [string, (p: ProbeInfo, size: number) => string][] = [ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { const toast = useToast(); const [confirm, setConfirm] = useState(null); - const hp = m.master_probe; + const mp = m.master_probe; const sp = m.slave_probe; const v = m.verdict; const renamed = m.rel_master.split("/").pop() !== m.rel_slave.split("/").pop(); @@ -295,8 +302,8 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { return (
- {(!hp || !sp) && No codec details available (ffprobe missing or failed). Compare the files yourself before linking.} - {hp && sp && ( + {(!mp || !sp) && No codec details available (ffprobe missing or failed). Compare the files yourself before linking.} + {mp && sp && (
@@ -312,7 +319,7 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { {rows.map(([label, f]) => { - const a = hp.error ? "–" : f(hp, m.master_size); + const a = mp.error ? "–" : f(mp, m.master_size); const b = sp.error ? "–" : f(sp, m.slave_size); return ( @@ -327,11 +334,23 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) { )}
- {v?.notes?.map((n) => ( + {/* Scans before this change stored an upgrade note; the green alert below covers it. */} + {v?.notes?.filter((n) => !n.startsWith("master is lossless")).map((n) => ( }> {n} ))} + {m.kind === "approximate" && ( + + Approximate match: {m.reason}. Check that the titles and durations really are the same song before linking. + + )} + {v?.better === "master" && ( + }> + Recommended: the master copy is better ({qualityLabel(mp)} vs {qualityLabel(sp)}). Linking upgrades the slave to it and frees the + slave's copy. + + )} {v?.better === "slave" && ( }> The master copy is lower quality. The master is normally kept because it belongs to a complete album. You can still link to it (the @@ -347,7 +366,8 @@ function ComparePanel({ m, onDone }: { m: Match; onDone: () => void }) {