Add opt-in approximate matching and upgrade recommendations

A per-pair "Approximate matching" setting adds a second pass for slave
files the strict rules could not place. It searches the same album with
disc folders (Disc 02, CD2, ...) ignored, then the top-level folder, and
scores titles by edit distance and word containment against a configurable
threshold (default 85%), with a small track-number bonus.

Guards against false matches: version words (live, remix, reprise,
instrumental, ...) must agree, near-tied candidates are left unmatched,
and approximate matches always go to manual review, never auto-link.
Each match records a human-readable reason.

Review now flags matches where the master copy is better as upgrades,
with an Upgrades filter and count and an "Upgrade to master copy" action.
The redundant "master is lossless" warning note is dropped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed 2026-09-22 16:58:53 -07:00
1 parent 5b088bb4a6
commit 381b804742
12 files changed
+552 -51

No files matched your search

+3
View File
@@ -306,12 +306,15 @@ type pairInput struct {
ScanIntervalMinutes int `json:"scan_interval_minutes"`
RetentionDays int `json:"retention_days"`
SettleMinutes int `json:"settle_minutes"`
ApproxMatching bool `json:"approx_matching"`
ApproxThreshold int `json:"approx_threshold"`
}
func (in pairInput) apply(p *store.Pair) {
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir = in.Name, in.MasterRoot, in.SlaveRoot, in.QuarantineDir
p.Extensions, p.AutoLinkIdentical = in.Extensions, in.AutoLinkIdentical
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes = in.ScanIntervalMinutes, in.RetentionDays, in.SettleMinutes
p.ApproxMatching, p.ApproxThreshold = in.ApproxMatching, in.ApproxThreshold
}
func (s *Server) handleValidatePair(w http.ResponseWriter, r *http.Request) {
+257
View File
@@ -0,0 +1,257 @@
package engine
import (
"fmt"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"unicode"
)
// Approximate matching is an opt-in, per-pair second pass for slave files the
// strict rules could not place. It tolerates disc subfolders ("Disc 02/",
// "CD2/") and slightly different titles, but never auto-links: every result
// goes to manual review.
// discFolder matches disc/side subfolders such as "Disc 02", "CD2", "Disk 1 - Bonus".
var discFolder = regexp.MustCompile(`(?i)^(disc|disk|cd|dvd|side|vol|volume)[\s._-]*\d+\b.*$`)
// trackNum extracts the leading track number ("07 Memory", "2-07 Memory").
var trackNum = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?(\d{1,3})(?:\s*[-._)]\s*|\s+)`)
// versionWords mark a different recording or arrangement. Two titles only
// match approximately if they carry the same set of these words, so
// "Memory (Live)" never matches "Memory".
var versionWords = map[string]bool{
"live": true, "remix": true, "mix": true, "acoustic": true, "demo": true, "instrumental": true,
"edit": true, "version": true, "reprise": true, "karaoke": true, "cover": true, "extended": true,
"radio": true, "unplugged": true, "orchestral": true, "piano": true, "clean": true, "acapella": true,
"cappella": true, "interlude": true, "intro": true, "outro": true, "skit": true, "vip": true, "dub": true,
"sped": true, "slowed": true, "nightcore": true, "reverb": true, "mashup": true, "bootleg": true,
}
// albumKey is the lower-cased directory with disc folders removed.
func albumKey(rel string) string {
var parts []string
for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) {
if p != "." && !discFolder.MatchString(p) {
parts = append(parts, strings.ToLower(p))
}
}
return strings.Join(parts, "/")
}
func hasDiscFolder(rel string) bool {
for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) {
if discFolder.MatchString(p) {
return true
}
}
return false
}
type titleInfo struct {
norm string // normTitle: letters+digits only
words []string // non-version words
versions map[string]bool // version words present
track int // 0 = unknown
}
func parseTitle(name string) titleInfo {
stem := strings.TrimSuffix(name, filepath.Ext(name))
ti := titleInfo{norm: normTitle(name), versions: map[string]bool{}}
if m := trackNum.FindStringSubmatch(stem); m != nil {
ti.track, _ = strconv.Atoi(m[1])
if s := trackNum.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" {
stem = s
}
}
for _, w := range strings.FieldsFunc(strings.ToLower(stem), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
}) {
if versionWords[w] {
ti.versions[w] = true
} else {
ti.words = append(ti.words, w)
}
}
return ti
}
func sameVersions(a, b map[string]bool) bool {
if len(a) != len(b) {
return false
}
for k := range a {
if !b[k] {
return false
}
}
return true
}
// levenshteinRatio is 1 - editDistance/maxLen over runes.
func levenshteinRatio(a, b string) float64 {
ra, rb := []rune(a), []rune(b)
if len(ra) == 0 || len(rb) == 0 {
return 0
}
prev := make([]int, len(rb)+1)
cur := make([]int, len(rb)+1)
for j := range prev {
prev[j] = j
}
for i := 1; i <= len(ra); i++ {
cur[0] = i
for j := 1; j <= len(rb); j++ {
cost := 1
if ra[i-1] == rb[j-1] {
cost = 0
}
cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
}
prev, cur = cur, prev
}
return 1 - float64(prev[len(rb)])/float64(max(len(ra), len(rb)))
}
// wordContainment scores titles where one adds words to the other
// ("Memory" vs "Memory from Cats"). It needs at least two shared words so a
// one-word title like "Love" cannot swallow "Love Story".
func wordContainment(a, b []string) float64 {
if len(a) > len(b) {
a, b = b, a
}
if len(a) < 2 {
return 0
}
set := map[string]bool{}
for _, w := range b {
set[w] = true
}
hit := 0
for _, w := range a {
if set[w] {
hit++
}
}
if hit != len(a) {
return 0
}
return 0.9 // a full subset is strong but never a perfect match
}
// titleScore is the similarity of two titles in [0,1], or -1 if they are
// different versions of a song.
func titleScore(a, b titleInfo) float64 {
if a.norm == "" || b.norm == "" || !sameVersions(a.versions, b.versions) {
return -1
}
s := max(levenshteinRatio(a.norm, b.norm), wordContainment(a.words, b.words))
if a.track > 0 && b.track > 0 {
if a.track == b.track {
s += 0.05
} else {
s -= 0.05
}
}
return min(s, 1)
}
type approxIndex struct {
byAlbum map[string][]string // albumKey -> master rels
byTop map[string][]string // lower(top folder) -> master rels
titles map[string]titleInfo
}
func buildApproxIndex(files []string) *approxIndex {
ax := &approxIndex{byAlbum: map[string][]string{}, byTop: map[string][]string{}, titles: map[string]titleInfo{}}
for _, rel := range files {
ax.titles[rel] = parseTitle(filepath.Base(rel))
k := albumKey(rel)
ax.byAlbum[k] = append(ax.byAlbum[k], rel)
if top := firstComponent(rel); top != "" {
t := strings.ToLower(top)
ax.byTop[t] = append(ax.byTop[t], rel)
}
}
return ax
}
type scored struct {
rel string
score float64
}
// approxMargin is how far the best candidate must beat the runner-up.
const approxMargin = 0.05
// find returns the best approximate master match for a slave file, plus a
// human-readable reason. The album (disc folders ignored) is searched first,
// then the whole top-level folder.
func (ax *approxIndex) find(relSlave string, threshold float64) (string, string) {
st := parseTitle(filepath.Base(relSlave))
scopes := []struct {
name string
cands []string
}{
{"same album", ax.byAlbum[albumKey(relSlave)]},
}
if top := firstComponent(relSlave); top != "" {
scopes = append(scopes, struct {
name string
cands []string
}{"same top-level folder, different album folder", ax.byTop[strings.ToLower(top)]})
}
for _, sc := range scopes {
var best []scored
for _, rel := range sc.cands {
if s := titleScore(st, ax.titles[rel]); s >= threshold {
best = append(best, scored{rel, s})
}
}
if len(best) == 0 {
continue
}
sort.Slice(best, func(i, j int) bool {
if best[i].score != best[j].score {
return best[i].score > best[j].score
}
return extRank(best[i].rel) < extRank(best[j].rel)
})
// Several formats of the same title in one folder are one song.
top := best[0]
ambiguous := false
for _, o := range best[1:] {
sameSong := filepath.Dir(o.rel) == filepath.Dir(top.rel) &&
strings.TrimSuffix(filepath.Base(o.rel), filepath.Ext(o.rel)) == strings.TrimSuffix(filepath.Base(top.rel), filepath.Ext(top.rel))
if !sameSong && top.score-o.score < approxMargin {
ambiguous = true
break
}
}
if ambiguous {
return "", ""
}
return top.rel, approxReason(relSlave, top.rel, st, ax.titles[top.rel], top.score, sc.name)
}
return "", ""
}
func approxReason(relSlave, relMaster string, st, mt titleInfo, score float64, scope string) string {
parts := []string{fmt.Sprintf("title %d%% similar", int(score*100+0.5))}
if st.track > 0 && mt.track > 0 {
if st.track == mt.track {
parts = append(parts, fmt.Sprintf("track %02d matches", st.track))
} else {
parts = append(parts, fmt.Sprintf("track numbers differ (%02d vs %02d)", st.track, mt.track))
}
}
if filepath.Dir(relSlave) != filepath.Dir(relMaster) && albumKey(relSlave) == albumKey(relMaster) {
parts = append(parts, "disc folder ignored")
}
parts = append(parts, scope)
return strings.Join(parts, " · ")
}
+126
View File
@@ -0,0 +1,126 @@
package engine
import (
"path/filepath"
"strings"
"testing"
"github.com/cdnutter/playlink/internal/store"
)
func enableApprox(t *testing.T, ev *env) {
t.Helper()
ev.pair.ApproxMatching = true
ev.pair.ApproxThreshold = 85
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
t.Fatal(err)
}
}
// The user's example: the master album has disc folders, the slave does not.
func TestApproxDiscFolderUpgrade(t *testing.T) {
ev := setup(t, false)
master := ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac", "lossless")
ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 01/07 Macavity.flac", "other")
ev.write(ev.master, "Andrew Lloyd Webber/Evita/03 Memory.flac", "same title, other album")
ev.write(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.mp3", "lossy")
// Off by default: the title exists on two albums, so nothing matches.
ev.scan()
if m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3"); m.Status != store.StatusUnmatched {
t.Fatalf("approximate matching must be opt-in, got %s/%s", m.Status, m.Kind)
}
enableApprox(t, ev)
ev.scan()
m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3")
if m.Kind != store.KindApprox || m.Status != store.StatusSimilar || m.RelMaster != "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac" {
t.Fatalf("unexpected match %s %s -> %s", m.Status, m.Kind, m.RelMaster)
}
if !strings.Contains(m.Reason, "disc folder ignored") || !strings.Contains(m.Reason, "track 07 matches") {
t.Fatalf("reason should explain the match, got %q", m.Reason)
}
// Linking upgrades the slave: it gets the master's FLAC in its own folder.
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); err != nil {
t.Fatal(err)
}
upgraded := filepath.Join(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.flac")
if !sameInode(t, master, upgraded) {
t.Fatal("slave should now hold a hardlink of the master FLAC")
}
}
func TestApproxTitleVariations(t *testing.T) {
ev := setup(t, false)
enableApprox(t, ev)
ev.write(ev.master, "Band/Album/CD1/03 - Dont Stop Believin.flac", "a")
ev.write(ev.slave, "Band/Album/3 Don't Stop Believing.mp3", "b") // typo-level difference
ev.write(ev.master, "Band/Album/CD1/04 - Song.flac", "c")
ev.write(ev.slave, "Band/Album/04 Song (Live).mp3", "d") // different version: must not match
ev.write(ev.master, "Band/Album/CD2/05 Totally Different.flac", "e")
ev.write(ev.slave, "Band/Album/05 Something Else.mp3", "f") // too dissimilar
ev.scan()
if m := ev.match("Band/Album/3 Don't Stop Believing.mp3"); m.Kind != store.KindApprox {
t.Fatalf("near-identical title should match approximately, got %s", m.Kind)
}
if m := ev.match("Band/Album/04 Song (Live).mp3"); m.Status != store.StatusUnmatched {
t.Fatalf("live version must not match the studio track, got %s -> %s", m.Kind, m.RelMaster)
}
if m := ev.match("Band/Album/05 Something Else.mp3"); m.Status != store.StatusUnmatched {
t.Fatalf("dissimilar titles must not match, got %s -> %s", m.Kind, m.RelMaster)
}
}
func TestApproxAmbiguousIsLeftAlone(t *testing.T) {
ev := setup(t, false)
enableApprox(t, ev)
// Two discs each carry an equally good candidate: refuse to guess.
ev.write(ev.master, "Band/Album/Disc 1/Anthem.flac", "a")
ev.write(ev.master, "Band/Album/Disc 2/Anthem.flac", "b")
ev.write(ev.slave, "Band/Album/Anthem.mp3", "c")
ev.scan()
if m := ev.match("Band/Album/Anthem.mp3"); m.Status != store.StatusUnmatched {
t.Fatalf("ambiguous candidates must not be matched, got %s -> %s", m.Kind, m.RelMaster)
}
}
func TestApproxAlbumFolderRename(t *testing.T) {
ev := setup(t, false)
enableApprox(t, ev)
ev.write(ev.master, "Cast/Cats (Original London Cast)/Disc 2/07 Memory.flac", "a")
ev.write(ev.slave, "Cast/Cats/07. Memory.mp3", "b")
ev.scan()
m := ev.match("Cast/Cats/07. Memory.mp3")
if m.Kind != store.KindApprox || !strings.Contains(m.Reason, "different album folder") {
t.Fatalf("should match across album folder names, got %s %q", m.Kind, m.Reason)
}
}
func TestTitleScore(t *testing.T) {
cases := []struct {
a, b string
min float64
max float64
}{
{"07 Memory.flac", "07 Memory.mp3", 1, 1},
{"07 Memory.flac", "08 Memory.mp3", 0.9, 0.96},
{"Memory.flac", "Memory (Live).flac", -1, -1},
{"Song (Remix).flac", "Song (Remix).mp3", 1, 1},
{"Love.flac", "Love Story.flac", 0, 0.6},
{"Memory from Cats.flac", "Memory from Cats Remastered.flac", 0.85, 1},
}
for _, c := range cases {
got := titleScore(parseTitle(c.a), parseTitle(c.b))
if got < c.min-1e-9 || got > c.max+1e-9 {
t.Errorf("titleScore(%q, %q) = %.3f, want [%.2f, %.2f]", c.a, c.b, got, c.min, c.max)
}
}
if k := albumKey("A/Cats/Disc 02/07 Memory.flac"); k != "a/cats" {
t.Errorf("albumKey = %q", k)
}
if k := albumKey("A/Cats/CD2 - Bonus/x.flac"); k != "a/cats" {
t.Errorf("albumKey = %q", k)
}
}
+6
View File
@@ -280,6 +280,12 @@ func (e *Engine) NormalizePair(p *store.Pair) ([]string, error) {
if p.RetentionDays < 0 || p.ScanIntervalMinutes < 0 || p.SettleMinutes < 0 {
return nil, errors.New("intervals must not be negative")
}
if p.ApproxThreshold == 0 {
p.ApproxThreshold = 85
}
if p.ApproxThreshold < 60 || p.ApproxThreshold > 100 {
return nil, errors.New("approximate match threshold must be between 60 and 100 percent")
}
if p.ScanIntervalMinutes > 0 && p.ScanIntervalMinutes < 5 {
return nil, errors.New("scan interval must be 0 (manual) or at least 5 minutes")
}
+31 -14
View File
@@ -104,11 +104,19 @@ type masterIndex struct {
lower map[string]string // lower(rel) -> rel
byDir map[string][]string // lower(dir) -> rels
byTitle map[string][]string // lower(top folder)|normTitle -> rels
approx *approxIndex // nil unless the pair enables approximate matching
threshold float64
}
func buildIndex(files map[string]fsops.Info) *masterIndex {
func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex {
ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
for _, rel := range sortedKeys(files) {
rels := sortedKeys(files)
if p.ApproxMatching {
ix.approx = buildApproxIndex(rels)
ix.threshold = float64(p.ApproxThreshold) / 100
}
for _, rel := range rels {
ix.lower[strings.ToLower(rel)] = rel
d := strings.ToLower(filepath.Dir(rel))
ix.byDir[d] = append(ix.byDir[d], rel)
@@ -130,13 +138,14 @@ func extRank(rel string) int {
return 2
}
// find locates the master counterpart of a slave file.
func (ix *masterIndex) find(relSlave string) (string, string) {
// find locates the master counterpart of a slave file. The reason is only
// set for approximate matches.
func (ix *masterIndex) find(relSlave string) (rel, kind, reason string) {
if _, ok := ix.files[relSlave]; ok {
return relSlave, store.KindPath
return relSlave, store.KindPath, ""
}
if r, ok := ix.lower[strings.ToLower(relSlave)]; ok {
return r, store.KindPathCase
return r, store.KindPathCase, ""
}
dir := strings.ToLower(filepath.Dir(relSlave))
base := filepath.Base(relSlave)
@@ -158,17 +167,25 @@ func (ix *masterIndex) find(relSlave string) (string, string) {
best = r
}
}
return best, store.KindExtension
return best, store.KindExtension, ""
}
if len(fuzzyHits) == 1 {
return fuzzyHits[0], store.KindFuzzy
return fuzzyHits[0], store.KindFuzzy, ""
}
// Approximate matching runs before the top-folder fallback because it
// searches the album (disc folders ignored) first, which resolves titles
// that appear on several albums.
if ix.approx != nil {
if r, why := ix.approx.find(relSlave, ix.threshold); r != "" {
return r, store.KindApprox, why
}
}
if top := firstComponent(relSlave); top != "" && norm != "" {
if hits := ix.byTitle[strings.ToLower(top)+"|"+norm]; len(hits) == 1 {
return hits[0], store.KindElsewhere
return hits[0], store.KindElsewhere, ""
}
}
return "", store.KindNone
return "", store.KindNone, ""
}
// classify fills in status, probes and verdict for a match, reusing cached
@@ -269,7 +286,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
return err
}
ix := buildIndex(masterFiles)
ix := buildIndex(masterFiles, l.pair)
prev, err := e.st.AllMatches(ctx, pairID)
if err != nil {
return err
@@ -284,7 +301,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
}
si := slaveFiles[rel]
m := store.Match{PairID: pairID, RelSlave: rel}
m.RelMaster, m.Kind = ix.find(rel)
m.RelMaster, m.Kind, m.Reason = ix.find(rel)
var pm *store.Match
if p, ok := prev[rel]; ok {
pm = &p
@@ -304,7 +321,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
// Layout warnings based on what was actually found.
if ss.files > 0 {
if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files {
if n := kinds[store.KindElsewhere] + kinds[store.KindApprox] + kinds[store.KindNone]; n*4 > ss.files {
warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the master library. "+
"Check that both libraries use the same folder layout.", n, ss.files))
}
@@ -426,7 +443,7 @@ func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) {
if pm != nil && pm.RelMaster != "" {
if in, err := fsops.RegularFile(filepath.Join(l.master.Path, pm.RelMaster)); err == nil {
mi = in
m.RelMaster, m.Kind = pm.RelMaster, pm.Kind
m.RelMaster, m.Kind, m.Reason = pm.RelMaster, pm.Kind, pm.Reason
}
}
}
+2 -3
View File
@@ -206,9 +206,8 @@ func Compare(master, slave Info) Verdict {
v.Better = "slave"
}
if master.Lossless != slave.Lossless {
if master.Lossless {
v.Notes = append(v.Notes, "master is lossless, slave is lossy")
} else {
// Only the downgrade case is a warning; an upgrade is covered by Better.
if !master.Lossless {
v.Notes = append(v.Notes, "slave is lossless but master is lossy: linking would downgrade the slave")
}
}
+38 -20
View File
@@ -104,6 +104,8 @@ type Pair struct {
ScanIntervalMinutes int `json:"scan_interval_minutes"`
RetentionDays int `json:"retention_days"`
SettleMinutes int `json:"settle_minutes"`
ApproxMatching bool `json:"approx_matching"`
ApproxThreshold int `json:"approx_threshold"` // percent
LastScanAt *int64 `json:"last_scan_at"`
LastScanError string `json:"last_scan_error"`
LastScanWarnings []string `json:"last_scan_warnings"`
@@ -111,7 +113,8 @@ type Pair struct {
}
const pairCols = `id, name, master_root, slave_root, quarantine_dir, extensions, auto_link_identical,
scan_interval_minutes, retention_days, settle_minutes, last_scan_at, last_scan_error, last_scan_warnings, created_at`
scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, last_scan_at, last_scan_error,
last_scan_warnings, created_at`
type scanner interface{ Scan(...any) error }
@@ -120,7 +123,8 @@ func scanPair(r scanner) (Pair, error) {
var exts, warns string
var last sql.NullInt64
err := r.Scan(&p.ID, &p.Name, &p.MasterRoot, &p.SlaveRoot, &p.QuarantineDir, &exts, &p.AutoLinkIdentical,
&p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &last, &p.LastScanError, &warns, &p.CreatedAt)
&p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &p.ApproxMatching, &p.ApproxThreshold, &last, &p.LastScanError,
&warns, &p.CreatedAt)
if err != nil {
return p, err
}
@@ -164,10 +168,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error {
exts := strings.Join(p.Extensions, ",")
if p.ID == 0 {
res, err := s.DB.ExecContext(ctx, `INSERT INTO pairs (name, master_root, slave_root, quarantine_dir, extensions,
auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical,
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, now())
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, now())
if err != nil {
return err
}
@@ -175,10 +179,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error {
return nil
}
_, err := s.DB.ExecContext(ctx, `UPDATE pairs SET name = ?, master_root = ?, slave_root = ?, quarantine_dir = ?,
extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?
WHERE id = ?`,
extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?,
approx_matching = ?, approx_threshold = ? WHERE id = ?`,
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical,
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ID)
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, p.ID)
return err
}
@@ -216,11 +220,12 @@ const (
// Match kinds (how the master candidate was found).
const (
KindPath = "path" // identical relative path
KindPathCase = "path_case" // relative path differs only by case
KindExtension = "extension" // same directory and title, different extension
KindFuzzy = "fuzzy" // same directory, normalized title matches
KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder
KindPath = "path" // identical relative path
KindPathCase = "path_case" // relative path differs only by case
KindExtension = "extension" // same directory and title, different extension
KindFuzzy = "fuzzy" // same directory, normalized title matches
KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder
KindApprox = "approximate" // opt-in: similar title, disc folders ignored
KindNone = "none"
)
@@ -241,18 +246,19 @@ type Match struct {
SlaveProbe json.RawMessage `json:"slave_probe,omitempty"`
Verdict json.RawMessage `json:"verdict,omitempty"`
Note string `json:"note"`
Reason string `json:"reason"` // why an approximate match was chosen
Ignored bool `json:"ignored"`
UpdatedAt int64 `json:"updated_at"`
}
const matchCols = `id, pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, master_mtime,
slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at`
slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at`
func scanMatch(r scanner) (Match, error) {
var m Match
var mp, sp, v string
err := r.Scan(&m.ID, &m.PairID, &m.RelSlave, &m.RelMaster, &m.Kind, &m.Status, &m.MasterIno, &m.MasterSize, &m.MasterMtime,
&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Ignored, &m.UpdatedAt)
&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Ignored, &m.UpdatedAt)
if mp != "" {
m.MasterProbe = json.RawMessage(mp)
}
@@ -316,6 +322,8 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat
where = append(where, "status IN ('different','similar') AND ignored = 0")
case "ignored":
where = append(where, "ignored = 1")
case "upgrade":
where = append(where, "status IN ('different','similar') AND ignored = 0 AND "+upgradeCond)
default:
where = append(where, "status = ? AND ignored = 0")
args = append(args, f.Status)
@@ -371,15 +379,15 @@ func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error)
func (s *Store) UpsertMatch(ctx context.Context, m *Match) error {
nullable := func(r json.RawMessage) string { return string(r) }
_, err := s.DB.ExecContext(ctx, `INSERT INTO matches (pair_id, rel_slave, rel_master, kind, status, master_ino, master_size,
master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(pair_id, rel_slave) DO UPDATE SET rel_master = excluded.rel_master, kind = excluded.kind,
status = excluded.status, master_ino = excluded.master_ino, master_size = excluded.master_size,
master_mtime = excluded.master_mtime, slave_ino = excluded.slave_ino, slave_size = excluded.slave_size,
slave_mtime = excluded.slave_mtime, master_probe = excluded.master_probe, slave_probe = excluded.slave_probe,
verdict = excluded.verdict, note = excluded.note, ignored = excluded.ignored, updated_at = excluded.updated_at`,
verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, ignored = excluded.ignored, updated_at = excluded.updated_at`,
m.PairID, m.RelSlave, m.RelMaster, m.Kind, m.Status, m.MasterIno, m.MasterSize, m.MasterMtime, m.SlaveIno, m.SlaveSize,
m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Ignored, now())
m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Ignored, now())
return err
}
@@ -433,9 +441,14 @@ func (s *Store) LinkedMasterPaths(ctx context.Context, pairID int64, prefix stri
return out, rows.Err()
}
// upgradeCond selects review matches where the master copy scored better, so
// linking upgrades the slave.
const upgradeCond = `verdict LIKE '%"better":"master"%'`
type Summary struct {
Counts map[string]int `json:"counts"`
Review int `json:"review"`
Upgrades int `json:"upgrades"`
Ignored int `json:"ignored"`
ReclaimableSize int64 `json:"reclaimable_bytes"`
LinkedSize int64 `json:"linked_bytes"`
@@ -470,7 +483,12 @@ func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) {
sum.Review += n
}
}
return sum, rows.Err()
if err := rows.Err(); err != nil {
return sum, err
}
err = s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0
AND status IN ('different','similar') AND `+upgradeCond, pairID).Scan(&sum.Upgrades)
return sum, err
}
// ---- operations ----
+5
View File
@@ -104,6 +104,11 @@ var migrations = []string{
UPDATE operations SET quarantine_side = 'master' WHERE quarantine_side = 'host';
UPDATE matches SET verdict = replace(verdict, 'host', 'master');
UPDATE pairs SET last_scan_warnings = replace(last_scan_warnings, 'host', 'master');`,
// 3: opt-in approximate matching, and the reason shown for such matches.
`ALTER TABLE pairs ADD COLUMN approx_matching INTEGER NOT NULL DEFAULT 0;
ALTER TABLE pairs ADD COLUMN approx_threshold INTEGER NOT NULL DEFAULT 85;
ALTER TABLE matches ADD COLUMN reason TEXT NOT NULL DEFAULT '';`,
}
// Open opens (and migrates) the database in dataDir.