Add opt-in approximate matching and upgrade recommendations
A per-pair "Approximate matching" setting adds a second pass for slave files the strict rules could not place. It searches the same album with disc folders (Disc 02, CD2, ...) ignored, then the top-level folder, and scores titles by edit distance and word containment against a configurable threshold (default 85%), with a small track-number bonus. Guards against false matches: version words (live, remix, reprise, instrumental, ...) must agree, near-tied candidates are left unmatched, and approximate matches always go to manual review, never auto-link. Each match records a human-readable reason. Review now flags matches where the master copy is better as upgrades, with an Upgrades filter and count and an "Upgrade to master copy" action. The redundant "master is lossless" warning note is dropped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5b088bb4a6
commit
381b804742
12 files changed
+552
-51
No files matched your search
@@ -306,12 +306,15 @@ type pairInput struct {
|
||||
ScanIntervalMinutes int `json:"scan_interval_minutes"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
SettleMinutes int `json:"settle_minutes"`
|
||||
ApproxMatching bool `json:"approx_matching"`
|
||||
ApproxThreshold int `json:"approx_threshold"`
|
||||
}
|
||||
|
||||
func (in pairInput) apply(p *store.Pair) {
|
||||
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir = in.Name, in.MasterRoot, in.SlaveRoot, in.QuarantineDir
|
||||
p.Extensions, p.AutoLinkIdentical = in.Extensions, in.AutoLinkIdentical
|
||||
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes = in.ScanIntervalMinutes, in.RetentionDays, in.SettleMinutes
|
||||
p.ApproxMatching, p.ApproxThreshold = in.ApproxMatching, in.ApproxThreshold
|
||||
}
|
||||
|
||||
func (s *Server) handleValidatePair(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Approximate matching is an opt-in, per-pair second pass for slave files the
|
||||
// strict rules could not place. It tolerates disc subfolders ("Disc 02/",
|
||||
// "CD2/") and slightly different titles, but never auto-links: every result
|
||||
// goes to manual review.
|
||||
|
||||
// discFolder matches disc/side subfolders such as "Disc 02", "CD2", "Disk 1 - Bonus".
|
||||
var discFolder = regexp.MustCompile(`(?i)^(disc|disk|cd|dvd|side|vol|volume)[\s._-]*\d+\b.*$`)
|
||||
|
||||
// trackNum extracts the leading track number ("07 Memory", "2-07 Memory").
|
||||
var trackNum = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?(\d{1,3})(?:\s*[-._)]\s*|\s+)`)
|
||||
|
||||
// versionWords mark a different recording or arrangement. Two titles only
|
||||
// match approximately if they carry the same set of these words, so
|
||||
// "Memory (Live)" never matches "Memory".
|
||||
var versionWords = map[string]bool{
|
||||
"live": true, "remix": true, "mix": true, "acoustic": true, "demo": true, "instrumental": true,
|
||||
"edit": true, "version": true, "reprise": true, "karaoke": true, "cover": true, "extended": true,
|
||||
"radio": true, "unplugged": true, "orchestral": true, "piano": true, "clean": true, "acapella": true,
|
||||
"cappella": true, "interlude": true, "intro": true, "outro": true, "skit": true, "vip": true, "dub": true,
|
||||
"sped": true, "slowed": true, "nightcore": true, "reverb": true, "mashup": true, "bootleg": true,
|
||||
}
|
||||
|
||||
// albumKey is the lower-cased directory with disc folders removed.
|
||||
func albumKey(rel string) string {
|
||||
var parts []string
|
||||
for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) {
|
||||
if p != "." && !discFolder.MatchString(p) {
|
||||
parts = append(parts, strings.ToLower(p))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func hasDiscFolder(rel string) bool {
|
||||
for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) {
|
||||
if discFolder.MatchString(p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type titleInfo struct {
|
||||
norm string // normTitle: letters+digits only
|
||||
words []string // non-version words
|
||||
versions map[string]bool // version words present
|
||||
track int // 0 = unknown
|
||||
}
|
||||
|
||||
func parseTitle(name string) titleInfo {
|
||||
stem := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
ti := titleInfo{norm: normTitle(name), versions: map[string]bool{}}
|
||||
if m := trackNum.FindStringSubmatch(stem); m != nil {
|
||||
ti.track, _ = strconv.Atoi(m[1])
|
||||
if s := trackNum.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" {
|
||||
stem = s
|
||||
}
|
||||
}
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(stem), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
}) {
|
||||
if versionWords[w] {
|
||||
ti.versions[w] = true
|
||||
} else {
|
||||
ti.words = append(ti.words, w)
|
||||
}
|
||||
}
|
||||
return ti
|
||||
}
|
||||
|
||||
func sameVersions(a, b map[string]bool) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k := range a {
|
||||
if !b[k] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// levenshteinRatio is 1 - editDistance/maxLen over runes.
|
||||
func levenshteinRatio(a, b string) float64 {
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
if len(ra) == 0 || len(rb) == 0 {
|
||||
return 0
|
||||
}
|
||||
prev := make([]int, len(rb)+1)
|
||||
cur := make([]int, len(rb)+1)
|
||||
for j := range prev {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= len(ra); i++ {
|
||||
cur[0] = i
|
||||
for j := 1; j <= len(rb); j++ {
|
||||
cost := 1
|
||||
if ra[i-1] == rb[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
cur[j] = min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
return 1 - float64(prev[len(rb)])/float64(max(len(ra), len(rb)))
|
||||
}
|
||||
|
||||
// wordContainment scores titles where one adds words to the other
|
||||
// ("Memory" vs "Memory from Cats"). It needs at least two shared words so a
|
||||
// one-word title like "Love" cannot swallow "Love Story".
|
||||
func wordContainment(a, b []string) float64 {
|
||||
if len(a) > len(b) {
|
||||
a, b = b, a
|
||||
}
|
||||
if len(a) < 2 {
|
||||
return 0
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for _, w := range b {
|
||||
set[w] = true
|
||||
}
|
||||
hit := 0
|
||||
for _, w := range a {
|
||||
if set[w] {
|
||||
hit++
|
||||
}
|
||||
}
|
||||
if hit != len(a) {
|
||||
return 0
|
||||
}
|
||||
return 0.9 // a full subset is strong but never a perfect match
|
||||
}
|
||||
|
||||
// titleScore is the similarity of two titles in [0,1], or -1 if they are
|
||||
// different versions of a song.
|
||||
func titleScore(a, b titleInfo) float64 {
|
||||
if a.norm == "" || b.norm == "" || !sameVersions(a.versions, b.versions) {
|
||||
return -1
|
||||
}
|
||||
s := max(levenshteinRatio(a.norm, b.norm), wordContainment(a.words, b.words))
|
||||
if a.track > 0 && b.track > 0 {
|
||||
if a.track == b.track {
|
||||
s += 0.05
|
||||
} else {
|
||||
s -= 0.05
|
||||
}
|
||||
}
|
||||
return min(s, 1)
|
||||
}
|
||||
|
||||
type approxIndex struct {
|
||||
byAlbum map[string][]string // albumKey -> master rels
|
||||
byTop map[string][]string // lower(top folder) -> master rels
|
||||
titles map[string]titleInfo
|
||||
}
|
||||
|
||||
func buildApproxIndex(files []string) *approxIndex {
|
||||
ax := &approxIndex{byAlbum: map[string][]string{}, byTop: map[string][]string{}, titles: map[string]titleInfo{}}
|
||||
for _, rel := range files {
|
||||
ax.titles[rel] = parseTitle(filepath.Base(rel))
|
||||
k := albumKey(rel)
|
||||
ax.byAlbum[k] = append(ax.byAlbum[k], rel)
|
||||
if top := firstComponent(rel); top != "" {
|
||||
t := strings.ToLower(top)
|
||||
ax.byTop[t] = append(ax.byTop[t], rel)
|
||||
}
|
||||
}
|
||||
return ax
|
||||
}
|
||||
|
||||
type scored struct {
|
||||
rel string
|
||||
score float64
|
||||
}
|
||||
|
||||
// approxMargin is how far the best candidate must beat the runner-up.
|
||||
const approxMargin = 0.05
|
||||
|
||||
// find returns the best approximate master match for a slave file, plus a
|
||||
// human-readable reason. The album (disc folders ignored) is searched first,
|
||||
// then the whole top-level folder.
|
||||
func (ax *approxIndex) find(relSlave string, threshold float64) (string, string) {
|
||||
st := parseTitle(filepath.Base(relSlave))
|
||||
scopes := []struct {
|
||||
name string
|
||||
cands []string
|
||||
}{
|
||||
{"same album", ax.byAlbum[albumKey(relSlave)]},
|
||||
}
|
||||
if top := firstComponent(relSlave); top != "" {
|
||||
scopes = append(scopes, struct {
|
||||
name string
|
||||
cands []string
|
||||
}{"same top-level folder, different album folder", ax.byTop[strings.ToLower(top)]})
|
||||
}
|
||||
for _, sc := range scopes {
|
||||
var best []scored
|
||||
for _, rel := range sc.cands {
|
||||
if s := titleScore(st, ax.titles[rel]); s >= threshold {
|
||||
best = append(best, scored{rel, s})
|
||||
}
|
||||
}
|
||||
if len(best) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(best, func(i, j int) bool {
|
||||
if best[i].score != best[j].score {
|
||||
return best[i].score > best[j].score
|
||||
}
|
||||
return extRank(best[i].rel) < extRank(best[j].rel)
|
||||
})
|
||||
// Several formats of the same title in one folder are one song.
|
||||
top := best[0]
|
||||
ambiguous := false
|
||||
for _, o := range best[1:] {
|
||||
sameSong := filepath.Dir(o.rel) == filepath.Dir(top.rel) &&
|
||||
strings.TrimSuffix(filepath.Base(o.rel), filepath.Ext(o.rel)) == strings.TrimSuffix(filepath.Base(top.rel), filepath.Ext(top.rel))
|
||||
if !sameSong && top.score-o.score < approxMargin {
|
||||
ambiguous = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ambiguous {
|
||||
return "", ""
|
||||
}
|
||||
return top.rel, approxReason(relSlave, top.rel, st, ax.titles[top.rel], top.score, sc.name)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func approxReason(relSlave, relMaster string, st, mt titleInfo, score float64, scope string) string {
|
||||
parts := []string{fmt.Sprintf("title %d%% similar", int(score*100+0.5))}
|
||||
if st.track > 0 && mt.track > 0 {
|
||||
if st.track == mt.track {
|
||||
parts = append(parts, fmt.Sprintf("track %02d matches", st.track))
|
||||
} else {
|
||||
parts = append(parts, fmt.Sprintf("track numbers differ (%02d vs %02d)", st.track, mt.track))
|
||||
}
|
||||
}
|
||||
if filepath.Dir(relSlave) != filepath.Dir(relMaster) && albumKey(relSlave) == albumKey(relMaster) {
|
||||
parts = append(parts, "disc folder ignored")
|
||||
}
|
||||
parts = append(parts, scope)
|
||||
return strings.Join(parts, " · ")
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/cdnutter/playlink/internal/store"
|
||||
)
|
||||
|
||||
func enableApprox(t *testing.T, ev *env) {
|
||||
t.Helper()
|
||||
ev.pair.ApproxMatching = true
|
||||
ev.pair.ApproxThreshold = 85
|
||||
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The user's example: the master album has disc folders, the slave does not.
|
||||
func TestApproxDiscFolderUpgrade(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
master := ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac", "lossless")
|
||||
ev.write(ev.master, "Andrew Lloyd Webber/Cats/Disc 01/07 Macavity.flac", "other")
|
||||
ev.write(ev.master, "Andrew Lloyd Webber/Evita/03 Memory.flac", "same title, other album")
|
||||
ev.write(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.mp3", "lossy")
|
||||
|
||||
// Off by default: the title exists on two albums, so nothing matches.
|
||||
ev.scan()
|
||||
if m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3"); m.Status != store.StatusUnmatched {
|
||||
t.Fatalf("approximate matching must be opt-in, got %s/%s", m.Status, m.Kind)
|
||||
}
|
||||
|
||||
enableApprox(t, ev)
|
||||
ev.scan()
|
||||
m := ev.match("Andrew Lloyd Webber/Cats/07 Memory.mp3")
|
||||
if m.Kind != store.KindApprox || m.Status != store.StatusSimilar || m.RelMaster != "Andrew Lloyd Webber/Cats/Disc 02/07 Memory.flac" {
|
||||
t.Fatalf("unexpected match %s %s -> %s", m.Status, m.Kind, m.RelMaster)
|
||||
}
|
||||
if !strings.Contains(m.Reason, "disc folder ignored") || !strings.Contains(m.Reason, "track 07 matches") {
|
||||
t.Fatalf("reason should explain the match, got %q", m.Reason)
|
||||
}
|
||||
|
||||
// Linking upgrades the slave: it gets the master's FLAC in its own folder.
|
||||
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
upgraded := filepath.Join(ev.slave, "Andrew Lloyd Webber/Cats/07 Memory.flac")
|
||||
if !sameInode(t, master, upgraded) {
|
||||
t.Fatal("slave should now hold a hardlink of the master FLAC")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxTitleVariations(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
enableApprox(t, ev)
|
||||
ev.write(ev.master, "Band/Album/CD1/03 - Dont Stop Believin.flac", "a")
|
||||
ev.write(ev.slave, "Band/Album/3 Don't Stop Believing.mp3", "b") // typo-level difference
|
||||
ev.write(ev.master, "Band/Album/CD1/04 - Song.flac", "c")
|
||||
ev.write(ev.slave, "Band/Album/04 Song (Live).mp3", "d") // different version: must not match
|
||||
ev.write(ev.master, "Band/Album/CD2/05 Totally Different.flac", "e")
|
||||
ev.write(ev.slave, "Band/Album/05 Something Else.mp3", "f") // too dissimilar
|
||||
ev.scan()
|
||||
|
||||
if m := ev.match("Band/Album/3 Don't Stop Believing.mp3"); m.Kind != store.KindApprox {
|
||||
t.Fatalf("near-identical title should match approximately, got %s", m.Kind)
|
||||
}
|
||||
if m := ev.match("Band/Album/04 Song (Live).mp3"); m.Status != store.StatusUnmatched {
|
||||
t.Fatalf("live version must not match the studio track, got %s -> %s", m.Kind, m.RelMaster)
|
||||
}
|
||||
if m := ev.match("Band/Album/05 Something Else.mp3"); m.Status != store.StatusUnmatched {
|
||||
t.Fatalf("dissimilar titles must not match, got %s -> %s", m.Kind, m.RelMaster)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxAmbiguousIsLeftAlone(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
enableApprox(t, ev)
|
||||
// Two discs each carry an equally good candidate: refuse to guess.
|
||||
ev.write(ev.master, "Band/Album/Disc 1/Anthem.flac", "a")
|
||||
ev.write(ev.master, "Band/Album/Disc 2/Anthem.flac", "b")
|
||||
ev.write(ev.slave, "Band/Album/Anthem.mp3", "c")
|
||||
ev.scan()
|
||||
if m := ev.match("Band/Album/Anthem.mp3"); m.Status != store.StatusUnmatched {
|
||||
t.Fatalf("ambiguous candidates must not be matched, got %s -> %s", m.Kind, m.RelMaster)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApproxAlbumFolderRename(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
enableApprox(t, ev)
|
||||
ev.write(ev.master, "Cast/Cats (Original London Cast)/Disc 2/07 Memory.flac", "a")
|
||||
ev.write(ev.slave, "Cast/Cats/07. Memory.mp3", "b")
|
||||
ev.scan()
|
||||
m := ev.match("Cast/Cats/07. Memory.mp3")
|
||||
if m.Kind != store.KindApprox || !strings.Contains(m.Reason, "different album folder") {
|
||||
t.Fatalf("should match across album folder names, got %s %q", m.Kind, m.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleScore(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b string
|
||||
min float64
|
||||
max float64
|
||||
}{
|
||||
{"07 Memory.flac", "07 Memory.mp3", 1, 1},
|
||||
{"07 Memory.flac", "08 Memory.mp3", 0.9, 0.96},
|
||||
{"Memory.flac", "Memory (Live).flac", -1, -1},
|
||||
{"Song (Remix).flac", "Song (Remix).mp3", 1, 1},
|
||||
{"Love.flac", "Love Story.flac", 0, 0.6},
|
||||
{"Memory from Cats.flac", "Memory from Cats Remastered.flac", 0.85, 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := titleScore(parseTitle(c.a), parseTitle(c.b))
|
||||
if got < c.min-1e-9 || got > c.max+1e-9 {
|
||||
t.Errorf("titleScore(%q, %q) = %.3f, want [%.2f, %.2f]", c.a, c.b, got, c.min, c.max)
|
||||
}
|
||||
}
|
||||
if k := albumKey("A/Cats/Disc 02/07 Memory.flac"); k != "a/cats" {
|
||||
t.Errorf("albumKey = %q", k)
|
||||
}
|
||||
if k := albumKey("A/Cats/CD2 - Bonus/x.flac"); k != "a/cats" {
|
||||
t.Errorf("albumKey = %q", k)
|
||||
}
|
||||
}
|
||||
@@ -280,6 +280,12 @@ func (e *Engine) NormalizePair(p *store.Pair) ([]string, error) {
|
||||
if p.RetentionDays < 0 || p.ScanIntervalMinutes < 0 || p.SettleMinutes < 0 {
|
||||
return nil, errors.New("intervals must not be negative")
|
||||
}
|
||||
if p.ApproxThreshold == 0 {
|
||||
p.ApproxThreshold = 85
|
||||
}
|
||||
if p.ApproxThreshold < 60 || p.ApproxThreshold > 100 {
|
||||
return nil, errors.New("approximate match threshold must be between 60 and 100 percent")
|
||||
}
|
||||
if p.ScanIntervalMinutes > 0 && p.ScanIntervalMinutes < 5 {
|
||||
return nil, errors.New("scan interval must be 0 (manual) or at least 5 minutes")
|
||||
}
|
||||
|
||||
+31
-14
@@ -104,11 +104,19 @@ type masterIndex struct {
|
||||
lower map[string]string // lower(rel) -> rel
|
||||
byDir map[string][]string // lower(dir) -> rels
|
||||
byTitle map[string][]string // lower(top folder)|normTitle -> rels
|
||||
|
||||
approx *approxIndex // nil unless the pair enables approximate matching
|
||||
threshold float64
|
||||
}
|
||||
|
||||
func buildIndex(files map[string]fsops.Info) *masterIndex {
|
||||
func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex {
|
||||
ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
|
||||
for _, rel := range sortedKeys(files) {
|
||||
rels := sortedKeys(files)
|
||||
if p.ApproxMatching {
|
||||
ix.approx = buildApproxIndex(rels)
|
||||
ix.threshold = float64(p.ApproxThreshold) / 100
|
||||
}
|
||||
for _, rel := range rels {
|
||||
ix.lower[strings.ToLower(rel)] = rel
|
||||
d := strings.ToLower(filepath.Dir(rel))
|
||||
ix.byDir[d] = append(ix.byDir[d], rel)
|
||||
@@ -130,13 +138,14 @@ func extRank(rel string) int {
|
||||
return 2
|
||||
}
|
||||
|
||||
// find locates the master counterpart of a slave file.
|
||||
func (ix *masterIndex) find(relSlave string) (string, string) {
|
||||
// find locates the master counterpart of a slave file. The reason is only
|
||||
// set for approximate matches.
|
||||
func (ix *masterIndex) find(relSlave string) (rel, kind, reason string) {
|
||||
if _, ok := ix.files[relSlave]; ok {
|
||||
return relSlave, store.KindPath
|
||||
return relSlave, store.KindPath, ""
|
||||
}
|
||||
if r, ok := ix.lower[strings.ToLower(relSlave)]; ok {
|
||||
return r, store.KindPathCase
|
||||
return r, store.KindPathCase, ""
|
||||
}
|
||||
dir := strings.ToLower(filepath.Dir(relSlave))
|
||||
base := filepath.Base(relSlave)
|
||||
@@ -158,17 +167,25 @@ func (ix *masterIndex) find(relSlave string) (string, string) {
|
||||
best = r
|
||||
}
|
||||
}
|
||||
return best, store.KindExtension
|
||||
return best, store.KindExtension, ""
|
||||
}
|
||||
if len(fuzzyHits) == 1 {
|
||||
return fuzzyHits[0], store.KindFuzzy
|
||||
return fuzzyHits[0], store.KindFuzzy, ""
|
||||
}
|
||||
// Approximate matching runs before the top-folder fallback because it
|
||||
// searches the album (disc folders ignored) first, which resolves titles
|
||||
// that appear on several albums.
|
||||
if ix.approx != nil {
|
||||
if r, why := ix.approx.find(relSlave, ix.threshold); r != "" {
|
||||
return r, store.KindApprox, why
|
||||
}
|
||||
}
|
||||
if top := firstComponent(relSlave); top != "" && norm != "" {
|
||||
if hits := ix.byTitle[strings.ToLower(top)+"|"+norm]; len(hits) == 1 {
|
||||
return hits[0], store.KindElsewhere
|
||||
return hits[0], store.KindElsewhere, ""
|
||||
}
|
||||
}
|
||||
return "", store.KindNone
|
||||
return "", store.KindNone, ""
|
||||
}
|
||||
|
||||
// classify fills in status, probes and verdict for a match, reusing cached
|
||||
@@ -269,7 +286,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
|
||||
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
|
||||
return err
|
||||
}
|
||||
ix := buildIndex(masterFiles)
|
||||
ix := buildIndex(masterFiles, l.pair)
|
||||
prev, err := e.st.AllMatches(ctx, pairID)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -284,7 +301,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
|
||||
}
|
||||
si := slaveFiles[rel]
|
||||
m := store.Match{PairID: pairID, RelSlave: rel}
|
||||
m.RelMaster, m.Kind = ix.find(rel)
|
||||
m.RelMaster, m.Kind, m.Reason = ix.find(rel)
|
||||
var pm *store.Match
|
||||
if p, ok := prev[rel]; ok {
|
||||
pm = &p
|
||||
@@ -304,7 +321,7 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
|
||||
|
||||
// Layout warnings based on what was actually found.
|
||||
if ss.files > 0 {
|
||||
if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files {
|
||||
if n := kinds[store.KindElsewhere] + kinds[store.KindApprox] + kinds[store.KindNone]; n*4 > ss.files {
|
||||
warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the master library. "+
|
||||
"Check that both libraries use the same folder layout.", n, ss.files))
|
||||
}
|
||||
@@ -426,7 +443,7 @@ func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) {
|
||||
if pm != nil && pm.RelMaster != "" {
|
||||
if in, err := fsops.RegularFile(filepath.Join(l.master.Path, pm.RelMaster)); err == nil {
|
||||
mi = in
|
||||
m.RelMaster, m.Kind = pm.RelMaster, pm.Kind
|
||||
m.RelMaster, m.Kind, m.Reason = pm.RelMaster, pm.Kind, pm.Reason
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,9 +206,8 @@ func Compare(master, slave Info) Verdict {
|
||||
v.Better = "slave"
|
||||
}
|
||||
if master.Lossless != slave.Lossless {
|
||||
if master.Lossless {
|
||||
v.Notes = append(v.Notes, "master is lossless, slave is lossy")
|
||||
} else {
|
||||
// Only the downgrade case is a warning; an upgrade is covered by Better.
|
||||
if !master.Lossless {
|
||||
v.Notes = append(v.Notes, "slave is lossless but master is lossy: linking would downgrade the slave")
|
||||
}
|
||||
}
|
||||
|
||||
+38
-20
@@ -104,6 +104,8 @@ type Pair struct {
|
||||
ScanIntervalMinutes int `json:"scan_interval_minutes"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
SettleMinutes int `json:"settle_minutes"`
|
||||
ApproxMatching bool `json:"approx_matching"`
|
||||
ApproxThreshold int `json:"approx_threshold"` // percent
|
||||
LastScanAt *int64 `json:"last_scan_at"`
|
||||
LastScanError string `json:"last_scan_error"`
|
||||
LastScanWarnings []string `json:"last_scan_warnings"`
|
||||
@@ -111,7 +113,8 @@ type Pair struct {
|
||||
}
|
||||
|
||||
const pairCols = `id, name, master_root, slave_root, quarantine_dir, extensions, auto_link_identical,
|
||||
scan_interval_minutes, retention_days, settle_minutes, last_scan_at, last_scan_error, last_scan_warnings, created_at`
|
||||
scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, last_scan_at, last_scan_error,
|
||||
last_scan_warnings, created_at`
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
@@ -120,7 +123,8 @@ func scanPair(r scanner) (Pair, error) {
|
||||
var exts, warns string
|
||||
var last sql.NullInt64
|
||||
err := r.Scan(&p.ID, &p.Name, &p.MasterRoot, &p.SlaveRoot, &p.QuarantineDir, &exts, &p.AutoLinkIdentical,
|
||||
&p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &last, &p.LastScanError, &warns, &p.CreatedAt)
|
||||
&p.ScanIntervalMinutes, &p.RetentionDays, &p.SettleMinutes, &p.ApproxMatching, &p.ApproxThreshold, &last, &p.LastScanError,
|
||||
&warns, &p.CreatedAt)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
@@ -164,10 +168,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error {
|
||||
exts := strings.Join(p.Extensions, ",")
|
||||
if p.ID == 0 {
|
||||
res, err := s.DB.ExecContext(ctx, `INSERT INTO pairs (name, master_root, slave_root, quarantine_dir, extensions,
|
||||
auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
auto_link_identical, scan_interval_minutes, retention_days, settle_minutes, approx_matching, approx_threshold, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical,
|
||||
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, now())
|
||||
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -175,10 +179,10 @@ func (s *Store) SavePair(ctx context.Context, p *Pair) error {
|
||||
return nil
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE pairs SET name = ?, master_root = ?, slave_root = ?, quarantine_dir = ?,
|
||||
extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?
|
||||
WHERE id = ?`,
|
||||
extensions = ?, auto_link_identical = ?, scan_interval_minutes = ?, retention_days = ?, settle_minutes = ?,
|
||||
approx_matching = ?, approx_threshold = ? WHERE id = ?`,
|
||||
p.Name, p.MasterRoot, p.SlaveRoot, p.QuarantineDir, exts, p.AutoLinkIdentical,
|
||||
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ID)
|
||||
p.ScanIntervalMinutes, p.RetentionDays, p.SettleMinutes, p.ApproxMatching, p.ApproxThreshold, p.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -216,11 +220,12 @@ const (
|
||||
|
||||
// Match kinds (how the master candidate was found).
|
||||
const (
|
||||
KindPath = "path" // identical relative path
|
||||
KindPathCase = "path_case" // relative path differs only by case
|
||||
KindExtension = "extension" // same directory and title, different extension
|
||||
KindFuzzy = "fuzzy" // same directory, normalized title matches
|
||||
KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder
|
||||
KindPath = "path" // identical relative path
|
||||
KindPathCase = "path_case" // relative path differs only by case
|
||||
KindExtension = "extension" // same directory and title, different extension
|
||||
KindFuzzy = "fuzzy" // same directory, normalized title matches
|
||||
KindElsewhere = "elsewhere" // same top-level folder and title, different subfolder
|
||||
KindApprox = "approximate" // opt-in: similar title, disc folders ignored
|
||||
KindNone = "none"
|
||||
)
|
||||
|
||||
@@ -241,18 +246,19 @@ type Match struct {
|
||||
SlaveProbe json.RawMessage `json:"slave_probe,omitempty"`
|
||||
Verdict json.RawMessage `json:"verdict,omitempty"`
|
||||
Note string `json:"note"`
|
||||
Reason string `json:"reason"` // why an approximate match was chosen
|
||||
Ignored bool `json:"ignored"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
const matchCols = `id, pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, master_mtime,
|
||||
slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at`
|
||||
slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at`
|
||||
|
||||
func scanMatch(r scanner) (Match, error) {
|
||||
var m Match
|
||||
var mp, sp, v string
|
||||
err := r.Scan(&m.ID, &m.PairID, &m.RelSlave, &m.RelMaster, &m.Kind, &m.Status, &m.MasterIno, &m.MasterSize, &m.MasterMtime,
|
||||
&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Ignored, &m.UpdatedAt)
|
||||
&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Ignored, &m.UpdatedAt)
|
||||
if mp != "" {
|
||||
m.MasterProbe = json.RawMessage(mp)
|
||||
}
|
||||
@@ -316,6 +322,8 @@ func (s *Store) Matches(ctx context.Context, pairID int64, f MatchFilter) ([]Mat
|
||||
where = append(where, "status IN ('different','similar') AND ignored = 0")
|
||||
case "ignored":
|
||||
where = append(where, "ignored = 1")
|
||||
case "upgrade":
|
||||
where = append(where, "status IN ('different','similar') AND ignored = 0 AND "+upgradeCond)
|
||||
default:
|
||||
where = append(where, "status = ? AND ignored = 0")
|
||||
args = append(args, f.Status)
|
||||
@@ -371,15 +379,15 @@ func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error)
|
||||
func (s *Store) UpsertMatch(ctx context.Context, m *Match) error {
|
||||
nullable := func(r json.RawMessage) string { return string(r) }
|
||||
_, err := s.DB.ExecContext(ctx, `INSERT INTO matches (pair_id, rel_slave, rel_master, kind, status, master_ino, master_size,
|
||||
master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, ignored, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(pair_id, rel_slave) DO UPDATE SET rel_master = excluded.rel_master, kind = excluded.kind,
|
||||
status = excluded.status, master_ino = excluded.master_ino, master_size = excluded.master_size,
|
||||
master_mtime = excluded.master_mtime, slave_ino = excluded.slave_ino, slave_size = excluded.slave_size,
|
||||
slave_mtime = excluded.slave_mtime, master_probe = excluded.master_probe, slave_probe = excluded.slave_probe,
|
||||
verdict = excluded.verdict, note = excluded.note, ignored = excluded.ignored, updated_at = excluded.updated_at`,
|
||||
verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, ignored = excluded.ignored, updated_at = excluded.updated_at`,
|
||||
m.PairID, m.RelSlave, m.RelMaster, m.Kind, m.Status, m.MasterIno, m.MasterSize, m.MasterMtime, m.SlaveIno, m.SlaveSize,
|
||||
m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Ignored, now())
|
||||
m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Ignored, now())
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -433,9 +441,14 @@ func (s *Store) LinkedMasterPaths(ctx context.Context, pairID int64, prefix stri
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// upgradeCond selects review matches where the master copy scored better, so
|
||||
// linking upgrades the slave.
|
||||
const upgradeCond = `verdict LIKE '%"better":"master"%'`
|
||||
|
||||
type Summary struct {
|
||||
Counts map[string]int `json:"counts"`
|
||||
Review int `json:"review"`
|
||||
Upgrades int `json:"upgrades"`
|
||||
Ignored int `json:"ignored"`
|
||||
ReclaimableSize int64 `json:"reclaimable_bytes"`
|
||||
LinkedSize int64 `json:"linked_bytes"`
|
||||
@@ -470,7 +483,12 @@ func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) {
|
||||
sum.Review += n
|
||||
}
|
||||
}
|
||||
return sum, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return sum, err
|
||||
}
|
||||
err = s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0
|
||||
AND status IN ('different','similar') AND `+upgradeCond, pairID).Scan(&sum.Upgrades)
|
||||
return sum, err
|
||||
}
|
||||
|
||||
// ---- operations ----
|
||||
|
||||
@@ -104,6 +104,11 @@ var migrations = []string{
|
||||
UPDATE operations SET quarantine_side = 'master' WHERE quarantine_side = 'host';
|
||||
UPDATE matches SET verdict = replace(verdict, 'host', 'master');
|
||||
UPDATE pairs SET last_scan_warnings = replace(last_scan_warnings, 'host', 'master');`,
|
||||
|
||||
// 3: opt-in approximate matching, and the reason shown for such matches.
|
||||
`ALTER TABLE pairs ADD COLUMN approx_matching INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE pairs ADD COLUMN approx_threshold INTEGER NOT NULL DEFAULT 85;
|
||||
ALTER TABLE matches ADD COLUMN reason TEXT NOT NULL DEFAULT '';`,
|
||||
}
|
||||
|
||||
// Open opens (and migrates) the database in dataDir.
|
||||
|
||||
Reference in new issue
Block a user