Flag slave-only songs whose album or artist the master has

Scans now mark slave-only songs as missing from the master when the master
has their album folder or at least their artist folder. A new Missing from
master filter in Review lists them, split by album and artist.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/export.go

@@ -63,6 +63,10 @@ func recommendation(m store.Match, v probe.Verdict) string {
 		return "review"
 	case m.Status == store.StatusError:
 		return "fix error"
+	case m.Missing == "album":
+		return "not in master: master has the album"
+	case m.Missing == "artist":
+		return "not in master: master has the artist"
 	}
 	return ""
 }

internal/engine/missing.go

@@ -0,0 +1,91 @@
+package engine
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// A slave-only song is "missing from the master" when the master has its
+// album folder (the song was probably never added there) or at least its
+// artist folder (the album is missing). Folders are compared with disc
+// folders ignored and release labels such as "Deluxe Edition" stripped, so
+// "Robot Face (Deluxe)" in the slave finds "Robot Face" in the master.
+
+// Missing levels.
+const (
+	MissingAlbum  = "album"
+	MissingArtist = "artist"
+)
+
+// folderNorm is a folder name reduced to letters and digits, release labels removed.
+func folderNorm(name string) string {
+	if w := words(stripRelease(name)); len(w) > 0 {
+		return strings.Join(w, "")
+	}
+	return strings.ToLower(name)
+}
+
+// folderKeys returns the normalized artist key and album key of a song's
+// folders; the album key is empty for a song directly in the artist folder.
+func folderKeys(rel string) (artist, album string) {
+	var parts []string
+	for _, p := range strings.Split(filepath.Dir(rel), string(filepath.Separator)) {
+		if p != "." && !discFolder.MatchString(p) {
+			parts = append(parts, folderNorm(p))
+		}
+	}
+	if len(parts) == 0 {
+		return "", ""
+	}
+	if len(parts) > 1 {
+		album = strings.Join(parts, "/")
+	}
+	return parts[0], album
+}
+
+// masterFolders indexes the master's artist and album folders by key, to
+// their real spelling.
+type masterFolders struct {
+	artists map[string]string
+	albums  map[string]string
+}
+
+func buildMasterFolders(rels []string) masterFolders {
+	mf := masterFolders{artists: map[string]string{}, albums: map[string]string{}}
+	for _, rel := range rels {
+		artist, album := folderKeys(rel)
+		if artist == "" {
+			continue
+		}
+		if _, ok := mf.artists[artist]; !ok {
+			mf.artists[artist] = firstComponent(rel)
+		}
+		if album != "" {
+			if _, ok := mf.albums[album]; !ok {
+				d := filepath.Dir(rel)
+				for d != "." && discFolder.MatchString(filepath.Base(d)) {
+					d = filepath.Dir(d)
+				}
+				mf.albums[album] = d
+			}
+		}
+	}
+	return mf
+}
+
+// missing reports whether the master has a slave-only song's album or
+// artist, with a reason naming the master's folder.
+func (mf masterFolders) missing(relSlave string) (level, reason string) {
+	artist, album := folderKeys(relSlave)
+	if artist == "" {
+		return "", ""
+	}
+	if d, ok := mf.albums[album]; ok && album != "" {
+		return MissingAlbum, fmt.Sprintf("the master has the album %q but not this song", d)
+	}
+	if d, ok := mf.artists[artist]; ok {
+		return MissingArtist, fmt.Sprintf("the master has the artist %q but not this album", d)
+	}
+	return "", ""
+}

internal/engine/missing_test.go

@@ -0,0 +1,64 @@
+package engine
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/cdnutter/playlink/internal/store"
+)
+
+func TestMissingFromMaster(t *testing.T) {
+	ev := setup(t, false)
+	m, s := ev.master, ev.slave
+	ev.write(m, "Caravan Palace/Robot Face/01 Lone Digger.flac", "a")
+	ev.write(m, "Caravan Palace/Chronologic/Disc 1/01 Miracle.flac", "b")
+	ev.write(s, "Caravan Palace/Robot Face/01 Lone Digger.flac", "a")
+	ev.write(s, "Caravan Palace/Robot Face/05 Waterguns.mp3", "song")              // album in master
+	ev.write(s, "Caravan Palace/robot face (Deluxe Edition)/12 Bonus.mp3", "song") // same album, other spelling
+	ev.write(s, "Caravan Palace/Chronologic/Disc 2/03 Plume.mp3", "song")          // disc folders ignored
+	ev.write(s, "Caravan Palace/Panic/01 Dragons.mp3", "song")                     // only the artist
+	ev.write(s, "Caravan Palace/Loose.mp3", "song")                                // only the artist
+	ev.write(s, "Unknown Artist/Album/Song.mp3", "song")                           // neither
+	ev.scan()
+
+	want := map[string]string{
+		"Caravan Palace/Robot Face/05 Waterguns.mp3":              "album",
+		"Caravan Palace/robot face (Deluxe Edition)/12 Bonus.mp3": "album",
+		"Caravan Palace/Chronologic/Disc 2/03 Plume.mp3":          "album",
+		"Caravan Palace/Panic/01 Dragons.mp3":                     "artist",
+		"Caravan Palace/Loose.mp3":                                "artist",
+		"Unknown Artist/Album/Song.mp3":                           "",
+		"Caravan Palace/Robot Face/01 Lone Digger.flac":           "",
+	}
+	for rel, level := range want {
+		mt, err := ev.st.MatchBySlave(ctx, ev.pair.ID, rel)
+		if err != nil {
+			t.Fatal(rel, err)
+		}
+		if mt.Missing != level {
+			t.Errorf("%s: missing %q, want %q", rel, mt.Missing, level)
+		}
+	}
+	mt, _ := ev.st.MatchBySlave(ctx, ev.pair.ID, "Caravan Palace/Chronologic/Disc 2/03 Plume.mp3")
+	if !strings.Contains(mt.Reason, `"Caravan Palace/Chronologic"`) {
+		t.Errorf("reason should name the master's album folder: %q", mt.Reason)
+	}
+
+	sum, err := ev.st.Summary(ctx, ev.pair.ID)
+	if err != nil || sum.Missing["album"] != 3 || sum.Missing["artist"] != 2 {
+		t.Fatalf("summary %+v %v", sum.Missing, err)
+	}
+	if _, n, _ := ev.st.Matches(ctx, ev.pair.ID, store.MatchFilter{Status: "missing"}); n != 5 {
+		t.Fatalf("missing filter: %d", n)
+	}
+	if _, n, _ := ev.st.Matches(ctx, ev.pair.ID, store.MatchFilter{Status: "missing_album"}); n != 3 {
+		t.Fatalf("missing_album filter: %d", n)
+	}
+
+	// Once the master gets the song, it is no longer missing.
+	ev.write(m, "Caravan Palace/Robot Face/05 Waterguns.mp3", "song")
+	ev.scan()
+	if mt, _ := ev.st.MatchBySlave(ctx, ev.pair.ID, "Caravan Palace/Robot Face/05 Waterguns.mp3"); mt.Missing != "" || mt.Status == store.StatusUnmatched {
+		t.Fatalf("now in master: %+v", mt)
+	}
+}

internal/engine/scan.go

@@ -105,6 +105,8 @@ type masterIndex struct {
 	byDir   map[string][]string // lower(dir) -> rels
 	byTitle map[string][]string // lower(top folder)|normTitle -> rels
 
+	folders masterFolders // artist and album folders, for songs missing from the master
+
 	approx    *approxIndex // nil unless the pair enables approximate matching
 	threshold float64
 	verify    VerifyFunc // confirms library-wide approximate matches; nil disables that scope
@@ -113,6 +115,7 @@ type masterIndex struct {
 func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex {
 	ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
 	rels := sortedKeys(files)
+	ix.folders = buildMasterFolders(rels)
 	if p.ApproxMatching {
 		ix.approx = buildApproxIndex(rels)
 		ix.threshold = float64(p.ApproxThreshold) / 100
@@ -327,6 +330,9 @@ func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string)
 			pm = &p
 		}
 		e.classify(ctx, l, &m, masterFiles[m.RelMaster], si, pm)
+		if m.Status == store.StatusUnmatched {
+			m.Missing, m.Reason = ix.folders.missing(rel)
+		}
 		if err := e.st.UpsertMatch(ctx, &m); err != nil {
 			return err
 		}
@@ -483,6 +489,9 @@ func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string, hint ...
 		m.Kind = store.KindNone
 	}
 	e.classify(ctx, l, &m, mi, si, pm)
+	if m.Status == store.StatusUnmatched && pm != nil && pm.Status == store.StatusUnmatched {
+		m.Missing, m.Reason = pm.Missing, pm.Reason // the next scan re-checks the master's folders
+	}
 	_ = e.st.UpsertMatch(ctx, &m)
 }
 

internal/store/models.go

@@ -286,20 +286,21 @@ type Match struct {
 	SlaveProbe  json.RawMessage `json:"slave_probe,omitempty"`
 	Verdict     json.RawMessage `json:"verdict,omitempty"`
 	Note        string          `json:"note"`
-	Reason      string          `json:"reason"`      // why an approximate match was chosen
+	Reason      string          `json:"reason"`      // why an approximate match was chosen, or what the master has of a missing song
+	Missing     string          `json:"missing"`     // slave-only song: the master has its "album" or only its "artist"
 	LinkTarget  string          `json:"link_target"` // where linking puts the slave file (filled by the API)
 	Ignored     bool            `json:"ignored"`
 	UpdatedAt   int64           `json:"updated_at"`
 }
 
 const matchCols = `id, pair_id, rel_slave, rel_master, kind, status, master_ino, master_size, master_mtime,
-	slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at`
+	slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, missing, ignored, updated_at`
 
 func scanMatch(r scanner) (Match, error) {
 	var m Match
 	var mp, sp, v string
 	err := r.Scan(&m.ID, &m.PairID, &m.RelSlave, &m.RelMaster, &m.Kind, &m.Status, &m.MasterIno, &m.MasterSize, &m.MasterMtime,
-		&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Ignored, &m.UpdatedAt)
+		&m.SlaveIno, &m.SlaveSize, &m.SlaveMtime, &mp, &sp, &v, &m.Note, &m.Reason, &m.Missing, &m.Ignored, &m.UpdatedAt)
 	if mp != "" {
 		m.MasterProbe = json.RawMessage(mp)
 	}
@@ -348,7 +349,7 @@ func (s *Store) AllMatches(ctx context.Context, pairID int64) (map[string]Match,
 }
 
 type MatchFilter struct {
-	Status string // a status, "review" (different+similar), "ignored" or ""
+	Status string // a status, "review" (different+similar), "upgrade", "missing[_album|_artist]", "ignored" or ""
 	Query  string
 	IDs    []int64 // restrict to these matches (at most maxIDFilter)
 	Limit  int
@@ -367,6 +368,11 @@ func matchWhere(pairID int64, f MatchFilter) (string, []any) {
 		where = append(where, "ignored = 1")
 	case "upgrade":
 		where = append(where, "status IN ('different','similar') AND ignored = 0 AND "+upgradeCond)
+	case "missing":
+		where = append(where, "status = 'unmatched' AND missing != '' AND ignored = 0")
+	case "missing_album", "missing_artist":
+		where = append(where, "status = 'unmatched' AND missing = ? AND ignored = 0")
+		args = append(args, strings.TrimPrefix(f.Status, "missing_"))
 	default:
 		where = append(where, "status = ? AND ignored = 0")
 		args = append(args, f.Status)
@@ -508,15 +514,15 @@ func (s *Store) IdenticalIDs(ctx context.Context, pairID int64) ([]int64, error)
 func (s *Store) UpsertMatch(ctx context.Context, m *Match) error {
 	nullable := func(r json.RawMessage) string { return string(r) }
 	_, err := s.DB.ExecContext(ctx, `INSERT INTO matches (pair_id, rel_slave, rel_master, kind, status, master_ino, master_size,
-		master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, ignored, updated_at)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		master_mtime, slave_ino, slave_size, slave_mtime, master_probe, slave_probe, verdict, note, reason, missing, ignored, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 		ON CONFLICT(pair_id, rel_slave) DO UPDATE SET rel_master = excluded.rel_master, kind = excluded.kind,
 		status = excluded.status, master_ino = excluded.master_ino, master_size = excluded.master_size,
 		master_mtime = excluded.master_mtime, slave_ino = excluded.slave_ino, slave_size = excluded.slave_size,
 		slave_mtime = excluded.slave_mtime, master_probe = excluded.master_probe, slave_probe = excluded.slave_probe,
-		verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, ignored = excluded.ignored, updated_at = excluded.updated_at`,
+		verdict = excluded.verdict, note = excluded.note, reason = excluded.reason, missing = excluded.missing, ignored = excluded.ignored, updated_at = excluded.updated_at`,
 		m.PairID, m.RelSlave, m.RelMaster, m.Kind, m.Status, m.MasterIno, m.MasterSize, m.MasterMtime, m.SlaveIno, m.SlaveSize,
-		m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Ignored, now())
+		m.SlaveMtime, nullable(m.MasterProbe), nullable(m.SlaveProbe), nullable(m.Verdict), m.Note, m.Reason, m.Missing, m.Ignored, now())
 	return err
 }
 
@@ -579,12 +585,13 @@ type Summary struct {
 	Review          int            `json:"review"`
 	Upgrades        int            `json:"upgrades"`
 	Ignored         int            `json:"ignored"`
+	Missing         map[string]int `json:"missing"` // slave-only songs by what the master has: album, artist
 	ReclaimableSize int64          `json:"reclaimable_bytes"`
 	LinkedSize      int64          `json:"linked_bytes"`
 }
 
 func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) {
-	sum := Summary{Counts: map[string]int{}}
+	sum := Summary{Counts: map[string]int{}, Missing: map[string]int{}}
 	rows, err := s.DB.QueryContext(ctx, `SELECT status, ignored, COUNT(*), COALESCE(SUM(slave_size), 0) FROM matches WHERE pair_id = ? GROUP BY status, ignored`, pairID)
 	if err != nil {
 		return sum, err
@@ -617,7 +624,24 @@ func (s *Store) Summary(ctx context.Context, pairID int64) (Summary, error) {
 	}
 	err = s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0
 		AND status IN ('different','similar') AND `+upgradeCond, pairID).Scan(&sum.Upgrades)
-	return sum, err
+	if err != nil {
+		return sum, err
+	}
+	mrows, err := s.DB.QueryContext(ctx, `SELECT missing, COUNT(*) FROM matches WHERE pair_id = ? AND ignored = 0
+		AND status = 'unmatched' AND missing != '' GROUP BY missing`, pairID)
+	if err != nil {
+		return sum, err
+	}
+	defer mrows.Close()
+	for mrows.Next() {
+		var k string
+		var n int
+		if err := mrows.Scan(&k, &n); err != nil {
+			return sum, err
+		}
+		sum.Missing[k] = n
+	}
+	return sum, mrows.Err()
 }
 
 // ---- operations ----

internal/store/store.go

@@ -140,6 +140,9 @@ var migrations = []string{
 	// 10: library type (music or soundtracks) and the step-by-step fixer's progress.
 	`ALTER TABLE pairs ADD COLUMN library_type TEXT NOT NULL DEFAULT 'music';
 	ALTER TABLE pairs ADD COLUMN fixer_state TEXT NOT NULL DEFAULT '';`,
+
+	// 11: slave-only songs whose album or artist the master has.
+	`ALTER TABLE matches ADD COLUMN missing TEXT NOT NULL DEFAULT '';`,
 }
 
 // SchemaVersion is the newest database schema this build knows.