playlink/internal/engine/scan.go

package engine

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"path/filepath"
	"regexp"
	"strings"
	"time"
	"unicode"

	"github.com/cdnutter/playlink/internal/fsops"
	"github.com/cdnutter/playlink/internal/notify"
	"github.com/cdnutter/playlink/internal/probe"
	"github.com/cdnutter/playlink/internal/store"
)

type walkStats struct {
	files    int
	symlinks int
}

// walk lists audio files beneath root. Hidden entries, playlink staging files,
// the quarantine and symlinks are never followed or returned.
func walk(ctx context.Context, l *libs, root fsops.Root, out map[string]fsops.Info) (walkStats, error) {
	var ws walkStats
	err := filepath.WalkDir(root.Path, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			if p == root.Path {
				return err
			}
			return nil // unreadable subtree: skip rather than abort
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		name := d.Name()
		if p != root.Path && strings.HasPrefix(name, ".") {
			if d.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		if d.IsDir() {
			if p == l.quarantine {
				return filepath.SkipDir
			}
			return nil
		}
		if d.Type()&fs.ModeSymlink != 0 {
			ws.symlinks++
			return nil
		}
		if !d.Type().IsRegular() || !l.hasExt(name) {
			return nil
		}
		in, err := fsops.Lstat(p)
		if err != nil || !in.Mode.IsRegular() {
			return nil
		}
		rel, err := root.Rel(p)
		if err != nil {
			return nil
		}
		out[rel] = in
		ws.files++
		return nil
	})
	return ws, err
}

var trackPrefix = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?\d{1,3}(?:\s*[-._)]\s*|\s+)`)

// normTitle makes a conservative comparison key from a file name: the track
// number prefix is stripped and punctuation/case ignored. Parenthesised text
// is kept, so "Song (Live)" never matches "Song".
func normTitle(name string) string {
	stem := strings.TrimSuffix(name, filepath.Ext(name))
	if s := trackPrefix.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" {
		stem = s
	}
	var b strings.Builder
	for _, r := range strings.ToLower(stem) {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			b.WriteRune(r)
		}
	}
	return b.String()
}

func firstComponent(rel string) string {
	if i := strings.IndexByte(rel, filepath.Separator); i >= 0 {
		return rel[:i]
	}
	return ""
}

// masterIndex supports the fallback lookups for slave files without an exact
// path counterpart.
type masterIndex struct {
	files   map[string]fsops.Info
	lower   map[string]string   // lower(rel) -> rel
	byDir   map[string][]string // lower(dir) -> rels
	byTitle map[string][]string // lower(top folder)|normTitle -> rels

	approx    *approxIndex // nil unless the pair enables approximate matching
	threshold float64
	verify    VerifyFunc // confirms library-wide approximate matches; nil disables that scope
}

func buildIndex(files map[string]fsops.Info, p store.Pair) *masterIndex {
	ix := &masterIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
	rels := sortedKeys(files)
	if p.ApproxMatching {
		ix.approx = buildApproxIndex(rels)
		ix.threshold = float64(p.ApproxThreshold) / 100
	}
	for _, rel := range rels {
		ix.lower[strings.ToLower(rel)] = rel
		d := strings.ToLower(filepath.Dir(rel))
		ix.byDir[d] = append(ix.byDir[d], rel)
		k := strings.ToLower(firstComponent(rel)) + "|" + normTitle(filepath.Base(rel))
		ix.byTitle[k] = append(ix.byTitle[k], rel)
	}
	return ix
}

// losslessRank orders extension candidates so a lossless master file is
// preferred when several share the same title.
func extRank(rel string) int {
	switch strings.ToLower(filepath.Ext(rel)) {
	case ".flac", ".wav", ".aiff", ".aif", ".alac", ".ape", ".wv", ".dsf", ".dff":
		return 0
	case ".m4a", ".opus", ".ogg":
		return 1
	}
	return 2
}

// find locates the master counterpart of a slave file. The reason is only
// set for approximate matches. With approximate matching on, a match that
// lands on a single, EP or compilation is replaced by the same recording on
// a studio album when one exists, so the slave is linked to the album
// version and takes on the album's folder structure.
func (ix *masterIndex) find(relSlave string) (rel, kind, reason string) {
	rel, kind, reason = ix.strictFind(relSlave)
	if ix.approx != nil && rel != "" {
		if alt, why := ix.approx.albumAlternative(relSlave, rel, ix.threshold); alt != "" {
			return alt, store.KindApprox, why
		}
	}
	return rel, kind, reason
}

func (ix *masterIndex) strictFind(relSlave string) (rel, kind, reason string) {
	if _, ok := ix.files[relSlave]; ok {
		return relSlave, store.KindPath, ""
	}
	if r, ok := ix.lower[strings.ToLower(relSlave)]; ok {
		return r, store.KindPathCase, ""
	}
	dir := strings.ToLower(filepath.Dir(relSlave))
	base := filepath.Base(relSlave)
	stem := strings.ToLower(strings.TrimSuffix(base, filepath.Ext(base)))
	var extHits, fuzzyHits []string
	norm := normTitle(base)
	for _, r := range ix.byDir[dir] {
		b := filepath.Base(r)
		if strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) == stem {
			extHits = append(extHits, r)
		} else if norm != "" && normTitle(b) == norm {
			fuzzyHits = append(fuzzyHits, r)
		}
	}
	if len(extHits) > 0 {
		best := extHits[0]
		for _, r := range extHits[1:] {
			if extRank(r) < extRank(best) {
				best = r
			}
		}
		return best, store.KindExtension, ""
	}
	if len(fuzzyHits) == 1 {
		return fuzzyHits[0], store.KindFuzzy, ""
	}
	// Approximate matching runs before the top-folder fallback because it
	// searches the album (disc folders ignored) first, which resolves titles
	// that appear on several albums.
	if ix.approx != nil {
		if r, why := ix.approx.find(relSlave, ix.threshold, ix.verify); r != "" {
			return r, store.KindApprox, why
		}
	}
	if top := firstComponent(relSlave); top != "" && norm != "" {
		// The hit is in another album folder, so it must be the same kind of
		// recording: a studio track never matches one on a live album.
		if hits := ix.byTitle[strings.ToLower(top)+"|"+norm]; len(hits) == 1 &&
			sameVersions(parseTitle(relSlave).versions, parseTitle(hits[0]).versions) {
			return hits[0], store.KindElsewhere, ""
		}
	}
	return "", store.KindNone, ""
}

// classify fills in status, probes and verdict for a match, reusing cached
// results from prev when neither file changed.
func (e *Engine) classify(ctx context.Context, l *libs, m *store.Match, mi, si fsops.Info, prev *store.Match) {
	m.SlaveIno, m.SlaveSize, m.SlaveMtime = si.Ino, si.Size, si.MtimeNs
	m.MasterIno, m.MasterSize, m.MasterMtime = 0, 0, 0
	m.MasterProbe, m.SlaveProbe, m.Verdict, m.Note = nil, nil, nil, ""
	if m.Kind == store.KindNone {
		m.Status = store.StatusUnmatched
		return
	}
	m.MasterIno, m.MasterSize, m.MasterMtime = mi.Ino, mi.Size, mi.MtimeNs

	unchanged := prev != nil && prev.RelMaster == m.RelMaster &&
		prev.MasterIno == mi.Ino && prev.MasterSize == mi.Size && prev.MasterMtime == mi.MtimeNs &&
		prev.SlaveIno == si.Ino && prev.SlaveSize == si.Size && prev.SlaveMtime == si.MtimeNs
	if prev != nil && prev.RelMaster == m.RelMaster {
		m.Ignored = prev.Ignored
	}

	switch {
	case mi.Dev != si.Dev:
		m.Status, m.Note = store.StatusError, "files are on different filesystems"
		return
	case mi.SameInode(si):
		m.Status = store.StatusLinked
		return
	}

	pathKind := m.Kind == store.KindPath || m.Kind == store.KindPathCase
	if pathKind && mi.Size == si.Size {
		if unchanged && (prev.Status == store.StatusIdentical || prev.Status == store.StatusDifferent) {
			m.Status = prev.Status
		} else {
			masterAbs := filepath.Join(l.master.Path, m.RelMaster)
			slaveAbs := filepath.Join(l.slave.Path, m.RelSlave)
			same, err := fsops.SameContent(masterAbs, slaveAbs)
			switch {
			case err != nil:
				m.Status, m.Note = store.StatusError, "compare failed: "+err.Error()
				return
			case same:
				m.Status = store.StatusIdentical
			default:
				m.Status = store.StatusDifferent
			}
		}
		if m.Status == store.StatusIdentical {
			return
		}
	} else if pathKind {
		m.Status = store.StatusDifferent
	} else {
		m.Status = store.StatusSimilar
	}

	// Manual review: gather codec details for both sides.
	if unchanged && len(prev.MasterProbe) > 0 && len(prev.SlaveProbe) > 0 {
		m.MasterProbe, m.SlaveProbe, m.Verdict = prev.MasterProbe, prev.SlaveProbe, prev.Verdict
		return
	}
	mp := e.prober.Probe(ctx, filepath.Join(l.master.Path, m.RelMaster))
	sp := e.prober.Probe(ctx, filepath.Join(l.slave.Path, m.RelSlave))
	v := probe.Compare(mp, sp)
	m.MasterProbe, _ = json.Marshal(mp)
	m.SlaveProbe, _ = json.Marshal(sp)
	m.Verdict, _ = json.Marshal(v)
}

// StartScan launches a background scan of a pair.
func (e *Engine) StartScan(pairID int64, trigger string) (*Job, error) {
	return e.startJob(pairID, "scan", func(ctx context.Context, j *Job) error {
		return e.scan(ctx, j, pairID, trigger)
	})
}

func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) error {
	started := time.Now()
	l, err := e.loadPair(ctx, pairID)
	if err != nil {
		_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), nil)
		return err
	}
	warns := layoutWarnings(l)

	j.phase("Indexing master library", 0)
	masterFiles := map[string]fsops.Info{}
	ms, err := walk(ctx, l, l.master, masterFiles)
	if err != nil {
		_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
		return err
	}
	j.phase("Indexing slave library", 0)
	slaveFiles := map[string]fsops.Info{}
	ss, err := walk(ctx, l, l.slave, slaveFiles)
	if err != nil {
		_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
		return err
	}
	ix := buildIndex(masterFiles, l.pair)
	if ix.approx != nil {
		ix.verify = e.artistVerifier(ctx, l)
	}
	prev, err := e.st.AllMatches(ctx, pairID)
	if err != nil {
		return err
	}

	j.phase("Comparing files", int64(len(slaveFiles)))
	keep := map[string]bool{}
	var layout layoutStats
	// Songs waiting for review, and how many of them weren't before this scan.
	review, newReview := 0, 0
	for _, rel := range sortedKeys(slaveFiles) {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		si := slaveFiles[rel]
		m := store.Match{PairID: pairID, RelSlave: rel}
		m.RelMaster, m.Kind, m.Reason = ix.find(rel)
		var pm *store.Match
		if p, ok := prev[rel]; ok {
			pm = &p
		}
		e.classify(ctx, l, &m, masterFiles[m.RelMaster], si, pm)
		if err := e.st.UpsertMatch(ctx, &m); err != nil {
			return err
		}
		keep[rel] = true
		if needsReview(m) {
			review++
			if pm == nil || !needsReview(*pm) {
				newReview++
			}
		}
		layout.add(m.RelSlave, m.RelMaster, m.Kind, m.Reason)
		j.count(m.Status)
		j.tick()
	}
	if err := e.st.DeleteMatchesExcept(ctx, pairID, keep); err != nil {
		return err
	}

	// Layout warnings based on what was actually found.
	if w := layout.warning(); w != nil {
		warns = append(warns, *w)
	}
	if ms.symlinks+ss.symlinks > 0 {
		warns = append(warns, store.Warning{Title: fmt.Sprintf("Ignored %d symlinks", ms.symlinks+ss.symlinks),
			Detail: "Playlink never follows or replaces symlinks."})
	}
	if e.prober.Bin == "" {
		warns = append(warns, store.Warning{Title: "ffprobe was not found",
			Detail: "Codec and quality details are unavailable for manual review."})
	}
	if err := e.st.FinishScan(ctx, pairID, time.Now().Unix(), "", warns); err != nil {
		return err
	}
	j.update(func(j *Job) {
		j.Message = fmt.Sprintf("Scanned %d master and %d slave files in %s", ms.files, ss.files, time.Since(started).Round(time.Second))
	})

	if newReview > 0 {
		msg := plural(newReview, "song needs", "songs need") + " review."
		if review > newReview {
			msg = fmt.Sprintf("%s %d are waiting in total.", msg, review)
		}
		e.emit(ctx, l, notify.Event{Kind: notify.EventReview, Title: "New songs to review", Message: msg,
			Path: fmt.Sprintf("#/pair/%d/review?status=review", l.pair.ID)})
	}

	if l.pair.AutoLinkIdentical {
		if err := e.autoLink(ctx, j, l, trigger); err != nil {
			return err
		}
	}
	if l.pair.AutoCleanupEmpty {
		return e.autoCleanup(ctx, j, l, trigger)
	}
	return nil
}

// needsReview matches the Review page's "Needs review" filter.
func needsReview(m store.Match) bool {
	return (m.Status == store.StatusDifferent || m.Status == store.StatusSimilar) && !m.Ignored
}

// autoLink links settled, byte-identical, same-path files after a scan.
func (e *Engine) autoLink(ctx context.Context, j *Job, l *libs, trigger string) error {
	ids, err := e.st.IdenticalIDs(ctx, l.pair.ID)
	if err != nil || len(ids) == 0 {
		return err
	}
	if e.st.DryRun(ctx) {
		j.update(func(j *Job) {
			j.Message += fmt.Sprintf(". Dry run is on: %d identical files would be auto-linked.", len(ids))
		})
		return nil
	}
	if err := e.BeforeAction(ctx, "auto-link"); err != nil {
		return err
	}
	linked, freed, err := e.linkIDs(ctx, j, l.pair.ID, ids, "auto:"+trigger, true)
	if linked > 0 {
		e.emit(ctx, l, notify.Event{Kind: notify.EventLinked, Title: "Identical files linked",
			Message: fmt.Sprintf("Replaced %s with hardlinks to the master, freeing %s.", plural(linked, "identical file", "identical files"), formatBytes(freed)),
			Path:    "#/activity"})
	}
	return err
}

// linkIDs links a batch of identical matches and returns how many were
// linked and the space that freed.
func (e *Engine) linkIDs(ctx context.Context, j *Job, pairID int64, ids []int64, trigger string, onlySettled bool) (linked int, freed int64, err error) {
	j.phase("Linking identical files", int64(len(ids)))
	settle := time.Duration(0)
	if onlySettled {
		if p, err := e.st.Pair(ctx, pairID); err == nil {
			settle = time.Duration(p.SettleMinutes) * time.Minute
		}
	}
	cutoff := time.Now().Add(-settle).UnixNano()
	for _, id := range ids {
		if ctx.Err() != nil {
			return linked, freed, ctx.Err()
		}
		m, err := e.st.Match(ctx, id)
		if err != nil {
			j.tick()
			continue
		}
		if onlySettled && m.Kind != store.KindPath {
			j.skip(m.RelSlave + ": path differs by case, link manually")
			j.tick()
			continue
		}
		if settle > 0 && (m.SlaveMtime > cutoff || m.MasterMtime > cutoff) {
			j.skip(m.RelSlave + ": modified recently, waiting for it to settle")
			j.tick()
			continue
		}
		op, err := e.Link(ctx, id, LinkOptions{Trigger: trigger})
		switch {
		case err != nil:
			j.skip(m.RelSlave + ": " + err.Error())
		case op.DryRun:
			j.count("planned")
		default:
			j.count("linked")
			linked++
			freed += op.Bytes
		}
		j.tick()
	}
	return linked, freed, nil
}

// StartLinkIdentical links every identical match of a pair in the background.
func (e *Engine) StartLinkIdentical(pairID int64) (*Job, error) {
	return e.startJob(pairID, "link-identical", func(ctx context.Context, j *Job) error {
		ids, err := e.st.IdenticalIDs(ctx, pairID)
		if err != nil {
			return err
		}
		if len(ids) == 0 {
			j.update(func(j *Job) { j.Message = "Nothing to link" })
			return nil
		}
		_, _, err = e.linkIDs(ctx, j, pairID, ids, "bulk", false)
		return err
	})
}

// refresh re-evaluates a single slave file after an operation. The optional
// hint names the master file the operation paired it with; it takes
// precedence over the stored match, which may describe the pre-operation
// state (or be gone, after a move).
func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string, hint ...store.Match) {
	si, err := fsops.RegularFile(filepath.Join(l.slave.Path, relSlave))
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			_ = e.st.DeleteMatch(ctx, l.pair.ID, relSlave)
		}
		return
	}
	m := store.Match{PairID: l.pair.ID, RelSlave: relSlave}
	var pm *store.Match
	if p, err := e.st.MatchBySlave(ctx, l.pair.ID, relSlave); err == nil {
		pm = &p
	}
	var mi fsops.Info
	if mp, err := l.master.Resolve(relSlave); err == nil {
		if in, err := fsops.RegularFile(mp); err == nil {
			mi = in
			m.RelMaster, m.Kind = relSlave, store.KindPath
		}
	}
	candidates := hint
	if pm != nil {
		candidates = append(candidates, *pm)
	}
	for _, c := range candidates {
		if m.Kind != "" || c.RelMaster == "" || c.Kind == "" || c.Kind == store.KindNone {
			continue
		}
		if mp, err := l.master.Resolve(c.RelMaster); err == nil {
			if in, err := fsops.RegularFile(mp); err == nil {
				mi = in
				m.RelMaster, m.Kind, m.Reason = c.RelMaster, c.Kind, c.Reason
			}
		}
	}
	if m.Kind == "" {
		m.Kind = store.KindNone
	}
	e.classify(ctx, l, &m, mi, si, pm)
	_ = e.st.UpsertMatch(ctx, &m)
}

// artistVerifier confirms a library-wide approximate match by comparing the
// artist tags of both files. Without ffprobe, or when either file has no
// artist tag, the match is rejected: a title alone is not enough evidence
// once the folder structure no longer agrees.
func (e *Engine) artistVerifier(ctx context.Context, l *libs) VerifyFunc {
	if e.prober.Bin == "" {
		return nil
	}
	cache := map[string]string{}
	artist := func(abs string) string {
		if a, ok := cache[abs]; ok {
			return a
		}
		a := strings.Join(words(e.prober.Probe(ctx, abs).Artist), " ")
		cache[abs] = a
		return a
	}
	return func(relSlave, relMaster string) bool {
		sa := artist(filepath.Join(l.slave.Path, relSlave))
		ma := artist(filepath.Join(l.master.Path, relMaster))
		if sa == "" || ma == "" {
			return false
		}
		// "Avenged Sevenfold" matches "Avenged Sevenfold feat Someone".
		return sa == ma || strings.HasPrefix(ma, sa+" ") || strings.HasPrefix(sa, ma+" ")
	}
}