Initial commit: playlink music library hardlink deduplicator
Go backend (chi, SQLite) with an embedded React/TypeScript UI that finds songs present in both a host and a slave music library and replaces the slave copy with a hardlink to the host copy. - Path-first matching with format/title/album-folder fallbacks - Auto-linking of byte-identical files, re-verified with cmp before linking - Manual review with ffprobe quality comparison, downgrade guard and promote-to-host override - Build mode to populate an empty slave library from the host - Dry run by default, atomic stage+rename replacement, quarantine with retention, undo, path confinement and authenticated UI - Dockerfile and compose file for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
commit
ac47e61b08
42 files changed
+8892
No files matched your search
@@ -0,0 +1,448 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/cdnutter/playlink/internal/fsops"
|
||||
"github.com/cdnutter/playlink/internal/probe"
|
||||
"github.com/cdnutter/playlink/internal/store"
|
||||
)
|
||||
|
||||
type walkStats struct {
|
||||
files int
|
||||
symlinks int
|
||||
offDepth int // files not at Artist/Album/Title depth
|
||||
}
|
||||
|
||||
// walk lists audio files beneath root. Hidden entries, playlink staging files,
|
||||
// the quarantine and symlinks are never followed or returned.
|
||||
func walk(ctx context.Context, l *libs, root fsops.Root, out map[string]fsops.Info) (walkStats, error) {
|
||||
var ws walkStats
|
||||
err := filepath.WalkDir(root.Path, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if p == root.Path {
|
||||
return err
|
||||
}
|
||||
return nil // unreadable subtree: skip rather than abort
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
name := d.Name()
|
||||
if p != root.Path && strings.HasPrefix(name, ".") {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
if p == l.quarantine {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.Type()&fs.ModeSymlink != 0 {
|
||||
ws.symlinks++
|
||||
return nil
|
||||
}
|
||||
if !d.Type().IsRegular() || !l.hasExt(name) {
|
||||
return nil
|
||||
}
|
||||
in, err := fsops.Lstat(p)
|
||||
if err != nil || !in.Mode.IsRegular() {
|
||||
return nil
|
||||
}
|
||||
rel, err := root.Rel(p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out[rel] = in
|
||||
ws.files++
|
||||
if strings.Count(rel, string(filepath.Separator)) != 2 {
|
||||
ws.offDepth++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return ws, err
|
||||
}
|
||||
|
||||
var trackPrefix = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?\d{1,3}(?:\s*[-._)]\s*|\s+)`)
|
||||
|
||||
// normTitle makes a conservative comparison key from a file name: the track
|
||||
// number prefix is stripped and punctuation/case ignored. Parenthesised text
|
||||
// is kept, so "Song (Live)" never matches "Song".
|
||||
func normTitle(name string) string {
|
||||
stem := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
if s := trackPrefix.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" {
|
||||
stem = s
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(stem) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func firstComponent(rel string) string {
|
||||
if i := strings.IndexByte(rel, filepath.Separator); i >= 0 {
|
||||
return rel[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hostIndex supports the fallback lookups for slave files without an exact
|
||||
// path counterpart.
|
||||
type hostIndex struct {
|
||||
files map[string]fsops.Info
|
||||
lower map[string]string // lower(rel) -> rel
|
||||
byDir map[string][]string // lower(dir) -> rels
|
||||
byTitle map[string][]string // lower(artist)|normTitle -> rels
|
||||
}
|
||||
|
||||
func buildIndex(files map[string]fsops.Info) *hostIndex {
|
||||
ix := &hostIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
|
||||
for _, rel := range sortedKeys(files) {
|
||||
ix.lower[strings.ToLower(rel)] = rel
|
||||
d := strings.ToLower(filepath.Dir(rel))
|
||||
ix.byDir[d] = append(ix.byDir[d], rel)
|
||||
k := strings.ToLower(firstComponent(rel)) + "|" + normTitle(filepath.Base(rel))
|
||||
ix.byTitle[k] = append(ix.byTitle[k], rel)
|
||||
}
|
||||
return ix
|
||||
}
|
||||
|
||||
// losslessRank orders extension candidates so a lossless host file is
|
||||
// preferred when several share the same title.
|
||||
func extRank(rel string) int {
|
||||
switch strings.ToLower(filepath.Ext(rel)) {
|
||||
case ".flac", ".wav", ".aiff", ".aif", ".alac", ".ape", ".wv", ".dsf", ".dff":
|
||||
return 0
|
||||
case ".m4a", ".opus", ".ogg":
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
// find locates the host counterpart of a slave file.
|
||||
func (ix *hostIndex) find(relSlave string) (string, string) {
|
||||
if _, ok := ix.files[relSlave]; ok {
|
||||
return relSlave, store.KindPath
|
||||
}
|
||||
if r, ok := ix.lower[strings.ToLower(relSlave)]; ok {
|
||||
return r, store.KindPathCase
|
||||
}
|
||||
dir := strings.ToLower(filepath.Dir(relSlave))
|
||||
base := filepath.Base(relSlave)
|
||||
stem := strings.ToLower(strings.TrimSuffix(base, filepath.Ext(base)))
|
||||
var extHits, fuzzyHits []string
|
||||
norm := normTitle(base)
|
||||
for _, r := range ix.byDir[dir] {
|
||||
b := filepath.Base(r)
|
||||
if strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) == stem {
|
||||
extHits = append(extHits, r)
|
||||
} else if norm != "" && normTitle(b) == norm {
|
||||
fuzzyHits = append(fuzzyHits, r)
|
||||
}
|
||||
}
|
||||
if len(extHits) > 0 {
|
||||
best := extHits[0]
|
||||
for _, r := range extHits[1:] {
|
||||
if extRank(r) < extRank(best) {
|
||||
best = r
|
||||
}
|
||||
}
|
||||
return best, store.KindExtension
|
||||
}
|
||||
if len(fuzzyHits) == 1 {
|
||||
return fuzzyHits[0], store.KindFuzzy
|
||||
}
|
||||
if artist := firstComponent(relSlave); artist != "" && norm != "" {
|
||||
if hits := ix.byTitle[strings.ToLower(artist)+"|"+norm]; len(hits) == 1 {
|
||||
return hits[0], store.KindElsewhere
|
||||
}
|
||||
}
|
||||
return "", store.KindNone
|
||||
}
|
||||
|
||||
// classify fills in status, probes and verdict for a match, reusing cached
|
||||
// results from prev when neither file changed.
|
||||
func (e *Engine) classify(ctx context.Context, l *libs, m *store.Match, hi, si fsops.Info, prev *store.Match) {
|
||||
m.SlaveIno, m.SlaveSize, m.SlaveMtime = si.Ino, si.Size, si.MtimeNs
|
||||
m.HostIno, m.HostSize, m.HostMtime = 0, 0, 0
|
||||
m.HostProbe, m.SlaveProbe, m.Verdict, m.Note = nil, nil, nil, ""
|
||||
if m.Kind == store.KindNone {
|
||||
m.Status = store.StatusUnmatched
|
||||
return
|
||||
}
|
||||
m.HostIno, m.HostSize, m.HostMtime = hi.Ino, hi.Size, hi.MtimeNs
|
||||
|
||||
unchanged := prev != nil && prev.RelHost == m.RelHost &&
|
||||
prev.HostIno == hi.Ino && prev.HostSize == hi.Size && prev.HostMtime == hi.MtimeNs &&
|
||||
prev.SlaveIno == si.Ino && prev.SlaveSize == si.Size && prev.SlaveMtime == si.MtimeNs
|
||||
if prev != nil && prev.RelHost == m.RelHost {
|
||||
m.Ignored = prev.Ignored
|
||||
}
|
||||
|
||||
switch {
|
||||
case hi.Dev != si.Dev:
|
||||
m.Status, m.Note = store.StatusError, "files are on different filesystems"
|
||||
return
|
||||
case hi.SameInode(si):
|
||||
m.Status = store.StatusLinked
|
||||
return
|
||||
}
|
||||
|
||||
pathKind := m.Kind == store.KindPath || m.Kind == store.KindPathCase
|
||||
if pathKind && hi.Size == si.Size {
|
||||
if unchanged && (prev.Status == store.StatusIdentical || prev.Status == store.StatusDifferent) {
|
||||
m.Status = prev.Status
|
||||
} else {
|
||||
hostAbs := filepath.Join(l.host.Path, m.RelHost)
|
||||
slaveAbs := filepath.Join(l.slave.Path, m.RelSlave)
|
||||
same, err := fsops.SameContent(hostAbs, slaveAbs)
|
||||
switch {
|
||||
case err != nil:
|
||||
m.Status, m.Note = store.StatusError, "compare failed: "+err.Error()
|
||||
return
|
||||
case same:
|
||||
m.Status = store.StatusIdentical
|
||||
default:
|
||||
m.Status = store.StatusDifferent
|
||||
}
|
||||
}
|
||||
if m.Status == store.StatusIdentical {
|
||||
return
|
||||
}
|
||||
} else if pathKind {
|
||||
m.Status = store.StatusDifferent
|
||||
} else {
|
||||
m.Status = store.StatusSimilar
|
||||
}
|
||||
|
||||
// Manual review: gather codec details for both sides.
|
||||
if unchanged && len(prev.HostProbe) > 0 && len(prev.SlaveProbe) > 0 {
|
||||
m.HostProbe, m.SlaveProbe, m.Verdict = prev.HostProbe, prev.SlaveProbe, prev.Verdict
|
||||
return
|
||||
}
|
||||
hp := e.prober.Probe(ctx, filepath.Join(l.host.Path, m.RelHost))
|
||||
sp := e.prober.Probe(ctx, filepath.Join(l.slave.Path, m.RelSlave))
|
||||
v := probe.Compare(hp, sp)
|
||||
m.HostProbe, _ = json.Marshal(hp)
|
||||
m.SlaveProbe, _ = json.Marshal(sp)
|
||||
m.Verdict, _ = json.Marshal(v)
|
||||
}
|
||||
|
||||
// StartScan launches a background scan of a pair.
|
||||
func (e *Engine) StartScan(pairID int64, trigger string) (*Job, error) {
|
||||
return e.startJob(pairID, "scan", func(ctx context.Context, j *Job) error {
|
||||
return e.scan(ctx, j, pairID, trigger)
|
||||
})
|
||||
}
|
||||
|
||||
func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) error {
|
||||
started := time.Now()
|
||||
l, err := e.loadPair(ctx, pairID)
|
||||
if err != nil {
|
||||
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), nil)
|
||||
return err
|
||||
}
|
||||
warns := layoutWarnings(l)
|
||||
|
||||
j.phase("Indexing host library", 0)
|
||||
hostFiles := map[string]fsops.Info{}
|
||||
hs, err := walk(ctx, l, l.host, hostFiles)
|
||||
if err != nil {
|
||||
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
|
||||
return err
|
||||
}
|
||||
j.phase("Indexing slave library", 0)
|
||||
slaveFiles := map[string]fsops.Info{}
|
||||
ss, err := walk(ctx, l, l.slave, slaveFiles)
|
||||
if err != nil {
|
||||
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
|
||||
return err
|
||||
}
|
||||
ix := buildIndex(hostFiles)
|
||||
prev, err := e.st.AllMatches(ctx, pairID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
j.phase("Comparing files", int64(len(slaveFiles)))
|
||||
keep := map[string]bool{}
|
||||
kinds := map[string]int{}
|
||||
for _, rel := range sortedKeys(slaveFiles) {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
si := slaveFiles[rel]
|
||||
m := store.Match{PairID: pairID, RelSlave: rel}
|
||||
m.RelHost, m.Kind = ix.find(rel)
|
||||
var pm *store.Match
|
||||
if p, ok := prev[rel]; ok {
|
||||
pm = &p
|
||||
}
|
||||
e.classify(ctx, l, &m, hostFiles[m.RelHost], si, pm)
|
||||
if err := e.st.UpsertMatch(ctx, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
keep[rel] = true
|
||||
kinds[m.Kind]++
|
||||
j.count(m.Status)
|
||||
j.tick()
|
||||
}
|
||||
if err := e.st.DeleteMatchesExcept(ctx, pairID, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Layout warnings based on what was actually found.
|
||||
if ss.files > 0 {
|
||||
if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files {
|
||||
warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the host library. "+
|
||||
"Check that both libraries use the same Artist/Album/Title.ext layout.", n, ss.files))
|
||||
}
|
||||
if ss.offDepth > 0 {
|
||||
warns = append(warns, fmt.Sprintf("%d slave files are not at Artist/Album/Title.ext depth.", ss.offDepth))
|
||||
}
|
||||
}
|
||||
if hs.offDepth > 0 {
|
||||
warns = append(warns, fmt.Sprintf("%d host files are not at Artist/Album/Title.ext depth.", hs.offDepth))
|
||||
}
|
||||
if hs.symlinks+ss.symlinks > 0 {
|
||||
warns = append(warns, fmt.Sprintf("Ignored %d symlinks (playlink never follows or replaces symlinks).", hs.symlinks+ss.symlinks))
|
||||
}
|
||||
if e.prober.Bin == "" {
|
||||
warns = append(warns, "ffprobe was not found, so codec and quality details are unavailable for manual review.")
|
||||
}
|
||||
if err := e.st.FinishScan(ctx, pairID, time.Now().Unix(), "", warns); err != nil {
|
||||
return err
|
||||
}
|
||||
j.update(func(j *Job) {
|
||||
j.Message = fmt.Sprintf("Scanned %d host and %d slave files in %s", hs.files, ss.files, time.Since(started).Round(time.Second))
|
||||
})
|
||||
|
||||
if l.pair.AutoLinkIdentical {
|
||||
return e.autoLink(ctx, j, l, trigger)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// autoLink links settled, byte-identical, same-path files after a scan.
|
||||
func (e *Engine) autoLink(ctx context.Context, j *Job, l *libs, trigger string) error {
|
||||
ids, err := e.st.IdenticalIDs(ctx, l.pair.ID)
|
||||
if err != nil || len(ids) == 0 {
|
||||
return err
|
||||
}
|
||||
if e.st.DryRun(ctx) {
|
||||
j.update(func(j *Job) {
|
||||
j.Message += fmt.Sprintf(". Dry run is on: %d identical files would be auto-linked.", len(ids))
|
||||
})
|
||||
return nil
|
||||
}
|
||||
return e.linkIDs(ctx, j, l.pair.ID, ids, "auto:"+trigger, true)
|
||||
}
|
||||
|
||||
// linkIDs links a batch of identical matches.
|
||||
func (e *Engine) linkIDs(ctx context.Context, j *Job, pairID int64, ids []int64, trigger string, onlySettled bool) error {
|
||||
j.phase("Linking identical files", int64(len(ids)))
|
||||
settle := time.Duration(0)
|
||||
if onlySettled {
|
||||
if p, err := e.st.Pair(ctx, pairID); err == nil {
|
||||
settle = time.Duration(p.SettleMinutes) * time.Minute
|
||||
}
|
||||
}
|
||||
cutoff := time.Now().Add(-settle).UnixNano()
|
||||
for _, id := range ids {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
m, err := e.st.Match(ctx, id)
|
||||
if err != nil {
|
||||
j.tick()
|
||||
continue
|
||||
}
|
||||
if onlySettled && m.Kind != store.KindPath {
|
||||
j.skip(m.RelSlave + ": path differs by case, link manually")
|
||||
j.tick()
|
||||
continue
|
||||
}
|
||||
if settle > 0 && (m.SlaveMtime > cutoff || m.HostMtime > cutoff) {
|
||||
j.skip(m.RelSlave + ": modified recently, waiting for it to settle")
|
||||
j.tick()
|
||||
continue
|
||||
}
|
||||
op, err := e.Link(ctx, id, LinkOptions{Trigger: trigger})
|
||||
switch {
|
||||
case err != nil:
|
||||
j.skip(m.RelSlave + ": " + err.Error())
|
||||
case op.DryRun:
|
||||
j.count("planned")
|
||||
default:
|
||||
j.count("linked")
|
||||
}
|
||||
j.tick()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartLinkIdentical links every identical match of a pair in the background.
|
||||
func (e *Engine) StartLinkIdentical(pairID int64) (*Job, error) {
|
||||
return e.startJob(pairID, "link-identical", func(ctx context.Context, j *Job) error {
|
||||
ids, err := e.st.IdenticalIDs(ctx, pairID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
j.update(func(j *Job) { j.Message = "Nothing to link" })
|
||||
return nil
|
||||
}
|
||||
return e.linkIDs(ctx, j, pairID, ids, "bulk", false)
|
||||
})
|
||||
}
|
||||
|
||||
// refresh re-evaluates a single slave file after an operation.
|
||||
func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) {
|
||||
si, err := fsops.RegularFile(filepath.Join(l.slave.Path, relSlave))
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
_ = e.st.DeleteMatch(ctx, l.pair.ID, relSlave)
|
||||
}
|
||||
return
|
||||
}
|
||||
m := store.Match{PairID: l.pair.ID, RelSlave: relSlave}
|
||||
var pm *store.Match
|
||||
if p, err := e.st.MatchBySlave(ctx, l.pair.ID, relSlave); err == nil {
|
||||
pm = &p
|
||||
}
|
||||
var hi fsops.Info
|
||||
if hp, err := l.host.Resolve(relSlave); err == nil {
|
||||
if in, err := fsops.RegularFile(hp); err == nil {
|
||||
hi = in
|
||||
m.RelHost, m.Kind = relSlave, store.KindPath
|
||||
}
|
||||
}
|
||||
if m.Kind == "" {
|
||||
if pm != nil && pm.RelHost != "" {
|
||||
if in, err := fsops.RegularFile(filepath.Join(l.host.Path, pm.RelHost)); err == nil {
|
||||
hi = in
|
||||
m.RelHost, m.Kind = pm.RelHost, pm.Kind
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.Kind == "" {
|
||||
m.Kind = store.KindNone
|
||||
}
|
||||
e.classify(ctx, l, &m, hi, si, pm)
|
||||
_ = e.st.UpsertMatch(ctx, &m)
|
||||
}
|
||||
Reference in new issue
Block a user