Initial commit: playlink music library hardlink deduplicator

Go backend (chi, SQLite) with an embedded React/TypeScript UI that finds
songs present in both a host and a slave music library and replaces the
slave copy with a hardlink to the host copy.

- Path-first matching with format/title/album-folder fallbacks
- Auto-linking of byte-identical files, re-verified with cmp before linking
- Manual review with ffprobe quality comparison, downgrade guard and
  promote-to-host override
- Build mode to populate an empty slave library from the host
- Dry run by default, atomic stage+rename replacement, quarantine with
  retention, undo, path confinement and authenticated UI
- Dockerfile and compose file for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed 2026-09-22 15:38:25 -07:00
commit ac47e61b08
42 files changed
+8892

No files matched your search

+352
View File
@@ -0,0 +1,352 @@
// Package engine implements scanning, matching and the hardlink operations.
package engine
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/cdnutter/playlink/internal/fsops"
"github.com/cdnutter/playlink/internal/probe"
"github.com/cdnutter/playlink/internal/store"
)
// DefaultExtensions are the audio file types considered by default.
var DefaultExtensions = []string{"flac", "mp3", "m4a", "aac", "alac", "ogg", "oga", "opus", "wav", "aif", "aiff", "wma", "ape", "wv", "dsf", "dff", "mka"}
// QuarantineDirName is the default quarantine folder created in the slave root.
const QuarantineDirName = ".playlink-trash"
var (
ErrBusy = errors.New("another job is already running for this library pair")
ErrDowngrade = errors.New("the host copy is lower quality than the slave copy; confirm the downgrade to continue")
ErrConfirm = errors.New("this action modifies the host library and must be confirmed")
)
type Engine struct {
st *store.Store
prober probe.Prober
allowedRoots []string
log *slog.Logger
// mu serializes every filesystem mutation across all pairs. Throughput
// is irrelevant next to the guarantee that two operations never race on
// the same files.
mu sync.Mutex
jobsMu sync.Mutex
jobs map[int64]*Job
nextID int64
}
func New(st *store.Store, prober probe.Prober, allowedRoots []string, log *slog.Logger) *Engine {
return &Engine{st: st, prober: prober, allowedRoots: allowedRoots, log: log, jobs: map[int64]*Job{}}
}
func (e *Engine) AllowedRoots() []string { return e.allowedRoots }
// ---- jobs ----
type Job struct {
ID int64 `json:"id"`
PairID int64 `json:"pair_id"`
Kind string `json:"kind"`
State string `json:"state"` // running, done, failed, cancelled
Phase string `json:"phase"`
Done int64 `json:"done"`
Total int64 `json:"total"`
Message string `json:"message"`
Error string `json:"error"`
Results map[string]int `json:"results"`
Skipped []string `json:"skipped"`
StartedAt int64 `json:"started_at"`
FinishedAt int64 `json:"finished_at,omitempty"`
cancel context.CancelFunc
e *Engine
}
func (j *Job) update(f func(j *Job)) {
j.e.jobsMu.Lock()
f(j)
j.e.jobsMu.Unlock()
}
func (j *Job) phase(p string, total int64) {
j.update(func(j *Job) { j.Phase, j.Total, j.Done = p, total, 0 })
}
func (j *Job) tick() { j.update(func(j *Job) { j.Done++ }) }
func (j *Job) count(key string) { j.update(func(j *Job) { j.Results[key]++ }) }
func (j *Job) skip(msg string) {
j.update(func(j *Job) {
j.Results["skipped"]++
if len(j.Skipped) < 200 {
j.Skipped = append(j.Skipped, msg)
}
})
}
// Job returns a snapshot of the latest job for a pair.
func (e *Engine) Job(pairID int64) *Job {
e.jobsMu.Lock()
defer e.jobsMu.Unlock()
j, ok := e.jobs[pairID]
if !ok {
return nil
}
cp := *j
cp.Results = map[string]int{}
for k, v := range j.Results {
cp.Results[k] = v
}
cp.Skipped = append([]string{}, j.Skipped...) // never nil: the UI expects an array
return &cp
}
func (e *Engine) CancelJob(pairID int64) bool {
e.jobsMu.Lock()
defer e.jobsMu.Unlock()
if j, ok := e.jobs[pairID]; ok && j.State == "running" {
j.cancel()
return true
}
return false
}
func (e *Engine) startJob(pairID int64, kind string, run func(ctx context.Context, j *Job) error) (*Job, error) {
e.jobsMu.Lock()
if j, ok := e.jobs[pairID]; ok && j.State == "running" {
e.jobsMu.Unlock()
return nil, ErrBusy
}
e.nextID++
ctx, cancel := context.WithCancel(context.Background())
j := &Job{ID: e.nextID, PairID: pairID, Kind: kind, State: "running", Results: map[string]int{},
Skipped: []string{}, StartedAt: time.Now().Unix(), cancel: cancel, e: e}
e.jobs[pairID] = j
e.jobsMu.Unlock()
go func() {
defer cancel()
err := run(ctx, j)
j.update(func(j *Job) {
j.FinishedAt = time.Now().Unix()
switch {
case errors.Is(err, context.Canceled):
j.State, j.Error = "cancelled", "cancelled"
case err != nil:
j.State, j.Error = "failed", err.Error()
default:
j.State = "done"
}
})
if err != nil {
e.log.Warn("job finished with error", "pair", pairID, "kind", kind, "err", err)
} else {
e.log.Info("job finished", "pair", pairID, "kind", kind)
}
}()
return e.Job(pairID), nil
}
// ---- pair validation ----
type libs struct {
pair store.Pair
host fsops.Root
slave fsops.Root
quarantine string
dev uint64
exts map[string]bool
}
func (l *libs) hasExt(name string) bool {
return l.exts[strings.TrimPrefix(strings.ToLower(filepath.Ext(name)), ".")]
}
func (e *Engine) withinAllowed(p string) bool {
for _, r := range e.allowedRoots {
if fsops.Within(r, p) {
return true
}
}
return false
}
// open resolves a pair's directories and re-checks every invariant. It is
// called before every scan and operation so configuration drift (a remount,
// a moved folder) is caught before anything is touched.
func (e *Engine) open(p store.Pair) (*libs, error) {
host, err := fsops.NewRoot(p.HostRoot)
if err != nil {
return nil, fmt.Errorf("host library: %w", err)
}
slave, err := fsops.NewRoot(p.SlaveRoot)
if err != nil {
return nil, fmt.Errorf("slave library: %w", err)
}
for _, r := range []string{host.Path, slave.Path} {
if !e.withinAllowed(r) {
return nil, fmt.Errorf("%s is outside the allowed roots %v", r, e.allowedRoots)
}
}
if fsops.Within(host.Path, slave.Path) || fsops.Within(slave.Path, host.Path) {
return nil, errors.New("host and slave libraries must not overlap or contain each other")
}
hi, err := fsops.Lstat(host.Path)
if err != nil {
return nil, err
}
si, err := fsops.Lstat(slave.Path)
if err != nil {
return nil, err
}
if hi.Dev != si.Dev {
return nil, fmt.Errorf("host and slave are on different filesystems (device %d vs %d): %w. "+
"In Docker, mount a single common parent folder instead of two separate volumes", hi.Dev, si.Dev, fsops.ErrCrossDevice)
}
q := filepath.Clean(p.QuarantineDir)
if !filepath.IsAbs(q) {
return nil, errors.New("quarantine directory must be absolute")
}
if fsops.Within(host.Path, q) {
return nil, errors.New("quarantine directory must not be inside the host library")
}
if q == slave.Path || !e.withinAllowed(q) {
return nil, errors.New("quarantine directory must be inside the allowed roots and not the slave root itself")
}
if err := os.MkdirAll(q, 0o775); err != nil {
return nil, fmt.Errorf("quarantine directory: %w", err)
}
if rq, err := filepath.EvalSymlinks(q); err != nil || (rq != q && !e.withinAllowed(rq)) {
return nil, errors.New("quarantine directory resolves outside the allowed roots")
}
qi, err := fsops.Lstat(q)
if err != nil {
return nil, err
}
if qi.Dev != hi.Dev {
return nil, errors.New("quarantine directory must be on the same filesystem as the libraries")
}
exts := map[string]bool{}
for _, x := range p.Extensions {
x = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(x)), ".")
if x != "" {
exts[x] = true
}
}
if len(exts) == 0 {
return nil, errors.New("no file extensions configured")
}
return &libs{pair: p, host: host, slave: slave, quarantine: q, dev: hi.Dev, exts: exts}, nil
}
// NormalizePair fills defaults and validates a pair. It returns non-fatal
// warnings (such as layout mismatches) for the UI.
func (e *Engine) NormalizePair(p *store.Pair) ([]string, error) {
p.Name = strings.TrimSpace(p.Name)
if p.Name == "" {
return nil, errors.New("name is required")
}
p.HostRoot = filepath.Clean(strings.TrimSpace(p.HostRoot))
p.SlaveRoot = filepath.Clean(strings.TrimSpace(p.SlaveRoot))
if strings.TrimSpace(p.QuarantineDir) == "" {
p.QuarantineDir = filepath.Join(p.SlaveRoot, QuarantineDirName)
}
p.QuarantineDir = filepath.Clean(strings.TrimSpace(p.QuarantineDir))
if len(p.Extensions) == 0 || (len(p.Extensions) == 1 && strings.TrimSpace(p.Extensions[0]) == "") {
p.Extensions = DefaultExtensions
}
var clean []string
for _, x := range p.Extensions {
x = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(x)), ".")
if x != "" && !strings.ContainsAny(x, ",/") {
clean = append(clean, x)
}
}
p.Extensions = clean
if p.RetentionDays < 0 || p.ScanIntervalMinutes < 0 || p.SettleMinutes < 0 {
return nil, errors.New("intervals must not be negative")
}
if p.ScanIntervalMinutes > 0 && p.ScanIntervalMinutes < 5 {
return nil, errors.New("scan interval must be 0 (manual) or at least 5 minutes")
}
l, err := e.open(*p)
if err != nil {
return nil, err
}
// Store resolved paths so later containment checks compare like with like.
p.HostRoot, p.SlaveRoot, p.QuarantineDir = l.host.Path, l.slave.Path, l.quarantine
return layoutWarnings(l), nil
}
// layoutWarnings does a cheap top-level comparison of the two libraries.
func layoutWarnings(l *libs) []string {
var warns []string
list := func(root string) (dirs map[string]bool, looseFiles int) {
dirs = map[string]bool{}
ents, _ := os.ReadDir(root)
for _, en := range ents {
if strings.HasPrefix(en.Name(), ".") {
continue
}
if en.IsDir() {
dirs[strings.ToLower(en.Name())] = true
} else if l.hasExt(en.Name()) {
looseFiles++
}
}
return
}
hd, hl := list(l.host.Path)
sd, sl := list(l.slave.Path)
if len(sd) == 0 && sl == 0 {
warns = append(warns, "The slave library is empty. Use Build to pick songs from the host library to hardlink into it.")
return warns
}
if hl > 0 || sl > 0 {
warns = append(warns, fmt.Sprintf("Found audio files directly in a library root (host: %d, slave: %d). Both libraries should use Artist/Album/Title.ext.", hl, sl))
}
shared := 0
for d := range sd {
if hd[d] {
shared++
}
}
if len(sd) > 0 && shared*2 < len(sd) {
warns = append(warns, fmt.Sprintf("Only %d of %d artist folders in the slave library also exist in the host library. "+
"The two libraries should share the same Artist/Album/Title.ext layout, or most files will not match.", shared, len(sd)))
}
return warns
}
// ---- helpers ----
func (e *Engine) loadPair(ctx context.Context, id int64) (*libs, error) {
p, err := e.st.Pair(ctx, id)
if err != nil {
return nil, err
}
return e.open(p)
}
func sortedKeys(m map[string]fsops.Info) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
+481
View File
@@ -0,0 +1,481 @@
package engine
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"os"
"path/filepath"
"testing"
"time"
"github.com/cdnutter/playlink/internal/fsops"
"github.com/cdnutter/playlink/internal/probe"
"github.com/cdnutter/playlink/internal/store"
)
type env struct {
t *testing.T
root string
host string
slave string
st *store.Store
e *Engine
pair store.Pair
}
func setup(t *testing.T, dryRun bool) *env {
t.Helper()
root, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
ev := &env{t: t, root: root, host: filepath.Join(root, "Albums"), slave: filepath.Join(root, "Singles")}
for _, d := range []string{ev.host, ev.slave, filepath.Join(root, "db")} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
st, err := store.Open(filepath.Join(root, "db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.DB.Close() })
v := "0"
if dryRun {
v = "1"
}
if err := st.SetSetting(context.Background(), "dry_run", v); err != nil {
t.Fatal(err)
}
ev.st = st
ev.e = New(st, probe.Prober{}, []string{root}, slog.New(slog.NewTextHandler(io.Discard, nil)))
ev.pair = store.Pair{Name: "test", HostRoot: ev.host, SlaveRoot: ev.slave, RetentionDays: 30}
if _, err := ev.e.NormalizePair(&ev.pair); err != nil {
t.Fatal(err)
}
if err := st.SavePair(context.Background(), &ev.pair); err != nil {
t.Fatal(err)
}
return ev
}
func (ev *env) write(base, rel, content string) string {
ev.t.Helper()
p := filepath.Join(base, rel)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
ev.t.Fatal(err)
}
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
ev.t.Fatal(err)
}
return p
}
func (ev *env) read(p string) string {
ev.t.Helper()
b, err := os.ReadFile(p)
if err != nil {
ev.t.Fatal(err)
}
return string(b)
}
func (ev *env) scan() {
ev.t.Helper()
if _, err := ev.e.StartScan(ev.pair.ID, "test"); err != nil {
ev.t.Fatal(err)
}
ev.wait()
}
func (ev *env) wait() *Job {
ev.t.Helper()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if j := ev.e.Job(ev.pair.ID); j != nil && j.State != "running" {
if j.State != "done" {
ev.t.Fatalf("job %s: %s", j.State, j.Error)
}
return j
}
time.Sleep(10 * time.Millisecond)
}
ev.t.Fatal("job did not finish")
return nil
}
func (ev *env) match(rel string) store.Match {
ev.t.Helper()
m, err := ev.st.MatchBySlave(context.Background(), ev.pair.ID, rel)
if err != nil {
ev.t.Fatalf("match %s: %v", rel, err)
}
return m
}
func sameInode(t *testing.T, a, b string) bool {
t.Helper()
ia, err := fsops.Lstat(a)
if err != nil {
t.Fatal(err)
}
ib, err := fsops.Lstat(b)
if err != nil {
t.Fatal(err)
}
return ia.SameInode(ib)
}
var ctx = context.Background()
func TestIdenticalLinkAndUndo(t *testing.T) {
ev := setup(t, false)
rel := "bbno$/Bag or Die/edamame.flac"
h := ev.write(ev.host, rel, "same-bytes")
s := ev.write(ev.slave, rel, "same-bytes")
ev.scan()
m := ev.match(rel)
if m.Status != store.StatusIdentical || m.Kind != store.KindPath {
t.Fatalf("want identical/path, got %s/%s", m.Status, m.Kind)
}
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
if err != nil {
t.Fatal(err)
}
if op.QuarantinePath != "" {
t.Fatal("identical link must not quarantine")
}
if !sameInode(t, h, s) {
t.Fatal("slave is not a hardlink of host")
}
if ev.match(rel).Status != store.StatusLinked {
t.Fatal("match not marked linked")
}
if err := ev.e.Undo(ctx, op.ID); err != nil {
t.Fatal(err)
}
if sameInode(t, h, s) || ev.read(s) != "same-bytes" {
t.Fatal("undo should leave an independent identical copy")
}
}
func TestDifferentLinkQuarantinesAndUndoRestores(t *testing.T) {
ev := setup(t, false)
rel := "Artist/Album/song.flac"
h := ev.write(ev.host, rel, "host-version-longer")
s := ev.write(ev.slave, rel, "slave-version")
ev.scan()
m := ev.match(rel)
if m.Status != store.StatusDifferent {
t.Fatalf("want different, got %s", m.Status)
}
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
if err != nil {
t.Fatal(err)
}
if !sameInode(t, h, s) {
t.Fatal("not linked")
}
if op.QuarantinePath == "" || ev.read(op.QuarantinePath) != "slave-version" {
t.Fatal("original slave file not quarantined")
}
if err := ev.e.Undo(ctx, op.ID); err != nil {
t.Fatal(err)
}
if ev.read(s) != "slave-version" || ev.read(h) != "host-version-longer" {
t.Fatal("undo did not restore the original")
}
if _, err := os.Stat(op.QuarantinePath); !errors.Is(err, os.ErrNotExist) {
t.Fatal("quarantine entry should be consumed by the restore")
}
if err := ev.e.Undo(ctx, op.ID); err == nil {
t.Fatal("double undo must fail")
}
}
func TestSimilarExtensionLinkRenamesAndUndo(t *testing.T) {
ev := setup(t, false)
h := ev.write(ev.host, "A/B/01 - Song.flac", "flac-data")
s := ev.write(ev.slave, "A/B/01 - Song.mp3", "mp3")
ev.scan()
m := ev.match("A/B/01 - Song.mp3")
if m.Status != store.StatusSimilar || m.Kind != store.KindExtension || m.RelHost != "A/B/01 - Song.flac" {
t.Fatalf("unexpected match %+v", m)
}
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
if err != nil {
t.Fatal(err)
}
newSlave := filepath.Join(ev.slave, "A/B/01 - Song.flac")
if !sameInode(t, h, newSlave) {
t.Fatal("slave should now hold the host's .flac as a hardlink")
}
if _, err := os.Stat(s); !errors.Is(err, os.ErrNotExist) {
t.Fatal("old .mp3 should be moved to quarantine")
}
if err := ev.e.Undo(ctx, op.ID); err != nil {
t.Fatal(err)
}
if ev.read(s) != "mp3" {
t.Fatal(".mp3 not restored")
}
if _, err := os.Stat(newSlave); !errors.Is(err, os.ErrNotExist) {
t.Fatal("linked .flac should be removed on undo")
}
if ev.read(h) != "flac-data" {
t.Fatal("host must be untouched")
}
}
func TestFuzzyTitleMatch(t *testing.T) {
ev := setup(t, false)
ev.write(ev.host, "A/B/03. My Song!.flac", "x")
ev.write(ev.slave, "A/B/My Song.mp3", "y")
ev.write(ev.slave, "A/B/My Song (Live).mp3", "z")
ev.write(ev.host, "A/Other Album/Lonely.flac", "x")
ev.write(ev.slave, "A/Single/Lonely.flac", "x")
ev.scan()
if m := ev.match("A/B/My Song.mp3"); m.Kind != store.KindFuzzy {
t.Fatalf("want fuzzy, got %s", m.Kind)
}
if m := ev.match("A/B/My Song (Live).mp3"); m.Status != store.StatusUnmatched {
t.Fatalf("live version must not match studio version, got %s", m.Status)
}
if m := ev.match("A/Single/Lonely.flac"); m.Kind != store.KindElsewhere || m.Status != store.StatusSimilar {
t.Fatalf("want elsewhere/similar, got %s/%s", m.Kind, m.Status)
}
}
func TestDowngradeGuardAndPromote(t *testing.T) {
ev := setup(t, false)
rel := "A/B/song.flac"
h := ev.write(ev.host, rel, "lossy-host")
s := ev.write(ev.slave, rel, "lossless-slave!")
ev.scan()
m := ev.match(rel)
// Simulate the probe verdict that the slave is better.
v, _ := json.Marshal(probe.Verdict{Better: "slave"})
m.Verdict = v
if err := ev.st.UpsertMatch(ctx, &m); err != nil {
t.Fatal(err)
}
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); !errors.Is(err, ErrDowngrade) {
t.Fatalf("want ErrDowngrade, got %v", err)
}
if _, err := ev.e.Promote(ctx, m.ID, false); !errors.Is(err, ErrConfirm) {
t.Fatalf("promote without confirm must fail, got %v", err)
}
op, err := ev.e.Promote(ctx, m.ID, true)
if err != nil {
t.Fatal(err)
}
if !sameInode(t, h, s) || ev.read(h) != "lossless-slave!" {
t.Fatal("host should now be the slave's file")
}
if ev.read(op.QuarantinePath) != "lossy-host" {
t.Fatal("host original not quarantined")
}
if err := ev.e.Undo(ctx, op.ID); err != nil {
t.Fatal(err)
}
if ev.read(h) != "lossy-host" || ev.read(s) != "lossless-slave!" || sameInode(t, h, s) {
t.Fatal("promote undo failed")
}
}
func TestChangedSinceScanIsRefused(t *testing.T) {
ev := setup(t, false)
rel := "A/B/c.flac"
ev.write(ev.host, rel, "aaa")
s := ev.write(ev.slave, rel, "bbb")
ev.scan()
m := ev.match(rel)
time.Sleep(10 * time.Millisecond)
if err := os.WriteFile(s, []byte("user edited this"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); !errors.Is(err, fsops.ErrChanged) {
t.Fatalf("want ErrChanged, got %v", err)
}
if ev.read(s) != "user edited this" {
t.Fatal("file must be untouched")
}
}
func TestDryRunChangesNothing(t *testing.T) {
ev := setup(t, true)
rel := "A/B/d.flac"
h := ev.write(ev.host, rel, "same")
s := ev.write(ev.slave, rel, "same")
ev.scan()
op, err := ev.e.Link(ctx, ev.match(rel).ID, LinkOptions{})
if err != nil {
t.Fatal(err)
}
if !op.DryRun || op.Status != store.OpPlanned {
t.Fatalf("expected planned dry-run op, got %+v", op)
}
if sameInode(t, h, s) {
t.Fatal("dry run must not link")
}
}
func TestAutoLinkRespectsSettleTime(t *testing.T) {
ev := setup(t, false)
ev.pair.AutoLinkIdentical = true
ev.pair.SettleMinutes = 60
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
t.Fatal(err)
}
h := ev.write(ev.host, "A/B/e.flac", "same")
s := ev.write(ev.slave, "A/B/e.flac", "same")
ev.scan()
if sameInode(t, h, s) {
t.Fatal("freshly written files must not be auto-linked")
}
ev.pair.SettleMinutes = 0
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
t.Fatal(err)
}
ev.scan()
if !sameInode(t, h, s) {
t.Fatal("settled identical file should be auto-linked")
}
}
func TestBuildAndRemove(t *testing.T) {
ev := setup(t, false)
h1 := ev.write(ev.host, "bbno$/Album/one.flac", "1")
ev.write(ev.host, "bbno$/Album/two.flac", "2")
ev.write(ev.host, "bbno$/Album/cover.jpg", "img")
ev.write(ev.slave, "bbno$/Album/two.flac", "user's own different file")
if _, err := ev.e.StartBuild(ev.pair.ID, []string{"bbno$"}); err != nil {
t.Fatal(err)
}
j := ev.wait()
if j.Results["linked"] != 1 || j.Results["skipped"] != 1 {
t.Fatalf("unexpected results %+v", j.Results)
}
s1 := filepath.Join(ev.slave, "bbno$/Album/one.flac")
if !sameInode(t, h1, s1) {
t.Fatal("build did not hardlink")
}
if ev.read(filepath.Join(ev.slave, "bbno$/Album/two.flac")) != "user's own different file" {
t.Fatal("build must never overwrite an existing slave file")
}
if _, err := os.Stat(filepath.Join(ev.slave, "bbno$/Album/cover.jpg")); err == nil {
t.Fatal("non-audio files must not be linked")
}
if _, err := ev.e.StartRemove(ev.pair.ID, []string{"bbno$"}); err != nil {
t.Fatal(err)
}
j = ev.wait()
if j.Results["removed"] != 1 || j.Results["skipped"] != 1 {
t.Fatalf("unexpected remove results %+v", j.Results)
}
if _, err := os.Stat(s1); !errors.Is(err, os.ErrNotExist) {
t.Fatal("linked file should be removed")
}
if ev.read(h1) != "1" {
t.Fatal("host must be untouched")
}
if ev.read(filepath.Join(ev.slave, "bbno$/Album/two.flac")) != "user's own different file" {
t.Fatal("remove must never delete a non-linked file")
}
}
func TestSymlinkEscapeIgnored(t *testing.T) {
ev := setup(t, false)
outside := t.TempDir()
os.WriteFile(filepath.Join(outside, "secret.flac"), []byte("x"), 0o644)
os.MkdirAll(filepath.Join(ev.slave, "Evil"), 0o755)
if err := os.Symlink(outside, filepath.Join(ev.slave, "Evil", "Album")); err != nil {
t.Fatal(err)
}
ev.write(ev.host, "Evil/Album/secret.flac", "x")
ev.scan()
if _, err := ev.st.MatchBySlave(ctx, ev.pair.ID, "Evil/Album/secret.flac"); err == nil {
t.Fatal("files behind a symlinked directory must not be scanned")
}
root, _ := fsops.NewRoot(ev.slave)
if _, err := root.Resolve("Evil/Album/secret.flac"); !errors.Is(err, fsops.ErrOutsideRoot) {
t.Fatalf("resolve through an escaping symlink must fail, got %v", err)
}
if _, err := root.Resolve("../Albums/x.flac"); !errors.Is(err, fsops.ErrOutsideRoot) {
t.Fatalf("dot-dot must fail, got %v", err)
}
}
func TestPurgeAndOverlapValidation(t *testing.T) {
ev := setup(t, false)
rel := "A/B/f.flac"
ev.write(ev.host, rel, "one")
ev.write(ev.slave, rel, "two!")
ev.scan()
op, err := ev.e.Link(ctx, ev.match(rel).ID, LinkOptions{})
if err != nil {
t.Fatal(err)
}
if err := ev.e.Purge(ctx, op.ID); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(op.QuarantinePath); !errors.Is(err, os.ErrNotExist) {
t.Fatal("purge did not delete quarantine")
}
if err := ev.e.Undo(ctx, op.ID); err == nil {
t.Fatal("undo after purge must fail")
}
bad := store.Pair{Name: "bad", HostRoot: ev.host, SlaveRoot: filepath.Join(ev.host, "A")}
if _, err := ev.e.NormalizePair(&bad); err == nil {
t.Fatal("nested libraries must be rejected")
}
outside := store.Pair{Name: "bad", HostRoot: ev.host, SlaveRoot: t.TempDir()}
if _, err := ev.e.NormalizePair(&outside); err == nil {
t.Fatal("paths outside allowed roots must be rejected")
}
}
func TestSameContent(t *testing.T) {
dir := t.TempDir()
big := bytes.Repeat([]byte("abcdefgh"), 300_000) // spans several chunks
a, b, c := filepath.Join(dir, "a"), filepath.Join(dir, "b"), filepath.Join(dir, "c")
os.WriteFile(a, big, 0o644)
os.WriteFile(b, big, 0o644)
alt := append([]byte(nil), big...)
alt[len(alt)-1] = 'X'
os.WriteFile(c, alt, 0o644)
if same, err := fsops.SameContent(a, b); err != nil || !same {
t.Fatalf("a,b should match: %v %v", same, err)
}
if same, _ := fsops.SameContent(a, c); same {
t.Fatal("a,c differ in the last byte")
}
}
func TestNormTitle(t *testing.T) {
cases := map[string]string{
"01 - Song.flac": "song",
"1-03 Song.mp3": "song",
"03. Song!.flac": "song",
"22.flac": "22",
"Song (Live).flac": "songlive",
"07 Song (Remix).flac": "songremix",
}
for in, want := range cases {
if got := normTitle(in); got != want {
t.Errorf("normTitle(%q) = %q, want %q", in, got, want)
}
}
}
+834
View File
@@ -0,0 +1,834 @@
package engine
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/cdnutter/playlink/internal/fsops"
"github.com/cdnutter/playlink/internal/probe"
"github.com/cdnutter/playlink/internal/store"
)
type LinkOptions struct {
// AllowDowngrade confirms linking when the host copy scored lower.
AllowDowngrade bool
Trigger string
}
// snapshot holds the verified state of both files right before an operation.
type snapshot struct {
hostAbs, slaveAbs string
hi, si fsops.Info
}
// preflight re-validates a match against the filesystem: both files must be
// plain regular files inside their roots, on one device, and unchanged since
// the scan recorded them.
func preflight(l *libs, m store.Match) (snapshot, error) {
var s snapshot
var err error
if s.hostAbs, err = l.host.Resolve(m.RelHost); err != nil {
return s, err
}
if s.slaveAbs, err = l.slave.Resolve(m.RelSlave); err != nil {
return s, err
}
if s.hi, err = fsops.RegularFile(s.hostAbs); err != nil {
return s, fmt.Errorf("host file: %w", err)
}
if s.si, err = fsops.RegularFile(s.slaveAbs); err != nil {
return s, fmt.Errorf("slave file: %w", err)
}
if s.hi.Ino != m.HostIno || !s.hi.Unchanged(m.HostSize, m.HostMtime) {
return s, fmt.Errorf("host %w", fsops.ErrChanged)
}
if s.si.Ino != m.SlaveIno || !s.si.Unchanged(m.SlaveSize, m.SlaveMtime) {
return s, fmt.Errorf("slave %w", fsops.ErrChanged)
}
if s.hi.Dev != s.si.Dev || s.hi.Dev != l.dev {
return s, fsops.ErrCrossDevice
}
if s.hi.SameInode(s.si) {
return s, errors.New("already linked")
}
return s, nil
}
func quarantinePath(l *libs, opID int64, side, rel string) string {
return filepath.Join(l.quarantine, strconv.FormatInt(opID, 10), side, rel)
}
// quarantine adds an extra hardlink of p (whose identity is in) inside the
// quarantine. The original stays in place; nothing is lost if a later step
// fails. The caller removes or replaces p afterwards.
func quarantine(p string, in fsops.Info, q string) error {
if err := fsops.MkdirAllLike(filepath.Dir(q)); err != nil {
return err
}
return fsops.LinkNew(p, in, q)
}
// dropQuarantine removes a quarantine link that turned out not to be needed.
func dropQuarantine(l *libs, q string, keep fsops.Info) {
if err := fsops.RemoveLink(q, keep); err == nil {
pruneEmpty(filepath.Dir(q), l.quarantine)
}
}
// removeOriginal deletes an original file after it was safely superseded. The
// inode must still be the one we verified.
func removeOriginal(p string, expect fsops.Info) error {
now, err := fsops.RegularFile(p)
if err != nil {
return err
}
if !now.SameInode(expect) || !now.Unchanged(expect.Size, expect.MtimeNs) {
return fmt.Errorf("%s %w", p, fsops.ErrChanged)
}
if err := os.Remove(p); err != nil {
return err
}
return fsops.SyncDir(filepath.Dir(p))
}
// pruneEmpty removes empty directories from dir upward, stopping at stop.
func pruneEmpty(dir, stop string) {
for dir != stop && fsops.Within(stop, dir) {
if err := os.Remove(dir); err != nil {
return
}
dir = filepath.Dir(dir)
}
}
func (e *Engine) begin(ctx context.Context, o *store.Operation) error {
o.DryRun = e.st.DryRun(ctx)
if o.Trigger == "" {
o.Trigger = "manual"
}
o.Status = store.OpRunning
if o.DryRun {
o.Status = store.OpPlanned
}
return e.st.CreateOp(ctx, o)
}
func (e *Engine) finish(ctx context.Context, o *store.Operation, err error) error {
if err != nil {
o.Status, o.Error = store.OpFailed, err.Error()
e.log.Warn("operation failed", "op", o.ID, "action", o.Action, "slave", o.RelSlave, "err", err)
} else {
o.Status = store.OpDone
e.log.Info("operation done", "op", o.ID, "action", o.Action, "slave", o.RelSlave, "host", o.RelHost)
}
if ferr := e.st.FinishOp(ctx, o.ID, o.Status, o.Error); ferr != nil && err == nil {
return ferr
}
return err
}
func verdictOf(m store.Match) probe.Verdict {
var v probe.Verdict
_ = json.Unmarshal(m.Verdict, &v)
return v
}
// Link replaces the slave file with a hardlink of the host file.
//
// Identical files are re-compared byte for byte right before linking and no
// quarantine copy is kept (the data exists in the host). Non-identical files
// are quarantined first so the operation can be undone.
func (e *Engine) Link(ctx context.Context, matchID int64, opt LinkOptions) (store.Operation, error) {
e.mu.Lock()
defer e.mu.Unlock()
m, err := e.st.Match(ctx, matchID)
if err != nil {
return store.Operation{}, err
}
l, err := e.loadPair(ctx, m.PairID)
if err != nil {
return store.Operation{}, err
}
switch m.Status {
case store.StatusIdentical, store.StatusDifferent, store.StatusSimilar:
default:
return store.Operation{}, fmt.Errorf("cannot link a match with status %q", m.Status)
}
if m.Ignored {
return store.Operation{}, errors.New("match is ignored; un-ignore it first")
}
identical := m.Status == store.StatusIdentical
if !identical && verdictOf(m).Better == "slave" && !opt.AllowDowngrade {
return store.Operation{}, ErrDowngrade
}
s, err := preflight(l, m)
if err != nil {
return store.Operation{}, err
}
if identical {
same, err := fsops.SameContent(s.hostAbs, s.slaveAbs)
if err != nil {
return store.Operation{}, err
}
if !same {
e.refresh(ctx, l, m.RelSlave)
return store.Operation{}, errors.New("files are no longer identical; moved to manual review")
}
}
newRel := filepath.Join(filepath.Dir(m.RelSlave), filepath.Base(m.RelHost))
newAbs, err := l.slave.Resolve(newRel)
if err != nil {
return store.Operation{}, err
}
rename := newRel != m.RelSlave
if rename {
if _, err := os.Lstat(newAbs); err == nil {
return store.Operation{}, fmt.Errorf("slave already has %s: %w", newRel, fsops.ErrExists)
}
}
o := store.Operation{PairID: l.pair.ID, Action: store.ActLink, RelHost: m.RelHost, RelSlave: m.RelSlave,
RelNew: newRel, Trigger: opt.Trigger}
if identical {
o.Bytes = s.si.Size // space freed
} else {
o.QuarantineSide = "slave"
}
if err := e.begin(ctx, &o); err != nil {
return o, err
}
if o.DryRun {
return o, nil
}
err = func() error {
var q string
if !identical {
q = quarantinePath(l, o.ID, "slave", m.RelSlave)
if err := quarantine(s.slaveAbs, s.si, q); err != nil {
return fmt.Errorf("quarantine: %w", err)
}
o.QuarantinePath = q
if _, err := e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = ? WHERE id = ?`, q, o.ID); err != nil {
dropQuarantine(l, q, s.si)
return err
}
}
if !rename {
if err := fsops.ReplaceWithLink(s.hostAbs, s.hi, s.slaveAbs); err != nil {
if q != "" {
if cur, e2 := fsops.Lstat(s.slaveAbs); e2 == nil && cur.SameInode(s.si) {
dropQuarantine(l, q, s.si) // original untouched
e.clearQuarantine(ctx, o.ID)
}
}
return err
}
return nil
}
if err := fsops.LinkNew(s.hostAbs, s.hi, newAbs); err != nil {
if q != "" {
dropQuarantine(l, q, s.si)
e.clearQuarantine(ctx, o.ID)
}
return err
}
return removeOriginal(s.slaveAbs, s.si)
}()
err = e.finish(ctx, &o, err)
e.refresh(ctx, l, m.RelSlave)
if rename {
e.refresh(ctx, l, newRel)
}
return o, err
}
func (e *Engine) clearQuarantine(ctx context.Context, opID int64) {
_, _ = e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = '' WHERE id = ?`, opID)
}
// Promote is the override path: the slave copy is better, so it replaces the
// host file (whose original is quarantined) and both libraries share it.
func (e *Engine) Promote(ctx context.Context, matchID int64, confirmed bool) (store.Operation, error) {
if !confirmed {
return store.Operation{}, ErrConfirm
}
e.mu.Lock()
defer e.mu.Unlock()
m, err := e.st.Match(ctx, matchID)
if err != nil {
return store.Operation{}, err
}
l, err := e.loadPair(ctx, m.PairID)
if err != nil {
return store.Operation{}, err
}
if m.Status != store.StatusDifferent && m.Status != store.StatusSimilar {
return store.Operation{}, fmt.Errorf("cannot promote a match with status %q", m.Status)
}
s, err := preflight(l, m)
if err != nil {
return store.Operation{}, err
}
newHostRel := filepath.Join(filepath.Dir(m.RelHost), filepath.Base(m.RelSlave))
newHostAbs, err := l.host.Resolve(newHostRel)
if err != nil {
return store.Operation{}, err
}
rename := newHostRel != m.RelHost
if rename {
if _, err := os.Lstat(newHostAbs); err == nil {
return store.Operation{}, fmt.Errorf("host already has %s: %w", newHostRel, fsops.ErrExists)
}
}
o := store.Operation{PairID: l.pair.ID, Action: store.ActPromote, RelHost: m.RelHost, RelSlave: m.RelSlave,
RelNew: newHostRel, QuarantineSide: "host", Trigger: "manual"}
if err := e.begin(ctx, &o); err != nil {
return o, err
}
if o.DryRun {
return o, nil
}
err = func() error {
q := quarantinePath(l, o.ID, "host", m.RelHost)
if err := quarantine(s.hostAbs, s.hi, q); err != nil {
return fmt.Errorf("quarantine: %w", err)
}
o.QuarantinePath = q
if _, err := e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = ? WHERE id = ?`, q, o.ID); err != nil {
dropQuarantine(l, q, s.hi)
return err
}
if !rename {
if err := fsops.ReplaceWithLink(s.slaveAbs, s.si, s.hostAbs); err != nil {
if cur, e2 := fsops.Lstat(s.hostAbs); e2 == nil && cur.SameInode(s.hi) {
dropQuarantine(l, q, s.hi)
e.clearQuarantine(ctx, o.ID)
}
return err
}
return nil
}
if err := fsops.LinkNew(s.slaveAbs, s.si, newHostAbs); err != nil {
dropQuarantine(l, q, s.hi)
e.clearQuarantine(ctx, o.ID)
return err
}
return removeOriginal(s.hostAbs, s.hi)
}()
err = e.finish(ctx, &o, err)
e.refresh(ctx, l, m.RelSlave)
return o, err
}
// ---- build (fresh slave library) and remove ----
// expand turns a list of relative files/folders into audio files under root.
func expand(ctx context.Context, l *libs, root fsops.Root, rels []string) ([]string, error) {
set := map[string]fsops.Info{}
for _, rel := range rels {
abs, err := root.Resolve(rel)
if err != nil {
return nil, err
}
fi, err := os.Lstat(abs)
if err != nil {
return nil, err
}
switch {
case fi.IsDir():
sub, err := fsops.NewRoot(abs)
if err != nil {
return nil, err
}
files := map[string]fsops.Info{}
if _, err := walk(ctx, l, sub, files); err != nil {
return nil, err
}
for r, in := range files {
set[filepath.Join(rel, r)] = in
}
case fi.Mode().IsRegular() && l.hasExt(abs):
set[filepath.Clean(rel)] = fsops.Info{}
}
}
return sortedKeys(set), nil
}
// StartBuild hardlinks the chosen host files/folders into the slave library,
// mirroring the host path. Existing slave files are never overwritten.
func (e *Engine) StartBuild(pairID int64, rels []string) (*Job, error) {
return e.startJob(pairID, "build", func(ctx context.Context, j *Job) error {
l, err := e.loadPair(ctx, pairID)
if err != nil {
return err
}
files, err := expand(ctx, l, l.host, rels)
if err != nil {
return err
}
j.phase("Linking into slave library", int64(len(files)))
for _, rel := range files {
if ctx.Err() != nil {
return ctx.Err()
}
e.buildOne(ctx, j, l, rel)
j.tick()
}
return nil
})
}
func (e *Engine) buildOne(ctx context.Context, j *Job, l *libs, rel string) {
e.mu.Lock()
defer e.mu.Unlock()
hostAbs, err := l.host.Resolve(rel)
if err != nil {
j.skip(rel + ": " + err.Error())
return
}
slaveAbs, err := l.slave.Resolve(rel)
if err != nil {
j.skip(rel + ": " + err.Error())
return
}
hi, err := fsops.RegularFile(hostAbs)
if err != nil {
j.skip(rel + ": " + err.Error())
return
}
if si, err := fsops.Lstat(slaveAbs); err == nil {
if si.SameInode(hi) {
j.count("already linked")
} else {
j.skip(rel + ": a different file already exists in the slave library (scan, then use Review)")
}
return
}
o := store.Operation{PairID: l.pair.ID, Action: store.ActBuild, RelHost: rel, RelSlave: rel, RelNew: rel, Trigger: "build"}
if err := e.begin(ctx, &o); err != nil {
j.skip(rel + ": " + err.Error())
return
}
if o.DryRun {
j.count("planned")
return
}
err = fsops.MkdirAllLike(filepath.Dir(slaveAbs))
if err == nil {
err = fsops.LinkNew(hostAbs, hi, slaveAbs)
}
if err = e.finish(ctx, &o, err); err != nil {
j.skip(rel + ": " + err.Error())
return
}
j.count("linked")
e.refresh(ctx, l, rel)
}
// StartRemove deletes slave files that are hardlinks of the host file at the
// same path. Anything that is not such a link is left alone.
func (e *Engine) StartRemove(pairID int64, rels []string) (*Job, error) {
return e.startJob(pairID, "remove", func(ctx context.Context, j *Job) error {
l, err := e.loadPair(ctx, pairID)
if err != nil {
return err
}
files, err := expand(ctx, l, l.slave, rels)
if err != nil {
return err
}
j.phase("Removing links from slave library", int64(len(files)))
for _, rel := range files {
if ctx.Err() != nil {
return ctx.Err()
}
e.removeOne(ctx, j, l, rel)
j.tick()
}
return nil
})
}
func (e *Engine) removeOne(ctx context.Context, j *Job, l *libs, rel string) {
e.mu.Lock()
defer e.mu.Unlock()
hostAbs, err1 := l.host.Resolve(rel)
slaveAbs, err2 := l.slave.Resolve(rel)
if err1 != nil || err2 != nil {
j.skip(rel + ": invalid path")
return
}
hi, err := fsops.RegularFile(hostAbs)
if err != nil {
j.skip(rel + ": no host file at the same path, so it is not removed")
return
}
si, err := fsops.RegularFile(slaveAbs)
if err != nil {
j.skip(rel + ": " + err.Error())
return
}
if !si.SameInode(hi) {
j.skip(rel + ": not a hardlink of the host file, so it is not removed")
return
}
o := store.Operation{PairID: l.pair.ID, Action: store.ActRemove, RelHost: rel, RelSlave: rel, Trigger: "build"}
if err := e.begin(ctx, &o); err != nil {
j.skip(rel + ": " + err.Error())
return
}
if o.DryRun {
j.count("planned")
return
}
err = fsops.RemoveLink(slaveAbs, hi)
if err = e.finish(ctx, &o, err); err != nil {
j.skip(rel + ": " + err.Error())
return
}
pruneEmpty(filepath.Dir(slaveAbs), l.slave.Path)
j.count("removed")
e.refresh(ctx, l, rel)
}
// ---- undo, restore and purge ----
// Undo reverts a completed operation. Every branch verifies that the current
// file is still exactly what the operation left behind before touching it.
func (e *Engine) Undo(ctx context.Context, opID int64) error {
e.mu.Lock()
defer e.mu.Unlock()
o, err := e.st.Op(ctx, opID)
if err != nil {
return err
}
if o.Status != store.OpDone || o.UndoneAt != nil {
return errors.New("only completed operations that were not undone can be reverted")
}
if o.PurgedAt != nil {
return errors.New("the quarantined original was already purged")
}
l, err := e.loadPair(ctx, o.PairID)
if err != nil {
return err
}
res := func(r fsops.Root, rel string) string {
p, e := r.Resolve(rel)
if e != nil && err == nil {
err = e
}
return p
}
hostAbs := res(l.host, o.RelHost)
slaveAbs := res(l.slave, o.RelSlave)
if err != nil {
return err
}
switch o.Action {
case store.ActBuild:
hi, err := fsops.RegularFile(hostAbs)
if err != nil {
return err
}
if err := fsops.RemoveLink(slaveAbs, hi); err != nil {
return err
}
pruneEmpty(filepath.Dir(slaveAbs), l.slave.Path)
case store.ActRemove:
hi, err := fsops.RegularFile(hostAbs)
if err != nil {
return err
}
if err := fsops.MkdirAllLike(filepath.Dir(slaveAbs)); err != nil {
return err
}
if err := fsops.LinkNew(hostAbs, hi, slaveAbs); err != nil {
return err
}
case store.ActLink:
newAbs := res(l.slave, o.RelNew)
if err != nil {
return err
}
hi, err := fsops.RegularFile(hostAbs)
if err != nil {
return err
}
cur, err := fsops.RegularFile(newAbs)
if err != nil {
return err
}
if !cur.SameInode(hi) {
return errors.New("the slave file is no longer linked to the host file; refusing to undo")
}
renamed := o.RelNew != o.RelSlave
if renamed {
if _, err := os.Lstat(slaveAbs); err == nil {
return fmt.Errorf("%s: %w", o.RelSlave, fsops.ErrExists)
}
}
if o.QuarantinePath == "" {
// Identical link: give the slave its own copy of the bytes again.
if err := fsops.CopyReplace(hostAbs, slaveAbs, hi.Mode); err != nil {
return err
}
} else {
qi, err := e.quarantined(l, o.QuarantinePath)
if err != nil {
return err
}
if renamed {
err = fsops.LinkNew(o.QuarantinePath, qi, slaveAbs)
} else {
err = fsops.ReplaceWithLink(o.QuarantinePath, qi, slaveAbs)
}
if err != nil {
return err
}
dropQuarantine(l, o.QuarantinePath, qi)
}
if renamed {
if err := fsops.RemoveLink(newAbs, hi); err != nil {
return err
}
}
e.refresh(ctx, l, o.RelNew)
case store.ActPromote:
newHostAbs := res(l.host, o.RelNew)
if err != nil {
return err
}
si, err := fsops.RegularFile(slaveAbs)
if err != nil {
return err
}
cur, err := fsops.RegularFile(newHostAbs)
if err != nil {
return err
}
if !cur.SameInode(si) {
return errors.New("the host file changed after promotion; refusing to undo")
}
qi, err := e.quarantined(l, o.QuarantinePath)
if err != nil {
return err
}
if o.RelNew == o.RelHost {
err = fsops.ReplaceWithLink(o.QuarantinePath, qi, hostAbs)
} else {
if _, e2 := os.Lstat(hostAbs); e2 == nil {
return fmt.Errorf("%s: %w", o.RelHost, fsops.ErrExists)
}
if err = fsops.LinkNew(o.QuarantinePath, qi, hostAbs); err == nil {
err = fsops.RemoveLink(newHostAbs, si)
}
}
if err != nil {
return err
}
dropQuarantine(l, o.QuarantinePath, qi)
default:
return fmt.Errorf("cannot undo %q", o.Action)
}
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
return err
}
e.log.Info("operation undone", "op", o.ID, "action", o.Action)
e.refresh(ctx, l, o.RelSlave)
return nil
}
// quarantined validates a quarantine path recorded in the database.
func (e *Engine) quarantined(l *libs, q string) (fsops.Info, error) {
q = filepath.Clean(q)
if !fsops.Within(l.quarantine, q) || q == l.quarantine {
return fsops.Info{}, fmt.Errorf("quarantine path %s: %w", q, fsops.ErrOutsideRoot)
}
return fsops.RegularFile(q)
}
// Purge permanently deletes a quarantined original.
func (e *Engine) Purge(ctx context.Context, opID int64) error {
e.mu.Lock()
defer e.mu.Unlock()
o, err := e.st.Op(ctx, opID)
if err != nil {
return err
}
return e.purge(ctx, o)
}
func (e *Engine) purge(ctx context.Context, o store.Operation) error {
if o.QuarantinePath == "" || o.PurgedAt != nil || o.UndoneAt != nil {
return errors.New("nothing to purge for this operation")
}
l, err := e.loadPair(ctx, o.PairID)
if err != nil {
return err
}
opDir := filepath.Join(l.quarantine, strconv.FormatInt(o.ID, 10))
if !fsops.Within(l.quarantine, filepath.Clean(o.QuarantinePath)) || !fsops.Within(opDir, filepath.Clean(o.QuarantinePath)) {
return fsops.ErrOutsideRoot
}
if err := os.RemoveAll(opDir); err != nil {
return err
}
e.log.Info("quarantine purged", "op", o.ID, "path", o.QuarantinePath)
return e.st.MarkPurged(ctx, o.ID)
}
// Maintenance purges expired quarantine entries and prunes sessions.
func (e *Engine) Maintenance(ctx context.Context) {
e.st.PruneSessions(ctx)
pairs, err := e.st.Pairs(ctx)
if err != nil {
return
}
for _, p := range pairs {
if p.RetentionDays <= 0 {
continue // keep forever
}
cutoff := time.Now().Add(-time.Duration(p.RetentionDays) * 24 * time.Hour).Unix()
ops, err := e.st.ExpiredQuarantine(ctx, p.ID, cutoff)
if err != nil {
continue
}
for _, o := range ops {
e.mu.Lock()
if err := e.purge(ctx, o); err != nil {
e.log.Warn("retention purge failed", "op", o.ID, "err", err)
}
e.mu.Unlock()
}
}
}
// Recover runs at startup: operations left running by a crash are marked
// interrupted and their disposable staging files are removed. Originals are
// never at risk because every replacement is an atomic rename.
func (e *Engine) Recover(ctx context.Context) {
ops, err := e.st.RunningOps(ctx)
if err != nil {
return
}
for _, o := range ops {
if p, err := e.st.Pair(ctx, o.PairID); err == nil {
for _, root := range []string{p.SlaveRoot, p.HostRoot} {
for _, rel := range []string{o.RelSlave, o.RelNew, o.RelHost} {
if rel == "" {
continue
}
dir := filepath.Join(root, filepath.Dir(rel))
ents, _ := os.ReadDir(dir)
for _, en := range ents {
if strings.HasPrefix(en.Name(), fsops.TmpPrefix) && en.Type().IsRegular() {
_ = os.Remove(filepath.Join(dir, en.Name()))
}
}
}
}
}
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, "interrupted by shutdown; rescan to verify state")
e.log.Warn("marked interrupted operation", "op", o.ID)
}
}
// ---- browsing (build mode) ----
type TreeEntry struct {
Name string `json:"name"`
Rel string `json:"rel"`
Dir bool `json:"dir"`
Size int64 `json:"size,omitempty"`
State string `json:"state,omitempty"` // files: linked, conflict, missing
Linked int `json:"linked,omitempty"`
}
// Tree lists a host directory and shows how much of it is in the slave.
func (e *Engine) Tree(ctx context.Context, pairID int64, rel string) ([]TreeEntry, error) {
l, err := e.loadPair(ctx, pairID)
if err != nil {
return nil, err
}
dir := l.host.Path
if rel != "" {
if dir, err = l.host.Resolve(rel); err != nil {
return nil, err
}
}
ents, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
linked, _ := e.st.LinkedHostPaths(ctx, pairID, rel)
perChild := map[string]int{}
for _, p := range linked {
sub := p
if rel != "" {
sub = strings.TrimPrefix(p, rel+string(filepath.Separator))
}
if i := strings.IndexByte(sub, filepath.Separator); i > 0 {
perChild[sub[:i]]++
}
}
out := []TreeEntry{}
for _, en := range ents {
name := en.Name()
if strings.HasPrefix(name, ".") || en.Type()&fs.ModeSymlink != 0 {
continue
}
child := name
if rel != "" {
child = filepath.Join(rel, name)
}
te := TreeEntry{Name: name, Rel: child, Dir: en.IsDir()}
if en.IsDir() {
te.Linked = perChild[name]
} else {
if !l.hasExt(name) {
continue
}
hi, err := fsops.Lstat(filepath.Join(dir, name))
if err != nil {
continue
}
te.Size = hi.Size
te.State = "missing"
if sp, err := l.slave.Resolve(child); err == nil {
if si, err := fsops.Lstat(sp); err == nil {
te.State = "conflict"
if si.SameInode(hi) {
te.State = "linked"
}
}
}
}
out = append(out, te)
}
sort.Slice(out, func(i, j int) bool {
if out[i].Dir != out[j].Dir {
return out[i].Dir
}
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
})
return out, nil
}
+448
View File
@@ -0,0 +1,448 @@
package engine
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"path/filepath"
"regexp"
"strings"
"time"
"unicode"
"github.com/cdnutter/playlink/internal/fsops"
"github.com/cdnutter/playlink/internal/probe"
"github.com/cdnutter/playlink/internal/store"
)
type walkStats struct {
files int
symlinks int
offDepth int // files not at Artist/Album/Title depth
}
// walk lists audio files beneath root. Hidden entries, playlink staging files,
// the quarantine and symlinks are never followed or returned.
func walk(ctx context.Context, l *libs, root fsops.Root, out map[string]fsops.Info) (walkStats, error) {
var ws walkStats
err := filepath.WalkDir(root.Path, func(p string, d fs.DirEntry, err error) error {
if err != nil {
if p == root.Path {
return err
}
return nil // unreadable subtree: skip rather than abort
}
if ctx.Err() != nil {
return ctx.Err()
}
name := d.Name()
if p != root.Path && strings.HasPrefix(name, ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if d.IsDir() {
if p == l.quarantine {
return filepath.SkipDir
}
return nil
}
if d.Type()&fs.ModeSymlink != 0 {
ws.symlinks++
return nil
}
if !d.Type().IsRegular() || !l.hasExt(name) {
return nil
}
in, err := fsops.Lstat(p)
if err != nil || !in.Mode.IsRegular() {
return nil
}
rel, err := root.Rel(p)
if err != nil {
return nil
}
out[rel] = in
ws.files++
if strings.Count(rel, string(filepath.Separator)) != 2 {
ws.offDepth++
}
return nil
})
return ws, err
}
var trackPrefix = regexp.MustCompile(`^\s*(?:\d{1,2}[-.]\s*)?\d{1,3}(?:\s*[-._)]\s*|\s+)`)
// normTitle makes a conservative comparison key from a file name: the track
// number prefix is stripped and punctuation/case ignored. Parenthesised text
// is kept, so "Song (Live)" never matches "Song".
func normTitle(name string) string {
stem := strings.TrimSuffix(name, filepath.Ext(name))
if s := trackPrefix.ReplaceAllString(stem, ""); strings.TrimSpace(s) != "" {
stem = s
}
var b strings.Builder
for _, r := range strings.ToLower(stem) {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
b.WriteRune(r)
}
}
return b.String()
}
func firstComponent(rel string) string {
if i := strings.IndexByte(rel, filepath.Separator); i >= 0 {
return rel[:i]
}
return ""
}
// hostIndex supports the fallback lookups for slave files without an exact
// path counterpart.
type hostIndex struct {
files map[string]fsops.Info
lower map[string]string // lower(rel) -> rel
byDir map[string][]string // lower(dir) -> rels
byTitle map[string][]string // lower(artist)|normTitle -> rels
}
func buildIndex(files map[string]fsops.Info) *hostIndex {
ix := &hostIndex{files: files, lower: map[string]string{}, byDir: map[string][]string{}, byTitle: map[string][]string{}}
for _, rel := range sortedKeys(files) {
ix.lower[strings.ToLower(rel)] = rel
d := strings.ToLower(filepath.Dir(rel))
ix.byDir[d] = append(ix.byDir[d], rel)
k := strings.ToLower(firstComponent(rel)) + "|" + normTitle(filepath.Base(rel))
ix.byTitle[k] = append(ix.byTitle[k], rel)
}
return ix
}
// losslessRank orders extension candidates so a lossless host file is
// preferred when several share the same title.
func extRank(rel string) int {
switch strings.ToLower(filepath.Ext(rel)) {
case ".flac", ".wav", ".aiff", ".aif", ".alac", ".ape", ".wv", ".dsf", ".dff":
return 0
case ".m4a", ".opus", ".ogg":
return 1
}
return 2
}
// find locates the host counterpart of a slave file.
func (ix *hostIndex) find(relSlave string) (string, string) {
if _, ok := ix.files[relSlave]; ok {
return relSlave, store.KindPath
}
if r, ok := ix.lower[strings.ToLower(relSlave)]; ok {
return r, store.KindPathCase
}
dir := strings.ToLower(filepath.Dir(relSlave))
base := filepath.Base(relSlave)
stem := strings.ToLower(strings.TrimSuffix(base, filepath.Ext(base)))
var extHits, fuzzyHits []string
norm := normTitle(base)
for _, r := range ix.byDir[dir] {
b := filepath.Base(r)
if strings.ToLower(strings.TrimSuffix(b, filepath.Ext(b))) == stem {
extHits = append(extHits, r)
} else if norm != "" && normTitle(b) == norm {
fuzzyHits = append(fuzzyHits, r)
}
}
if len(extHits) > 0 {
best := extHits[0]
for _, r := range extHits[1:] {
if extRank(r) < extRank(best) {
best = r
}
}
return best, store.KindExtension
}
if len(fuzzyHits) == 1 {
return fuzzyHits[0], store.KindFuzzy
}
if artist := firstComponent(relSlave); artist != "" && norm != "" {
if hits := ix.byTitle[strings.ToLower(artist)+"|"+norm]; len(hits) == 1 {
return hits[0], store.KindElsewhere
}
}
return "", store.KindNone
}
// classify fills in status, probes and verdict for a match, reusing cached
// results from prev when neither file changed.
func (e *Engine) classify(ctx context.Context, l *libs, m *store.Match, hi, si fsops.Info, prev *store.Match) {
m.SlaveIno, m.SlaveSize, m.SlaveMtime = si.Ino, si.Size, si.MtimeNs
m.HostIno, m.HostSize, m.HostMtime = 0, 0, 0
m.HostProbe, m.SlaveProbe, m.Verdict, m.Note = nil, nil, nil, ""
if m.Kind == store.KindNone {
m.Status = store.StatusUnmatched
return
}
m.HostIno, m.HostSize, m.HostMtime = hi.Ino, hi.Size, hi.MtimeNs
unchanged := prev != nil && prev.RelHost == m.RelHost &&
prev.HostIno == hi.Ino && prev.HostSize == hi.Size && prev.HostMtime == hi.MtimeNs &&
prev.SlaveIno == si.Ino && prev.SlaveSize == si.Size && prev.SlaveMtime == si.MtimeNs
if prev != nil && prev.RelHost == m.RelHost {
m.Ignored = prev.Ignored
}
switch {
case hi.Dev != si.Dev:
m.Status, m.Note = store.StatusError, "files are on different filesystems"
return
case hi.SameInode(si):
m.Status = store.StatusLinked
return
}
pathKind := m.Kind == store.KindPath || m.Kind == store.KindPathCase
if pathKind && hi.Size == si.Size {
if unchanged && (prev.Status == store.StatusIdentical || prev.Status == store.StatusDifferent) {
m.Status = prev.Status
} else {
hostAbs := filepath.Join(l.host.Path, m.RelHost)
slaveAbs := filepath.Join(l.slave.Path, m.RelSlave)
same, err := fsops.SameContent(hostAbs, slaveAbs)
switch {
case err != nil:
m.Status, m.Note = store.StatusError, "compare failed: "+err.Error()
return
case same:
m.Status = store.StatusIdentical
default:
m.Status = store.StatusDifferent
}
}
if m.Status == store.StatusIdentical {
return
}
} else if pathKind {
m.Status = store.StatusDifferent
} else {
m.Status = store.StatusSimilar
}
// Manual review: gather codec details for both sides.
if unchanged && len(prev.HostProbe) > 0 && len(prev.SlaveProbe) > 0 {
m.HostProbe, m.SlaveProbe, m.Verdict = prev.HostProbe, prev.SlaveProbe, prev.Verdict
return
}
hp := e.prober.Probe(ctx, filepath.Join(l.host.Path, m.RelHost))
sp := e.prober.Probe(ctx, filepath.Join(l.slave.Path, m.RelSlave))
v := probe.Compare(hp, sp)
m.HostProbe, _ = json.Marshal(hp)
m.SlaveProbe, _ = json.Marshal(sp)
m.Verdict, _ = json.Marshal(v)
}
// StartScan launches a background scan of a pair.
func (e *Engine) StartScan(pairID int64, trigger string) (*Job, error) {
return e.startJob(pairID, "scan", func(ctx context.Context, j *Job) error {
return e.scan(ctx, j, pairID, trigger)
})
}
func (e *Engine) scan(ctx context.Context, j *Job, pairID int64, trigger string) error {
started := time.Now()
l, err := e.loadPair(ctx, pairID)
if err != nil {
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), nil)
return err
}
warns := layoutWarnings(l)
j.phase("Indexing host library", 0)
hostFiles := map[string]fsops.Info{}
hs, err := walk(ctx, l, l.host, hostFiles)
if err != nil {
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
return err
}
j.phase("Indexing slave library", 0)
slaveFiles := map[string]fsops.Info{}
ss, err := walk(ctx, l, l.slave, slaveFiles)
if err != nil {
_ = e.st.FinishScan(ctx, pairID, started.Unix(), err.Error(), warns)
return err
}
ix := buildIndex(hostFiles)
prev, err := e.st.AllMatches(ctx, pairID)
if err != nil {
return err
}
j.phase("Comparing files", int64(len(slaveFiles)))
keep := map[string]bool{}
kinds := map[string]int{}
for _, rel := range sortedKeys(slaveFiles) {
if ctx.Err() != nil {
return ctx.Err()
}
si := slaveFiles[rel]
m := store.Match{PairID: pairID, RelSlave: rel}
m.RelHost, m.Kind = ix.find(rel)
var pm *store.Match
if p, ok := prev[rel]; ok {
pm = &p
}
e.classify(ctx, l, &m, hostFiles[m.RelHost], si, pm)
if err := e.st.UpsertMatch(ctx, &m); err != nil {
return err
}
keep[rel] = true
kinds[m.Kind]++
j.count(m.Status)
j.tick()
}
if err := e.st.DeleteMatchesExcept(ctx, pairID, keep); err != nil {
return err
}
// Layout warnings based on what was actually found.
if ss.files > 0 {
if n := kinds[store.KindElsewhere] + kinds[store.KindNone]; n*4 > ss.files {
warns = append(warns, fmt.Sprintf("%d of %d slave files have no counterpart at the same path in the host library. "+
"Check that both libraries use the same Artist/Album/Title.ext layout.", n, ss.files))
}
if ss.offDepth > 0 {
warns = append(warns, fmt.Sprintf("%d slave files are not at Artist/Album/Title.ext depth.", ss.offDepth))
}
}
if hs.offDepth > 0 {
warns = append(warns, fmt.Sprintf("%d host files are not at Artist/Album/Title.ext depth.", hs.offDepth))
}
if hs.symlinks+ss.symlinks > 0 {
warns = append(warns, fmt.Sprintf("Ignored %d symlinks (playlink never follows or replaces symlinks).", hs.symlinks+ss.symlinks))
}
if e.prober.Bin == "" {
warns = append(warns, "ffprobe was not found, so codec and quality details are unavailable for manual review.")
}
if err := e.st.FinishScan(ctx, pairID, time.Now().Unix(), "", warns); err != nil {
return err
}
j.update(func(j *Job) {
j.Message = fmt.Sprintf("Scanned %d host and %d slave files in %s", hs.files, ss.files, time.Since(started).Round(time.Second))
})
if l.pair.AutoLinkIdentical {
return e.autoLink(ctx, j, l, trigger)
}
return nil
}
// autoLink links settled, byte-identical, same-path files after a scan.
func (e *Engine) autoLink(ctx context.Context, j *Job, l *libs, trigger string) error {
ids, err := e.st.IdenticalIDs(ctx, l.pair.ID)
if err != nil || len(ids) == 0 {
return err
}
if e.st.DryRun(ctx) {
j.update(func(j *Job) {
j.Message += fmt.Sprintf(". Dry run is on: %d identical files would be auto-linked.", len(ids))
})
return nil
}
return e.linkIDs(ctx, j, l.pair.ID, ids, "auto:"+trigger, true)
}
// linkIDs links a batch of identical matches.
func (e *Engine) linkIDs(ctx context.Context, j *Job, pairID int64, ids []int64, trigger string, onlySettled bool) error {
j.phase("Linking identical files", int64(len(ids)))
settle := time.Duration(0)
if onlySettled {
if p, err := e.st.Pair(ctx, pairID); err == nil {
settle = time.Duration(p.SettleMinutes) * time.Minute
}
}
cutoff := time.Now().Add(-settle).UnixNano()
for _, id := range ids {
if ctx.Err() != nil {
return ctx.Err()
}
m, err := e.st.Match(ctx, id)
if err != nil {
j.tick()
continue
}
if onlySettled && m.Kind != store.KindPath {
j.skip(m.RelSlave + ": path differs by case, link manually")
j.tick()
continue
}
if settle > 0 && (m.SlaveMtime > cutoff || m.HostMtime > cutoff) {
j.skip(m.RelSlave + ": modified recently, waiting for it to settle")
j.tick()
continue
}
op, err := e.Link(ctx, id, LinkOptions{Trigger: trigger})
switch {
case err != nil:
j.skip(m.RelSlave + ": " + err.Error())
case op.DryRun:
j.count("planned")
default:
j.count("linked")
}
j.tick()
}
return nil
}
// StartLinkIdentical links every identical match of a pair in the background.
func (e *Engine) StartLinkIdentical(pairID int64) (*Job, error) {
return e.startJob(pairID, "link-identical", func(ctx context.Context, j *Job) error {
ids, err := e.st.IdenticalIDs(ctx, pairID)
if err != nil {
return err
}
if len(ids) == 0 {
j.update(func(j *Job) { j.Message = "Nothing to link" })
return nil
}
return e.linkIDs(ctx, j, pairID, ids, "bulk", false)
})
}
// refresh re-evaluates a single slave file after an operation.
func (e *Engine) refresh(ctx context.Context, l *libs, relSlave string) {
si, err := fsops.RegularFile(filepath.Join(l.slave.Path, relSlave))
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
_ = e.st.DeleteMatch(ctx, l.pair.ID, relSlave)
}
return
}
m := store.Match{PairID: l.pair.ID, RelSlave: relSlave}
var pm *store.Match
if p, err := e.st.MatchBySlave(ctx, l.pair.ID, relSlave); err == nil {
pm = &p
}
var hi fsops.Info
if hp, err := l.host.Resolve(relSlave); err == nil {
if in, err := fsops.RegularFile(hp); err == nil {
hi = in
m.RelHost, m.Kind = relSlave, store.KindPath
}
}
if m.Kind == "" {
if pm != nil && pm.RelHost != "" {
if in, err := fsops.RegularFile(filepath.Join(l.host.Path, pm.RelHost)); err == nil {
hi = in
m.RelHost, m.Kind = pm.RelHost, pm.Kind
}
}
}
if m.Kind == "" {
m.Kind = store.KindNone
}
e.classify(ctx, l, &m, hi, si, pm)
_ = e.st.UpsertMatch(ctx, &m)
}