Initial commit: playlink music library hardlink deduplicator
Go backend (chi, SQLite) with an embedded React/TypeScript UI that finds songs present in both a host and a slave music library and replaces the slave copy with a hardlink to the host copy. - Path-first matching with format/title/album-folder fallbacks - Auto-linking of byte-identical files, re-verified with cmp before linking - Manual review with ffprobe quality comparison, downgrade guard and promote-to-host override - Build mode to populate an empty slave library from the host - Dry run by default, atomic stage+rename replacement, quarantine with retention, undo, path confinement and authenticated UI - Dockerfile and compose file for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
commit
ac47e61b08
42 files changed
+8892
No files matched your search
@@ -0,0 +1,481 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cdnutter/playlink/internal/fsops"
|
||||
"github.com/cdnutter/playlink/internal/probe"
|
||||
"github.com/cdnutter/playlink/internal/store"
|
||||
)
|
||||
|
||||
type env struct {
|
||||
t *testing.T
|
||||
root string
|
||||
host string
|
||||
slave string
|
||||
st *store.Store
|
||||
e *Engine
|
||||
pair store.Pair
|
||||
}
|
||||
|
||||
func setup(t *testing.T, dryRun bool) *env {
|
||||
t.Helper()
|
||||
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev := &env{t: t, root: root, host: filepath.Join(root, "Albums"), slave: filepath.Join(root, "Singles")}
|
||||
for _, d := range []string{ev.host, ev.slave, filepath.Join(root, "db")} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
st, err := store.Open(filepath.Join(root, "db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.DB.Close() })
|
||||
v := "0"
|
||||
if dryRun {
|
||||
v = "1"
|
||||
}
|
||||
if err := st.SetSetting(context.Background(), "dry_run", v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev.st = st
|
||||
ev.e = New(st, probe.Prober{}, []string{root}, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
ev.pair = store.Pair{Name: "test", HostRoot: ev.host, SlaveRoot: ev.slave, RetentionDays: 30}
|
||||
if _, err := ev.e.NormalizePair(&ev.pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SavePair(context.Background(), &ev.pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ev
|
||||
}
|
||||
|
||||
func (ev *env) write(base, rel, content string) string {
|
||||
ev.t.Helper()
|
||||
p := filepath.Join(base, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
ev.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
ev.t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (ev *env) read(p string) string {
|
||||
ev.t.Helper()
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
ev.t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (ev *env) scan() {
|
||||
ev.t.Helper()
|
||||
if _, err := ev.e.StartScan(ev.pair.ID, "test"); err != nil {
|
||||
ev.t.Fatal(err)
|
||||
}
|
||||
ev.wait()
|
||||
}
|
||||
|
||||
func (ev *env) wait() *Job {
|
||||
ev.t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if j := ev.e.Job(ev.pair.ID); j != nil && j.State != "running" {
|
||||
if j.State != "done" {
|
||||
ev.t.Fatalf("job %s: %s", j.State, j.Error)
|
||||
}
|
||||
return j
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
ev.t.Fatal("job did not finish")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ev *env) match(rel string) store.Match {
|
||||
ev.t.Helper()
|
||||
m, err := ev.st.MatchBySlave(context.Background(), ev.pair.ID, rel)
|
||||
if err != nil {
|
||||
ev.t.Fatalf("match %s: %v", rel, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func sameInode(t *testing.T, a, b string) bool {
|
||||
t.Helper()
|
||||
ia, err := fsops.Lstat(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ib, err := fsops.Lstat(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ia.SameInode(ib)
|
||||
}
|
||||
|
||||
var ctx = context.Background()
|
||||
|
||||
func TestIdenticalLinkAndUndo(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
rel := "bbno$/Bag or Die/edamame.flac"
|
||||
h := ev.write(ev.host, rel, "same-bytes")
|
||||
s := ev.write(ev.slave, rel, "same-bytes")
|
||||
ev.scan()
|
||||
|
||||
m := ev.match(rel)
|
||||
if m.Status != store.StatusIdentical || m.Kind != store.KindPath {
|
||||
t.Fatalf("want identical/path, got %s/%s", m.Status, m.Kind)
|
||||
}
|
||||
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if op.QuarantinePath != "" {
|
||||
t.Fatal("identical link must not quarantine")
|
||||
}
|
||||
if !sameInode(t, h, s) {
|
||||
t.Fatal("slave is not a hardlink of host")
|
||||
}
|
||||
if ev.match(rel).Status != store.StatusLinked {
|
||||
t.Fatal("match not marked linked")
|
||||
}
|
||||
|
||||
if err := ev.e.Undo(ctx, op.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sameInode(t, h, s) || ev.read(s) != "same-bytes" {
|
||||
t.Fatal("undo should leave an independent identical copy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDifferentLinkQuarantinesAndUndoRestores(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
rel := "Artist/Album/song.flac"
|
||||
h := ev.write(ev.host, rel, "host-version-longer")
|
||||
s := ev.write(ev.slave, rel, "slave-version")
|
||||
ev.scan()
|
||||
|
||||
m := ev.match(rel)
|
||||
if m.Status != store.StatusDifferent {
|
||||
t.Fatalf("want different, got %s", m.Status)
|
||||
}
|
||||
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sameInode(t, h, s) {
|
||||
t.Fatal("not linked")
|
||||
}
|
||||
if op.QuarantinePath == "" || ev.read(op.QuarantinePath) != "slave-version" {
|
||||
t.Fatal("original slave file not quarantined")
|
||||
}
|
||||
if err := ev.e.Undo(ctx, op.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ev.read(s) != "slave-version" || ev.read(h) != "host-version-longer" {
|
||||
t.Fatal("undo did not restore the original")
|
||||
}
|
||||
if _, err := os.Stat(op.QuarantinePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("quarantine entry should be consumed by the restore")
|
||||
}
|
||||
if err := ev.e.Undo(ctx, op.ID); err == nil {
|
||||
t.Fatal("double undo must fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimilarExtensionLinkRenamesAndUndo(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
h := ev.write(ev.host, "A/B/01 - Song.flac", "flac-data")
|
||||
s := ev.write(ev.slave, "A/B/01 - Song.mp3", "mp3")
|
||||
ev.scan()
|
||||
|
||||
m := ev.match("A/B/01 - Song.mp3")
|
||||
if m.Status != store.StatusSimilar || m.Kind != store.KindExtension || m.RelHost != "A/B/01 - Song.flac" {
|
||||
t.Fatalf("unexpected match %+v", m)
|
||||
}
|
||||
op, err := ev.e.Link(ctx, m.ID, LinkOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newSlave := filepath.Join(ev.slave, "A/B/01 - Song.flac")
|
||||
if !sameInode(t, h, newSlave) {
|
||||
t.Fatal("slave should now hold the host's .flac as a hardlink")
|
||||
}
|
||||
if _, err := os.Stat(s); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("old .mp3 should be moved to quarantine")
|
||||
}
|
||||
if err := ev.e.Undo(ctx, op.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ev.read(s) != "mp3" {
|
||||
t.Fatal(".mp3 not restored")
|
||||
}
|
||||
if _, err := os.Stat(newSlave); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("linked .flac should be removed on undo")
|
||||
}
|
||||
if ev.read(h) != "flac-data" {
|
||||
t.Fatal("host must be untouched")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFuzzyTitleMatch(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
ev.write(ev.host, "A/B/03. My Song!.flac", "x")
|
||||
ev.write(ev.slave, "A/B/My Song.mp3", "y")
|
||||
ev.write(ev.slave, "A/B/My Song (Live).mp3", "z")
|
||||
ev.write(ev.host, "A/Other Album/Lonely.flac", "x")
|
||||
ev.write(ev.slave, "A/Single/Lonely.flac", "x")
|
||||
ev.scan()
|
||||
if m := ev.match("A/B/My Song.mp3"); m.Kind != store.KindFuzzy {
|
||||
t.Fatalf("want fuzzy, got %s", m.Kind)
|
||||
}
|
||||
if m := ev.match("A/B/My Song (Live).mp3"); m.Status != store.StatusUnmatched {
|
||||
t.Fatalf("live version must not match studio version, got %s", m.Status)
|
||||
}
|
||||
if m := ev.match("A/Single/Lonely.flac"); m.Kind != store.KindElsewhere || m.Status != store.StatusSimilar {
|
||||
t.Fatalf("want elsewhere/similar, got %s/%s", m.Kind, m.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDowngradeGuardAndPromote(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
rel := "A/B/song.flac"
|
||||
h := ev.write(ev.host, rel, "lossy-host")
|
||||
s := ev.write(ev.slave, rel, "lossless-slave!")
|
||||
ev.scan()
|
||||
m := ev.match(rel)
|
||||
|
||||
// Simulate the probe verdict that the slave is better.
|
||||
v, _ := json.Marshal(probe.Verdict{Better: "slave"})
|
||||
m.Verdict = v
|
||||
if err := ev.st.UpsertMatch(ctx, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); !errors.Is(err, ErrDowngrade) {
|
||||
t.Fatalf("want ErrDowngrade, got %v", err)
|
||||
}
|
||||
if _, err := ev.e.Promote(ctx, m.ID, false); !errors.Is(err, ErrConfirm) {
|
||||
t.Fatalf("promote without confirm must fail, got %v", err)
|
||||
}
|
||||
op, err := ev.e.Promote(ctx, m.ID, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sameInode(t, h, s) || ev.read(h) != "lossless-slave!" {
|
||||
t.Fatal("host should now be the slave's file")
|
||||
}
|
||||
if ev.read(op.QuarantinePath) != "lossy-host" {
|
||||
t.Fatal("host original not quarantined")
|
||||
}
|
||||
if err := ev.e.Undo(ctx, op.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ev.read(h) != "lossy-host" || ev.read(s) != "lossless-slave!" || sameInode(t, h, s) {
|
||||
t.Fatal("promote undo failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedSinceScanIsRefused(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
rel := "A/B/c.flac"
|
||||
ev.write(ev.host, rel, "aaa")
|
||||
s := ev.write(ev.slave, rel, "bbb")
|
||||
ev.scan()
|
||||
m := ev.match(rel)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if err := os.WriteFile(s, []byte("user edited this"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ev.e.Link(ctx, m.ID, LinkOptions{}); !errors.Is(err, fsops.ErrChanged) {
|
||||
t.Fatalf("want ErrChanged, got %v", err)
|
||||
}
|
||||
if ev.read(s) != "user edited this" {
|
||||
t.Fatal("file must be untouched")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunChangesNothing(t *testing.T) {
|
||||
ev := setup(t, true)
|
||||
rel := "A/B/d.flac"
|
||||
h := ev.write(ev.host, rel, "same")
|
||||
s := ev.write(ev.slave, rel, "same")
|
||||
ev.scan()
|
||||
op, err := ev.e.Link(ctx, ev.match(rel).ID, LinkOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !op.DryRun || op.Status != store.OpPlanned {
|
||||
t.Fatalf("expected planned dry-run op, got %+v", op)
|
||||
}
|
||||
if sameInode(t, h, s) {
|
||||
t.Fatal("dry run must not link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoLinkRespectsSettleTime(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
ev.pair.AutoLinkIdentical = true
|
||||
ev.pair.SettleMinutes = 60
|
||||
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := ev.write(ev.host, "A/B/e.flac", "same")
|
||||
s := ev.write(ev.slave, "A/B/e.flac", "same")
|
||||
ev.scan()
|
||||
if sameInode(t, h, s) {
|
||||
t.Fatal("freshly written files must not be auto-linked")
|
||||
}
|
||||
ev.pair.SettleMinutes = 0
|
||||
if err := ev.st.SavePair(ctx, &ev.pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev.scan()
|
||||
if !sameInode(t, h, s) {
|
||||
t.Fatal("settled identical file should be auto-linked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAndRemove(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
h1 := ev.write(ev.host, "bbno$/Album/one.flac", "1")
|
||||
ev.write(ev.host, "bbno$/Album/two.flac", "2")
|
||||
ev.write(ev.host, "bbno$/Album/cover.jpg", "img")
|
||||
ev.write(ev.slave, "bbno$/Album/two.flac", "user's own different file")
|
||||
|
||||
if _, err := ev.e.StartBuild(ev.pair.ID, []string{"bbno$"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := ev.wait()
|
||||
if j.Results["linked"] != 1 || j.Results["skipped"] != 1 {
|
||||
t.Fatalf("unexpected results %+v", j.Results)
|
||||
}
|
||||
s1 := filepath.Join(ev.slave, "bbno$/Album/one.flac")
|
||||
if !sameInode(t, h1, s1) {
|
||||
t.Fatal("build did not hardlink")
|
||||
}
|
||||
if ev.read(filepath.Join(ev.slave, "bbno$/Album/two.flac")) != "user's own different file" {
|
||||
t.Fatal("build must never overwrite an existing slave file")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(ev.slave, "bbno$/Album/cover.jpg")); err == nil {
|
||||
t.Fatal("non-audio files must not be linked")
|
||||
}
|
||||
|
||||
if _, err := ev.e.StartRemove(ev.pair.ID, []string{"bbno$"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j = ev.wait()
|
||||
if j.Results["removed"] != 1 || j.Results["skipped"] != 1 {
|
||||
t.Fatalf("unexpected remove results %+v", j.Results)
|
||||
}
|
||||
if _, err := os.Stat(s1); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("linked file should be removed")
|
||||
}
|
||||
if ev.read(h1) != "1" {
|
||||
t.Fatal("host must be untouched")
|
||||
}
|
||||
if ev.read(filepath.Join(ev.slave, "bbno$/Album/two.flac")) != "user's own different file" {
|
||||
t.Fatal("remove must never delete a non-linked file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSymlinkEscapeIgnored(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
outside := t.TempDir()
|
||||
os.WriteFile(filepath.Join(outside, "secret.flac"), []byte("x"), 0o644)
|
||||
os.MkdirAll(filepath.Join(ev.slave, "Evil"), 0o755)
|
||||
if err := os.Symlink(outside, filepath.Join(ev.slave, "Evil", "Album")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev.write(ev.host, "Evil/Album/secret.flac", "x")
|
||||
ev.scan()
|
||||
if _, err := ev.st.MatchBySlave(ctx, ev.pair.ID, "Evil/Album/secret.flac"); err == nil {
|
||||
t.Fatal("files behind a symlinked directory must not be scanned")
|
||||
}
|
||||
root, _ := fsops.NewRoot(ev.slave)
|
||||
if _, err := root.Resolve("Evil/Album/secret.flac"); !errors.Is(err, fsops.ErrOutsideRoot) {
|
||||
t.Fatalf("resolve through an escaping symlink must fail, got %v", err)
|
||||
}
|
||||
if _, err := root.Resolve("../Albums/x.flac"); !errors.Is(err, fsops.ErrOutsideRoot) {
|
||||
t.Fatalf("dot-dot must fail, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPurgeAndOverlapValidation(t *testing.T) {
|
||||
ev := setup(t, false)
|
||||
rel := "A/B/f.flac"
|
||||
ev.write(ev.host, rel, "one")
|
||||
ev.write(ev.slave, rel, "two!")
|
||||
ev.scan()
|
||||
op, err := ev.e.Link(ctx, ev.match(rel).ID, LinkOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ev.e.Purge(ctx, op.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(op.QuarantinePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("purge did not delete quarantine")
|
||||
}
|
||||
if err := ev.e.Undo(ctx, op.ID); err == nil {
|
||||
t.Fatal("undo after purge must fail")
|
||||
}
|
||||
|
||||
bad := store.Pair{Name: "bad", HostRoot: ev.host, SlaveRoot: filepath.Join(ev.host, "A")}
|
||||
if _, err := ev.e.NormalizePair(&bad); err == nil {
|
||||
t.Fatal("nested libraries must be rejected")
|
||||
}
|
||||
outside := store.Pair{Name: "bad", HostRoot: ev.host, SlaveRoot: t.TempDir()}
|
||||
if _, err := ev.e.NormalizePair(&outside); err == nil {
|
||||
t.Fatal("paths outside allowed roots must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
big := bytes.Repeat([]byte("abcdefgh"), 300_000) // spans several chunks
|
||||
a, b, c := filepath.Join(dir, "a"), filepath.Join(dir, "b"), filepath.Join(dir, "c")
|
||||
os.WriteFile(a, big, 0o644)
|
||||
os.WriteFile(b, big, 0o644)
|
||||
alt := append([]byte(nil), big...)
|
||||
alt[len(alt)-1] = 'X'
|
||||
os.WriteFile(c, alt, 0o644)
|
||||
if same, err := fsops.SameContent(a, b); err != nil || !same {
|
||||
t.Fatalf("a,b should match: %v %v", same, err)
|
||||
}
|
||||
if same, _ := fsops.SameContent(a, c); same {
|
||||
t.Fatal("a,c differ in the last byte")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormTitle(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"01 - Song.flac": "song",
|
||||
"1-03 Song.mp3": "song",
|
||||
"03. Song!.flac": "song",
|
||||
"22.flac": "22",
|
||||
"Song (Live).flac": "songlive",
|
||||
"07 Song (Remix).flac": "songremix",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := normTitle(in); got != want {
|
||||
t.Errorf("normTitle(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user