Initial commit: playlink music library hardlink deduplicator
Go backend (chi, SQLite) with an embedded React/TypeScript UI that finds songs present in both a host and a slave music library and replaces the slave copy with a hardlink to the host copy. - Path-first matching with format/title/album-folder fallbacks - Auto-linking of byte-identical files, re-verified with cmp before linking - Manual review with ffprobe quality comparison, downgrade guard and promote-to-host override - Build mode to populate an empty slave library from the host - Dry run by default, atomic stage+rename replacement, quarantine with retention, undo, path confinement and authenticated UI - Dockerfile and compose file for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
commit
ac47e61b08
42 files changed
+8892
No files matched your search
@@ -0,0 +1,405 @@
|
||||
// Package fsops contains the low-level, safety-critical filesystem primitives
|
||||
// used by playlink: confined path resolution, byte comparison, atomic
|
||||
// hardlink replacement and directory fsyncs.
|
||||
//
|
||||
// Every mutating primitive in this package follows the same rule: a path that
|
||||
// holds user data is never left missing. Replacements are staged as a
|
||||
// temporary hardlink in the destination directory and then atomically renamed
|
||||
// over the target.
|
||||
package fsops
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// TmpPrefix marks staging files created by playlink. Anything carrying this
|
||||
// prefix is disposable: it is either a hardlink to a host file or a copy that
|
||||
// has not been renamed into place yet.
|
||||
const TmpPrefix = ".playlink-tmp-"
|
||||
|
||||
var (
|
||||
ErrOutsideRoot = errors.New("path escapes library root")
|
||||
ErrNotRegular = errors.New("not a regular file")
|
||||
ErrCrossDevice = errors.New("files are on different filesystems; hardlinks are impossible")
|
||||
ErrChanged = errors.New("file changed since it was scanned; rescan first")
|
||||
ErrExists = errors.New("destination already exists")
|
||||
ErrVerifyFailed = errors.New("post-operation verification failed")
|
||||
)
|
||||
|
||||
// Info is the subset of stat(2) that playlink cares about.
|
||||
type Info struct {
|
||||
Dev uint64 `json:"dev"`
|
||||
Ino uint64 `json:"ino"`
|
||||
Nlink uint64 `json:"nlink"`
|
||||
Size int64 `json:"size"`
|
||||
MtimeNs int64 `json:"mtime_ns"`
|
||||
Mode fs.FileMode `json:"-"`
|
||||
}
|
||||
|
||||
// SameInode reports whether a and b are the same file on disk.
|
||||
func (i Info) SameInode(o Info) bool { return i.Dev == o.Dev && i.Ino == o.Ino }
|
||||
|
||||
// Unchanged reports whether size and mtime still match a previous snapshot.
|
||||
func (i Info) Unchanged(size, mtimeNs int64) bool { return i.Size == size && i.MtimeNs == mtimeNs }
|
||||
|
||||
// Lstat stats p without following a trailing symlink.
|
||||
func Lstat(p string) (Info, error) {
|
||||
fi, err := os.Lstat(p)
|
||||
if err != nil {
|
||||
return Info{}, err
|
||||
}
|
||||
return infoFrom(fi), nil
|
||||
}
|
||||
|
||||
func infoFrom(fi fs.FileInfo) Info {
|
||||
in := Info{Size: fi.Size(), MtimeNs: fi.ModTime().UnixNano(), Mode: fi.Mode()}
|
||||
if st, ok := fi.Sys().(*syscall.Stat_t); ok {
|
||||
in.Dev = uint64(st.Dev)
|
||||
in.Ino = uint64(st.Ino)
|
||||
in.Nlink = uint64(st.Nlink)
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// RegularFile lstat's p and fails unless it is a plain regular file (not a
|
||||
// symlink, device, directory, ...).
|
||||
func RegularFile(p string) (Info, error) {
|
||||
in, err := Lstat(p)
|
||||
if err != nil {
|
||||
return in, err
|
||||
}
|
||||
if !in.Mode.IsRegular() {
|
||||
return in, fmt.Errorf("%s: %w", p, ErrNotRegular)
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// Root is an absolute, symlink-free directory that confines every path
|
||||
// playlink touches for a library.
|
||||
type Root struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
// NewRoot validates dir and resolves symlinks so containment checks are exact.
|
||||
func NewRoot(dir string) (Root, error) {
|
||||
if !filepath.IsAbs(dir) {
|
||||
return Root{}, fmt.Errorf("%q is not an absolute path", dir)
|
||||
}
|
||||
real, err := filepath.EvalSymlinks(filepath.Clean(dir))
|
||||
if err != nil {
|
||||
return Root{}, err
|
||||
}
|
||||
fi, err := os.Stat(real)
|
||||
if err != nil {
|
||||
return Root{}, err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return Root{}, fmt.Errorf("%q is not a directory", dir)
|
||||
}
|
||||
return Root{Path: real}, nil
|
||||
}
|
||||
|
||||
// Within reports whether p equals base or lies beneath it. Both must be clean.
|
||||
func Within(base, p string) bool {
|
||||
if p == base {
|
||||
return true
|
||||
}
|
||||
if base == "/" {
|
||||
return strings.HasPrefix(p, "/")
|
||||
}
|
||||
return strings.HasPrefix(p, base+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// CleanRel validates a library-relative path coming from the database or an
|
||||
// API request.
|
||||
func CleanRel(rel string) (string, error) {
|
||||
if rel == "" || filepath.IsAbs(rel) || strings.ContainsRune(rel, 0) {
|
||||
return "", fmt.Errorf("invalid relative path %q", rel)
|
||||
}
|
||||
c := filepath.Clean(rel)
|
||||
if c == "." || c == ".." || strings.HasPrefix(c, "../") {
|
||||
return "", fmt.Errorf("%q: %w", rel, ErrOutsideRoot)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Resolve maps a relative path to an absolute one inside the root. The
|
||||
// deepest existing ancestor is resolved through symlinks and must still be
|
||||
// inside the root, so a symlinked directory can never redirect a write
|
||||
// outside the library. The final component itself is not followed.
|
||||
func (r Root) Resolve(rel string) (string, error) {
|
||||
c, err := CleanRel(rel)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
full := filepath.Join(r.Path, c)
|
||||
anc := filepath.Dir(full)
|
||||
for {
|
||||
real, err := filepath.EvalSymlinks(anc)
|
||||
if err == nil {
|
||||
if !Within(r.Path, real) {
|
||||
return "", fmt.Errorf("%q: %w", rel, ErrOutsideRoot)
|
||||
}
|
||||
// Re-anchor the missing tail on the resolved ancestor.
|
||||
tail, _ := filepath.Rel(anc, full)
|
||||
return filepath.Join(real, tail), nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
return "", err
|
||||
}
|
||||
if anc == r.Path || anc == "/" {
|
||||
return "", err
|
||||
}
|
||||
anc = filepath.Dir(anc)
|
||||
}
|
||||
}
|
||||
|
||||
// Rel converts an absolute path inside the root back to a relative one.
|
||||
func (r Root) Rel(abs string) (string, error) {
|
||||
if !Within(r.Path, abs) || abs == r.Path {
|
||||
return "", ErrOutsideRoot
|
||||
}
|
||||
return filepath.Rel(r.Path, abs)
|
||||
}
|
||||
|
||||
// SameContent compares two files byte for byte (the equivalent of cmp(1)).
|
||||
// It refuses to report equality if either file changed while being read.
|
||||
func SameContent(a, b string) (bool, error) {
|
||||
fa, err := os.Open(a)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer fa.Close()
|
||||
fb, err := os.Open(b)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer fb.Close()
|
||||
|
||||
sa, err := fa.Stat()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
sb, err := fb.Stat()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !sa.Mode().IsRegular() || !sb.Mode().IsRegular() {
|
||||
return false, ErrNotRegular
|
||||
}
|
||||
if sa.Size() != sb.Size() {
|
||||
return false, nil
|
||||
}
|
||||
ia, ib := infoFrom(sa), infoFrom(sb)
|
||||
if ia.SameInode(ib) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
const chunk = 1 << 20
|
||||
bufA := make([]byte, chunk)
|
||||
bufB := make([]byte, chunk)
|
||||
for {
|
||||
na, errA := io.ReadFull(fa, bufA)
|
||||
nb, errB := io.ReadFull(fb, bufB)
|
||||
if na != nb || !bytes.Equal(bufA[:na], bufB[:nb]) {
|
||||
return false, nil
|
||||
}
|
||||
endA := errA == io.EOF || errA == io.ErrUnexpectedEOF
|
||||
endB := errB == io.EOF || errB == io.ErrUnexpectedEOF
|
||||
if errA != nil && !endA {
|
||||
return false, errA
|
||||
}
|
||||
if errB != nil && !endB {
|
||||
return false, errB
|
||||
}
|
||||
if endA || endB {
|
||||
if endA != endB {
|
||||
return false, nil
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Guard against a writer modifying either file mid-compare.
|
||||
for _, pair := range []struct {
|
||||
p string
|
||||
prev Info
|
||||
}{{a, ia}, {b, ib}} {
|
||||
now, err := Lstat(pair.p)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !now.SameInode(pair.prev) || !now.Unchanged(pair.prev.Size, pair.prev.MtimeNs) {
|
||||
return false, ErrChanged
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// SyncDir fsyncs a directory so renames and new links inside it are durable.
|
||||
func SyncDir(dir string) error {
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
if err := d.Sync(); err != nil && !errors.Is(err, syscall.EINVAL) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tmpName(dir string) (string, error) {
|
||||
var b [8]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(dir, TmpPrefix+hex.EncodeToString(b[:])), nil
|
||||
}
|
||||
|
||||
// StageLink creates a temporary hardlink to src inside dstDir and verifies it
|
||||
// points at the same inode. The caller must rename or remove the result.
|
||||
func StageLink(src string, srcInfo Info, dstDir string) (string, error) {
|
||||
tmp, err := tmpName(dstDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Link(src, tmp); err != nil {
|
||||
if errors.Is(err, syscall.EXDEV) {
|
||||
return "", ErrCrossDevice
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
got, err := Lstat(tmp)
|
||||
if err != nil || !got.SameInode(srcInfo) {
|
||||
_ = os.Remove(tmp)
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%w: staged link does not share the source inode", ErrVerifyFailed)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return tmp, nil
|
||||
}
|
||||
|
||||
// LinkNew hardlinks src to dst, which must not exist. Verified by inode.
|
||||
func LinkNew(src string, srcInfo Info, dst string) error {
|
||||
if _, err := os.Lstat(dst); err == nil {
|
||||
return fmt.Errorf("%s: %w", dst, ErrExists)
|
||||
}
|
||||
if err := os.Link(src, dst); err != nil {
|
||||
if errors.Is(err, syscall.EXDEV) {
|
||||
return ErrCrossDevice
|
||||
}
|
||||
return err
|
||||
}
|
||||
got, err := Lstat(dst)
|
||||
if err != nil || !got.SameInode(srcInfo) {
|
||||
// dst was created by us a moment ago and is only an extra link.
|
||||
_ = os.Remove(dst)
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%w: %s does not share the source inode", ErrVerifyFailed, dst)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return SyncDir(filepath.Dir(dst))
|
||||
}
|
||||
|
||||
// ReplaceWithLink atomically replaces dst with a hardlink to src. dst must be
|
||||
// a regular file on the same device. The staged link is renamed over dst, so
|
||||
// dst never disappears, even on a crash.
|
||||
func ReplaceWithLink(src string, srcInfo Info, dst string) error {
|
||||
tmp, err := StageLink(src, srcInfo, filepath.Dir(dst))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := SyncDir(filepath.Dir(dst)); err != nil {
|
||||
return err
|
||||
}
|
||||
got, err := Lstat(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !got.SameInode(srcInfo) {
|
||||
return fmt.Errorf("%w: %s does not share the source inode after rename", ErrVerifyFailed, dst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveLink deletes p only if it is a hardlink of keep (same inode and at
|
||||
// least one other link). This is the only way playlink deletes a file outside
|
||||
// the quarantine, and it never destroys the last copy of any data.
|
||||
func RemoveLink(p string, keep Info) error {
|
||||
in, err := RegularFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !in.SameInode(keep) || in.Nlink < 2 {
|
||||
return fmt.Errorf("refusing to remove %s: it is not a hardlink of the file being kept", p)
|
||||
}
|
||||
if err := os.Remove(p); err != nil {
|
||||
return err
|
||||
}
|
||||
return SyncDir(filepath.Dir(p))
|
||||
}
|
||||
|
||||
// CopyReplace replaces dst with an independent byte copy of src (used to undo
|
||||
// a link between identical files). The copy is fsynced and verified before
|
||||
// being renamed into place.
|
||||
func CopyReplace(src, dst string, mode fs.FileMode) error {
|
||||
tmp, err := tmpName(filepath.Dir(dst))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.Perm())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanup := func(e error) error { out.Close(); _ = os.Remove(tmp); return e }
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return cleanup(err)
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
return cleanup(err)
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
same, err := SameContent(src, tmp)
|
||||
if err != nil || !same {
|
||||
_ = os.Remove(tmp)
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%w: copy differs from source", ErrVerifyFailed)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return SyncDir(filepath.Dir(dst))
|
||||
}
|
||||
|
||||
// MkdirAllLike creates dir and any missing parents with 0775 permissions.
|
||||
func MkdirAllLike(dir string) error {
|
||||
return os.MkdirAll(dir, 0o775)
|
||||
}
|
||||
Reference in new issue
Block a user