cdnutterandClaude Opus 5.5
428935dc58
Each pair is a Music (Artist/Album/01 Title) or Soundtracks (Album/01 Title) library. The new Fixer tab takes the master, then the slave, through four reviewed steps: folder structure, album tags (optional MusicBrainz lookup; the slave reuses the fixed master's tags), folder names and file names, then links the two libraries. Every fix is an undoable operation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1021 lines
29 KiB
Go
1021 lines
29 KiB
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cdnutter/playlink/internal/fsops"
|
|
"github.com/cdnutter/playlink/internal/probe"
|
|
"github.com/cdnutter/playlink/internal/store"
|
|
)
|
|
|
|
type LinkOptions struct {
|
|
// AllowDowngrade confirms linking when the master copy scored lower.
|
|
AllowDowngrade bool
|
|
// KeepFolder keeps a cross-folder match in the slave's current folder
|
|
// instead of mirroring the master's folder structure.
|
|
KeepFolder bool
|
|
Trigger string
|
|
}
|
|
|
|
// LinkTarget is where the slave file ends up after linking. A match in the
|
|
// same folder keeps the slave's folder (only the file name follows the
|
|
// master). A match found in another folder, such as a single matched to its
|
|
// album version, mirrors the master's path so the slave gets the album's
|
|
// folder structure, unless keepFolder is set.
|
|
func LinkTarget(relSlave, relMaster string, keepFolder bool) string {
|
|
if relMaster == "" {
|
|
return relSlave
|
|
}
|
|
sd := filepath.Dir(relSlave)
|
|
if keepFolder || strings.EqualFold(sd, filepath.Dir(relMaster)) {
|
|
return filepath.Join(sd, filepath.Base(relMaster))
|
|
}
|
|
return relMaster
|
|
}
|
|
|
|
// snapshot holds the verified state of both files right before an operation.
|
|
type snapshot struct {
|
|
masterAbs, slaveAbs string
|
|
mi, si fsops.Info
|
|
}
|
|
|
|
// preflight re-validates a match against the filesystem: both files must be
|
|
// plain regular files inside their roots, on one device, and unchanged since
|
|
// the scan recorded them.
|
|
func preflight(l *libs, m store.Match) (snapshot, error) {
|
|
var s snapshot
|
|
var err error
|
|
if s.masterAbs, err = l.master.Resolve(m.RelMaster); err != nil {
|
|
return s, err
|
|
}
|
|
if s.slaveAbs, err = l.slave.Resolve(m.RelSlave); err != nil {
|
|
return s, err
|
|
}
|
|
if s.mi, err = fsops.RegularFile(s.masterAbs); err != nil {
|
|
return s, fmt.Errorf("master file: %w", err)
|
|
}
|
|
if s.si, err = fsops.RegularFile(s.slaveAbs); err != nil {
|
|
return s, fmt.Errorf("slave file: %w", err)
|
|
}
|
|
if s.mi.Ino != m.MasterIno || !s.mi.Unchanged(m.MasterSize, m.MasterMtime) {
|
|
return s, fmt.Errorf("master %w", fsops.ErrChanged)
|
|
}
|
|
if s.si.Ino != m.SlaveIno || !s.si.Unchanged(m.SlaveSize, m.SlaveMtime) {
|
|
return s, fmt.Errorf("slave %w", fsops.ErrChanged)
|
|
}
|
|
if s.mi.Dev != s.si.Dev || s.mi.Dev != l.dev {
|
|
return s, fsops.ErrCrossDevice
|
|
}
|
|
if s.mi.SameInode(s.si) {
|
|
return s, errors.New("already linked")
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func quarantinePath(l *libs, opID int64, side, rel string) string {
|
|
return filepath.Join(l.quarantine, strconv.FormatInt(opID, 10), side, rel)
|
|
}
|
|
|
|
// quarantine adds an extra hardlink of p (whose identity is in) inside the
|
|
// quarantine. The original stays in place; nothing is lost if a later step
|
|
// fails. The caller removes or replaces p afterwards.
|
|
func quarantine(p string, in fsops.Info, q string) error {
|
|
if err := fsops.MkdirAllLike(filepath.Dir(q)); err != nil {
|
|
return err
|
|
}
|
|
return fsops.LinkNew(p, in, q)
|
|
}
|
|
|
|
// dropQuarantine removes a quarantine link that turned out not to be needed.
|
|
func dropQuarantine(l *libs, q string, keep fsops.Info) {
|
|
if err := fsops.RemoveLink(q, keep); err == nil {
|
|
pruneEmpty(filepath.Dir(q), l.quarantine)
|
|
}
|
|
}
|
|
|
|
// removeOriginal deletes an original file after it was safely superseded. The
|
|
// inode must still be the one we verified.
|
|
//
|
|
// With a quarantine copy q, identity is proven by p and q being the same
|
|
// file right now. Timestamps are deliberately not compared: FUSE filesystems
|
|
// such as Unraid's /mnt/user can report them inconsistently between calls,
|
|
// and any change to the file is preserved in quarantine anyway because both
|
|
// names point at the same data. Without a quarantine copy (a byte-identical
|
|
// file), the size and modification time must also be unchanged.
|
|
func removeOriginal(p string, expect fsops.Info, q string) error {
|
|
now, err := fsops.RegularFile(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if q != "" {
|
|
qi, err := fsops.RegularFile(q)
|
|
if err != nil {
|
|
return fmt.Errorf("quarantine copy: %w", err)
|
|
}
|
|
if !now.SameInode(qi) {
|
|
return fmt.Errorf("%s is no longer the file that was quarantined: %w", p, fsops.ErrChanged)
|
|
}
|
|
} else {
|
|
if !now.SameInode(expect) {
|
|
return fmt.Errorf("%s was replaced by another file: %w", p, fsops.ErrChanged)
|
|
}
|
|
if !now.Unchanged(expect.Size, expect.MtimeNs) {
|
|
return fmt.Errorf("%s changed after it was compared (size %d -> %d, mtime %d -> %d): %w",
|
|
p, expect.Size, now.Size, expect.MtimeNs, now.MtimeNs, fsops.ErrChanged)
|
|
}
|
|
}
|
|
if err := os.Remove(p); err != nil {
|
|
return err
|
|
}
|
|
return fsops.SyncDir(filepath.Dir(p))
|
|
}
|
|
|
|
// testHookBeforeRemove lets tests change the filesystem between creating the
|
|
// new link and removing the original. Always nil outside tests.
|
|
var testHookBeforeRemove func(slaveAbs string)
|
|
|
|
// stillQuarantined reports whether the original at p is intact and still the
|
|
// quarantined file, i.e. the quarantine copy is not needed after a rollback.
|
|
func stillQuarantined(p, q string) bool {
|
|
pi, err1 := fsops.RegularFile(p)
|
|
qi, err2 := fsops.RegularFile(q)
|
|
return err1 == nil && err2 == nil && pi.SameInode(qi)
|
|
}
|
|
|
|
// pruneEmpty removes empty directories from dir upward, stopping at stop.
|
|
func pruneEmpty(dir, stop string) {
|
|
for dir != stop && fsops.Within(stop, dir) {
|
|
if err := os.Remove(dir); err != nil {
|
|
return
|
|
}
|
|
dir = filepath.Dir(dir)
|
|
}
|
|
}
|
|
|
|
func (e *Engine) begin(ctx context.Context, o *store.Operation) error {
|
|
o.DryRun = e.st.DryRun(ctx)
|
|
if o.Trigger == "" {
|
|
o.Trigger = "manual"
|
|
}
|
|
o.Status = store.OpRunning
|
|
if o.DryRun {
|
|
o.Status = store.OpPlanned
|
|
}
|
|
return e.st.CreateOp(ctx, o)
|
|
}
|
|
|
|
func (e *Engine) finish(ctx context.Context, o *store.Operation, err error) error {
|
|
if err != nil {
|
|
o.Status, o.Error = store.OpFailed, err.Error()
|
|
e.log.Warn("operation failed", "op", o.ID, "action", o.Action, "slave", o.RelSlave, "err", err)
|
|
} else {
|
|
o.Status = store.OpDone
|
|
e.log.Info("operation done", "op", o.ID, "action", o.Action, "slave", o.RelSlave, "master", o.RelMaster, "new", o.RelNew)
|
|
}
|
|
if ferr := e.st.FinishOp(ctx, o.ID, o.Status, o.Error); ferr != nil && err == nil {
|
|
return ferr
|
|
}
|
|
return err
|
|
}
|
|
|
|
func verdictOf(m store.Match) probe.Verdict {
|
|
var v probe.Verdict
|
|
_ = json.Unmarshal(m.Verdict, &v)
|
|
return v
|
|
}
|
|
|
|
// Link replaces the slave file with a hardlink of the master file.
|
|
//
|
|
// Identical files are re-compared byte for byte right before linking and no
|
|
// quarantine copy is kept (the data exists in the master). Non-identical files
|
|
// are quarantined first so the operation can be undone.
|
|
func (e *Engine) Link(ctx context.Context, matchID int64, opt LinkOptions) (store.Operation, error) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
|
|
m, err := e.st.Match(ctx, matchID)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
l, err := e.loadPair(ctx, m.PairID)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
switch m.Status {
|
|
case store.StatusIdentical, store.StatusDifferent, store.StatusSimilar:
|
|
default:
|
|
return store.Operation{}, fmt.Errorf("cannot link a match with status %q", m.Status)
|
|
}
|
|
if m.Ignored {
|
|
return store.Operation{}, errors.New("match is ignored; un-ignore it first")
|
|
}
|
|
identical := m.Status == store.StatusIdentical
|
|
if !identical && verdictOf(m).Better == "slave" && !opt.AllowDowngrade {
|
|
return store.Operation{}, ErrDowngrade
|
|
}
|
|
s, err := preflight(l, m)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
if identical {
|
|
same, err := fsops.SameContent(s.masterAbs, s.slaveAbs)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
if !same {
|
|
e.refresh(ctx, l, m.RelSlave)
|
|
return store.Operation{}, errors.New("files are no longer identical; moved to manual review")
|
|
}
|
|
}
|
|
|
|
newRel := LinkTarget(m.RelSlave, m.RelMaster, opt.KeepFolder)
|
|
newAbs, err := l.slave.Resolve(newRel)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
rename := newRel != m.RelSlave
|
|
// preexisting: the target already is a hardlink of the master (e.g. left
|
|
// behind by an earlier failed link). Then only the original is retired.
|
|
preexisting := false
|
|
if rename {
|
|
if cur, err := fsops.Lstat(newAbs); err == nil {
|
|
if !cur.Mode.IsRegular() || !cur.SameInode(s.mi) {
|
|
return store.Operation{}, fmt.Errorf("slave already has a different file at %s: %w", newRel, fsops.ErrExists)
|
|
}
|
|
preexisting = true
|
|
}
|
|
}
|
|
|
|
o := store.Operation{PairID: l.pair.ID, Action: store.ActLink, RelMaster: m.RelMaster, RelSlave: m.RelSlave,
|
|
RelNew: newRel, Trigger: opt.Trigger, MatchKind: m.Kind, MatchReason: m.Reason, LinkPreexisted: preexisting}
|
|
if identical {
|
|
o.Bytes = s.si.Size // space freed
|
|
} else {
|
|
o.QuarantineSide = "slave"
|
|
}
|
|
if err := e.begin(ctx, &o); err != nil {
|
|
return o, err
|
|
}
|
|
if o.DryRun {
|
|
return o, nil
|
|
}
|
|
|
|
err = func() error {
|
|
var q string
|
|
if !identical {
|
|
q = quarantinePath(l, o.ID, "slave", m.RelSlave)
|
|
if err := quarantine(s.slaveAbs, s.si, q); err != nil {
|
|
return fmt.Errorf("quarantine: %w", err)
|
|
}
|
|
o.QuarantinePath = q
|
|
if _, err := e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = ? WHERE id = ?`, q, o.ID); err != nil {
|
|
dropQuarantine(l, q, s.si)
|
|
return err
|
|
}
|
|
}
|
|
if !rename {
|
|
if err := fsops.ReplaceWithLink(s.masterAbs, s.mi, s.slaveAbs); err != nil {
|
|
if q != "" {
|
|
if cur, e2 := fsops.Lstat(s.slaveAbs); e2 == nil && cur.SameInode(s.si) {
|
|
dropQuarantine(l, q, s.si) // original untouched
|
|
e.clearQuarantine(ctx, o.ID)
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
if !preexisting {
|
|
err := fsops.MkdirAllLike(filepath.Dir(newAbs))
|
|
if err == nil {
|
|
err = fsops.LinkNew(s.masterAbs, s.mi, newAbs)
|
|
}
|
|
if err != nil {
|
|
pruneEmpty(filepath.Dir(newAbs), l.slave.Path)
|
|
if q != "" {
|
|
dropQuarantine(l, q, s.si)
|
|
e.clearQuarantine(ctx, o.ID)
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
if testHookBeforeRemove != nil {
|
|
testHookBeforeRemove(s.slaveAbs)
|
|
}
|
|
if err := removeOriginal(s.slaveAbs, s.si, q); err != nil {
|
|
// Never leave both copies behind: undo the new link (it is only
|
|
// an extra name for the master file) and the quarantine copy.
|
|
if !preexisting {
|
|
if rbErr := fsops.RemoveLink(newAbs, s.mi); rbErr == nil {
|
|
pruneEmpty(filepath.Dir(newAbs), l.slave.Path)
|
|
}
|
|
}
|
|
if q != "" && stillQuarantined(s.slaveAbs, q) {
|
|
dropQuarantine(l, q, s.si)
|
|
e.clearQuarantine(ctx, o.ID)
|
|
}
|
|
return fmt.Errorf("could not remove the original slave file, so nothing was changed: %w", err)
|
|
}
|
|
pruneEmpty(filepath.Dir(s.slaveAbs), l.slave.Path) // e.g. the single's now-empty folder
|
|
return nil
|
|
}()
|
|
err = e.finish(ctx, &o, err)
|
|
e.refresh(ctx, l, m.RelSlave, m)
|
|
if rename {
|
|
e.refresh(ctx, l, newRel, m)
|
|
}
|
|
return o, err
|
|
}
|
|
|
|
func (e *Engine) clearQuarantine(ctx context.Context, opID int64) {
|
|
_, _ = e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = '' WHERE id = ?`, opID)
|
|
}
|
|
|
|
// Promote is the override path: the slave copy is better, so it replaces the
|
|
// master file (whose original is quarantined) and both libraries share it.
|
|
func (e *Engine) Promote(ctx context.Context, matchID int64, confirmed bool) (store.Operation, error) {
|
|
if !confirmed {
|
|
return store.Operation{}, ErrConfirm
|
|
}
|
|
return e.promote(ctx, matchID, "manual")
|
|
}
|
|
|
|
func (e *Engine) promote(ctx context.Context, matchID int64, trigger string) (store.Operation, error) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
|
|
m, err := e.st.Match(ctx, matchID)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
l, err := e.loadPair(ctx, m.PairID)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
if m.Status != store.StatusDifferent && m.Status != store.StatusSimilar {
|
|
return store.Operation{}, fmt.Errorf("cannot promote a match with status %q", m.Status)
|
|
}
|
|
s, err := preflight(l, m)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
newMasterRel := filepath.Join(filepath.Dir(m.RelMaster), filepath.Base(m.RelSlave))
|
|
newMasterAbs, err := l.master.Resolve(newMasterRel)
|
|
if err != nil {
|
|
return store.Operation{}, err
|
|
}
|
|
rename := newMasterRel != m.RelMaster
|
|
if rename {
|
|
if _, err := os.Lstat(newMasterAbs); err == nil {
|
|
return store.Operation{}, fmt.Errorf("master already has %s: %w", newMasterRel, fsops.ErrExists)
|
|
}
|
|
}
|
|
|
|
o := store.Operation{PairID: l.pair.ID, Action: store.ActPromote, RelMaster: m.RelMaster, RelSlave: m.RelSlave,
|
|
RelNew: newMasterRel, QuarantineSide: "master", Trigger: trigger, MatchKind: m.Kind, MatchReason: m.Reason}
|
|
if err := e.begin(ctx, &o); err != nil {
|
|
return o, err
|
|
}
|
|
if o.DryRun {
|
|
return o, nil
|
|
}
|
|
err = func() error {
|
|
q := quarantinePath(l, o.ID, "master", m.RelMaster)
|
|
if err := quarantine(s.masterAbs, s.mi, q); err != nil {
|
|
return fmt.Errorf("quarantine: %w", err)
|
|
}
|
|
o.QuarantinePath = q
|
|
if _, err := e.st.DB.ExecContext(ctx, `UPDATE operations SET quarantine_path = ? WHERE id = ?`, q, o.ID); err != nil {
|
|
dropQuarantine(l, q, s.mi)
|
|
return err
|
|
}
|
|
if !rename {
|
|
if err := fsops.ReplaceWithLink(s.slaveAbs, s.si, s.masterAbs); err != nil {
|
|
if cur, e2 := fsops.Lstat(s.masterAbs); e2 == nil && cur.SameInode(s.mi) {
|
|
dropQuarantine(l, q, s.mi)
|
|
e.clearQuarantine(ctx, o.ID)
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
if err := fsops.LinkNew(s.slaveAbs, s.si, newMasterAbs); err != nil {
|
|
dropQuarantine(l, q, s.mi)
|
|
e.clearQuarantine(ctx, o.ID)
|
|
return err
|
|
}
|
|
if err := removeOriginal(s.masterAbs, s.mi, q); err != nil {
|
|
_ = fsops.RemoveLink(newMasterAbs, s.si)
|
|
if stillQuarantined(s.masterAbs, q) {
|
|
dropQuarantine(l, q, s.mi)
|
|
e.clearQuarantine(ctx, o.ID)
|
|
}
|
|
return fmt.Errorf("could not remove the original master file, so nothing was changed: %w", err)
|
|
}
|
|
return nil
|
|
}()
|
|
err = e.finish(ctx, &o, err)
|
|
promoted := m
|
|
if err == nil {
|
|
promoted.RelMaster = newMasterRel // the master file may have been renamed
|
|
}
|
|
e.refresh(ctx, l, m.RelSlave, promoted)
|
|
return o, err
|
|
}
|
|
|
|
// ---- build (fresh slave library) and remove ----
|
|
|
|
// expand turns a list of relative files/folders into audio files under root.
|
|
func expand(ctx context.Context, l *libs, root fsops.Root, rels []string) ([]string, error) {
|
|
set := map[string]fsops.Info{}
|
|
for _, rel := range rels {
|
|
abs, err := root.Resolve(rel)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
fi, err := os.Lstat(abs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
switch {
|
|
case fi.IsDir():
|
|
sub, err := fsops.NewRoot(abs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
files := map[string]fsops.Info{}
|
|
if _, err := walk(ctx, l, sub, files); err != nil {
|
|
return nil, err
|
|
}
|
|
for r, in := range files {
|
|
set[filepath.Join(rel, r)] = in
|
|
}
|
|
case fi.Mode().IsRegular() && l.hasExt(abs):
|
|
set[filepath.Clean(rel)] = fsops.Info{}
|
|
}
|
|
}
|
|
return sortedKeys(set), nil
|
|
}
|
|
|
|
// StartBuild hardlinks the chosen master files/folders into the slave library,
|
|
// mirroring the master path. Existing slave files are never overwritten.
|
|
func (e *Engine) StartBuild(pairID int64, rels []string) (*Job, error) {
|
|
return e.startJob(pairID, "build", func(ctx context.Context, j *Job) error {
|
|
l, err := e.loadPair(ctx, pairID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files, err := expand(ctx, l, l.master, rels)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
j.phase("Linking into slave library", int64(len(files)))
|
|
for _, rel := range files {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
e.buildOne(ctx, j, l, rel)
|
|
j.tick()
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (e *Engine) buildOne(ctx context.Context, j *Job, l *libs, rel string) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
masterAbs, err := l.master.Resolve(rel)
|
|
if err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
slaveAbs, err := l.slave.Resolve(rel)
|
|
if err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
mi, err := fsops.RegularFile(masterAbs)
|
|
if err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
if si, err := fsops.Lstat(slaveAbs); err == nil {
|
|
if si.SameInode(mi) {
|
|
j.count("already linked")
|
|
} else {
|
|
j.skip(rel + ": a different file already exists in the slave library (scan, then use Review)")
|
|
}
|
|
return
|
|
}
|
|
o := store.Operation{PairID: l.pair.ID, Action: store.ActBuild, RelMaster: rel, RelSlave: rel, RelNew: rel, Trigger: "build"}
|
|
if err := e.begin(ctx, &o); err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
if o.DryRun {
|
|
j.count("planned")
|
|
return
|
|
}
|
|
err = fsops.MkdirAllLike(filepath.Dir(slaveAbs))
|
|
if err == nil {
|
|
err = fsops.LinkNew(masterAbs, mi, slaveAbs)
|
|
}
|
|
if err = e.finish(ctx, &o, err); err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
j.count("linked")
|
|
e.refresh(ctx, l, rel)
|
|
}
|
|
|
|
// StartRemove deletes slave files that are hardlinks of the master file at the
|
|
// same path. Anything that is not such a link is left alone.
|
|
func (e *Engine) StartRemove(pairID int64, rels []string) (*Job, error) {
|
|
return e.startJob(pairID, "remove", func(ctx context.Context, j *Job) error {
|
|
l, err := e.loadPair(ctx, pairID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files, err := expand(ctx, l, l.slave, rels)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
j.phase("Removing links from slave library", int64(len(files)))
|
|
for _, rel := range files {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
e.removeOne(ctx, j, l, rel)
|
|
j.tick()
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (e *Engine) removeOne(ctx context.Context, j *Job, l *libs, rel string) {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
masterAbs, err1 := l.master.Resolve(rel)
|
|
slaveAbs, err2 := l.slave.Resolve(rel)
|
|
if err1 != nil || err2 != nil {
|
|
j.skip(rel + ": invalid path")
|
|
return
|
|
}
|
|
mi, err := fsops.RegularFile(masterAbs)
|
|
if err != nil {
|
|
j.skip(rel + ": no master file at the same path, so it is not removed")
|
|
return
|
|
}
|
|
si, err := fsops.RegularFile(slaveAbs)
|
|
if err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
if !si.SameInode(mi) {
|
|
j.skip(rel + ": not a hardlink of the master file, so it is not removed")
|
|
return
|
|
}
|
|
o := store.Operation{PairID: l.pair.ID, Action: store.ActRemove, RelMaster: rel, RelSlave: rel, Trigger: "build"}
|
|
if err := e.begin(ctx, &o); err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
if o.DryRun {
|
|
j.count("planned")
|
|
return
|
|
}
|
|
err = fsops.RemoveLink(slaveAbs, mi)
|
|
if err = e.finish(ctx, &o, err); err != nil {
|
|
j.skip(rel + ": " + err.Error())
|
|
return
|
|
}
|
|
pruneEmpty(filepath.Dir(slaveAbs), l.slave.Path)
|
|
j.count("removed")
|
|
e.refresh(ctx, l, rel)
|
|
}
|
|
|
|
// ---- undo, restore and purge ----
|
|
|
|
// Undo reverts a completed operation. Every branch verifies that the current
|
|
// file is still exactly what the operation left behind before touching it.
|
|
func (e *Engine) Undo(ctx context.Context, opID int64) error {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
o, err := e.st.Op(ctx, opID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if o.Status != store.OpDone || o.UndoneAt != nil {
|
|
return errors.New("only completed operations that were not undone can be reverted")
|
|
}
|
|
if o.PurgedAt != nil {
|
|
return errors.New("the quarantined original was already purged")
|
|
}
|
|
l, err := e.loadPair(ctx, o.PairID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if o.Action == store.ActCover {
|
|
if err := e.undoCover(l, o); err != nil {
|
|
return err
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("folder cover undone", "op", o.ID)
|
|
return nil
|
|
}
|
|
if o.Action == store.ActMoveDir {
|
|
if err := undoMoveDir(l, o); err != nil {
|
|
return err
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("folder move undone", "op", o.ID, "folder", o.RelNew)
|
|
_, _ = e.StartScan(o.PairID, "undo")
|
|
return nil
|
|
}
|
|
if o.Action == store.ActMove {
|
|
if err := undoMove(l, o); err != nil {
|
|
return err
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("file move undone", "op", o.ID)
|
|
_, _ = e.StartScan(o.PairID, "undo")
|
|
return nil
|
|
}
|
|
if o.Action == store.ActTags {
|
|
if err := e.undoTags(ctx, l, o); err != nil {
|
|
return err
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("tag edit undone", "op", o.ID)
|
|
return nil
|
|
}
|
|
if o.Action == store.ActRename {
|
|
if err := undoRename(l, o); err != nil {
|
|
return err
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("folder rename undone", "op", o.ID, "folder", o.RelNew)
|
|
_, _ = e.StartScan(o.PairID, "undo") // paths changed; ErrBusy just means a job already runs
|
|
return nil
|
|
}
|
|
res := func(r fsops.Root, rel string) string {
|
|
p, e := r.Resolve(rel)
|
|
if e != nil && err == nil {
|
|
err = e
|
|
}
|
|
return p
|
|
}
|
|
masterAbs := res(l.master, o.RelMaster)
|
|
slaveAbs := res(l.slave, o.RelSlave)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
switch o.Action {
|
|
case store.ActBuild:
|
|
mi, err := fsops.RegularFile(masterAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := fsops.RemoveLink(slaveAbs, mi); err != nil {
|
|
return err
|
|
}
|
|
pruneEmpty(filepath.Dir(slaveAbs), l.slave.Path)
|
|
|
|
case store.ActRemove:
|
|
mi, err := fsops.RegularFile(masterAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := fsops.MkdirAllLike(filepath.Dir(slaveAbs)); err != nil {
|
|
return err
|
|
}
|
|
if err := fsops.LinkNew(masterAbs, mi, slaveAbs); err != nil {
|
|
return err
|
|
}
|
|
|
|
case store.ActLink:
|
|
newAbs := res(l.slave, o.RelNew)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
mi, err := fsops.RegularFile(masterAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cur, err := fsops.RegularFile(newAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !cur.SameInode(mi) {
|
|
return errors.New("the slave file is no longer linked to the master file; refusing to undo")
|
|
}
|
|
renamed := o.RelNew != o.RelSlave
|
|
if renamed {
|
|
if _, err := os.Lstat(slaveAbs); err == nil {
|
|
return fmt.Errorf("%s: %w", o.RelSlave, fsops.ErrExists)
|
|
}
|
|
// The original folder may have been pruned when the link moved
|
|
// the file into the master's folder structure.
|
|
if err := fsops.MkdirAllLike(filepath.Dir(slaveAbs)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if o.QuarantinePath == "" {
|
|
// Identical link: give the slave its own copy of the bytes again.
|
|
if err := fsops.CopyReplace(masterAbs, slaveAbs, mi.Mode); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
qi, err := e.quarantined(l, o.QuarantinePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if renamed {
|
|
err = fsops.LinkNew(o.QuarantinePath, qi, slaveAbs)
|
|
} else {
|
|
err = fsops.ReplaceWithLink(o.QuarantinePath, qi, slaveAbs)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dropQuarantine(l, o.QuarantinePath, qi)
|
|
}
|
|
if renamed && !o.LinkPreexisted {
|
|
if err := fsops.RemoveLink(newAbs, mi); err != nil {
|
|
return err
|
|
}
|
|
pruneEmpty(filepath.Dir(newAbs), l.slave.Path)
|
|
}
|
|
e.refresh(ctx, l, o.RelNew)
|
|
|
|
case store.ActPromote:
|
|
newMasterAbs := res(l.master, o.RelNew)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
si, err := fsops.RegularFile(slaveAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cur, err := fsops.RegularFile(newMasterAbs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !cur.SameInode(si) {
|
|
return errors.New("the master file changed after promotion; refusing to undo")
|
|
}
|
|
qi, err := e.quarantined(l, o.QuarantinePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if o.RelNew == o.RelMaster {
|
|
err = fsops.ReplaceWithLink(o.QuarantinePath, qi, masterAbs)
|
|
} else {
|
|
if _, e2 := os.Lstat(masterAbs); e2 == nil {
|
|
return fmt.Errorf("%s: %w", o.RelMaster, fsops.ErrExists)
|
|
}
|
|
if err = fsops.LinkNew(o.QuarantinePath, qi, masterAbs); err == nil {
|
|
err = fsops.RemoveLink(newMasterAbs, si)
|
|
}
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dropQuarantine(l, o.QuarantinePath, qi)
|
|
|
|
default:
|
|
return fmt.Errorf("cannot undo %q", o.Action)
|
|
}
|
|
if err := e.st.MarkUndone(ctx, o.ID); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("operation undone", "op", o.ID, "action", o.Action)
|
|
// Restore the original pairing (the master path before any promote rename).
|
|
e.refresh(ctx, l, o.RelSlave, store.Match{RelMaster: o.RelMaster, Kind: o.MatchKind, Reason: o.MatchReason})
|
|
return nil
|
|
}
|
|
|
|
// quarantined validates a quarantine path recorded in the database.
|
|
func (e *Engine) quarantined(l *libs, q string) (fsops.Info, error) {
|
|
q = filepath.Clean(q)
|
|
if !fsops.Within(l.quarantine, q) || q == l.quarantine {
|
|
return fsops.Info{}, fmt.Errorf("quarantine path %s: %w", q, fsops.ErrOutsideRoot)
|
|
}
|
|
return fsops.RegularFile(q)
|
|
}
|
|
|
|
// Purge permanently deletes a quarantined original.
|
|
func (e *Engine) Purge(ctx context.Context, opID int64) error {
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
o, err := e.st.Op(ctx, opID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return e.purge(ctx, o)
|
|
}
|
|
|
|
func (e *Engine) purge(ctx context.Context, o store.Operation) error {
|
|
if o.QuarantinePath == "" || o.PurgedAt != nil || o.UndoneAt != nil {
|
|
return errors.New("nothing to purge for this operation")
|
|
}
|
|
l, err := e.loadPair(ctx, o.PairID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
opDir := filepath.Join(l.quarantine, strconv.FormatInt(o.ID, 10))
|
|
if !fsops.Within(l.quarantine, filepath.Clean(o.QuarantinePath)) || !fsops.Within(opDir, filepath.Clean(o.QuarantinePath)) {
|
|
return fsops.ErrOutsideRoot
|
|
}
|
|
if err := os.RemoveAll(opDir); err != nil {
|
|
return err
|
|
}
|
|
e.log.Info("quarantine purged", "op", o.ID, "path", o.QuarantinePath)
|
|
return e.st.MarkPurged(ctx, o.ID)
|
|
}
|
|
|
|
// Maintenance purges expired quarantine entries and prunes sessions.
|
|
func (e *Engine) Maintenance(ctx context.Context) {
|
|
e.st.PruneSessions(ctx)
|
|
pairs, err := e.st.Pairs(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, p := range pairs {
|
|
if p.RetentionDays <= 0 {
|
|
continue // keep forever
|
|
}
|
|
cutoff := time.Now().Add(-time.Duration(p.RetentionDays) * 24 * time.Hour).Unix()
|
|
ops, err := e.st.ExpiredQuarantine(ctx, p.ID, cutoff)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, o := range ops {
|
|
e.mu.Lock()
|
|
if err := e.purge(ctx, o); err != nil {
|
|
e.log.Warn("retention purge failed", "op", o.ID, "err", err)
|
|
}
|
|
e.mu.Unlock()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Recover runs at startup: operations left running by a crash are marked
|
|
// interrupted and their disposable staging files are removed. Originals are
|
|
// never at risk because every replacement is an atomic rename.
|
|
func (e *Engine) Recover(ctx context.Context) {
|
|
ops, err := e.st.RunningOps(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, o := range ops {
|
|
if o.Action == store.ActMoveDir {
|
|
if l, err := e.loadPair(ctx, o.PairID); err == nil {
|
|
recoverMoveDir(l, o, e.log)
|
|
}
|
|
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, "interrupted by shutdown; rescan to verify state")
|
|
continue
|
|
}
|
|
if o.Action == store.ActMove {
|
|
if l, err := e.loadPair(ctx, o.PairID); err == nil {
|
|
recoverMove(l, o)
|
|
}
|
|
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, "interrupted by shutdown; rescan to verify state")
|
|
continue
|
|
}
|
|
if o.Action == store.ActTags {
|
|
msg := e.recoverTags(ctx, o)
|
|
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, msg)
|
|
e.log.Warn("recovered interrupted tag edit", "op", o.ID, "result", msg)
|
|
continue
|
|
}
|
|
if o.Action == store.ActRename {
|
|
if p, err := e.st.Pair(ctx, o.PairID); err == nil {
|
|
recoverRename(p, o, e.log)
|
|
}
|
|
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, "interrupted by shutdown; rescan to verify state")
|
|
continue
|
|
}
|
|
if p, err := e.st.Pair(ctx, o.PairID); err == nil {
|
|
for _, root := range []string{p.SlaveRoot, p.MasterRoot} {
|
|
for _, rel := range []string{o.RelSlave, o.RelNew, o.RelMaster} {
|
|
if rel == "" {
|
|
continue
|
|
}
|
|
dir := filepath.Join(root, filepath.Dir(rel))
|
|
ents, _ := os.ReadDir(dir)
|
|
for _, en := range ents {
|
|
if strings.HasPrefix(en.Name(), fsops.TmpPrefix) && en.Type().IsRegular() {
|
|
_ = os.Remove(filepath.Join(dir, en.Name()))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
_ = e.st.FinishOp(ctx, o.ID, store.OpInterrupted, "interrupted by shutdown; rescan to verify state")
|
|
e.log.Warn("marked interrupted operation", "op", o.ID)
|
|
}
|
|
}
|
|
|
|
// ---- browsing (build mode) ----
|
|
|
|
type TreeEntry struct {
|
|
Name string `json:"name"`
|
|
Rel string `json:"rel"`
|
|
Dir bool `json:"dir"`
|
|
Size int64 `json:"size,omitempty"`
|
|
State string `json:"state,omitempty"` // files: linked, conflict, missing
|
|
Linked int `json:"linked,omitempty"`
|
|
}
|
|
|
|
// Tree lists a master directory and shows how much of it is in the slave.
|
|
func (e *Engine) Tree(ctx context.Context, pairID int64, rel string) ([]TreeEntry, error) {
|
|
l, err := e.loadPair(ctx, pairID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dir := l.master.Path
|
|
if rel != "" {
|
|
if dir, err = l.master.Resolve(rel); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
ents, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
linked, _ := e.st.LinkedMasterPaths(ctx, pairID, rel)
|
|
perChild := map[string]int{}
|
|
for _, p := range linked {
|
|
sub := p
|
|
if rel != "" {
|
|
sub = strings.TrimPrefix(p, rel+string(filepath.Separator))
|
|
}
|
|
if i := strings.IndexByte(sub, filepath.Separator); i > 0 {
|
|
perChild[sub[:i]]++
|
|
}
|
|
}
|
|
out := []TreeEntry{}
|
|
for _, en := range ents {
|
|
name := en.Name()
|
|
if strings.HasPrefix(name, ".") || en.Type()&fs.ModeSymlink != 0 {
|
|
continue
|
|
}
|
|
child := name
|
|
if rel != "" {
|
|
child = filepath.Join(rel, name)
|
|
}
|
|
te := TreeEntry{Name: name, Rel: child, Dir: en.IsDir()}
|
|
if en.IsDir() {
|
|
te.Linked = perChild[name]
|
|
} else {
|
|
if !l.hasExt(name) {
|
|
continue
|
|
}
|
|
mi, err := fsops.Lstat(filepath.Join(dir, name))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
te.Size = mi.Size
|
|
te.State = "missing"
|
|
if sp, err := l.slave.Resolve(child); err == nil {
|
|
if si, err := fsops.Lstat(sp); err == nil {
|
|
te.State = "conflict"
|
|
if si.SameInode(mi) {
|
|
te.State = "linked"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out = append(out, te)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if out[i].Dir != out[j].Dir {
|
|
return out[i].Dir
|
|
}
|
|
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
|
|
})
|
|
return out, nil
|
|
}
|