From 0fc35b13e813a7dab842b74e8c954d3db16214cc Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 29 Oct 2026 10:28:35 -0700 Subject: [PATCH] Prune stems whose source song was removed or replaced Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + README.md | 3 ++ desktop/stemify/library_core.py | 47 +++++++++++++++++++++++++++++++ server/README.md | 3 ++ server/prune.py | 49 +++++++++++++++++++++++++++++++++ server/stemd.Dockerfile | 2 +- server/stemd.py | 31 +++++++++++++++++++-- 7 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 server/prune.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d27fbf0..0a519ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,3 +16,4 @@ 2026-10-28 iOS: Removed drag reorder from the Tab Bar settings page 2026-10-28 iOS: CarPlay Downloads Playlists, Albums and Artists load from local downloads, work offline 2026-10-28 iOS: Stem playlist metadata refresh matches by key, file, then title/artist with a search fallback, and also runs when the app returns to the foreground +2026-10-29 server: stemd prunes stems whose source song is gone (orphans). PRUNE, PRUNE_CHANGED, PRUNE_MAX env. server/prune.py is a dry-run CLI. diff --git a/README.md b/README.md index 7ccb0f5..b3cd56f 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,9 @@ To update, copy the new files, rebuild, `docker rm -f` the container and run it | `MODEL` | `roformer_bs_sw` | Any id from [the model table](#separation-models) | | `DEVICE` | `cuda` | `cuda`, `cuda:N` or `cpu` | | `RESCAN_MIN` | `30` | Minutes between library scans once caught up | +| `PRUNE` | `1` | Remove stems whose song is gone from the library, at start and every `RESCAN_MIN` | +| `PRUNE_CHANGED` | `0` | Also remove stems of songs replaced in place (size or mtime changed) | +| `PRUNE_MAX` | `0.2` | Skip a prune that would remove more than this share of the stems. `server/prune.py` is the manual dry run, `--apply` to remove | | `AUTO_SCAN` | `0` | `1` to scan the library on a timer. Off by default: only Stem Now requests are separated | | `FP16` | `1` | Half precision on the GPU (`0` for fp32) | | `OVERLAP`, `SEGMENT`, `SHIFTS` | `0.25`, `0`, `0` | Separation quality, same as the desktop app | diff --git a/desktop/stemify/library_core.py b/desktop/stemify/library_core.py index 7959074..d042cd7 100644 --- a/desktop/stemify/library_core.py +++ b/desktop/stemify/library_core.py @@ -12,6 +12,7 @@ import os import shutil import subprocess import time +import unicodedata import numpy as np import soundfile as sf @@ -105,3 +106,49 @@ def write_track(dest, stems, meta): for name in [s["file"] for s in written] + [MARKER]: os.replace(os.path.join(part, name), os.path.join(dest, name)) os.rmdir(part) + + +def _norm(path): + return unicodedata.normalize("NFC", os.path.normpath(path)).casefold() + + +def find_stale(src, dst): + """Stems folders that no longer match the library. + + Returns (orphans, changed): orphans have no source track any more (removed, renamed or + moved); changed still have one, but its size or mtime differs from the marker's, so the + stems are of an older version of the song. + """ + expected = {_norm(d): p for p, d in iter_tracks(src, dst)} + orphans, changed = [], [] + for root, dirs, files in os.walk(dst): + dirs[:] = [d for d in dirs if not d.startswith(".")] + if MARKER not in files: + continue + dirs[:] = [] + source = expected.get(_norm(root)) + if source is None: + orphans.append(root) + continue + try: + with open(os.path.join(root, MARKER)) as f: + m = json.load(f) + if [m["size"], m["mtime_ns"]] != signature(source): + changed.append(root) + except (OSError, ValueError, KeyError): + pass # unreadable marker or source: leave it alone + return sorted(orphans), sorted(changed) + + +def remove_stems(dst, folders): + """Delete stems folders, then any parent folders left empty (never dst itself).""" + dst = os.path.realpath(dst) + for folder in folders: + shutil.rmtree(folder) + parent = os.path.dirname(os.path.realpath(folder)) + while parent != dst and parent.startswith(dst + os.sep): + try: + os.rmdir(parent) + except OSError: + break + parent = os.path.dirname(parent) diff --git a/server/README.md b/server/README.md index d1f08ba..d172f1e 100644 --- a/server/README.md +++ b/server/README.md @@ -130,6 +130,9 @@ the app would briefly see no stems. | `DEVICE` | `cuda` | `cuda`, `cuda:N` or `cpu` | | `RESCAN_MIN` | `30` | minutes between scans once caught up | | `AUTO_SCAN` | `0` | `1` = scan the library every `RESCAN_MIN`; `0` = only do Stem Now requests | +| `PRUNE` | `1` | Remove stems whose song is gone from the library, at start and every `RESCAN_MIN` | +| `PRUNE_CHANGED` | `0` | Also remove stems of songs replaced in place (size or mtime changed) | +| `PRUNE_MAX` | `0.2` | Skip a prune that would remove more than this share of the stems. `server/prune.py` is the manual dry run, `--apply` to remove | | `FP16` | `1` | half precision on the GPU (`0` for fp32) | | `OVERLAP`, `SEGMENT`, `SHIFTS` | `0.25`, `0`, `0` | as in the desktop app's settings | diff --git a/server/prune.py b/server/prune.py new file mode 100644 index 0000000..816cd71 --- /dev/null +++ b/server/prune.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""prune: list or remove stems whose song is no longer in the music library. + + prune.py dry run: print orphans (song gone) and changed (song replaced) + prune.py --apply remove the orphans + prune.py --apply --changed also remove stems of replaced songs, so stemd redoes them + +MUSIC_DIR and STEMS_DIR as for stemd. Refuses when the music folder holds no tracks. +Without --force it also refuses to remove more than 20% of the stems in one go. +""" + +import argparse +import os +import sys + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_ROOT, "desktop")) +sys.path.insert(0, _ROOT) + +from stemify.library_core import find_stale, iter_tracks, remove_stems # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--music", default=os.environ.get("MUSIC_DIR", "/music")) + ap.add_argument("--stems", default=os.environ.get("STEMS_DIR", "/stems")) + ap.add_argument("--apply", action="store_true") + ap.add_argument("--changed", action="store_true") + ap.add_argument("--force", action="store_true") + a = ap.parse_args() + total = sum(1 for _ in iter_tracks(a.music, a.stems)) + if not total: + sys.exit(f"no tracks in {a.music}; is the share mounted?") + orphans, changed = find_stale(a.music, a.stems) + drop = orphans + (changed if a.changed else []) + for label, items in (("orphan", orphans), ("changed", changed)): + for p in items: + print(f"{label}\t{os.path.relpath(p, a.stems)}") + print(f"{len(orphans)} orphans, {len(changed)} changed, {total} tracks in the library") + if not a.apply or not drop: + return + if len(drop) > 0.2 * (total + len(orphans)) and not a.force: + sys.exit("that is over 20% of the stems; check the paths or pass --force") + remove_stems(a.stems, drop) + print(f"removed {len(drop)}") + + +if __name__ == "__main__": + main() diff --git a/server/stemd.Dockerfile b/server/stemd.Dockerfile index 917fda7..1000792 100644 --- a/server/stemd.Dockerfile +++ b/server/stemd.Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev \ COPY desktop/stemify/__init__.py desktop/stemify/catalog.py desktop/stemify/library_core.py /app/stemify/ COPY desktop/stemify/worker/__init__.py desktop/stemify/worker/__main__.py desktop/stemify/worker/audio_io.py \ desktop/stemify/worker/backends.py /app/stemify/worker/ -COPY server/stemd.py /app/server/ +COPY server/stemd.py server/prune.py /app/server/ ENV MUSIC_DIR=/music STEMS_DIR=/stems STATE_DIR=/state MODELS_DIR=/models \ TORCH_HOME=/models/torch HF_HOME=/models/hf NUMBA_CACHE_DIR=/tmp/numba HOME=/tmp \ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONWARNINGS=ignore::FutureWarning diff --git a/server/stemd.py b/server/stemd.py index bdd94a7..2dedce4 100644 --- a/server/stemd.py +++ b/server/stemd.py @@ -21,6 +21,9 @@ Settings (environment): DEVICE cuda cuda, cuda:N or cpu RESCAN_MIN 30 minutes between scans once caught up AUTO_SCAN 0 1 = scan the library on a timer; 0 = only separate Stem Now requests + PRUNE 1 remove stems whose song left the library; 0 = off + PRUNE_CHANGED 0 1 = also remove stems of songs replaced in place (redone by the next scan) + PRUNE_MAX 0.2 skip a prune that would remove more than this share of the stems FP16 1 half precision on the GPU OVERLAP / SEGMENT / SHIFTS as in the desktop app's settings """ @@ -40,8 +43,8 @@ sys.path.insert(0, os.path.join(_ROOT, "desktop")) # in the image the package s sys.path.insert(0, _ROOT) from stemify.catalog import MODELS_BY_ID # noqa: E402 -from stemify.library_core import (MARKER, MAX_SECONDS, SOURCE_RANK, duration, is_done, is_share_error, # noqa: E402 - iter_tracks, signature, write_track) +from stemify.library_core import (MARKER, MAX_SECONDS, SOURCE_RANK, duration, find_stale, is_done, # noqa: E402 + is_share_error, iter_tracks, remove_stems, signature, write_track) MUSIC_DIR = os.environ.get("MUSIC_DIR", "/music") STEMS_DIR = os.environ.get("STEMS_DIR", "/stems") @@ -51,6 +54,9 @@ MODEL = os.environ.get("MODEL", "roformer_bs_sw") DEVICE = os.environ.get("DEVICE", "cuda") RESCAN_S = float(os.environ.get("RESCAN_MIN", "30")) * 60 AUTO_SCAN = os.environ.get("AUTO_SCAN", "0") in ("1", "true", "yes") +PRUNE = os.environ.get("PRUNE", "1") not in ("0", "false", "no", "") +PRUNE_CHANGED = os.environ.get("PRUNE_CHANGED", "0") in ("1", "true", "yes") +PRUNE_MAX = float(os.environ.get("PRUNE_MAX", "0.2")) FP16 = os.environ.get("FP16", "1") not in ("0", "false", "no", "") OVERLAP = float(os.environ.get("OVERLAP", "0.25")) SEGMENT = float(os.environ.get("SEGMENT", "0")) @@ -331,6 +337,26 @@ def serve_queue(sep, failed): return used +def prune(): + """Remove stems whose song is gone from MUSIC_DIR (and optionally replaced ones).""" + if not PRUNE: + return + orphans, changed = find_stale(MUSIC_DIR, STEMS_DIR) + drop = orphans + (changed if PRUNE_CHANGED else []) + if changed and not PRUNE_CHANGED: + log(f"prune: {len(changed):,} stems are of a replaced song (PRUNE_CHANGED=1 removes them)") + if not drop: + return + total = sum(1 for _ in iter_tracks(MUSIC_DIR, STEMS_DIR)) + kept = total - len(orphans) + if kept <= 0 or len(drop) > PRUNE_MAX * (kept + len(drop)): + log(f"prune: refusing to remove {len(drop):,} of {kept + len(drop):,} stems " + f"(over PRUNE_MAX {PRUNE_MAX:.0%}, or the music share looks empty); run server/prune.py --apply to force") + return + remove_stems(STEMS_DIR, drop) + log(f"prune: removed {len(drop):,} stems folders no longer in the library") + + def run_pass(sep, failed): """Separate everything that needs it. Returns (tracks done, whether the GPU was used).""" t0 = time.monotonic() @@ -404,6 +430,7 @@ def main(): log("AUTO_SCAN off: only Stem Now requests will be separated") while not stop.is_set(): try: + prune() if not AUTO_SCAN: idle(sep, failed, RESCAN_S) continue