stemify/stemroller/worker/__main__.py

"""Separation worker process.

Connects to the UI's local socket and speaks newline-delimited JSON:

  UI -> worker   start {id, file, model, device, two_stems, out_dir, segment, shifts, overlap, fp16}
                 cancel {id}
                 release {}   drop cached models and free VRAM
                 quit {}
  worker -> UI   hello {devices[], torch, hip}
                 vram {device, free_mb, total_mb}
                 progress {id, pct, stage}
                 done {id, stems[], peaks_path, samplerate, duration, out_dir}
                 error {id, msg}
                 cancelled {id}

Runs one job at a time; the UI owns the queue.
"""

import argparse
import json
import os
import queue
import re
import shutil
import socket
import subprocess
import sys
import threading
import time
import traceback

# Fast MIOpen kernel selection: avoids multi-minute first-run tuning on RDNA4.
os.environ.setdefault("MIOPEN_FIND_MODE", "FAST")
os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")
os.environ.setdefault("MIOPEN_LOG_LEVEL", "2")  # silence workspace warnings

PROGRESS_INTERVAL = 1 / 30  # doc: at most 30 progress updates per second
MODEL_DIR = os.path.join(os.path.expanduser("~/.cache/stemroller"), "models")


class Conn:
    def __init__(self, path):
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.sock.connect(path)
        self.lock = threading.Lock()
        self.rfile = self.sock.makefile("r", encoding="utf-8")

    def send(self, type_, **payload):
        line = json.dumps({"type": type_, **payload}) + "\n"
        with self.lock:
            self.sock.sendall(line.encode())

    def __iter__(self):
        for line in self.rfile:
            line = line.strip()
            if line:
                yield json.loads(line)


class Job:
    def __init__(self, msg, conn):
        from stemroller.catalog import MODELS_BY_ID
        self.id = msg["id"]
        self.file = msg["file"]
        self.model = MODELS_BY_ID[msg["model"]]
        self.device = msg.get("device", "auto")
        self.two_stems = bool(msg.get("two_stems"))
        self.out_dir = msg["out_dir"]
        self.segment = float(msg.get("segment") or 0)
        self.shifts = int(msg.get("shifts") or 0)
        self.overlap = float(msg.get("overlap", 0.25))
        self.fp16 = bool(msg.get("fp16", True))
        # audio-separator expresses overlap as a divisor of the chunk (step = chunk / overlap);
        # RoFormer's windowed overlap-add wants at least 4 (75%).
        self.overlap_int = max(4, round(1 / max(0.05, 1 - self.overlap)))
        self.cancel_event = threading.Event()
        self._conn = conn
        self._last_sent = 0.0

    def check(self):
        if self.cancel_event.is_set():
            from stemroller.worker.backends import Cancelled
            raise Cancelled()

    def report(self, stage, pct, force=False):
        now = time.monotonic()
        if force or now - self._last_sent >= PROGRESS_INTERVAL:
            self._last_sent = now
            self._conn.send("progress", id=self.id, pct=round(float(pct), 4), stage=stage)


def gpu_marketing_names():
    """gfx arch -> marketing name from rocminfo (torch reports a generic name on ROCm)."""
    names = {}
    if not shutil.which("rocminfo"):
        return names
    try:
        out = subprocess.run(["rocminfo"], capture_output=True, text=True, timeout=5).stdout
    except Exception:
        return names
    arch = None
    for line in out.splitlines():
        m = re.match(r"\s*Name:\s+(gfx\w+)\s*$", line)
        if m:
            arch = m.group(1)
        m = re.match(r"\s*Marketing Name:\s+(.+?)\s*$", line)
        if m and arch:
            names.setdefault(arch, m.group(1))
            arch = None
    return names


def pci_address(props):
    bus = getattr(props, "pci_bus_id", None)
    if bus is None:
        return None
    return f"{props.pci_domain_id:04x}:{bus:02x}:{props.pci_device_id:02x}.0"


def sysfs_vram_paths(props):
    pci = pci_address(props)
    if pci is None:
        return None
    base = f"/sys/bus/pci/devices/{pci}"
    total, used = base + "/mem_info_vram_total", base + "/mem_info_vram_used"
    return (total, used) if os.path.exists(used) else None


def enumerate_devices():
    import torch
    devices = []
    if torch.cuda.is_available():
        names = gpu_marketing_names() if torch.version.hip else {}
        for i in range(torch.cuda.device_count()):
            p = torch.cuda.get_device_properties(i)
            if getattr(p, "is_integrated", False):
                continue  # skip APU iGPUs (e.g. the 9800X3D's gfx1036)
            arch = getattr(p, "gcnArchName", "").split(":")[0]
            devices.append({
                "id": f"cuda:{i}",
                "name": names.get(arch, p.name),
                "arch": arch,
                "total_mb": p.total_memory // 2**20,
                "pci": pci_address(p),
                "_sysfs": sysfs_vram_paths(p),
            })
    return devices


def vram_loop(conn, devices):
    import torch
    while True:
        for d in devices:
            try:
                if d["_sysfs"]:
                    total = int(open(d["_sysfs"][0]).read()) // 2**20
                    used = int(open(d["_sysfs"][1]).read()) // 2**20
                    free = total - used
                else:
                    free, total = (v // 2**20 for v in torch.cuda.mem_get_info(int(d["id"].split(":")[1])))
                conn.send("vram", device=d["id"], free_mb=free, total_mb=total)
            except Exception:
                pass
        time.sleep(1.0)


def resolve_device(choice, devices):
    import torch
    if choice == "cpu" or not devices:
        return torch.device("cpu")
    if choice != "auto" and any(d["id"] == choice for d in devices):
        return torch.device(choice)
    best = max(devices, key=lambda d: d["total_mb"])
    return torch.device(best["id"])


def run_job(job, backends, devices, conn):
    import numpy as np
    import torch
    from stemroller.worker.audio_io import save_peaks, write_stem

    device = resolve_device(job.device, devices)
    backend = backends[job.model["family"]]
    os.makedirs(job.out_dir, exist_ok=True)
    try:
        stems, sr = backend.run(job, device)
    except torch.OutOfMemoryError:
        torch.cuda.empty_cache()
        raise RuntimeError("Out of GPU memory. Lower the segment size in Settings or pick CPU.")

    if job.two_stems and "vocals" in stems and len(stems) > 2:
        rest = [a for k, a in stems.items() if k != "vocals"]
        stems = {"vocals": stems["vocals"], "instrumental": np.sum(rest, axis=0)}

    job.report("Writing stems", 0.94, force=True)
    out = []
    for name, audio in stems.items():
        job.check()
        path = os.path.join(job.out_dir, f"{name}.wav")
        write_stem(path, audio, sr)
        out.append({"name": name, "path": path})
    job.report("Building waveforms", 0.98, force=True)
    peaks_path = os.path.join(job.out_dir, "peaks.npz")
    save_peaks(peaks_path, stems, sr)
    duration = next(iter(stems.values())).shape[1] / sr
    meta = {"source": job.file, "model": job.model["id"], "two_stems": job.two_stems,
            "stems": out, "peaks_path": peaks_path, "samplerate": sr, "duration": duration,
            "device": str(device), "finished": time.time()}
    with open(os.path.join(job.out_dir, "meta.json"), "w") as f:
        json.dump(meta, f)
    if device.type == "cuda":
        torch.cuda.empty_cache()
    return {"stems": out, "peaks_path": peaks_path, "samplerate": sr, "duration": duration,
            "out_dir": job.out_dir, "device": str(device)}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--socket", required=True)
    args = ap.parse_args()

    try:
        os.nice(5)  # UI first; the GPU does the heavy lifting anyway
    except OSError:
        pass
    conn = Conn(args.socket)
    conn.send("booting")

    import torch
    torch.set_num_threads(max(1, (os.cpu_count() or 4) - 2))
    from stemroller.worker.backends import Cancelled, DemucsBackend, RoformerBackend

    devices = enumerate_devices()
    conn.send("hello", devices=[{k: v for k, v in d.items() if not k.startswith("_")} for d in devices],
              torch=torch.__version__, hip=torch.version.hip)
    threading.Thread(target=vram_loop, args=(conn, devices), daemon=True).start()

    jobs: queue.Queue = queue.Queue()
    current: dict = {}

    def reader():
        try:
            for msg in conn:
                t = msg.get("type")
                if t in ("start", "release"):
                    jobs.put(msg)
                elif t == "cancel":
                    job = current.get("job")
                    if job and job.id == msg.get("id"):
                        job.cancel_event.set()
                elif t == "quit":
                    break
        except (OSError, ValueError):
            pass
        # UI went away or asked us to quit: no point finishing work nobody will read.
        os._exit(0)

    threading.Thread(target=reader, daemon=True).start()

    backends = {"demucs": DemucsBackend(), "roformer": RoformerBackend(MODEL_DIR)}
    while True:
        msg = jobs.get()
        if msg.get("type") == "release":
            for b in backends.values():
                b.release()
            continue
        try:
            job = Job(msg, conn)
        except Exception as e:
            conn.send("error", id=msg.get("id"), msg=f"Bad job: {e}")
            continue
        current["job"] = job
        try:
            result = run_job(job, backends, devices, conn)
            conn.send("done", id=job.id, **result)
        except Cancelled:
            shutil.rmtree(job.out_dir, ignore_errors=True)
            conn.send("cancelled", id=job.id)
        except Exception as e:
            traceback.print_exc()
            shutil.rmtree(job.out_dir, ignore_errors=True)
            conn.send("error", id=job.id, msg=str(e) or e.__class__.__name__)
        finally:
            current.pop("job", None)
            sys.stdout.flush()


if __name__ == "__main__":
    main()