Find and remove duplicate songs within the slave library

A new Duplicates filter in Review finds songs filed more than once in the
slave library, such as a single next to its album version, and recommends
which copy to keep. Removed copies go to the quarantine and can be restored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

web/src/api.ts

@@ -139,7 +139,7 @@ export type Match = {
 export type Operation = {
   id: number;
   pair_id: number;
-  action: "link" | "promote" | "build" | "remove" | "rename" | "remove-folder";
+  action: "link" | "promote" | "build" | "remove" | "rename" | "remove-folder" | "remove-duplicate";
   rel_master: string;
   rel_slave: string;
   rel_new: string;

web/src/pages/Activity.tsx

@@ -6,6 +6,7 @@ import {
   ChevronLeft,
   ChevronRight,
   CircleAlert,
+  CopyMinus,
   Crown,
   FolderPen,
   FolderPlus,
@@ -65,6 +66,7 @@ const actionLabel: Record<Operation["action"], string> = {
   remove: "Removed from slave",
   rename: "Renamed slave folder",
   "remove-folder": "Removed folder without music",
+  "remove-duplicate": "Removed duplicate song",
 };
 
 const actionIcon: Record<Operation["action"], typeof Link2> = {
@@ -74,6 +76,7 @@ const actionIcon: Record<Operation["action"], typeof Link2> = {
   remove: Unlink,
   rename: FolderPen,
   "remove-folder": FolderX,
+  "remove-duplicate": CopyMinus,
 };
 
 // Only exceptions get a badge: "done" is the normal case.
@@ -306,5 +309,7 @@ function undoText(o: Operation) {
       return `The slave folder is renamed back from "${o.rel_new}" to "${o.rel_slave}", and the library is rescanned.`;
     case "remove-folder":
       return `The folder "${o.rel_slave || o.rel_master}" is moved back from the quarantine into the ${o.quarantine_side} library, with everything that was in it.`;
+    case "remove-duplicate":
+      return `The song "${o.rel_slave}" is moved back from the quarantine into the slave library.`;
   }
 }

web/src/pages/Duplicates.tsx

@@ -0,0 +1,327 @@
+import { useEffect, useMemo, useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { AlertTriangle, AudioLines, CopyMinus, Link2, Play, Search, Star } from "lucide-react";
+import { api, type Pair, type ProbeInfo } from "../api";
+import { Alert, Badge, Button, Card, Modal, cx, formatAgo, formatBytes, formatDuration, inputCls } from "../ui";
+import { useToast } from "../toast";
+import { JobBar } from "./PairPage";
+import { MiniPlayer, type Track } from "./Player";
+
+type DupeFile = { rel: string; size: number; release: "album" | "single" | "compilation"; linked: boolean; probe?: ProbeInfo; keep: boolean };
+type DupeGroup = { id: string; title: string; files: DupeFile[]; notes?: string[] | null };
+type DupeScan = { created_at: number; files: number; groups: DupeGroup[]; probed: boolean };
+
+const SHOW = 100;
+
+function quality(p?: ProbeInfo) {
+  if (!p || p.error) return null;
+  if (p.lossless) return `${p.codec.toUpperCase()} ${p.bit_depth || "?"}/${(p.sample_rate / 1000).toFixed(1)}`;
+  return `${p.codec.toUpperCase()} ${Math.round(p.bitrate / 1000)}k`;
+}
+
+function split(rel: string) {
+  const i = rel.lastIndexOf("/");
+  return i >= 0 ? [rel.slice(0, i + 1), rel.slice(i + 1)] : ["", rel];
+}
+
+/**
+ * Songs that are in the slave library more than once, such as a single next
+ * to the album version. Separate from master/slave matching: this only looks
+ * inside the slave library.
+ */
+export function Duplicates({ pair }: { pair: Pair }) {
+  const qc = useQueryClient();
+  const toast = useToast();
+  const [sel, setSel] = useState<Set<string>>(new Set());
+  const [q, setQ] = useState("");
+  const [show, setShow] = useState(SHOW);
+  const [confirm, setConfirm] = useState(false);
+  const [player, setPlayer] = useState<{ queue: Track[]; index: number } | null>(null);
+  const nowPlaying = player ? player.queue[player.index].rel : null;
+
+  const data = useQuery({
+    queryKey: ["duplicates", pair.id],
+    queryFn: () => api.get<{ scan: DupeScan | null; dry_run: boolean }>(`/pairs/${pair.id}/duplicates`),
+  });
+  const scan = data.data?.scan ?? null;
+  const running = pair.job?.state === "running";
+  const ourJob = running && pair.job!.kind.includes("duplicates");
+  const refresh = () => {
+    qc.invalidateQueries({ queryKey: ["pair", pair.id] });
+    qc.invalidateQueries({ queryKey: ["duplicates", pair.id] });
+  };
+  // Every copy except the recommended one starts selected.
+  useEffect(() => {
+    setSel(new Set((scan?.groups ?? []).flatMap((g) => g.files.filter((f) => !f.keep).map((f) => f.rel))));
+  }, [scan?.created_at]); // eslint-disable-line react-hooks/exhaustive-deps
+  // A removal shrinks the list: drop selections of files that are gone.
+  useEffect(() => {
+    if (!scan) return;
+    const listed = new Set(scan.groups.flatMap((g) => g.files.map((f) => f.rel)));
+    setSel((s) => new Set([...s].filter((r) => listed.has(r))));
+  }, [scan]);
+
+  const find = useMutation({
+    mutationFn: () => api.post(`/pairs/${pair.id}/duplicates/scan`),
+    onSuccess: refresh,
+    onError: (e) => toast("err", e.message),
+  });
+
+  const groups = scan?.groups ?? [];
+  const needle = q.trim().toLowerCase();
+  const shown = useMemo(
+    () => (needle ? groups.filter((g) => g.title.toLowerCase().includes(needle) || g.files.some((f) => f.rel.toLowerCase().includes(needle))) : groups),
+    [groups, needle],
+  );
+  const chosen = groups.flatMap((g) => g.files.filter((f) => sel.has(f.rel)));
+  const extra = groups.reduce((n, g) => n + g.files.length - 1, 0);
+
+  const toggle = (g: DupeGroup, rel: string) =>
+    setSel((s) => {
+      const n = new Set(s);
+      if (n.has(rel)) n.delete(rel);
+      else if (g.files.filter((f) => f.rel !== rel).some((f) => !n.has(f.rel))) n.add(rel); // always keep one copy
+      return n;
+    });
+  const play = (g: DupeGroup, rel: string) => {
+    const queue = g.files.map((f) => ({ rel: f.rel, name: split(f.rel)[1], side: "slave" as const }));
+    setPlayer({ queue, index: queue.findIndex((t) => t.rel === rel) });
+  };
+
+  return (
+    <div className={cx("space-y-4", player && "pb-28")}>
+      <Card className="space-y-4 p-5">
+        <div className="flex flex-wrap items-start justify-between gap-3">
+          <div className="max-w-2xl">
+            <h2 className="flex items-center gap-2 font-semibold">
+              <CopyMinus className="size-4 text-accent" /> Duplicate songs in the slave library
+            </h2>
+            <p className="mt-1 text-sm text-muted">
+              Finds songs that are in the slave library more than once, such as a single next to its album version, a compilation track, or one
+              song in two formats. This only looks inside the slave library and is separate from matching against the master. Live, remix,
+              acoustic and other versions are kept apart. Removed copies go to the quarantine and can be restored from History.
+            </p>
+          </div>
+          <Button variant="primary" onClick={() => find.mutate()} disabled={running} loading={find.isPending}>
+            <Search className="size-4" /> Find duplicates
+          </Button>
+        </div>
+        {ourJob && <JobBar job={pair.job!} />}
+      </Card>
+
+      {scan && !ourJob && (
+        <Card className="overflow-hidden">
+          <div className="flex flex-wrap items-center justify-between gap-3 border-b border-line px-4 py-3">
+            <div className="text-sm">
+              {groups.length === 0 ? (
+                "No duplicates"
+              ) : (
+                <>
+                  <span className="font-medium">{groups.length}</span> {groups.length === 1 ? "song is" : "songs are"} in the slave library more than
+                  once ({extra} extra {extra === 1 ? "copy" : "copies"})
+                </>
+              )}
+              <span className="text-muted">
+                {" "}
+                · {scan.files} songs checked {formatAgo(scan.created_at)}
+              </span>
+            </div>
+            {groups.length > 0 && (
+              <div className="flex w-full items-center gap-2 sm:w-auto">
+                <div className="relative flex-1 sm:w-56">
+                  <Search className="absolute left-2.5 top-2.5 size-4 text-muted" />
+                  <input className={inputCls + " pl-8"} placeholder="Filter by path…" value={q} onChange={(e) => setQ(e.target.value)} />
+                </div>
+                <Button variant="danger" disabled={chosen.length === 0 || running} onClick={() => setConfirm(true)}>
+                  <CopyMinus className="size-4" /> Remove {chosen.length}…
+                </Button>
+              </div>
+            )}
+          </div>
+          {!scan.probed && groups.length > 0 && (
+            <div className="border-b border-line px-4 py-2 text-xs text-amber-300/90">
+              ffprobe isn't available, so artist tags, lengths and quality weren't compared. Check each group before removing.
+            </div>
+          )}
+          {groups.length === 0 ? (
+            <div className="px-4 py-8 text-center text-sm text-muted">Every song is in the slave library once.</div>
+          ) : (
+            <>
+              <div className="border-b border-line bg-panel-2 px-4 py-2 text-xs text-muted">
+                <Star className="mb-0.5 inline size-3 fill-current text-emerald-300" /> marks the copy recommended to keep: the album version first, then one
+                already linked to the master, then the better quality. The others start selected; one copy of every song is always kept.
+              </div>
+              <div className="divide-y divide-line">
+                {shown.slice(0, show).map((g) => (
+                  <div key={g.id} className="px-4 py-3">
+                    <div className="mb-1.5 flex flex-wrap items-baseline gap-x-2 text-sm font-medium text-gray-100">
+                      {g.title}
+                      <span className="text-xs font-normal text-muted">{g.files.length} copies</span>
+                    </div>
+                    {g.notes?.map((n) => (
+                      <div key={n} className="mb-1.5 flex items-start gap-1.5 text-xs text-amber-300/90">
+                        <AlertTriangle className="mt-px size-3.5 shrink-0" /> {n}
+                      </div>
+                    ))}
+                    <div className="space-y-0.5">
+                      {g.files.map((f) => {
+                        const [dir, base] = split(f.rel);
+                        const on = sel.has(f.rel);
+                        const lastKept = !on && g.files.every((o) => o.rel === f.rel || sel.has(o.rel));
+                        const playing = nowPlaying === f.rel;
+                        return (
+                          <div key={f.rel} className={cx("flex items-center gap-3 rounded-md px-2 py-1.5", on ? "bg-red-500/5" : "hover:bg-panel-2/60")}>
+                            <input
+                              type="checkbox"
+                              className="shrink-0 cursor-pointer disabled:cursor-not-allowed"
+                              aria-label={`Remove ${f.rel}`}
+                              checked={on}
+                              disabled={lastKept}
+                              title={lastKept ? "One copy is always kept" : on ? "Will be removed" : "Select to remove"}
+                              onChange={() => toggle(g, f.rel)}
+                            />
+                            <button
+                              className={cx(
+                                "grid size-6 shrink-0 place-items-center rounded-full cursor-pointer",
+                                playing ? "brand-gradient text-on-accent" : "text-muted hover:bg-accent/15 hover:text-accent",
+                              )}
+                              title={playing ? "Playing" : "Listen"}
+                              onClick={() => play(g, f.rel)}
+                            >
+                              {playing ? <AudioLines className="size-3.5" /> : <Play className="ml-px size-3 fill-current" />}
+                            </button>
+                            <div className={cx("min-w-0 flex-1 break-words text-sm sm:truncate", on && "text-gray-400 line-through decoration-red-400/50")} title={f.rel}>
+                              <span className="text-muted">{dir}</span>
+                              <span className={cx(playing ? "font-medium text-accent" : "text-gray-100")}>{base}</span>
+                            </div>
+                            <div className="flex shrink-0 items-center gap-1.5">
+                              {f.keep && (
+                                <Badge tone="green">
+                                  <Star className="size-3 fill-current" /> keep
+                                </Badge>
+                              )}
+                              {f.release !== "album" && <Badge tone="amber">{f.release}</Badge>}
+                              {f.linked && (
+                                <Badge tone="accent">
+                                  <Link2 className="size-3" /> linked
+                                </Badge>
+                              )}
+                            </div>
+                            <div className="hidden w-24 shrink-0 text-right text-xs text-muted md:block">{quality(f.probe) ?? ""}</div>
+                            <div className="hidden w-12 shrink-0 text-right text-xs tabular-nums text-muted sm:block">
+                              {f.probe?.duration ? formatDuration(f.probe.duration) : ""}
+                            </div>
+                            <div className="hidden w-16 shrink-0 text-right text-xs tabular-nums text-muted sm:block">{formatBytes(f.size)}</div>
+                          </div>
+                        );
+                      })}
+                    </div>
+                  </div>
+                ))}
+              </div>
+              {shown.length === 0 && <div className="px-4 py-8 text-center text-sm text-muted">No duplicates match this filter.</div>}
+              {shown.length > show && (
+                <div className="border-t border-line px-4 py-3 text-center">
+                  <Button size="sm" onClick={() => setShow(show + SHOW)}>
+                    Show more ({shown.length - show} left)
+                  </Button>
+                </div>
+              )}
+            </>
+          )}
+        </Card>
+      )}
+
+      {player && (
+        <MiniPlayer pairId={pair.id} queue={player.queue} index={player.index} onIndex={(index) => setPlayer({ ...player, index })} onClose={() => setPlayer(null)} />
+      )}
+
+      {confirm && scan && (
+        <ConfirmRemove
+          pair={pair}
+          groups={groups}
+          files={chosen}
+          dry={data.data?.dry_run ?? true}
+          onClose={() => setConfirm(false)}
+          onDone={() => {
+            setConfirm(false);
+            setPlayer(null);
+            refresh();
+          }}
+        />
+      )}
+    </div>
+  );
+}
+
+function ConfirmRemove({
+  pair,
+  groups,
+  files,
+  dry,
+  onClose,
+  onDone,
+}: {
+  pair: Pair;
+  groups: DupeGroup[];
+  files: DupeFile[];
+  dry: boolean;
+  onClose: () => void;
+  onDone: () => void;
+}) {
+  const toast = useToast();
+  const chosen = new Set(files.map((f) => f.rel));
+  // Groups where the recommended copy is the one being removed, or whose lengths differ.
+  const overridden = groups.filter((g) => g.files.some((f) => f.keep && chosen.has(f.rel))).length;
+  const lengths = groups.filter((g) => g.files.some((f) => chosen.has(f.rel)) && (g.notes ?? []).some((n) => n.startsWith("Lengths"))).length;
+  const bytes = files.filter((f) => !f.linked).reduce((n, f) => n + f.size, 0);
+  const remove = useMutation({
+    mutationFn: () => api.post(`/pairs/${pair.id}/duplicates/remove`, { files: files.map((f) => f.rel) }),
+    onSuccess: onDone,
+    onError: (e) => toast("err", e.message),
+  });
+  const n = files.length;
+  return (
+    <Modal open onClose={onClose} title={`Remove ${n} duplicate ${n === 1 ? "copy" : "copies"}?`}>
+      <div className="space-y-4 text-sm">
+        <p className="text-gray-300">
+          These files are moved from the slave library into the quarantine. Each is checked right before it moves: it's skipped if it changed since
+          the search or if no other copy of the song is left. Emptied folders are removed. You can restore them from History
+          {pair.retention_days > 0 ? `; they are deleted for good after ${pair.retention_days} days` : "; they stay in the quarantine until you purge them"}
+          {bytes > 0 && <>, which frees up to {formatBytes(bytes)}</>}.
+        </p>
+        <div className="max-h-48 overflow-y-auto rounded-lg border border-line bg-bg/60 px-3 py-2 mono text-xs text-gray-300">
+          {files.map((f) => (
+            <div key={f.rel} className="truncate" title={f.rel}>
+              {f.rel}
+            </div>
+          ))}
+        </div>
+        {(overridden > 0 || lengths > 0) && (
+          <Alert tone="amber" icon={<AlertTriangle className="mt-0.5 size-4 shrink-0" />}>
+            {overridden > 0 && (
+              <div>
+                In {overridden} {overridden === 1 ? "song" : "songs"}, you're removing the copy recommended to keep.
+              </div>
+            )}
+            {lengths > 0 && (
+              <div>
+                {lengths} {lengths === 1 ? "song has" : "songs have"} copies of different lengths, which may be different edits.
+              </div>
+            )}
+          </Alert>
+        )}
+        <p className="text-xs text-muted">Playlists that point at the removed files will need updating.</p>
+        {dry && <Alert tone="blue">Dry run is on: the removals are only recorded in Activity, and no file moves.</Alert>}
+        <div className="flex justify-end gap-2">
+          <Button variant="ghost" onClick={onClose}>
+            Cancel
+          </Button>
+          <Button variant="danger" loading={remove.isPending} onClick={() => remove.mutate()}>
+            <CopyMinus className="size-4" /> Remove {n} {n === 1 ? "copy" : "copies"}
+          </Button>
+        </div>
+      </div>
+    </Modal>
+  );
+}

web/src/pages/PairPage.tsx

@@ -40,6 +40,7 @@ function useJobFinished(pair: Pair | undefined) {
       qc.invalidateQueries({ queryKey: ["ops"] });
       qc.invalidateQueries({ queryKey: ["pairs"] });
       qc.invalidateQueries({ queryKey: ["empty-folders"] });
+      qc.invalidateQueries({ queryKey: ["duplicates"] });
       const j = pair.job!;
       if (st === "done") toast("ok", `${j.kind} finished${j.message ? `: ${j.message}` : ""}`);
       else toast("err", `${j.kind} ${st}: ${j.error}`);

web/src/pages/Player.tsx

@@ -2,11 +2,11 @@ import { useEffect, useRef, useState } from "react";
 import { CircleAlert, Disc3, Pause, Play, SkipBack, SkipForward, Volume1, Volume2, VolumeX, X } from "lucide-react";
 import { cx } from "../ui";
 
-/** A song the player can play: its path in the master library and a display name. */
-export type Track = { rel: string; name: string };
+/** A song the player can play: its path in a library (the master unless side says otherwise) and a display name. */
+export type Track = { rel: string; name: string; side?: "master" | "slave" };
 
-export function listenURL(pairId: number, rel: string) {
-  return `/api/pairs/${pairId}/listen?side=master&path=${encodeURIComponent(rel)}`;
+export function listenURL(pairId: number, rel: string, side: "master" | "slave" = "master") {
+  return `/api/pairs/${pairId}/listen?side=${side}&path=${encodeURIComponent(rel)}`;
 }
 
 function clock(sec: number) {
@@ -59,9 +59,9 @@ export function MiniPlayer({
     setError(null);
     setTime(0);
     setDuration(0);
-    a.src = listenURL(pairId, track.rel);
+    a.src = listenURL(pairId, track.rel, track.side);
     a.play().catch(() => {}); // a refused autoplay just leaves it paused
-  }, [pairId, track.rel]);
+  }, [pairId, track.rel, track.side]);
 
   useEffect(() => {
     if (audio.current) audio.current.volume = volume;
@@ -126,7 +126,7 @@ export function MiniPlayer({
                 </span>
               ) : (
                 <>
-                  {folder || "master"} · {index + 1} of {queue.length}
+                  {folder || track.side || "master"} · {index + 1} of {queue.length}
                 </>
               )}
             </div>

web/src/pages/Review.tsx

@@ -1,10 +1,11 @@
 import { useEffect, useState } from "react";
 import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { AlertTriangle, ArrowRight, ArrowUpCircle, Disc3, Download, ListMusic, ChevronDown, ChevronRight, CopyCheck, EyeOff, Eye, Link2, Search, Crown } from "lucide-react";
+import { AlertTriangle, ArrowRight, ArrowUpCircle, Disc3, Download, ListMusic, ChevronDown, ChevronRight, CopyCheck, CopyMinus, EyeOff, Eye, Link2, Search, Crown } from "lucide-react";
 import { ApiError, api, type Match, type Page, type Pair, type ProbeInfo } from "../api";
 import { Alert, Badge, Button, Card, Empty, Modal, Spinner, cx, formatBytes, formatDuration, inputCls, statusTone } from "../ui";
 import { useToast } from "../toast";
 import { AlbumView, BulkBar, TriCheckbox, toggleIDs, type Selection } from "./Bulk";
+import { Duplicates } from "./Duplicates";
 
 const filters = [
   { id: "review", label: "Needs review" },
@@ -72,6 +73,7 @@ export function Review({ pair }: { pair: Pair }) {
     queryFn: () =>
       api.get<Page<Match>>(`/pairs/${pair.id}/matches?status=${status}&q=${encodeURIComponent(debounced)}&limit=${PAGE}&offset=${page * PAGE}`),
     placeholderData: keepPreviousData,
+    enabled: status !== "duplicates",
   });
 
   const bulk = useMutation({
@@ -105,6 +107,19 @@ export function Review({ pair }: { pair: Pair }) {
             {count(f.id) !== undefined && <span className="tabular-nums opacity-70">{count(f.id)}</span>}
           </button>
         ))}
+        {/* Not a match status: duplicates within the slave library. */}
+        <span className="mx-1 h-4 w-px bg-line" aria-hidden />
+        <button
+          onClick={() => setStatus("duplicates")}
+          title="Songs that are in the slave library more than once, such as a single and its album version"
+          className={cx(
+            "flex items-center gap-1.5 rounded-full border px-3 py-1 text-xs cursor-pointer",
+            status === "duplicates" ? "border-accent bg-accent/15 text-white" : "border-line text-muted hover:text-white",
+          )}
+        >
+          <CopyMinus className="size-3.5" /> Duplicates
+        </button>
+        {status !== "duplicates" && (
         <div className="ml-auto flex w-full items-center gap-2 sm:w-auto">
           <div className="relative flex-1 sm:w-64">
             <Search className="absolute left-2.5 top-2.5 size-4 text-muted" />
@@ -126,8 +141,14 @@ export function Review({ pair }: { pair: Pair }) {
           </div>
           <ExportMenu pairId={pair.id} status={status} query={debounced} viewCount={list.data?.total} filterLabel={filters.find((f) => f.id === status)?.label ?? "All"} />
         </div>
+        )}
       </div>
 
+      {status === "duplicates" ? (
+        <Duplicates pair={pair} />
+      ) : (
+      <>
+
       {status === "identical" && identical > 0 && (
         <Card className="flex flex-wrap items-center justify-between gap-3 p-4">
           <div className="text-sm">
@@ -191,6 +212,8 @@ export function Review({ pair }: { pair: Pair }) {
       )}
 
       <BulkBar pair={pair} status={status} sel={sel} clear={() => setSel(new Set())} />
+      </>
+      )}
 
       <Modal open={confirmBulk} onClose={() => setConfirmBulk(false)} title="Link all identical files?">
         <p className="text-sm text-gray-300">