Import lists from CSV or text and match each entry
The list editor gets an Import button. A .csv/.tsv/.txt file or pasted text is parsed on the server: header columns (Title, Year, Artist or Director, Note, Rank), numbered lines and trailing "(1999)" years are understood, and Canon's own export round-trips. Each row is then searched with the list's provider and scored. A row is matched automatically only when one result fits exactly and nothing else fits as well; a year or artist in the file breaks ties, an exact title beats a bracketed version like "(Live)", and "Artist - Title" lines are tried both ways. Custom lists need a year or subtitle to be certain. Everything else is shown for review with a recommended pick, the other results, "Add as written", Skip, and a search-again box. Entries are added in file order after the existing ones, with any notes from the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aadacaff95
commit
dadf835be3
10 files changed
+1131
No files matched your search
@@ -129,6 +129,15 @@ A list can hold 1 to 65,535 entries. The editor recommends **100 or fewer** and
|
|||||||
|
|
||||||
To share with friends and family without being fully public, set a password under **Settings → Privacy → Site password**. Visitors then see a password prompt over a blurred placeholder; none of your lists, images or profile are sent until they enter it. Once they do, that browser stays unlocked for 90 days. Changing the password asks everyone for the new one, and you never need it while signed in. A protected site also tells search engines not to index it.
|
To share with friends and family without being fully public, set a password under **Settings → Privacy → Site password**. Visitors then see a password prompt over a blurred placeholder; none of your lists, images or profile are sent until they enter it. Once they do, that browser stays unlocked for 90 days. Changing the password asks everyone for the new one, and you never need it while signed in. A protected site also tells search engines not to index it.
|
||||||
|
|
||||||
|
## Importing a list
|
||||||
|
|
||||||
|
**Import** in the list editor adds entries from a file, after the ones already there and in the file's order:
|
||||||
|
|
||||||
|
- **Text** (`.txt`, or paste): one title per line. Numbering like `1.` is dropped when every line has it, and a trailing `(1999)` is read as the year. For songs and albums, `Artist - Title` works either way round.
|
||||||
|
- **CSV / TSV**: a header row picks the columns: `Title` (or Name), `Year`, `Artist`/`Director`/`Developer`, `Note` (or Why), and `Rank` to set the order. Without a header the first column is the title. Canon's own export imports as-is.
|
||||||
|
|
||||||
|
Each entry is searched for and matched. Canon only adds a match on its own when it's certain: the title matches exactly and nothing else fits as well (a year or artist in the file helps). Everything else is left for you to check, with a recommended pick, the other results, **Add as written** to make a new entry from your text, **Skip**, and a box to search again. Custom lists search all of Wikipedia, so there a match is only certain when a year or subtitle confirms it.
|
||||||
|
|
||||||
## Exporting a list
|
## Exporting a list
|
||||||
|
|
||||||
Every list can be downloaded as its titles only, in rank order, one per line. There are no ranks, header row or other fields.
|
Every list can be downloaded as its titles only, in rank order, one per line. There are no ranks, header row or other fields.
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/cdnutter/canon/internal/providers"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseText(t *testing.T) {
|
||||||
|
got, err := Parse([]byte("1. Heat (1995)\n2) 2001: A Space Odyssey\n\n3 - Alien [1979]\n"), "list.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []Row{{Title: "Heat", Year: "1995"}, {Title: "2001: A Space Odyssey"}, {Title: "Alien", Year: "1979"}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
// Not every line is numbered, so nothing is stripped.
|
||||||
|
got, _ = Parse([]byte("2001: A Space Odyssey\nHeat\n1917\n"), "x.txt")
|
||||||
|
if got[0].Title != "2001: A Space Odyssey" || got[2].Title != "1917" {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCSV(t *testing.T) {
|
||||||
|
csv := "\xef\xbb\xbfRank,Title,Year,Director,Why\n2,Alien,1979,Ridley Scott,\"Tense, perfect\"\n1,Heat,1995,Michael Mann,\n"
|
||||||
|
got, err := Parse([]byte(csv), "films.csv")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []Row{
|
||||||
|
{Title: "Heat", Year: "1995", Subtitle: "Michael Mann"},
|
||||||
|
{Title: "Alien", Year: "1979", Subtitle: "Ridley Scott", Note: "Tense, perfect"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
// Canon's own export: one quoted column, no header.
|
||||||
|
got, _ = Parse([]byte("\xef\xbb\xbf\"Heat\"\r\n\"Crouching Tiger, Hidden Dragon\"\r\n"), "films.csv")
|
||||||
|
if len(got) != 2 || got[1].Title != "Crouching Tiger, Hidden Dragon" {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
// Headerless rank,title,year.
|
||||||
|
got, _ = Parse([]byte("1,Heat,1995\n2,Alien,1979\n"), "x.csv")
|
||||||
|
if got[1] != (Row{Title: "Alien", Year: "1979"}) {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
if _, err := Parse([]byte("\n \n"), "x.txt"); err != ErrEmpty {
|
||||||
|
t.Fatalf("empty file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func res(title, year, sub string) providers.Result {
|
||||||
|
return providers.Result{Provider: "t", ProviderID: title + year + sub, Title: title, Year: year, Subtitle: sub}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRank(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
row Row
|
||||||
|
category string
|
||||||
|
results []providers.Result
|
||||||
|
status string
|
||||||
|
top string
|
||||||
|
}{
|
||||||
|
{"exact and alone", Row{Title: "Alien"}, "movie", []providers.Result{res("Alien", "1979", ""), res("Aliens", "1986", "")}, Matched, "Alien1979"},
|
||||||
|
{"remake, no year", Row{Title: "Heat"}, "movie", []providers.Result{res("Heat", "1995", ""), res("Heat", "1986", "")}, Review, "Heat1995"},
|
||||||
|
{"remake, year given", Row{Title: "Heat", Year: "1986"}, "movie", []providers.Result{res("Heat", "1995", ""), res("Heat", "1986", "")}, Matched, "Heat1986"},
|
||||||
|
{"loose title", Row{Title: "Star Wars"}, "movie", []providers.Result{res("Star Wars: A New Hope", "1977", "")}, Review, "Star Wars: A New Hope1977"},
|
||||||
|
{"punctuation and accents", Row{Title: "amelie"}, "movie", []providers.Result{res("Amélie", "2001", "")}, Matched, "Amélie2001"},
|
||||||
|
{"wikipedia qualifier", Row{Title: "The Matrix", Year: "1999"}, "custom", []providers.Result{res("The Matrix (1999 film)", "1999", "")}, Matched, "The Matrix (1999 film)1999"},
|
||||||
|
{"artist - title", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Creep", "1992", "TLC"), res("Creep", "1992", "Radiohead")}, Matched, "Creep1992Radiohead"},
|
||||||
|
{"title, no artist", Row{Title: "Creep"}, "song", []providers.Result{res("Creep", "1992", "TLC"), res("Creep", "1992", "Radiohead")}, Review, "Creep1992TLC"},
|
||||||
|
{"same album twice", Row{Title: "OK Computer"}, "album", []providers.Result{res("OK Computer", "1997", "Radiohead"), res("OK Computer", "1997", "Radiohead")}, Matched, "OK Computer1997Radiohead"},
|
||||||
|
{"wrong year", Row{Title: "Alien", Year: "1990"}, "movie", []providers.Result{res("Alien", "1979", "")}, Review, "Alien1979"},
|
||||||
|
{"exact beats a bracketed version", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Creep (Acoustic)", "", "Radiohead"), res("Creep", "", "Radiohead")}, Matched, "CreepRadiohead"},
|
||||||
|
{"only bracketed versions", Row{Title: "Hey Jude"}, "song", []providers.Result{res("Hey Jude (Remastered 2015)", "", "The Beatles")}, Matched, "Hey Jude (Remastered 2015)The Beatles"},
|
||||||
|
{"year beats no year", Row{Title: "The Matrix", Year: "1999"}, "custom", []providers.Result{res("The Matrix", "", "American media franchise"), res("The Matrix", "1999", "Film by the Wachowskis")}, Matched, "The Matrix1999Film by the Wachowskis"},
|
||||||
|
{"artist confirmed beats a literal title", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Radiohead - Creep", "", "Klangsberg"), res("Creep", "", "Radiohead")}, Matched, "CreepRadiohead"},
|
||||||
|
{"custom needs confirming", Row{Title: "Heat"}, "custom", []providers.Result{res("Heat", "", "Type of energy transfer")}, Review, "HeatType of energy transfer"},
|
||||||
|
{"nothing", Row{Title: "zzz"}, "movie", nil, None, ""},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
status, scored := Rank(c.row, c.category, c.results)
|
||||||
|
if status != c.status {
|
||||||
|
t.Errorf("%s: status %s, want %s", c.name, status, c.status)
|
||||||
|
}
|
||||||
|
if len(scored) > 0 && scored[0].ProviderID != c.top {
|
||||||
|
t.Errorf("%s: top %s, want %s", c.name, scored[0].ProviderID, c.top)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"github.com/cdnutter/canon/internal/providers"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Match statuses.
|
||||||
|
const (
|
||||||
|
Matched = "matched" // exactly one result fits: add it without asking
|
||||||
|
Review = "review" // there are results, but the owner should pick
|
||||||
|
None = "none" // nothing found
|
||||||
|
)
|
||||||
|
|
||||||
|
// Scored is a search result with how well it fits the row.
|
||||||
|
type Scored struct {
|
||||||
|
providers.Result
|
||||||
|
Score int `json:"score"`
|
||||||
|
// Tier is how the title fits, given it agrees with any year or artist
|
||||||
|
// given: 2 exactly, 1 once bracketed qualifiers like "(Live)" are
|
||||||
|
// ignored, 0 not at all.
|
||||||
|
Tier int `json:"-"`
|
||||||
|
// Confirms counts what the row gave besides the title (year, artist)
|
||||||
|
// that this result agrees with.
|
||||||
|
Confirms int `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fit is how one reading of a row fits one result.
|
||||||
|
type fit struct{ score, tier, confirms int }
|
||||||
|
|
||||||
|
func (f fit) better(g fit) bool {
|
||||||
|
if f.tier != g.tier {
|
||||||
|
return f.tier > g.tier
|
||||||
|
}
|
||||||
|
if f.confirms != g.confirms {
|
||||||
|
return f.confirms > g.confirms
|
||||||
|
}
|
||||||
|
return f.score > g.score
|
||||||
|
}
|
||||||
|
|
||||||
|
// music categories may write "Artist - Title" on one line.
|
||||||
|
func isMusic(category string) bool { return category == "album" || category == "song" }
|
||||||
|
|
||||||
|
// Query is what to search the provider for.
|
||||||
|
func Query(r Row, category string) string {
|
||||||
|
q := r.Title
|
||||||
|
if isMusic(category) {
|
||||||
|
q = dashes.Replace(q)
|
||||||
|
if r.Subtitle != "" {
|
||||||
|
q += " " + r.Subtitle
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
var dashes = strings.NewReplacer(" - ", " ", " – ", " ", " — ", " ")
|
||||||
|
|
||||||
|
// readings are the ways a row can be understood: as written, and for music
|
||||||
|
// "A - B" as either title/artist or artist/title.
|
||||||
|
type reading struct{ title, year, artist string }
|
||||||
|
|
||||||
|
func readings(r Row, category string) []reading {
|
||||||
|
out := []reading{{r.Title, r.Year, r.Subtitle}}
|
||||||
|
if isMusic(category) && r.Subtitle == "" {
|
||||||
|
for _, sep := range []string{" - ", " – ", " — "} {
|
||||||
|
if a, b, ok := strings.Cut(r.Title, sep); ok {
|
||||||
|
out = append(out, reading{b, r.Year, a}, reading{a, r.Year, b})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank scores results against the row, best first, and decides whether the
|
||||||
|
// best one is certain.
|
||||||
|
func Rank(r Row, category string, results []providers.Result) (string, []Scored) {
|
||||||
|
results = slices.DeleteFunc(slices.Clone(results), isDisambiguation)
|
||||||
|
if len(results) == 0 {
|
||||||
|
return None, nil
|
||||||
|
}
|
||||||
|
reads := readings(r, category)
|
||||||
|
scored := make([]Scored, len(results))
|
||||||
|
for i, res := range results {
|
||||||
|
best := fit{score: -1 << 30}
|
||||||
|
for _, rd := range reads {
|
||||||
|
if f := score(rd, res); f.better(best) {
|
||||||
|
best = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The provider's own relevance order breaks ties.
|
||||||
|
scored[i] = Scored{Result: res, Score: best.score - 2*i, Tier: best.tier, Confirms: best.confirms}
|
||||||
|
}
|
||||||
|
slices.SortStableFunc(scored, func(a, b Scored) int {
|
||||||
|
if a.Tier != b.Tier {
|
||||||
|
return b.Tier - a.Tier
|
||||||
|
}
|
||||||
|
if a.Confirms != b.Confirms {
|
||||||
|
return b.Confirms - a.Confirms
|
||||||
|
}
|
||||||
|
return b.Score - a.Score
|
||||||
|
})
|
||||||
|
top := scored[0]
|
||||||
|
if top.Tier == 0 {
|
||||||
|
return Review, scored
|
||||||
|
}
|
||||||
|
// Custom lists search all of Wikipedia, so a bare title ("Heat") could
|
||||||
|
// be anything; only a year or subtitle that agrees makes it certain.
|
||||||
|
if category == "custom" && top.Confirms == 0 {
|
||||||
|
return Review, scored
|
||||||
|
}
|
||||||
|
// Two different things that fit equally well (Heat 1995 and Heat 1986,
|
||||||
|
// with no year given) need the owner to choose. A result that agrees
|
||||||
|
// with more of what the row says (its year, its artist) wins outright.
|
||||||
|
for _, other := range scored[1:] {
|
||||||
|
if other.Tier < top.Tier || other.Confirms < top.Confirms {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !sameEntry(top.Result, other.Result) {
|
||||||
|
return Review, scored
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Matched, scored
|
||||||
|
}
|
||||||
|
|
||||||
|
// isDisambiguation drops Wikipedia's "may refer to" pages, which are never
|
||||||
|
// the thing itself.
|
||||||
|
func isDisambiguation(r providers.Result) bool {
|
||||||
|
return r.Provider == "wikipedia" && strings.Contains(strings.ToLower(r.Subtitle), "same term")
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameEntry treats one work listed twice (Deezer and MusicBrainz) as one.
|
||||||
|
func sameEntry(a, b providers.Result) bool {
|
||||||
|
return normTitle(a.Title) == normTitle(b.Title) && a.Year == b.Year && normWords(a.Subtitle) == normWords(b.Subtitle)
|
||||||
|
}
|
||||||
|
|
||||||
|
func score(rd reading, res providers.Result) fit {
|
||||||
|
want, got := normTitle(rd.title), normTitle(res.Title)
|
||||||
|
if want == "" || got == "" {
|
||||||
|
return fit{}
|
||||||
|
}
|
||||||
|
f := fit{score: int(60 * similarity(want, got))}
|
||||||
|
switch {
|
||||||
|
case normFull(rd.title) == normFull(res.Title):
|
||||||
|
f.score, f.tier = 110, 2
|
||||||
|
case want == got:
|
||||||
|
f.score, f.tier = 100, 1
|
||||||
|
}
|
||||||
|
if rd.year != "" {
|
||||||
|
switch d := yearDiff(rd.year, res.Year); {
|
||||||
|
case d == 0:
|
||||||
|
f.score += 25
|
||||||
|
f.confirms++
|
||||||
|
case d == 1:
|
||||||
|
f.score += 8
|
||||||
|
f.tier = 0
|
||||||
|
case d > 1:
|
||||||
|
f.score -= 20
|
||||||
|
f.tier = 0
|
||||||
|
}
|
||||||
|
// A result without a year (common on Wikipedia) neither helps nor
|
||||||
|
// rules it out.
|
||||||
|
}
|
||||||
|
if rd.artist != "" {
|
||||||
|
if sharesName(rd.artist, res.Subtitle) {
|
||||||
|
f.score += 25
|
||||||
|
f.confirms++
|
||||||
|
} else {
|
||||||
|
f.score -= 10
|
||||||
|
f.tier = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func yearDiff(a, b string) int {
|
||||||
|
x, err1 := strconv.Atoi(a)
|
||||||
|
y, err2 := strconv.Atoi(b)
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
if x > y {
|
||||||
|
return x - y
|
||||||
|
}
|
||||||
|
return y - x
|
||||||
|
}
|
||||||
|
|
||||||
|
// sharesName is true when one name contains the other as whole words:
|
||||||
|
// "Radiohead" in "Radiohead", "Nolan" in "Christopher Nolan".
|
||||||
|
func sharesName(a, b string) bool {
|
||||||
|
a, b = " "+normWords(a)+" ", " "+normWords(b)+" "
|
||||||
|
if strings.TrimSpace(a) == "" || strings.TrimSpace(b) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.Contains(a, b) || strings.Contains(b, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// similarity is the share of words two titles have in common (Dice).
|
||||||
|
func similarity(a, b string) float64 {
|
||||||
|
wa, wb := strings.Fields(a), strings.Fields(b)
|
||||||
|
if len(wa) == 0 || len(wb) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
count := map[string]int{}
|
||||||
|
for _, w := range wa {
|
||||||
|
count[w]++
|
||||||
|
}
|
||||||
|
common := 0
|
||||||
|
for _, w := range wb {
|
||||||
|
if count[w] > 0 {
|
||||||
|
count[w]--
|
||||||
|
common++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 2 * float64(common) / float64(len(wa)+len(wb))
|
||||||
|
}
|
||||||
|
|
||||||
|
// normTitle compares titles loosely: case, accents, punctuation, "&" vs
|
||||||
|
// "and", a leading "The", and bracketed qualifiers like "(1995 film)" or
|
||||||
|
// "(Remastered)" don't matter.
|
||||||
|
func normTitle(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
depth := 0
|
||||||
|
for _, r := range s {
|
||||||
|
switch r {
|
||||||
|
case '(', '[', '{':
|
||||||
|
depth++
|
||||||
|
case ')', ']', '}':
|
||||||
|
if depth > 0 {
|
||||||
|
depth--
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if depth == 0 {
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normFull(b.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// normFull is normTitle keeping bracketed parts: "Creep (Acoustic)" is not
|
||||||
|
// "Creep".
|
||||||
|
func normFull(s string) string {
|
||||||
|
w := normWords(s)
|
||||||
|
for _, article := range []string{"the ", "a ", "an "} {
|
||||||
|
if rest, ok := strings.CutPrefix(w, article); ok && rest != "" {
|
||||||
|
return rest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func normWords(s string) string {
|
||||||
|
s = strings.ReplaceAll(strings.ToLower(s), "&", " and ")
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range s {
|
||||||
|
if f, ok := fold[r]; ok {
|
||||||
|
b.WriteString(f)
|
||||||
|
} else if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||||
|
b.WriteRune(r)
|
||||||
|
} else if r == '\'' || r == '’' || r == '.' {
|
||||||
|
// "Don't" = "Dont", "S.O.S" = "SOS"
|
||||||
|
} else {
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(strings.Fields(b.String()), " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
var fold = map[rune]string{
|
||||||
|
'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae",
|
||||||
|
'ç': "c", 'è': "e", 'é': "e", 'ê': "e", 'ë': "e",
|
||||||
|
'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ñ': "n",
|
||||||
|
'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'œ': "oe",
|
||||||
|
'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'ÿ': "y", 'ß': "ss",
|
||||||
|
}
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
// Package importer reads a list of titles from a CSV or text file and
|
||||||
|
// matches each one to a search result.
|
||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/csv"
|
||||||
|
"errors"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MaxRows caps one import.
|
||||||
|
const MaxRows = 2000
|
||||||
|
|
||||||
|
// Row is one entry read from the file, in the file's order.
|
||||||
|
type Row struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
Year string `json:"year,omitempty"`
|
||||||
|
Subtitle string `json:"subtitle,omitempty"` // artist, director, developer…
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var ErrEmpty = errors.New("no titles found in that file")
|
||||||
|
|
||||||
|
// Parse reads rows from data. CSV and TSV files may have a header row
|
||||||
|
// naming their columns; text files hold one title per line.
|
||||||
|
func Parse(data []byte, filename string) ([]Row, error) {
|
||||||
|
data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
|
||||||
|
if !utf8.Valid(data) {
|
||||||
|
data = []byte(strings.ToValidUTF8(latin1(data), "�"))
|
||||||
|
}
|
||||||
|
text := strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||||
|
var rows []Row
|
||||||
|
var err error
|
||||||
|
switch ext := strings.ToLower(filename); {
|
||||||
|
case strings.HasSuffix(ext, ".csv"):
|
||||||
|
rows, err = parseDelimited(text, sniffDelimiter(text))
|
||||||
|
case strings.HasSuffix(ext, ".tsv"):
|
||||||
|
rows, err = parseDelimited(text, '\t')
|
||||||
|
default:
|
||||||
|
rows = parseLines(text)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return nil, ErrEmpty
|
||||||
|
}
|
||||||
|
if len(rows) > MaxRows {
|
||||||
|
rows = rows[:MaxRows]
|
||||||
|
}
|
||||||
|
return rows, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// latin1 decodes bytes as Windows-1252/Latin-1, which is what spreadsheet
|
||||||
|
// exports that aren't UTF-8 usually are.
|
||||||
|
func latin1(b []byte) string {
|
||||||
|
r := make([]rune, len(b))
|
||||||
|
for i, c := range b {
|
||||||
|
r[i] = rune(c)
|
||||||
|
}
|
||||||
|
return string(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sniffDelimiter(text string) rune {
|
||||||
|
first, _, _ := strings.Cut(text, "\n")
|
||||||
|
best, n := ',', strings.Count(first, ",")
|
||||||
|
for _, d := range []rune{'\t', ';'} {
|
||||||
|
if c := strings.Count(first, string(d)); c > n {
|
||||||
|
best, n = d, c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
|
// numbered matches a leading rank: "1. ", "#2 ", "3) ", "4 - ".
|
||||||
|
var numbered = regexp.MustCompile(`^\s*#?\d{1,5}\s*[.):\-–—]?\s+`)
|
||||||
|
|
||||||
|
// trailingYear matches "Heat (1995)" or "Heat [1995]".
|
||||||
|
var trailingYear = regexp.MustCompile(`\s*[(\[]((?:1[5-9]|20)\d\d)[)\]]\s*$`)
|
||||||
|
|
||||||
|
func parseLines(text string) []Row {
|
||||||
|
var lines []string
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
|
lines = append(lines, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Only strip numbering when every line has it, so a list that merely
|
||||||
|
// contains "2001: A Space Odyssey" keeps its title.
|
||||||
|
strip := len(lines) > 0
|
||||||
|
for _, l := range lines {
|
||||||
|
if !numbered.MatchString(l) {
|
||||||
|
strip = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows := make([]Row, 0, len(lines))
|
||||||
|
for _, l := range lines {
|
||||||
|
if strip {
|
||||||
|
l = numbered.ReplaceAllString(l, "")
|
||||||
|
}
|
||||||
|
if r := rowFromTitle(l); r.Title != "" {
|
||||||
|
rows = append(rows, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
func rowFromTitle(s string) Row {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
r := Row{Title: s}
|
||||||
|
if m := trailingYear.FindStringSubmatchIndex(s); m != nil {
|
||||||
|
r.Title = strings.TrimSpace(s[:m[0]])
|
||||||
|
r.Year = s[m[2]:m[3]]
|
||||||
|
}
|
||||||
|
if r.Title == "" {
|
||||||
|
r.Title = s
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
var headerNames = map[string]string{
|
||||||
|
"title": "title", "name": "title", "film": "title", "movie": "title", "show": "title",
|
||||||
|
"album": "title", "song": "title", "track": "title", "game": "title", "item": "title",
|
||||||
|
"year": "year", "release year": "year", "released": "year", "date": "year",
|
||||||
|
"artist": "subtitle", "director": "subtitle", "creator": "subtitle", "developer": "subtitle",
|
||||||
|
"author": "subtitle", "by": "subtitle", "subtitle": "subtitle", "network": "subtitle",
|
||||||
|
"note": "note", "notes": "note", "why": "note", "comment": "note", "comments": "note", "review": "note",
|
||||||
|
"rank": "rank", "#": "rank", "position": "rank", "no": "rank", "no.": "rank",
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDelimited(text string, delim rune) ([]Row, error) {
|
||||||
|
cr := csv.NewReader(strings.NewReader(text))
|
||||||
|
cr.Comma = delim
|
||||||
|
cr.FieldsPerRecord = -1
|
||||||
|
cr.LazyQuotes = true
|
||||||
|
cr.TrimLeadingSpace = true
|
||||||
|
records, err := cr.ReadAll()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("that CSV file couldn't be read: " + err.Error())
|
||||||
|
}
|
||||||
|
records = slices.DeleteFunc(records, func(rec []string) bool {
|
||||||
|
return strings.TrimSpace(strings.Join(rec, "")) == ""
|
||||||
|
})
|
||||||
|
if len(records) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cols := map[string]int{}
|
||||||
|
if head := records[0]; isHeader(head) {
|
||||||
|
for i, name := range head {
|
||||||
|
if role, ok := headerNames[strings.ToLower(strings.TrimSpace(name))]; ok {
|
||||||
|
if _, taken := cols[role]; !taken {
|
||||||
|
cols[role] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
records = records[1:]
|
||||||
|
} else {
|
||||||
|
cols = guessColumns(records)
|
||||||
|
}
|
||||||
|
if _, ok := cols["title"]; !ok {
|
||||||
|
cols["title"] = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
type ranked struct {
|
||||||
|
Row
|
||||||
|
rank int
|
||||||
|
}
|
||||||
|
var out []ranked
|
||||||
|
cell := func(rec []string, role string) string {
|
||||||
|
i, ok := cols[role]
|
||||||
|
if !ok || i >= len(rec) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(rec[i])
|
||||||
|
}
|
||||||
|
for i, rec := range records {
|
||||||
|
r := Row{Title: cell(rec, "title"), Subtitle: cell(rec, "subtitle"), Note: cell(rec, "note")}
|
||||||
|
if y := yearIn(cell(rec, "year")); y != "" {
|
||||||
|
r.Year = y
|
||||||
|
} else {
|
||||||
|
t := rowFromTitle(r.Title)
|
||||||
|
r.Title, r.Year = t.Title, t.Year
|
||||||
|
}
|
||||||
|
if r.Title == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rank, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#"))
|
||||||
|
if err != nil {
|
||||||
|
rank = 1 << 30
|
||||||
|
}
|
||||||
|
out = append(out, ranked{r, rank*MaxRows*2 + i}) // stable within equal ranks
|
||||||
|
}
|
||||||
|
if _, ok := cols["rank"]; ok {
|
||||||
|
slices.SortStableFunc(out, func(a, b ranked) int { return a.rank - b.rank })
|
||||||
|
}
|
||||||
|
rows := make([]Row, len(out))
|
||||||
|
for i, r := range out {
|
||||||
|
rows[i] = r.Row
|
||||||
|
}
|
||||||
|
return rows, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isHeader(rec []string) bool {
|
||||||
|
for _, c := range rec {
|
||||||
|
if headerNames[strings.ToLower(strings.TrimSpace(c))] == "title" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var yearOnly = regexp.MustCompile(`\b((?:1[5-9]|20)\d\d)\b`)
|
||||||
|
|
||||||
|
func yearIn(s string) string { return yearOnly.FindString(s) }
|
||||||
|
|
||||||
|
// guessColumns handles headerless files: "1,Heat,1995" or "Heat,1995".
|
||||||
|
func guessColumns(records [][]string) map[string]int {
|
||||||
|
cols := map[string]int{}
|
||||||
|
all := func(col int, ok func(string) bool) bool {
|
||||||
|
for _, rec := range records {
|
||||||
|
if col >= len(rec) || !ok(strings.TrimSpace(rec[col])) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
isInt := func(s string) bool { _, err := strconv.Atoi(strings.TrimPrefix(s, "#")); return err == nil }
|
||||||
|
isYear := func(s string) bool { return s == yearIn(s) && s != "" }
|
||||||
|
title := 0
|
||||||
|
width := len(records[0])
|
||||||
|
if width >= 2 && all(0, isInt) && !all(1, isInt) {
|
||||||
|
cols["rank"] = 0
|
||||||
|
title = 1
|
||||||
|
}
|
||||||
|
cols["title"] = title
|
||||||
|
if title+1 < width && all(title+1, isYear) {
|
||||||
|
cols["year"] = title + 1
|
||||||
|
}
|
||||||
|
return cols
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/cdnutter/canon/internal/importer"
|
||||||
|
"github.com/cdnutter/canon/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxImportBytes = 2 << 20
|
||||||
|
|
||||||
|
// apiImportParse reads an uploaded CSV/TXT file, or pasted text, into rows.
|
||||||
|
func (s *Server) apiImportParse(w http.ResponseWriter, r *http.Request) {
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, maxImportBytes+64<<10)
|
||||||
|
var data []byte
|
||||||
|
name := "pasted.txt"
|
||||||
|
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/") {
|
||||||
|
f, hdr, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
s.writeErr(w, store.ValidationError{Msg: "Choose a .csv or .txt file (up to 2 MB)."})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if data, err = io.ReadAll(f); err != nil {
|
||||||
|
s.writeErr(w, store.ValidationError{Msg: "That file is too big (2 MB at most)."})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name = hdr.Filename
|
||||||
|
} else {
|
||||||
|
var in struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &in); err != nil {
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data = []byte(in.Text)
|
||||||
|
}
|
||||||
|
rows, err := importer.Parse(data, name)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, importer.ErrEmpty) {
|
||||||
|
err = store.ValidationError{Msg: "No titles found. Use one title per line, or a CSV with a Title column."}
|
||||||
|
} else {
|
||||||
|
err = store.ValidationError{Msg: err.Error()}
|
||||||
|
}
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]any{"rows": rows, "max": importer.MaxRows})
|
||||||
|
}
|
||||||
|
|
||||||
|
type importMatch struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Results []searchResult `json:"results"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// apiImportMatch searches for one imported row and says how sure the best
|
||||||
|
// result is.
|
||||||
|
func (s *Server) apiImportMatch(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
listID, err := pathID(r)
|
||||||
|
if err != nil {
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var row importer.Row
|
||||||
|
if err := decodeJSON(r, &row); err != nil {
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
row.Title = strings.TrimSpace(row.Title)
|
||||||
|
if row.Title == "" || utf8.RuneCountInString(row.Title) > 300 {
|
||||||
|
writeError(w, http.StatusBadRequest, "Each entry needs a title.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
list, err := s.store.ListByID(ctx, listID)
|
||||||
|
if err != nil {
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p, err := s.providers.For(ctx, list.Category)
|
||||||
|
if err != nil {
|
||||||
|
s.writeErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
results, err := p.Search(sctx, importer.Query(row, list.Category))
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("import search failed", "provider", p.Name(), "err", err)
|
||||||
|
writeError(w, http.StatusBadGateway, fmt.Sprintf("%s search failed: %v", p.Label(), err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
status, scored := importer.Rank(row, list.Category, results)
|
||||||
|
out := importMatch{Status: status, Results: []searchResult{}}
|
||||||
|
for _, sc := range scored[:min(len(scored), 6)] {
|
||||||
|
sr := searchResult{Result: sc.Result}
|
||||||
|
if sc.Provider == "deezer" && hasPreviews(list.Category) {
|
||||||
|
sr.PreviewRef = sc.ProviderID
|
||||||
|
}
|
||||||
|
if it, err := s.store.FindItemByProvider(ctx, listID, sc.Provider, sc.ProviderID); err == nil {
|
||||||
|
sr.ExistingRank = it.Rank
|
||||||
|
}
|
||||||
|
out.Results = append(out.Results, sr)
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, out)
|
||||||
|
}
|
||||||
@@ -146,6 +146,8 @@ func (s *Server) Handler() http.Handler {
|
|||||||
api("POST /api/items/{id}/refresh", s.apiRefreshItem)
|
api("POST /api/items/{id}/refresh", s.apiRefreshItem)
|
||||||
api("DELETE /api/items/{id}", s.apiDeleteItem)
|
api("DELETE /api/items/{id}", s.apiDeleteItem)
|
||||||
api("GET /api/search", s.apiSearch)
|
api("GET /api/search", s.apiSearch)
|
||||||
|
api("POST /api/import/parse", s.apiImportParse)
|
||||||
|
api("POST /api/lists/{id}/import/match", s.apiImportMatch)
|
||||||
api("GET /api/settings", s.apiSettings)
|
api("GET /api/settings", s.apiSettings)
|
||||||
api("PUT /api/settings", s.apiUpdateSettings)
|
api("PUT /api/settings", s.apiUpdateSettings)
|
||||||
api("POST /api/settings/avatar", s.apiUploadAvatar)
|
api("POST /api/settings/avatar", s.apiUploadAvatar)
|
||||||
|
|||||||
@@ -827,3 +827,44 @@ input[type="checkbox"] { accent-color: var(--accent); width: 18px; height: 18px;
|
|||||||
.backup-list .backup-when { flex: 1; min-width: 0; font-size: 14px; font-weight: 550; }
|
.backup-list .backup-when { flex: 1; min-width: 0; font-size: 14px; font-weight: 550; }
|
||||||
.backup-list .backup-meta { display: block; color: var(--muted); font-size: 12.5px; font-weight: 400; margin-top: 2px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
.backup-list .backup-meta { display: block; color: var(--muted); font-size: 12.5px; font-weight: 400; margin-top: 2px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
.backup-list .backup-actions { display: flex; gap: 2px; flex: none; }
|
.backup-list .backup-actions { display: flex; gap: 2px; flex: none; }
|
||||||
|
|
||||||
|
/* ---------- import ---------- */
|
||||||
|
.import-drop {
|
||||||
|
display: flex; align-items: center; gap: 14px; padding: 18px; cursor: pointer;
|
||||||
|
border: 1.5px dashed var(--border); border-radius: var(--radius); background: var(--surface);
|
||||||
|
transition: border-color 0.15s, background 0.15s;
|
||||||
|
}
|
||||||
|
.import-drop:hover, .import-drop.is-over { border-color: var(--accent); background: var(--accent-soft); }
|
||||||
|
.import-drop > .icon { width: 24px; height: 24px; color: var(--accent); flex: none; }
|
||||||
|
.import-drop-text { display: flex; flex-direction: column; gap: 2px; min-width: 0; }
|
||||||
|
.import-file { position: absolute; width: 1px; height: 1px; opacity: 0; pointer-events: none; }
|
||||||
|
.import-dialog { max-width: min(760px, calc(100vw - 32px)); width: 100%; }
|
||||||
|
.import-dialog-inner { display: flex; flex-direction: column; max-height: calc(100dvh - 48px); }
|
||||||
|
.import-body { display: flex; flex-direction: column; gap: 12px; overflow-y: auto; min-height: 0; }
|
||||||
|
.import-top { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; }
|
||||||
|
.import-summary { font-weight: 600; font-size: 14px; }
|
||||||
|
.import-list { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 8px; }
|
||||||
|
.import-row { border: 1px solid var(--border); border-radius: var(--radius); padding: 10px 12px; display: flex; flex-direction: column; gap: 10px; background: var(--surface); }
|
||||||
|
.import-row.is-review, .import-row.is-none, .import-row.is-error { border-color: color-mix(in srgb, var(--warn) 45%, var(--border)); }
|
||||||
|
.import-row-head { display: flex; align-items: center; gap: 10px; min-width: 0; }
|
||||||
|
.import-num { color: var(--faint); font-size: 13px; font-variant-numeric: tabular-nums; min-width: 2.5ch; text-align: right; }
|
||||||
|
.import-src { flex: 1; min-width: 0; font-size: 14px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.import-src b { font-weight: 600; }
|
||||||
|
.badge--ok { background: color-mix(in srgb, var(--success) 16%, transparent); color: var(--success); }
|
||||||
|
.badge .icon { width: 12px; height: 12px; }
|
||||||
|
.import-chosen { display: flex; align-items: center; gap: 10px; justify-content: space-between; padding-left: calc(2.5ch + 10px); }
|
||||||
|
.import-card { display: flex; align-items: center; gap: 10px; min-width: 0; }
|
||||||
|
.import-card .result-art { width: 38px; flex: none; }
|
||||||
|
.import-card-text { display: flex; flex-direction: column; gap: 2px; min-width: 0; }
|
||||||
|
.import-card .result-title { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
||||||
|
.import-opts { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 6px; }
|
||||||
|
.import-opt { display: flex; align-items: center; gap: 10px; padding: 6px 8px; border: 1px solid var(--border); border-radius: var(--radius); cursor: pointer; min-width: 0; }
|
||||||
|
.import-opt:has(input:checked) { border-color: var(--accent); background: var(--accent-soft); }
|
||||||
|
.import-opt input { accent-color: var(--accent); margin: 0; flex: none; }
|
||||||
|
.import-opt.is-new, .import-opt.is-skip { font-size: 13.5px; }
|
||||||
|
.import-new { display: flex; align-items: center; gap: 8px; min-width: 0; }
|
||||||
|
.import-new .icon { width: 16px; height: 16px; color: var(--accent); flex: none; }
|
||||||
|
.import-again { display: flex; align-items: center; gap: 8px; }
|
||||||
|
.import-again > .icon { width: 16px; height: 16px; color: var(--faint); flex: none; }
|
||||||
|
.import-row .import-again input[type="search"] { flex: 1; min-width: 0; height: 30px; font-size: 13px; }
|
||||||
|
.dialog-foot > .alert { margin: 0; display: flex; align-items: center; gap: 8px; flex-wrap: wrap; font-size: 13px; }
|
||||||
@@ -3,6 +3,7 @@ import { h, icon, api, readBoot, toast, confirmDialog, chooseDialog, openDialog,
|
|||||||
import { sortable } from "./sortable.js";
|
import { sortable } from "./sortable.js";
|
||||||
import { openListForm } from "./list-form.js";
|
import { openListForm } from "./list-form.js";
|
||||||
import { previewButton } from "./player.js";
|
import { previewButton } from "./player.js";
|
||||||
|
import { openImport } from "./import.js";
|
||||||
|
|
||||||
const boot = readBoot();
|
const boot = readBoot();
|
||||||
const { categories, limits, productions } = boot;
|
const { categories, limits, productions } = boot;
|
||||||
@@ -261,6 +262,12 @@ async function runRefreshAll(targets, replaceCustomArt) {
|
|||||||
|
|
||||||
document.querySelector("[data-action=refresh-all]")?.addEventListener("click", refreshAll);
|
document.querySelector("[data-action=refresh-all]")?.addEventListener("click", refreshAll);
|
||||||
|
|
||||||
|
document.querySelector("[data-action=import]").addEventListener("click", () => openImport({
|
||||||
|
list, cat,
|
||||||
|
onListChange: (l) => { list = l; renderHeader(); },
|
||||||
|
onDone: () => load(pages()),
|
||||||
|
}));
|
||||||
|
|
||||||
// Songs and albums from Deezer, or matched to Deezer when saved, can play.
|
// Songs and albums from Deezer, or matched to Deezer when saved, can play.
|
||||||
function canPreview(it) {
|
function canPreview(it) {
|
||||||
return (cat.key === "song" || cat.key === "album") && (it.provider === "deezer" || !!it.extra?.deezer);
|
return (cat.key === "song" || cat.key === "album") && (it.provider === "deezer" || !!it.extra?.deezer);
|
||||||
|
|||||||
@@ -0,0 +1,337 @@
|
|||||||
|
// Import a list from a CSV or text file: read it, search for each entry,
|
||||||
|
// add the certain matches as-is and ask about the rest.
|
||||||
|
import { h, icon, api, toast, openDialog, dialogHead, busy, fmt } from "./common.js";
|
||||||
|
import { previewButton } from "./player.js";
|
||||||
|
|
||||||
|
const CONCURRENCY = 3;
|
||||||
|
|
||||||
|
export function openImport({ list, cat, onListChange, onDone }) {
|
||||||
|
let dlg;
|
||||||
|
const file = h("input", { type: "file", accept: ".csv,.tsv,.txt,text/csv,text/plain,text/tab-separated-values", class: "import-file" });
|
||||||
|
const text = h("textarea", { rows: 6, placeholder: "Or paste titles here, one per line", "aria-label": "Titles to import" });
|
||||||
|
const error = h("p", { class: "alert alert--error", role: "alert", hidden: true });
|
||||||
|
const next = h("button", { type: "submit", class: "btn btn--primary" }, "Find matches");
|
||||||
|
const fileLabel = h("span", { class: "import-drop-text" }, h("b", {}, "Choose a .csv or .txt file"), h("span", { class: "muted small" }, "or drop it here"));
|
||||||
|
const drop = h("label", { class: "import-drop" }, icon("upload"), fileLabel, file);
|
||||||
|
|
||||||
|
file.addEventListener("change", () => {
|
||||||
|
const f = file.files[0];
|
||||||
|
fileLabel.replaceChildren(h("b", {}, f ? f.name : "Choose a .csv or .txt file"), h("span", { class: "muted small" }, f ? `${Math.ceil(f.size / 1024)} KB · click to change` : "or drop it here"));
|
||||||
|
});
|
||||||
|
for (const ev of ["dragenter", "dragover"]) drop.addEventListener(ev, (e) => { e.preventDefault(); drop.classList.add("is-over"); });
|
||||||
|
for (const ev of ["dragleave", "drop"]) drop.addEventListener(ev, () => drop.classList.remove("is-over"));
|
||||||
|
drop.addEventListener("drop", (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (e.dataTransfer.files.length) {
|
||||||
|
file.files = e.dataTransfer.files;
|
||||||
|
file.dispatchEvent(new Event("change"));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const form = h("form", {},
|
||||||
|
dialogHead("Import a list", () => dlg),
|
||||||
|
h("div", { class: "dialog-body stack" },
|
||||||
|
h("p", { class: "muted" },
|
||||||
|
`Entries are added after #${fmt.format(list.itemCount)} in the file's order. Use one title per line, or a CSV with columns like Title, Year, ${artistWord(cat)} and Note. `,
|
||||||
|
"Each one is looked up and matched for you; anything Canon isn't sure about is left for you to check."),
|
||||||
|
drop,
|
||||||
|
text,
|
||||||
|
error,
|
||||||
|
),
|
||||||
|
h("div", { class: "dialog-foot" },
|
||||||
|
h("button", { type: "button", class: "btn", onclick: () => dlg.close() }, "Cancel"),
|
||||||
|
next,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
form.addEventListener("submit", async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
error.hidden = true;
|
||||||
|
let body;
|
||||||
|
if (file.files[0]) {
|
||||||
|
body = new FormData();
|
||||||
|
body.append("file", file.files[0]);
|
||||||
|
} else if (text.value.trim()) {
|
||||||
|
body = { text: text.value };
|
||||||
|
} else {
|
||||||
|
error.hidden = false;
|
||||||
|
error.textContent = "Choose a file or paste some titles first.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
busy(next, true);
|
||||||
|
try {
|
||||||
|
const res = await api("POST", "/api/import/parse", body);
|
||||||
|
dlg.close();
|
||||||
|
review(res.rows, res.rows.length >= res.max ? res.max : 0);
|
||||||
|
} catch (err) {
|
||||||
|
error.hidden = false;
|
||||||
|
error.textContent = err.message;
|
||||||
|
} finally {
|
||||||
|
busy(next, false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
dlg = openDialog(form);
|
||||||
|
dlg.style.maxWidth = "560px";
|
||||||
|
|
||||||
|
// ---------- step 2: match and review ----------
|
||||||
|
|
||||||
|
function review(rows, capped) {
|
||||||
|
let closed = false;
|
||||||
|
let matching = true;
|
||||||
|
let onlyCheck = false;
|
||||||
|
const entries = rows.map((row, i) => ({ row, i, state: "pending", results: [], choice: null, open: false }));
|
||||||
|
|
||||||
|
const bar = h("span", { style: { width: "0%" } });
|
||||||
|
const meter = h("div", { class: "meter", style: { margin: 0 } }, bar);
|
||||||
|
const summary = h("p", { class: "import-summary", "aria-live": "polite" });
|
||||||
|
const filter = h("label", { class: "check small" }, h("input", {
|
||||||
|
type: "checkbox", onchange: (e) => { onlyCheck = e.target.checked; renderAll(); },
|
||||||
|
}), "Show only ones to check");
|
||||||
|
const listEl = h("ol", { class: "import-list" });
|
||||||
|
const room = h("p", { class: "alert alert--warn", hidden: true });
|
||||||
|
const go = h("button", { type: "button", class: "btn btn--primary", disabled: true }, "Import");
|
||||||
|
const body = h("div", { class: "dialog-body import-body" },
|
||||||
|
capped ? h("p", { class: "alert alert--warn" }, icon("info"), `Only the first ${fmt.format(capped)} entries are imported at a time.`) : null,
|
||||||
|
h("div", { class: "import-top" }, summary, filter), meter, listEl);
|
||||||
|
const foot = h("div", { class: "dialog-foot" }, room, h("span", { class: "spacer" }),
|
||||||
|
h("button", { type: "button", class: "btn", onclick: () => rdlg.close() }, "Cancel"), go);
|
||||||
|
const rdlg = openDialog(h("div", { class: "import-dialog-inner" }, dialogHead("Check the matches", () => rdlg), body, foot), {
|
||||||
|
className: "import-dialog",
|
||||||
|
onClose: () => { closed = true; },
|
||||||
|
});
|
||||||
|
|
||||||
|
const chosen = () => entries.filter((e) => e.choice !== null && e.choice !== "skip");
|
||||||
|
const needsCheck = (e) => e.state === "review" || e.state === "none" || e.state === "error";
|
||||||
|
|
||||||
|
function renderTop() {
|
||||||
|
const done = entries.filter((e) => e.state !== "pending").length;
|
||||||
|
bar.style.width = `${Math.round((done / entries.length) * 100)}%`;
|
||||||
|
meter.hidden = !matching;
|
||||||
|
const count = (s) => entries.filter((e) => e.state === s).length;
|
||||||
|
const parts = matching
|
||||||
|
? [`Matching ${fmt.format(done)} of ${fmt.format(entries.length)}…`]
|
||||||
|
: [
|
||||||
|
`${fmt.format(count("matched"))} matched`,
|
||||||
|
count("review") && `${fmt.format(count("review"))} to check`,
|
||||||
|
count("none") + count("error") && `${fmt.format(count("none") + count("error"))} not found`,
|
||||||
|
count("dupe") && `${fmt.format(count("dupe"))} already on the list`,
|
||||||
|
].filter(Boolean);
|
||||||
|
summary.textContent = parts.join(" · ");
|
||||||
|
|
||||||
|
const n = chosen().length;
|
||||||
|
const free = list.maxItems - list.itemCount;
|
||||||
|
go.disabled = matching || n === 0 || n > free;
|
||||||
|
go.textContent = matching ? "Matching…" : `Import ${fmt.format(n)} ${n === 1 ? "entry" : "entries"}`;
|
||||||
|
room.hidden = matching || n <= free;
|
||||||
|
if (!room.hidden) {
|
||||||
|
const want = Math.min(list.itemCount + n, 65535);
|
||||||
|
room.replaceChildren(icon("info"), `This list has room for ${fmt.format(free)} more. `,
|
||||||
|
h("button", {
|
||||||
|
type: "button", class: "btn btn--sm", onclick: async (e) => {
|
||||||
|
busy(e.currentTarget, true);
|
||||||
|
try {
|
||||||
|
Object.assign(list, await api("PUT", `/api/lists/${list.id}`, { ...list, maxItems: want }));
|
||||||
|
onListChange?.(list);
|
||||||
|
renderTop();
|
||||||
|
} catch (err) { toast(err.message, { error: true }); busy(e.currentTarget, false); }
|
||||||
|
},
|
||||||
|
}, `Raise the size to ${fmt.format(want)}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAll() {
|
||||||
|
listEl.replaceChildren(...entries.filter((e) => !onlyCheck || needsCheck(e)).map(renderEntry));
|
||||||
|
renderTop();
|
||||||
|
}
|
||||||
|
|
||||||
|
function rerender(e) {
|
||||||
|
const old = listEl.querySelector(`[data-i="${e.i}"]`);
|
||||||
|
if (old) old.replaceWith(renderEntry(e));
|
||||||
|
renderTop();
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderEntry(e) {
|
||||||
|
const { row } = e;
|
||||||
|
const asWritten = [row.subtitle, row.year].filter(Boolean).join(" · ");
|
||||||
|
const badge = {
|
||||||
|
pending: h("span", { class: "badge badge--muted" }, "Matching…"),
|
||||||
|
matched: h("span", { class: "badge badge--ok" }, icon("check"), "Matched"),
|
||||||
|
review: h("span", { class: "badge badge--warn" }, "Check this"),
|
||||||
|
none: h("span", { class: "badge badge--warn" }, "Not found"),
|
||||||
|
error: h("span", { class: "badge badge--warn", title: e.error }, "Search failed"),
|
||||||
|
dupe: h("span", { class: "badge badge--muted" }, `Already #${e.results[0]?.existingRank}`),
|
||||||
|
}[e.state];
|
||||||
|
|
||||||
|
const li = h("li", { class: `import-row is-${e.state}`, "data-i": e.i },
|
||||||
|
h("div", { class: "import-row-head" },
|
||||||
|
h("span", { class: "import-num" }, `${e.i + 1}.`),
|
||||||
|
h("div", { class: "import-src" }, h("b", {}, row.title), asWritten && h("span", { class: "muted small" }, ` ${asWritten}`)),
|
||||||
|
badge),
|
||||||
|
);
|
||||||
|
if (e.state === "pending") return li;
|
||||||
|
|
||||||
|
// Certain matches collapse to one line until the owner wants to change them.
|
||||||
|
if (!e.open && (e.state === "matched" || e.state === "dupe")) {
|
||||||
|
const pick = typeof e.choice === "number" ? e.results[e.choice] : null;
|
||||||
|
li.append(h("div", { class: "import-chosen" },
|
||||||
|
pick ? resultCard(pick) : h("span", { class: "muted small" }, e.choice === "skip" ? "Skipped" : `Adding “${row.title}” as written`),
|
||||||
|
h("button", { type: "button", class: "btn btn--sm btn--ghost", onclick: () => { e.open = true; rerender(e); } }, "Change")));
|
||||||
|
return li;
|
||||||
|
}
|
||||||
|
|
||||||
|
const name = `imp-${e.i}`;
|
||||||
|
const option = (value, content, extraClass = "") => h("label", { class: "import-opt " + extraClass },
|
||||||
|
h("input", {
|
||||||
|
type: "radio", name, checked: e.choice === value,
|
||||||
|
onchange: () => { e.choice = value; renderTop(); },
|
||||||
|
}),
|
||||||
|
content);
|
||||||
|
const opts = e.results.map((r, idx) => option(idx, resultCard(r, idx === 0 && e.state !== "none" ? "Recommended" : "")));
|
||||||
|
opts.push(option("manual", h("span", { class: "import-new" }, icon("plus"), h("span", {}, "Add ", h("b", {}, `“${row.title}”`), " as written")), "is-new"));
|
||||||
|
opts.push(option("skip", h("span", { class: "muted" }, "Skip it"), "is-skip"));
|
||||||
|
|
||||||
|
const q = h("input", { type: "search", value: row.title, "aria-label": `Search again for ${row.title}` });
|
||||||
|
const again = h("form", { class: "import-again" }, icon("search"), q,
|
||||||
|
h("button", { type: "submit", class: "btn btn--sm" }, "Search again"));
|
||||||
|
again.addEventListener("submit", async (ev) => {
|
||||||
|
ev.preventDefault();
|
||||||
|
const btn = again.querySelector("button");
|
||||||
|
busy(btn, true);
|
||||||
|
// The new spelling is only for searching; "as written" keeps the original.
|
||||||
|
await match(e, { ...row, title: q.value.trim() || row.title });
|
||||||
|
e.open = true;
|
||||||
|
rerender(e);
|
||||||
|
});
|
||||||
|
li.append(e.error ? h("p", { class: "small muted" }, e.error) : null,
|
||||||
|
h("div", { class: "import-opts" }, ...opts), again);
|
||||||
|
return li;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resultCard(r, flag) {
|
||||||
|
const sub = [r.subtitle, r.year, r.extra?.type].filter(Boolean).join(" · ");
|
||||||
|
const art = h("span", { class: "result-art" + (cat.aspect === "square" ? " is-square" : "") });
|
||||||
|
if (r.imageUrl) {
|
||||||
|
const img = h("img", { src: r.imageUrl, alt: "", loading: "lazy", referrerpolicy: "no-referrer" });
|
||||||
|
img.addEventListener("error", () => img.replaceWith(icon(cat.icon)), { once: true });
|
||||||
|
art.append(img);
|
||||||
|
} else {
|
||||||
|
art.append(icon(cat.icon));
|
||||||
|
}
|
||||||
|
if (r.previewRef) art.append(previewButton(`/admin/preview?ref=${encodeURIComponent(r.previewRef)}`, `Play a preview of ${r.title}`, "play-btn play-btn--mini"));
|
||||||
|
return h("span", { class: "import-card" }, art,
|
||||||
|
h("span", { class: "import-card-text" },
|
||||||
|
h("span", { class: "result-title" }, r.title, flag && h("span", { class: "badge" }, flag)),
|
||||||
|
sub && h("span", { class: "result-sub" }, sub),
|
||||||
|
r.existingRank && h("span", { class: "small muted" }, `Already on this list at #${r.existingRank}`)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function match(e, row = e.row) {
|
||||||
|
try {
|
||||||
|
const res = await api("POST", `/api/lists/${list.id}/import/match`, { title: row.title, year: row.year || "", subtitle: row.subtitle || "" });
|
||||||
|
e.results = res.results;
|
||||||
|
e.error = "";
|
||||||
|
e.state = res.status;
|
||||||
|
e.choice = res.status === "none" ? "manual" : 0;
|
||||||
|
if (res.results[0]?.existingRank && res.status === "matched") {
|
||||||
|
e.state = "dupe";
|
||||||
|
e.choice = "skip";
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
e.state = "error";
|
||||||
|
e.error = err.message;
|
||||||
|
e.results = [];
|
||||||
|
e.choice = "manual";
|
||||||
|
if (err.status === 424) throw err; // no API key: every row would fail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function matchAll() {
|
||||||
|
let next = 0;
|
||||||
|
let fatal = null;
|
||||||
|
const worker = async () => {
|
||||||
|
while (!closed && !fatal && next < entries.length) {
|
||||||
|
const e = entries[next++];
|
||||||
|
try {
|
||||||
|
await match(e);
|
||||||
|
} catch (err) {
|
||||||
|
fatal = err;
|
||||||
|
}
|
||||||
|
rerender(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
|
||||||
|
matching = false;
|
||||||
|
if (fatal) {
|
||||||
|
for (const e of entries) if (e.state === "pending") { e.state = "error"; e.error = fatal.message; e.choice = "manual"; }
|
||||||
|
body.prepend(h("p", { class: "alert alert--warn" }, icon("info"), fatal.message, " ",
|
||||||
|
h("a", { href: "/admin/settings#integrations" }, "Open integrations →")));
|
||||||
|
}
|
||||||
|
renderAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
go.addEventListener("click", () => runImport(entries.filter((e) => e.choice !== null && e.choice !== "skip"), rdlg));
|
||||||
|
renderAll();
|
||||||
|
matchAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- step 3: add them ----------
|
||||||
|
|
||||||
|
async function runImport(todo, reviewDlg) {
|
||||||
|
reviewDlg.close();
|
||||||
|
let stopped = false;
|
||||||
|
const bar = h("span", { style: { width: "0%" } });
|
||||||
|
const status = h("p", { class: "muted small", "aria-live": "polite" }, "Starting…");
|
||||||
|
const body = h("div", { class: "dialog-body stack" }, h("div", { class: "meter", style: { margin: 0 } }, bar), status);
|
||||||
|
const stop = h("button", { type: "button", class: "btn", onclick: () => { stopped = true; stop.disabled = true; stop.textContent = "Stopping…"; } }, "Stop");
|
||||||
|
const foot = h("div", { class: "dialog-foot" }, stop);
|
||||||
|
const pdlg = openDialog(h("div", {}, h("div", { class: "dialog-head" }, h("h2", {}, "Importing…")), body, foot), {
|
||||||
|
onClose: () => { stopped = true; onDone?.(); },
|
||||||
|
});
|
||||||
|
pdlg.style.maxWidth = "520px";
|
||||||
|
|
||||||
|
let added = 0, dupes = 0, done = 0;
|
||||||
|
const failed = [];
|
||||||
|
for (const e of todo) {
|
||||||
|
if (stopped) break;
|
||||||
|
status.textContent = `${fmt.format(done + 1)} of ${fmt.format(todo.length)} · ${e.row.title}`;
|
||||||
|
const r = typeof e.choice === "number" ? e.results[e.choice] : null;
|
||||||
|
const payload = r
|
||||||
|
? {
|
||||||
|
title: r.title, subtitle: r.subtitle, year: r.year, externalUrl: r.externalUrl, imageUrl: r.imageUrl,
|
||||||
|
provider: r.provider, providerId: r.providerId, extra: { ...(r.extra || {}) }, description: r.overview || "",
|
||||||
|
details: true, note: e.row.note || "",
|
||||||
|
}
|
||||||
|
: { title: e.row.title, subtitle: e.row.subtitle || "", year: e.row.year || "", note: e.row.note || "", extra: {} };
|
||||||
|
try {
|
||||||
|
const res = await api("POST", `/api/lists/${list.id}/items`, payload);
|
||||||
|
added++;
|
||||||
|
list.itemCount++;
|
||||||
|
if (res.warning) failed.push({ title: e.row.title, error: res.warning });
|
||||||
|
} catch (err) {
|
||||||
|
if (err.status === 409) dupes++;
|
||||||
|
else failed.push({ title: e.row.title, error: err.message });
|
||||||
|
if (err.status === 400 && /full/i.test(err.message)) break;
|
||||||
|
}
|
||||||
|
done++;
|
||||||
|
bar.style.width = `${Math.round((done / todo.length) * 100)}%`;
|
||||||
|
}
|
||||||
|
onListChange?.(list);
|
||||||
|
pdlg.querySelector(".dialog-head h2").textContent = done < todo.length ? "Import stopped" : "Import finished";
|
||||||
|
body.replaceChildren(...[
|
||||||
|
h("p", {}, [
|
||||||
|
`${fmt.format(added)} added`,
|
||||||
|
dupes && `${fmt.format(dupes)} skipped (already on the list)`,
|
||||||
|
failed.length && `${fmt.format(failed.length)} with problems`,
|
||||||
|
done < todo.length && `${fmt.format(todo.length - done)} not reached`,
|
||||||
|
].filter(Boolean).join(" · ")),
|
||||||
|
failed.length > 0 && h("ul", { class: "refresh-failures" },
|
||||||
|
failed.slice(0, 12).map((f) => h("li", {}, h("b", {}, f.title), " — ", f.error)),
|
||||||
|
failed.length > 12 && h("li", { class: "muted" }, `…and ${failed.length - 12} more`)),
|
||||||
|
].filter(Boolean));
|
||||||
|
foot.replaceChildren(h("button", { type: "button", class: "btn btn--primary", onclick: () => pdlg.close() }, "Done"));
|
||||||
|
if (added) toast(`Imported ${fmt.format(added)} ${added === 1 ? "entry" : "entries"}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function artistWord(cat) {
|
||||||
|
return { movie: "Director", tv: "Network", album: "Artist", song: "Artist", game: "Developer" }[cat.key] || "Subtitle";
|
||||||
|
}
|
||||||
@@ -13,6 +13,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="admin-head-actions">
|
<div class="admin-head-actions">
|
||||||
<a class="btn btn--ghost" data-bind="public" href="/l/{{.Data.List.Slug}}">{{icon "external"}}View</a>
|
<a class="btn btn--ghost" data-bind="public" href="/l/{{.Data.List.Slug}}">{{icon "external"}}View</a>
|
||||||
|
<button class="btn btn--ghost" type="button" data-action="import" title="Add entries from a CSV or text file">{{icon "upload"}}Import</button>
|
||||||
{{template "export" .Data.List.Slug}}
|
{{template "export" .Data.List.Slug}}
|
||||||
{{if ne $c.Key "custom"}}<button class="btn btn--ghost" type="button" data-action="refresh-all" title="Pull the latest details and artwork from each entry's source">{{icon "refresh"}}Refresh all</button>{{end}}
|
{{if ne $c.Key "custom"}}<button class="btn btn--ghost" type="button" data-action="refresh-all" title="Pull the latest details and artwork from each entry's source">{{icon "refresh"}}Refresh all</button>{{end}}
|
||||||
<button class="btn" type="button" data-action="edit-list">{{icon "settings"}}List settings</button>
|
<button class="btn" type="button" data-action="edit-list">{{icon "settings"}}List settings</button>
|
||||||
|
|||||||
Reference in new issue
Block a user