Import lists from CSV or text and match each entry
The list editor gets an Import button. A .csv/.tsv/.txt file or pasted text is parsed on the server: header columns (Title, Year, Artist or Director, Note, Rank), numbered lines and trailing "(1999)" years are understood, and Canon's own export round-trips. Each row is then searched with the list's provider and scored. A row is matched automatically only when one result fits exactly and nothing else fits as well; a year or artist in the file breaks ties, an exact title beats a bracketed version like "(Live)", and "Artist - Title" lines are tried both ways. Custom lists need a year or subtitle to be certain. Everything else is shown for review with a recommended pick, the other results, "Add as written", Skip, and a search-again box. Entries are added in file order after the existing ones, with any notes from the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
1 parent
aadacaff95
commit
dadf835be3
10 files changed
+1131
No files matched your search
@@ -0,0 +1,93 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/cdnutter/canon/internal/providers"
|
||||
)
|
||||
|
||||
func TestParseText(t *testing.T) {
|
||||
got, err := Parse([]byte("1. Heat (1995)\n2) 2001: A Space Odyssey\n\n3 - Alien [1979]\n"), "list.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Row{{Title: "Heat", Year: "1995"}, {Title: "2001: A Space Odyssey"}, {Title: "Alien", Year: "1979"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
// Not every line is numbered, so nothing is stripped.
|
||||
got, _ = Parse([]byte("2001: A Space Odyssey\nHeat\n1917\n"), "x.txt")
|
||||
if got[0].Title != "2001: A Space Odyssey" || got[2].Title != "1917" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSV(t *testing.T) {
|
||||
csv := "\xef\xbb\xbfRank,Title,Year,Director,Why\n2,Alien,1979,Ridley Scott,\"Tense, perfect\"\n1,Heat,1995,Michael Mann,\n"
|
||||
got, err := Parse([]byte(csv), "films.csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Row{
|
||||
{Title: "Heat", Year: "1995", Subtitle: "Michael Mann"},
|
||||
{Title: "Alien", Year: "1979", Subtitle: "Ridley Scott", Note: "Tense, perfect"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
// Canon's own export: one quoted column, no header.
|
||||
got, _ = Parse([]byte("\xef\xbb\xbf\"Heat\"\r\n\"Crouching Tiger, Hidden Dragon\"\r\n"), "films.csv")
|
||||
if len(got) != 2 || got[1].Title != "Crouching Tiger, Hidden Dragon" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
// Headerless rank,title,year.
|
||||
got, _ = Parse([]byte("1,Heat,1995\n2,Alien,1979\n"), "x.csv")
|
||||
if got[1] != (Row{Title: "Alien", Year: "1979"}) {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
if _, err := Parse([]byte("\n \n"), "x.txt"); err != ErrEmpty {
|
||||
t.Fatalf("empty file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func res(title, year, sub string) providers.Result {
|
||||
return providers.Result{Provider: "t", ProviderID: title + year + sub, Title: title, Year: year, Subtitle: sub}
|
||||
}
|
||||
|
||||
func TestRank(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
row Row
|
||||
category string
|
||||
results []providers.Result
|
||||
status string
|
||||
top string
|
||||
}{
|
||||
{"exact and alone", Row{Title: "Alien"}, "movie", []providers.Result{res("Alien", "1979", ""), res("Aliens", "1986", "")}, Matched, "Alien1979"},
|
||||
{"remake, no year", Row{Title: "Heat"}, "movie", []providers.Result{res("Heat", "1995", ""), res("Heat", "1986", "")}, Review, "Heat1995"},
|
||||
{"remake, year given", Row{Title: "Heat", Year: "1986"}, "movie", []providers.Result{res("Heat", "1995", ""), res("Heat", "1986", "")}, Matched, "Heat1986"},
|
||||
{"loose title", Row{Title: "Star Wars"}, "movie", []providers.Result{res("Star Wars: A New Hope", "1977", "")}, Review, "Star Wars: A New Hope1977"},
|
||||
{"punctuation and accents", Row{Title: "amelie"}, "movie", []providers.Result{res("Amélie", "2001", "")}, Matched, "Amélie2001"},
|
||||
{"wikipedia qualifier", Row{Title: "The Matrix", Year: "1999"}, "custom", []providers.Result{res("The Matrix (1999 film)", "1999", "")}, Matched, "The Matrix (1999 film)1999"},
|
||||
{"artist - title", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Creep", "1992", "TLC"), res("Creep", "1992", "Radiohead")}, Matched, "Creep1992Radiohead"},
|
||||
{"title, no artist", Row{Title: "Creep"}, "song", []providers.Result{res("Creep", "1992", "TLC"), res("Creep", "1992", "Radiohead")}, Review, "Creep1992TLC"},
|
||||
{"same album twice", Row{Title: "OK Computer"}, "album", []providers.Result{res("OK Computer", "1997", "Radiohead"), res("OK Computer", "1997", "Radiohead")}, Matched, "OK Computer1997Radiohead"},
|
||||
{"wrong year", Row{Title: "Alien", Year: "1990"}, "movie", []providers.Result{res("Alien", "1979", "")}, Review, "Alien1979"},
|
||||
{"exact beats a bracketed version", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Creep (Acoustic)", "", "Radiohead"), res("Creep", "", "Radiohead")}, Matched, "CreepRadiohead"},
|
||||
{"only bracketed versions", Row{Title: "Hey Jude"}, "song", []providers.Result{res("Hey Jude (Remastered 2015)", "", "The Beatles")}, Matched, "Hey Jude (Remastered 2015)The Beatles"},
|
||||
{"year beats no year", Row{Title: "The Matrix", Year: "1999"}, "custom", []providers.Result{res("The Matrix", "", "American media franchise"), res("The Matrix", "1999", "Film by the Wachowskis")}, Matched, "The Matrix1999Film by the Wachowskis"},
|
||||
{"artist confirmed beats a literal title", Row{Title: "Radiohead - Creep"}, "song", []providers.Result{res("Radiohead - Creep", "", "Klangsberg"), res("Creep", "", "Radiohead")}, Matched, "CreepRadiohead"},
|
||||
{"custom needs confirming", Row{Title: "Heat"}, "custom", []providers.Result{res("Heat", "", "Type of energy transfer")}, Review, "HeatType of energy transfer"},
|
||||
{"nothing", Row{Title: "zzz"}, "movie", nil, None, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
status, scored := Rank(c.row, c.category, c.results)
|
||||
if status != c.status {
|
||||
t.Errorf("%s: status %s, want %s", c.name, status, c.status)
|
||||
}
|
||||
if len(scored) > 0 && scored[0].ProviderID != c.top {
|
||||
t.Errorf("%s: top %s, want %s", c.name, scored[0].ProviderID, c.top)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/cdnutter/canon/internal/providers"
|
||||
)
|
||||
|
||||
// Match statuses.
|
||||
const (
|
||||
Matched = "matched" // exactly one result fits: add it without asking
|
||||
Review = "review" // there are results, but the owner should pick
|
||||
None = "none" // nothing found
|
||||
)
|
||||
|
||||
// Scored is a search result with how well it fits the row.
|
||||
type Scored struct {
|
||||
providers.Result
|
||||
Score int `json:"score"`
|
||||
// Tier is how the title fits, given it agrees with any year or artist
|
||||
// given: 2 exactly, 1 once bracketed qualifiers like "(Live)" are
|
||||
// ignored, 0 not at all.
|
||||
Tier int `json:"-"`
|
||||
// Confirms counts what the row gave besides the title (year, artist)
|
||||
// that this result agrees with.
|
||||
Confirms int `json:"-"`
|
||||
}
|
||||
|
||||
// fit is how one reading of a row fits one result.
|
||||
type fit struct{ score, tier, confirms int }
|
||||
|
||||
func (f fit) better(g fit) bool {
|
||||
if f.tier != g.tier {
|
||||
return f.tier > g.tier
|
||||
}
|
||||
if f.confirms != g.confirms {
|
||||
return f.confirms > g.confirms
|
||||
}
|
||||
return f.score > g.score
|
||||
}
|
||||
|
||||
// music categories may write "Artist - Title" on one line.
|
||||
func isMusic(category string) bool { return category == "album" || category == "song" }
|
||||
|
||||
// Query is what to search the provider for.
|
||||
func Query(r Row, category string) string {
|
||||
q := r.Title
|
||||
if isMusic(category) {
|
||||
q = dashes.Replace(q)
|
||||
if r.Subtitle != "" {
|
||||
q += " " + r.Subtitle
|
||||
}
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
var dashes = strings.NewReplacer(" - ", " ", " – ", " ", " — ", " ")
|
||||
|
||||
// readings are the ways a row can be understood: as written, and for music
|
||||
// "A - B" as either title/artist or artist/title.
|
||||
type reading struct{ title, year, artist string }
|
||||
|
||||
func readings(r Row, category string) []reading {
|
||||
out := []reading{{r.Title, r.Year, r.Subtitle}}
|
||||
if isMusic(category) && r.Subtitle == "" {
|
||||
for _, sep := range []string{" - ", " – ", " — "} {
|
||||
if a, b, ok := strings.Cut(r.Title, sep); ok {
|
||||
out = append(out, reading{b, r.Year, a}, reading{a, r.Year, b})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rank scores results against the row, best first, and decides whether the
|
||||
// best one is certain.
|
||||
func Rank(r Row, category string, results []providers.Result) (string, []Scored) {
|
||||
results = slices.DeleteFunc(slices.Clone(results), isDisambiguation)
|
||||
if len(results) == 0 {
|
||||
return None, nil
|
||||
}
|
||||
reads := readings(r, category)
|
||||
scored := make([]Scored, len(results))
|
||||
for i, res := range results {
|
||||
best := fit{score: -1 << 30}
|
||||
for _, rd := range reads {
|
||||
if f := score(rd, res); f.better(best) {
|
||||
best = f
|
||||
}
|
||||
}
|
||||
// The provider's own relevance order breaks ties.
|
||||
scored[i] = Scored{Result: res, Score: best.score - 2*i, Tier: best.tier, Confirms: best.confirms}
|
||||
}
|
||||
slices.SortStableFunc(scored, func(a, b Scored) int {
|
||||
if a.Tier != b.Tier {
|
||||
return b.Tier - a.Tier
|
||||
}
|
||||
if a.Confirms != b.Confirms {
|
||||
return b.Confirms - a.Confirms
|
||||
}
|
||||
return b.Score - a.Score
|
||||
})
|
||||
top := scored[0]
|
||||
if top.Tier == 0 {
|
||||
return Review, scored
|
||||
}
|
||||
// Custom lists search all of Wikipedia, so a bare title ("Heat") could
|
||||
// be anything; only a year or subtitle that agrees makes it certain.
|
||||
if category == "custom" && top.Confirms == 0 {
|
||||
return Review, scored
|
||||
}
|
||||
// Two different things that fit equally well (Heat 1995 and Heat 1986,
|
||||
// with no year given) need the owner to choose. A result that agrees
|
||||
// with more of what the row says (its year, its artist) wins outright.
|
||||
for _, other := range scored[1:] {
|
||||
if other.Tier < top.Tier || other.Confirms < top.Confirms {
|
||||
continue
|
||||
}
|
||||
if !sameEntry(top.Result, other.Result) {
|
||||
return Review, scored
|
||||
}
|
||||
}
|
||||
return Matched, scored
|
||||
}
|
||||
|
||||
// isDisambiguation drops Wikipedia's "may refer to" pages, which are never
|
||||
// the thing itself.
|
||||
func isDisambiguation(r providers.Result) bool {
|
||||
return r.Provider == "wikipedia" && strings.Contains(strings.ToLower(r.Subtitle), "same term")
|
||||
}
|
||||
|
||||
// sameEntry treats one work listed twice (Deezer and MusicBrainz) as one.
|
||||
func sameEntry(a, b providers.Result) bool {
|
||||
return normTitle(a.Title) == normTitle(b.Title) && a.Year == b.Year && normWords(a.Subtitle) == normWords(b.Subtitle)
|
||||
}
|
||||
|
||||
func score(rd reading, res providers.Result) fit {
|
||||
want, got := normTitle(rd.title), normTitle(res.Title)
|
||||
if want == "" || got == "" {
|
||||
return fit{}
|
||||
}
|
||||
f := fit{score: int(60 * similarity(want, got))}
|
||||
switch {
|
||||
case normFull(rd.title) == normFull(res.Title):
|
||||
f.score, f.tier = 110, 2
|
||||
case want == got:
|
||||
f.score, f.tier = 100, 1
|
||||
}
|
||||
if rd.year != "" {
|
||||
switch d := yearDiff(rd.year, res.Year); {
|
||||
case d == 0:
|
||||
f.score += 25
|
||||
f.confirms++
|
||||
case d == 1:
|
||||
f.score += 8
|
||||
f.tier = 0
|
||||
case d > 1:
|
||||
f.score -= 20
|
||||
f.tier = 0
|
||||
}
|
||||
// A result without a year (common on Wikipedia) neither helps nor
|
||||
// rules it out.
|
||||
}
|
||||
if rd.artist != "" {
|
||||
if sharesName(rd.artist, res.Subtitle) {
|
||||
f.score += 25
|
||||
f.confirms++
|
||||
} else {
|
||||
f.score -= 10
|
||||
f.tier = 0
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func yearDiff(a, b string) int {
|
||||
x, err1 := strconv.Atoi(a)
|
||||
y, err2 := strconv.Atoi(b)
|
||||
if err1 != nil || err2 != nil {
|
||||
return -1
|
||||
}
|
||||
if x > y {
|
||||
return x - y
|
||||
}
|
||||
return y - x
|
||||
}
|
||||
|
||||
// sharesName is true when one name contains the other as whole words:
|
||||
// "Radiohead" in "Radiohead", "Nolan" in "Christopher Nolan".
|
||||
func sharesName(a, b string) bool {
|
||||
a, b = " "+normWords(a)+" ", " "+normWords(b)+" "
|
||||
if strings.TrimSpace(a) == "" || strings.TrimSpace(b) == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(a, b) || strings.Contains(b, a)
|
||||
}
|
||||
|
||||
// similarity is the share of words two titles have in common (Dice).
|
||||
func similarity(a, b string) float64 {
|
||||
wa, wb := strings.Fields(a), strings.Fields(b)
|
||||
if len(wa) == 0 || len(wb) == 0 {
|
||||
return 0
|
||||
}
|
||||
count := map[string]int{}
|
||||
for _, w := range wa {
|
||||
count[w]++
|
||||
}
|
||||
common := 0
|
||||
for _, w := range wb {
|
||||
if count[w] > 0 {
|
||||
count[w]--
|
||||
common++
|
||||
}
|
||||
}
|
||||
return 2 * float64(common) / float64(len(wa)+len(wb))
|
||||
}
|
||||
|
||||
// normTitle compares titles loosely: case, accents, punctuation, "&" vs
|
||||
// "and", a leading "The", and bracketed qualifiers like "(1995 film)" or
|
||||
// "(Remastered)" don't matter.
|
||||
func normTitle(s string) string {
|
||||
var b strings.Builder
|
||||
depth := 0
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '(', '[', '{':
|
||||
depth++
|
||||
case ')', ']', '}':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
default:
|
||||
if depth == 0 {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return normFull(b.String())
|
||||
}
|
||||
|
||||
// normFull is normTitle keeping bracketed parts: "Creep (Acoustic)" is not
|
||||
// "Creep".
|
||||
func normFull(s string) string {
|
||||
w := normWords(s)
|
||||
for _, article := range []string{"the ", "a ", "an "} {
|
||||
if rest, ok := strings.CutPrefix(w, article); ok && rest != "" {
|
||||
return rest
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func normWords(s string) string {
|
||||
s = strings.ReplaceAll(strings.ToLower(s), "&", " and ")
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if f, ok := fold[r]; ok {
|
||||
b.WriteString(f)
|
||||
} else if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
} else if r == '\'' || r == '’' || r == '.' {
|
||||
// "Don't" = "Dont", "S.O.S" = "SOS"
|
||||
} else {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
return strings.Join(strings.Fields(b.String()), " ")
|
||||
}
|
||||
|
||||
var fold = map[rune]string{
|
||||
'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae",
|
||||
'ç': "c", 'è': "e", 'é': "e", 'ê': "e", 'ë': "e",
|
||||
'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ñ': "n",
|
||||
'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'œ': "oe",
|
||||
'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'ÿ': "y", 'ß': "ss",
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
// Package importer reads a list of titles from a CSV or text file and
|
||||
// matches each one to a search result.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MaxRows caps one import.
|
||||
const MaxRows = 2000
|
||||
|
||||
// Row is one entry read from the file, in the file's order.
|
||||
type Row struct {
|
||||
Title string `json:"title"`
|
||||
Year string `json:"year,omitempty"`
|
||||
Subtitle string `json:"subtitle,omitempty"` // artist, director, developer…
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
var ErrEmpty = errors.New("no titles found in that file")
|
||||
|
||||
// Parse reads rows from data. CSV and TSV files may have a header row
|
||||
// naming their columns; text files hold one title per line.
|
||||
func Parse(data []byte, filename string) ([]Row, error) {
|
||||
data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
|
||||
if !utf8.Valid(data) {
|
||||
data = []byte(strings.ToValidUTF8(latin1(data), "�"))
|
||||
}
|
||||
text := strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||
var rows []Row
|
||||
var err error
|
||||
switch ext := strings.ToLower(filename); {
|
||||
case strings.HasSuffix(ext, ".csv"):
|
||||
rows, err = parseDelimited(text, sniffDelimiter(text))
|
||||
case strings.HasSuffix(ext, ".tsv"):
|
||||
rows, err = parseDelimited(text, '\t')
|
||||
default:
|
||||
rows = parseLines(text)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, ErrEmpty
|
||||
}
|
||||
if len(rows) > MaxRows {
|
||||
rows = rows[:MaxRows]
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// latin1 decodes bytes as Windows-1252/Latin-1, which is what spreadsheet
|
||||
// exports that aren't UTF-8 usually are.
|
||||
func latin1(b []byte) string {
|
||||
r := make([]rune, len(b))
|
||||
for i, c := range b {
|
||||
r[i] = rune(c)
|
||||
}
|
||||
return string(r)
|
||||
}
|
||||
|
||||
func sniffDelimiter(text string) rune {
|
||||
first, _, _ := strings.Cut(text, "\n")
|
||||
best, n := ',', strings.Count(first, ",")
|
||||
for _, d := range []rune{'\t', ';'} {
|
||||
if c := strings.Count(first, string(d)); c > n {
|
||||
best, n = d, c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// numbered matches a leading rank: "1. ", "#2 ", "3) ", "4 - ".
|
||||
var numbered = regexp.MustCompile(`^\s*#?\d{1,5}\s*[.):\-–—]?\s+`)
|
||||
|
||||
// trailingYear matches "Heat (1995)" or "Heat [1995]".
|
||||
var trailingYear = regexp.MustCompile(`\s*[(\[]((?:1[5-9]|20)\d\d)[)\]]\s*$`)
|
||||
|
||||
func parseLines(text string) []Row {
|
||||
var lines []string
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
// Only strip numbering when every line has it, so a list that merely
|
||||
// contains "2001: A Space Odyssey" keeps its title.
|
||||
strip := len(lines) > 0
|
||||
for _, l := range lines {
|
||||
if !numbered.MatchString(l) {
|
||||
strip = false
|
||||
break
|
||||
}
|
||||
}
|
||||
rows := make([]Row, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
if strip {
|
||||
l = numbered.ReplaceAllString(l, "")
|
||||
}
|
||||
if r := rowFromTitle(l); r.Title != "" {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func rowFromTitle(s string) Row {
|
||||
s = strings.TrimSpace(s)
|
||||
r := Row{Title: s}
|
||||
if m := trailingYear.FindStringSubmatchIndex(s); m != nil {
|
||||
r.Title = strings.TrimSpace(s[:m[0]])
|
||||
r.Year = s[m[2]:m[3]]
|
||||
}
|
||||
if r.Title == "" {
|
||||
r.Title = s
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
var headerNames = map[string]string{
|
||||
"title": "title", "name": "title", "film": "title", "movie": "title", "show": "title",
|
||||
"album": "title", "song": "title", "track": "title", "game": "title", "item": "title",
|
||||
"year": "year", "release year": "year", "released": "year", "date": "year",
|
||||
"artist": "subtitle", "director": "subtitle", "creator": "subtitle", "developer": "subtitle",
|
||||
"author": "subtitle", "by": "subtitle", "subtitle": "subtitle", "network": "subtitle",
|
||||
"note": "note", "notes": "note", "why": "note", "comment": "note", "comments": "note", "review": "note",
|
||||
"rank": "rank", "#": "rank", "position": "rank", "no": "rank", "no.": "rank",
|
||||
}
|
||||
|
||||
func parseDelimited(text string, delim rune) ([]Row, error) {
|
||||
cr := csv.NewReader(strings.NewReader(text))
|
||||
cr.Comma = delim
|
||||
cr.FieldsPerRecord = -1
|
||||
cr.LazyQuotes = true
|
||||
cr.TrimLeadingSpace = true
|
||||
records, err := cr.ReadAll()
|
||||
if err != nil {
|
||||
return nil, errors.New("that CSV file couldn't be read: " + err.Error())
|
||||
}
|
||||
records = slices.DeleteFunc(records, func(rec []string) bool {
|
||||
return strings.TrimSpace(strings.Join(rec, "")) == ""
|
||||
})
|
||||
if len(records) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cols := map[string]int{}
|
||||
if head := records[0]; isHeader(head) {
|
||||
for i, name := range head {
|
||||
if role, ok := headerNames[strings.ToLower(strings.TrimSpace(name))]; ok {
|
||||
if _, taken := cols[role]; !taken {
|
||||
cols[role] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
records = records[1:]
|
||||
} else {
|
||||
cols = guessColumns(records)
|
||||
}
|
||||
if _, ok := cols["title"]; !ok {
|
||||
cols["title"] = 0
|
||||
}
|
||||
|
||||
type ranked struct {
|
||||
Row
|
||||
rank int
|
||||
}
|
||||
var out []ranked
|
||||
cell := func(rec []string, role string) string {
|
||||
i, ok := cols[role]
|
||||
if !ok || i >= len(rec) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(rec[i])
|
||||
}
|
||||
for i, rec := range records {
|
||||
r := Row{Title: cell(rec, "title"), Subtitle: cell(rec, "subtitle"), Note: cell(rec, "note")}
|
||||
if y := yearIn(cell(rec, "year")); y != "" {
|
||||
r.Year = y
|
||||
} else {
|
||||
t := rowFromTitle(r.Title)
|
||||
r.Title, r.Year = t.Title, t.Year
|
||||
}
|
||||
if r.Title == "" {
|
||||
continue
|
||||
}
|
||||
rank, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#"))
|
||||
if err != nil {
|
||||
rank = 1 << 30
|
||||
}
|
||||
out = append(out, ranked{r, rank*MaxRows*2 + i}) // stable within equal ranks
|
||||
}
|
||||
if _, ok := cols["rank"]; ok {
|
||||
slices.SortStableFunc(out, func(a, b ranked) int { return a.rank - b.rank })
|
||||
}
|
||||
rows := make([]Row, len(out))
|
||||
for i, r := range out {
|
||||
rows[i] = r.Row
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func isHeader(rec []string) bool {
|
||||
for _, c := range rec {
|
||||
if headerNames[strings.ToLower(strings.TrimSpace(c))] == "title" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var yearOnly = regexp.MustCompile(`\b((?:1[5-9]|20)\d\d)\b`)
|
||||
|
||||
func yearIn(s string) string { return yearOnly.FindString(s) }
|
||||
|
||||
// guessColumns handles headerless files: "1,Heat,1995" or "Heat,1995".
|
||||
func guessColumns(records [][]string) map[string]int {
|
||||
cols := map[string]int{}
|
||||
all := func(col int, ok func(string) bool) bool {
|
||||
for _, rec := range records {
|
||||
if col >= len(rec) || !ok(strings.TrimSpace(rec[col])) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
isInt := func(s string) bool { _, err := strconv.Atoi(strings.TrimPrefix(s, "#")); return err == nil }
|
||||
isYear := func(s string) bool { return s == yearIn(s) && s != "" }
|
||||
title := 0
|
||||
width := len(records[0])
|
||||
if width >= 2 && all(0, isInt) && !all(1, isInt) {
|
||||
cols["rank"] = 0
|
||||
title = 1
|
||||
}
|
||||
cols["title"] = title
|
||||
if title+1 < width && all(title+1, isYear) {
|
||||
cols["year"] = title + 1
|
||||
}
|
||||
return cols
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/cdnutter/canon/internal/importer"
|
||||
"github.com/cdnutter/canon/internal/store"
|
||||
)
|
||||
|
||||
const maxImportBytes = 2 << 20
|
||||
|
||||
// apiImportParse reads an uploaded CSV/TXT file, or pasted text, into rows.
|
||||
func (s *Server) apiImportParse(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxImportBytes+64<<10)
|
||||
var data []byte
|
||||
name := "pasted.txt"
|
||||
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/") {
|
||||
f, hdr, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
s.writeErr(w, store.ValidationError{Msg: "Choose a .csv or .txt file (up to 2 MB)."})
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
if data, err = io.ReadAll(f); err != nil {
|
||||
s.writeErr(w, store.ValidationError{Msg: "That file is too big (2 MB at most)."})
|
||||
return
|
||||
}
|
||||
name = hdr.Filename
|
||||
} else {
|
||||
var in struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := decodeJSON(r, &in); err != nil {
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
data = []byte(in.Text)
|
||||
}
|
||||
rows, err := importer.Parse(data, name)
|
||||
if err != nil {
|
||||
if errors.Is(err, importer.ErrEmpty) {
|
||||
err = store.ValidationError{Msg: "No titles found. Use one title per line, or a CSV with a Title column."}
|
||||
} else {
|
||||
err = store.ValidationError{Msg: err.Error()}
|
||||
}
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"rows": rows, "max": importer.MaxRows})
|
||||
}
|
||||
|
||||
type importMatch struct {
|
||||
Status string `json:"status"`
|
||||
Results []searchResult `json:"results"`
|
||||
}
|
||||
|
||||
// apiImportMatch searches for one imported row and says how sure the best
|
||||
// result is.
|
||||
func (s *Server) apiImportMatch(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
listID, err := pathID(r)
|
||||
if err != nil {
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
var row importer.Row
|
||||
if err := decodeJSON(r, &row); err != nil {
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
row.Title = strings.TrimSpace(row.Title)
|
||||
if row.Title == "" || utf8.RuneCountInString(row.Title) > 300 {
|
||||
writeError(w, http.StatusBadRequest, "Each entry needs a title.")
|
||||
return
|
||||
}
|
||||
list, err := s.store.ListByID(ctx, listID)
|
||||
if err != nil {
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
p, err := s.providers.For(ctx, list.Category)
|
||||
if err != nil {
|
||||
s.writeErr(w, err)
|
||||
return
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
results, err := p.Search(sctx, importer.Query(row, list.Category))
|
||||
if err != nil {
|
||||
s.log.Warn("import search failed", "provider", p.Name(), "err", err)
|
||||
writeError(w, http.StatusBadGateway, fmt.Sprintf("%s search failed: %v", p.Label(), err))
|
||||
return
|
||||
}
|
||||
status, scored := importer.Rank(row, list.Category, results)
|
||||
out := importMatch{Status: status, Results: []searchResult{}}
|
||||
for _, sc := range scored[:min(len(scored), 6)] {
|
||||
sr := searchResult{Result: sc.Result}
|
||||
if sc.Provider == "deezer" && hasPreviews(list.Category) {
|
||||
sr.PreviewRef = sc.ProviderID
|
||||
}
|
||||
if it, err := s.store.FindItemByProvider(ctx, listID, sc.Provider, sc.ProviderID); err == nil {
|
||||
sr.ExistingRank = it.Rank
|
||||
}
|
||||
out.Results = append(out.Results, sr)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
@@ -146,6 +146,8 @@ func (s *Server) Handler() http.Handler {
|
||||
api("POST /api/items/{id}/refresh", s.apiRefreshItem)
|
||||
api("DELETE /api/items/{id}", s.apiDeleteItem)
|
||||
api("GET /api/search", s.apiSearch)
|
||||
api("POST /api/import/parse", s.apiImportParse)
|
||||
api("POST /api/lists/{id}/import/match", s.apiImportMatch)
|
||||
api("GET /api/settings", s.apiSettings)
|
||||
api("PUT /api/settings", s.apiUpdateSettings)
|
||||
api("POST /api/settings/avatar", s.apiUploadAvatar)
|
||||
|
||||
Reference in new issue
Block a user