Import lists from CSV or text and match each entry
The list editor gets an Import button. A .csv/.tsv/.txt file or pasted text is parsed on the server: header columns (Title, Year, Artist or Director, Note, Rank), numbered lines and trailing "(1999)" years are understood, and Canon's own export round-trips. Each row is then searched with the list's provider and scored. A row is matched automatically only when one result fits exactly and nothing else fits as well; a year or artist in the file breaks ties, an exact title beats a bracketed version like "(Live)", and "Artist - Title" lines are tried both ways. Custom lists need a year or subtitle to be certain. Everything else is shown for review with a recommended pick, the other results, "Add as written", Skip, and a search-again box. Entries are added in file order after the existing ones, with any notes from the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
1 parent
aadacaff95
commit
dadf835be3
10 files changed
+1131
No files matched your search
@@ -0,0 +1,247 @@
|
||||
// Package importer reads a list of titles from a CSV or text file and
|
||||
// matches each one to a search result.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MaxRows caps one import.
|
||||
const MaxRows = 2000
|
||||
|
||||
// Row is one entry read from the file, in the file's order.
|
||||
type Row struct {
|
||||
Title string `json:"title"`
|
||||
Year string `json:"year,omitempty"`
|
||||
Subtitle string `json:"subtitle,omitempty"` // artist, director, developer…
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
var ErrEmpty = errors.New("no titles found in that file")
|
||||
|
||||
// Parse reads rows from data. CSV and TSV files may have a header row
|
||||
// naming their columns; text files hold one title per line.
|
||||
func Parse(data []byte, filename string) ([]Row, error) {
|
||||
data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
|
||||
if !utf8.Valid(data) {
|
||||
data = []byte(strings.ToValidUTF8(latin1(data), "�"))
|
||||
}
|
||||
text := strings.ReplaceAll(string(data), "\r\n", "\n")
|
||||
var rows []Row
|
||||
var err error
|
||||
switch ext := strings.ToLower(filename); {
|
||||
case strings.HasSuffix(ext, ".csv"):
|
||||
rows, err = parseDelimited(text, sniffDelimiter(text))
|
||||
case strings.HasSuffix(ext, ".tsv"):
|
||||
rows, err = parseDelimited(text, '\t')
|
||||
default:
|
||||
rows = parseLines(text)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, ErrEmpty
|
||||
}
|
||||
if len(rows) > MaxRows {
|
||||
rows = rows[:MaxRows]
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// latin1 decodes bytes as Windows-1252/Latin-1, which is what spreadsheet
|
||||
// exports that aren't UTF-8 usually are.
|
||||
func latin1(b []byte) string {
|
||||
r := make([]rune, len(b))
|
||||
for i, c := range b {
|
||||
r[i] = rune(c)
|
||||
}
|
||||
return string(r)
|
||||
}
|
||||
|
||||
func sniffDelimiter(text string) rune {
|
||||
first, _, _ := strings.Cut(text, "\n")
|
||||
best, n := ',', strings.Count(first, ",")
|
||||
for _, d := range []rune{'\t', ';'} {
|
||||
if c := strings.Count(first, string(d)); c > n {
|
||||
best, n = d, c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// numbered matches a leading rank: "1. ", "#2 ", "3) ", "4 - ".
|
||||
var numbered = regexp.MustCompile(`^\s*#?\d{1,5}\s*[.):\-–—]?\s+`)
|
||||
|
||||
// trailingYear matches "Heat (1995)" or "Heat [1995]".
|
||||
var trailingYear = regexp.MustCompile(`\s*[(\[]((?:1[5-9]|20)\d\d)[)\]]\s*$`)
|
||||
|
||||
func parseLines(text string) []Row {
|
||||
var lines []string
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
// Only strip numbering when every line has it, so a list that merely
|
||||
// contains "2001: A Space Odyssey" keeps its title.
|
||||
strip := len(lines) > 0
|
||||
for _, l := range lines {
|
||||
if !numbered.MatchString(l) {
|
||||
strip = false
|
||||
break
|
||||
}
|
||||
}
|
||||
rows := make([]Row, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
if strip {
|
||||
l = numbered.ReplaceAllString(l, "")
|
||||
}
|
||||
if r := rowFromTitle(l); r.Title != "" {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func rowFromTitle(s string) Row {
|
||||
s = strings.TrimSpace(s)
|
||||
r := Row{Title: s}
|
||||
if m := trailingYear.FindStringSubmatchIndex(s); m != nil {
|
||||
r.Title = strings.TrimSpace(s[:m[0]])
|
||||
r.Year = s[m[2]:m[3]]
|
||||
}
|
||||
if r.Title == "" {
|
||||
r.Title = s
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
var headerNames = map[string]string{
|
||||
"title": "title", "name": "title", "film": "title", "movie": "title", "show": "title",
|
||||
"album": "title", "song": "title", "track": "title", "game": "title", "item": "title",
|
||||
"year": "year", "release year": "year", "released": "year", "date": "year",
|
||||
"artist": "subtitle", "director": "subtitle", "creator": "subtitle", "developer": "subtitle",
|
||||
"author": "subtitle", "by": "subtitle", "subtitle": "subtitle", "network": "subtitle",
|
||||
"note": "note", "notes": "note", "why": "note", "comment": "note", "comments": "note", "review": "note",
|
||||
"rank": "rank", "#": "rank", "position": "rank", "no": "rank", "no.": "rank",
|
||||
}
|
||||
|
||||
func parseDelimited(text string, delim rune) ([]Row, error) {
|
||||
cr := csv.NewReader(strings.NewReader(text))
|
||||
cr.Comma = delim
|
||||
cr.FieldsPerRecord = -1
|
||||
cr.LazyQuotes = true
|
||||
cr.TrimLeadingSpace = true
|
||||
records, err := cr.ReadAll()
|
||||
if err != nil {
|
||||
return nil, errors.New("that CSV file couldn't be read: " + err.Error())
|
||||
}
|
||||
records = slices.DeleteFunc(records, func(rec []string) bool {
|
||||
return strings.TrimSpace(strings.Join(rec, "")) == ""
|
||||
})
|
||||
if len(records) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
cols := map[string]int{}
|
||||
if head := records[0]; isHeader(head) {
|
||||
for i, name := range head {
|
||||
if role, ok := headerNames[strings.ToLower(strings.TrimSpace(name))]; ok {
|
||||
if _, taken := cols[role]; !taken {
|
||||
cols[role] = i
|
||||
}
|
||||
}
|
||||
}
|
||||
records = records[1:]
|
||||
} else {
|
||||
cols = guessColumns(records)
|
||||
}
|
||||
if _, ok := cols["title"]; !ok {
|
||||
cols["title"] = 0
|
||||
}
|
||||
|
||||
type ranked struct {
|
||||
Row
|
||||
rank int
|
||||
}
|
||||
var out []ranked
|
||||
cell := func(rec []string, role string) string {
|
||||
i, ok := cols[role]
|
||||
if !ok || i >= len(rec) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(rec[i])
|
||||
}
|
||||
for i, rec := range records {
|
||||
r := Row{Title: cell(rec, "title"), Subtitle: cell(rec, "subtitle"), Note: cell(rec, "note")}
|
||||
if y := yearIn(cell(rec, "year")); y != "" {
|
||||
r.Year = y
|
||||
} else {
|
||||
t := rowFromTitle(r.Title)
|
||||
r.Title, r.Year = t.Title, t.Year
|
||||
}
|
||||
if r.Title == "" {
|
||||
continue
|
||||
}
|
||||
rank, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#"))
|
||||
if err != nil {
|
||||
rank = 1 << 30
|
||||
}
|
||||
out = append(out, ranked{r, rank*MaxRows*2 + i}) // stable within equal ranks
|
||||
}
|
||||
if _, ok := cols["rank"]; ok {
|
||||
slices.SortStableFunc(out, func(a, b ranked) int { return a.rank - b.rank })
|
||||
}
|
||||
rows := make([]Row, len(out))
|
||||
for i, r := range out {
|
||||
rows[i] = r.Row
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func isHeader(rec []string) bool {
|
||||
for _, c := range rec {
|
||||
if headerNames[strings.ToLower(strings.TrimSpace(c))] == "title" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var yearOnly = regexp.MustCompile(`\b((?:1[5-9]|20)\d\d)\b`)
|
||||
|
||||
func yearIn(s string) string { return yearOnly.FindString(s) }
|
||||
|
||||
// guessColumns handles headerless files: "1,Heat,1995" or "Heat,1995".
|
||||
func guessColumns(records [][]string) map[string]int {
|
||||
cols := map[string]int{}
|
||||
all := func(col int, ok func(string) bool) bool {
|
||||
for _, rec := range records {
|
||||
if col >= len(rec) || !ok(strings.TrimSpace(rec[col])) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
isInt := func(s string) bool { _, err := strconv.Atoi(strings.TrimPrefix(s, "#")); return err == nil }
|
||||
isYear := func(s string) bool { return s == yearIn(s) && s != "" }
|
||||
title := 0
|
||||
width := len(records[0])
|
||||
if width >= 2 && all(0, isInt) && !all(1, isInt) {
|
||||
cols["rank"] = 0
|
||||
title = 1
|
||||
}
|
||||
cols["title"] = title
|
||||
if title+1 < width && all(title+1, isYear) {
|
||||
cols["year"] = title + 1
|
||||
}
|
||||
return cols
|
||||
}
|
||||
Reference in new issue
Block a user