Import lists from CSV or text and match each entry
The list editor gets an Import button. A .csv/.tsv/.txt file or pasted text is parsed on the server: header columns (Title, Year, Artist or Director, Note, Rank), numbered lines and trailing "(1999)" years are understood, and Canon's own export round-trips. Each row is then searched with the list's provider and scored. A row is matched automatically only when one result fits exactly and nothing else fits as well; a year or artist in the file breaks ties, an exact title beats a bracketed version like "(Live)", and "Artist - Title" lines are tried both ways. Custom lists need a year or subtitle to be certain. Everything else is shown for review with a recommended pick, the other results, "Add as written", Skip, and a search-again box. Entries are added in file order after the existing ones, with any notes from the file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
cdnutterandClaude Opus 5.5 committed
1 parent
aadacaff95
commit
dadf835be3
10 files changed
+1131
No files matched your search
@@ -0,0 +1,280 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/cdnutter/canon/internal/providers"
|
||||
)
|
||||
|
||||
// Match statuses.
|
||||
const (
|
||||
Matched = "matched" // exactly one result fits: add it without asking
|
||||
Review = "review" // there are results, but the owner should pick
|
||||
None = "none" // nothing found
|
||||
)
|
||||
|
||||
// Scored is a search result with how well it fits the row.
|
||||
type Scored struct {
|
||||
providers.Result
|
||||
Score int `json:"score"`
|
||||
// Tier is how the title fits, given it agrees with any year or artist
|
||||
// given: 2 exactly, 1 once bracketed qualifiers like "(Live)" are
|
||||
// ignored, 0 not at all.
|
||||
Tier int `json:"-"`
|
||||
// Confirms counts what the row gave besides the title (year, artist)
|
||||
// that this result agrees with.
|
||||
Confirms int `json:"-"`
|
||||
}
|
||||
|
||||
// fit is how one reading of a row fits one result.
|
||||
type fit struct{ score, tier, confirms int }
|
||||
|
||||
func (f fit) better(g fit) bool {
|
||||
if f.tier != g.tier {
|
||||
return f.tier > g.tier
|
||||
}
|
||||
if f.confirms != g.confirms {
|
||||
return f.confirms > g.confirms
|
||||
}
|
||||
return f.score > g.score
|
||||
}
|
||||
|
||||
// music categories may write "Artist - Title" on one line.
|
||||
func isMusic(category string) bool { return category == "album" || category == "song" }
|
||||
|
||||
// Query is what to search the provider for.
|
||||
func Query(r Row, category string) string {
|
||||
q := r.Title
|
||||
if isMusic(category) {
|
||||
q = dashes.Replace(q)
|
||||
if r.Subtitle != "" {
|
||||
q += " " + r.Subtitle
|
||||
}
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
var dashes = strings.NewReplacer(" - ", " ", " – ", " ", " — ", " ")
|
||||
|
||||
// readings are the ways a row can be understood: as written, and for music
|
||||
// "A - B" as either title/artist or artist/title.
|
||||
type reading struct{ title, year, artist string }
|
||||
|
||||
func readings(r Row, category string) []reading {
|
||||
out := []reading{{r.Title, r.Year, r.Subtitle}}
|
||||
if isMusic(category) && r.Subtitle == "" {
|
||||
for _, sep := range []string{" - ", " – ", " — "} {
|
||||
if a, b, ok := strings.Cut(r.Title, sep); ok {
|
||||
out = append(out, reading{b, r.Year, a}, reading{a, r.Year, b})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rank scores results against the row, best first, and decides whether the
|
||||
// best one is certain.
|
||||
func Rank(r Row, category string, results []providers.Result) (string, []Scored) {
|
||||
results = slices.DeleteFunc(slices.Clone(results), isDisambiguation)
|
||||
if len(results) == 0 {
|
||||
return None, nil
|
||||
}
|
||||
reads := readings(r, category)
|
||||
scored := make([]Scored, len(results))
|
||||
for i, res := range results {
|
||||
best := fit{score: -1 << 30}
|
||||
for _, rd := range reads {
|
||||
if f := score(rd, res); f.better(best) {
|
||||
best = f
|
||||
}
|
||||
}
|
||||
// The provider's own relevance order breaks ties.
|
||||
scored[i] = Scored{Result: res, Score: best.score - 2*i, Tier: best.tier, Confirms: best.confirms}
|
||||
}
|
||||
slices.SortStableFunc(scored, func(a, b Scored) int {
|
||||
if a.Tier != b.Tier {
|
||||
return b.Tier - a.Tier
|
||||
}
|
||||
if a.Confirms != b.Confirms {
|
||||
return b.Confirms - a.Confirms
|
||||
}
|
||||
return b.Score - a.Score
|
||||
})
|
||||
top := scored[0]
|
||||
if top.Tier == 0 {
|
||||
return Review, scored
|
||||
}
|
||||
// Custom lists search all of Wikipedia, so a bare title ("Heat") could
|
||||
// be anything; only a year or subtitle that agrees makes it certain.
|
||||
if category == "custom" && top.Confirms == 0 {
|
||||
return Review, scored
|
||||
}
|
||||
// Two different things that fit equally well (Heat 1995 and Heat 1986,
|
||||
// with no year given) need the owner to choose. A result that agrees
|
||||
// with more of what the row says (its year, its artist) wins outright.
|
||||
for _, other := range scored[1:] {
|
||||
if other.Tier < top.Tier || other.Confirms < top.Confirms {
|
||||
continue
|
||||
}
|
||||
if !sameEntry(top.Result, other.Result) {
|
||||
return Review, scored
|
||||
}
|
||||
}
|
||||
return Matched, scored
|
||||
}
|
||||
|
||||
// isDisambiguation drops Wikipedia's "may refer to" pages, which are never
|
||||
// the thing itself.
|
||||
func isDisambiguation(r providers.Result) bool {
|
||||
return r.Provider == "wikipedia" && strings.Contains(strings.ToLower(r.Subtitle), "same term")
|
||||
}
|
||||
|
||||
// sameEntry treats one work listed twice (Deezer and MusicBrainz) as one.
|
||||
func sameEntry(a, b providers.Result) bool {
|
||||
return normTitle(a.Title) == normTitle(b.Title) && a.Year == b.Year && normWords(a.Subtitle) == normWords(b.Subtitle)
|
||||
}
|
||||
|
||||
func score(rd reading, res providers.Result) fit {
|
||||
want, got := normTitle(rd.title), normTitle(res.Title)
|
||||
if want == "" || got == "" {
|
||||
return fit{}
|
||||
}
|
||||
f := fit{score: int(60 * similarity(want, got))}
|
||||
switch {
|
||||
case normFull(rd.title) == normFull(res.Title):
|
||||
f.score, f.tier = 110, 2
|
||||
case want == got:
|
||||
f.score, f.tier = 100, 1
|
||||
}
|
||||
if rd.year != "" {
|
||||
switch d := yearDiff(rd.year, res.Year); {
|
||||
case d == 0:
|
||||
f.score += 25
|
||||
f.confirms++
|
||||
case d == 1:
|
||||
f.score += 8
|
||||
f.tier = 0
|
||||
case d > 1:
|
||||
f.score -= 20
|
||||
f.tier = 0
|
||||
}
|
||||
// A result without a year (common on Wikipedia) neither helps nor
|
||||
// rules it out.
|
||||
}
|
||||
if rd.artist != "" {
|
||||
if sharesName(rd.artist, res.Subtitle) {
|
||||
f.score += 25
|
||||
f.confirms++
|
||||
} else {
|
||||
f.score -= 10
|
||||
f.tier = 0
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func yearDiff(a, b string) int {
|
||||
x, err1 := strconv.Atoi(a)
|
||||
y, err2 := strconv.Atoi(b)
|
||||
if err1 != nil || err2 != nil {
|
||||
return -1
|
||||
}
|
||||
if x > y {
|
||||
return x - y
|
||||
}
|
||||
return y - x
|
||||
}
|
||||
|
||||
// sharesName is true when one name contains the other as whole words:
|
||||
// "Radiohead" in "Radiohead", "Nolan" in "Christopher Nolan".
|
||||
func sharesName(a, b string) bool {
|
||||
a, b = " "+normWords(a)+" ", " "+normWords(b)+" "
|
||||
if strings.TrimSpace(a) == "" || strings.TrimSpace(b) == "" {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(a, b) || strings.Contains(b, a)
|
||||
}
|
||||
|
||||
// similarity is the share of words two titles have in common (Dice).
|
||||
func similarity(a, b string) float64 {
|
||||
wa, wb := strings.Fields(a), strings.Fields(b)
|
||||
if len(wa) == 0 || len(wb) == 0 {
|
||||
return 0
|
||||
}
|
||||
count := map[string]int{}
|
||||
for _, w := range wa {
|
||||
count[w]++
|
||||
}
|
||||
common := 0
|
||||
for _, w := range wb {
|
||||
if count[w] > 0 {
|
||||
count[w]--
|
||||
common++
|
||||
}
|
||||
}
|
||||
return 2 * float64(common) / float64(len(wa)+len(wb))
|
||||
}
|
||||
|
||||
// normTitle compares titles loosely: case, accents, punctuation, "&" vs
|
||||
// "and", a leading "The", and bracketed qualifiers like "(1995 film)" or
|
||||
// "(Remastered)" don't matter.
|
||||
func normTitle(s string) string {
|
||||
var b strings.Builder
|
||||
depth := 0
|
||||
for _, r := range s {
|
||||
switch r {
|
||||
case '(', '[', '{':
|
||||
depth++
|
||||
case ')', ']', '}':
|
||||
if depth > 0 {
|
||||
depth--
|
||||
}
|
||||
default:
|
||||
if depth == 0 {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return normFull(b.String())
|
||||
}
|
||||
|
||||
// normFull is normTitle keeping bracketed parts: "Creep (Acoustic)" is not
|
||||
// "Creep".
|
||||
func normFull(s string) string {
|
||||
w := normWords(s)
|
||||
for _, article := range []string{"the ", "a ", "an "} {
|
||||
if rest, ok := strings.CutPrefix(w, article); ok && rest != "" {
|
||||
return rest
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func normWords(s string) string {
|
||||
s = strings.ReplaceAll(strings.ToLower(s), "&", " and ")
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if f, ok := fold[r]; ok {
|
||||
b.WriteString(f)
|
||||
} else if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
} else if r == '\'' || r == '’' || r == '.' {
|
||||
// "Don't" = "Dont", "S.O.S" = "SOS"
|
||||
} else {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
return strings.Join(strings.Fields(b.String()), " ")
|
||||
}
|
||||
|
||||
var fold = map[rune]string{
|
||||
'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae",
|
||||
'ç': "c", 'è': "e", 'é': "e", 'ê': "e", 'ë': "e",
|
||||
'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ñ': "n",
|
||||
'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'œ': "oe",
|
||||
'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'ÿ': "y", 'ß': "ss",
|
||||
}
|
||||
Reference in new issue
Block a user