canon/internal/importer/parse.go

// Package importer reads a list of titles from a CSV or text file and
// matches each one to a search result.
package importer

import (
	"bytes"
	"encoding/csv"
	"errors"
	"io"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"unicode/utf8"
)

// MaxRows caps one import.
const MaxRows = 2000

// Row is one entry read from the file.
type Row struct {
	// Place is where it goes on the list: its line number (blank lines
	// leave a place empty), or the number it was given ("38. Title" in a
	// numbered list, or a Rank column).
	Place    int    `json:"place"`
	Title    string `json:"title"`
	Year     string `json:"year,omitempty"`
	Subtitle string `json:"subtitle,omitempty"` // artist, director, developer…
	Note     string `json:"note,omitempty"`
}

var ErrEmpty = errors.New("no titles found in that file")

// Parse reads rows from data. CSV and TSV files may have a header row
// naming their columns; text files hold one title per line.
func Parse(data []byte, filename string) ([]Row, error) {
	data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
	if !utf8.Valid(data) {
		data = []byte(strings.ToValidUTF8(latin1(data), "�"))
	}
	text := strings.ReplaceAll(string(data), "\r\n", "\n")
	var rows []Row
	var err error
	switch ext := strings.ToLower(filename); {
	case strings.HasSuffix(ext, ".csv"):
		rows, err = parseDelimited(text, sniffDelimiter(text))
	case strings.HasSuffix(ext, ".tsv"):
		rows, err = parseDelimited(text, '\t')
	default:
		rows = parseLines(text)
	}
	if err != nil {
		return nil, err
	}
	if len(rows) == 0 {
		return nil, ErrEmpty
	}
	rows = orderPlaces(rows)
	if len(rows) > MaxRows {
		rows = rows[:MaxRows]
	}
	return rows, nil
}

// orderPlaces sorts rows by place and gives a repeated place (two rows both
// numbered 3) the next free one, keeping the file's order.
func orderPlaces(rows []Row) []Row {
	slices.SortStableFunc(rows, func(a, b Row) int { return a.Place - b.Place })
	prev := 0
	for i := range rows {
		if rows[i].Place <= prev {
			rows[i].Place = prev + 1
		}
		prev = rows[i].Place
	}
	return rows
}

// latin1 decodes bytes as Windows-1252/Latin-1, which is what spreadsheet
// exports that aren't UTF-8 usually are.
func latin1(b []byte) string {
	r := make([]rune, len(b))
	for i, c := range b {
		r[i] = rune(c)
	}
	return string(r)
}

func sniffDelimiter(text string) rune {
	first, _, _ := strings.Cut(text, "\n")
	best, n := ',', strings.Count(first, ",")
	for _, d := range []rune{'\t', ';'} {
		if c := strings.Count(first, string(d)); c > n {
			best, n = d, c
		}
	}
	return best
}

// numbered matches a leading rank: "1. ", "#2 ", "3) ", "4 - ".
var numbered = regexp.MustCompile(`^\s*#?\d{1,5}\s*[.):\-–—]?\s+`)

// trailingYear matches "Heat (1995)" or "Heat [1995]".
var trailingYear = regexp.MustCompile(`\s*[(\[]((?:1[5-9]|20)\d\d)[)\]]\s*$`)

func parseLines(text string) []Row {
	lines := strings.Split(text, "\n")
	// Only use numbering when every non-blank line has it, so a list that
	// merely contains "2001: A Space Odyssey" keeps its title.
	numberedAll, any := true, false
	for _, l := range lines {
		if l = strings.TrimSpace(l); l != "" {
			any = true
			if !numbered.MatchString(l) {
				numberedAll = false
				break
			}
		}
	}
	numberedAll = numberedAll && any
	var rows []Row
	for i, l := range lines {
		l = strings.TrimSpace(l)
		if l == "" {
			continue // a blank line is an empty place
		}
		place := i + 1
		if numberedAll {
			if n, err := strconv.Atoi(strings.Trim(numbered.FindString(l), " \t#.):-–—")); err == nil && n > 0 {
				place = n
			}
			l = numbered.ReplaceAllString(l, "")
		}
		if r := rowFromTitle(l); r.Title != "" {
			r.Place = place
			rows = append(rows, r)
		}
	}
	return rows
}

func rowFromTitle(s string) Row {
	s = strings.TrimSpace(s)
	r := Row{Title: s}
	if m := trailingYear.FindStringSubmatchIndex(s); m != nil {
		r.Title = strings.TrimSpace(s[:m[0]])
		r.Year = s[m[2]:m[3]]
	}
	if r.Title == "" {
		r.Title = s
	}
	return r
}

var headerNames = map[string]string{
	"title": "title", "name": "title", "film": "title", "movie": "title", "show": "title",
	"album": "title", "song": "title", "track": "title", "game": "title", "item": "title",
	"year": "year", "release year": "year", "released": "year", "date": "year",
	"artist": "subtitle", "director": "subtitle", "creator": "subtitle", "developer": "subtitle",
	"author": "subtitle", "by": "subtitle", "subtitle": "subtitle", "network": "subtitle",
	"note": "note", "notes": "note", "why": "note", "comment": "note", "comments": "note", "review": "note",
	"rank": "rank", "#": "rank", "position": "rank", "no": "rank", "no.": "rank",
}

func parseDelimited(text string, delim rune) ([]Row, error) {
	cr := csv.NewReader(strings.NewReader(text))
	cr.Comma = delim
	cr.FieldsPerRecord = -1
	cr.LazyQuotes = true
	cr.TrimLeadingSpace = true
	// The reader skips blank lines, so keep each record's line number: a
	// blank line is an empty place.
	var records [][]string
	var lines []int
	for {
		rec, err := cr.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, errors.New("that CSV file couldn't be read: " + err.Error())
		}
		line, _ := cr.FieldPos(0)
		records = append(records, rec)
		lines = append(lines, line)
	}
	if len(records) == 0 {
		return nil, nil
	}

	cols := map[string]int{}
	first := 1 // the line place #1 is on
	if head := records[0]; isHeader(head) {
		for i, name := range head {
			if role, ok := headerNames[strings.ToLower(strings.TrimSpace(name))]; ok {
				if _, taken := cols[role]; !taken {
					cols[role] = i
				}
			}
		}
		first = lines[0] + 1
		records, lines = records[1:], lines[1:]
	} else {
		cols = guessColumns(records)
	}
	if _, ok := cols["title"]; !ok {
		cols["title"] = 0
	}

	cell := func(rec []string, role string) string {
		i, ok := cols[role]
		if !ok || i >= len(rec) {
			return ""
		}
		return strings.TrimSpace(rec[i])
	}
	var rows []Row
	for i, rec := range records {
		r := Row{Title: cell(rec, "title"), Subtitle: cell(rec, "subtitle"), Note: cell(rec, "note")}
		if y := yearIn(cell(rec, "year")); y != "" {
			r.Year = y
		} else {
			t := rowFromTitle(r.Title)
			r.Title, r.Year = t.Title, t.Year
		}
		if r.Title == "" {
			continue
		}
		r.Place = lines[i] - first + 1
		if n, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#")); err == nil && n > 0 {
			r.Place = n
		}
		rows = append(rows, r)
	}
	return rows, nil
}

func isHeader(rec []string) bool {
	for _, c := range rec {
		if headerNames[strings.ToLower(strings.TrimSpace(c))] == "title" {
			return true
		}
	}
	return false
}

var yearOnly = regexp.MustCompile(`\b((?:1[5-9]|20)\d\d)\b`)

func yearIn(s string) string { return yearOnly.FindString(s) }

// guessColumns handles headerless files: "1,Heat,1995" or "Heat,1995".
func guessColumns(records [][]string) map[string]int {
	cols := map[string]int{}
	all := func(col int, ok func(string) bool) bool {
		for _, rec := range records {
			if col >= len(rec) || !ok(strings.TrimSpace(rec[col])) {
				return false
			}
		}
		return true
	}
	isInt := func(s string) bool { _, err := strconv.Atoi(strings.TrimPrefix(s, "#")); return err == nil }
	isYear := func(s string) bool { return s == yearIn(s) && s != "" }
	title := 0
	width := len(records[0])
	if width >= 2 && all(0, isInt) && !all(1, isInt) {
		cols["rank"] = 0
		title = 1
	}
	cols["title"] = title
	if title+1 < width && all(title+1, isYear) {
		cols["year"] = title + 1
	}
	return cols
}