diff --git a/README.md b/README.md index 992ac72..e2c2693 100644 --- a/README.md +++ b/README.md @@ -144,16 +144,16 @@ Edits show instantly and save in the background. ## Importing a list -**Import** in the list editor adds entries from a file, after the ones already there and in the file's order: +**Import** in the list editor adds entries from a file. **Line numbers are places**: line 1 becomes #1, line 38 becomes #38, and a blank line leaves its place empty (entries already at those places move down to make room). -- **Text** (`.txt`, or paste): one title per line. Numbering like `1.` is dropped when every line has it, and a trailing `(1999)` is read as the year. For songs and albums, `Artist - Title` works either way round. -- **CSV / TSV**: a header row picks the columns: `Title` (or Name), `Year`, `Artist`/`Director`/`Developer`, `Note` (or Why), and `Rank` to set the order. Without a header the first column is the title. Canon's own export imports as-is. +- **Text** (`.txt`, or paste): one title per line. If every line is numbered (`38. Super Mario World`), the numbers are the places instead. A trailing `(1999)` is read as the year. For songs and albums, `Artist - Title` works either way round. +- **CSV / TSV**: a header row picks the columns: `Title` (or Name), `Year`, `Artist`/`Director`/`Developer`, `Note` (or Why), and `Rank` to give places directly. Without a header the first column is the title. The first row after the header is #1. Each entry is searched for and matched. Canon only adds a match on its own when it's certain: the title matches exactly and nothing else fits as well (a year or artist in the file helps). Everything else is left for you to check, with a recommended pick, the other results, **Add as written** to make a new entry from your text, **Skip**, and a box to search again. Custom lists search all of Wikipedia, so there a match is only certain when a year or subtitle confirms it. ## Exporting a list -Every list can be downloaded as its titles only, in rank order, one per line. There are no ranks, header row or other fields. +Every list can be downloaded as its titles only, one per line, where line N is place #N: an empty place is a blank line. There are no ranks, header row or other fields, and importing the file puts everything back where it was. - **Export → Titles as CSV** (`/l//export?format=csv`): a single column. It starts with a UTF-8 BOM so Excel shows accented titles correctly. - **Export → Titles as text** (`/l//export?format=txt`): plain lines. diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go index 98f4999..5c73f2a 100644 --- a/internal/importer/importer_test.go +++ b/internal/importer/importer_test.go @@ -1,6 +1,7 @@ package importer import ( + "fmt" "reflect" "testing" @@ -8,17 +9,28 @@ import ( ) func TestParseText(t *testing.T) { - got, err := Parse([]byte("1. Heat (1995)\n2) 2001: A Space Odyssey\n\n3 - Alien [1979]\n"), "list.txt") + got, err := Parse([]byte("1. Heat (1995)\n2) 2001: A Space Odyssey\n\n4 - Alien [1979]\n"), "list.txt") if err != nil { t.Fatal(err) } - want := []Row{{Title: "Heat", Year: "1995"}, {Title: "2001: A Space Odyssey"}, {Title: "Alien", Year: "1979"}} + want := []Row{{Place: 1, Title: "Heat", Year: "1995"}, {Place: 2, Title: "2001: A Space Odyssey"}, {Place: 4, Title: "Alien", Year: "1979"}} if !reflect.DeepEqual(got, want) { t.Fatalf("got %+v", got) } - // Not every line is numbered, so nothing is stripped. - got, _ = Parse([]byte("2001: A Space Odyssey\nHeat\n1917\n"), "x.txt") - if got[0].Title != "2001: A Space Odyssey" || got[2].Title != "1917" { + // Not every line is numbered, so nothing is stripped; line numbers are + // the places, and a blank line leaves one empty. + got, _ = Parse([]byte("2001: A Space Odyssey\nHeat\n\n1917\n"), "x.txt") + if got[0].Title != "2001: A Space Odyssey" || got[2].Title != "1917" || got[2].Place != 4 { + t.Fatalf("got %+v", got) + } + // Numbers win over line numbers when every line has one. + got, _ = Parse([]byte("1. Super Mario Sunshine\n38. Super Mario World\n"), "games.txt") + if got[0].Place != 1 || got[1].Place != 38 || got[1].Title != "Super Mario World" { + t.Fatalf("got %+v", got) + } + // A repeated number takes the next free place, keeping the file's order. + got, _ = Parse([]byte("3. A\n3. B\n1. C\n"), "x.txt") + if fmt.Sprint(got) != fmt.Sprint([]Row{{Place: 1, Title: "C"}, {Place: 3, Title: "A"}, {Place: 4, Title: "B"}}) { t.Fatalf("got %+v", got) } } @@ -30,20 +42,26 @@ func TestParseCSV(t *testing.T) { t.Fatal(err) } want := []Row{ - {Title: "Heat", Year: "1995", Subtitle: "Michael Mann"}, - {Title: "Alien", Year: "1979", Subtitle: "Ridley Scott", Note: "Tense, perfect"}, + {Place: 1, Title: "Heat", Year: "1995", Subtitle: "Michael Mann"}, + {Place: 2, Title: "Alien", Year: "1979", Subtitle: "Ridley Scott", Note: "Tense, perfect"}, } if !reflect.DeepEqual(got, want) { t.Fatalf("got %+v", got) } - // Canon's own export: one quoted column, no header. - got, _ = Parse([]byte("\xef\xbb\xbf\"Heat\"\r\n\"Crouching Tiger, Hidden Dragon\"\r\n"), "films.csv") - if len(got) != 2 || got[1].Title != "Crouching Tiger, Hidden Dragon" { + // Canon's own export: one column, no header, a blank line for an empty + // place. + got, _ = Parse([]byte("\xef\xbb\xbfHeat\r\n\r\n\"Crouching Tiger, Hidden Dragon\"\r\n"), "films.csv") + if len(got) != 2 || got[1].Title != "Crouching Tiger, Hidden Dragon" || got[1].Place != 3 { + t.Fatalf("got %+v", got) + } + // With a header, the first row after it is #1. + got, _ = Parse([]byte("Title\nHeat\n\nAlien\n"), "x.csv") + if got[0].Place != 1 || got[1].Place != 3 { t.Fatalf("got %+v", got) } // Headerless rank,title,year. - got, _ = Parse([]byte("1,Heat,1995\n2,Alien,1979\n"), "x.csv") - if got[1] != (Row{Title: "Alien", Year: "1979"}) { + got, _ = Parse([]byte("1,Heat,1995\n5,Alien,1979\n"), "x.csv") + if got[1] != (Row{Place: 5, Title: "Alien", Year: "1979"}) { t.Fatalf("got %+v", got) } if _, err := Parse([]byte("\n \n"), "x.txt"); err != ErrEmpty { diff --git a/internal/importer/parse.go b/internal/importer/parse.go index be12b00..5eb1b9c 100644 --- a/internal/importer/parse.go +++ b/internal/importer/parse.go @@ -6,6 +6,7 @@ import ( "bytes" "encoding/csv" "errors" + "io" "regexp" "slices" "strconv" @@ -16,8 +17,12 @@ import ( // MaxRows caps one import. const MaxRows = 2000 -// Row is one entry read from the file, in the file's order. +// Row is one entry read from the file. type Row struct { + // Place is where it goes on the list: its line number (blank lines + // leave a place empty), or the number it was given ("38. Title" in a + // numbered list, or a Rank column). + Place int `json:"place"` Title string `json:"title"` Year string `json:"year,omitempty"` Subtitle string `json:"subtitle,omitempty"` // artist, director, developer… @@ -50,12 +55,27 @@ func Parse(data []byte, filename string) ([]Row, error) { if len(rows) == 0 { return nil, ErrEmpty } + rows = orderPlaces(rows) if len(rows) > MaxRows { rows = rows[:MaxRows] } return rows, nil } +// orderPlaces sorts rows by place and gives a repeated place (two rows both +// numbered 3) the next free one, keeping the file's order. +func orderPlaces(rows []Row) []Row { + slices.SortStableFunc(rows, func(a, b Row) int { return a.Place - b.Place }) + prev := 0 + for i := range rows { + if rows[i].Place <= prev { + rows[i].Place = prev + 1 + } + prev = rows[i].Place + } + return rows +} + // latin1 decodes bytes as Windows-1252/Latin-1, which is what spreadsheet // exports that aren't UTF-8 usually are. func latin1(b []byte) string { @@ -84,27 +104,35 @@ var numbered = regexp.MustCompile(`^\s*#?\d{1,5}\s*[.):\-–—]?\s+`) var trailingYear = regexp.MustCompile(`\s*[(\[]((?:1[5-9]|20)\d\d)[)\]]\s*$`) func parseLines(text string) []Row { - var lines []string - for _, l := range strings.Split(text, "\n") { + lines := strings.Split(text, "\n") + // Only use numbering when every non-blank line has it, so a list that + // merely contains "2001: A Space Odyssey" keeps its title. + numberedAll, any := true, false + for _, l := range lines { if l = strings.TrimSpace(l); l != "" { - lines = append(lines, l) + any = true + if !numbered.MatchString(l) { + numberedAll = false + break + } } } - // Only strip numbering when every line has it, so a list that merely - // contains "2001: A Space Odyssey" keeps its title. - strip := len(lines) > 0 - for _, l := range lines { - if !numbered.MatchString(l) { - strip = false - break + numberedAll = numberedAll && any + var rows []Row + for i, l := range lines { + l = strings.TrimSpace(l) + if l == "" { + continue // a blank line is an empty place } - } - rows := make([]Row, 0, len(lines)) - for _, l := range lines { - if strip { + place := i + 1 + if numberedAll { + if n, err := strconv.Atoi(strings.Trim(numbered.FindString(l), " \t#.):-–—")); err == nil && n > 0 { + place = n + } l = numbered.ReplaceAllString(l, "") } if r := rowFromTitle(l); r.Title != "" { + r.Place = place rows = append(rows, r) } } @@ -140,18 +168,28 @@ func parseDelimited(text string, delim rune) ([]Row, error) { cr.FieldsPerRecord = -1 cr.LazyQuotes = true cr.TrimLeadingSpace = true - records, err := cr.ReadAll() - if err != nil { - return nil, errors.New("that CSV file couldn't be read: " + err.Error()) + // The reader skips blank lines, so keep each record's line number: a + // blank line is an empty place. + var records [][]string + var lines []int + for { + rec, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, errors.New("that CSV file couldn't be read: " + err.Error()) + } + line, _ := cr.FieldPos(0) + records = append(records, rec) + lines = append(lines, line) } - records = slices.DeleteFunc(records, func(rec []string) bool { - return strings.TrimSpace(strings.Join(rec, "")) == "" - }) if len(records) == 0 { return nil, nil } cols := map[string]int{} + first := 1 // the line place #1 is on if head := records[0]; isHeader(head) { for i, name := range head { if role, ok := headerNames[strings.ToLower(strings.TrimSpace(name))]; ok { @@ -160,7 +198,8 @@ func parseDelimited(text string, delim rune) ([]Row, error) { } } } - records = records[1:] + first = lines[0] + 1 + records, lines = records[1:], lines[1:] } else { cols = guessColumns(records) } @@ -168,11 +207,6 @@ func parseDelimited(text string, delim rune) ([]Row, error) { cols["title"] = 0 } - type ranked struct { - Row - rank int - } - var out []ranked cell := func(rec []string, role string) string { i, ok := cols[role] if !ok || i >= len(rec) { @@ -180,6 +214,7 @@ func parseDelimited(text string, delim rune) ([]Row, error) { } return strings.TrimSpace(rec[i]) } + var rows []Row for i, rec := range records { r := Row{Title: cell(rec, "title"), Subtitle: cell(rec, "subtitle"), Note: cell(rec, "note")} if y := yearIn(cell(rec, "year")); y != "" { @@ -191,18 +226,11 @@ func parseDelimited(text string, delim rune) ([]Row, error) { if r.Title == "" { continue } - rank, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#")) - if err != nil { - rank = 1 << 30 + r.Place = lines[i] - first + 1 + if n, err := strconv.Atoi(strings.TrimPrefix(cell(rec, "rank"), "#")); err == nil && n > 0 { + r.Place = n } - out = append(out, ranked{r, rank*MaxRows*2 + i}) // stable within equal ranks - } - if _, ok := cols["rank"]; ok { - slices.SortStableFunc(out, func(a, b ranked) int { return a.rank - b.rank }) - } - rows := make([]Row, len(out)) - for i, r := range out { - rows[i] = r.Row + rows = append(rows, r) } return rows, nil } diff --git a/internal/server/export.go b/internal/server/export.go index a29e4be..e0ca027 100644 --- a/internal/server/export.go +++ b/internal/server/export.go @@ -14,8 +14,9 @@ var exportFormats = map[string]struct{ ext, contentType string }{ "txt": {"txt", "text/plain; charset=utf-8"}, } -// handleExport downloads a list's titles in rank order, one per line, with -// nothing else: no ranks, header row or metadata. +// handleExport downloads a list's titles, one per line, with nothing else: +// no ranks, header row or metadata. Line N is place #N, so an empty place +// is a blank line, and importing the file puts everything back where it was. func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) { format := r.URL.Query().Get("format") if format == "" { @@ -35,9 +36,12 @@ func (s *Server) handleExport(w http.ResponseWriter, r *http.Request) { s.serverError(w, r, err) return } - titles := make([]string, len(items)) - for i, it := range items { - titles[i] = it.Title + var titles []string + for _, it := range items { + for len(titles) < it.Rank-1 { + titles = append(titles, "") + } + titles = append(titles, it.Title) } w.Header().Set("Content-Type", f.contentType) w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.%s"`, l.Slug, f.ext)) diff --git a/internal/server/server.go b/internal/server/server.go index c66fb64..895e4ec 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -321,20 +321,45 @@ func (s *Server) isHTTPS(r *http.Request) bool { func (s *Server) staticHandler() http.Handler { fsrv := http.FileServerFS(s.static) + etags := staticETags(s.static) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/") { s.notFound(w, r) return } + if tag, ok := etags[strings.TrimPrefix(r.URL.Path, "/")]; ok { + w.Header().Set("ETag", tag) // FileServer answers If-None-Match with 304 + } if r.URL.Query().Get("v") != "" { w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") } else { - w.Header().Set("Cache-Control", "public, max-age=3600") + // Scripts load each other without a ?v= (import "./places.js"), + // so unversioned files are re-checked on every load; the ETag + // makes that a quick 304 until a new build changes them. + w.Header().Set("Cache-Control", "no-cache") } fsrv.ServeHTTP(w, r) }) } +// staticETags fingerprints every embedded static file once at startup. +func staticETags(fsys fs.FS) map[string]string { + tags := map[string]string{} + fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + b, err := fs.ReadFile(fsys, p) + if err != nil { + return nil + } + sum := sha256.Sum256(b) + tags[p] = `"` + hex.EncodeToString(sum[:8]) + `"` + return nil + }) + return tags +} + func (s *Server) handleImage(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") if !imgcache.ValidName(name) { diff --git a/web/static/css/app.css b/web/static/css/app.css index 2da24e5..2d1417e 100644 --- a/web/static/css/app.css +++ b/web/static/css/app.css @@ -889,12 +889,12 @@ input[type="checkbox"] { accent-color: var(--accent); width: 18px; height: 18px; .import-row { border: 1px solid var(--border); border-radius: var(--radius); padding: 10px 12px; display: flex; flex-direction: column; gap: 10px; background: var(--surface); } .import-row.is-review, .import-row.is-none, .import-row.is-error { border-color: color-mix(in srgb, var(--warn) 45%, var(--border)); } .import-row-head { display: flex; align-items: center; gap: 10px; min-width: 0; } -.import-num { color: var(--faint); font-size: 13px; font-variant-numeric: tabular-nums; min-width: 2.5ch; text-align: right; } +.import-num { color: var(--faint); font-size: 13px; font-variant-numeric: tabular-nums; min-width: 4.5ch; text-align: right; } .import-src { flex: 1; min-width: 0; font-size: 14px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .import-src b { font-weight: 600; } .badge--ok { background: color-mix(in srgb, var(--success) 16%, transparent); color: var(--success); } .badge .icon { width: 12px; height: 12px; } -.import-chosen { display: flex; align-items: center; gap: 10px; justify-content: space-between; padding-left: calc(2.5ch + 10px); } +.import-chosen { display: flex; align-items: center; gap: 10px; justify-content: space-between; padding-left: calc(4.5ch + 10px); } .import-card { display: flex; align-items: center; gap: 10px; min-width: 0; } .import-card .result-art { width: 38px; flex: none; } .import-card-text { display: flex; flex-direction: column; gap: 2px; min-width: 0; } diff --git a/web/static/js/import.js b/web/static/js/import.js index d7334a2..2145abc 100644 --- a/web/static/js/import.js +++ b/web/static/js/import.js @@ -32,7 +32,7 @@ export function openImport({ list, cat, onListChange, onDone }) { dialogHead("Import a list", () => dlg), h("div", { class: "dialog-body stack" }, h("p", { class: "muted" }, - `Entries are added after your last one, in the file's order. Use one title per line, or a CSV with columns like Title, Year, ${artistWord(cat)} and Note. `, + `Line 1 becomes #1, line 38 becomes #38, and a blank line leaves its place empty${list.itemCount ? " (entries already at those places move down to make room)" : ""}. Use one title per line, or a CSV with columns like Title, Year, ${artistWord(cat)} and Note. `, "Each one is looked up and matched for you; anything Canon isn't sure about is left for you to check."), drop, text, @@ -117,14 +117,18 @@ export function openImport({ list, cat, onListChange, onDone }) { ].filter(Boolean); summary.textContent = parts.join(" · "); - const n = chosen().length; - const free = list.maxItems - list.itemCount; - go.disabled = matching || n === 0 || n > free; + const picked = chosen(); + const n = picked.length; + const lastPlace = Math.max(0, ...picked.map((e) => e.row.place)); + const fits = list.itemCount + n <= list.maxItems && lastPlace <= list.maxItems; + go.disabled = matching || n === 0 || !fits; go.textContent = matching ? "Matching…" : `Import ${fmt.format(n)} ${n === 1 ? "entry" : "entries"}`; - room.hidden = matching || n <= free; + room.hidden = matching || n === 0 || fits; if (!room.hidden) { - const want = Math.min(list.itemCount + n, 65535); - room.replaceChildren(icon("info"), `This list has room for ${fmt.format(free)} more. `, + const want = Math.min(Math.max(list.itemCount + n, lastPlace), 65535); + room.replaceChildren(icon("info"), lastPlace > list.maxItems + ? `The file goes up to #${fmt.format(lastPlace)}, but this list ends at #${fmt.format(list.maxItems)}. ` + : `This list has room for ${fmt.format(list.maxItems - list.itemCount)} more. `, h("button", { type: "button", class: "btn btn--sm", onclick: async (e) => { busy(e.currentTarget, true); @@ -163,7 +167,7 @@ export function openImport({ list, cat, onListChange, onDone }) { const li = h("li", { class: `import-row is-${e.state}`, "data-i": e.i }, h("div", { class: "import-row-head" }, - h("span", { class: "import-num" }, `${e.i + 1}.`), + h("span", { class: "import-num" }, `#${row.place}`), h("div", { class: "import-src" }, h("b", {}, row.title), asWritten && h("span", { class: "muted small" }, ` ${asWritten}`)), badge), ); @@ -299,9 +303,9 @@ export function openImport({ list, cat, onListChange, onDone }) { ? { title: r.title, subtitle: r.subtitle, year: r.year, externalUrl: r.externalUrl, imageUrl: r.imageUrl, provider: r.provider, providerId: r.providerId, extra: { ...(r.extra || {}) }, description: r.overview || "", - details: true, note: e.row.note || "", + details: true, note: e.row.note || "", rank: e.row.place, } - : { title: e.row.title, subtitle: e.row.subtitle || "", year: e.row.year || "", note: e.row.note || "", extra: {} }; + : { title: e.row.title, subtitle: e.row.subtitle || "", year: e.row.year || "", note: e.row.note || "", extra: {}, rank: e.row.place }; try { const res = await api("POST", `/api/lists/${list.id}/items`, payload); added++;