Compare commits
39
Commits
e82f7b0d13
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddc98788c6 | ||
|
|
682fc4a6fb | ||
|
|
08b15c42b4 | ||
|
|
03f0a454be | ||
|
|
bf6a8d0129 | ||
|
|
7851d8e231 | ||
|
|
5abf062c24 | ||
|
|
9e521ddaef | ||
|
|
50551114cd | ||
|
|
13c81440ef | ||
|
|
248038ec97 | ||
|
|
78fc38e99d | ||
|
|
8d0516bedc | ||
|
|
e5ca78ba41 | ||
|
|
3c4f8de3a4 | ||
|
|
dd91b83c27 | ||
|
|
815e7307d1 | ||
|
|
9af6fedbe0 | ||
|
|
6016745297 | ||
|
|
8a355b637f | ||
|
|
8889a4e351 | ||
|
|
0f17480be7 | ||
|
|
525ddfd88d | ||
|
|
f1b1e7fd98 | ||
|
|
ce9b629115 | ||
|
|
36424784af | ||
|
|
cd2bbe8840 | ||
|
|
8b56a65bad | ||
|
|
1089c29179 | ||
|
|
7063b835db | ||
|
|
57ba47a5a0 | ||
|
|
861ae6cb5e | ||
|
|
af70da1493 | ||
|
|
055b1df769 | ||
|
|
22cfb34788 | ||
|
|
154731a003 | ||
|
|
a78e44f56e | ||
|
|
2d857e3e10 | ||
|
|
f0c0927ab8 |
No files matched your search
Generated
+965
File diff suppressed because it is too large.
Load diff
@@ -6,11 +6,13 @@ edition = "2024"
|
||||
[dependencies]
|
||||
ammonia = "4.1.4"
|
||||
anyhow = "1.0.104"
|
||||
arborium = { version = "2.18.2", default-features = false, features = ["lang-asm", "lang-awk", "lang-bash", "lang-c", "lang-c-sharp", "lang-clojure", "lang-cmake", "lang-commonlisp", "lang-cpp", "lang-css", "lang-d", "lang-dart", "lang-diff", "lang-dockerfile", "lang-elisp", "lang-elixir", "lang-elm", "lang-erlang", "lang-fish", "lang-fsharp", "lang-gleam", "lang-glsl", "lang-go", "lang-graphql", "lang-groovy", "lang-haskell", "lang-hcl", "lang-html", "lang-ini", "lang-java", "lang-javascript", "lang-jq", "lang-json", "lang-julia", "lang-kotlin", "lang-lean", "lang-lua", "lang-make", "lang-matlab", "lang-nix", "lang-objc", "lang-ocaml", "lang-odin", "lang-perl", "lang-php", "lang-powershell", "lang-prolog", "lang-proto", "lang-python", "lang-r", "lang-ruby", "lang-rust", "lang-scala", "lang-scheme", "lang-scss", "lang-solidity", "lang-sql", "lang-starlark", "lang-swift", "lang-toml", "lang-tsx", "lang-typescript", "lang-verilog", "lang-vhdl", "lang-vim", "lang-x86asm", "lang-xml", "lang-yaml", "lang-zig", "lang-zsh"] }
|
||||
askama = "0.16.0"
|
||||
async-trait = "0.1.92"
|
||||
axum = "0.8.9"
|
||||
axum-login = { git = "https://github.com/maxcountryman/axum-login.git", rev = "151c72d7a1b4646830f86b4332e6bd6e34d719a7", version = "0.18.0" }
|
||||
base64 = "0.23.1"
|
||||
betlang = "0.1.1"
|
||||
clap = { version = "4.6.6", features = ["derive"] }
|
||||
dom_smoothie = "0.18.0"
|
||||
epub-builder = "0.8.3"
|
||||
|
||||
@@ -8,8 +8,8 @@ the articles, enriches them with HackerNews/Lobsters/Reddit social proof, has
|
||||
DeepSeek triage every eligible opening and closely assess a 120-article union,
|
||||
then utility-ranks and diversity-caps a 60-item shortlist for Claude Opus 5 — the editor —
|
||||
which assembles the issue
|
||||
(no minimum size, a hard ceiling), writes a one-line *why* under every headline,
|
||||
the summaries and *The Brief*. It assembles two EPUB editions (a standard one
|
||||
(no minimum size, a hard ceiling), then writes the summaries and *The Brief*.
|
||||
It assembles two EPUB editions (a standard one
|
||||
and one tuned for the Xteink X4 e-ink reader), converts the X4 edition to XTC, and
|
||||
publishes the lot over its own OPDS catalog — which doubles as a
|
||||
[BookOrbit](https://github.com/thallada/bookorbit) watched folder if you run one.
|
||||
@@ -99,8 +99,7 @@ enabled on every request to an `anthropic`-kind provider.
|
||||
| **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node <repo>/cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. |
|
||||
| A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. |
|
||||
|
||||
`data/profile.md` is the hand-maintained reader profile; its optional interests are merged
|
||||
with `data/scour-interests.opml`. Both paths are configurable.
|
||||
`data/profile.md` is the hand-maintained reader profile; its path is configurable.
|
||||
|
||||
---
|
||||
|
||||
@@ -136,6 +135,8 @@ daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes]
|
||||
daily-epub features prune # stale embeddings, old telemetry and assessments
|
||||
daily-epub backfill-social [--days 7] # re-poll social scores for recent articles
|
||||
daily-epub feeds discover [--days 14] [--limit 50] # seed feed candidates from recent aggregator articles
|
||||
daily-epub interests import [--opml PATH] [--profile PATH] # one-time standing-interest import
|
||||
daily-epub interests backfill # match compatible cached article embeddings to interests
|
||||
daily-epub db migrate # run migrations (also automatic on every start)
|
||||
daily-epub config check # validate the config, print the resolved roles, keys and paths
|
||||
daily-epub users add USER [--admin] [--password-stdin]
|
||||
@@ -206,14 +207,17 @@ the database or takes the lock, so it is safe to run next to a live `generate`.
|
||||
## Web site and dashboard
|
||||
|
||||
The server is both the public newspaper index and the private operator UI. An
|
||||
anonymous visitor sees titles, authors, sources, metadata, AI summaries, why
|
||||
lines and outbound comment links; article bodies, the Brief, the World Briefing
|
||||
anonymous visitor sees titles, authors, sources, metadata, AI summaries and
|
||||
outbound comment links; article bodies, the Brief, the World Briefing
|
||||
and comments stay private. A signed-in `user`
|
||||
sees complete issues and article chapters and can download available formats
|
||||
from a single download menu. An
|
||||
`admin` can additionally follow direct dashboard links from issue entries and
|
||||
article chapters, rate articles, and use every `/dashboard/*` page, including
|
||||
settings and jobs. Personalization is shared across accounts for now.
|
||||
settings and jobs. `/dashboard/interests` lists the standing interests with
|
||||
their LLM-assigned category and a rating-derived weight, adds or deletes them,
|
||||
and links every interest name in the UI to the articles it matched.
|
||||
Personalization is shared across accounts for now.
|
||||
|
||||
Visitors can request an account with their preferred username at
|
||||
`/request-access`; admins review open requests on `/dashboard/users`. Approving
|
||||
@@ -249,6 +253,7 @@ Set `server.jobs_enabled = false` to make starts unavailable.
|
||||
| `backfill-social` | Refresh recent social scores. |
|
||||
| `features-prune` | Remove stale embeddings and curation telemetry. |
|
||||
| `import-ratings` | Fetch, embed, and rate URLs queued from the Ratings page. |
|
||||
| `interests-categorize` | File uncategorized interests under categories with the bulk model. |
|
||||
|
||||
The Ratings page accepts up to 500 historical article URLs at a time with one
|
||||
verdict and optional note. Imports run in the background and show per-URL
|
||||
@@ -285,7 +290,7 @@ articles do not yet have embeddings to compare.
|
||||
|
||||
| Route | Access | Purpose |
|
||||
|---|---|---|
|
||||
| `GET /`, `/issues`, `/issues/{date}`, `/feed.xml` | Public | Latest issue, archive, stripped issue index, and equivalent Atom feed. Signed-in issue views expand to the complete issue. |
|
||||
| `GET /`, `/issues`, `/issues/{date}`, `/feed.xml`, `/feed.json` | Public | Latest issue, archive, stripped issue index, and the equivalent Atom and JSON Feed 1.1 feeds. Signed-in issue views expand to the complete issue. |
|
||||
| `GET /issues/{date}/articles/{id}`, `/world`, `/behind` | User or admin | Private article, World Briefing, and Behind the paper chapters. |
|
||||
| `GET /issues/{date}/read` | User or admin | Open the Standard edition in BookOrbit's web reader when the integration is enabled. |
|
||||
| `GET /robots.txt`, `/static/{file}` | Public | Crawler policy and embedded CSS, JavaScript, and favicon. |
|
||||
@@ -366,7 +371,6 @@ prints what resolved.
|
||||
| `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. |
|
||||
| `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. |
|
||||
| `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. |
|
||||
| `interests_opml` | `data/scour-interests.opml` | Scour interests merged with the profile interests. |
|
||||
| `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). |
|
||||
| `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. |
|
||||
| `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. |
|
||||
@@ -409,6 +413,12 @@ prints what resolved.
|
||||
| `curation.recent_rejection_days` | `7` | Churn window for recent low triage/deep assessments. |
|
||||
| `curation.recent_rejection_floor` | `3.0` | Scores below this floor are excluded during the churn window (except auto-includes). |
|
||||
| `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. |
|
||||
| `curation.ranking.affinity_floor` / `affinity_full` | `15` / `40` | Interest-attributable ratings where affinity starts and reaches full weight. |
|
||||
| `curation.ranking.weights.preliminary.affinity` | `0.10` | Rating-derived interest affinity in the preliminary blend. |
|
||||
| `curation.ranking.weights.preliminary.interest` | `0.30` | Interest similarity in the preliminary blend. |
|
||||
| `curation.ranking.weights.preliminary.social` | `0.05` | Social signal in the preliminary blend. |
|
||||
| `curation.ranking.weights.utility.affinity` | `0.05` | Rating-derived interest affinity in the utility score. |
|
||||
| `curation.ranking.weights.utility.knn` | `0.10` | Rated-neighbour preference in the utility score. |
|
||||
| `editorial.summary_model` | `editor` | Which `[llm]` role writes the per-article summaries: `editor` (with per-article bulk fallback) or `bulk`. |
|
||||
| `editorial.summary_input_tokens` | `3000` | Article text offered to the summary prompt. |
|
||||
| `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. |
|
||||
@@ -448,7 +458,9 @@ prints what resolved.
|
||||
gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to
|
||||
`knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from
|
||||
`feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the
|
||||
signal is absent. Ratings decay with `rating_half_life_days` (60) over
|
||||
signal is absent. `affinity` (rating-derived interest affinity) likewise ramps
|
||||
from `affinity_floor` (15) to `affinity_full` (40) interest-attributable ratings.
|
||||
Ratings decay with `rating_half_life_days` (60) over
|
||||
`rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and
|
||||
`negative_coefficient` (0.75) shape the signal. `slop_author_penalty` (0.75)
|
||||
is the fraction of the blend and utility removed from every candidate whose
|
||||
@@ -457,13 +469,13 @@ limit. `triage_max` (800),
|
||||
`deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3),
|
||||
`semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]`
|
||||
(`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]`
|
||||
(`quality` 0.40 · `fit` 0.20 · `knn` 0.15 · `interest` 0.10 · `feed` 0.05 ·
|
||||
(`quality` 0.40 · `fit` 0.20 · `knn` 0.10 · `affinity` 0.05 · `interest` 0.10 · `feed` 0.05 ·
|
||||
`triage` 0.05 · `social` 0.03 · `heuristic` 0.02, over the signals present for
|
||||
each article of the deep set) and `[curation.ranking.diversity]`
|
||||
(`cluster_threshold` 0.85, `per_cluster_cap` 2, `utility_protected` 10) drive
|
||||
the LLM triage, deep assessment, utility ranking and diversification stages.
|
||||
`[curation.ranking.weights.preliminary]` (`interest` 0.35 · `knn` 0.25 ·
|
||||
`heuristic` 0.20 · `feed` 0.10 · `social` 0.10) blends the cheap signals; weights
|
||||
`[curation.ranking.weights.preliminary]` (`interest` 0.30 · `knn` 0.25 ·
|
||||
`affinity` 0.10 · `heuristic` 0.20 · `feed` 0.10 · `social` 0.05) blends the cheap signals; weights
|
||||
are renormalized over the signals present for each article, so they need not sum
|
||||
to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are
|
||||
what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥
|
||||
@@ -709,7 +721,7 @@ toggle and nothing else. The whole matrix:
|
||||
|
||||
| route | `Cache-Control` |
|
||||
|---|---|
|
||||
| `/`, `/issues`, `/issues/{date}`, `/feed.xml`, `/issues.json` (anonymous) | `public, max-age=300` |
|
||||
| `/`, `/issues`, `/issues/{date}`, `/feed.xml`, `/feed.json`, `/issues.json` (anonymous) | `public, max-age=300` |
|
||||
| the same pages with a `daily_session=` cookie | `private, no-store` |
|
||||
| `/robots.txt` | `public, max-age=86400` |
|
||||
| `/static/*?v=<hash>` | `public, max-age=31536000, immutable` |
|
||||
@@ -843,7 +855,7 @@ epubcheck "./out/The Daily EPUB - $(date +%F).epub" # expect zero errors
|
||||
|
||||
# 4. Now with the bulk and editor providers, still not publishing
|
||||
daily-epub generate --dry-run --out ./out --max-articles 6
|
||||
# → check the lineup is sane (at most 6 picks, each with a "why" line) and the
|
||||
# → check the lineup is sane (at most 6 picks) and the
|
||||
# printed per-provider cost is well under $1
|
||||
|
||||
# 5. Full live run
|
||||
@@ -923,6 +935,7 @@ images/ article images comments.rs discussion chapters
|
||||
encode re-encode publish.rs BookOrbit + XTC
|
||||
embed into the page server.rs ratings, OPDS
|
||||
html.rs markup helpers db.rs SQLite
|
||||
interests.rs standing interests and stored article matches
|
||||
lock.rs one writer at a time (flock on <database_path>.lock)
|
||||
```
|
||||
|
||||
@@ -992,11 +1005,12 @@ From spec §7, plus what implementation turned up:
|
||||
ceiling; a protocol that is neither means another impl. Voyage AI embeddings
|
||||
sit behind the analogous `EmbeddingBackend` trait in `curate/embedding.rs`.
|
||||
- **Triage and union admission replace the heuristic gate.** Every eligible
|
||||
article gets interest, rated-neighbour, feed-affinity, social and heuristic
|
||||
signals, then DeepSeek reads its opening (up to `triage_max`). The deep set is
|
||||
article gets interest, rated-neighbour, feed-affinity, interest-affinity,
|
||||
social and heuristic signals, then DeepSeek reads its opening (up to
|
||||
`triage_max`). The deep set is
|
||||
the union of triage, interest, neighbour, exploration, blend and auto-include
|
||||
retrievers. `explain` shows the assessment and `admitted_by`. Learned signals
|
||||
stay absent until their gates open (8 and 15 ratings respectively).
|
||||
stay absent until their gates open (8, 15, and 15 ratings respectively).
|
||||
- **Deep assessment and diversity are live.** DeepSeek reads a representative
|
||||
beginning/middle/end sample, separates editorial quality from reader fit, and
|
||||
records descriptive facets. Utility is normalized over the deep set; embedding
|
||||
|
||||
+10
-7
@@ -24,9 +24,8 @@ database_path = "/var/lib/daily-epub/daily-epub.db"
|
||||
# Default output directory for generated artifacts (overridden by `--out`).
|
||||
out_dir = "/var/lib/daily-epub/out"
|
||||
|
||||
# Hand-maintained reader profile and Scour interests merged into the system prompt.
|
||||
# Hand-maintained reader profile loaded into the system prompt.
|
||||
profile_path = "data/profile.md"
|
||||
interests_opml = "data/scour-interests.opml"
|
||||
|
||||
[miniflux]
|
||||
base_url = "http://127.0.0.1:8082"
|
||||
@@ -138,7 +137,7 @@ slop_value = -1.0 # AI slop: a full negative; the author pena
|
||||
verdicts_in_prompt = 60
|
||||
|
||||
# Every weight, quota, gate and threshold of the personalized ranker. The
|
||||
# learned signals (`knn`, `feed`) contribute nothing until their gates open:
|
||||
# learned signals (`knn`, `feed`, `affinity`) contribute nothing until their gates open:
|
||||
# the weight ramps linearly from `*_floor` to `*_full` rated articles.
|
||||
[curation.ranking]
|
||||
triage_max = 800 # eligible articles the triage LLM reads
|
||||
@@ -153,10 +152,12 @@ knn_floor = 8
|
||||
knn_full = 25
|
||||
feed_floor = 15
|
||||
feed_full = 40
|
||||
affinity_floor = 15
|
||||
affinity_full = 40
|
||||
slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict
|
||||
semantic_min_words = 300
|
||||
exploration_slots = 5
|
||||
embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors
|
||||
embedding_retention_days = 0 # `features prune`: unrated, unpublished vectors; 0 = keep forever
|
||||
telemetry_retention_days = 180 # `features prune`: candidate_runs and article_assessments
|
||||
|
||||
[curation.ranking.quotas]
|
||||
@@ -166,16 +167,18 @@ knn = 20
|
||||
|
||||
# Weights need not sum to 1; they are renormalized over the present signals.
|
||||
[curation.ranking.weights.preliminary]
|
||||
interest = 0.35
|
||||
interest = 0.30
|
||||
knn = 0.25
|
||||
affinity = 0.10
|
||||
heuristic = 0.20
|
||||
feed = 0.10
|
||||
social = 0.10
|
||||
social = 0.05
|
||||
|
||||
[curation.ranking.weights.utility]
|
||||
quality = 0.40
|
||||
fit = 0.20
|
||||
knn = 0.15
|
||||
knn = 0.10
|
||||
affinity = 0.05
|
||||
interest = 0.10
|
||||
feed = 0.05
|
||||
triage = 0.05
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Dashboard tables
|
||||
|
||||
How every data table on the dashboard is laid out, and the rules a new table
|
||||
must follow so it never grows a horizontal scrollbar on a desktop screen.
|
||||
Everything here is plain CSS in `src/web/tailwind.css` plus four cell classes;
|
||||
there is no table library and no JavaScript involved.
|
||||
|
||||
## The problem this solves
|
||||
|
||||
A table used to be `min-width: max-content` inside a `.scroll-x` wrapper, and
|
||||
most cells were `white-space: nowrap`. Any table whose natural width exceeded
|
||||
the page column (80rem, ~1355px at the site's 110% scale) became a horizontal
|
||||
scroller, even on a 2560px monitor, and the columns that mattered were hidden
|
||||
behind the scrollbar. Wrapping columns were squeezed to their minimum while the
|
||||
nowrap ones kept everything.
|
||||
|
||||
## The three rules
|
||||
|
||||
1. **Tables get the whole viewport, everything else keeps the page column.**
|
||||
`.dashboard` is a CSS grid with two named column spans: `content` (at most
|
||||
80rem minus the page gutters, exactly what the header uses) and `wide` (the
|
||||
viewport minus the same gutters). Every direct child sits in `content`; a
|
||||
direct-child `.scroll-x` sits in `wide`. The table inside a wide wrapper is
|
||||
`width: auto` (shrink-to-fit) with `min-width` equal to the content column
|
||||
and `max-width: 100%`, centred. A table that fits the page column therefore
|
||||
looks exactly as before; a table that needs more grows, centred, up to the
|
||||
viewport edge; the page itself never scrolls sideways.
|
||||
|
||||
2. **Columns wrap before the table scrolls.** The `min-width: max-content` rule
|
||||
is gone. A table shrinks by wrapping its text columns down to their floors,
|
||||
and only when the floors alone no longer fit (phones, mostly) does the
|
||||
`.scroll-x` wrapper scroll. Headers always wrap.
|
||||
|
||||
3. **A cell declares what it holds, and nothing else decides its width.**
|
||||
There are exactly four cell classes:
|
||||
|
||||
| class | use it for | behaviour |
|
||||
| ------------ | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
|
||||
| *(none)* | short prose, names, tokens, badges and badge lists | wraps at spaces (a `.badge` is nowrap on its own, so lists wrap between badges) |
|
||||
| `num` | numbers, money, counts | right-aligned tabular figures; `td.num` never wraps, `th.num` may |
|
||||
| `cell-tight` | timestamps, dates, ids, a single short token that must not split | `white-space: nowrap`. Nothing longer than ~20 characters |
|
||||
| `cell-wrap` | titles, URLs, notes, messages, anything free-form | floor 10rem, ceiling 32rem, `overflow-wrap: anywhere` |
|
||||
|
||||
Anything long that must not wrap gets an inner block with a Tailwind clamp
|
||||
(`<div class="line-clamp-2" title="…">`) inside a `cell-wrap` cell; the cell
|
||||
ceiling bounds the column, the clamp bounds the row height. Do not put
|
||||
`truncate`/`line-clamp-*` on a `td` itself (a table cell cannot be a
|
||||
`-webkit-box`), and do not use `table-fixed` on a page-level table (fixed
|
||||
layout needs a definite width and is silently ignored with `width: auto`).
|
||||
|
||||
## Checklist for a new table
|
||||
|
||||
- Wrap it in `<div class="scroll-x">` and make that wrapper a **direct child** of `<section class="dashboard">` if the
|
||||
table has more than a handful of columns. Tables inside `.card`, `.cards` or
|
||||
`details` stay inside their box and just wrap/scroll there.
|
||||
- Let the page scroll. A table that is the page's feature (paginated lists,
|
||||
anything that is the last thing on the page) shows every row at full height;
|
||||
a scrolling island inside a scrolling page just fights the wheel. Add `tall`
|
||||
(70vh cap, own scrollbar) only when the table is a side feature with real
|
||||
content below it, as the profile page's version history is.
|
||||
- Give every `td` one of the four classes above according to its content.
|
||||
When in doubt, leave it unclassed: wrapping is the safe default.
|
||||
- Never `cell-tight` a name, a title, a reason sentence or a free-text field.
|
||||
- Check the page at 390, 1280 and 2560px. At 1440 and above the wrapper's
|
||||
`scrollWidth` must equal its `clientWidth`; at 390 a wide table is allowed to
|
||||
scroll. A table's floor is the sum of its nowrap columns plus 10rem per
|
||||
`cell-wrap` column: the 13-column articles table bottoms out near 1270px, so
|
||||
it still scrolls a little on a 1280px viewport. Tables inside a `.card`
|
||||
cannot grow past the card (20rem minimum), so a card table with more than
|
||||
four numeric columns (run page: provider usage) scrolls inside its card at
|
||||
1280px; move such a table to page level if that matters.
|
||||
|
||||
## Measuring
|
||||
|
||||
`docs/plans/briefs/dashboard-tables/measure.mjs` logs in as the dev-seed admin,
|
||||
visits every dashboard page at several widths and prints, per page, whether the
|
||||
document overflows and which table wrappers scroll. Run it against a seeded
|
||||
dev server (`cargo run --example seed_dev_db -- ./dev`) before and after a CSS
|
||||
change; the "scrolling" count for widths ≥ 1280 should be zero.
|
||||
@@ -0,0 +1,247 @@
|
||||
# First-class interests: a table, a page, categories, links, and a rating-driven weight
|
||||
|
||||
**Date:** 2026-09-12
|
||||
**Repository:** `thallada/the-daily-epub`
|
||||
**Status:** implementation plan, ready to execute
|
||||
**Builds on:** `docs/plans/2026-09-02-personalized-curation-v2.md` (§8 profile and interests, §9 cheap signals, §12 blend and utility), `docs/plans/2026-09-03-web-dashboard.md` (page conventions), `docs/plans/2026-09-07-feed-discovery.md` (the most recent page + job + CLI addition; copy its shapes)
|
||||
|
||||
Written for a fresh implementation agent. Facts about this repo were checked against `main` (`e82f7b0`) on 2026-09-12. Nothing here depends on an external service beyond the LLM providers and Voyage that the pipeline already uses.
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Today the reader's ~230 standing interests live in a Scour OPML export (`data/scour-interests.opml`) plus an optional `## Interests` section of `profile.md`. They are parsed on every run, embedded, matched against each day's articles (`signals::interest_matches`), grouped for the system prompt by a hand-written keyword table (`profile/themes.rs`), and — since the last commit — shown to the reader as the **Matches:** line under every article header. They cannot be added from the dashboard, ratings never touch them, and nothing links from an interest to the articles that matched it.
|
||||
|
||||
This plan makes interests a first-class record:
|
||||
|
||||
1. **A database table is the only source of interests.** The OPML and the profile's `## Interests` section become one-time import inputs.
|
||||
2. **An Interests dashboard page** lists them with category, weight, and match counts; adds new ones; filters by category; changes a category by hand; deletes.
|
||||
3. **Categories** are kept (the grouped view the operator likes) and assigned by an LLM in one daily batch, only when uncategorized interests exist, with a manual **Categorize now** button.
|
||||
4. **Every interest name in the web UI is a link** to the Articles page filtered (and sorted) to the articles that matched it, with a cut-off so unrelated articles never appear. The Articles page gains an interest filter and an interests column.
|
||||
5. **Ratings nudge interests.** Each explicit rating credits the interests the article matched, scaled by how strongly it matched; the resulting per-interest weight is shown on the Interests page (default sort: highest first) and feeds curation as one bounded signal, so diversity is preserved.
|
||||
6. **Backfill** is two CLI commands the operator runs once.
|
||||
|
||||
## 2. Verified facts
|
||||
|
||||
### Interests today
|
||||
|
||||
- `profile::parse_interests(opml)` reads `<outline text="…">` names; `parse_profile_str` strips `## Interests` from `profile.md` and returns its lines; `union_interests` dedupes case-insensitively (`src/curate/profile/mod.rs:36–150`). `load_standing_interests(opml, profile)` is called from `pipeline::prepare_features` (`src/pipeline.rs:946`), `embedding::plan_backfill` (`src/curate/embedding.rs:901`), and `profile::load_or_build`/`rebuild` (through `prompt_inputs`). `config.interests_opml` is referenced in 20 places in `src/` (mostly tests that write a temp OPML), `config.example.toml:29`, README (lines 103 and 369), and `docs/runbooks/curation-v2-migration.md`.
|
||||
- The system prompt's "Standing interests" section groups names with `profile::group_into_themes` (`src/curate/profile/themes.rs`): twelve keyword themes plus "Other standing interests", deterministic, sorted. The profile dashboard page renders the same grouping (`src/web/dashboard/profile.rs:249`, template `dashboard/profile.html:42`).
|
||||
- `interest_embeddings` (migration 0002) is keyed by the interest **name**; `EmbeddingService::interests(&[String])` returns cached vectors and fetches misses (`src/curate/embedding.rs:743`). A new name is embedded automatically the next time it is passed.
|
||||
- `signals::interest_matches(article_embeddings, interest_embeddings)` (`src/curate/signals.rs:492`) computes cosine per interest × article, z-scores each interest across the day's embedded articles (std floored at 1e-3; raw top-1 cosine fallback under 30 articles), keeps the top `RECORDED_TOP = 3` per article as `TopInterest { name, z, cos }`, and scores `0.7·z₁ + 0.3·mean(top-3 z)`. `signals.top_interests` is serialized into `candidate_runs.signals_json` (`telemetry::serialize_signals`), copied into `Pick.top_interests` as bare names by the editor (`src/curate/editor.rs:598`), and rendered as the Matches line by `chapters::understanding` → `_understanding.html` (web) and `chapter.xhtml` / `in_this_issue.xhtml` (EPUB). Prompts list interests with `z ≥ 1.5` as "matches" (`triage.rs:135`, `assess.rs:192`, `editor.rs:159`).
|
||||
- Interest names also appear on: the dashboard article detail (`dashboard/article.html:61`, "Top interests" table), `_signals_table.html:5` (runs and article history), and the Feeds page "Why" badges (`dashboard/feeds.html:14`, from `discovery::why`).
|
||||
- Daily volume: ~365 entries → a few hundred articles per run; 512-dim `voyage-4-lite` vectors; `embedding_retention_days = 120`, so the embedding cache holds on the order of 30–50 k vectors (≈2 KB each). Scanning the cache per web request is not an option; a per-run cosine pass is milliseconds (§9.1 of the curation plan).
|
||||
|
||||
### Ratings and the learned signals
|
||||
|
||||
- `rating_events` is append-only; `Db::current_ratings(lookback_days) -> Vec<RatedArticle>` returns the latest explicit verdict per article with `value` (`loved` 1.0, `good` 0.35, `not_for_me` −1.0, `slop` per `feedback.slop_value`) and `event_at` (`src/db.rs:713`). Ratings are written by `web::rate::post` (dashboard and issue pages), `rate::record_explicit` (CLI), and `imports::run`.
|
||||
- `signals::PreferenceState::load` (`src/curate/signals.rs:238`) builds the run's learned state from current ratings: decayed weights `value × 0.5^(age/half_life)`, kNN over rated embeddings, and **feed/author affinity** as Beta-smoothed rates `(up+1)/(up+down+2)` with a gate `gate(n, feed_floor=15, feed_full=40)`. Every cheap signal is `Option<f64>`, percentile-normalized over the eligible set (`normalize`, `PERCENTILE_SIGNALS`), blended with renormalized weights (`preliminary_blend`; `rank::calculate_utility_for` for the deep set). Absent is never zero. The slop-author factor scales both blends. Diversity is enforced downstream by `rank::shortlist` (cluster threshold 0.85, `per_cluster_cap = 2`, `utility_protected = 10`) and by the editor prompt.
|
||||
- Signal names are enumerated in: `signals::PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend`'s candidate list, `rank::calculate_utility_for`'s weighted list, `telemetry::serialize_signals` (raw list) and `RENDERED_SIGNALS`, `dashboard::SIGNAL_NAMES`, `config::{PreliminaryWeights, UtilityWeights}`, and `settings::SETTINGS_HELP`. A new signal touches all of them.
|
||||
- The Ratings dashboard page computes each verdict's feed credit on the fly with a pure function (`ratings::contribution`) rather than storing it. Follow that precedent.
|
||||
|
||||
### Dashboard, jobs, CLI, config
|
||||
|
||||
- One submodule per page group under `src/web/dashboard/`, each with `routes()`, merged in `dashboard::router()`; admin gating is applied by the caller. Nav tabs are hard-coded in `src/web/templates/layout.html` (keyed on `page.active_nav`); overview tiles in `dashboard/overview.html`. POST → flash → redirect via `jobs::set_flash`. `Page::is_admin()` is available in every template. Pager partial `dashboard/_pager.html`; client-side row filter `data-table-filter`.
|
||||
- The Articles list (`src/web/dashboard/articles.rs`) builds `ARTICLE_INNER` (articles ⨝ best entry ⨝ latest `candidate_runs` row ⨝ assessments, with correlated subqueries for rating and publication) and wraps it in `SELECT * FROM (…) x WHERE 1=1 {clauses} ORDER BY {sort}`. Filters are allow-listed; sorts come from `ARTICLE_SORTS`; `Pager::new(pagination, path, &filters.params())` round-trips them.
|
||||
- Jobs are a fixed catalogue (`jobs::Job`, `CATALOGUE`, `parse`, `name`, `description`, `takes_lock`, `dangerous`) run as `daily-epub-job@<name>.service` and dispatched in `main::run_job`. The profile page starts `profile-rebuild` with a plain form posting to `/dashboard/jobs/profile-rebuild`. The only daily entry point is the `daily-epub-generate.timer` (05:30 America/New_York).
|
||||
- `pipeline::build_llms` (`src/pipeline.rs:1133`) builds the taste-profile prompt, the clients, and runs the weekly learned-adjustments rebuild, then rebuilds the clients with the new prompt. An LLM step that must precede the prompt goes here.
|
||||
- Config sections are `#[serde(deny_unknown_fields, default)]`; an unknown top-level key makes `Config` fail to load. The settings page has a hard-coded `GROUP_ORDER`, `PATH_KEYS`, and `SETTINGS_HELP`, with a test over the section list. Migrations: `sqlx::migrate!("./migrations")`; latest is `0012_article_publication.sql`, so the new file is `0013_interests.sql`. `db.rs` has a migration test asserting a table list (`src/db.rs:1720`).
|
||||
|
||||
## 3. Options considered
|
||||
|
||||
### 3.1 Where "articles matching interest X" comes from
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. Compute on request | Load the interest vector and every cached article embedding, dot, sort. | **Rejected.** 30–50 k × 2 KB per request; the dashboard is 1–7 ms today and should stay there. |
|
||||
| B. Query `candidate_runs.signals_json` with `json_each` | The top-3 names are already persisted per run. | **Rejected.** A JSON scan over every telemetry row (hundreds of thousands, pruned at 180 days) per request, no index, and rows vanish with telemetry retention. |
|
||||
| C. **A junction table written by the signals stage** | `article_interests(article_id, interest_id, cos, z)`: the same top-3 the run already computes, one row each, indexed by interest. The filter is an indexed join. | **Chosen.** ~1 k rows/day, zero extra computation, and the backfill is one pass over cached embeddings. |
|
||||
|
||||
### 3.2 How an interest weight is computed and stored
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. Stored counters updated on every rating event | Add `up/down` columns to `interests`; the rating handlers, CLI, and importer bump them; write a backfill migration script. | **Rejected.** Four write paths to keep in sync, decay cannot be stored (it is a function of *now*), and it duplicates the ratings history that already exists. |
|
||||
| B. **Derived on the fly from current ratings × matches** | One pure function over `current_ratings` and their `article_interests` rows, exactly like feed affinity. The run computes it in `PreferenceState::load`; the Interests page computes it on render. | **Chosen.** No new write path, always current, decay and lookback for free, backfill is "make sure rated articles have match rows". ≤ a few hundred ratings × 3 rows: microseconds. |
|
||||
| C. Ask the weekly learned-adjustments rebuild to write per-interest weights | An LLM judges the rating history per interest. | **Rejected.** Non-deterministic, weekly, and the numbers would not be explainable. The existing prose rebuild already sees the ratings. |
|
||||
|
||||
### 3.3 How the weight enters curation
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. **A new bounded cheap signal, `affinity`** | Per article: match-strength-weighted mean of its matched interests' weights; percentile-normalized; gated on rating count; small configured weight in the preliminary blend and the utility. | **Chosen.** Fits the existing design (absent ≠ zero, renormalized weights, `explain` shows it), and its influence is capped at its weight share, so one runaway interest cannot dominate. Diversity machinery downstream is untouched. |
|
||||
| B. Multiply each interest's z by its weight before the top-3 is taken | Changes which interests appear as matches. | **Rejected.** Entangles "what does this article match" with "what does he like", and the Matches line would drift with ratings. |
|
||||
| C. Annotate the prompt's Standing interests with ↑/↓ | Cheap and the LLM would use it. | **Deferred** (§9). Worth adding once the weights have a few weeks of ratings behind them; it is a five-line change on top of this plan. |
|
||||
| D. A new deep-set admission retriever by affinity | Like the `interest` and `knn` quotas. | **Rejected.** More slots for the same signal; the blend fill already admits high-affinity articles. |
|
||||
|
||||
### 3.4 Categories
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| A. **`interests.category TEXT NULL`; the category set is the distinct values** | **Chosen.** No FK, no second page, renaming is an `UPDATE`. |
|
||||
| B. A separate `interest_categories` table with FK | Rejected: a table with one meaningful column. |
|
||||
| C. Keep the keyword table in `themes.rs` | Rejected as the source of truth (the user wants DB-tracked interests and LLM categorization), but **kept for the one-time import** so the current grouping survives unchanged. |
|
||||
|
||||
### 3.5 When the categorizer runs
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| A. **Inside `generate`, before the prompt is built, only when uncategorized interests exist; plus a catalogue job for the button** | **Chosen.** The morning timer is the only daily trigger that exists; a run without new interests spends nothing. |
|
||||
| B. Its own systemd timer | Rejected: another unit to install for a call that takes seconds. |
|
||||
| C. Synchronously in the Add handler | Rejected: an LLM call in a request path, and the user asked for a daily batch. |
|
||||
|
||||
## 4. Design decisions (settled)
|
||||
|
||||
| Topic | Decision |
|
||||
|---|---|
|
||||
| Match rule (one definition everywhere) | An interest **matches** an article when it is among the article's top three interests by z **and** `z ≥ MATCH_MIN_Z = 1.0`. `interest_matches` applies this when it truncates, so `signals.top_interests`, the Matches line, the stored rows, the Articles filter, and the interest weights all agree. The score formula is unchanged (computed before the cut). The prompts keep their stricter `z ≥ 1.5` for "matches interests". |
|
||||
| Stored rows | `article_interests(article_id, interest_id, cos, z, run_id)`, upserted per eligible article per run (the same article can be eligible on consecutive days; the latest run wins; cosine is stable, z is that day's). Rows are never pruned (≈40 bytes each, ~1 k/day). |
|
||||
| Link target and filter key | `/dashboard/articles?interest=<name>`: names are unique (case-insensitive), human-readable, and `Pick.top_interests` already carries names, so no id has to travel through `issue_json`. The handler resolves the name to an id; an unknown name yields an empty list, never an error. When `interest` is set and `sort` is absent, the sort defaults to `match` (cosine descending). |
|
||||
| Who sees links | The dashboard is admin-only, so the Matches line links only for admin viewers (`page.is_admin()`); readers and anonymous visitors see plain text as today. The EPUB never links. |
|
||||
| Interest weight | Beta-smoothed rate over decayed, strength-scaled credits (§5.2): `(up + 1) / (up + down + 2)`, in (0, 1), 0.5 = no information. Shown with two decimals plus `up`, `down`, and the number of rated matches. |
|
||||
| Match strength | `s = clamp(z / 3, 0, 1)`: a z of 3 credits the full rating, a bare match (z = 1) a third. |
|
||||
| Curation signal | `affinity` (§5.3): signed, centred on zero, gated on the number of ratings that credited at least one interest (`affinity_floor = 15`, `affinity_full = 40`, same shape as the feed gate). Default weights: preliminary 0.10 (taken from `interest` 0.35→0.30 and `social` 0.10→0.05), utility 0.05 (taken from `knn` 0.15→0.10). |
|
||||
| Diversity | Guaranteed by construction: the signal's share of the blend is its configured weight (≤ 10 % / 5 %), percentile normalization caps a favourite interest's articles at percentile 1.0 of that one signal, Beta smoothing means a single *loved* moves a weight from 0.50 to at most 0.67, ratings decay with the 60-day half-life, and `rank::shortlist`'s per-cluster cap and the editor's diversity instructions are unchanged. No new cap is needed. |
|
||||
| Source of truth | The `interests` table. `interests_opml` is **removed** from config; `profile.md`'s `## Interests` section is still stripped by the parser but no longer read (the importer consumes it once; the profile page says so). |
|
||||
| Prompt grouping | "Standing interests" groups by `interests.category` (sorted by category name, members sorted case-insensitively); `NULL` renders under the existing label "Other standing interests". `themes.rs` is used only by the importer. |
|
||||
| Categorizer | One JSON call on the bulk provider over every uncategorized interest, given the existing category list; may create a category only when none fits. Unassigned names stay `NULL` and are retried the next day. Also the catalogue job `interests-categorize` behind the **Categorize now** button. |
|
||||
| Interests page actions | Add (name + optional category), change category (per-row select + Save), Delete (confirm; cascades match rows, deletes the cached embedding). No rename (delete + add). |
|
||||
| Backfill | `daily-epub interests import` (OPML + profile section → rows, categorized by the keyword table) and `daily-epub interests backfill` (match rows for every cached embedding). Weights need no backfill: they are derived. |
|
||||
| Config | No new section. `[curation.ranking]` gains `affinity_floor`, `affinity_full`; the two weight tables gain `affinity`. |
|
||||
|
||||
## 5. The numbers
|
||||
|
||||
### 5.1 Match rows
|
||||
|
||||
In `prepare_features` (`src/pipeline.rs`), after `signals::compute_all` and before the candidates are handed on: for every candidate with non-empty `signals.top_interests`, upsert `(article_id, interest_id, cos, z, run_id)`. One transaction, `INSERT … ON CONFLICT(article_id, interest_id) DO UPDATE SET cos, z, run_id`. Names map to ids through the `interests` rows loaded at the top of the stage (the same rows whose names go to `service.interests`). Best effort: a failure is a report warning, never a failed run.
|
||||
|
||||
### 5.2 Interest weight (pure, `interests::rates`)
|
||||
|
||||
Inputs: `current_ratings(rating_lookback_days)` and the `article_interests` rows of those articles.
|
||||
|
||||
```text
|
||||
for each current rating r on article a (value v_r, decay d_r = 0.5^(age_days / half_life_days))
|
||||
for each match row (a, i, z):
|
||||
s = clamp(z / 3, 0, 1)
|
||||
credit = v_r × d_r × s
|
||||
up_i += max(credit, 0)
|
||||
down_i += max(−credit, 0)
|
||||
n_i += 1
|
||||
weight_i = (up_i + 1) / (up_i + down_i + 2) # 0.5 when n_i = 0
|
||||
```
|
||||
|
||||
`cleared` verdicts are already excluded by `current_ratings`; `slop` carries `feedback.slop_value` like everywhere else. A rating is *attributable to interests* when it credits at least one interest; that count drives the gate.
|
||||
|
||||
### 5.3 The `affinity` signal (pure, in `PreferenceState`)
|
||||
|
||||
```text
|
||||
matched = the article's top_interests with n_i > 0
|
||||
affinity = Σ s_i × (weight_i − 0.5) / Σ s_i absent when matched is empty or the gate is 0
|
||||
gate = gate(attributable_interest_ratings, affinity_floor, affinity_full)
|
||||
```
|
||||
|
||||
The raw value lives in [−0.5, 0.5]; it is percentile-normalized with the other cheap signals, weighted by `weights.preliminary.affinity × gate` in the blend and `weights.utility.affinity × gate` in the utility. `explain` and `_signals_table.html` show it like any other signal. The once-per-run preference log line gains `affinity gate … (n=…)`.
|
||||
|
||||
## 6. Data model — `migrations/0013_interests.sql`
|
||||
|
||||
```sql
|
||||
CREATE TABLE interests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
category TEXT, -- NULL until categorized
|
||||
created_at TEXT NOT NULL,
|
||||
categorized_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_interests_category ON interests(category);
|
||||
|
||||
CREATE TABLE article_interests (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||
cos REAL NOT NULL,
|
||||
z REAL NOT NULL,
|
||||
run_id INTEGER, -- NULL for backfilled rows
|
||||
PRIMARY KEY (article_id, interest_id)
|
||||
);
|
||||
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||
```
|
||||
|
||||
`interest_embeddings` stays keyed by name (no migration): a deleted interest also deletes its embedding row; a renamed one is a new interest.
|
||||
|
||||
## 7. Implementation steps
|
||||
|
||||
Sizes are rough line counts including tests. Steps 1–3 must be in order; 4–8 can proceed in parallel after 3; 9 last.
|
||||
|
||||
1. **Migration + `src/interests.rs`** (~350). The table above; the db-list test in `db.rs` gains both tables. Module (following `discovery.rs`: data access, pure functions, and the LLM step in one file):
|
||||
- `Interest { id, name, category, created_at, categorized_at }`; `list(db) -> Vec<Interest>` (ordered by name), `add(db, name, category, now) -> Result<id, Duplicate>` (trimmed, 1–80 chars, unique case-insensitive), `set_category(db, id, Option<category>, now)`, `delete(db, id)` (also `DELETE FROM interest_embeddings WHERE interest = name`), `names(db) -> Vec<String>`, `grouped(db) -> Vec<(String, Vec<String>)>` (for the prompt), `uncategorized(db)`.
|
||||
- `replace_matches(db, run_id, &[(ArticleId, &[TopInterest])], names→ids)`; `matches_for_articles(db, ids) -> Vec<MatchRow { article_id, interest_id, name, cos, z }>`; `match_counts(db) -> HashMap<interest_id, i64>`.
|
||||
- `pub fn rates(ratings: &[(ArticleId, value, decay)], rows: &[(ArticleId, interest_id, z)]) -> (HashMap<interest_id, Rate { up, down, n }>, attributable)` per §5.2, and `Rate::weight()`.
|
||||
- Tests: uniqueness is case-insensitive; `rates` matches hand-checked numbers (a loved z=3 → up 1.0; a good z=1.5 → up 0.175; a not-for-me z=0.9 → nothing, below the cut); delete cascades; `grouped` puts `NULL` last under "Other standing interests".
|
||||
|
||||
2. **Signals** (~250). In `signals.rs`: `MATCH_MIN_Z = 1.0` applied inside `interest_matches` after truncation; `Signals.affinity: Option<f64>`; `"affinity"` in `PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend` (gate `affinity_gate`); `PreferenceState` gains `interest_rates: HashMap<String, Rate>` (keyed by name — `TopInterest` carries names), `affinity_gate`, `attributable_interest_ratings`, `fn affinity(&self, top: &[TopInterest]) -> Option<f64>`, loaded in `load` from `interests::matches_for_articles(rated ids)` and exposed on `PreferenceSummary` and in `log`. `rank::calculate_utility_for` adds `("affinity", configured.affinity, candidate.signals.affinity_gate)`. `telemetry`: `serialize_signals` raw list and `RENDERED_SIGNALS`; `render_explain` prints it with the others. `dashboard::SIGNAL_NAMES` and `_signals_table.html` pick it up automatically once the name is in the list. Config: `RankingConfig { affinity_floor: 15, affinity_full: 40 }`, `PreliminaryWeights { affinity: 0.10, interest: 0.30, social: 0.05 }`, `UtilityWeights { affinity: 0.05, knn: 0.10 }`, `SETTINGS_HELP` lines, `config.example.toml`, README table.
|
||||
Tests: the top-3 cut drops a z=0.4 third interest; affinity is absent under the gate and with no rated interests; a candidate matching a 0.8-weight interest outranks an otherwise identical one matching a 0.3-weight interest in the blend; blend weights still renormalize to 1.
|
||||
|
||||
3. **Replace the OPML/profile plumbing** (~300, mostly deletions and test edits). `profile::load_standing_interests`, `parse_interests`, `union_interests`, and `prompt_inputs`' OPML argument go; `build`, `load_or_build`, `rebuild`, and `weekly_rebuild_if_due` take `grouped: Vec<(String, Vec<String>)>` from `interests::grouped(db)` instead of `interests: &[String]` + `group_into_themes`. `pipeline::prepare_features` and `embedding::plan_backfill` take names from `interests::names(db)`. `themes.rs` stays but is only referenced by the importer (step 8); `pub use themes::group_into_themes` is removed from the profile module. Remove `Config.interests_opml` (struct, `Default`, `config check` line, the `deny_unknown_fields` implication is a rollout note in §10), `settings::PATH_KEYS`/`SETTINGS_HELP`, `config.example.toml`, README (lines 103, 369), and the runbook mention. Every test that writes a temp OPML instead inserts rows with `interests::add`. The profile page loses the OPML card and the "Extracted `## Interests` lines" preview; in their place one line: "N standing interests in M categories — manage them on the Interests page", and the editor note says the `## Interests` section is ignored.
|
||||
|
||||
4. **Match rows in the run** (~80). `prepare_features` writes them per §5.1 (timing folded into the existing `signals` timing; count `interest_matches` on the report counts is optional — skip unless free).
|
||||
|
||||
5. **Articles page** (~200). `ArticlesQuery.interest: Option<String>` → `ArticleFilters.interest: Option<(i64, String)>` resolved by name (case-insensitive) in `from_query`'s caller (it needs the db; resolve in `list` before building filters, or make `from_query` async — pick the former). When set: `ARTICLE_INNER` gains `JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?` (inside the inner query so `idx_article_interests_interest` drives it; the join is a `{interest_join}` placeholder that is empty otherwise) and exposes `ai.cos AS match_cos`; `ARTICLE_SORTS` gains `("match", "x.match_cos DESC, x.id DESC")`, which is the default when `interest` is set and `sort` is absent, and is ignored (falls back to `first_seen`) when it is not. Filter UI: a `<select name="interest">` over all interest names (blank = any), populated from `interests::list`. New column **interests**: for the page's ≤ 50 rows, one query through `matches_for_articles`, grouped per article, rendered as badges linking to `?interest=<name>` (ordered by z desc). `params()` round-trips `interest` and `sort=match`. Test: a seeded match row is found by the filter, an unknown name yields an empty table (200), and the column links.
|
||||
|
||||
6. **Interests page** (~450). `src/web/dashboard/interests.rs` + `dashboard/interests.html`; routes `GET /dashboard/interests`, `POST /dashboard/interests` (add), `POST /dashboard/interests/{id}/category`, `POST /dashboard/interests/{id}/delete`. Nav tab **Interests** after Ratings; overview tile "N interests · M uncategorized". Page: add form (name, category select with a blank "let the categorizer decide" option), category filter tabs or select (`?category=<name>` and `?category=uncategorized`), sort select (`weight` default, `name`, `matches`, `added`), the `data-table-filter` row filter, no pager (≤ a few hundred rows). Columns: interest (link → `/dashboard/articles?interest=<name>`), category (link → `?category=`), weight (2 decimals, tabular; `—` when n = 0), up, down, rated matches n, matched articles (count), added (`format_time`), actions (category select + Save; Delete with `confirm`). Weights come from `interests::rates` over `current_ratings(rating_lookback_days)` computed on render; the header says which lookback and half-life it uses and shows the affinity gate state ("affinity gate 0.35 · 22 of 40 attributable ratings"), mirroring the Ratings page. **Categorize now** button posts to `/dashboard/jobs/interests-categorize` (the existing generic job route), shown enabled when `jobs_enabled` and disabled with the manual command otherwise, with the uncategorized count next to it. Flash messages for add/duplicate/deleted. Tests: page renders with weights sorted descending; add rejects a duplicate differing only in case; delete removes match rows and the embedding row; routes are admin-only.
|
||||
|
||||
7. **Links everywhere** (~120). `chapters::Understanding.interests` becomes `Vec<InterestRef { name, href }>` (`href = "/dashboard/articles?interest=" + encode_component(name)`) with `interests_line()` for the EPUB templates; `_understanding.html` renders `{% for %}…{% if page.is_admin() %}<a href>…` joined by ` · `. Dashboard article detail "Top interests" table, `_signals_table.html`, and the Feeds page "Why" badges link the same way. Existing tests asserting `Matches: Filesystems · Rust` keep passing for non-admin views; add one admin assertion.
|
||||
|
||||
8. **Categorizer + job + CLI** (~350).
|
||||
- `interests::categorize(config, db) -> Result<String>`: load uncategorized; if none, return "nothing to categorize"; build `Llms::from_config` with the current taste prompt (as `cmd_profile_rebuild` does) and take `bulk` (fall back to `editor_or_bulk`); one `complete_json` at temperature 0.2 with:
|
||||
```text
|
||||
TASK: file each new standing interest under one of the reader's interest categories.
|
||||
Existing categories (reuse these names verbatim): <list>
|
||||
Create a new category only when none of the existing ones fits; a new category must be broad enough to hold several interests and named like the existing ones (two to five words, sentence case). Every interest gets exactly one category.
|
||||
New interests: <one per line>
|
||||
Return JSON exactly: {"assignments":[{"interest":"…","category":"…"}]}
|
||||
```
|
||||
Validate: interest must be one of the batch (case-insensitive), category trimmed, 1–60 chars; unknown names are ignored; anything unassigned stays `NULL`. Log "categorized N (M new categories: …)". Return that as the job message.
|
||||
- `pipeline::build_llms`: after the clients exist, if `interests::uncategorized(db)` is non-empty and a client is available, run `categorize` (warning on error), then rebuild the prompt and the clients exactly as the weekly rebuild path does (the grouping changed). Both the categorizer and the weekly rebuild share that "prompt changed, remake clients" tail.
|
||||
- `jobs::Job::InterestsCategorize` (`interests-categorize`, description "File uncategorized interests under categories with the bulk model, creating new ones only when needed.", `takes_lock` None, not dangerous); `main::run_job` dispatch; catalogue tests.
|
||||
- CLI `daily-epub interests <import|backfill>` in `main.rs`:
|
||||
- `import [--opml PATH] [--profile PATH]` (defaults `data/scour-interests.opml` and `config.profile_path`): parse the OPML with the old `parse_interests` (moved into `interests.rs` as a private helper) and the profile's `## Interests` lines, insert with `add` skipping duplicates, assign each new row's category with `themes::group_into_themes` mapped back per name ("Other standing interests" → `NULL` so the LLM gets a go at them). Print "imported N, skipped M existing, K left for the categorizer".
|
||||
- `backfill`: load every `article_embeddings` row compatible with the configured model/dimension in chunks, load interest vectors from `interest_embeddings` (fetching misses through the service if Voyage is enabled), run `interest_matches` over the whole set (z over the whole cached set stands in for the per-day cohort; say so in the docstring), and `INSERT OR IGNORE` the rows with `run_id NULL` so real-run rows win. Print the row count. Takes the run lock (it writes what the run writes).
|
||||
- Tests: the categorizer with `MockBackend` assigns two interests, creates one new category, ignores a name not in the batch; `import` is idempotent; `backfill` on a two-article fixture writes the expected rows and leaves an existing run-written row alone.
|
||||
|
||||
9. **Docs** (~40). README: the "Web site and dashboard" paragraph gets a sentence on Interests; the Commands section gets `interests import` / `interests backfill`; the configuration table gains the three keys and loses `interests_opml`; the Layout section lists `src/interests.rs`. `docs/ideas.md`: strike the "what interests most match" item.
|
||||
|
||||
Expect roughly 2 100 lines including tests.
|
||||
|
||||
## 8. Verification
|
||||
|
||||
- `cargo test`, `cargo clippy`, `cargo fmt` green.
|
||||
- Dev seed (`dev/`): `interests import --opml data/scour-interests.opml`, then `interests backfill` (no embeddings in the seed → "0 rows", exits 0), open `/dashboard/interests`: 230 rows, every one categorized except a handful under Uncategorized, weight `—` everywhere; add "Rust macros" → appears uncategorized; **Categorize now** with jobs disabled shows the manual command.
|
||||
- After deploy, on the server: run the two commands (§10), open the page, click **Categorize now**, watch the job page, refresh: no Uncategorized left, new categories (if any) look sane. Rate one article from the issue page; its matched interests' weights move on the next refresh with the expected magnitude (`0.5 → 0.67` for a full-strength *loved*).
|
||||
- Next morning's run: `explain` on a selected article shows `affinity` present with a weight, or absent with the gate at 0 if fewer than 15 ratings credited an interest (expected at first; the log line says which). The Articles page filtered by an interest the reader knows well returns articles that obviously belong, sorted by cosine; nothing unrelated appears at the bottom of the first page. Dashboard timing for that page stays single-digit ms (`Server-Timing` header).
|
||||
- The system prompt on the profile page groups Standing interests by the DB categories with the same members as before the change (byte-stable apart from category names that the LLM added).
|
||||
|
||||
## 9. Deferred, with triggers
|
||||
|
||||
- **Prompt annotation** of Standing interests with the strongest and weakest weights (e.g. `Rust ↑`, `Coffee ↓`) — once the affinity gate is open and the weights look right for a couple of weeks.
|
||||
- **Renaming** an interest in place, and merging duplicates — when the operator actually needs it (delete + add loses nothing but the match rows, which the next runs rebuild).
|
||||
- **Match-row retention** — if `article_interests` ever exceeds a few million rows; today it grows ~1 k/day.
|
||||
- **Implicit ratings** feeding the weights — already designed for in the curation plan §6.4; nothing here blocks it.
|
||||
- **Scour** — the Scour feeds in Miniflux keep working as sources; nothing here syncs interests back to Scour.
|
||||
|
||||
## 10. Rollout notes
|
||||
|
||||
1. Deploy the binary. **Before restarting the service, delete `interests_opml` from `/etc/daily-epub/config.toml`** (the key is gone and `Config` rejects unknown keys). Migrations run on the first command.
|
||||
2. `daily-epub interests import --opml /var/lib/daily-epub/data/scour-interests.opml` (the profile's `## Interests` section is read from `profile_path`; it has been empty so far). Then delete that section from `profile.md`, or leave it — it is ignored either way.
|
||||
3. `daily-epub features backfill --rated-only` if any rated article lacks an embedding (it prints the count), then `daily-epub interests backfill`.
|
||||
4. Open `/dashboard/interests`, press **Categorize now** for the leftovers, and read the weights: they are live from that moment because they are derived from the existing ratings.
|
||||
5. Optional: `daily-epub generate --dry-run` to see `affinity` in a run before the morning.
|
||||
|
||||
## 11. Non-goals (do not add)
|
||||
|
||||
- A category editor page, category colours, or ordering beyond alphabetical.
|
||||
- Per-interest notes, pinning, or manual weight overrides — the weight is derived from ratings only, so the operator changes it by rating.
|
||||
- Interest suggestions from articles ("you seem to like X") — a separate feature with its own plan if wanted.
|
||||
- Syncing interests to or from Scour, or OPML export.
|
||||
- A stored weight column, incremental counters, or a scheduled recompute.
|
||||
- Configurable `MATCH_MIN_Z` or match strength curve — constants until there is a reason.
|
||||
- Public (anonymous or reader-role) links from the Matches line; the dashboard is admin-only.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Shared brief — dashboard tables (2026-09-20)
|
||||
|
||||
The operator sees horizontal scrollbars on most dashboard tables, even on an
|
||||
ultrawide monitor, because every table is capped at the 80rem page column and
|
||||
`min-width: max-content` forbids wrapping. The fix is a small, reusable table
|
||||
framework described in **`docs/dashboard-tables.md` — read it first; it is the
|
||||
spec.** Two tasks run in parallel in separate git worktrees:
|
||||
|
||||
- `01-css.md` — the CSS side (`src/web/tailwind.css`, rebuilt `app.css`).
|
||||
- `02-templates.md` — the template audit (every `td` gets the right cell class).
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Work only in the worktree named in your brief. **Never touch the main checkout**
|
||||
at `/home/thallada/workspace/the-daily-epub`. **Do not `git commit`** (the sandbox
|
||||
cannot write `.git/worktrees`); leave the work in the tree, the orchestrator
|
||||
commits.
|
||||
- Styling lives in **`src/web/tailwind.css`** (Tailwind v4, `@apply` inside
|
||||
`@layer components`, semantic colour tokens). After every CSS change run
|
||||
`npm run css` (writes `src/web/static/app.css`, which the binary embeds with
|
||||
`include_str!`). `npm run css:check` must print no diff at the end.
|
||||
- CSP forbids inline `<style>`, `<script>` and `style=""` attributes. No new JS,
|
||||
no new dependencies, no table library: this is plain CSS + four cell classes.
|
||||
- Keep every hook `app.js` relies on: `table[data-filter]`, `.table-filter`,
|
||||
`input[data-table-filter]`, `.scroll-x` (the filter script does
|
||||
`table.closest(".scroll-x")`), `form[data-confirm]`, `details[id]`, `[data-refresh]`.
|
||||
- Keep `cargo fmt`, `cargo clippy --all-targets -- -D warnings` and the tests
|
||||
green (`cargo test --lib web::` while iterating, one full `cargo test` at the
|
||||
end). **Sandbox note:** inside the Codex sandbox `bind()` on 127.0.0.1 is
|
||||
forbidden; ignore every test that fails with `PermissionDenied`/bind errors
|
||||
(about 17 pre-existing ones). Do not "fix" them.
|
||||
- 4 cores / 7 GB shared with another agent building Rust at the same time: one
|
||||
cargo process at a time, be patient.
|
||||
- KISS / YAGNI: no config knobs, no per-table width tables, no JS resizing, no
|
||||
responsive card-ification of rows. Where the brief and the code disagree,
|
||||
follow the code and note the deviation in your handoff.
|
||||
- Finish by writing `docs/plans/briefs/dashboard-tables/handoff-<task>.md`:
|
||||
what changed (file by file), deviations, what you verified and how, anything
|
||||
left open.
|
||||
|
||||
## Previewing (optional — the sandbox usually cannot bind a port)
|
||||
|
||||
Seed a throwaway database with `cargo run --example seed_dev_db -- ./dev`
|
||||
(git-ignored) and serve it with
|
||||
`DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) DAILY_EPUB_SERVER__BIND=127.0.0.1:<port> cargo run -- --config ./dev/config.toml serve`.
|
||||
`measure.mjs` in this directory (needs `playwright` on the module path; a copy
|
||||
lives at `~/.npm/_npx/705bc6b22212b352/node_modules`) prints per page and width
|
||||
whether the document overflows and which table wrappers scroll. If you cannot
|
||||
bind, rely on the router tests and say so; the orchestrator screenshots every
|
||||
page at 390/768/1280/1920/2560 after merging.
|
||||
@@ -0,0 +1,85 @@
|
||||
# Task 01 — table framework CSS
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-tables-css` (branch `tables-css`).
|
||||
Read `00-shared.md` and `docs/dashboard-tables.md` first. Only `src/web/tailwind.css`
|
||||
and the rebuilt `src/web/static/app.css` change in this task (plus your handoff).
|
||||
The template audit happens in a sibling worktree; do not edit templates.
|
||||
|
||||
## 1. `.dashboard` becomes a content grid
|
||||
|
||||
Replace `.dashboard { @apply mx-auto my-8 max-w-7xl px-4 pb-6 … sm:px-6; }` with a
|
||||
grid that keeps the page column pixel-identical to the header
|
||||
(`mx-auto max-w-7xl px-4 sm:px-6` = content width `80rem - 2 * gutter`, centred):
|
||||
|
||||
```css
|
||||
.dashboard {
|
||||
--gutter: 1rem;
|
||||
@apply my-8 grid pb-6 font-sans text-base leading-normal;
|
||||
grid-template-columns:
|
||||
[full-start] var(--gutter)
|
||||
[wide-start] 1fr
|
||||
[content-start] minmax(0, calc(80rem - 2 * var(--gutter)))
|
||||
[content-end] 1fr
|
||||
[wide-end] var(--gutter)
|
||||
[full-end];
|
||||
}
|
||||
@media (width >= 40rem) { .dashboard { --gutter: 1.5rem; } } /* Tailwind `sm` */
|
||||
.dashboard > * { grid-column: content; min-width: 0; }
|
||||
.dashboard > .scroll-x { grid-column: wide; }
|
||||
```
|
||||
|
||||
Keep `.dashboard > * + * { mt-4 }` and every other `.dashboard …` rule as they
|
||||
are (margins on grid items work). Verify in the templates that nothing relied
|
||||
on the old padding: `.save-bar` uses `-mx-4 sm:-mx-6` to bleed to the edges;
|
||||
under the grid it still overflows its `content` area into the gutter columns
|
||||
by the same amount, so leave it unless you see a problem. The `sm` breakpoint
|
||||
is `40rem` in Tailwind v4 and `@media` rem is always 16px-based, so the query
|
||||
above matches `sm:` exactly.
|
||||
|
||||
Prototype measured in Chromium and Firefox at 390/1280/2560: `h1` left edge and
|
||||
width identical to today, no document overflow, table centred and wider than
|
||||
the column only when its content needs it.
|
||||
|
||||
## 2. Tables shrink by wrapping; page-level tables may grow past the column
|
||||
|
||||
- Delete `.scroll-x > table { min-width:max-content; }`.
|
||||
- Add, for the wide wrapper only:
|
||||
|
||||
```css
|
||||
.dashboard > .scroll-x > table {
|
||||
width: auto; /* shrink-to-fit */
|
||||
min-width: min(100%, calc(80rem - 2 * var(--gutter)));
|
||||
max-width: 100%;
|
||||
margin-inline: auto;
|
||||
}
|
||||
```
|
||||
|
||||
A table narrower than the page column fills the column as today; a wider one
|
||||
grows, centred, to the viewport edge; the wrapper scrolls only past that.
|
||||
Tables that are not direct children (cards, disclosures, signal tables inside
|
||||
cells) keep `w-full` from the base rule and simply wrap, then scroll.
|
||||
|
||||
## 3. Cell vocabulary (exactly four classes, see the doc's table)
|
||||
|
||||
- `.num` → `@apply text-right tabular-nums;` and `td.num { @apply whitespace-nowrap; }`.
|
||||
Header cells with `.num` may wrap ("cache write", "matched articles").
|
||||
- `.cell-tight` unchanged (`whitespace-nowrap`).
|
||||
- `.cell-wrap` → `@apply min-w-[10rem] max-w-[32rem] whitespace-normal; overflow-wrap:anywhere;`
|
||||
(ceiling raised from 24rem so titles get a single line when the screen has
|
||||
room; `max-width` on a cell is honoured by Chromium and Firefox for the
|
||||
column's max-content contribution — measured).
|
||||
- `th` keeps wrapping (no nowrap); keep the sticky `thead`.
|
||||
- `td > pre.preview { max-w-md }` and `.rating-cell` rules stay.
|
||||
|
||||
## 4. Small things to keep honest
|
||||
|
||||
- `.prose-body table` (reader pages) keeps its own rules; do not touch.
|
||||
- Put a short comment above the table rules pointing at `docs/dashboard-tables.md`.
|
||||
- `npm run css`, then `cargo build` (the binary embeds app.css) and
|
||||
`cargo test --lib web::`. Router tests assert on markup, not on these rules,
|
||||
so nothing should break; if something does, say what and why in the handoff.
|
||||
- If you can run the dev server + `measure.mjs`, do it and paste the ≥1280 lines
|
||||
for `/dashboard/articles`, `/dashboard/runs` and `/dashboard/runs/1` into the
|
||||
handoff. With unchanged templates some tables will still scroll at 1280 —
|
||||
that is expected until the template audit lands — but at 2560 the articles
|
||||
and candidates tables must fit without scrolling.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Task 02 — template audit: every cell says what it holds
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-tables-tpl` (branch `tables-tpl`).
|
||||
Read `00-shared.md` and `docs/dashboard-tables.md` first. Only files under
|
||||
`src/web/templates/` change in this task (plus your handoff). The CSS lands in a
|
||||
sibling worktree; assume the four cell classes behave exactly as the doc's
|
||||
table says (`cell-wrap` floor 10rem / ceiling 32rem with `overflow-wrap:anywhere`,
|
||||
`cell-tight` nowrap, `td.num` nowrap right-aligned, unclassed cells wrap at
|
||||
spaces, `th` always wraps, page-level tables grow past the page column when
|
||||
their content needs it).
|
||||
|
||||
Go through every `<table>` in `src/web/templates/dashboard/*.html`,
|
||||
`src/web/templates/_candidate_row.html` and `src/web/templates/_signals_table.html`
|
||||
and apply these rules to each `td`:
|
||||
|
||||
1. **`cell-tight` only on timestamps, dates, ids and single tokens ≤ ~20
|
||||
characters** (`first_seen`, `when`, `started`, `finished`, `requested`,
|
||||
`created`, `last_login`, `saved_at`, `changed_at`, `event_at`, `issue_date`,
|
||||
`published`, `run.date`, `run.funnel` "310 → 200 → …", `age → decay`,
|
||||
`#id`). **Remove it** from feed names/titles (`article.feed`, `candidate.feed`,
|
||||
`miss.feed`, `row.feed_credits`, nearest-article `feed`), reasons
|
||||
(`article.reason`, `row.reason`, `candidate.reason`), "admitted by" cells,
|
||||
usernames / `requested_by` / `saved_by`, roles, sources, kinds, job names,
|
||||
and from any cell that combines a badge with more text (stage + run date).
|
||||
A `.badge` is already `whitespace-nowrap` on its own, so a badge-only cell
|
||||
needs no class.
|
||||
2. **`cell-wrap` on every free-form column**: titles, URLs, notes, messages,
|
||||
"reason or comment", facet values, badge lists (interests), config diff
|
||||
before/after, anything a user typed. Keep it where it already is.
|
||||
3. **`num` on numbers only** (it already is; just make sure no text column
|
||||
uses it because it wanted right alignment — `#id` cells may keep `num`).
|
||||
4. **Long text that should not stretch a row**: if a column can hold hundreds of
|
||||
characters (job `message`, rating-import `message`, `note`), keep
|
||||
`cell-wrap` and wrap the text in `<div class="line-clamp-3" title="{{ … }}">`
|
||||
**only** if the full text is reachable somewhere else on the page or via the
|
||||
`title`; otherwise let it wrap. Never put `truncate`/`line-clamp-*` on a `td`.
|
||||
5. **`dashboard/feeds.html`**: drop `table-fixed` (fixed layout is ignored once
|
||||
the page-level table is `width:auto`). Keep the `w-*` hints on the `th`s
|
||||
(they act as minimums under auto layout), keep the `line-clamp-*`/`truncate`
|
||||
inner elements — they already sit on block children, which is the pattern —
|
||||
and put `cell-wrap` on the Feed and Why cells if it is not there. The
|
||||
actions cell keeps its forms; give it `cell-tight` only if the buttons wrap
|
||||
badly, otherwise leave it unclassed.
|
||||
6. **`dashboard/run.html` Feeds card** (`<table class="table-fixed">` inside
|
||||
`.card`): this table is `w-full` inside a card, so `table-fixed` still works
|
||||
there; leave it, but it is the only allowed `table-fixed`.
|
||||
7. **Header cells**: remove nothing, but make sure no `th` has `whitespace-nowrap`
|
||||
or `cell-tight`; long headers such as
|
||||
"considered → eligible → triaged → assessed → shortlisted → selected" may wrap.
|
||||
8. **Wrappers**: every page-level table (more than a handful of columns:
|
||||
runs, candidates, articles, ratings current/events, interests, feeds, users,
|
||||
jobs, stats runs/costs, settings history, profile versions, near misses,
|
||||
funnel) must sit in a `<div class="scroll-x …">` that is a **direct child**
|
||||
of `<section class="dashboard …">` — check that no such wrapper is nested in
|
||||
a stray `<div>`; tables inside `.card`/`.cards`/`details` stay where they are.
|
||||
Keep `tall` where it is today.
|
||||
|
||||
Do not change the columns themselves, their order, the text, or any
|
||||
`data-*`/`id`/`name`/`action`/`aria-*` attribute. Do not touch reader/public
|
||||
templates (`issue_*.html`, `article.html`, `world.html`, `behind.html`, `_toc.html`).
|
||||
|
||||
Verify: `cargo test --lib web::` (router tests assert on text and some class
|
||||
names; if one breaks only because a class moved, update the test and say so),
|
||||
then `cargo fmt`/`clippy`. List in the handoff, per template, which cells lost
|
||||
`cell-tight`, which gained `cell-wrap`, and any clamp you added.
|
||||
@@ -0,0 +1,75 @@
|
||||
# Dashboard tables CSS handoff
|
||||
|
||||
## What changed
|
||||
|
||||
- `src/web/tailwind.css`
|
||||
- Disabled Tailwind v4 automatic repository scanning with
|
||||
`@import "tailwindcss" source(none);`; retained the explicit template and
|
||||
`app.js` sources.
|
||||
- Replaced the padded/max-width dashboard container with the specified named
|
||||
`content`/`wide` grid and 1rem/1.5rem gutter variable.
|
||||
- Let direct-child `.scroll-x` wrappers occupy the wide track and made their
|
||||
tables shrink-to-fit between the content-column floor and wrapper ceiling.
|
||||
- Removed the global `.scroll-x > table { min-width:max-content; }` rule.
|
||||
- Kept `.num` alignment/tabular figures on headers and cells, while limiting
|
||||
nowrap to `td.num`.
|
||||
- Raised `.cell-wrap`'s maximum width from 24rem to 32rem.
|
||||
- Added a pointer from the table rules to `docs/dashboard-tables.md`.
|
||||
- `src/web/static/app.css`
|
||||
- Rebuilt from the Tailwind source and checked for reproducibility.
|
||||
|
||||
No templates, JavaScript, reader table rules, preview rules, or rating-cell
|
||||
rules changed.
|
||||
|
||||
## Generated CSS diff audit
|
||||
|
||||
Compared committed and rebuilt CSS one closing brace per line with:
|
||||
|
||||
```sh
|
||||
diff -U0 \
|
||||
<(git show HEAD:src/web/static/app.css | tr '}' '\n') \
|
||||
<(tr '}' '\n' < src/web/static/app.css)
|
||||
```
|
||||
|
||||
The 177-line diff contained 111 removed and 12 added content lines (excluding
|
||||
the two diff file headers). The component-rule changes were exactly the table
|
||||
framework changes above: remove `.scroll-x > table`, replace `.dashboard`, add
|
||||
the three direct-child dashboard selectors, split `td.num` nowrap from `.num`,
|
||||
and change `.cell-wrap` from 24rem to 32rem. The remainder was removal of
|
||||
utilities, unused theme symbols, and supporting custom properties that had
|
||||
previously been emitted by repository-wide automatic source discovery. There
|
||||
were no unrelated added selectors.
|
||||
|
||||
The rebuilt file shrank from 77,721 to 70,697 bytes. The documentation-only
|
||||
utility selectors `.line-clamp-3`, `.max-w-[32rem]`, `.min-w-[10rem]`, and
|
||||
`.pb-6` are absent. The semantic `.cell-wrap` component still contains
|
||||
`min-width:10rem` and `max-width:32rem`, and `.dashboard` still contains its
|
||||
requested `padding-bottom` declaration.
|
||||
|
||||
## Verification
|
||||
|
||||
- `npm run css` — passed.
|
||||
- `npm run css:check` — passed with no diff.
|
||||
- `cargo build` — passed after the final CSS build.
|
||||
- `cargo test --lib web::` — 135 passed; the two failures were
|
||||
`web::dashboard::feeds::tests::a_duplicate_subscription_still_decides_the_row`
|
||||
and `web::dashboard::feeds::tests::add_subscribes_and_records_the_miniflux_feed_id`.
|
||||
Both failed at the test bind setup with the documented
|
||||
`PermissionDenied` / `Operation not permitted` sandbox error.
|
||||
- `cargo fmt` — passed without changes.
|
||||
- `cargo clippy --all-targets -- -D warnings` — passed.
|
||||
- `git diff --check` — passed.
|
||||
|
||||
An extra full `cargo test` required by the shared brief was attempted, but the
|
||||
filesystem ran out of space while creating another test archive before tests
|
||||
started. I removed only this package's generated artifacts with
|
||||
`cargo clean -p daily-epub` (2.0 GiB), then reran the complete user-requested
|
||||
command sequence above successfully. The full suite remains unverified; the
|
||||
focused web suite is verified subject only to the two allowed bind failures.
|
||||
|
||||
## Deviations and open items
|
||||
|
||||
There are no implementation deviations. I did not run the browser measurement
|
||||
script because the sandbox cannot bind a preview server, as independently
|
||||
confirmed by the focused test failures. Visual viewport measurement remains for
|
||||
the post-merge environment described in the shared brief.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Dashboard tables template handoff
|
||||
|
||||
Implemented `02-templates.md` against `00-shared.md` and
|
||||
`docs/dashboard-tables.md`. No columns, content, hooks, or attributes were
|
||||
changed. The existing page-level `.scroll-x` wrappers already had the required
|
||||
direct-child topology, so none moved.
|
||||
|
||||
## Cell audit by template
|
||||
|
||||
| Template | Cells that lost `cell-tight` | Cells that gained `cell-wrap` | Clamp added |
|
||||
| --- | --- | --- | --- |
|
||||
| `_candidate_row.html` | Feed, stage, reason, admitted by, flags | Flags (badge list) | None |
|
||||
| `_signals_table.html` | None | None | None |
|
||||
| `dashboard/article.html` | Run history: stage, reason, admitted by; nearest articles: feed, rating; rating events: label, kind, source, user | Top-interests interest name | Rating-event note: inner `line-clamp-3` with the full note in `title` |
|
||||
| `dashboard/articles.html` | Feed, last stage (badge plus run date), reason, rating | None | None |
|
||||
| `dashboard/feeds.html` | None | None (Feed and Why already had it) | None (existing Feed/Why inner clamps retained) |
|
||||
| `dashboard/interests.html` | Category | None | None |
|
||||
| `dashboard/jobs.html` | Job name, requested by, status | None (message already had it) | Message: inner `line-clamp-3` with the full message in `title` |
|
||||
| `dashboard/profile.html` | Saved by, restore action | Preview | None |
|
||||
| `dashboard/ratings.html` | Imports: verdict, status; events: verdict, source, by; current: when/by, feed credit, used last run | Current feed credit | Import message, event note, and current note: inner `line-clamp-3` with full text in `title` |
|
||||
| `dashboard/run.html` | Funnel stage; near misses: feed, stage | None | None |
|
||||
| `dashboard/runs.html` | Status (badge plus dry-run text) | None | None |
|
||||
| `dashboard/settings_history.html` | Who | None | None |
|
||||
| `dashboard/stats.html` | Runs status | None | None |
|
||||
| `dashboard/users.html` | None | None | None |
|
||||
|
||||
`dashboard/feeds.html` also lost page-level `table-fixed`; its existing header
|
||||
width hints and inner Feed/Why clamps remain. The card-local `table-fixed` feeds
|
||||
table in `dashboard/run.html` remains as specified.
|
||||
|
||||
## Deviations
|
||||
|
||||
None. Templates whose cells were already compliant were left unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo test --lib web::`: 135 passed; two Miniflux mock-server tests failed
|
||||
only because sandbox `bind()` returned `PermissionDenied` at
|
||||
`src/web/dashboard/feeds.rs:497`, as permitted by the brief.
|
||||
- `cargo fmt`: passed.
|
||||
- `cargo clippy --all-targets -- -D warnings`: passed.
|
||||
- `git diff --check`: passed.
|
||||
|
||||
The first clippy attempt exhausted the filesystem while writing generated Rust
|
||||
metadata. Removing the disposable `target/debug/incremental` cache freed enough
|
||||
space; the exact clippy command then completed successfully.
|
||||
@@ -0,0 +1,42 @@
|
||||
// usage: node measure.mjs <port> <outdir> [widths]
|
||||
import { chromium } from "playwright";
|
||||
import fs from "node:fs";
|
||||
const [port, outdir, widthsArg] = process.argv.slice(2);
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
const widths = (widthsArg || "390,768,1280,1920,2560").split(",").map(Number);
|
||||
const pages = ["/dashboard","/dashboard/runs","/dashboard/runs/1","/dashboard/runs/2","/dashboard/articles","/dashboard/articles/1","/dashboard/ratings","/dashboard/ratings?tab=events","/dashboard/interests","/dashboard/feeds","/dashboard/feeds?status=added","/dashboard/profile","/dashboard/stats","/dashboard/jobs","/dashboard/settings","/dashboard/settings/history","/dashboard/users","/","/issues","/account"];
|
||||
fs.mkdirSync(outdir, { recursive: true });
|
||||
const browser = await chromium.launch();
|
||||
const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } });
|
||||
const page = await ctx.newPage();
|
||||
await page.goto(`${base}/login`);
|
||||
await page.fill('input[name="username"]', "admin");
|
||||
await page.fill('input[name="password"]', "adminpassword123");
|
||||
await page.click('button[type="submit"]');
|
||||
await page.waitForLoadState("networkidle");
|
||||
const report = [];
|
||||
for (const w of widths) {
|
||||
await page.setViewportSize({ width: w, height: 1000 });
|
||||
for (const p of pages) {
|
||||
const resp = await page.goto(`${base}${p}`, { waitUntil: "networkidle" });
|
||||
const status = resp?.status();
|
||||
const info = await page.evaluate(() => {
|
||||
const de = document.documentElement;
|
||||
const container = document.querySelector(".dashboard, main > section, main > div, main");
|
||||
const tables = [...document.querySelectorAll("table")].map((t) => {
|
||||
const host = t.closest(".scroll-x") || t.parentElement;
|
||||
const r = t.getBoundingClientRect();
|
||||
return { cls: t.className, ths: t.querySelectorAll("thead th").length, rows: t.querySelectorAll("tbody tr").length, tableW: Math.round(r.width), hostW: Math.round(host.getBoundingClientRect().width), hostScroll: host.scrollWidth, hostClient: host.clientWidth, overflow: host.scrollWidth > host.clientWidth + 1 };
|
||||
});
|
||||
const culprits = [...document.querySelectorAll("body *")].filter((el) => el.scrollWidth > el.clientWidth + 1 && getComputedStyle(el).overflowX === "visible" && !el.closest("table")).slice(0, 5).map((el) => `${el.tagName.toLowerCase()}.${String(el.className).split(" ").slice(0,2).join(".")} ${el.scrollWidth}>${el.clientWidth}`);
|
||||
return { pageOverflow: de.scrollWidth > de.clientWidth, docW: de.clientWidth, containerW: container ? Math.round(container.getBoundingClientRect().width) : null, tables, culprits };
|
||||
});
|
||||
const name = p.replace(/[\/?=]/g, "_").replace(/^_/, "") || "root";
|
||||
if (w === 1280 || w === 2560 || w === 390) await page.screenshot({ path: `${outdir}/${w}-${name}.png`, fullPage: true });
|
||||
report.push({ w, p, status, ...info });
|
||||
const bad = info.tables.filter((t) => t.overflow);
|
||||
console.log(`${w} ${p} status=${status} page=${info.pageOverflow ? "OVERFLOW" : "ok"} container=${info.containerW} tables=${info.tables.length} scrolling=${bad.length}${bad.length ? " [" + bad.map((t) => `${t.cls || "-"}:${t.hostScroll}>${t.hostClient}`).join(", ") + "]" : ""}${info.culprits.length ? " culprits=" + info.culprits.join(" | ") : ""}`);
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(`${outdir}/report.json`, JSON.stringify(report, null, 1));
|
||||
await browser.close();
|
||||
@@ -9,7 +9,7 @@ What changes for the operator, in one paragraph: the binary is replaced; the SQL
|
||||
tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first);
|
||||
`config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a
|
||||
`profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained
|
||||
`profile.md` is installed next to the OPML; the systemd units are unchanged.
|
||||
`profile.md` is installed for the hand-maintained reader profile; the systemd units are unchanged.
|
||||
|
||||
## 0. Before touching the server
|
||||
|
||||
@@ -70,12 +70,13 @@ Everything you do not mention keeps its documented default, so the edit is small
|
||||
|
||||
| Old key | Why |
|
||||
|---|---|
|
||||
| `interests_opml = …` (top level) | standing interests now live in SQLite; remove this before rollout because unknown keys fail startup |
|
||||
| `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) |
|
||||
| `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` |
|
||||
| the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) |
|
||||
| any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` |
|
||||
|
||||
**Add** near the top, next to `interests_opml`:
|
||||
**Add** near the top:
|
||||
|
||||
```toml
|
||||
profile_path = "/var/lib/daily-epub/data/profile.md"
|
||||
@@ -83,8 +84,7 @@ profile_path = "/var/lib/daily-epub/data/profile.md"
|
||||
|
||||
Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which
|
||||
under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an
|
||||
explicit path survives running one-off commands from another directory. Point
|
||||
`interests_opml` at an absolute path too if it is still relative.
|
||||
explicit path survives running one-off commands from another directory.
|
||||
|
||||
**Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old
|
||||
`[deepseek]` table if you had changed them; the values shown are the defaults.
|
||||
@@ -165,11 +165,9 @@ Voyage dashboards: the in-app `max_daily_usd` meters are runaway guards, not acc
|
||||
```sh
|
||||
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data
|
||||
sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md
|
||||
# if the OPML is not already there:
|
||||
sudo install -m0640 -o daily-epub -g daily-epub data/scour-interests.opml /var/lib/daily-epub/data/
|
||||
```
|
||||
|
||||
If the file is missing the run does not fail; it logs a warning and uses the OPML interests only,
|
||||
If the file is missing the run does not fail; it logs a warning and uses empty profile prose,
|
||||
which is a much worse prompt. `config check` in the next step tells you whether it was found.
|
||||
|
||||
## 6. Check the config as the service user
|
||||
@@ -214,7 +212,7 @@ de generate --dry-run --out /var/lib/daily-epub/out-check
|
||||
The dry run makes real DeepSeek, Claude and Voyage calls but publishes nothing and writes no
|
||||
`issues` row. Read the printed lineup and the four report lines (`curation:`, `admission:`,
|
||||
`preference:`, `providers:`); the cost should be well under $1. Then read the EPUB it wrote
|
||||
(Calibre or KOReader): The Brief, the `Why it's here` line under each headline, and the new
|
||||
(Calibre or KOReader): The Brief and the new
|
||||
Behind-the-paper chapter before the colophon. Finally:
|
||||
|
||||
```sh
|
||||
@@ -263,7 +261,7 @@ content filter (those are bisected, retried once on the editor provider, and the
|
||||
`provider_rejected` so they are not retried daily). The `triage:` and `assess:` log lines say how
|
||||
many were reused versus requested. The bulk-side cost of the second run is therefore small; the
|
||||
editor, summaries and the Brief are the real spend. Compare the two
|
||||
lineups, the `why` lines and the Brief side by side, and the `providers:` cost line. To switch
|
||||
lineups and the Brief side by side, and the `providers:` cost line. To switch
|
||||
for good, set `editor = "gemini"` in `[llm]` (and `summary_model` stays `editor`, so summaries
|
||||
move with it). The same trick works for the bulk role: `DAILY_EPUB_LLM__BULK=gemini`.
|
||||
|
||||
|
||||
@@ -370,7 +370,6 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
|
||||
section: target.issue.clone(),
|
||||
position: 0,
|
||||
is_lead: false,
|
||||
why: None,
|
||||
summary: None,
|
||||
llm: None,
|
||||
top_interests: Vec::new(),
|
||||
|
||||
+24
-19
@@ -29,14 +29,13 @@ use jiff::civil::Date;
|
||||
const ADMIN_PASSWORD: &str = "adminpassword123";
|
||||
const READER_PASSWORD: &str = "readerpassword123";
|
||||
|
||||
/// (section, title, feed, summary, why, words)
|
||||
const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
/// (section, title, feed, summary, words)
|
||||
const STORIES: &[(&str, &str, &str, &str, i64)] = &[
|
||||
(
|
||||
"Top Stories",
|
||||
"How a Forty-Year-Old Filesystem Quietly Rewrote Its Write Path",
|
||||
"Systems Weekly",
|
||||
"A long, careful account of the redesign, with the benchmarks that justified it and the two regressions that nearly sank it.",
|
||||
"The systems story with enough operational detail to matter to you",
|
||||
3100,
|
||||
),
|
||||
(
|
||||
@@ -44,7 +43,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"The Case Against Feature Flags",
|
||||
"Alice on Software",
|
||||
"Argues that flags outlive their purpose and proposes a retirement discipline, with examples from three codebases.",
|
||||
"A contrarian take on tooling you use every day",
|
||||
1850,
|
||||
),
|
||||
(
|
||||
@@ -52,7 +50,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"What the Latest Battery Chemistry Actually Changes",
|
||||
"Ars Technica",
|
||||
"Separates the press-release claims from the measurable improvements in energy density and cycle life.",
|
||||
"You keep an eye on energy storage; this one is unusually sober",
|
||||
2200,
|
||||
),
|
||||
(
|
||||
@@ -60,7 +57,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"A Field Guide to Distributed Consensus, Told Through One Outage",
|
||||
"The Morning Paper",
|
||||
"Walks through a real incident to explain leader election, log replication and why the fix was a config change.",
|
||||
"Distributed systems explained with an actual outage rather than diagrams",
|
||||
4200,
|
||||
),
|
||||
(
|
||||
@@ -68,7 +64,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"Why Every Map Is a Lie & How Cartographers Choose Which One to Tell",
|
||||
"Longreads",
|
||||
"A history of projections and the politics behind them, from Mercator to the maps in your phone.",
|
||||
"Long-form nonfiction outside the technical orbit you asked for",
|
||||
5100,
|
||||
),
|
||||
(
|
||||
@@ -76,7 +71,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"Notes on Writing a Rust Linter That People Actually Enable",
|
||||
"Rust Blog",
|
||||
"Design notes on false-positive budgets, fix suggestions and the social side of shipping a lint.",
|
||||
"Rust tooling with a practical bent",
|
||||
1600,
|
||||
),
|
||||
(
|
||||
@@ -84,7 +78,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"The Terminal Is the Best UI We Have and It Is Getting Better",
|
||||
"Julia's Notebook",
|
||||
"A tour of modern terminal features (hyperlinks, images, synchronized output) and which tools use them.",
|
||||
"Terminal ergonomics, one of your recurring interests",
|
||||
1300,
|
||||
),
|
||||
(
|
||||
@@ -92,7 +85,6 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"A Small-Town Bakery's Sourdough Starter Turns One Hundred",
|
||||
"Saveur",
|
||||
"A charming profile of a starter kept alive across four generations and the bread it still makes.",
|
||||
"A small-scene delight outside the usual technical orbit",
|
||||
900,
|
||||
),
|
||||
(
|
||||
@@ -100,15 +92,14 @@ const STORIES: &[(&str, &str, &str, &str, &str, i64)] = &[
|
||||
"Inside the Community Keeping 1990s Synthesizers Alive",
|
||||
"Sound on Sound",
|
||||
"Repair collectives, replacement parts and the odd economics of vintage gear.",
|
||||
"Music hardware, for the weekend",
|
||||
1700,
|
||||
),
|
||||
];
|
||||
|
||||
fn story_article(index: usize, story: &(&str, &str, &str, &str, &str, i64)) -> Article {
|
||||
fn story_article(index: usize, story: &(&str, &str, &str, &str, i64)) -> Article {
|
||||
let id = index as i64 + 1;
|
||||
let entry_id = 1000 + id;
|
||||
let (_, title, feed, _, _, words) = *story;
|
||||
let (_, title, feed, _, words) = *story;
|
||||
let mut article = fixtures::article(id, entry_id, title);
|
||||
article.feed_title = feed.to_string();
|
||||
article.feed_id = 7 + index as i64;
|
||||
@@ -140,9 +131,25 @@ fn body_html(title: &str, words: i64) -> String {
|
||||
html.push_str("<blockquote><p>A pull quote that the author would rather you remembered than the rest.</p></blockquote>");
|
||||
}
|
||||
if index == 7 {
|
||||
html.push_str(
|
||||
"<pre><code>fn main() {\n println!(\"hello, reader\");\n}</code></pre>",
|
||||
);
|
||||
// One block the source marked as Rust, one it left for detection.
|
||||
html.push_str(concat!(
|
||||
"<pre data-lang=\"rust\"><code>/// Greets whoever is reading.\n",
|
||||
"fn main() {\n",
|
||||
" let reader: &str = \"reader\";\n",
|
||||
" for n in 0..3 {\n",
|
||||
" println!(\"hello, {reader} #{n}\"); // three times\n",
|
||||
" }\n",
|
||||
"}</code></pre>",
|
||||
"<pre><code>import json\n\n",
|
||||
"def load(path: str) -> dict:\n",
|
||||
" \"\"\"Read a JSON config, or an empty one.\"\"\"\n",
|
||||
" try:\n",
|
||||
" with open(path) as f:\n",
|
||||
" return json.load(f)\n",
|
||||
" except FileNotFoundError:\n",
|
||||
" return {} # first run\n",
|
||||
"</code></pre>",
|
||||
));
|
||||
}
|
||||
html.push_str("<p>Paragraph ");
|
||||
html.push_str(&(index + 1).to_string());
|
||||
@@ -167,7 +174,7 @@ fn dev_issue(date: Date, issue_number: i64, generated_at: Timestamp) -> Issue {
|
||||
let mut picks = Vec::new();
|
||||
let mut summaries = BTreeMap::new();
|
||||
for (index, story) in STORIES.iter().enumerate() {
|
||||
let (section, _, _, summary, why, _) = *story;
|
||||
let (section, _, _, summary, _) = *story;
|
||||
if !section_order.iter().any(|s| s == section) {
|
||||
section_order.push(section.to_string());
|
||||
}
|
||||
@@ -237,7 +244,6 @@ fn dev_issue(date: Date, issue_number: i64, generated_at: Timestamp) -> Issue {
|
||||
section: section.to_string(),
|
||||
position,
|
||||
is_lead: index == 0,
|
||||
why: Some(why.to_string()),
|
||||
summary: Some(summary.to_string()),
|
||||
llm,
|
||||
top_interests,
|
||||
@@ -418,7 +424,6 @@ async fn seed_near_misses(db: &Db, run_id: i64) -> anyhow::Result<()> {
|
||||
rank_utility: Some(index + 10),
|
||||
cluster_id: Some(index),
|
||||
cluster_rank: Some(1),
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
//! Preview an article the way the Xteink X4 will show it.
|
||||
//!
|
||||
//! Browsers are a poor stand-in for the X4: its pages are rendered ahead of
|
||||
//! time by the XTC converter's CREngine, which has its own defaults (`code` is a
|
||||
//! block, `pre` has no margin) and only four grey levels. This builds both
|
||||
//! EPUB editions around one article, runs the real converter on the X4 one and
|
||||
//! decodes every page to `page-NNN.png`.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --example x4_preview -- <article URL | HTML file> <out dir>
|
||||
//! cargo run --example x4_preview -- --convert <x4.epub> <out dir>
|
||||
//! ```
|
||||
//!
|
||||
//! A URL goes through the real extraction path; an HTML file is used as the
|
||||
//! stored body as-is. `--convert` skips the build, for iterating on a
|
||||
//! stylesheet inside an unzipped EPUB. Needs the converter at
|
||||
//! `/opt/epub-to-xtc-converter` and runs it with `xtc-settings.example.json`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use daily_epub::config::Config;
|
||||
use daily_epub::epub;
|
||||
use daily_epub::extract::Extractor;
|
||||
use daily_epub::types::Edition;
|
||||
|
||||
const CONVERTER: &str = "/opt/epub-to-xtc-converter/cli/index.js";
|
||||
const SETTINGS: &str = "xtc-settings.example.json";
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.as_slice() {
|
||||
[flag, epub, out] if flag == "--convert" => {
|
||||
let pages = convert(Path::new(epub), Path::new(out))?;
|
||||
println!("{pages} pages in {out}");
|
||||
}
|
||||
[source, out] => {
|
||||
let out = PathBuf::from(out);
|
||||
std::fs::create_dir_all(&out)?;
|
||||
let html = if source.starts_with("http") {
|
||||
let extractor = Extractor::new(reqwest::Client::new(), vec![]);
|
||||
let page = extractor.fetch_readable(source).await?;
|
||||
extractor.finish_readable(source, &page).content_html
|
||||
} else {
|
||||
std::fs::read_to_string(source)?
|
||||
};
|
||||
std::fs::write(out.join("stored.html"), &html)?;
|
||||
|
||||
let mut issue = epub::fixtures::issue();
|
||||
issue.lineup.picks.truncate(1);
|
||||
issue.lineup.picks[0].article.content_html = html;
|
||||
let config = Config::default();
|
||||
for edition in [Edition::Standard, Edition::X4] {
|
||||
let artifact =
|
||||
epub::build_edition_with_images(&issue, edition, &config, &out, &[])?;
|
||||
println!("{edition:?}: {}", artifact.path.display());
|
||||
if edition == Edition::X4 {
|
||||
let pages = convert(&artifact.path, &out)?;
|
||||
println!("{pages} X4 pages in {}", out.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!(
|
||||
"usage: x4_preview <URL | HTML file> <out dir> | --convert <epub> <out dir>"
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the converter on `epub` and decode its pages into `out`.
|
||||
fn convert(epub: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
std::fs::create_dir_all(out)?;
|
||||
let xtch = out.join("x4.xtch");
|
||||
let status = Command::new("node")
|
||||
.args([CONVERTER, "convert"])
|
||||
.arg(epub)
|
||||
.arg("-o")
|
||||
.arg(&xtch)
|
||||
.args(["-f", "xtch", "-c", SETTINGS])
|
||||
.stdout(Stdio::null())
|
||||
.status()?;
|
||||
anyhow::ensure!(status.success(), "converter failed: {status}");
|
||||
decode_xtch(&xtch, out)
|
||||
}
|
||||
|
||||
fn u16_at(bytes: &[u8], at: usize) -> usize {
|
||||
u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize
|
||||
}
|
||||
|
||||
fn u64_at(bytes: &[u8], at: usize) -> usize {
|
||||
u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize
|
||||
}
|
||||
|
||||
/// Decode every page of an XTCH container to a PNG.
|
||||
///
|
||||
/// Layout per the converter's `docs/xtc-format-spec.md`: the page index is at
|
||||
/// the offset in header bytes 0x18..0x20, 16 bytes per page (offset, size,
|
||||
/// width, height). Each page is a 22-byte header and two bit planes stored
|
||||
/// column by column, right to left, 8 vertical pixels per byte. The plane
|
||||
/// order follows the encoder (`cli/encoder.js`), not the spec's table: the
|
||||
/// first plane holds the level's low bit, and 0b01 is dark grey.
|
||||
fn decode_xtch(path: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
let bytes = std::fs::read(path)?;
|
||||
let pages = u16_at(&bytes, 0x06);
|
||||
let index = u64_at(&bytes, 0x18);
|
||||
for page in 0..pages {
|
||||
let entry = index + page * 16;
|
||||
let (offset, width, height) = (
|
||||
u64_at(&bytes, entry),
|
||||
u16_at(&bytes, entry + 12),
|
||||
u16_at(&bytes, entry + 14),
|
||||
);
|
||||
let plane = (width * height).div_ceil(8);
|
||||
let column = height.div_ceil(8);
|
||||
let data = &bytes[offset + 22..];
|
||||
let mut image = image::GrayImage::new(width as u32, height as u32);
|
||||
for x in 0..width {
|
||||
for y in 0..height {
|
||||
let byte = (width - 1 - x) * column + y / 8;
|
||||
let bit = 7 - (y % 8);
|
||||
let low = (data[byte] >> bit) & 1;
|
||||
let high = (data[plane + byte] >> bit) & 1;
|
||||
let shade = [255, 85, 170, 0][usize::from(high << 1 | low)];
|
||||
image.put_pixel(x as u32, y as u32, image::Luma([shade]));
|
||||
}
|
||||
}
|
||||
image.save(out.join(format!("page-{page:03}.png")))?;
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
@@ -117,7 +117,7 @@ users-passwd *args:
|
||||
users *args:
|
||||
{{run}} users "$@"
|
||||
|
||||
# Run one catalogue job in-process (generate, dry-run, profile-rebuild, features-backfill, backfill-social, features-prune, import-ratings).
|
||||
# Run one catalogue job in-process (generate, dry-run, profile-rebuild, features-backfill, backfill-social, features-prune, import-ratings, interests-categorize).
|
||||
job name:
|
||||
{{run}} job run {{name}}
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE interests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
category TEXT, -- NULL until categorized
|
||||
created_at TEXT NOT NULL,
|
||||
categorized_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_interests_category ON interests(category);
|
||||
|
||||
CREATE TABLE article_interests (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||
cos REAL NOT NULL,
|
||||
z REAL NOT NULL,
|
||||
run_id INTEGER, -- NULL for backfilled rows
|
||||
PRIMARY KEY (article_id, interest_id)
|
||||
);
|
||||
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Related articles under each web issue article: the nearest embedded articles
|
||||
-- across the whole archive, computed when the issue is published (and by
|
||||
-- `features related` for older issues). Stored rather than computed per page
|
||||
-- view because finding them is a scan of every cached embedding.
|
||||
CREATE TABLE issue_related (
|
||||
issue_date TEXT NOT NULL REFERENCES issues(date) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL, -- 0 = closest
|
||||
related_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
cosine REAL NOT NULL,
|
||||
PRIMARY KEY (issue_date, article_id, position)
|
||||
);
|
||||
+38
-18
@@ -37,12 +37,9 @@ impl From<figment::Error> for ConfigError {
|
||||
/// Legacy/alternate env var for the rating-link HMAC key (spec §1).
|
||||
pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET";
|
||||
|
||||
/// Root configuration document (§3.14).
|
||||
///
|
||||
/// Unknown *top-level* keys are ignored on purpose: the prefix `DAILY_EPUB_` is
|
||||
/// shared with plain operator env vars such as [`ENV_SECRET_ALIAS`].
|
||||
/// Root configuration document; unknown keys fail so retired settings stay visible.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct Config {
|
||||
/// IANA tz used for day boundaries and `--date` (§3.14, notes §2).
|
||||
pub timezone: String,
|
||||
@@ -65,8 +62,6 @@ pub struct Config {
|
||||
pub database_path: PathBuf,
|
||||
/// Default artifact output directory (overridden by `generate --out`).
|
||||
pub out_dir: PathBuf,
|
||||
/// Scour interests OPML used to seed the taste profile (§3.6).
|
||||
pub interests_opml: PathBuf,
|
||||
/// Hand-maintained reader profile loaded for every curation run (§8.2).
|
||||
pub profile_path: PathBuf,
|
||||
|
||||
@@ -99,7 +94,6 @@ impl Default for Config {
|
||||
world_briefing: true,
|
||||
database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"),
|
||||
out_dir: PathBuf::from("/var/lib/daily-epub/out"),
|
||||
interests_opml: PathBuf::from("data/scour-interests.opml"),
|
||||
profile_path: PathBuf::from("data/profile.md"),
|
||||
miniflux: MinifluxConfig::default(),
|
||||
llm: LlmConfig::default(),
|
||||
@@ -488,12 +482,16 @@ pub struct RankingConfig {
|
||||
pub knn_full: usize,
|
||||
pub feed_floor: usize,
|
||||
pub feed_full: usize,
|
||||
pub affinity_floor: usize,
|
||||
pub affinity_full: usize,
|
||||
/// Fraction of the preliminary blend and the utility removed from any
|
||||
/// candidate whose author has a current *AI slop* verdict (§9.3). `1.0`
|
||||
/// zeroes such candidates; `0.0` disables the penalty.
|
||||
pub slop_author_penalty: f64,
|
||||
pub semantic_min_words: i64,
|
||||
pub exploration_slots: usize,
|
||||
/// Days before `features prune` drops unrated, unpublished vectors; 0 keeps
|
||||
/// them forever (the default, so related articles span the whole archive).
|
||||
pub embedding_retention_days: i64,
|
||||
pub telemetry_retention_days: i64,
|
||||
pub quotas: RankingQuotas,
|
||||
@@ -516,10 +514,12 @@ impl Default for RankingConfig {
|
||||
knn_full: 25,
|
||||
feed_floor: 15,
|
||||
feed_full: 40,
|
||||
affinity_floor: 15,
|
||||
affinity_full: 40,
|
||||
slop_author_penalty: 0.75,
|
||||
semantic_min_words: 300,
|
||||
exploration_slots: 5,
|
||||
embedding_retention_days: 120,
|
||||
embedding_retention_days: 0,
|
||||
telemetry_retention_days: 180,
|
||||
quotas: RankingQuotas::default(),
|
||||
weights: RankingWeights::default(),
|
||||
@@ -558,6 +558,7 @@ pub struct RankingWeights {
|
||||
pub struct PreliminaryWeights {
|
||||
pub interest: f64,
|
||||
pub knn: f64,
|
||||
pub affinity: f64,
|
||||
pub heuristic: f64,
|
||||
pub feed: f64,
|
||||
pub social: f64,
|
||||
@@ -566,11 +567,12 @@ pub struct PreliminaryWeights {
|
||||
impl Default for PreliminaryWeights {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
interest: 0.35,
|
||||
interest: 0.30,
|
||||
knn: 0.25,
|
||||
affinity: 0.10,
|
||||
heuristic: 0.20,
|
||||
feed: 0.10,
|
||||
social: 0.10,
|
||||
social: 0.05,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -581,6 +583,7 @@ pub struct UtilityWeights {
|
||||
pub quality: f64,
|
||||
pub fit: f64,
|
||||
pub knn: f64,
|
||||
pub affinity: f64,
|
||||
pub interest: f64,
|
||||
pub feed: f64,
|
||||
pub triage: f64,
|
||||
@@ -593,7 +596,8 @@ impl Default for UtilityWeights {
|
||||
Self {
|
||||
quality: 0.40,
|
||||
fit: 0.20,
|
||||
knn: 0.15,
|
||||
knn: 0.10,
|
||||
affinity: 0.05,
|
||||
interest: 0.10,
|
||||
feed: 0.05,
|
||||
triage: 0.05,
|
||||
@@ -934,7 +938,11 @@ impl Config {
|
||||
fig = fig.merge(Toml::file(p));
|
||||
}
|
||||
}
|
||||
Ok(fig.merge(Env::prefixed(ENV_PREFIX).split(ENV_SPLIT)))
|
||||
Ok(fig.merge(
|
||||
Env::prefixed(ENV_PREFIX)
|
||||
.ignore(&["secret"])
|
||||
.split(ENV_SPLIT),
|
||||
))
|
||||
}
|
||||
|
||||
/// The file `load` reads: the explicit `--config` path, else `./config.toml`
|
||||
@@ -1059,7 +1067,6 @@ impl Config {
|
||||
});
|
||||
lines.push(file_line("database_path", &self.database_path));
|
||||
lines.push(file_line("profile_path", &self.profile_path));
|
||||
lines.push(file_line("interests_opml", &self.interests_opml));
|
||||
for (role, name) in self.llm.roles() {
|
||||
match self.providers.get(name) {
|
||||
Some(provider) => lines.push(provider_line(&format!("llm.{role}"), name, provider)),
|
||||
@@ -1204,7 +1211,10 @@ impl Config {
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
if ranking.knn_full <= ranking.knn_floor || ranking.feed_full <= ranking.feed_floor {
|
||||
if ranking.knn_full <= ranking.knn_floor
|
||||
|| ranking.feed_full <= ranking.feed_floor
|
||||
|| ranking.affinity_full <= ranking.affinity_floor
|
||||
{
|
||||
return Err(ConfigError::Invalid(
|
||||
"curation.ranking *_full must be > *_floor >= 0".into(),
|
||||
));
|
||||
@@ -1229,12 +1239,14 @@ impl Config {
|
||||
let weights = [
|
||||
preliminary.interest,
|
||||
preliminary.knn,
|
||||
preliminary.affinity,
|
||||
preliminary.heuristic,
|
||||
preliminary.feed,
|
||||
preliminary.social,
|
||||
utility.quality,
|
||||
utility.fit,
|
||||
utility.knn,
|
||||
utility.affinity,
|
||||
utility.interest,
|
||||
utility.feed,
|
||||
utility.triage,
|
||||
@@ -2018,7 +2030,7 @@ mod tests {
|
||||
for (key, default) in defaults.as_object().expect("config is a table") {
|
||||
let section = match key.as_str() {
|
||||
"curation" | "llm" | "providers" | "voyage" | "editorial" => key,
|
||||
"target_article_count" | "profile_path" | "interests_opml" => key,
|
||||
"target_article_count" | "profile_path" => key,
|
||||
_ => continue,
|
||||
};
|
||||
let documented = documented
|
||||
@@ -2084,12 +2096,17 @@ mod tests {
|
||||
);
|
||||
assert_eq!((ranking.knn_floor, ranking.knn_full), (8, 25));
|
||||
assert_eq!((ranking.feed_floor, ranking.feed_full), (15, 40));
|
||||
assert_eq!((ranking.affinity_floor, ranking.affinity_full), (15, 40));
|
||||
assert_eq!(ranking.rating_half_life_days, 60.0);
|
||||
assert_eq!(ranking.negative_coefficient, 0.75);
|
||||
assert_eq!(ranking.weights.preliminary.interest, 0.35);
|
||||
assert_eq!(ranking.weights.preliminary.interest, 0.30);
|
||||
assert_eq!(ranking.weights.preliminary.affinity, 0.10);
|
||||
assert_eq!(ranking.weights.preliminary.social, 0.05);
|
||||
assert_eq!(ranking.weights.utility.quality, 0.40);
|
||||
assert_eq!(ranking.weights.utility.knn, 0.10);
|
||||
assert_eq!(ranking.weights.utility.affinity, 0.05);
|
||||
assert_eq!(ranking.diversity.per_cluster_cap, 2);
|
||||
assert_eq!(ranking.embedding_retention_days, 120);
|
||||
assert_eq!(ranking.embedding_retention_days, 0);
|
||||
assert_eq!(ranking.telemetry_retention_days, 180);
|
||||
cfg.validate().unwrap();
|
||||
|
||||
@@ -2106,6 +2123,9 @@ mod tests {
|
||||
bad.curation.ranking.knn_full = bad.curation.ranking.knn_floor;
|
||||
assert!(bad.validate().is_err(), "*_full must exceed *_floor");
|
||||
let mut bad = Config::default();
|
||||
bad.curation.ranking.affinity_full = bad.curation.ranking.affinity_floor;
|
||||
assert!(bad.validate().is_err(), "*_full must exceed *_floor");
|
||||
let mut bad = Config::default();
|
||||
bad.curation.ranking.shortlist_keep = bad.curation.ranking.deep_keep + 1;
|
||||
assert!(bad.validate().is_err(), "deep_keep >= shortlist_keep");
|
||||
let mut bad = Config::default();
|
||||
|
||||
+2
-76
@@ -19,8 +19,6 @@ pub struct SelectionItem {
|
||||
pub position: i64,
|
||||
#[serde(default)]
|
||||
pub lead_story: bool,
|
||||
#[serde(default)]
|
||||
pub why: Option<String>,
|
||||
}
|
||||
|
||||
pub const EDITOR_INSTRUCTIONS: &str = r#"TASK: assemble today's issue of The Daily EPUB from the shortlist below.
|
||||
@@ -39,16 +37,6 @@ RULES
|
||||
6. Never select two articles that tell the same story.
|
||||
7. SIZE: aim for about {soft_target}; never more than {hard_max}; there is NO minimum.
|
||||
If only nine pieces deserve the reader's morning, publish nine. Never pad.
|
||||
8. For every pick write "why": at most 14 words. It is printed under the headline
|
||||
as "Why it's here", above the summary, with the matched interests listed right
|
||||
beneath it. So it must NOT describe the piece (the summary does that) and must
|
||||
NOT just list interests (already shown). Say why THIS reader gets THIS piece
|
||||
today: the itch it scratches, the rated piece it rhymes with, the argument he
|
||||
will want to pick, the gap in today's paper it fills, or the exploration bet it
|
||||
is. Second person is fine.
|
||||
Bad: "A candid Rust query engine post-mortem showing why io_uring lost to mmap."
|
||||
Good: "The io_uring-versus-mmap verdict you'd want before touching your own engine."
|
||||
|
||||
EDITORIAL JUDGEMENT
|
||||
- Depth over coverage. Drop anything you would not defend to him in person.
|
||||
- Diversity is a feature: do not let one subject, one format, or one feed dominate,
|
||||
@@ -62,7 +50,7 @@ EDITORIAL JUDGEMENT
|
||||
- Scores are evidence, not instructions. Overrule them when the paper reads better.
|
||||
|
||||
Return JSON exactly:
|
||||
{"picks": [{"id": 123, "section": "Top Stories", "position": 1, "lead_story": true, "why": "…"}]}"#;
|
||||
{"picks": [{"id": 123, "section": "Top Stories", "position": 1, "lead_story": true}]}"#;
|
||||
|
||||
pub fn build_prompt(
|
||||
shortlist: &[Candidate],
|
||||
@@ -398,16 +386,6 @@ pub fn parse_selection_response(raw: &str) -> Vec<SelectionItem> {
|
||||
})
|
||||
})
|
||||
.unwrap_or(false),
|
||||
why: object
|
||||
.get("why")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(|why| {
|
||||
why.split_whitespace()
|
||||
.take(14)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
})
|
||||
.filter(|why| !why.is_empty()),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -483,7 +461,6 @@ pub async fn select(
|
||||
section: "From the Blogroll".into(),
|
||||
position: i64::MAX,
|
||||
lead_story: false,
|
||||
why: Some("A standing source you always want represented".into()),
|
||||
},
|
||||
candidate.clone(),
|
||||
));
|
||||
@@ -592,7 +569,6 @@ fn assemble(
|
||||
section: item.section,
|
||||
position: *position,
|
||||
is_lead: Some(item.id) == lead_id,
|
||||
why: item.why,
|
||||
summary: None,
|
||||
llm: candidate.assessment.deep,
|
||||
top_interests: candidate
|
||||
@@ -639,7 +615,6 @@ pub fn select_without_llm(
|
||||
section: heuristic_section(&candidate, sections),
|
||||
position: chosen.len() as i64 + 1,
|
||||
lead_story: false,
|
||||
why: None,
|
||||
},
|
||||
candidate,
|
||||
));
|
||||
@@ -742,7 +717,7 @@ mod tests {
|
||||
let picks: Vec<String> = (1..=n)
|
||||
.map(|i| {
|
||||
format!(
|
||||
r#"{{"id":{i},"section":"Top Stories","position":{i},"lead_story":{},"why":"pick {i} because"}}"#,
|
||||
r#"{{"id":{i},"section":"Top Stories","position":{i},"lead_story":{}}}"#,
|
||||
i == 1
|
||||
)
|
||||
})
|
||||
@@ -809,34 +784,10 @@ mod tests {
|
||||
assert!(items[0].lead_story);
|
||||
assert_eq!(items[0].section, "Top Stories");
|
||||
assert_eq!(items.iter().filter(|i| i.lead_story).count(), 1);
|
||||
assert!(items[0].why.as_deref().is_some_and(|w| !w.is_empty()));
|
||||
// Junk entries in the fixture are dropped, not fatal.
|
||||
assert!(items.iter().all(|i| i.id != 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn why_lines_are_optional_and_capped_at_fourteen_words() {
|
||||
let long = (1..=30)
|
||||
.map(|i| format!("w{i}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let items = parse_selection_response(&format!(
|
||||
r#"{{"picks":[{{"id":1,"section":"Top Stories","why":"{long}"}},
|
||||
{{"id":2,"section":"Top Stories","why":" "}},
|
||||
{{"id":3,"section":"Top Stories"}}]}}"#
|
||||
));
|
||||
assert_eq!(items.len(), 3);
|
||||
assert_eq!(
|
||||
items[0]
|
||||
.why
|
||||
.as_deref()
|
||||
.map(|w| w.split_whitespace().count()),
|
||||
Some(14)
|
||||
);
|
||||
assert!(items[1].why.is_none());
|
||||
assert!(items[2].why.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_prompt_substitutes_the_size_targets_and_renders_the_shortlist() {
|
||||
let prompt = build_prompt(&candidates(3), §ions(), 6, 11);
|
||||
@@ -963,29 +914,6 @@ mod tests {
|
||||
assert!(prompt.contains("aim for about 6; never more than 11"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn why_lines_land_on_picks() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(picks_json(3), TokenUsage::default());
|
||||
let lineup = select(
|
||||
&bulk_only(backend),
|
||||
candidates(5),
|
||||
§ions(),
|
||||
3,
|
||||
5,
|
||||
date(),
|
||||
)
|
||||
.await
|
||||
.expect("selection");
|
||||
assert_eq!(lineup.picks.len(), 3);
|
||||
for pick in &lineup.picks {
|
||||
assert_eq!(
|
||||
pick.why.as_deref(),
|
||||
Some(format!("pick {} because", pick.article.id).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hallucinated_ids_and_missing_leads_are_repaired() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
@@ -1086,7 +1014,6 @@ mod tests {
|
||||
.find(|p| p.article.id == 30)
|
||||
.expect("reinserted");
|
||||
assert_eq!(reinserted.section, "From the Blogroll");
|
||||
assert!(reinserted.why.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1195,7 +1122,6 @@ mod tests {
|
||||
for pick in &lineup.picks {
|
||||
assert!(sections().contains(&pick.section));
|
||||
assert!(pick.summary.is_none());
|
||||
assert!(pick.why.is_none());
|
||||
}
|
||||
assert!(!lineup.section_order.is_empty());
|
||||
}
|
||||
|
||||
@@ -275,10 +275,9 @@ pub fn build_brief_prompt(lineup: &Lineup, summaries: &BTreeMap<ArticleId, Strin
|
||||
.unwrap_or_else(|| excerpt_summary(pick));
|
||||
let _ = writeln!(
|
||||
prompt,
|
||||
"- {}\n feed: {}\n why: {}\n quality: {}\n fit: {}\n summary: {}",
|
||||
"- {}\n feed: {}\n quality: {}\n fit: {}\n summary: {}",
|
||||
pick.article.title.trim(),
|
||||
pick.article.feed_title.trim(),
|
||||
pick.why.as_deref().unwrap_or("not supplied"),
|
||||
quality,
|
||||
fit,
|
||||
summary
|
||||
@@ -459,7 +458,6 @@ mod tests {
|
||||
section: section.into(),
|
||||
position: 1,
|
||||
is_lead,
|
||||
why: Some(format!("the {title} piece you'd argue with")),
|
||||
summary: None,
|
||||
llm: None,
|
||||
top_interests: Vec::new(),
|
||||
@@ -635,7 +633,6 @@ mod tests {
|
||||
assert!(prompt.contains("## Top Stories"));
|
||||
assert!(prompt.contains("## Boston & Local"));
|
||||
assert!(prompt.contains("- Migrating 40TB off Postgres"));
|
||||
assert!(prompt.contains("why: the Migrating 40TB off Postgres piece you'd argue with"));
|
||||
assert!(prompt.contains("summary: A migration story with numbers."));
|
||||
assert!(prompt.contains("quality: unassessed"));
|
||||
assert!(prompt.contains("fit: unassessed"));
|
||||
|
||||
+12
-23
@@ -19,9 +19,10 @@ use sha2::{Digest as _, Sha256};
|
||||
use sqlx::Row as _;
|
||||
|
||||
use crate::config::{Config, VoyageConfig};
|
||||
use crate::curate::{approx_tokens, profile, prompt_text};
|
||||
use crate::curate::{approx_tokens, prompt_text};
|
||||
use crate::db::{Db, fmt_ts};
|
||||
use crate::http::RetryPolicy;
|
||||
use crate::interests;
|
||||
use crate::types::{Article, ArticleId};
|
||||
|
||||
/// The only place the Voyage key comes from (§4.3).
|
||||
@@ -443,10 +444,10 @@ pub fn dot(left: &[f32], right: &[f32]) -> Result<f64, EmbeddingError> {
|
||||
|
||||
/// The `limit` closest compatible cached article vectors, highest cosine first.
|
||||
///
|
||||
/// A brute-force scan: the cache holds at most a few thousand rows after
|
||||
/// pruning, so one statement plus a sort is cheap. The caller supplies the
|
||||
/// target row's model, dimension and decoded vector so pages that already
|
||||
/// load that row do not query it a second time. Malformed candidate blobs are
|
||||
/// A brute-force scan of every compatible row: fine for the admin dashboard,
|
||||
/// too slow for reader pages (see [`crate::related`], which stores its
|
||||
/// results). The caller supplies the target row's model, dimension and decoded
|
||||
/// vector so pages that already load that row do not query it a second time. Malformed candidate blobs are
|
||||
/// ignored like malformed entries in the normal cache loader.
|
||||
pub async fn nearest_articles(
|
||||
db: &Db,
|
||||
@@ -898,14 +899,7 @@ pub async fn plan_backfill(
|
||||
}
|
||||
}
|
||||
|
||||
let interests =
|
||||
match profile::load_standing_interests(&config.interests_opml, &config.profile_path) {
|
||||
Ok(interests) => interests,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "could not load standing interests; skipping them");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let interest_names = interests::names(db).await?;
|
||||
|
||||
let mut plan = BackfillPlan::default();
|
||||
let mut keep = |articles: Vec<Article>, misses: Vec<(ArticleId, i64)>| -> Vec<Article> {
|
||||
@@ -922,8 +916,8 @@ pub async fn plan_backfill(
|
||||
let other_misses = service.uncached_articles(&others).await?;
|
||||
plan.others = keep(others, other_misses);
|
||||
|
||||
let interest_misses = service.uncached_interests(&interests).await?;
|
||||
plan.cached += interests.len() - interest_misses.len();
|
||||
let interest_misses = service.uncached_interests(&interest_names).await?;
|
||||
plan.cached += interest_names.len() - interest_misses.len();
|
||||
plan.estimated_tokens += interest_misses
|
||||
.iter()
|
||||
.map(|interest| approx_tokens(interest) as i64)
|
||||
@@ -1391,7 +1385,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn backfill_prioritizes_the_learned_set_and_is_idempotent() {
|
||||
let (dir, db) = db_with_articles(&[1, 2, 3]).await;
|
||||
let (_dir, db) = db_with_articles(&[1, 2, 3]).await;
|
||||
// Article 1 is rated, article 2 is published, article 3 is neither.
|
||||
sqlx::query(
|
||||
"INSERT INTO rating_events (article_id, kind, source, label, value, event_at)
|
||||
@@ -1418,16 +1412,11 @@ mod tests {
|
||||
|
||||
let config = Config {
|
||||
voyage: small_config(),
|
||||
interests_opml: dir.path().join("interests.opml"),
|
||||
profile_path: dir.path().join("profile.md"),
|
||||
..Config::default()
|
||||
};
|
||||
std::fs::write(
|
||||
&config.interests_opml,
|
||||
"<opml><body><outline text=\"Writerdeck\"/></body></opml>",
|
||||
)
|
||||
interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(&config.profile_path, "# Reader profile\n").unwrap();
|
||||
|
||||
let backend = Arc::new(MockBackend::auto(4));
|
||||
let svc = service(db.clone(), config.voyage.clone(), backend.clone());
|
||||
|
||||
+50
-117
@@ -3,7 +3,6 @@
|
||||
//! Every run rebuilds one byte-stable prompt from the hand-maintained profile,
|
||||
//! standing interests, stored weekly adjustments, and current explicit verdicts.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Write as _;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -13,6 +12,7 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::llm::LlmClient;
|
||||
use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
||||
use crate::interests;
|
||||
use crate::types::{Facets, RatedArticle, TasteProfile};
|
||||
|
||||
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
||||
@@ -29,50 +29,9 @@ pub const NO_LEARNED_ADJUSTMENTS: &str = "No reader ratings have been collected
|
||||
const EDITOR_IN_CHIEF_FRAMING: &str = "You are the editor-in-chief of *The Daily EPUB*, a personal morning newspaper assembled every day for exactly one reader. Everything you are asked to do — score, select, place, summarize, introduce — serves his taste, not a general audience's. When a judgement call is close, re-read this profile and decide the way he would.";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Interest and profile-file parsing
|
||||
// Profile-file parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn parse_interests(opml_path: &Path) -> anyhow::Result<Vec<String>> {
|
||||
let raw = std::fs::read_to_string(opml_path)
|
||||
.with_context(|| format!("reading the interests OPML at {}", opml_path.display()))?;
|
||||
let interests = parse_interests_str(&raw);
|
||||
if interests.is_empty() {
|
||||
anyhow::bail!(
|
||||
"no <outline text=\"…\"> interests found in {}",
|
||||
opml_path.display()
|
||||
);
|
||||
}
|
||||
tracing::debug!(count = interests.len(), "parsed scour interests");
|
||||
Ok(interests)
|
||||
}
|
||||
|
||||
pub fn parse_interests_str(raw: &str) -> Vec<String> {
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut out = Vec::new();
|
||||
for chunk in raw.split("text=\"").skip(1) {
|
||||
let Some((value, _)) = chunk.split_once('"') else {
|
||||
continue;
|
||||
};
|
||||
let name = xml_unescape(value).trim().to_string();
|
||||
if !name.is_empty() && seen.insert(name.to_lowercase()) {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn xml_unescape(s: &str) -> String {
|
||||
if !s.contains('&') {
|
||||
return s.to_string();
|
||||
}
|
||||
s.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("'", "'")
|
||||
.replace("&", "&")
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProfileFile {
|
||||
/// Original Markdown with every `## Interests` section removed.
|
||||
@@ -116,7 +75,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result<ProfileFile> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(raw) => Ok(parse_profile_str(&raw)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
tracing::warn!(path = %path.display(), "profile file is missing; using OPML interests only");
|
||||
tracing::warn!(path = %path.display(), "profile file is missing; using empty profile prose");
|
||||
Ok(ProfileFile {
|
||||
body: String::new(),
|
||||
interests: Vec::new(),
|
||||
@@ -128,30 +87,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result<ProfileFile> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the exact standing-interest union used in the system prompt.
|
||||
pub fn load_standing_interests(
|
||||
opml_path: &Path,
|
||||
profile_path: &Path,
|
||||
) -> anyhow::Result<Vec<String>> {
|
||||
let opml = parse_interests(opml_path)?;
|
||||
let profile = load_profile(profile_path)?;
|
||||
Ok(union_interests(opml, profile.interests))
|
||||
}
|
||||
|
||||
fn union_interests(opml: Vec<String>, profile: Vec<String>) -> Vec<String> {
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut out = Vec::new();
|
||||
for interest in opml.into_iter().chain(profile) {
|
||||
let interest = interest.trim();
|
||||
if !interest.is_empty() && seen.insert(interest.to_lowercase()) {
|
||||
out.push(interest.to_string());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub mod themes;
|
||||
pub use themes::group_into_themes;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prompt assembly
|
||||
@@ -174,7 +110,7 @@ fn one_line(text: &str) -> String {
|
||||
/// Assemble sections in the exact cache-friendly order required by §8.4.
|
||||
pub fn build(
|
||||
profile_body: &str,
|
||||
interests: &[String],
|
||||
grouped: &[(String, Vec<String>)],
|
||||
learned_adjustments: &str,
|
||||
ratings: &[RatedArticle],
|
||||
verdict_limit: usize,
|
||||
@@ -193,8 +129,8 @@ pub fn build(
|
||||
|
||||
doc.push_str("## Standing interests\n\n");
|
||||
doc.push_str("These are his subscribed interest topics, grouped. They raise the floor for a match, but never cap the paper: an outstanding article on none of these still belongs.\n\n");
|
||||
for (theme, members) in group_into_themes(interests) {
|
||||
let _ = writeln!(doc, "- **{}**: {}", theme, members.join(", "));
|
||||
for (category, members) in grouped {
|
||||
let _ = writeln!(doc, "- **{}**: {}", category, members.join(", "));
|
||||
}
|
||||
|
||||
doc.push_str("\n## Learned adjustments (rebuilt weekly from ratings)\n\n");
|
||||
@@ -272,26 +208,27 @@ async fn store_version(db: &Db, version: i64, built_at: Timestamp) -> anyhow::Re
|
||||
|
||||
async fn prompt_inputs(
|
||||
db: &Db,
|
||||
opml_path: &Path,
|
||||
profile_path: &Path,
|
||||
) -> anyhow::Result<(ProfileFile, Vec<String>, Vec<RatedArticle>, String)> {
|
||||
let opml = parse_interests(opml_path)?;
|
||||
) -> anyhow::Result<(
|
||||
ProfileFile,
|
||||
Vec<(String, Vec<String>)>,
|
||||
Vec<RatedArticle>,
|
||||
String,
|
||||
)> {
|
||||
let profile = load_profile(profile_path)?;
|
||||
let interests = union_interests(opml, profile.interests.clone());
|
||||
let grouped = interests::grouped(db).await?;
|
||||
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
||||
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||
Ok((profile, interests, ratings, learned))
|
||||
Ok((profile, grouped, ratings, learned))
|
||||
}
|
||||
|
||||
/// Rebuild the complete system prompt from its live inputs on every run.
|
||||
pub async fn load_or_build(
|
||||
db: &Db,
|
||||
opml_path: &Path,
|
||||
profile_path: &Path,
|
||||
verdict_limit: usize,
|
||||
) -> anyhow::Result<TasteProfile> {
|
||||
let (profile_file, interests, ratings, learned) =
|
||||
prompt_inputs(db, opml_path, profile_path).await?;
|
||||
let (profile_file, grouped, ratings, learned) = prompt_inputs(db, profile_path).await?;
|
||||
let (version, built_at) = match stored_version(db).await? {
|
||||
Some(stored) => stored,
|
||||
None => {
|
||||
@@ -303,7 +240,7 @@ pub async fn load_or_build(
|
||||
let profile = TasteProfile {
|
||||
text: build(
|
||||
&profile_file.body,
|
||||
&interests,
|
||||
&grouped,
|
||||
&learned,
|
||||
&ratings,
|
||||
verdict_limit,
|
||||
@@ -315,7 +252,10 @@ pub async fn load_or_build(
|
||||
db.kv_set(KV_TASTE_PROFILE, &profile.text).await?;
|
||||
tracing::debug!(
|
||||
version,
|
||||
interests = interests.len(),
|
||||
interests = grouped
|
||||
.iter()
|
||||
.map(|(_, members)| members.len())
|
||||
.sum::<usize>(),
|
||||
verdicts = ratings.len().min(verdict_limit),
|
||||
chars = profile.text.len(),
|
||||
"rebuilt the taste profile prompt"
|
||||
@@ -334,7 +274,6 @@ pub async fn is_stale(db: &Db) -> anyhow::Result<bool> {
|
||||
pub async fn weekly_rebuild_if_due(
|
||||
db: &Db,
|
||||
llm: &LlmClient,
|
||||
opml_path: &Path,
|
||||
profile_path: &Path,
|
||||
verdict_limit: usize,
|
||||
) -> anyhow::Result<Option<TasteProfile>> {
|
||||
@@ -346,9 +285,7 @@ pub async fn weekly_rebuild_if_due(
|
||||
return Ok(None);
|
||||
}
|
||||
tracing::info!("taste profile is over a week old; rebuilding learned adjustments");
|
||||
Ok(Some(
|
||||
rebuild(db, llm, opml_path, profile_path, verdict_limit).await?,
|
||||
))
|
||||
Ok(Some(rebuild(db, llm, profile_path, verdict_limit).await?))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -429,15 +366,12 @@ pub fn build_rebuild_prompt(ratings: &[RatedArticle]) -> String {
|
||||
pub async fn rebuild(
|
||||
db: &Db,
|
||||
llm: &LlmClient,
|
||||
opml_path: &Path,
|
||||
profile_path: &Path,
|
||||
verdict_limit: usize,
|
||||
) -> anyhow::Result<TasteProfile> {
|
||||
// Read the prompt inputs first: a rebuild that dies on a missing OPML must
|
||||
// stay due and must not have spent a model call getting there.
|
||||
let opml = parse_interests(opml_path)?;
|
||||
// Read the prompt inputs before spending a model call.
|
||||
let profile_file = load_profile(profile_path)?;
|
||||
let interests = union_interests(opml, profile_file.interests.clone());
|
||||
let grouped = interests::grouped(db).await?;
|
||||
let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?;
|
||||
let previous = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||
let learned = if ratings.is_empty() {
|
||||
@@ -473,7 +407,7 @@ pub async fn rebuild(
|
||||
let profile = TasteProfile {
|
||||
text: build(
|
||||
&profile_file.body,
|
||||
&interests,
|
||||
&grouped,
|
||||
&learned,
|
||||
&ratings,
|
||||
verdict_limit,
|
||||
@@ -495,18 +429,15 @@ pub async fn rebuild(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const OPML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/scour-interests.opml");
|
||||
const PROFILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/profile.md");
|
||||
|
||||
#[test]
|
||||
fn profile_interests_are_removed_and_union_case_insensitively() {
|
||||
fn profile_interests_are_removed_for_the_importer() {
|
||||
let parsed = parse_profile_str(
|
||||
"# P\n\n## Interests\n- Rust\nBoston Tech\n- rust\n\n## Notes\nKeep this.\n",
|
||||
);
|
||||
assert_eq!(parsed.body, "# P\n\n## Notes\nKeep this.\n");
|
||||
assert_eq!(parsed.interests, ["Rust", "Boston Tech", "rust"]);
|
||||
let union = union_interests(vec!["rust".into(), "E-Ink".into()], parsed.interests);
|
||||
assert_eq!(union, ["rust", "E-Ink", "Boston Tech"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -526,7 +457,7 @@ mod tests {
|
||||
};
|
||||
let prompt = build(
|
||||
"# Reader profile\n\nProfile prose.",
|
||||
&["Rust".into()],
|
||||
&[("Software".into(), vec!["Rust".into(), "SQLite".into()])],
|
||||
"- Adjust.",
|
||||
&[rating],
|
||||
60,
|
||||
@@ -539,17 +470,16 @@ mod tests {
|
||||
assert!(
|
||||
framing < profile && profile < interests && interests < learned && learned < verdicts
|
||||
);
|
||||
assert!(prompt.contains("- **Software**: Rust, SQLite"));
|
||||
assert!(prompt.contains("NOT FOR ME | A title | A feed | A summary with whitespace."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shipped_profile_and_opml_parse() {
|
||||
fn shipped_profile_parses() {
|
||||
let profile = load_profile(Path::new(PROFILE_PATH)).unwrap();
|
||||
assert!(profile.body.contains("## Who he is"));
|
||||
assert!(!profile.body.contains("## Interests"));
|
||||
assert!(profile.interests.is_empty());
|
||||
let interests = parse_interests(Path::new(OPML_PATH)).unwrap();
|
||||
assert!(interests.iter().any(|interest| interest == "Rust"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -594,19 +524,21 @@ mod tests {
|
||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
let opml = dir.path().join("interests.opml");
|
||||
let profile_path = dir.path().join("profile.md");
|
||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
||||
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
&profile_path,
|
||||
"# Reader profile\n\nOriginal prose.\n\n## Interests\n- Custom Topic\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let first = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
||||
let first = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||
assert_eq!(first.version, 1);
|
||||
assert!(first.text.contains("Original prose."));
|
||||
assert!(first.text.contains("Custom Topic"));
|
||||
assert!(first.text.contains("- **Software**: Rust"));
|
||||
assert!(!first.text.contains("Custom Topic"));
|
||||
assert!(!first.text.contains("## Interests"));
|
||||
|
||||
std::fs::write(
|
||||
@@ -614,11 +546,11 @@ mod tests {
|
||||
"# Reader profile\n\nChanged prose.\n\n## Interests\n- Another Topic\n",
|
||||
)
|
||||
.unwrap();
|
||||
let second = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
||||
let second = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||
assert_eq!(second.version, first.version);
|
||||
assert_eq!(second.built_at, first.built_at);
|
||||
assert!(second.text.contains("Changed prose."));
|
||||
assert!(second.text.contains("Another Topic"));
|
||||
assert!(!second.text.contains("Another Topic"));
|
||||
assert!(!second.text.contains("Original prose."));
|
||||
|
||||
let missing = load_profile(&dir.path().join("missing.md")).unwrap();
|
||||
@@ -637,11 +569,12 @@ mod tests {
|
||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
let opml = dir.path().join("interests.opml");
|
||||
let profile_path = dir.path().join("profile.md");
|
||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
||||
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
||||
let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
||||
let initial = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||
assert_eq!(initial.version, 1);
|
||||
|
||||
sqlx::query(
|
||||
@@ -677,7 +610,7 @@ mod tests {
|
||||
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||
backend.clone(),
|
||||
);
|
||||
let rebuilt = rebuild(&db, &llm, &opml, &profile_path, 60).await.unwrap();
|
||||
let rebuilt = rebuild(&db, &llm, &profile_path, 60).await.unwrap();
|
||||
assert_eq!(rebuilt.version, 2);
|
||||
assert!(rebuilt.text.contains("Rank first-hand reports higher."));
|
||||
assert!(
|
||||
@@ -700,16 +633,16 @@ mod tests {
|
||||
let db = Db::open_and_migrate(&dir.path().join("profile.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
let opml = dir.path().join("interests.opml");
|
||||
let profile_path = dir.path().join("profile.md");
|
||||
std::fs::write(&opml, r#"<outline text="Rust"/>"#).unwrap();
|
||||
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap();
|
||||
let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap();
|
||||
let initial = load_or_build(&db, &profile_path, 60).await.unwrap();
|
||||
assert_eq!(initial.version, 1);
|
||||
|
||||
// The OPML goes missing the way a relative path does under a service
|
||||
// whose working directory is not the checkout.
|
||||
std::fs::remove_file(&opml).unwrap();
|
||||
std::fs::remove_file(&profile_path).unwrap();
|
||||
std::fs::create_dir(&profile_path).unwrap();
|
||||
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
let llm = LlmClient::with_backend(
|
||||
@@ -718,10 +651,10 @@ mod tests {
|
||||
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||
backend.clone(),
|
||||
);
|
||||
let error = rebuild(&db, &llm, &opml, &profile_path, 60)
|
||||
let error = rebuild(&db, &llm, &profile_path, 60)
|
||||
.await
|
||||
.expect_err("a missing OPML fails the rebuild");
|
||||
assert!(format!("{error:#}").contains("reading the interests OPML"));
|
||||
.expect_err("an unreadable profile fails the rebuild");
|
||||
assert!(format!("{error:#}").contains("reading the reader profile"));
|
||||
|
||||
// Still version 1, so the profile stays stale and the rebuild is retried.
|
||||
assert_eq!(stored_version(&db).await.unwrap().unwrap().0, 1);
|
||||
|
||||
+6
-1
@@ -62,6 +62,11 @@ fn calculate_utility_for(
|
||||
("quality", configured.quality, 1.0),
|
||||
("fit", configured.fit, 1.0),
|
||||
("knn", configured.knn, candidate.signals.knn_gate),
|
||||
(
|
||||
"affinity",
|
||||
configured.affinity,
|
||||
candidate.signals.affinity_gate,
|
||||
),
|
||||
("interest", configured.interest, 1.0),
|
||||
("feed", configured.feed, candidate.signals.feed_gate),
|
||||
("triage", configured.triage, 1.0),
|
||||
@@ -342,7 +347,7 @@ mod tests {
|
||||
}
|
||||
assert!(!b.signals.weights.contains_key("interest"));
|
||||
assert!(
|
||||
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.15 * 0.5) / 0.40).abs()
|
||||
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.10 * 0.5) / 0.40).abs()
|
||||
< 1e-9
|
||||
);
|
||||
}
|
||||
|
||||
+304
-10
@@ -14,11 +14,14 @@ use crate::config::{PreliminaryWeights, RankingConfig, VoyageConfig};
|
||||
use crate::curate::embedding::{dot, load_article_embeddings};
|
||||
use crate::curate::prefilter;
|
||||
use crate::db::Db;
|
||||
use crate::interests::{self, Rate};
|
||||
use crate::types::{Article, ArticleId, FeedId, SourceKind};
|
||||
|
||||
/// Below this many embedded eligible articles the z-score is too noisy, so the
|
||||
/// interest signal falls back to the raw top-1 cosine (§9.1).
|
||||
pub const INTEREST_ZSCORE_MIN_ARTICLES: usize = 30;
|
||||
/// Weak top-three matches are omitted everywhere they are presented or credited.
|
||||
pub const MATCH_MIN_Z: f64 = 1.0;
|
||||
/// Standard-deviation floor for the per-interest z-score (§9.1).
|
||||
const ZSCORE_STD_FLOOR: f64 = 1e-3;
|
||||
/// How many interests and rated neighbours `signals_json` records (§7.5).
|
||||
@@ -30,7 +33,8 @@ pub const AGGREGATOR_FEED_SHARE: f64 = 0.25;
|
||||
/// The signal names that go through the percentile normalizer, in the order
|
||||
/// they are rendered (§12.2). LLM scores (`triage`, `quality`, `fit`) are
|
||||
/// absolute and arrive in steps 4–5.
|
||||
pub const PERCENTILE_SIGNALS: [&str; 5] = ["interest", "knn", "feed", "social", "heuristic"];
|
||||
pub const PERCENTILE_SIGNALS: [&str; 6] =
|
||||
["interest", "knn", "feed", "affinity", "social", "heuristic"];
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TopInterest {
|
||||
@@ -56,6 +60,7 @@ pub struct Signals {
|
||||
pub interest_top1_cos: Option<f64>,
|
||||
pub knn: Option<f64>,
|
||||
pub feed: Option<f64>,
|
||||
pub affinity: Option<f64>,
|
||||
pub social: Option<f64>,
|
||||
pub heuristic: Option<f64>,
|
||||
/// Mid-rank percentiles of the present signals (§12.2).
|
||||
@@ -80,6 +85,8 @@ pub struct Signals {
|
||||
pub knn_gate: f64,
|
||||
#[serde(skip)]
|
||||
pub feed_gate: f64,
|
||||
#[serde(skip)]
|
||||
pub affinity_gate: f64,
|
||||
/// `ranking.slop_author_penalty`, applied when `slop_author` is set.
|
||||
#[serde(skip)]
|
||||
pub slop_penalty: f64,
|
||||
@@ -105,6 +112,7 @@ impl Signals {
|
||||
"interest_top1_cos" => self.interest_top1_cos,
|
||||
"knn" => self.knn,
|
||||
"feed" => self.feed,
|
||||
"affinity" => self.affinity,
|
||||
"social" => self.social,
|
||||
"heuristic" => self.heuristic,
|
||||
_ => None,
|
||||
@@ -130,8 +138,10 @@ impl Signals {
|
||||
pub struct PreferenceSummary {
|
||||
pub rated_with_embeddings: usize,
|
||||
pub attributable_feed_ratings: usize,
|
||||
pub attributable_interest_ratings: usize,
|
||||
pub knn_gate: f64,
|
||||
pub feed_gate: f64,
|
||||
pub affinity_gate: f64,
|
||||
}
|
||||
|
||||
/// One rated article with an embedding: the unit of the preference state (§9.2).
|
||||
@@ -179,12 +189,15 @@ pub struct PreferenceState {
|
||||
pub examples: Vec<RatedExample>,
|
||||
feed_rates: HashMap<FeedId, FeedRate>,
|
||||
author_rates: HashMap<String, FeedRate>,
|
||||
interest_rates: HashMap<String, Rate>,
|
||||
/// Normalized keys of authors with a current *AI slop* verdict (§9.3).
|
||||
slop_authors: HashSet<String>,
|
||||
pub slop_author_penalty: f64,
|
||||
pub attributable_feed_ratings: usize,
|
||||
pub attributable_interest_ratings: usize,
|
||||
pub knn_gate: f64,
|
||||
pub feed_gate: f64,
|
||||
pub affinity_gate: f64,
|
||||
}
|
||||
|
||||
impl PreferenceState {
|
||||
@@ -204,12 +217,28 @@ impl PreferenceState {
|
||||
examples,
|
||||
feed_rates,
|
||||
author_rates,
|
||||
interest_rates: HashMap::new(),
|
||||
slop_authors: HashSet::new(),
|
||||
slop_author_penalty: ranking.slop_author_penalty,
|
||||
attributable_feed_ratings,
|
||||
attributable_interest_ratings: 0,
|
||||
affinity_gate: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Register rating-derived interest rates after the embedding examples are built.
|
||||
pub fn with_interest_rates(
|
||||
mut self,
|
||||
rates_by_name: HashMap<String, Rate>,
|
||||
attributable: usize,
|
||||
ranking: &RankingConfig,
|
||||
) -> Self {
|
||||
self.interest_rates = rates_by_name;
|
||||
self.attributable_interest_ratings = attributable;
|
||||
self.affinity_gate = gate(attributable, ranking.affinity_floor, ranking.affinity_full);
|
||||
self
|
||||
}
|
||||
|
||||
/// Register the authors whose current verdict is *AI slop*; keys are
|
||||
/// normalized like [`normalize_author`] and empty ones are dropped.
|
||||
pub fn with_slop_authors<I, S>(mut self, authors: I) -> Self
|
||||
@@ -234,8 +263,7 @@ impl PreferenceState {
|
||||
self.slop_authors.len()
|
||||
}
|
||||
|
||||
/// Load `db::current_ratings(rating_lookback_days)` joined to
|
||||
/// `article_embeddings`; ratings without an embedding are skipped (§9.2).
|
||||
/// Load current ratings; only kNN/feed examples require an embedding.
|
||||
pub async fn load(
|
||||
db: &Db,
|
||||
voyage: &VoyageConfig,
|
||||
@@ -249,7 +277,7 @@ impl PreferenceState {
|
||||
.collect::<Vec<_>>();
|
||||
let embeddings = load_article_embeddings(db, voyage, &ids).await?;
|
||||
let mut examples = Vec::new();
|
||||
for rating in ratings {
|
||||
for rating in &ratings {
|
||||
let Some(embedding) = embeddings.get(&rating.article_id).cloned() else {
|
||||
continue;
|
||||
};
|
||||
@@ -264,8 +292,8 @@ impl PreferenceState {
|
||||
let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||
examples.push(RatedExample {
|
||||
article_id: rating.article_id,
|
||||
label: rating.label,
|
||||
title: rating.title,
|
||||
label: rating.label.clone(),
|
||||
title: rating.title.clone(),
|
||||
value: rating.value,
|
||||
decay: decay(age_days, ranking.rating_half_life_days),
|
||||
embedding,
|
||||
@@ -274,16 +302,51 @@ impl PreferenceState {
|
||||
aggregator_only,
|
||||
});
|
||||
}
|
||||
let match_rows = interests::matches_for_articles(db, &ids).await?;
|
||||
let rated = ratings
|
||||
.iter()
|
||||
.map(|rating| {
|
||||
let age_days =
|
||||
(now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||
(
|
||||
rating.article_id,
|
||||
rating.value,
|
||||
decay(age_days, ranking.rating_half_life_days),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let matched = match_rows
|
||||
.iter()
|
||||
.map(|row| (row.article_id, row.interest_id, row.z))
|
||||
.collect::<Vec<_>>();
|
||||
let rates = interests::rates(&rated, &matched);
|
||||
let names = match_rows
|
||||
.iter()
|
||||
.map(|row| (row.interest_id, row.name.as_str()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let rates_by_name = rates
|
||||
.by_interest
|
||||
.into_iter()
|
||||
.filter_map(|(interest_id, rate)| {
|
||||
names
|
||||
.get(&interest_id)
|
||||
.map(|name| ((*name).to_string(), rate))
|
||||
})
|
||||
.collect();
|
||||
let slop_authors = db.slop_authors().await?;
|
||||
Ok(Self::build(examples, ranking).with_slop_authors(slop_authors))
|
||||
Ok(Self::build(examples, ranking)
|
||||
.with_interest_rates(rates_by_name, rates.attributable, ranking)
|
||||
.with_slop_authors(slop_authors))
|
||||
}
|
||||
|
||||
pub fn summary(&self) -> PreferenceSummary {
|
||||
PreferenceSummary {
|
||||
rated_with_embeddings: self.examples.len(),
|
||||
attributable_feed_ratings: self.attributable_feed_ratings,
|
||||
attributable_interest_ratings: self.attributable_interest_ratings,
|
||||
knn_gate: self.knn_gate,
|
||||
feed_gate: self.feed_gate,
|
||||
affinity_gate: self.affinity_gate,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,21 +360,54 @@ impl PreferenceState {
|
||||
self.attributable_feed_ratings, ranking.feed_floor
|
||||
)
|
||||
};
|
||||
let affinity_detail = if self.affinity_gate > 0.0 {
|
||||
format!("(n={})", self.attributable_interest_ratings)
|
||||
} else {
|
||||
format!(
|
||||
"(n={} < {})",
|
||||
self.attributable_interest_ratings, ranking.affinity_floor
|
||||
)
|
||||
};
|
||||
tracing::info!(
|
||||
rated_with_embeddings = self.examples.len(),
|
||||
knn_gate = self.knn_gate,
|
||||
feed_gate = self.feed_gate,
|
||||
affinity_gate = self.affinity_gate,
|
||||
slop_authors = self.slop_authors.len(),
|
||||
"preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; {} slop authors (penalty {:.2})",
|
||||
"preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; affinity gate {:.1} {}; {} slop authors (penalty {:.2})",
|
||||
self.examples.len(),
|
||||
self.knn_gate,
|
||||
self.feed_gate,
|
||||
feed_detail,
|
||||
self.affinity_gate,
|
||||
affinity_detail,
|
||||
self.slop_authors.len(),
|
||||
self.slop_author_penalty
|
||||
);
|
||||
}
|
||||
|
||||
/// Match-strength-weighted preference for the article's rated interests.
|
||||
fn affinity(&self, top: &[TopInterest]) -> Option<f64> {
|
||||
if self.affinity_gate <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let mut weighted = 0.0;
|
||||
let mut strength_sum = 0.0;
|
||||
for interest in top {
|
||||
let Some(rate) = self
|
||||
.interest_rates
|
||||
.get(&interest.name)
|
||||
.filter(|rate| rate.n > 0)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let strength = (interest.z / 3.0).clamp(0.0, 1.0);
|
||||
weighted += strength * (rate.weight() - 0.5);
|
||||
strength_sum += strength;
|
||||
}
|
||||
(strength_sum > 0.0).then_some(weighted / strength_sum)
|
||||
}
|
||||
|
||||
/// Signed rated-neighbour preference and the three nearest rated articles
|
||||
/// (§9.2). Absent when the gate is closed or there are no examples.
|
||||
pub fn knn(&self, candidate: &[f32], ranking: &RankingConfig) -> (Option<f64>, Vec<Neighbour>) {
|
||||
@@ -552,6 +648,7 @@ pub fn interest_matches(
|
||||
let top_mean = all.iter().map(|item| item.z).sum::<f64>() / all.len() as f64;
|
||||
0.7 * all[0].z + 0.3 * top_mean
|
||||
};
|
||||
all.retain(|interest| interest.z >= MATCH_MIN_Z);
|
||||
(
|
||||
article_id,
|
||||
InterestMatch {
|
||||
@@ -580,10 +677,12 @@ pub fn compute(
|
||||
let mut signals = Signals::baseline(article);
|
||||
signals.knn_gate = preference.knn_gate;
|
||||
signals.feed_gate = preference.feed_gate;
|
||||
signals.affinity_gate = preference.affinity_gate;
|
||||
if let Some(matched) = interests.get(&article.id) {
|
||||
signals.interest = Some(matched.score);
|
||||
signals.interest_top1_cos = Some(matched.top1_cos);
|
||||
signals.top_interests = matched.top_interests.clone();
|
||||
signals.affinity = preference.affinity(&matched.top_interests);
|
||||
}
|
||||
if let Some(embedding) = article_embeddings.get(&article.id) {
|
||||
let (knn, neighbours) = preference.knn(embedding, ranking);
|
||||
@@ -680,6 +779,7 @@ pub fn preliminary_blend(signals: &mut Signals, configured: &PreliminaryWeights)
|
||||
let candidates = [
|
||||
("interest", configured.interest, 1.0),
|
||||
("knn", configured.knn, signals.knn_gate),
|
||||
("affinity", configured.affinity, signals.affinity_gate),
|
||||
("heuristic", configured.heuristic, 1.0),
|
||||
("feed", configured.feed, signals.feed_gate),
|
||||
("social", configured.social, 1.0),
|
||||
@@ -817,6 +917,47 @@ mod tests {
|
||||
assert!((matched[&2].top1_cos - 0.707).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_cut_keeps_the_score_from_the_uncut_top_three() {
|
||||
let mut articles = HashMap::new();
|
||||
for id in 1..=30 {
|
||||
let mut vector = vec![0.0; 30];
|
||||
vector[id - 1] = 1.0;
|
||||
articles.insert(id as ArticleId, vector);
|
||||
}
|
||||
let interest_at_z = |target: f64| {
|
||||
let mean = -target / 29.0;
|
||||
let spread = ((30.0 - target * target - target * target / 29.0) / 812.0).sqrt();
|
||||
let mut vector = vec![mean + spread; 30];
|
||||
vector[0] = target;
|
||||
vector[29] = mean - 28.0 * spread;
|
||||
unit(
|
||||
&vector
|
||||
.into_iter()
|
||||
.map(|value| value as f32)
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
};
|
||||
let interests = HashMap::from([
|
||||
("first".to_string(), interest_at_z(2.0)),
|
||||
("second".to_string(), interest_at_z(1.5)),
|
||||
("weak third".to_string(), interest_at_z(0.4)),
|
||||
]);
|
||||
|
||||
let matched = interest_matches(&articles, &interests);
|
||||
let first = &matched[&1];
|
||||
assert_eq!(
|
||||
first
|
||||
.top_interests
|
||||
.iter()
|
||||
.map(|interest| interest.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["first", "second"]
|
||||
);
|
||||
let uncut_score = 0.7 * 2.0 + 0.3 * ((2.0 + 1.5 + 0.4) / 3.0);
|
||||
assert!((first.score - uncut_score).abs() < 1e-5, "{}", first.score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interest_falls_back_to_raw_cosine_under_thirty_articles() {
|
||||
let (articles, interests) = interest_fixture(10);
|
||||
@@ -826,6 +967,11 @@ mod tests {
|
||||
(m.score - m.top1_cos).abs() < 1e-9,
|
||||
"article {id} should use raw top-1"
|
||||
);
|
||||
assert!(
|
||||
m.top_interests
|
||||
.iter()
|
||||
.all(|interest| interest.z >= MATCH_MIN_Z)
|
||||
);
|
||||
}
|
||||
assert!((matched[&2].score - 0.707).abs() < 0.01);
|
||||
}
|
||||
@@ -930,6 +1076,99 @@ mod tests {
|
||||
assert_eq!(state.knn(&unit(&[1.0, 0.0]), &ranking), (None, Vec::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affinity_is_absent_under_the_gate_and_without_rated_interests() {
|
||||
let top = [TopInterest {
|
||||
name: "Rust".into(),
|
||||
z: 3.0,
|
||||
cos: 0.8,
|
||||
}];
|
||||
let rates = HashMap::from([(
|
||||
"Rust".to_string(),
|
||||
Rate {
|
||||
up: 3.0,
|
||||
down: 0.0,
|
||||
n: 1,
|
||||
},
|
||||
)]);
|
||||
let closed = PreferenceState::build(Vec::new(), &ranking()).with_interest_rates(
|
||||
rates,
|
||||
1,
|
||||
&ranking(),
|
||||
);
|
||||
assert_eq!(closed.affinity(&top), None);
|
||||
|
||||
let mut open_ranking = ranking();
|
||||
open_ranking.affinity_floor = 0;
|
||||
open_ranking.affinity_full = 1;
|
||||
let empty = PreferenceState::build(Vec::new(), &open_ranking).with_interest_rates(
|
||||
HashMap::new(),
|
||||
1,
|
||||
&open_ranking,
|
||||
);
|
||||
assert_eq!(empty.affinity(&top), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn affinity_load_counts_ratings_without_embeddings() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("signals.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen)
|
||||
VALUES (1, 'https://example.com/1', 'Rated', '2026-09-13T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let now = Timestamp::now();
|
||||
let interests::AddOutcome::Added(interest_id) =
|
||||
interests::add(&db, "Rust", None, now).await.unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO article_interests (article_id, interest_id, cos, z)
|
||||
VALUES (1, ?, 0.8, 3.0)",
|
||||
)
|
||||
.bind(interest_id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO rating_events
|
||||
(article_id, kind, source, label, value, event_at)
|
||||
VALUES (1, 'explicit', 'test', 'loved', 1.0, ?)",
|
||||
)
|
||||
.bind(now.to_string())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut ranking = ranking();
|
||||
ranking.affinity_floor = 0;
|
||||
ranking.affinity_full = 1;
|
||||
let voyage = VoyageConfig {
|
||||
enabled: false,
|
||||
..VoyageConfig::default()
|
||||
};
|
||||
|
||||
let state = PreferenceState::load(&db, &voyage, &ranking, now)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(state.examples.is_empty());
|
||||
assert_eq!(state.attributable_interest_ratings, 1);
|
||||
assert_eq!(state.affinity_gate, 1.0);
|
||||
let affinity = state
|
||||
.affinity(&[TopInterest {
|
||||
name: "Rust".into(),
|
||||
z: 3.0,
|
||||
cos: 0.8,
|
||||
}])
|
||||
.unwrap();
|
||||
assert!((affinity - 1.0 / 6.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
// --- §9.3 feed affinity ---
|
||||
|
||||
#[test]
|
||||
@@ -1165,8 +1404,8 @@ mod tests {
|
||||
assert!((signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
assert!(!signals.weights.contains_key("knn"));
|
||||
assert!(!signals.weights.contains_key("social"));
|
||||
// 0.35/0.55 × 0.8 + 0.20/0.55 × 0.4 = 0.6545…
|
||||
assert!((blend - 65.4545).abs() < 0.01, "{blend}");
|
||||
// 0.30/0.50 × 0.8 + 0.20/0.50 × 0.4 = 0.64.
|
||||
assert!((blend - 64.0).abs() < 1e-9, "{blend}");
|
||||
|
||||
let mut only_heuristic = Signals {
|
||||
heuristic: Some(2.0),
|
||||
@@ -1192,6 +1431,61 @@ mod tests {
|
||||
assert!((signals.weights["knn"] - 0.125 / 0.325).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn liked_interest_outranks_disliked_interest_with_affinity_in_the_blend() {
|
||||
let mut ranking = ranking();
|
||||
ranking.affinity_floor = 0;
|
||||
ranking.affinity_full = 1;
|
||||
let rates = HashMap::from([
|
||||
(
|
||||
"liked".to_string(),
|
||||
Rate {
|
||||
up: 3.0,
|
||||
down: 0.0,
|
||||
n: 1,
|
||||
},
|
||||
),
|
||||
(
|
||||
"disliked".to_string(),
|
||||
Rate {
|
||||
up: 0.0,
|
||||
down: 4.0 / 3.0,
|
||||
n: 1,
|
||||
},
|
||||
),
|
||||
]);
|
||||
let state =
|
||||
PreferenceState::build(Vec::new(), &ranking).with_interest_rates(rates, 1, &ranking);
|
||||
let top = |name: &str| {
|
||||
vec![TopInterest {
|
||||
name: name.into(),
|
||||
z: 3.0,
|
||||
cos: 0.8,
|
||||
}]
|
||||
};
|
||||
let mut signals = [
|
||||
Signals {
|
||||
affinity: state.affinity(&top("liked")),
|
||||
heuristic: Some(1.0),
|
||||
affinity_gate: state.affinity_gate,
|
||||
..Signals::default()
|
||||
},
|
||||
Signals {
|
||||
affinity: state.affinity(&top("disliked")),
|
||||
heuristic: Some(1.0),
|
||||
affinity_gate: state.affinity_gate,
|
||||
..Signals::default()
|
||||
},
|
||||
];
|
||||
normalize(&mut signals.iter_mut().collect::<Vec<_>>());
|
||||
for signal in &mut signals {
|
||||
preliminary_blend(signal, &ranking.weights.preliminary);
|
||||
assert!((signal.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
assert!(signal.weights.contains_key("affinity"));
|
||||
}
|
||||
assert!(signals[0].preliminary > signals[1].preliminary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_scores_every_article_and_leaves_ungated_signals_absent() {
|
||||
let articles = vec![article(1, &[10]), article(2, &[20]), article(3, &[30])];
|
||||
|
||||
+70
-35
@@ -2,7 +2,7 @@
|
||||
//! the `explain` command and feature retention (plan §7.4–7.5, §15.2, §16).
|
||||
//!
|
||||
//! One row per considered article per run says where it stopped and why. Rows
|
||||
//! are upserted on every stage transition with every column set (never
|
||||
//! are upserted on every stage transition with every active field set (never
|
||||
//! `COALESCE`), so the last write for a run is the whole truth.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
@@ -19,10 +19,11 @@ use crate::report::RunReport;
|
||||
use crate::types::{ArticleId, Candidate, NearMiss};
|
||||
|
||||
/// Signal names rendered by `explain`, in the order of §7.5.
|
||||
const RENDERED_SIGNALS: [&str; 8] = [
|
||||
const RENDERED_SIGNALS: [&str; 9] = [
|
||||
"interest",
|
||||
"knn",
|
||||
"feed",
|
||||
"affinity",
|
||||
"social",
|
||||
"heuristic",
|
||||
"triage",
|
||||
@@ -44,16 +45,15 @@ pub struct CandidateRun<'a> {
|
||||
pub rank_utility: Option<i64>,
|
||||
pub cluster_id: Option<i64>,
|
||||
pub cluster_rank: Option<i64>,
|
||||
pub editor_why: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Upsert one row, setting every column (§7.4).
|
||||
/// Upsert one row, setting every active telemetry field (§7.4).
|
||||
pub async fn write(db: &Db, row: &CandidateRun<'_>) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"INSERT INTO candidate_runs
|
||||
(run_id, article_id, stage, excluded_reason, admitted_by, signals_json,
|
||||
utility, rank_utility, cluster_id, cluster_rank, editor_why)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
utility, rank_utility, cluster_id, cluster_rank)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id, article_id) DO UPDATE SET
|
||||
stage = excluded.stage,
|
||||
excluded_reason = excluded.excluded_reason,
|
||||
@@ -62,8 +62,7 @@ pub async fn write(db: &Db, row: &CandidateRun<'_>) -> Result<(), sqlx::Error> {
|
||||
utility = excluded.utility,
|
||||
rank_utility = excluded.rank_utility,
|
||||
cluster_id = excluded.cluster_id,
|
||||
cluster_rank = excluded.cluster_rank,
|
||||
editor_why = excluded.editor_why",
|
||||
cluster_rank = excluded.cluster_rank",
|
||||
)
|
||||
.bind(row.run_id)
|
||||
.bind(row.article_id)
|
||||
@@ -75,7 +74,6 @@ pub async fn write(db: &Db, row: &CandidateRun<'_>) -> Result<(), sqlx::Error> {
|
||||
.bind(row.rank_utility)
|
||||
.bind(row.cluster_id)
|
||||
.bind(row.cluster_rank)
|
||||
.bind(row.editor_why)
|
||||
.execute(db.pool())
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -102,7 +100,6 @@ pub async fn thin_excluded(
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -159,6 +156,7 @@ pub fn serialize_signals(signals: &Signals, auto_include: bool) -> String {
|
||||
"interest_top1_cos",
|
||||
"knn",
|
||||
"feed",
|
||||
"affinity",
|
||||
"social",
|
||||
"heuristic",
|
||||
] {
|
||||
@@ -234,7 +232,6 @@ pub struct ExplainRow {
|
||||
pub rank_utility: Option<i64>,
|
||||
pub cluster_id: Option<i64>,
|
||||
pub cluster_rank: Option<i64>,
|
||||
pub editor_why: Option<String>,
|
||||
}
|
||||
|
||||
impl ExplainRow {
|
||||
@@ -252,7 +249,6 @@ impl ExplainRow {
|
||||
rank_utility: row.get("rank_utility"),
|
||||
cluster_id: row.get("cluster_id"),
|
||||
cluster_rank: row.get("cluster_rank"),
|
||||
editor_why: row.get("editor_why"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +300,7 @@ pub async fn explain_row(
|
||||
"SELECT cr.run_id, cr.article_id, COALESCE(a.title, '') AS title,
|
||||
COALESCE(e.feed_title, '') AS feed_title,
|
||||
cr.stage, cr.excluded_reason, cr.admitted_by, cr.signals_json,
|
||||
cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank, cr.editor_why
|
||||
cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank
|
||||
FROM candidate_runs cr JOIN articles a ON a.id = cr.article_id
|
||||
LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||
WHERE cr.run_id = ? AND cr.article_id = ?",
|
||||
@@ -327,7 +323,7 @@ pub async fn near_misses(
|
||||
"SELECT cr.run_id, cr.article_id, COALESCE(a.title, '') AS title,
|
||||
COALESCE(e.feed_title, '') AS feed_title,
|
||||
cr.stage, cr.excluded_reason, cr.admitted_by, cr.signals_json,
|
||||
cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank, cr.editor_why
|
||||
cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank
|
||||
FROM candidate_runs cr JOIN articles a ON a.id = cr.article_id
|
||||
LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||
WHERE cr.run_id = ? AND cr.stage != 'selected' AND cr.stage != 'excluded'",
|
||||
@@ -488,9 +484,6 @@ pub async fn render_explain(db: &Db, row: &ExplainRow) -> Result<String, sqlx::E
|
||||
if let Some(admitted_by) = &row.admitted_by {
|
||||
let _ = writeln!(out, "admitted by: {admitted_by}");
|
||||
}
|
||||
if let Some(why) = &row.editor_why {
|
||||
let _ = writeln!(out, "editor: {why}");
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
@@ -1065,7 +1058,7 @@ pub fn render_stats_text(data: &StatsData) -> String {
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Pruned {
|
||||
/// `article_embeddings` of unrated, unpublished articles past
|
||||
/// `embedding_retention_days`.
|
||||
/// `embedding_retention_days`; always 0 when that is 0 (keep forever).
|
||||
pub embeddings: u64,
|
||||
/// `candidate_runs` rows of runs past `telemetry_retention_days`.
|
||||
pub telemetry: u64,
|
||||
@@ -1074,7 +1067,8 @@ pub struct Pruned {
|
||||
}
|
||||
|
||||
/// Delete `article_embeddings` for articles neither rated nor published that
|
||||
/// are older than `embedding_retention_days`, `candidate_runs` rows whose run
|
||||
/// are older than `embedding_retention_days` (0 or less keeps every vector, so
|
||||
/// related articles can reach the whole archive), `candidate_runs` rows whose run
|
||||
/// started more than `telemetry_retention_days` ago, and `article_assessments`
|
||||
/// older than the same window (§7.1, §7.4). Runs from `features prune` and
|
||||
/// once per `generate` after publishing.
|
||||
@@ -1088,7 +1082,10 @@ pub async fn prune(
|
||||
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
||||
.unwrap_or(Timestamp::UNIX_EPOCH)
|
||||
};
|
||||
let embeddings = sqlx::query(
|
||||
let embeddings = if embedding_retention_days <= 0 {
|
||||
0
|
||||
} else {
|
||||
sqlx::query(
|
||||
"DELETE FROM article_embeddings
|
||||
WHERE article_id IN (
|
||||
SELECT ae.article_id
|
||||
@@ -1101,7 +1098,8 @@ pub async fn prune(
|
||||
.bind(fmt_ts(cutoff(embedding_retention_days)))
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
.rows_affected()
|
||||
};
|
||||
|
||||
let telemetry = sqlx::query(
|
||||
"DELETE FROM candidate_runs
|
||||
@@ -1157,11 +1155,17 @@ mod tests {
|
||||
Signals {
|
||||
interest: Some(1.2),
|
||||
interest_top1_cos: Some(0.61),
|
||||
affinity: Some(0.2),
|
||||
heuristic: Some(heuristic),
|
||||
norm: BTreeMap::from([("heuristic".into(), norm), ("interest".into(), 0.9)]),
|
||||
norm: BTreeMap::from([
|
||||
("affinity".into(), 0.7),
|
||||
("heuristic".into(), norm),
|
||||
("interest".into(), 0.9),
|
||||
]),
|
||||
weights: BTreeMap::from([
|
||||
("heuristic".into(), 0.2 / 0.55),
|
||||
("interest".into(), 0.35 / 0.55),
|
||||
("affinity".into(), 0.1 / 0.6),
|
||||
("heuristic".into(), 0.2 / 0.6),
|
||||
("interest".into(), 0.3 / 0.6),
|
||||
]),
|
||||
top_interests: vec![TopInterest {
|
||||
name: "Gaussian Splatting".into(),
|
||||
@@ -1186,6 +1190,7 @@ mod tests {
|
||||
assert_eq!(parsed["v"], 1);
|
||||
assert_eq!(parsed["raw"]["heuristic"], 41.0);
|
||||
assert_eq!(parsed["raw"]["interest_top1_cos"], 0.61);
|
||||
assert_eq!(parsed["raw"]["affinity"], 0.2);
|
||||
assert_eq!(parsed["present"]["heuristic"], true);
|
||||
assert_eq!(parsed["present"]["knn"], false);
|
||||
assert_eq!(parsed["present"]["quality"], false);
|
||||
@@ -1246,7 +1251,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1280,7 +1284,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rows_are_upserted_with_every_column_replaced() {
|
||||
async fn rows_are_upserted_with_every_active_field_replaced() {
|
||||
let (_dir, db) = db_with_articles(&[1]).await;
|
||||
let run_id = db.start_run(date(), Timestamp::now()).await.unwrap();
|
||||
write(
|
||||
@@ -1296,7 +1300,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1314,7 +1317,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: Some("because"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1324,7 +1326,6 @@ mod tests {
|
||||
assert_eq!(row.excluded_reason, None, "no COALESCE");
|
||||
assert_eq!(row.utility, None);
|
||||
assert_eq!(row.admitted_by.as_deref(), Some("[\"prefilter\"]"));
|
||||
assert_eq!(row.editor_why.as_deref(), Some("because"));
|
||||
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM candidate_runs")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
@@ -1351,7 +1352,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1385,7 +1385,11 @@ mod tests {
|
||||
);
|
||||
let squashed = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
assert!(
|
||||
squashed.contains("heuristic 41.000 · 0.550 · 0.364"),
|
||||
squashed.contains("heuristic 41.000 · 0.550 · 0.333"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
squashed.contains("affinity 0.200 · 0.700 · 0.167"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(squashed.contains("knn absent"), "{text}");
|
||||
@@ -1480,7 +1484,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1569,7 +1572,6 @@ mod tests {
|
||||
rank_utility: Some(3),
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1587,7 +1589,6 @@ mod tests {
|
||||
rank_utility: Some(1),
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: Some("because"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1684,7 +1685,6 @@ mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1947,4 +1947,39 @@ mean generation time: n/a (0 runs)
|
||||
// A second pass finds nothing left to remove.
|
||||
assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_embedding_retention_keeps_every_vector() {
|
||||
let (_dir, db) = db_with_articles(&[1, 2]).await;
|
||||
let now = Timestamp::now();
|
||||
let old = fmt_ts(now - jiff::Span::new().hours(2000 * 24));
|
||||
sqlx::query("UPDATE articles SET first_seen = ?")
|
||||
.bind(&old)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let blob = encode_blob(&[0.5, 0.5]).unwrap();
|
||||
for id in 1..=2 {
|
||||
sqlx::query(
|
||||
"INSERT INTO article_embeddings
|
||||
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||
VALUES (?, 'voyage-4-lite', 2, 'h', ?, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&blob)
|
||||
.bind(&old)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// 0 must not mean "a cutoff of now", which would delete everything.
|
||||
let pruned = prune(&db, 0, 180, now).await.unwrap();
|
||||
assert_eq!(pruned.embeddings, 0);
|
||||
let remaining: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_embeddings")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 2);
|
||||
}
|
||||
}
|
||||
@@ -606,8 +606,8 @@ impl Db {
|
||||
.await?;
|
||||
for pick in picks {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position, is_lead, summary, why)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position, is_lead, summary)
|
||||
VALUES (?, ?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.bind(pick.article.id)
|
||||
@@ -615,7 +615,6 @@ impl Db {
|
||||
.bind(pick.position)
|
||||
.bind(pick.is_lead)
|
||||
.bind(pick.summary.as_deref())
|
||||
.bind(pick.why.as_deref())
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
@@ -1689,6 +1688,10 @@ mod tests {
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::raw_sql(include_str!("../migrations/0013_interests.sql"))
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let rows = sqlx::query(
|
||||
"SELECT article_id, issue_date, kind, source, label, value, event_at, user_id
|
||||
@@ -1724,6 +1727,8 @@ mod tests {
|
||||
"interest_embeddings",
|
||||
"article_assessments",
|
||||
"candidate_runs",
|
||||
"interests",
|
||||
"article_interests",
|
||||
"users",
|
||||
"sessions",
|
||||
"config_changes",
|
||||
|
||||
+62
-16
@@ -42,7 +42,6 @@ struct IndexEntry {
|
||||
source: String,
|
||||
reading_minutes: i64,
|
||||
summary: String,
|
||||
why: Option<String>,
|
||||
understanding: Understanding,
|
||||
}
|
||||
|
||||
@@ -82,7 +81,6 @@ struct ArticleChapter {
|
||||
meta_line: String,
|
||||
social_line: Option<String>,
|
||||
understanding: Understanding,
|
||||
why: Option<String>,
|
||||
summary: Option<String>,
|
||||
excerpt_only: bool,
|
||||
body_html: String,
|
||||
@@ -152,9 +150,10 @@ struct ProviderCostLine {
|
||||
|
||||
/// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids.
|
||||
pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String {
|
||||
// Sanitize first: image rewriting emits our own trusted markup (including the
|
||||
// `image-placeholder` class, which ammonia would otherwise strip).
|
||||
let cleaned = ammonia::clean(html);
|
||||
// Sanitize first: highlighting and image rewriting emit our own trusted
|
||||
// markup (the `hl-*` token classes and `image-placeholder`, which ammonia
|
||||
// would otherwise strip).
|
||||
let cleaned = crate::highlight::clean_and_highlight(html);
|
||||
let rewritten = images::rewrite_img_srcs(&cleaned, images_);
|
||||
to_xhtml(&rewritten)
|
||||
}
|
||||
@@ -232,7 +231,13 @@ fn facet_label(token: &str) -> Option<String> {
|
||||
pub struct Understanding {
|
||||
pub kicker: Option<String>,
|
||||
pub topics: Option<String>,
|
||||
pub interests: Option<String>,
|
||||
pub interests: Vec<InterestRef>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct InterestRef {
|
||||
pub name: String,
|
||||
pub href: String,
|
||||
}
|
||||
|
||||
/// Reader-facing assessment details, split so templates can give each part the
|
||||
@@ -257,7 +262,14 @@ pub fn understanding(pick: &Pick) -> Understanding {
|
||||
});
|
||||
(kicker, topics)
|
||||
});
|
||||
let interests = (!pick.top_interests.is_empty()).then(|| pick.top_interests.join(" \u{00b7} "));
|
||||
let interests = pick
|
||||
.top_interests
|
||||
.iter()
|
||||
.map(|name| InterestRef {
|
||||
name: name.clone(),
|
||||
href: crate::interests::articles_href(name),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Understanding {
|
||||
kicker,
|
||||
@@ -387,7 +399,6 @@ pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||
source: index_source_line(&pick.article),
|
||||
reading_minutes: pick.article.reading_minutes(),
|
||||
summary: summary_for(issue, pick).unwrap_or_default().to_string(),
|
||||
why: pick.why.clone(),
|
||||
understanding: understanding(pick),
|
||||
})
|
||||
.collect();
|
||||
@@ -402,7 +413,6 @@ pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||
source: "Wikipedia Current Events".into(),
|
||||
reading_minutes: 3,
|
||||
summary: "The day's events, as recorded by the Current Events portal.".into(),
|
||||
why: None,
|
||||
understanding: Understanding::default(),
|
||||
}],
|
||||
});
|
||||
@@ -480,7 +490,6 @@ pub fn render_article(
|
||||
meta_line: meta_parts.join(" \u{00b7} "),
|
||||
social_line: social_line(&article.social),
|
||||
understanding: understanding(pick),
|
||||
why: pick.why.clone(),
|
||||
summary: summary_for(issue, pick).map(str::to_string),
|
||||
excerpt_only: article.excerpt_only,
|
||||
body_html: prepare_body(&article.content_html, images_),
|
||||
@@ -811,7 +820,16 @@ mod tests {
|
||||
Understanding {
|
||||
kicker: Some("Software engineering · Analysis".into()),
|
||||
topics: Some("copy-on-write · ZFS".into()),
|
||||
interests: Some("Filesystems · Rust".into()),
|
||||
interests: vec![
|
||||
InterestRef {
|
||||
name: "Filesystems".into(),
|
||||
href: "/dashboard/articles?interest=Filesystems".into(),
|
||||
},
|
||||
InterestRef {
|
||||
name: "Rust".into(),
|
||||
href: "/dashboard/articles?interest=Rust".into(),
|
||||
},
|
||||
],
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -833,7 +851,10 @@ mod tests {
|
||||
assert_eq!(
|
||||
understanding(&pick),
|
||||
Understanding {
|
||||
interests: Some("Rust".into()),
|
||||
interests: vec![InterestRef {
|
||||
name: "Rust".into(),
|
||||
href: "/dashboard/articles?interest=Rust".into(),
|
||||
}],
|
||||
..Understanding::default()
|
||||
}
|
||||
);
|
||||
@@ -888,7 +909,21 @@ mod tests {
|
||||
// The lead's rubric and matches are on the index; the second pick has neither.
|
||||
assert_eq!(chapter.xhtml.matches("class=\"rubric\"").count(), 1);
|
||||
assert!(chapter.xhtml.contains("Software engineering · Analysis"));
|
||||
assert!(chapter.xhtml.contains("Matches: Filesystems · Rust"));
|
||||
assert!(
|
||||
chapter
|
||||
.xhtml
|
||||
.contains("<span class=\"interest\">Filesystems</span>")
|
||||
);
|
||||
assert!(
|
||||
chapter
|
||||
.xhtml
|
||||
.contains("<span class=\"interest\">Rust</span>")
|
||||
);
|
||||
let rubric_position = chapter.xhtml.find("class=\"rubric\"").unwrap();
|
||||
let matches_position = chapter.xhtml.find("class=\"matches\"").unwrap();
|
||||
let summary_position = chapter.xhtml.find("class=\"index-summary\"").unwrap();
|
||||
assert!(rubric_position < matches_position);
|
||||
assert!(matches_position < summary_position);
|
||||
// Titles are escaped (askama emits numeric references), never injected raw.
|
||||
assert!(chapter.xhtml.contains("A Niche Delight & Other Tales"));
|
||||
assert_xml_ok(&chapter.xhtml);
|
||||
@@ -961,11 +996,22 @@ mod tests {
|
||||
assert!(chapter.xhtml.contains("class=\"rubric\""));
|
||||
assert!(chapter.xhtml.contains("Software engineering · Analysis"));
|
||||
assert!(chapter.xhtml.contains("copy-on-write · ZFS"));
|
||||
assert!(chapter.xhtml.contains("Matches: Filesystems · Rust"));
|
||||
assert!(
|
||||
chapter
|
||||
.xhtml
|
||||
.contains("<span class=\"interest\">Filesystems</span>")
|
||||
);
|
||||
assert!(
|
||||
chapter
|
||||
.xhtml
|
||||
.contains("<span class=\"interest\">Rust</span>")
|
||||
);
|
||||
let rubric_position = chapter.xhtml.find("class=\"rubric\"").unwrap();
|
||||
let matches_position = chapter.xhtml.find("class=\"matches\"").unwrap();
|
||||
let summary_position = chapter.xhtml.find("class=\"summary\"").unwrap();
|
||||
let social_position = chapter.xhtml.find("class=\"social\"").unwrap();
|
||||
assert!(rubric_position < summary_position);
|
||||
assert!(rubric_position < matches_position);
|
||||
assert!(matches_position < summary_position);
|
||||
assert!(summary_position < social_position);
|
||||
let footer = chapter
|
||||
.xhtml
|
||||
@@ -973,7 +1019,7 @@ mod tests {
|
||||
.unwrap()
|
||||
.1;
|
||||
assert!(!footer.contains("class=\"rubric\""));
|
||||
assert!(!footer.contains("Matches:"));
|
||||
assert!(!footer.contains("class=\"matches\""));
|
||||
let second = render_article(
|
||||
&issue,
|
||||
&issue.lineup.picks[1],
|
||||
|
||||
@@ -87,7 +87,6 @@ pub fn issue() -> Issue {
|
||||
section: "Top Stories".into(),
|
||||
position: 0,
|
||||
is_lead: true,
|
||||
why: Some("The systems story with enough operational detail to matter".into()),
|
||||
summary: Some("What it argues, and why it is worth the time.".into()),
|
||||
llm: Some(Deep {
|
||||
quality: 9.0,
|
||||
@@ -115,7 +114,6 @@ pub fn issue() -> Issue {
|
||||
section: "Niche Corner".into(),
|
||||
position: 0,
|
||||
is_lead: false,
|
||||
why: Some("A small-scene delight outside the usual technical orbit".into()),
|
||||
summary: None,
|
||||
llm: None,
|
||||
top_interests: Vec::new(),
|
||||
|
||||
@@ -10,15 +10,8 @@
|
||||
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}
|
||||
<p class="rubric">{% if let Some(kicker) = understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
||||
{% endif %}
|
||||
{% if why.is_some() || understanding.interests.is_some() %}
|
||||
<div class="why">
|
||||
{% if let Some(text) = why %}
|
||||
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
||||
{% endif %}
|
||||
{% if let Some(interests) = understanding.interests %}
|
||||
<p class="why-matches">Matches: {{ interests }}</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if !understanding.interests.is_empty() %}
|
||||
<p class="matches">{% for interest in understanding.interests %}{% if !loop.first %}<span class="sep">|</span>{% endif %}<span class="interest">{{ interest.name }}</span>{% endfor %}</p>
|
||||
{% endif %}
|
||||
{% if let Some(text) = summary %}
|
||||
<p class="summary">{{ text }}</p>
|
||||
|
||||
@@ -13,15 +13,8 @@
|
||||
{% if entry.understanding.kicker.is_some() || entry.understanding.topics.is_some() %}
|
||||
<p class="rubric">{% if let Some(kicker) = entry.understanding.kicker %}<span class="kicker">{{ kicker }}</span>{% if let Some(topics) = entry.understanding.topics %}   {{ topics }}{% endif %}{% else %}{% if let Some(topics) = entry.understanding.topics %}{{ topics }}{% endif %}{% endif %}</p>
|
||||
{% endif %}
|
||||
{% if entry.why.is_some() || entry.understanding.interests.is_some() %}
|
||||
<div class="index-why">
|
||||
{% if let Some(text) = entry.why %}
|
||||
<p class="why-line"><em>Why it's here: {{ text }}</em></p>
|
||||
{% endif %}
|
||||
{% if let Some(interests) = entry.understanding.interests %}
|
||||
<p class="why-matches">Matches: {{ interests }}</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if !entry.understanding.interests.is_empty() %}
|
||||
<p class="matches">{% for interest in entry.understanding.interests %}{% if !loop.first %}<span class="sep">|</span>{% endif %}<span class="interest">{{ interest.name }}</span>{% endfor %}</p>
|
||||
{% endif %}
|
||||
{% if !entry.summary.is_empty() %}
|
||||
<p class="index-summary">{{ entry.summary }}</p>
|
||||
|
||||
@@ -95,22 +95,17 @@ img {
|
||||
font-variant: small-caps;
|
||||
}
|
||||
|
||||
.why,
|
||||
.index-why {
|
||||
.matches {
|
||||
margin: 0.2em 0 0 0;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.why-line {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.why-matches {
|
||||
margin: 0;
|
||||
font-size: 0.85em;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.sep {
|
||||
color: #888888;
|
||||
padding: 0 0.4em;
|
||||
}
|
||||
|
||||
.dateline,
|
||||
.stats {
|
||||
text-align: center;
|
||||
@@ -165,7 +160,16 @@ figcaption,
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
pre, code {
|
||||
/* The firmwares (CrossPoint, CrossInk) ignore `white-space` and treat `pre`
|
||||
as inline, so the X4 simplifier makes each listing line a `p.code-line`;
|
||||
the first and last lines carry the block's spacing, and a blank source
|
||||
line becomes a `p.code-gap` margin. `figure` centres for
|
||||
image captions, so lines reset alignment. The XTC converter's CREngine
|
||||
defaults `code` to a block. */
|
||||
pre {
|
||||
display: block;
|
||||
margin: 0;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
font-size: 0.85em;
|
||||
white-space: pre-wrap;
|
||||
@@ -174,6 +178,81 @@ pre, code {
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
p.code-line {
|
||||
margin: 0;
|
||||
text-indent: 0;
|
||||
text-align: left;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
p.code-first {
|
||||
margin-top: 0.6em;
|
||||
}
|
||||
|
||||
p.code-last {
|
||||
margin-bottom: 0.6em;
|
||||
}
|
||||
|
||||
/* Where the source had a blank line: half a line, not a whole one. */
|
||||
p.code-gap {
|
||||
margin-bottom: 0.5em;
|
||||
}
|
||||
|
||||
code {
|
||||
display: inline;
|
||||
font-family: monospace;
|
||||
font-size: 0.85em;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
/* Syntax tokens (src/highlight.rs), as in style.css but on the X4's four
|
||||
levels only (#000, #555, #aaa, white), so glyphs are not dithered: ink for
|
||||
names, dark grey for keywords, literals and punctuation, light grey for
|
||||
comments. */
|
||||
.hl-function,
|
||||
.hl-macro,
|
||||
.hl-title,
|
||||
.hl-strong {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.hl-type,
|
||||
.hl-namespace,
|
||||
.hl-constructor,
|
||||
.hl-attribute,
|
||||
.hl-emphasis {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.hl-keyword,
|
||||
.hl-tag {
|
||||
font-weight: bold;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-string {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-number,
|
||||
.hl-constant,
|
||||
.hl-label {
|
||||
font-style: italic;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-punctuation,
|
||||
.hl-operator {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
font-style: italic;
|
||||
color: #aaaaaa;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85em;
|
||||
|
||||
@@ -160,22 +160,19 @@ hr.rule {
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.why,
|
||||
.index-why {
|
||||
margin: 0.35em 0 0 0;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.why-line {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.why-matches {
|
||||
margin: 0.1em 0 0 0;
|
||||
font-size: 0.85em;
|
||||
/* Matched interests read like the rubric's topics, with a muted vertical
|
||||
bar between them instead of the topics' dot. */
|
||||
.matches {
|
||||
margin: 0.2em 0 0 0;
|
||||
font-size: 0.8em;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.sep {
|
||||
color: #aaaaaa;
|
||||
padding: 0 0.4em;
|
||||
}
|
||||
|
||||
.summary {
|
||||
margin: 0.5em 0 0 0;
|
||||
font-style: italic;
|
||||
@@ -224,13 +221,67 @@ hr.rule {
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
/* CREngine readers (KOReader, PocketBook) default `code` to a block. */
|
||||
.article-body code {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
/* Left even inside a `<figure>`, which centres for image captions. */
|
||||
.article-body pre {
|
||||
margin: 0.8em 0;
|
||||
text-align: left;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
border-left: 2px solid #cccccc;
|
||||
padding-left: 0.5em;
|
||||
}
|
||||
|
||||
/* Syntax tokens (src/highlight.rs), monochrome for e-ink, after zenbones and
|
||||
lackluster: what the code names stays in ink (functions bold, types italic),
|
||||
the language's own furniture steps back to grey (keywords bold grey,
|
||||
literals and punctuation grey), comments lightest. Flat selectors. */
|
||||
.hl-function,
|
||||
.hl-macro,
|
||||
.hl-title,
|
||||
.hl-strong {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.hl-type,
|
||||
.hl-namespace,
|
||||
.hl-constructor,
|
||||
.hl-attribute,
|
||||
.hl-emphasis {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.hl-keyword,
|
||||
.hl-tag {
|
||||
font-weight: bold;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-string {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-number,
|
||||
.hl-constant,
|
||||
.hl-label {
|
||||
font-style: italic;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-punctuation,
|
||||
.hl-operator {
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
font-style: italic;
|
||||
color: #888888;
|
||||
}
|
||||
|
||||
.article-body table {
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85em;
|
||||
|
||||
+415
-7
@@ -124,10 +124,295 @@ fn is_dropped_declaration(decl: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Class of every paragraph [`paragraph_code_lines`] makes from a listing
|
||||
/// line; the first and last also get [`CODE_FIRST_CLASS`] / [`CODE_LAST_CLASS`].
|
||||
pub const CODE_LINE_CLASS: &str = "code-line";
|
||||
pub const CODE_FIRST_CLASS: &str = "code-first";
|
||||
pub const CODE_LAST_CLASS: &str = "code-last";
|
||||
/// Marks a line followed by blank lines in the source; it gets a short gap.
|
||||
pub const CODE_GAP_CLASS: &str = "code-gap";
|
||||
|
||||
/// Strip layout constructs the X4 renderer handles poorly from chapter markup,
|
||||
/// then soft-hyphenate anything too long for it to wrap (§3.10).
|
||||
pub fn simplify_xhtml(xhtml: &str) -> String {
|
||||
break_long_words(&strip_attributes(xhtml, DROPPED_ATTRIBUTES))
|
||||
paragraph_code_lines(&break_long_words(&strip_attributes(
|
||||
&flatten_lossy_tables(xhtml),
|
||||
DROPPED_ATTRIBUTES,
|
||||
)))
|
||||
}
|
||||
|
||||
/// Table elements dropped outright when an image table is flattened.
|
||||
const TABLE_FRAME_ELEMENTS: &[&str] =
|
||||
&["table", "thead", "tbody", "tfoot", "tr", "colgroup", "col"];
|
||||
|
||||
/// Table elements whose content becomes a `<div>` when a table is flattened.
|
||||
const TABLE_CELL_ELEMENTS: &[&str] = &["td", "th", "caption"];
|
||||
|
||||
/// Elements the X4 firmwares treat as images.
|
||||
const IMAGE_ELEMENTS: &[&str] = &["img", "image"];
|
||||
|
||||
/// Turn every table the X4 firmwares would lose content from into plain
|
||||
/// `<div>` blocks: one that holds an image or another table.
|
||||
///
|
||||
/// The firmwares (CrossPoint and CrossInk) never draw an image inside a table
|
||||
/// cell: they print its alt text instead. Publishers that lay out a figure as a
|
||||
/// one-cell table with the caption beneath the image (The Space Review, many
|
||||
/// older sites) would lose every figure. CrossInk also throws away all the text
|
||||
/// of a table nested in another. Flattening keeps each cell's content in order,
|
||||
/// which is how the firmwares already fall back for any table they cannot lay
|
||||
/// out as a grid. Tables of plain text stay tables.
|
||||
fn flatten_lossy_tables(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
if let Some(name) = raw_text_name(tag) {
|
||||
let close = find_close_tag(html, end, name).unwrap_or(html.len());
|
||||
out.push_str(&html[start..close]);
|
||||
cursor = close;
|
||||
continue;
|
||||
}
|
||||
if !is_element_tag(tag, "table", false) {
|
||||
out.push_str(tag);
|
||||
cursor = end;
|
||||
continue;
|
||||
}
|
||||
let table_end = matching_table_end(html, start);
|
||||
let table = &html[start..table_end];
|
||||
// Past the outer `<table` tag, so it doesn't count as its own nesting.
|
||||
let inner = &table[end - start..];
|
||||
if contains_element(inner, IMAGE_ELEMENTS) || contains_element(inner, &["table"]) {
|
||||
out.push_str(&flatten_table(table));
|
||||
} else {
|
||||
out.push_str(table);
|
||||
}
|
||||
cursor = table_end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Byte offset just past the `</table>` that closes the table opened at
|
||||
/// `start`, counting nested tables; the end of `html` if it is never closed.
|
||||
fn matching_table_end(html: &str, start: usize) -> usize {
|
||||
let mut depth = 0usize;
|
||||
let mut cursor = start;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let tag_start = cursor + rel;
|
||||
let Some(end) = tag_end(html, tag_start) else {
|
||||
break;
|
||||
};
|
||||
let tag = &html[tag_start..end];
|
||||
if is_element_tag(tag, "table", false) && !tag.ends_with("/>") {
|
||||
depth += 1;
|
||||
} else if is_element_tag(tag, "table", true) {
|
||||
depth = depth.saturating_sub(1);
|
||||
if depth == 0 {
|
||||
return end;
|
||||
}
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
html.len()
|
||||
}
|
||||
|
||||
/// Whether any tag in `html` opens one of the named elements.
|
||||
fn contains_element(html: &str, names: &[&str]) -> bool {
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
return false;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
if names.iter().any(|name| is_element_tag(tag, name, false)) {
|
||||
return true;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Rewrite one table's markup (nested tables included) as `<div>` blocks.
|
||||
fn flatten_table(table: &str) -> String {
|
||||
let mut out = String::with_capacity(table.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = table[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&table[cursor..start]);
|
||||
let Some(end) = tag_end(table, start) else {
|
||||
out.push_str(&table[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &table[start..end];
|
||||
cursor = end;
|
||||
let is = |names: &[&str], closing: bool| {
|
||||
names.iter().any(|name| is_element_tag(tag, name, closing))
|
||||
};
|
||||
if is(TABLE_FRAME_ELEMENTS, false) || is(TABLE_FRAME_ELEMENTS, true) {
|
||||
// A newline, not nothing: the firmwares buffer the last word
|
||||
// before an element boundary until whitespace arrives.
|
||||
out.push('\n');
|
||||
} else if is(TABLE_CELL_ELEMENTS, false) {
|
||||
out.push_str(if tag.ends_with("/>") {
|
||||
"<div></div>"
|
||||
} else {
|
||||
"<div>"
|
||||
});
|
||||
} else if is(TABLE_CELL_ELEMENTS, true) {
|
||||
out.push_str("</div>");
|
||||
} else {
|
||||
out.push_str(tag);
|
||||
}
|
||||
}
|
||||
out.push_str(&table[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Turn each line of every `<pre>` into its own `<p class="code-line">`.
|
||||
///
|
||||
/// The X4 firmwares (CrossPoint and forks such as CrossInk) ignore
|
||||
/// `white-space` and lay out only `p`, `div`, `li`, `blockquote`, lists and
|
||||
/// `br` as blocks: a `<pre>` is inline, so its margins never apply and its
|
||||
/// `<br/>` lines are handled by each firmware's own rules. CrossInk glues the
|
||||
/// first line onto the paragraph before, and empty `<br/>` lines count as scene
|
||||
/// breaks. A paragraph per line takes the one path every firmware lays out the
|
||||
/// same way. The first and last lines carry the block's margins; blank lines
|
||||
/// become a half-line gap (see [`code_paragraphs`]).
|
||||
///
|
||||
/// Inline elements open across a line break (a `<code>`, a multi-line
|
||||
/// comment's span) are closed at the end of the line and reopened at the start
|
||||
/// of the next, so every paragraph is well formed.
|
||||
fn paragraph_code_lines(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len() + 256);
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
out.push_str(tag);
|
||||
cursor = end;
|
||||
if !is_element_tag(tag, "pre", false) {
|
||||
continue;
|
||||
}
|
||||
let Some(close) = find_close_tag(html, cursor, "pre") else {
|
||||
continue;
|
||||
};
|
||||
out.push_str(&code_paragraphs(&html[cursor..close]));
|
||||
cursor = close;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The paragraphs for one `<pre>` body whose line breaks are `<br/>`.
|
||||
fn code_paragraphs(body: &str) -> String {
|
||||
// Each line's markup, plus whether it shows any text.
|
||||
let mut lines: Vec<(String, bool)> = Vec::new();
|
||||
// Inline elements open at this point: (name, opening tag).
|
||||
let mut open: Vec<(String, &str)> = Vec::new();
|
||||
let mut line = String::new();
|
||||
let mut visible = false;
|
||||
let mut cursor = 0usize;
|
||||
let finish = |line: &mut String, visible: &mut bool, open: &[(String, &str)]| {
|
||||
let mut done = std::mem::take(line);
|
||||
if !*visible {
|
||||
done.push(NO_BREAK_SPACE);
|
||||
}
|
||||
for (name, _) in open.iter().rev() {
|
||||
done.push_str(&format!("</{name}>"));
|
||||
}
|
||||
let shown = std::mem::replace(visible, false);
|
||||
for (_, tag) in open {
|
||||
line.push_str(tag);
|
||||
}
|
||||
(done, shown)
|
||||
};
|
||||
while cursor < body.len() {
|
||||
let Some(rel) = body[cursor..].find('<') else {
|
||||
let text = &body[cursor..];
|
||||
visible |= shows_text(text);
|
||||
line.push_str(text);
|
||||
break;
|
||||
};
|
||||
let start = cursor + rel;
|
||||
let text = &body[cursor..start];
|
||||
visible |= shows_text(text);
|
||||
line.push_str(text);
|
||||
let Some(end) = tag_end(body, start) else {
|
||||
line.push_str(&body[start..]);
|
||||
break;
|
||||
};
|
||||
let tag = &body[start..end];
|
||||
cursor = end;
|
||||
let inner = tag.trim_start_matches('<').trim_end_matches('>');
|
||||
let name = crate::html::tag_name(inner);
|
||||
if name == "br" {
|
||||
lines.push(finish(&mut line, &mut visible, &open));
|
||||
} else if inner.starts_with('/') {
|
||||
if let Some(at) = open.iter().rposition(|(open_name, _)| *open_name == name) {
|
||||
open.remove(at);
|
||||
}
|
||||
line.push_str(tag);
|
||||
} else {
|
||||
if !inner.trim_end().ends_with('/') {
|
||||
open.push((name, tag));
|
||||
}
|
||||
line.push_str(tag);
|
||||
}
|
||||
}
|
||||
lines.push(finish(&mut line, &mut visible, &open));
|
||||
// Blank lines become a short gap after the line before them rather than
|
||||
// paragraphs of their own: with the firmware's paragraph spacing on, a
|
||||
// blank paragraph adds a full line plus two half-line gaps and reads as a
|
||||
// double newline. Leading, trailing and repeated blanks collapse.
|
||||
let mut kept: Vec<(String, bool)> = Vec::with_capacity(lines.len());
|
||||
for (line, shown) in lines {
|
||||
if shown {
|
||||
kept.push((line, false));
|
||||
} else if let Some(previous) = kept.last_mut() {
|
||||
previous.1 = true;
|
||||
}
|
||||
}
|
||||
if kept.is_empty() {
|
||||
// An all-blank listing still occupies one line.
|
||||
kept.push((NO_BREAK_SPACE.to_string(), false));
|
||||
}
|
||||
|
||||
let last = kept.len() - 1;
|
||||
let mut out = String::with_capacity(body.len() + kept.len() * 48);
|
||||
for (index, (line, gap_after)) in kept.into_iter().enumerate() {
|
||||
let mut class = String::from(CODE_LINE_CLASS);
|
||||
if index == 0 {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_FIRST_CLASS);
|
||||
}
|
||||
if index == last {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_LAST_CLASS);
|
||||
} else if gap_after {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_GAP_CLASS);
|
||||
}
|
||||
out.push_str(&format!("<p class=\"{class}\">{line}</p>"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a run of character data shows anything but spaces.
|
||||
fn shows_text(text: &str) -> bool {
|
||||
text.chars()
|
||||
.any(|c| !c.is_whitespace() && c != NO_BREAK_SPACE)
|
||||
}
|
||||
|
||||
/// Insert soft hyphens into over-long words and make `<pre>` whitespace
|
||||
@@ -609,7 +894,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pre_whitespace_is_made_explicit_inside_nested_tags() {
|
||||
let input = "<pre><code>fn main() {\n if x < 2 {\n <em>return</em>;\n }\n <strong><a href=\"#\">// done</a></strong>\n}</code></pre>";
|
||||
let out = simplify_xhtml(input);
|
||||
let out = break_long_words(input);
|
||||
assert_eq!(
|
||||
out,
|
||||
concat!(
|
||||
@@ -622,10 +907,40 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_code_line_becomes_a_paragraph() {
|
||||
let input = "<p>Before.</p><figure><pre><code>let a = 1;\n\n let b = 2;</code></pre></figure><p>After.</p>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
concat!(
|
||||
"<p>Before.</p><figure><pre>",
|
||||
"<p class=\"code-line code-first code-gap\"><code>let a = 1;</code></p>",
|
||||
"<p class=\"code-line code-last\"><code>\u{00a0}\u{00a0}let b = 2;</code></p>",
|
||||
"</pre></figure><p>After.</p>"
|
||||
)
|
||||
);
|
||||
// `<pre` as a prefix of another name is not a code block.
|
||||
assert_eq!(simplify_xhtml("<prefix>x</prefix>"), "<prefix>x</prefix>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spans_open_across_a_line_break_are_closed_and_reopened() {
|
||||
let input = "<pre><code><span class=\"hl-comment\">/* one\ntwo */</span> x\n</code></pre>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
concat!(
|
||||
"<pre>",
|
||||
"<p class=\"code-line code-first\"><code><span class=\"hl-comment\">/* one</span></code></p>",
|
||||
"<p class=\"code-line code-last\"><code><span class=\"hl-comment\">two */</span> x</code></p>",
|
||||
"</pre>"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tabs_in_pre_become_four_no_break_spaces() {
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre><code>one\ttwo</code></pre>"),
|
||||
break_long_words("<pre><code>one\ttwo</code></pre>"),
|
||||
"<pre><code>one\u{00a0}\u{00a0}\u{00a0}\u{00a0}two</code></pre>"
|
||||
);
|
||||
}
|
||||
@@ -634,7 +949,7 @@ mod tests {
|
||||
fn whitespace_outside_pre_is_untouched() {
|
||||
let input = "<p>outside text\n next\tword</p><pre>x y</pre><p>tail text</p>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
break_long_words(input),
|
||||
"<p>outside text\n next\tword</p><pre>x\u{00a0}\u{00a0}y</pre><p>tail text</p>"
|
||||
);
|
||||
}
|
||||
@@ -642,11 +957,11 @@ mod tests {
|
||||
#[test]
|
||||
fn newline_right_after_pre_open_is_not_a_blank_line() {
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre><code>\nlet a = 1;\n\nlet b = 2;</code></pre>"),
|
||||
break_long_words("<pre><code>\nlet a = 1;\n\nlet b = 2;</code></pre>"),
|
||||
"<pre><code>let a = 1;<br/><br/>let b = 2;</code></pre>"
|
||||
);
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre>\n x</pre>"),
|
||||
break_long_words("<pre>\n x</pre>"),
|
||||
"<pre>\u{00a0}\u{00a0}x</pre>"
|
||||
);
|
||||
}
|
||||
@@ -654,7 +969,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pre_without_a_trailing_newline_gets_no_extra_break() {
|
||||
let input = "<pre><code>let answer = 42;</code></pre>";
|
||||
assert_eq!(simplify_xhtml(input), input);
|
||||
assert_eq!(break_long_words(input), input);
|
||||
}
|
||||
|
||||
/// The shipped X4 stylesheet must already satisfy the X4 rules, so the
|
||||
@@ -794,4 +1109,97 @@ mod tests {
|
||||
"<?xml version=\"1.0\"?>\n<!DOCTYPE html>\n<html><body><p>2 < 3</p></body></html>";
|
||||
assert_eq!(simplify_xhtml(input), input);
|
||||
}
|
||||
|
||||
/// Whether any image in `html` sits inside a table, where the X4
|
||||
/// firmwares print its alt text instead of drawing it.
|
||||
fn image_inside_table(html: &str) -> bool {
|
||||
let mut depth = 0usize;
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
let end = tag_end(html, start).unwrap();
|
||||
let tag = &html[start..end];
|
||||
if is_element_tag(tag, "table", false) {
|
||||
depth += 1;
|
||||
} else if is_element_tag(tag, "table", true) {
|
||||
depth -= 1;
|
||||
} else if depth > 0 && IMAGE_ELEMENTS.iter().any(|n| is_element_tag(tag, n, false)) {
|
||||
return true;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_figure_laid_out_as_a_table_becomes_divs() {
|
||||
// The Space Review's figure markup (issue 2026-10-01).
|
||||
let input = "<p>Before.</p><table><tbody><tr><td><img src=\"images/a.jpg\" alt=\"ELINT images\"/><p>During the 1970s. (credit: NRO)</p></td></tr></tbody></table><h3>Next</h3>";
|
||||
let out = simplify_xhtml(input);
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
assert!(!out.contains("<t"), "{out}");
|
||||
assert!(
|
||||
out.contains("<div><img src=\"images/a.jpg\" alt=\"ELINT images\"/><p>During the 1970s. (credit: NRO)</p></div>"),
|
||||
"{out}"
|
||||
);
|
||||
assert!(
|
||||
out.starts_with("<p>Before.</p>") && out.ends_with("<h3>Next</h3>"),
|
||||
"{out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_cell_of_an_image_table_keeps_its_content_in_order() {
|
||||
let input = "<table class=\"x\" width=\"600\"><caption>Cap</caption><thead><tr><th>A</th><th>B</th></tr></thead><tr><td colspan=\"2\"><img src=\"1.jpg\"/></td></tr><tr><td>left</td><td>right</td></tr></table>";
|
||||
let out = flatten_lossy_tables(input);
|
||||
assert_eq!(
|
||||
out.split_whitespace().collect::<String>(),
|
||||
"<div>Cap</div><div>A</div><div>B</div><div><imgsrc=\"1.jpg\"/></div><div>left</div><div>right</div>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_image_tables_flatten_from_the_outside_in() {
|
||||
for input in [
|
||||
// Image in the inner table only.
|
||||
"<table><tr><td><table><tr><td><img src=\"1.jpg\"/></td></tr></table></td></tr></table><p>after</p>",
|
||||
// Image in the outer table after a text-only inner table.
|
||||
"<table><tr><td><table><tr><td>x</td></tr></table><img src=\"1.jpg\"/></td></tr></table><p>after</p>",
|
||||
// An SVG `<image>` counts too.
|
||||
"<table><tr><td><svg><image href=\"1.jpg\"/></svg></td></tr></table><p>after</p>",
|
||||
] {
|
||||
let out = simplify_xhtml(input);
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
assert!(!out.contains("<table"), "{out}");
|
||||
assert!(out.ends_with("<p>after</p>"), "{out}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_nested_text_table_flattens_its_outer_table() {
|
||||
let input = "<table><tr><td>outer<table><tr><td>inner</td></tr></table></td></tr></table>";
|
||||
let out = flatten_lossy_tables(input);
|
||||
assert_eq!(
|
||||
out.split_whitespace().collect::<String>(),
|
||||
"<div>outer<div>inner</div></div>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_tables_stay_tables() {
|
||||
let input = "<table><tr><th>Year</th><td>1971</td></tr></table><p><img src=\"1.jpg\"/></p><table><tr><td>2</td></tr></table>";
|
||||
assert_eq!(flatten_lossy_tables(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_markup_inside_raw_text_is_left_alone() {
|
||||
let input = "<script>let t = '<table><td><img src=x></td></table>';</script>";
|
||||
assert_eq!(flatten_lossy_tables(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unclosed_image_table_still_flattens() {
|
||||
let out = flatten_lossy_tables("<table><tr><td><img src=\"1.jpg\"/>");
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
}
|
||||
}
|
||||
+28
-3
@@ -15,6 +15,7 @@ use std::time::Duration;
|
||||
use futures::StreamExt;
|
||||
use url::Url;
|
||||
|
||||
use crate::highlight::annotate_hints;
|
||||
use crate::html::word_count;
|
||||
use crate::images::normalize::{normalize_img_tags, prepare_for_readability};
|
||||
use crate::images::refs::collect_image_urls;
|
||||
@@ -338,7 +339,8 @@ fn readable_page(
|
||||
html: &str,
|
||||
url: &str,
|
||||
) -> Result<(String, String, Option<String>, Option<String>), ExtractError> {
|
||||
let html = prepare_for_readability(html);
|
||||
// Readability strips classes, and with them code blocks' language hints.
|
||||
let html = prepare_for_readability(&annotate_hints(html));
|
||||
let config = dom_smoothie::Config {
|
||||
max_elements_to_parse: 60_000,
|
||||
..Default::default()
|
||||
@@ -447,6 +449,7 @@ fn builder() -> ammonia::Builder<'static> {
|
||||
.generic_attributes(HashSet::new())
|
||||
.link_rel(None)
|
||||
.strip_comments(true);
|
||||
crate::highlight::keep_lang_attr(&mut b);
|
||||
b
|
||||
}
|
||||
|
||||
@@ -454,8 +457,10 @@ fn builder() -> ammonia::Builder<'static> {
|
||||
///
|
||||
/// Allowed: `p`, `h1`–`h4`, `ul`/`ol`/`li`, `blockquote`, `pre`, `code`, `em`,
|
||||
/// `strong`, `a`, `img`, `figure`, `figcaption`, table basics, `hr`, `br`.
|
||||
/// The one attribute kept beyond links and images is a code block's language,
|
||||
/// as `<pre data-lang>` (see [`crate::highlight`]).
|
||||
pub fn sanitize(html: &str) -> String {
|
||||
builder().clean(html).to_string()
|
||||
builder().clean(&annotate_hints(html)).to_string()
|
||||
}
|
||||
|
||||
/// [`sanitize`], additionally rewriting relative `href`/`src` against `base_url`
|
||||
@@ -464,7 +469,7 @@ pub fn sanitize_with_base(html: &str, base_url: &str) -> String {
|
||||
match Url::parse(base_url) {
|
||||
Ok(base) => builder()
|
||||
.url_relative(ammonia::UrlRelative::RewriteWithBase(base))
|
||||
.clean(html)
|
||||
.clean(&annotate_hints(html))
|
||||
.to_string(),
|
||||
Err(_) => sanitize(html),
|
||||
}
|
||||
@@ -546,6 +551,26 @@ mod tests {
|
||||
format!("<p>{}</p>", "lorem ".repeat(words))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_keeps_code_block_languages_as_data_lang_only() {
|
||||
let dirty = r#"<div class="language-ruby highlighter-rouge"><pre class="highlight"><code>puts 1</code></pre></div>
|
||||
<pre><code class="language-rust" data-x="y">fn main() {}</code></pre>
|
||||
<pre data-lang="" onload="x">forged</pre>
|
||||
<p data-lang="rust">not a block</p>"#;
|
||||
let clean = sanitize(dirty);
|
||||
assert!(
|
||||
clean.contains(r#"<pre data-lang="ruby"><code>puts 1</code></pre>"#),
|
||||
"{clean}"
|
||||
);
|
||||
assert!(
|
||||
clean.contains(r#"<pre data-lang="rust"><code>fn main() {}</code></pre>"#),
|
||||
"{clean}"
|
||||
);
|
||||
assert!(clean.contains("<pre>forged</pre>"), "{clean}");
|
||||
assert!(clean.contains("<p>not a block</p>"), "{clean}");
|
||||
assert!(!clean.contains("class="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_keeps_the_allowlist_and_drops_everything_else() {
|
||||
let dirty = r#"
|
||||
|
||||
@@ -0,0 +1,743 @@
|
||||
//! Syntax highlighting for code blocks in article bodies.
|
||||
//!
|
||||
//! Two halves, run at different times:
|
||||
//!
|
||||
//! * [`annotate_hints`] runs on source markup at extraction, before
|
||||
//! readability and the sanitizer strip every `class`. It reads whatever
|
||||
//! language hint the page carries (`language-rust`, Rouge's wrapper `<div>`,
|
||||
//! SyntaxHighlighter's `brush: py`, `data-lang`, …) and records it as one
|
||||
//! normalized `data-lang` on the `<pre>`, which both of them keep.
|
||||
//! * [`highlight_blocks`] runs at render time, on sanitized markup, for the web
|
||||
//! article page and both EPUB editions. Blocks with a hint use it; blocks
|
||||
//! without one are guessed from their content ([`guess_language`]), and
|
||||
//! anything not confidently code is left exactly as it was.
|
||||
//!
|
||||
//! Tokens come out as flat `<span class="hl-…">` elements, one class each, so
|
||||
//! the X4 stylesheet (no descendant selectors) can style them too. Colours are
|
||||
//! the stylesheets' business: the web theme is in `src/web/tailwind.css`, the
|
||||
//! monochrome e-ink themes in `src/epub/templates/style*.css`.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use arborium::{Config, GrammarStore, Highlighter, HtmlFormat};
|
||||
|
||||
use crate::html::{attr_escape, decode_entities, parse_attrs, tag_end, tag_name};
|
||||
|
||||
/// The attribute that carries a block's language from extraction to render.
|
||||
pub const LANG_ATTR: &str = "data-lang";
|
||||
|
||||
/// The `data-lang` value for a block its source marked as not code (plain
|
||||
/// text, program output): never highlighted, never guessed.
|
||||
pub const PLAIN: &str = "text";
|
||||
|
||||
/// Prefix of every token class, e.g. `hl-keyword`.
|
||||
const CLASS_PREFIX: &str = "hl";
|
||||
|
||||
/// Hints that mean "this is not source code".
|
||||
const PLAIN_HINTS: &[&str] = &[
|
||||
"text",
|
||||
"txt",
|
||||
"plain",
|
||||
"plaintext",
|
||||
"none",
|
||||
"nohighlight",
|
||||
"no-highlight",
|
||||
"output",
|
||||
"console",
|
||||
"terminal",
|
||||
"log",
|
||||
];
|
||||
|
||||
/// Attributes that name a block's language outright.
|
||||
const LANG_ATTRIBUTES: &[&str] = &[
|
||||
"data-lang",
|
||||
"data-language",
|
||||
"data-code-block-lang",
|
||||
"data-code-language",
|
||||
];
|
||||
|
||||
/// Class prefixes highlighters put in front of a language name.
|
||||
const CLASS_PREFIXES: &[&str] = &[
|
||||
"language-",
|
||||
"lang-",
|
||||
"highlight-source-",
|
||||
"highlight-",
|
||||
"source-",
|
||||
];
|
||||
|
||||
/// Minimum model confidence for a guessed language. Measured on short real
|
||||
/// snippets, guesses at or above this were right; below it they were coin flips.
|
||||
const MIN_GUESS_SCORE: f32 = 0.5;
|
||||
|
||||
/// Unhinted blocks shorter than this are left alone: one-liners are usually a
|
||||
/// command or a path, and too short to classify.
|
||||
const MIN_GUESS_CHARS: usize = 40;
|
||||
|
||||
/// A block that is only plausibly code is weighed against the article's
|
||||
/// prevailing language using at most this many of its best candidates…
|
||||
const MAX_CANDIDATES: usize = 3;
|
||||
|
||||
/// …each scoring at least this much.
|
||||
const MIN_CANDIDATE_SCORE: f32 = 0.1;
|
||||
|
||||
/// The detector reads at most this much of a block.
|
||||
const MAX_GUESS_BYTES: usize = 4096;
|
||||
|
||||
/// Compiled grammars, shared across threads; each call gets its own parser.
|
||||
static GRAMMARS: LazyLock<Arc<GrammarStore>> = LazyLock::new(|| Arc::new(GrammarStore::new()));
|
||||
|
||||
fn new_highlighter() -> Highlighter {
|
||||
Highlighter::with_store_and_config(
|
||||
GRAMMARS.clone(),
|
||||
Config {
|
||||
html_format: HtmlFormat::ClassNamesWithPrefix(CLASS_PREFIX.into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether `name` (a language or one of arborium's aliases) has a grammar.
|
||||
fn supported(name: &str) -> bool {
|
||||
GRAMMARS.get(name).is_some()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hints (extraction time)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalize one hint token (`language-rust`, `py`, `nohighlight`) to the
|
||||
/// value stored in `data-lang`: a supported language name or [`PLAIN`].
|
||||
///
|
||||
/// A bare token (no `language-` style prefix) must be at least two characters,
|
||||
/// so stray one-letter classes don't turn into C or R.
|
||||
pub fn normalize_hint(token: &str) -> Option<String> {
|
||||
let token = token.trim().trim_matches(';').to_ascii_lowercase();
|
||||
let (name, prefixed) = CLASS_PREFIXES
|
||||
.iter()
|
||||
.find_map(|p| token.strip_prefix(p).map(|rest| (rest, true)))
|
||||
.unwrap_or((token.as_str(), false));
|
||||
if name.is_empty() || (!prefixed && name.len() < 2) {
|
||||
return None;
|
||||
}
|
||||
if PLAIN_HINTS.contains(&name) {
|
||||
return Some(PLAIN.to_string());
|
||||
}
|
||||
let name = canonical_name(name);
|
||||
supported(name).then(|| name.to_string())
|
||||
}
|
||||
|
||||
/// The name detection uses for the common aliases a hint may carry, so a
|
||||
/// `py` block and a guessed `python` block count as the same language.
|
||||
fn canonical_name(name: &str) -> &str {
|
||||
match name {
|
||||
"py" | "python3" => "python",
|
||||
"js" | "node" | "jsx" => "javascript",
|
||||
"ts" => "typescript",
|
||||
"sh" | "shell" => "bash",
|
||||
"rs" => "rust",
|
||||
"golang" => "go",
|
||||
"rb" => "ruby",
|
||||
"yml" => "yaml",
|
||||
"c++" | "cxx" | "cc" | "hpp" => "cpp",
|
||||
"cs" | "csharp" | "c#" => "c-sharp",
|
||||
"objective-c" | "objectivec" => "objc",
|
||||
"kt" => "kotlin",
|
||||
"ex" | "exs" => "elixir",
|
||||
"hs" => "haskell",
|
||||
"ps1" | "pwsh" => "powershell",
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// The language hint carried by one element's attributes, if any.
|
||||
fn hint_from_attrs(attrs: &[(String, String)]) -> Option<String> {
|
||||
for (name, value) in attrs {
|
||||
if LANG_ATTRIBUTES.contains(&name.as_str())
|
||||
&& let Some(hint) = normalize_hint(value)
|
||||
{
|
||||
return Some(hint);
|
||||
}
|
||||
}
|
||||
let class = attrs.iter().find(|(name, _)| name == "class")?.1.as_str();
|
||||
// SyntaxHighlighter writes `class="brush: python; gutter: false"`.
|
||||
let tokens: Vec<&str> = class
|
||||
.split(|c: char| c.is_whitespace() || c == ';')
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect();
|
||||
for (i, token) in tokens.iter().enumerate() {
|
||||
let candidate = match token.strip_prefix("brush:") {
|
||||
Some("") => tokens.get(i + 1).copied(),
|
||||
Some(rest) => Some(rest),
|
||||
None if i > 0 && tokens[i - 1] == "brush:" => None,
|
||||
None => Some(*token),
|
||||
};
|
||||
if let Some(hint) = candidate.and_then(normalize_hint) {
|
||||
return Some(hint);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Record each `<pre>`'s language hint as `data-lang`, before readability and
|
||||
/// the sanitizer strip the classes that carry it.
|
||||
///
|
||||
/// The hint may sit on the `<pre>`, on the `<code>` right inside it, or on an
|
||||
/// enclosing `<div>` (Jekyll's Rouge, GitHub's `highlight-source-*`). Markup is
|
||||
/// otherwise returned byte-identical.
|
||||
pub fn annotate_hints(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len() + 64);
|
||||
let mut cursor = 0usize;
|
||||
// Hint of every open `<div>`, innermost last.
|
||||
let mut divs: Vec<Option<String>> = Vec::new();
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
cursor = end;
|
||||
let inner = raw.trim_start_matches('<').trim_end_matches('>');
|
||||
let name = tag_name(inner);
|
||||
if inner.starts_with('/') {
|
||||
if name == "div" {
|
||||
divs.pop();
|
||||
}
|
||||
out.push_str(raw);
|
||||
continue;
|
||||
}
|
||||
match name.as_str() {
|
||||
"div" => {
|
||||
if !inner.trim_end().ends_with('/') {
|
||||
divs.push(hint_from_attrs(&parse_attrs(inner)));
|
||||
}
|
||||
out.push_str(raw);
|
||||
}
|
||||
"pre" => {
|
||||
let attrs = parse_attrs(inner.trim_end_matches('/'));
|
||||
let hint = hint_from_attrs(&attrs)
|
||||
.or_else(|| first_code_hint(html, end))
|
||||
.or_else(|| divs.iter().rev().find_map(Clone::clone));
|
||||
match hint {
|
||||
Some(hint) => out.push_str(&pre_tag(&attrs, &hint)),
|
||||
None => out.push_str(raw),
|
||||
}
|
||||
}
|
||||
_ => out.push_str(raw),
|
||||
}
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The hint on a `<code>` that opens right after a `<pre>` ending at `after`.
|
||||
fn first_code_hint(html: &str, after: usize) -> Option<String> {
|
||||
let rest = &html[after..];
|
||||
let start = after + (rest.len() - rest.trim_start().len());
|
||||
if !html[start..].starts_with('<') {
|
||||
return None;
|
||||
}
|
||||
let end = tag_end(html, start)?;
|
||||
let inner = html[start..end]
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>');
|
||||
(tag_name(inner) == "code").then(|| hint_from_attrs(&parse_attrs(inner)))?
|
||||
}
|
||||
|
||||
/// Rebuild a `<pre>` tag with `data-lang` set to `hint`.
|
||||
fn pre_tag(attrs: &[(String, String)], hint: &str) -> String {
|
||||
let mut tag = String::from("<pre");
|
||||
for (name, value) in attrs {
|
||||
if LANG_ATTRIBUTES.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
tag.push_str(&format!(" {name}=\"{}\"", attr_escape(value)));
|
||||
}
|
||||
tag.push_str(&format!(" {LANG_ATTR}=\"{}\">", attr_escape(hint)));
|
||||
tag
|
||||
}
|
||||
|
||||
/// Sanitizer filter for `data-lang`: keep it only if it is a value
|
||||
/// [`annotate_hints`] could have written.
|
||||
pub fn filter_lang_attr(value: &str) -> Option<String> {
|
||||
normalize_hint(value).filter(|hint| hint == &value.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
/// Let an ammonia builder keep a valid `data-lang` on `<pre>` (and nothing else
|
||||
/// it did not already allow).
|
||||
pub fn keep_lang_attr(builder: &mut ammonia::Builder<'_>) {
|
||||
builder
|
||||
.add_tag_attributes("pre", &[LANG_ATTR])
|
||||
.attribute_filter(|element, attribute, value| {
|
||||
if element == "pre" && attribute == LANG_ATTR {
|
||||
filter_lang_attr(value).map(Cow::Owned)
|
||||
} else {
|
||||
Some(Cow::Borrowed(value))
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// `ammonia::clean` that keeps block languages, then [`highlight_blocks`]:
|
||||
/// the render-time entry point for article bodies.
|
||||
pub fn clean_and_highlight(html: &str) -> String {
|
||||
let mut builder = ammonia::Builder::default();
|
||||
keep_lang_attr(&mut builder);
|
||||
highlight_blocks(&builder.clean(html).to_string())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// What an unhinted block's content says about its language.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Detected {
|
||||
/// Confidently this language.
|
||||
Sure(&'static str),
|
||||
/// Plausibly one of these, best first: taken only when the rest of the
|
||||
/// article is in one of them (see [`highlight_blocks`]).
|
||||
Maybe(Vec<&'static str>),
|
||||
/// Not code, or too little of it to tell.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Best guess at the language of an unhinted block, or `None` when it is not
|
||||
/// confidently code. A wrong colour scheme is worse than none.
|
||||
pub fn guess_language(code: &str) -> Option<&'static str> {
|
||||
match detect(code) {
|
||||
Detected::Sure(language) => Some(language),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn detect(code: &str) -> Detected {
|
||||
let text = code.trim();
|
||||
// Parsing is proof, at any length.
|
||||
if (text.starts_with('{') || text.starts_with('['))
|
||||
&& serde_json::from_str::<serde_json::Value>(text).is_ok()
|
||||
{
|
||||
return Detected::Sure("json");
|
||||
}
|
||||
if text.chars().count() < MIN_GUESS_CHARS && !text.starts_with("#!") {
|
||||
return Detected::Unknown;
|
||||
}
|
||||
if let Some(language) = first_line_language(text) {
|
||||
return Detected::Sure(language);
|
||||
}
|
||||
// A shell session is prompts interleaved with output; neither grammar fits.
|
||||
if text.starts_with("$ ") || text.starts_with("> ") || text.starts_with("% ") {
|
||||
return Detected::Unknown;
|
||||
}
|
||||
let sample = crate::html::truncate_utf8(text, MAX_GUESS_BYTES);
|
||||
let detection = betlang::detect(sample);
|
||||
let mut ranked = detection.top_languages().take(MAX_CANDIDATES);
|
||||
let Some((score, top)) = ranked.next() else {
|
||||
return Detected::Unknown;
|
||||
};
|
||||
// Prose, logs and program output all come out as Markdown.
|
||||
let Some(top) = betlang_language(top) else {
|
||||
return Detected::Unknown;
|
||||
};
|
||||
if score >= MIN_GUESS_SCORE {
|
||||
return Detected::Sure(top);
|
||||
}
|
||||
let mut candidates = vec![top];
|
||||
candidates.extend(
|
||||
ranked
|
||||
.filter(|(score, _)| *score >= MIN_CANDIDATE_SCORE)
|
||||
.filter_map(|(_, language)| betlang_language(language)),
|
||||
);
|
||||
Detected::Maybe(candidates)
|
||||
}
|
||||
|
||||
/// The grammar for a betlang label, if we have one.
|
||||
fn betlang_language(language: betlang::Language) -> Option<&'static str> {
|
||||
let name = match language.slug() {
|
||||
"markdown" => return None,
|
||||
"cs" => "c-sharp",
|
||||
"objectivec" => "objc",
|
||||
"lisp" => "commonlisp",
|
||||
"shell" => "bash",
|
||||
"gemfile" | "gemspec" => "ruby",
|
||||
"gradle" => "groovy",
|
||||
other => other,
|
||||
};
|
||||
supported(name).then_some(name)
|
||||
}
|
||||
|
||||
/// Languages a block announces on its first line: shebangs and XML-ish preludes.
|
||||
fn first_line_language(text: &str) -> Option<&'static str> {
|
||||
let first = text.lines().next()?.trim().to_ascii_lowercase();
|
||||
if let Some(shebang) = first.strip_prefix("#!") {
|
||||
let interpreter = shebang
|
||||
.split_whitespace()
|
||||
.flat_map(|word| word.rsplit('/').next())
|
||||
.find(|word| *word != "env" && !word.starts_with('-'))?;
|
||||
let name = match interpreter.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.') {
|
||||
"sh" | "bash" | "dash" | "ksh" => "bash",
|
||||
"zsh" => "zsh",
|
||||
"fish" => "fish",
|
||||
"python" => "python",
|
||||
"node" | "deno" | "bun" => "javascript",
|
||||
"ruby" => "ruby",
|
||||
"perl" => "perl",
|
||||
"php" => "php",
|
||||
"lua" => "lua",
|
||||
_ => return None,
|
||||
};
|
||||
return supported(name).then_some(name);
|
||||
}
|
||||
if first.starts_with("<?php") {
|
||||
return Some("php");
|
||||
}
|
||||
if first.starts_with("<?xml") {
|
||||
return Some("xml");
|
||||
}
|
||||
// Pages and the tags posts most often quote on their own.
|
||||
if ["<!doctype html", "<html", "<script", "<style", "<template"]
|
||||
.iter()
|
||||
.any(|prefix| first.starts_with(prefix))
|
||||
{
|
||||
return Some("html");
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rendering
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One `<pre>` block found by [`highlight_blocks`].
|
||||
struct Block {
|
||||
/// Byte range of the opening tag, and where `</pre>` starts.
|
||||
start: usize,
|
||||
open_end: usize,
|
||||
close: usize,
|
||||
attrs: Vec<(String, String)>,
|
||||
hint: Option<String>,
|
||||
code: Option<String>,
|
||||
detected: Detected,
|
||||
}
|
||||
|
||||
/// Highlight every `<pre>` block in sanitized markup.
|
||||
///
|
||||
/// A block's language is its `data-lang`, else what its content confidently
|
||||
/// says ([`guess_language`]). A block whose content is only plausibly code
|
||||
/// takes the article's prevailing language when that is one of its
|
||||
/// candidates, so a post in Go does not alternate between highlighted and
|
||||
/// plain listings. A highlighted block becomes `<pre><code>…spans…</code></pre>`;
|
||||
/// any other block is copied through untouched apart from losing `data-lang`,
|
||||
/// which is only for this step.
|
||||
pub fn highlight_blocks(html: &str) -> String {
|
||||
if !html.contains("<pre") {
|
||||
return html.to_string();
|
||||
}
|
||||
let blocks = find_blocks(html);
|
||||
let prevailing = prevailing_language(&blocks);
|
||||
let mut highlighter: Option<Highlighter> = None;
|
||||
let mut out = String::with_capacity(html.len() * 2);
|
||||
let mut cursor = 0usize;
|
||||
for block in &blocks {
|
||||
out.push_str(&html[cursor..block.start]);
|
||||
let language = match (block.hint.as_deref(), &block.detected) {
|
||||
(Some(PLAIN), _) => None,
|
||||
(Some(hint), _) => Some(hint),
|
||||
(None, Detected::Sure(language)) => Some(*language),
|
||||
(None, Detected::Maybe(candidates)) => {
|
||||
prevailing.filter(|language| candidates.contains(language))
|
||||
}
|
||||
(None, Detected::Unknown) => None,
|
||||
};
|
||||
let highlighted = match (language, &block.code) {
|
||||
(Some(language), Some(code)) => highlighter
|
||||
.get_or_insert_with(new_highlighter)
|
||||
.highlight(language, code)
|
||||
.ok()
|
||||
.filter(|spans| spans.contains("<span")),
|
||||
_ => None,
|
||||
};
|
||||
match highlighted {
|
||||
Some(spans) => {
|
||||
out.push_str("<pre><code>");
|
||||
out.push_str(&spans);
|
||||
out.push_str("</code></pre>");
|
||||
}
|
||||
None => {
|
||||
out.push_str(&pre_without_lang(
|
||||
&block.attrs,
|
||||
block.hint.is_some(),
|
||||
&html[block.start..block.open_end],
|
||||
));
|
||||
out.push_str(&html[block.open_end..block.close]);
|
||||
out.push_str("</pre>");
|
||||
}
|
||||
}
|
||||
cursor = block.close + "</pre>".len();
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Every complete `<pre>` block, in order, with its hint and detection.
|
||||
fn find_blocks(html: &str) -> Vec<Block> {
|
||||
let mut blocks = Vec::new();
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find("<pre") {
|
||||
let start = cursor + rel;
|
||||
let Some(open_end) = tag_end(html, start) else {
|
||||
break;
|
||||
};
|
||||
let inner = html[start..open_end]
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>');
|
||||
if tag_name(inner) != "pre" {
|
||||
cursor = open_end;
|
||||
continue;
|
||||
}
|
||||
let Some(close_rel) = html[open_end..].find("</pre>") else {
|
||||
break;
|
||||
};
|
||||
let close = open_end + close_rel;
|
||||
let attrs = parse_attrs(inner);
|
||||
let hint = attrs
|
||||
.iter()
|
||||
.find(|(name, _)| name == LANG_ATTR)
|
||||
.map(|(_, value)| value.clone());
|
||||
let code = block_text(&html[open_end..close]);
|
||||
let detected = match (&hint, &code) {
|
||||
(None, Some(code)) => detect(code),
|
||||
_ => Detected::Unknown,
|
||||
};
|
||||
blocks.push(Block {
|
||||
start,
|
||||
open_end,
|
||||
close,
|
||||
attrs,
|
||||
hint,
|
||||
code,
|
||||
detected,
|
||||
});
|
||||
cursor = close + "</pre>".len();
|
||||
}
|
||||
blocks
|
||||
}
|
||||
|
||||
/// The language most of an article's hinted and confidently detected blocks
|
||||
/// share, earliest first on a tie.
|
||||
fn prevailing_language(blocks: &[Block]) -> Option<&str> {
|
||||
let mut counts: Vec<(&str, usize)> = Vec::new();
|
||||
for block in blocks {
|
||||
let language = match (block.hint.as_deref(), &block.detected) {
|
||||
(Some(PLAIN), _) => continue,
|
||||
(Some(hint), _) => hint,
|
||||
(None, Detected::Sure(language)) => language,
|
||||
_ => continue,
|
||||
};
|
||||
match counts.iter_mut().find(|(name, _)| *name == language) {
|
||||
Some((_, count)) => *count += 1,
|
||||
None => counts.push((language, 1)),
|
||||
}
|
||||
}
|
||||
counts
|
||||
.iter()
|
||||
.rev()
|
||||
.max_by_key(|(_, count)| *count)
|
||||
.map(|(language, _)| *language)
|
||||
}
|
||||
|
||||
/// The `<pre>` opening tag minus `data-lang`, or `raw` when it had none.
|
||||
fn pre_without_lang(attrs: &[(String, String)], had_lang: bool, raw: &str) -> String {
|
||||
if !had_lang {
|
||||
return raw.to_string();
|
||||
}
|
||||
let mut tag = String::from("<pre");
|
||||
for (name, value) in attrs.iter().filter(|(name, _)| name != LANG_ATTR) {
|
||||
tag.push_str(&format!(" {name}=\"{}\"", attr_escape(value)));
|
||||
}
|
||||
tag.push('>');
|
||||
tag
|
||||
}
|
||||
|
||||
/// The source text of a `<pre>` body: an optional wrapping `<code>` removed,
|
||||
/// `<br>` as newlines, inline markup dropped, entities decoded, and the one
|
||||
/// leading newline HTML ignores stripped.
|
||||
///
|
||||
/// `None` when the body holds block markup (a table, a list) — that is a
|
||||
/// layout, not a listing, and is left alone.
|
||||
fn block_text(body: &str) -> Option<String> {
|
||||
let mut text = String::with_capacity(body.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = body[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
text.push_str(&body[cursor..start]);
|
||||
let end = tag_end(body, start)?;
|
||||
let inner = body[start..end]
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>');
|
||||
match tag_name(inner).as_str() {
|
||||
"br" => text.push('\n'),
|
||||
"code" | "span" | "em" | "strong" | "a" | "b" | "i" | "u" | "s" | "mark" | "kbd"
|
||||
| "samp" | "var" | "sub" | "sup" | "small" => {}
|
||||
_ => return None,
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
text.push_str(&body[cursor..]);
|
||||
let text = decode_entities(&text);
|
||||
let text = text
|
||||
.strip_prefix("\r\n")
|
||||
.or_else(|| text.strip_prefix('\n'))
|
||||
.unwrap_or(&text);
|
||||
let text = text.trim_end_matches(['\n', '\r']);
|
||||
(!text.trim().is_empty()).then(|| text.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn normalizes_prefixed_bare_and_plain_hints() {
|
||||
assert_eq!(normalize_hint("language-rust").as_deref(), Some("rust"));
|
||||
assert_eq!(normalize_hint("lang-py").as_deref(), Some("python"));
|
||||
assert_eq!(normalize_hint("highlight-source-go").as_deref(), Some("go"));
|
||||
assert_eq!(normalize_hint("Python").as_deref(), Some("python"));
|
||||
assert_eq!(normalize_hint("language-text").as_deref(), Some(PLAIN));
|
||||
assert_eq!(normalize_hint("nohighlight").as_deref(), Some(PLAIN));
|
||||
assert_eq!(normalize_hint("language-c").as_deref(), Some("c"));
|
||||
// Bare one-letter classes and framework classes are not hints.
|
||||
assert_eq!(normalize_hint("c"), None);
|
||||
assert_eq!(normalize_hint("highlight"), None);
|
||||
assert_eq!(normalize_hint("chroma"), None);
|
||||
assert_eq!(normalize_hint("language-klingon"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn annotates_hints_from_pre_code_and_wrapper_divs() {
|
||||
assert_eq!(
|
||||
annotate_hints(r#"<pre class="chroma"><code class="language-go">x</code></pre>"#),
|
||||
r#"<pre class="chroma" data-lang="go"><code class="language-go">x</code></pre>"#
|
||||
);
|
||||
assert_eq!(
|
||||
annotate_hints(r#"<pre class="brush: py; gutter: false">x</pre>"#),
|
||||
r#"<pre class="brush: py; gutter: false" data-lang="python">x</pre>"#
|
||||
);
|
||||
// Jekyll/Rouge: the hint is on a wrapper two levels up.
|
||||
assert_eq!(
|
||||
annotate_hints(
|
||||
r#"<div class="language-ruby highlighter-rouge"><div class="highlight"><pre class="highlight"><code>x</code></pre></div></div><pre>y</pre>"#
|
||||
),
|
||||
r#"<div class="language-ruby highlighter-rouge"><div class="highlight"><pre class="highlight" data-lang="ruby"><code>x</code></pre></div></div><pre>y</pre>"#
|
||||
);
|
||||
assert_eq!(
|
||||
annotate_hints(r#"<pre data-code-block-lang="JavaScript">x</pre>"#),
|
||||
r#"<pre data-lang="javascript">x</pre>"#
|
||||
);
|
||||
let plain = "<p>a</p><pre><code>no hint</code></pre>";
|
||||
assert_eq!(annotate_hints(plain), plain);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_sanitizer_filter_only_keeps_normalized_values() {
|
||||
assert_eq!(filter_lang_attr("rust").as_deref(), Some("rust"));
|
||||
assert_eq!(filter_lang_attr("text").as_deref(), Some(PLAIN));
|
||||
assert_eq!(filter_lang_attr("language-rust"), None);
|
||||
// Hints are stored canonical, so an alias is not a stored value.
|
||||
assert_eq!(filter_lang_attr("py"), None);
|
||||
assert_eq!(filter_lang_attr("\" onload=\"x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn highlights_hinted_blocks_with_flat_prefixed_spans() {
|
||||
let out = highlight_blocks(
|
||||
"<p>a</p><pre data-lang=\"rust\"><code>fn main() {\n let s = \"<&>\";\n}</code></pre><p>b</p>",
|
||||
);
|
||||
assert!(out.starts_with("<p>a</p><pre><code><span class=\"hl-keyword\">fn</span>"));
|
||||
assert!(out.contains("<span class=\"hl-string\">"<&>"</span>"));
|
||||
assert!(out.ends_with("</code></pre><p>b</p>"));
|
||||
assert!(!out.contains("data-lang"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn br_line_breaks_and_inline_markup_become_plain_source() {
|
||||
let out = highlight_blocks(
|
||||
"<pre data-lang=\"python\"><code>def f():<br> <em>return</em> 1</code></pre>",
|
||||
);
|
||||
assert!(out.contains("<span class=\"hl-keyword\">def</span>"));
|
||||
assert!(out.contains("\n <span class=\"hl-keyword\">return</span>"));
|
||||
assert!(!out.contains("<em>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_and_unrecognized_blocks_are_left_alone() {
|
||||
assert_eq!(
|
||||
highlight_blocks("<pre data-lang=\"text\"><code>def f(): return 1</code></pre>"),
|
||||
"<pre><code>def f(): return 1</code></pre>"
|
||||
);
|
||||
let short = "<pre><code>cargo build</code></pre>";
|
||||
assert_eq!(highlight_blocks(short), short);
|
||||
let prose = "<pre>Dear reader, this is a poem set in a preformatted block\nbecause the author liked the look of it.</pre>";
|
||||
assert_eq!(highlight_blocks(prose), prose);
|
||||
let session = "<pre><code>$ cargo build --release\n Compiling foo v0.1.0\n Finished release</code></pre>";
|
||||
assert_eq!(highlight_blocks(session), session);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guesses_unhinted_code_by_content() {
|
||||
assert_eq!(
|
||||
guess_language(
|
||||
"import os\n\ndef main():\n print(os.getcwd())\n\nif __name__ == '__main__':\n main()\n"
|
||||
),
|
||||
Some("python")
|
||||
);
|
||||
assert_eq!(
|
||||
guess_language(
|
||||
"SELECT id, name FROM users WHERE created_at > NOW() - INTERVAL '1 day' ORDER BY id;"
|
||||
),
|
||||
Some("sql")
|
||||
);
|
||||
assert_eq!(
|
||||
guess_language("#!/usr/bin/env bash\nset -euo pipefail"),
|
||||
Some("bash")
|
||||
);
|
||||
assert_eq!(
|
||||
guess_language("{\"a\": [1, 2, 3], \"b\": {\"c\": null}}"),
|
||||
Some("json")
|
||||
);
|
||||
let out = highlight_blocks(
|
||||
"<pre><code>#include <stdio.h>\n\nint main(void) {\n printf(\"hello\\n\");\n return 0;\n}</code></pre>",
|
||||
);
|
||||
assert!(out.contains("hl-"), "{out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_uncertain_block_takes_the_articles_prevailing_language() {
|
||||
// Both from go.dev/blog/range-functions, which marks no languages.
|
||||
let sure = "<pre><code>// Add adds an element to a set.\nfunc (s *Set[E]) Add(v E) {\n s.m[v] = struct{}{}\n}\n\n// Contains reports whether an element is in a set.\nfunc (s *Set[E]) Contains(v E) bool {\n _, ok := s.m[v]\n return ok\n}\n</code></pre>";
|
||||
let unsure = "<pre><code>func (s *Set[E]) Push(f func(E) bool) {\n for v := range s.m {\n if !f(v) {\n return\n }\n }\n}\n</code></pre>";
|
||||
let text = |pre: &str| block_text(&pre[5..pre.len() - 6]).unwrap();
|
||||
assert_eq!(guess_language(&text(sure)), Some("go"));
|
||||
assert_eq!(guess_language(&text(unsure)), None);
|
||||
assert!(matches!(detect(&text(unsure)), Detected::Maybe(c) if c.contains(&"go")));
|
||||
|
||||
let alone = highlight_blocks(unsure);
|
||||
assert_eq!(alone, unsure);
|
||||
let together = highlight_blocks(&format!("{sure}<p>and</p>{unsure}"));
|
||||
assert_eq!(
|
||||
together
|
||||
.matches("<span class=\"hl-keyword\">func</span>")
|
||||
.count(),
|
||||
4
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quoted_style_and_script_tags_are_html() {
|
||||
assert_eq!(
|
||||
guess_language("<style>\n* {\n box-sizing: border-box;\n}\n</style>"),
|
||||
Some("html")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,812 @@
|
||||
//! Standing-interest storage and rating-derived weights.
|
||||
//!
|
||||
//! Interest queries stay here so the central database layer remains focused on
|
||||
//! the pipeline's shared records.
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
use jiff::Timestamp;
|
||||
use serde::Deserialize;
|
||||
use sqlx::Row as _;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::curate::llm::{LlmClient, Llms, provider_meters};
|
||||
use crate::curate::profile;
|
||||
use crate::curate::signals::TopInterest;
|
||||
use crate::db::{Db, fmt_ts};
|
||||
use crate::types::ArticleId;
|
||||
|
||||
/// One standing interest and its optional prompt category.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Interest {
|
||||
pub id: i64,
|
||||
pub name: String,
|
||||
pub category: Option<String>,
|
||||
pub created_at: String,
|
||||
pub categorized_at: Option<String>,
|
||||
}
|
||||
|
||||
/// Result of adding a name whose uniqueness is case-insensitive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AddOutcome {
|
||||
Added(i64),
|
||||
Duplicate,
|
||||
}
|
||||
|
||||
/// One stored article-to-interest match.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct MatchRow {
|
||||
pub article_id: ArticleId,
|
||||
pub interest_id: i64,
|
||||
pub name: String,
|
||||
pub cos: f64,
|
||||
pub z: f64,
|
||||
}
|
||||
|
||||
/// Rating credit accumulated for one interest.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||
pub struct Rate {
|
||||
pub up: f64,
|
||||
pub down: f64,
|
||||
pub n: usize,
|
||||
}
|
||||
|
||||
impl Rate {
|
||||
/// Beta smoothing keeps an unrated interest neutral.
|
||||
pub fn weight(&self) -> f64 {
|
||||
(self.up + 1.0) / (self.up + self.down + 2.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Interest rates plus the number of ratings that could affect them.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct Rates {
|
||||
pub by_interest: HashMap<i64, Rate>,
|
||||
pub attributable: usize,
|
||||
}
|
||||
|
||||
const INTEREST_COLUMNS: &str = "id, name, category, created_at, categorized_at";
|
||||
|
||||
/// Dashboard article search for one interest name.
|
||||
pub fn articles_href(name: &str) -> String {
|
||||
format!(
|
||||
"/dashboard/articles?interest={}",
|
||||
crate::web::encode_component(name)
|
||||
)
|
||||
}
|
||||
|
||||
/// Parse an OPML export for the one-time interests importer.
|
||||
pub fn parse_opml(raw: &str) -> Vec<String> {
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut out = Vec::new();
|
||||
for chunk in raw.split("text=\"").skip(1) {
|
||||
let Some((value, _)) = chunk.split_once('"') else {
|
||||
continue;
|
||||
};
|
||||
let name = xml_unescape(value).trim().to_string();
|
||||
if !name.is_empty() && seen.insert(name.to_lowercase()) {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn xml_unescape(value: &str) -> String {
|
||||
if !value.contains('&') {
|
||||
return value.to_string();
|
||||
}
|
||||
value
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("'", "'")
|
||||
.replace("&", "&")
|
||||
}
|
||||
|
||||
fn interest_from(row: &sqlx::sqlite::SqliteRow) -> Interest {
|
||||
Interest {
|
||||
id: row.get("id"),
|
||||
name: row.get("name"),
|
||||
category: row.get("category"),
|
||||
created_at: row.get("created_at"),
|
||||
categorized_at: row.get("categorized_at"),
|
||||
}
|
||||
}
|
||||
|
||||
/// All interests, ordered case-insensitively by name.
|
||||
pub async fn list(db: &Db) -> Result<Vec<Interest>> {
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"SELECT {INTEREST_COLUMNS} FROM interests ORDER BY name COLLATE NOCASE, name"
|
||||
)))
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
Ok(rows.iter().map(interest_from).collect())
|
||||
}
|
||||
|
||||
/// Add one trimmed, non-empty name of at most 80 characters.
|
||||
pub async fn add(
|
||||
db: &Db,
|
||||
name: &str,
|
||||
category: Option<&str>,
|
||||
now: Timestamp,
|
||||
) -> Result<AddOutcome> {
|
||||
let name = name.trim();
|
||||
let len = name.chars().count();
|
||||
if !(1..=80).contains(&len) {
|
||||
bail!("interest name must be 1–80 characters");
|
||||
}
|
||||
|
||||
let result = sqlx::query(
|
||||
"INSERT OR IGNORE INTO interests (name, category, created_at, categorized_at)
|
||||
VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.bind(name)
|
||||
.bind(category)
|
||||
.bind(fmt_ts(now))
|
||||
.bind(category.map(|_| fmt_ts(now)))
|
||||
.execute(db.pool())
|
||||
.await?;
|
||||
if result.rows_affected() == 0 {
|
||||
Ok(AddOutcome::Duplicate)
|
||||
} else {
|
||||
Ok(AddOutcome::Added(result.last_insert_rowid()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Set or clear an interest category and its categorization timestamp together.
|
||||
pub async fn set_category(db: &Db, id: i64, category: Option<&str>, now: Timestamp) -> Result<()> {
|
||||
sqlx::query("UPDATE interests SET category = ?, categorized_at = ? WHERE id = ?")
|
||||
.bind(category)
|
||||
.bind(category.map(|_| fmt_ts(now)))
|
||||
.bind(id)
|
||||
.execute(db.pool())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete an interest, its match rows, and its name-keyed cached embedding.
|
||||
pub async fn delete(db: &Db, id: i64) -> Result<()> {
|
||||
let mut tx = db.pool().begin().await?;
|
||||
let name: Option<String> = sqlx::query_scalar("SELECT name FROM interests WHERE id = ?")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
sqlx::query("DELETE FROM interests WHERE id = ?")
|
||||
.bind(id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
if let Some(name) = name {
|
||||
sqlx::query("DELETE FROM interest_embeddings WHERE interest = ?")
|
||||
.bind(name)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// All names in the stable order used for embedding requests.
|
||||
pub async fn names(db: &Db) -> Result<Vec<String>> {
|
||||
Ok(
|
||||
sqlx::query_scalar("SELECT name FROM interests ORDER BY name COLLATE NOCASE, name")
|
||||
.fetch_all(db.pool())
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
/// Names grouped for the prompt, with uncategorized interests last.
|
||||
pub async fn grouped(db: &Db) -> Result<Vec<(String, Vec<String>)>> {
|
||||
let mut by_category: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut other = Vec::new();
|
||||
for interest in list(db).await? {
|
||||
if let Some(category) = interest.category {
|
||||
by_category.entry(category).or_default().push(interest.name);
|
||||
} else {
|
||||
other.push(interest.name);
|
||||
}
|
||||
}
|
||||
|
||||
let mut groups: Vec<_> = by_category.into_iter().collect();
|
||||
groups.sort_by(|left, right| {
|
||||
left.0
|
||||
.to_lowercase()
|
||||
.cmp(&right.0.to_lowercase())
|
||||
.then_with(|| left.0.cmp(&right.0))
|
||||
});
|
||||
for (_, members) in &mut groups {
|
||||
members.sort_by(|left, right| {
|
||||
left.to_lowercase()
|
||||
.cmp(&right.to_lowercase())
|
||||
.then_with(|| left.cmp(right))
|
||||
});
|
||||
}
|
||||
if !other.is_empty() {
|
||||
groups.push(("Other standing interests".to_string(), other));
|
||||
}
|
||||
Ok(groups)
|
||||
}
|
||||
|
||||
/// Interests awaiting the categorizer, ordered case-insensitively by name.
|
||||
pub async fn uncategorized(db: &Db) -> Result<Vec<Interest>> {
|
||||
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"SELECT {INTEREST_COLUMNS} FROM interests WHERE category IS NULL
|
||||
ORDER BY name COLLATE NOCASE, name"
|
||||
)))
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
Ok(rows.iter().map(interest_from).collect())
|
||||
}
|
||||
|
||||
/// Upsert the current run's recorded top-interest matches in one transaction.
|
||||
pub async fn replace_matches(
|
||||
db: &Db,
|
||||
run_id: Option<i64>,
|
||||
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||
ids: &HashMap<String, i64>,
|
||||
) -> Result<()> {
|
||||
write_matches(db, run_id, matches, ids, true).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Insert backfilled matches without disturbing rows a real run wrote.
|
||||
pub async fn insert_matches_if_absent(
|
||||
db: &Db,
|
||||
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||
ids: &HashMap<String, i64>,
|
||||
) -> Result<u64> {
|
||||
write_matches(db, None, matches, ids, false).await
|
||||
}
|
||||
|
||||
/// One transaction over the top interests of every article, either upserting
|
||||
/// (a run's own rows) or leaving whatever is already stored alone (a backfill).
|
||||
async fn write_matches(
|
||||
db: &Db,
|
||||
run_id: Option<i64>,
|
||||
matches: &[(ArticleId, Vec<TopInterest>)],
|
||||
ids: &HashMap<String, i64>,
|
||||
replace: bool,
|
||||
) -> Result<u64> {
|
||||
let sql = if replace {
|
||||
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(article_id, interest_id) DO UPDATE SET
|
||||
cos = excluded.cos, z = excluded.z, run_id = excluded.run_id"
|
||||
} else {
|
||||
"INSERT OR IGNORE INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||
VALUES (?, ?, ?, ?, ?)"
|
||||
};
|
||||
let mut tx = db.pool().begin().await?;
|
||||
let mut written = 0;
|
||||
for (article_id, top_interests) in matches {
|
||||
for top in top_interests {
|
||||
let Some(interest_id) = ids.get(&top.name) else {
|
||||
continue;
|
||||
};
|
||||
written += sqlx::query(sql)
|
||||
.bind(article_id)
|
||||
.bind(interest_id)
|
||||
.bind(top.cos)
|
||||
.bind(top.z)
|
||||
.bind(run_id)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
const CATEGORIZE_PROMPT: &str = r#"TASK: file each new standing interest under one of the reader's interest categories.
|
||||
Existing categories (reuse these names verbatim): {categories}
|
||||
Create a new category only when none of the existing ones fits; a new category must be broad enough to hold several interests and named like the existing ones (two to five words, sentence case). Every interest gets exactly one category.
|
||||
New interests: {interests}
|
||||
Return JSON exactly: {"assignments":[{"interest":"…","category":"…"}]}"#;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CategorizeResponse {
|
||||
#[serde(default)]
|
||||
assignments: Vec<CategoryAssignment>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CategoryAssignment {
|
||||
interest: String,
|
||||
category: String,
|
||||
}
|
||||
|
||||
/// File every currently uncategorized interest in one bulk-model call.
|
||||
pub async fn categorize(config: &Config, db: &Db) -> Result<String> {
|
||||
let pending = uncategorized(db).await?;
|
||||
if pending.is_empty() {
|
||||
return Ok("nothing to categorize".into());
|
||||
}
|
||||
let taste = profile::load_or_build(
|
||||
db,
|
||||
&config.profile_path,
|
||||
config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
.await?;
|
||||
let llms = Llms::from_config(config, taste.text, &provider_meters(config));
|
||||
let llm = llms
|
||||
.bulk
|
||||
.as_ref()
|
||||
.or_else(|| llms.editor_or_bulk())
|
||||
.context("no LLM provider is available for interest categorization")?;
|
||||
categorize_with_llm(db, &pending, llm).await
|
||||
}
|
||||
|
||||
async fn categorize_with_llm(db: &Db, pending: &[Interest], llm: &LlmClient) -> Result<String> {
|
||||
let all = list(db).await?;
|
||||
let categories = all
|
||||
.iter()
|
||||
.filter_map(|interest| interest.category.as_deref())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
let names = pending
|
||||
.iter()
|
||||
.map(|interest| interest.name.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
let prompt = CATEGORIZE_PROMPT
|
||||
.replace("{categories}", &categories.join(", "))
|
||||
.replace("{interests}", &format!("\n{}", names.join("\n")));
|
||||
let response: CategorizeResponse = llm.complete_json(&prompt, 0.2).await?;
|
||||
|
||||
let pending_by_name = pending
|
||||
.iter()
|
||||
.map(|interest| (interest.name.to_lowercase(), interest))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let existing_categories = categories
|
||||
.iter()
|
||||
.map(|category| (category.to_lowercase(), *category))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut assigned = BTreeSet::new();
|
||||
let mut new_categories = BTreeSet::new();
|
||||
let now = Timestamp::now();
|
||||
for assignment in response.assignments {
|
||||
let Some(interest) = pending_by_name.get(&assignment.interest.trim().to_lowercase()) else {
|
||||
continue;
|
||||
};
|
||||
let category = assignment.category.trim();
|
||||
if !(1..=60).contains(&category.chars().count()) || !assigned.insert(interest.id) {
|
||||
continue;
|
||||
}
|
||||
// A model that answers "software" for an existing "Software" must not
|
||||
// split the category.
|
||||
let category = match existing_categories.get(&category.to_lowercase()) {
|
||||
Some(existing) => existing,
|
||||
None => {
|
||||
new_categories.insert(category.to_string());
|
||||
category
|
||||
}
|
||||
};
|
||||
set_category(db, interest.id, Some(category), now).await?;
|
||||
}
|
||||
|
||||
let mut message = format!(
|
||||
"categorized {} ({} new categories:",
|
||||
assigned.len(),
|
||||
new_categories.len()
|
||||
);
|
||||
if !new_categories.is_empty() {
|
||||
let _ = write!(
|
||||
message,
|
||||
" {}",
|
||||
new_categories.into_iter().collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
}
|
||||
message.push(')');
|
||||
tracing::info!(%message);
|
||||
Ok(message)
|
||||
}
|
||||
|
||||
/// Stored matches for the requested articles.
|
||||
pub async fn matches_for_articles(db: &Db, article_ids: &[ArticleId]) -> Result<Vec<MatchRow>> {
|
||||
let mut matches = Vec::new();
|
||||
for chunk in article_ids.chunks(500) {
|
||||
let placeholders = vec!["?"; chunk.len()].join(", ");
|
||||
let mut query = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"SELECT ai.article_id, ai.interest_id, i.name, ai.cos, ai.z
|
||||
FROM article_interests ai
|
||||
JOIN interests i ON i.id = ai.interest_id
|
||||
WHERE ai.article_id IN ({placeholders})"
|
||||
)));
|
||||
for article_id in chunk {
|
||||
query = query.bind(article_id);
|
||||
}
|
||||
for row in query.fetch_all(db.pool()).await? {
|
||||
matches.push(MatchRow {
|
||||
article_id: row.get("article_id"),
|
||||
interest_id: row.get("interest_id"),
|
||||
name: row.get("name"),
|
||||
cos: row.get("cos"),
|
||||
z: row.get("z"),
|
||||
});
|
||||
}
|
||||
}
|
||||
matches.sort_by(|left, right| {
|
||||
left.article_id
|
||||
.cmp(&right.article_id)
|
||||
.then_with(|| right.z.total_cmp(&left.z))
|
||||
.then_with(|| left.interest_id.cmp(&right.interest_id))
|
||||
});
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
/// Number of stored article matches for each interest.
|
||||
pub async fn match_counts(db: &Db) -> Result<HashMap<i64, i64>> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT interest_id, COUNT(*) AS matches FROM article_interests GROUP BY interest_id",
|
||||
)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|row| (row.get("interest_id"), row.get("matches")))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Derive smoothed interest rates from current ratings and their match rows.
|
||||
pub fn rates(ratings: &[(ArticleId, f64, f64)], rows: &[(ArticleId, i64, f64)]) -> Rates {
|
||||
let mut rows_by_article: HashMap<ArticleId, Vec<(i64, f64)>> = HashMap::new();
|
||||
for &(article_id, interest_id, z) in rows {
|
||||
rows_by_article
|
||||
.entry(article_id)
|
||||
.or_default()
|
||||
.push((interest_id, z));
|
||||
}
|
||||
|
||||
let mut result = Rates::default();
|
||||
for &(article_id, value, decay) in ratings {
|
||||
let mut attributed = false;
|
||||
if let Some(matches) = rows_by_article.get(&article_id) {
|
||||
for &(interest_id, z) in matches {
|
||||
let strength = (z / 3.0).clamp(0.0, 1.0);
|
||||
if strength <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
attributed = true;
|
||||
let credit = value * decay * strength;
|
||||
let rate = result.by_interest.entry(interest_id).or_default();
|
||||
rate.up += credit.max(0.0);
|
||||
rate.down += (-credit).max(0.0);
|
||||
rate.n += 1;
|
||||
}
|
||||
}
|
||||
if attributed {
|
||||
result.attributable += 1;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ts(value: &str) -> Timestamp {
|
||||
value.parse().unwrap()
|
||||
}
|
||||
|
||||
async fn test_db() -> (tempfile::TempDir, Db) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opml_parser_unescapes_trims_and_deduplicates_names() {
|
||||
let interests = parse_opml(
|
||||
r#"<opml><body>
|
||||
<outline text=" Rust "/>
|
||||
<outline text="E-Ink & RSS"/>
|
||||
<outline text="rust"/>
|
||||
<outline text="Quotes "and" apostrophes 'x' 'y'"/>
|
||||
<outline text="Markup <tag>"/>
|
||||
<outline text=""/>
|
||||
</body></opml>"#,
|
||||
);
|
||||
assert_eq!(
|
||||
interests,
|
||||
[
|
||||
"Rust",
|
||||
"E-Ink & RSS",
|
||||
"Quotes \"and\" apostrophes 'x' 'y'",
|
||||
"Markup <tag>",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_article(db: &Db, id: ArticleId) {
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(format!("https://example.com/{id}"))
|
||||
.bind(format!("Article {id}"))
|
||||
.bind("2026-09-12T00:00:00Z")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_trims_names_and_uniqueness_is_case_insensitive() {
|
||||
let (_dir, db) = test_db().await;
|
||||
let now = ts("2026-09-12T12:00:00Z");
|
||||
let AddOutcome::Added(id) = add(&db, " Rust ", Some("Software"), now).await.unwrap()
|
||||
else {
|
||||
panic!("first insert should succeed");
|
||||
};
|
||||
assert_eq!(
|
||||
add(&db, "rust", None, now).await.unwrap(),
|
||||
AddOutcome::Duplicate
|
||||
);
|
||||
assert!(add(&db, " ", None, now).await.is_err());
|
||||
assert!(add(&db, &"x".repeat(81), None, now).await.is_err());
|
||||
|
||||
let interests = list(&db).await.unwrap();
|
||||
assert_eq!(interests.len(), 1);
|
||||
assert_eq!(interests[0].id, id);
|
||||
assert_eq!(interests[0].name, "Rust");
|
||||
assert_eq!(interests[0].category.as_deref(), Some("Software"));
|
||||
assert_eq!(
|
||||
interests[0].categorized_at.as_deref(),
|
||||
Some("2026-09-12T12:00:00Z")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rates_apply_value_decay_strength_and_negative_credit() {
|
||||
let ratings = [
|
||||
(1, 1.0, 1.0),
|
||||
(2, 0.35, 1.0),
|
||||
(3, 1.0, 0.5),
|
||||
(4, -1.0, 0.5),
|
||||
(5, -1.0, 1.0),
|
||||
];
|
||||
let rows = [
|
||||
(1, 10, 3.0),
|
||||
(2, 10, 1.5),
|
||||
(3, 11, 3.0),
|
||||
(4, 10, 0.9),
|
||||
(5, 12, 0.0),
|
||||
];
|
||||
let rates = rates(&ratings, &rows);
|
||||
|
||||
assert_eq!(rates.attributable, 4);
|
||||
let ten = rates.by_interest[&10];
|
||||
assert!((ten.up - 1.175).abs() < 1e-12);
|
||||
assert!((ten.down - 0.15).abs() < 1e-12);
|
||||
assert_eq!(ten.n, 3);
|
||||
assert!((ten.weight() - 2.175 / 3.325).abs() < 1e-12);
|
||||
assert_eq!(
|
||||
rates.by_interest[&11],
|
||||
Rate {
|
||||
up: 0.5,
|
||||
down: 0.0,
|
||||
n: 1
|
||||
}
|
||||
);
|
||||
assert!(!rates.by_interest.contains_key(&12));
|
||||
assert_eq!(Rate::default().weight(), 0.5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_cascades_matches_and_removes_the_embedding() {
|
||||
let (_dir, db) = test_db().await;
|
||||
seed_article(&db, 1).await;
|
||||
let now = ts("2026-09-12T12:00:00Z");
|
||||
let AddOutcome::Added(id) = add(&db, "Rust", None, now).await.unwrap() else {
|
||||
unreachable!();
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO interest_embeddings
|
||||
(interest, model, dimension, embedding, created_at) VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind("Rust")
|
||||
.bind("test")
|
||||
.bind(1_i64)
|
||||
.bind(vec![0_u8; 4])
|
||||
.bind("2026-09-12T12:00:00Z")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let ids = HashMap::from([("Rust".to_string(), id)]);
|
||||
replace_matches(
|
||||
&db,
|
||||
Some(7),
|
||||
&[(
|
||||
1,
|
||||
vec![
|
||||
TopInterest {
|
||||
name: "Rust".into(),
|
||||
cos: 0.7,
|
||||
z: 1.2,
|
||||
},
|
||||
TopInterest {
|
||||
name: "Unknown".into(),
|
||||
cos: 0.9,
|
||||
z: 2.0,
|
||||
},
|
||||
],
|
||||
)],
|
||||
&ids,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
replace_matches(
|
||||
&db,
|
||||
None,
|
||||
&[(
|
||||
1,
|
||||
vec![TopInterest {
|
||||
name: "Rust".into(),
|
||||
cos: 0.8,
|
||||
z: 1.5,
|
||||
}],
|
||||
)],
|
||||
&ids,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(match_counts(&db).await.unwrap(), HashMap::from([(id, 1)]));
|
||||
assert_eq!(
|
||||
matches_for_articles(&db, &[1]).await.unwrap(),
|
||||
[MatchRow {
|
||||
article_id: 1,
|
||||
interest_id: id,
|
||||
name: "Rust".into(),
|
||||
cos: 0.8,
|
||||
z: 1.5
|
||||
}]
|
||||
);
|
||||
let run_id: Option<i64> = sqlx::query_scalar(
|
||||
"SELECT run_id FROM article_interests WHERE article_id = 1 AND interest_id = ?",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(run_id, None);
|
||||
|
||||
delete(&db, id).await.unwrap();
|
||||
let match_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_interests")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let embedding_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interest_embeddings")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(match_rows, 0);
|
||||
assert_eq!(embedding_rows, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grouped_sorts_categories_and_puts_uncategorized_last() {
|
||||
let (_dir, db) = test_db().await;
|
||||
let now = ts("2026-09-12T12:00:00Z");
|
||||
add(&db, "zebra", Some("Animals"), now).await.unwrap();
|
||||
add(&db, "Alpaca", Some("Animals"), now).await.unwrap();
|
||||
let AddOutcome::Added(id) = add(&db, "rust", None, now).await.unwrap() else {
|
||||
unreachable!();
|
||||
};
|
||||
add(&db, "Baking", Some("cooking"), now).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
grouped(&db).await.unwrap(),
|
||||
[
|
||||
("Animals".into(), vec!["Alpaca".into(), "zebra".into()]),
|
||||
("cooking".into(), vec!["Baking".into()]),
|
||||
("Other standing interests".into(), vec!["rust".into()]),
|
||||
]
|
||||
);
|
||||
assert_eq!(uncategorized(&db).await.unwrap()[0].id, id);
|
||||
|
||||
set_category(&db, id, Some("Software"), now).await.unwrap();
|
||||
assert!(uncategorized(&db).await.unwrap().is_empty());
|
||||
set_category(&db, id, None, now).await.unwrap();
|
||||
let rust = uncategorized(&db).await.unwrap().pop().unwrap();
|
||||
assert_eq!(rust.categorized_at, None);
|
||||
assert_eq!(
|
||||
names(&db).await.unwrap(),
|
||||
["Alpaca", "Baking", "rust", "zebra"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_article_lookup_is_a_no_op() {
|
||||
let (_dir, db) = test_db().await;
|
||||
assert!(matches_for_articles(&db, &[]).await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn categorizer_short_circuits_without_uncategorized_interests() {
|
||||
let (_dir, db) = test_db().await;
|
||||
add(
|
||||
&db,
|
||||
"Databases",
|
||||
Some("Software"),
|
||||
ts("2026-09-12T12:00:00Z"),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
categorize(&Config::default(), &db).await.unwrap(),
|
||||
"nothing to categorize"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn categorizer_assigns_known_names_and_tracks_new_categories() {
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::config::ProviderConfig;
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::types::TokenUsage;
|
||||
|
||||
let (_dir, db) = test_db().await;
|
||||
let now = ts("2026-09-12T12:00:00Z");
|
||||
add(&db, "Databases", Some("Software"), now).await.unwrap();
|
||||
add(&db, "Rust macros", None, now).await.unwrap();
|
||||
add(&db, "Wheel-thrown pottery", None, now).await.unwrap();
|
||||
let pending = uncategorized(&db).await.unwrap();
|
||||
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"assignments":[
|
||||
{"interest":"RUST MACROS","category":" software "},
|
||||
{"interest":"Wheel-thrown pottery","category":"Creative crafts"},
|
||||
{"interest":"Not in the batch","category":"Made up"}
|
||||
]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = LlmClient::with_backend(
|
||||
"mock",
|
||||
"taste prompt".into(),
|
||||
UsageMeter::for_provider(&ProviderConfig::deepseek()),
|
||||
backend.clone(),
|
||||
);
|
||||
|
||||
let message = categorize_with_llm(&db, &pending, &llm).await.unwrap();
|
||||
assert_eq!(message, "categorized 2 (1 new categories: Creative crafts)");
|
||||
let stored = list(&db).await.unwrap();
|
||||
assert_eq!(
|
||||
stored
|
||||
.iter()
|
||||
.find(|interest| interest.name == "Rust macros")
|
||||
.and_then(|interest| interest.category.as_deref()),
|
||||
Some("Software")
|
||||
);
|
||||
assert_eq!(
|
||||
stored
|
||||
.iter()
|
||||
.find(|interest| interest.name == "Wheel-thrown pottery")
|
||||
.and_then(|interest| interest.category.as_deref()),
|
||||
Some("Creative crafts")
|
||||
);
|
||||
assert_eq!(backend.calls(), 1);
|
||||
let request = &backend.prompts()[0];
|
||||
assert_eq!(request.temperature, 0.2);
|
||||
assert!(request.json);
|
||||
assert!(
|
||||
request
|
||||
.user
|
||||
.contains("Existing categories (reuse these names verbatim): Software")
|
||||
);
|
||||
assert!(
|
||||
request
|
||||
.user
|
||||
.contains("New interests: \nRust macros\nWheel-thrown pottery")
|
||||
);
|
||||
}
|
||||
}
|
||||
+59
-8
@@ -32,6 +32,8 @@ pub const EXITED_BEFORE_START: &str = "unit exited before the job started; see t
|
||||
pub enum Job {
|
||||
/// `generate` (today) or `generate-YYYY-MM-DD`.
|
||||
Generate { date: Option<Date> },
|
||||
/// `rebuild-YYYY-MM-DD` → `rebuild --date`.
|
||||
Rebuild { date: Date },
|
||||
/// `dry-run` → `generate --dry-run`.
|
||||
DryRun,
|
||||
/// `profile-rebuild` → `profile rebuild`.
|
||||
@@ -44,11 +46,13 @@ pub enum Job {
|
||||
FeaturesPrune,
|
||||
/// `import-ratings` → process ratings-dashboard URL imports.
|
||||
ImportRatings,
|
||||
/// `interests-categorize` → file uncategorized standing interests.
|
||||
InterestsCategorize,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
/// The catalogue in the order the Jobs page lists it.
|
||||
pub const CATALOGUE: [Job; 7] = [
|
||||
pub const CATALOGUE: [Job; 8] = [
|
||||
Job::Generate { date: None },
|
||||
Job::DryRun,
|
||||
Job::ProfileRebuild,
|
||||
@@ -56,6 +60,7 @@ impl Job {
|
||||
Job::BackfillSocial,
|
||||
Job::FeaturesPrune,
|
||||
Job::ImportRatings,
|
||||
Job::InterestsCategorize,
|
||||
];
|
||||
|
||||
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
|
||||
@@ -67,8 +72,9 @@ impl Job {
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
}
|
||||
|
||||
/// Parse a catalogue name or the dated `generate-YYYY-MM-DD` form; anything
|
||||
/// else (including `../x`, uppercase, unknown names) is `None`.
|
||||
/// Parse a catalogue name or a dated `generate-YYYY-MM-DD` or
|
||||
/// `rebuild-YYYY-MM-DD` form; anything else (including `../x`, uppercase,
|
||||
/// unknown names) is `None`.
|
||||
pub fn parse(name: &str) -> Option<Job> {
|
||||
if !Self::valid_name(name) {
|
||||
return None;
|
||||
@@ -81,11 +87,19 @@ impl Job {
|
||||
"backfill-social" => Some(Job::BackfillSocial),
|
||||
"features-prune" => Some(Job::FeaturesPrune),
|
||||
"import-ratings" => Some(Job::ImportRatings),
|
||||
"interests-categorize" => Some(Job::InterestsCategorize),
|
||||
_ => {
|
||||
let date = name.strip_prefix("generate-")?;
|
||||
let (rebuild, date) = match name.strip_prefix("generate-") {
|
||||
Some(date) => (false, date),
|
||||
None => (true, name.strip_prefix("rebuild-")?),
|
||||
};
|
||||
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
||||
let parsed: Date = date.parse().ok()?;
|
||||
(parsed.to_string() == date).then_some(Job::Generate { date: Some(parsed) })
|
||||
(parsed.to_string() == date).then_some(if rebuild {
|
||||
Job::Rebuild { date: parsed }
|
||||
} else {
|
||||
Job::Generate { date: Some(parsed) }
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -94,12 +108,14 @@ impl Job {
|
||||
match self {
|
||||
Job::Generate { date: None } => "generate".into(),
|
||||
Job::Generate { date: Some(date) } => format!("generate-{date}"),
|
||||
Job::Rebuild { date } => format!("rebuild-{date}"),
|
||||
Job::DryRun => "dry-run".into(),
|
||||
Job::ProfileRebuild => "profile-rebuild".into(),
|
||||
Job::FeaturesBackfill => "features-backfill".into(),
|
||||
Job::BackfillSocial => "backfill-social".into(),
|
||||
Job::FeaturesPrune => "features-prune".into(),
|
||||
Job::ImportRatings => "import-ratings".into(),
|
||||
Job::InterestsCategorize => "interests-categorize".into(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +132,9 @@ impl Job {
|
||||
Job::Generate { date: Some(_) } => {
|
||||
"Build and publish the issue of a given date, republishing it if it exists."
|
||||
}
|
||||
Job::Rebuild { .. } => {
|
||||
"Re-render and republish a published issue's EPUBs from its stored content, without curating again."
|
||||
}
|
||||
Job::DryRun => "Run the whole pipeline but publish nothing and record no issue.",
|
||||
Job::ProfileRebuild => {
|
||||
"Regenerate the learned taste adjustments from ratings with the editor model."
|
||||
@@ -128,6 +147,9 @@ impl Job {
|
||||
"Drop stale embeddings, old candidate telemetry and old assessments per the retention config."
|
||||
}
|
||||
Job::ImportRatings => "Fetch, embed and rate the URLs queued from the Ratings page.",
|
||||
Job::InterestsCategorize => {
|
||||
"File uncategorized interests under categories with the bulk model, creating new ones only when needed."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,16 +158,17 @@ impl Job {
|
||||
pub fn takes_lock(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
Job::Generate { .. } | Job::DryRun => Some("generate"),
|
||||
Job::Rebuild { .. } => Some("rebuild"),
|
||||
Job::ProfileRebuild => Some("profile rebuild"),
|
||||
Job::FeaturesBackfill => Some("features backfill"),
|
||||
Job::BackfillSocial => Some("backfill-social"),
|
||||
Job::FeaturesPrune | Job::ImportRatings => None,
|
||||
Job::FeaturesPrune | Job::ImportRatings | Job::InterestsCategorize => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Needs a confirmation dialog: `generate` republishes an issue.
|
||||
/// Needs a confirmation dialog: `generate` and `rebuild` republish an issue.
|
||||
pub fn dangerous(&self) -> bool {
|
||||
matches!(self, Job::Generate { .. })
|
||||
matches!(self, Job::Generate { .. } | Job::Rebuild { .. })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -459,10 +482,38 @@ mod tests {
|
||||
Job::ImportRatings.description(),
|
||||
"Fetch, embed and rate the URLs queued from the Ratings page."
|
||||
);
|
||||
assert_eq!(
|
||||
Job::parse("interests-categorize"),
|
||||
Some(Job::InterestsCategorize)
|
||||
);
|
||||
assert_eq!(Job::InterestsCategorize.takes_lock(), None);
|
||||
assert_eq!(
|
||||
Job::InterestsCategorize.description(),
|
||||
"File uncategorized interests under categories with the bulk model, creating new ones only when needed."
|
||||
);
|
||||
assert!(!Job::InterestsCategorize.dangerous());
|
||||
assert!(Job::parse("generate-2026-09-03").unwrap().dangerous());
|
||||
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_the_dated_rebuild_form() {
|
||||
let job = Job::parse("rebuild-2026-10-01").unwrap();
|
||||
assert_eq!(
|
||||
job,
|
||||
Job::Rebuild {
|
||||
date: "2026-10-01".parse().unwrap()
|
||||
}
|
||||
);
|
||||
assert_eq!(job.name(), "rebuild-2026-10-01");
|
||||
assert_eq!(job.unit(), "daily-epub-job@rebuild-2026-10-01.service");
|
||||
assert_eq!(job.takes_lock(), Some("rebuild"));
|
||||
assert!(job.dangerous());
|
||||
for bad in ["rebuild", "rebuild-", "rebuild-2026-10-1", "rebuild-today"] {
|
||||
assert_eq!(Job::parse(bad), None, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_traversal_uppercase_and_unknown_names() {
|
||||
for name in [
|
||||
|
||||
@@ -22,10 +22,12 @@ pub mod dedupe;
|
||||
pub mod discovery;
|
||||
pub mod epub;
|
||||
pub mod extract;
|
||||
pub mod highlight;
|
||||
pub mod html;
|
||||
pub mod http;
|
||||
pub mod images;
|
||||
pub mod imports;
|
||||
pub mod interests;
|
||||
pub mod jobs;
|
||||
pub mod lock;
|
||||
pub mod mail;
|
||||
@@ -33,6 +35,7 @@ pub mod miniflux;
|
||||
pub mod pipeline;
|
||||
pub mod publish;
|
||||
pub mod rate;
|
||||
pub mod related;
|
||||
pub mod report;
|
||||
pub mod server;
|
||||
pub mod social;
|
||||
|
||||
+444
-16
@@ -3,6 +3,7 @@
|
||||
//! Everything of substance lives in the library (`src/lib.rs`); this binary only
|
||||
//! parses flags, loads config, opens the database and dispatches.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Write as _;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -17,7 +18,7 @@ use daily_epub::db::Db;
|
||||
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
||||
use daily_epub::report::{RunReport, VOYAGE_PROVIDER};
|
||||
use daily_epub::types::{ArticleId, Vote};
|
||||
use daily_epub::{curate, discovery, http, imports, jobs, lock, rate, server, social};
|
||||
use daily_epub::{curate, discovery, http, imports, interests, jobs, lock, rate, server, social};
|
||||
|
||||
/// A personalized daily newspaper, delivered as an EPUB.
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -35,6 +36,8 @@ struct Cli {
|
||||
enum Command {
|
||||
/// Build (and publish) one issue.
|
||||
Generate(GenerateArgs),
|
||||
/// Rebuild and republish a published issue's EPUBs from its stored content.
|
||||
Rebuild(RebuildArgs),
|
||||
/// Run the rating endpoints, the OPDS catalog and downloads.
|
||||
Serve,
|
||||
/// Taste-profile maintenance.
|
||||
@@ -64,6 +67,9 @@ enum Command {
|
||||
/// Feed subscription candidates (feed discovery plan §4 step 6).
|
||||
#[command(subcommand)]
|
||||
Feeds(FeedsCommand),
|
||||
/// Import and backfill standing interests.
|
||||
#[command(subcommand)]
|
||||
Interests(InterestsCommand),
|
||||
/// Operator jobs (what `daily-epub-job@<name>.service` runs).
|
||||
#[command(subcommand)]
|
||||
Job(JobCommand),
|
||||
@@ -74,7 +80,8 @@ enum JobCommand {
|
||||
/// Run one catalogue job in-process and record it in the `jobs` table.
|
||||
Run {
|
||||
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
|
||||
/// `features-backfill`, `backfill-social`, `features-prune` or `import-ratings`.
|
||||
/// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings`
|
||||
/// or `interests-categorize`.
|
||||
name: String,
|
||||
},
|
||||
}
|
||||
@@ -86,6 +93,24 @@ enum FeedsCommand {
|
||||
Discover(FeedsDiscoverArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum InterestsCommand {
|
||||
/// Import standing interests from OPML and the profile's Interests section.
|
||||
Import(InterestsImportArgs),
|
||||
/// Match every compatible cached article embedding to standing interests.
|
||||
Backfill,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct InterestsImportArgs {
|
||||
/// OPML input (defaults to data/scour-interests.opml).
|
||||
#[arg(long, value_name = "PATH")]
|
||||
opml: Option<PathBuf>,
|
||||
/// Profile input (defaults to profile_path from the configuration).
|
||||
#[arg(long, value_name = "PATH")]
|
||||
profile: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct FeedsDiscoverArgs {
|
||||
/// How far back to look for articles.
|
||||
@@ -136,6 +161,13 @@ enum ConfigCommand {
|
||||
Check,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RebuildArgs {
|
||||
/// Date of the published issue to rebuild.
|
||||
#[arg(long, value_name = "YYYY-MM-DD")]
|
||||
date: jiff::civil::Date,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct GenerateArgs {
|
||||
/// Issue date in the configured timezone (defaults to today).
|
||||
@@ -307,6 +339,15 @@ enum FeaturesCommand {
|
||||
Backfill(BackfillArgs),
|
||||
/// Drop stale embeddings, old candidate telemetry and old assessments per the retention config.
|
||||
Prune,
|
||||
/// Recompute the related articles under each issue's web articles.
|
||||
Related(RelatedArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RelatedArgs {
|
||||
/// Only this issue (YYYY-MM-DD); every issue when omitted.
|
||||
#[arg(long)]
|
||||
date: Option<jiff::civil::Date>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
@@ -359,6 +400,17 @@ async fn main() -> Result<()> {
|
||||
let outcome = pipeline::generate(&config, &db, &GenerateOptions::from(&args)).await?;
|
||||
print_outcome(&outcome);
|
||||
}
|
||||
Command::Rebuild(args) => {
|
||||
// No migration: this only re-renders what an existing schema holds.
|
||||
let db = Db::open(&config.database_path).await?;
|
||||
let published = pipeline::rebuild(&config, &db, args.date).await?;
|
||||
for artifact in &published.epubs {
|
||||
println!("{:?}: {}", artifact.edition, artifact.path.display());
|
||||
}
|
||||
if let Some(xtc) = &published.xtc {
|
||||
println!("XTC: {}", xtc.display());
|
||||
}
|
||||
}
|
||||
Command::Serve => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
let config_path = Config::resolve_path(cli.config.as_deref());
|
||||
@@ -410,6 +462,10 @@ async fn main() -> Result<()> {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
println!("{}", cmd_feeds_discover(&config, &db, args).await?);
|
||||
}
|
||||
Command::Interests(command) => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
println!("{}", cmd_interests(&config, &db, command).await?);
|
||||
}
|
||||
Command::Job(JobCommand::Run { name }) => {
|
||||
let Some(job) = jobs::Job::parse(&name) else {
|
||||
eprintln!("unknown job {name:?}; the catalogue is:");
|
||||
@@ -424,6 +480,14 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
.description()
|
||||
);
|
||||
eprintln!(
|
||||
" {:<20} {}",
|
||||
"rebuild-YYYY-MM-DD",
|
||||
jobs::Job::Rebuild {
|
||||
date: jiff::civil::Date::default()
|
||||
}
|
||||
.description()
|
||||
);
|
||||
std::process::exit(2);
|
||||
};
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
@@ -434,17 +498,19 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
/// The commands that write the database and provider budgets and so hold the
|
||||
/// run lock (§5): `generate`, `profile rebuild`, `features backfill`,
|
||||
/// `backfill-social`, and a `job run` of any of them. Everything else is
|
||||
/// read-only or its own writer.
|
||||
/// run lock (§5): `generate`, `rebuild`, `profile rebuild`, `features backfill`,
|
||||
/// `backfill-social`, `interests backfill`, and a `job run` of any of them.
|
||||
/// Everything else is read-only or its own writer.
|
||||
fn lock_holder(command: &Command) -> Option<&'static str> {
|
||||
match command {
|
||||
Command::Generate(_) => Some("generate"),
|
||||
Command::Rebuild(_) => Some("rebuild"),
|
||||
// Writes the same tables the discovery stage of a run writes.
|
||||
Command::Feeds(FeedsCommand::Discover(_)) => Some("feeds discover"),
|
||||
Command::Profile(ProfileCommand::Rebuild) => Some("profile rebuild"),
|
||||
Command::Features(FeaturesCommand::Backfill(_)) => Some("features backfill"),
|
||||
Command::BackfillSocial(_) => Some("backfill-social"),
|
||||
Command::Interests(InterestsCommand::Backfill) => Some("interests backfill"),
|
||||
// An unknown name takes no lock; the dispatch exits 2 before opening
|
||||
// the database.
|
||||
Command::Job(JobCommand::Run { name }) => {
|
||||
@@ -455,9 +521,11 @@ fn lock_holder(command: &Command) -> Option<&'static str> {
|
||||
| Command::Explain(_)
|
||||
| Command::Stats(_)
|
||||
| Command::Features(FeaturesCommand::Prune)
|
||||
| Command::Features(FeaturesCommand::Related(_))
|
||||
| Command::Db(_)
|
||||
| Command::Config(_)
|
||||
| Command::Users(_) => None,
|
||||
| Command::Users(_)
|
||||
| Command::Interests(InterestsCommand::Import(_)) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -494,6 +562,152 @@ async fn cmd_feeds_discover(config: &Config, db: &Db, args: FeedsDiscoverArgs) -
|
||||
))
|
||||
}
|
||||
|
||||
async fn cmd_interests(config: &Config, db: &Db, command: InterestsCommand) -> Result<String> {
|
||||
match command {
|
||||
InterestsCommand::Import(args) => cmd_interests_import(config, db, args).await,
|
||||
InterestsCommand::Backfill => cmd_interests_backfill(config, db).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn cmd_interests_import(
|
||||
config: &Config,
|
||||
db: &Db,
|
||||
args: InterestsImportArgs,
|
||||
) -> Result<String> {
|
||||
let opml_path = args
|
||||
.opml
|
||||
.unwrap_or_else(|| PathBuf::from("data/scour-interests.opml"));
|
||||
let profile_path = args.profile.unwrap_or_else(|| config.profile_path.clone());
|
||||
let opml = std::fs::read_to_string(&opml_path)
|
||||
.with_context(|| format!("reading interests OPML at {}", opml_path.display()))?;
|
||||
let profile = curate::profile::load_profile(&profile_path)?;
|
||||
let mut seen = HashSet::new();
|
||||
let names = interests::parse_opml(&opml)
|
||||
.into_iter()
|
||||
.chain(profile.interests)
|
||||
.filter(|name| seen.insert(name.to_lowercase()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let now = jiff::Timestamp::now();
|
||||
let mut added = Vec::new();
|
||||
let mut skipped = 0;
|
||||
for name in names {
|
||||
match interests::add(db, &name, None, now).await? {
|
||||
interests::AddOutcome::Added(id) => added.push((id, name)),
|
||||
interests::AddOutcome::Duplicate => skipped += 1,
|
||||
}
|
||||
}
|
||||
|
||||
let new_names = added
|
||||
.iter()
|
||||
.map(|(_, name)| name.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let categories = curate::profile::themes::group_into_themes(&new_names)
|
||||
.into_iter()
|
||||
.flat_map(|(category, members)| {
|
||||
members.into_iter().map(move |name| {
|
||||
let category = (category != "Other standing interests").then(|| category.clone());
|
||||
(name.to_lowercase(), category)
|
||||
})
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut uncategorized = 0;
|
||||
for (id, name) in &added {
|
||||
match categories
|
||||
.get(&name.to_lowercase())
|
||||
.and_then(Option::as_deref)
|
||||
{
|
||||
Some(category) => interests::set_category(db, *id, Some(category), now).await?,
|
||||
None => uncategorized += 1,
|
||||
}
|
||||
}
|
||||
|
||||
Ok(format!(
|
||||
"imported {}, skipped {} existing, {} left for the categorizer",
|
||||
added.len(),
|
||||
skipped,
|
||||
uncategorized
|
||||
))
|
||||
}
|
||||
|
||||
/// Backfill uses z-scores over the whole compatible cache as a stand-in for a
|
||||
/// run's per-day article cohort.
|
||||
async fn cmd_interests_backfill(config: &Config, db: &Db) -> Result<String> {
|
||||
use sqlx::Row as _;
|
||||
|
||||
let mut article_embeddings = HashMap::new();
|
||||
let mut after = i64::MIN;
|
||||
loop {
|
||||
let rows = sqlx::query(
|
||||
"SELECT article_id, embedding FROM article_embeddings
|
||||
WHERE model = ? AND dimension = ? AND article_id > ?
|
||||
ORDER BY article_id LIMIT 500",
|
||||
)
|
||||
.bind(&config.voyage.model)
|
||||
.bind(config.voyage.output_dimension as i64)
|
||||
.bind(after)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
if rows.is_empty() {
|
||||
break;
|
||||
}
|
||||
for row in &rows {
|
||||
let article_id: ArticleId = row.get("article_id");
|
||||
after = article_id;
|
||||
match embedding::decode_blob(
|
||||
&row.get::<Vec<u8>, _>("embedding"),
|
||||
config.voyage.output_dimension,
|
||||
) {
|
||||
Ok(vector) => {
|
||||
article_embeddings.insert(article_id, vector);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(article_id, %error, "ignoring a malformed cached embedding")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let rows = interests::list(db).await?;
|
||||
let names = rows
|
||||
.iter()
|
||||
.map(|interest| interest.name.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let ids = rows
|
||||
.into_iter()
|
||||
.map(|interest| (interest.name, interest.id))
|
||||
.collect::<HashMap<_, _>>();
|
||||
// Like the pipeline: a missing key means cached vectors only, not a failure.
|
||||
let service = match config.voyage.enabled {
|
||||
true => match embedding::EmbeddingService::real(db.clone(), config.voyage.clone()) {
|
||||
Ok(service) => Some(service),
|
||||
Err(embedding::EmbeddingError::MissingApiKey) => None,
|
||||
Err(error) => return Err(error).context("building the Voyage client"),
|
||||
},
|
||||
false => None,
|
||||
};
|
||||
let cached_only = service.is_none();
|
||||
let service = service.unwrap_or_else(|| {
|
||||
embedding::EmbeddingService::cached_only(db.clone(), config.voyage.clone())
|
||||
});
|
||||
let interest_embeddings = service.interests(&names).await?;
|
||||
let mut matches = curate::signals::interest_matches(&article_embeddings, &interest_embeddings)
|
||||
.into_iter()
|
||||
.filter_map(|(article_id, matched)| {
|
||||
(!matched.top_interests.is_empty()).then_some((article_id, matched.top_interests))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
matches.sort_by_key(|(article_id, _)| *article_id);
|
||||
let inserted = interests::insert_matches_if_absent(db, &matches, &ids).await?;
|
||||
if cached_only {
|
||||
Ok(format!(
|
||||
"no Voyage client; used cached interest vectors and wrote {inserted} interest match rows"
|
||||
))
|
||||
} else {
|
||||
Ok(format!("wrote {inserted} interest match rows"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn cmd_users(db: &Db, command: UsersCommand) -> Result<()> {
|
||||
use daily_epub::web::users;
|
||||
match command {
|
||||
@@ -724,7 +938,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result<String> {
|
||||
use curate::llm::{Llms, provider_meters};
|
||||
let profile = curate::profile::load_or_build(
|
||||
db,
|
||||
&config.interests_opml,
|
||||
&config.profile_path,
|
||||
config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
@@ -749,7 +962,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result<String> {
|
||||
let rebuilt = curate::profile::rebuild(
|
||||
db,
|
||||
llm,
|
||||
&config.interests_opml,
|
||||
&config.profile_path,
|
||||
config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
@@ -860,7 +1072,7 @@ async fn cmd_explain(db: &Db, args: ExplainArgs) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `features backfill` / `features prune`; returns the final summary line
|
||||
/// `features backfill` / `features prune` / `features related`; returns the final summary line
|
||||
/// (progress lines are printed as they happen).
|
||||
async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Result<String> {
|
||||
match command {
|
||||
@@ -918,13 +1130,25 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
||||
jiff::Timestamp::now(),
|
||||
)
|
||||
.await?;
|
||||
let embeddings = if ranking.embedding_retention_days <= 0 {
|
||||
"kept every embedding (retention 0)".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"pruned {} embeddings older than {} days",
|
||||
pruned.embeddings, ranking.embedding_retention_days
|
||||
)
|
||||
};
|
||||
Ok(format!(
|
||||
"pruned {} embeddings older than {} days, {} candidate rows and {} assessments older than {} days",
|
||||
pruned.embeddings,
|
||||
ranking.embedding_retention_days,
|
||||
pruned.telemetry,
|
||||
pruned.assessments,
|
||||
ranking.telemetry_retention_days
|
||||
"{embeddings}, {} candidate rows and {} assessments older than {} days",
|
||||
pruned.telemetry, pruned.assessments, ranking.telemetry_retention_days
|
||||
))
|
||||
}
|
||||
FeaturesCommand::Related(args) => {
|
||||
let dates: Vec<_> = args.date.into_iter().collect();
|
||||
let refreshed = daily_epub::related::refresh(db, &config.voyage, &dates).await?;
|
||||
Ok(format!(
|
||||
"stored {} related links across {} issues",
|
||||
refreshed.links, refreshed.issues
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -988,6 +1212,21 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
|
||||
jobs::Job::Generate { date } => {
|
||||
generate_job(config, db, date.map(|date| date.to_string()), false).await
|
||||
}
|
||||
jobs::Job::Rebuild { date } => {
|
||||
let published = pipeline::rebuild(config, db, *date).await?;
|
||||
let xtc = if published.xtc.is_some() {
|
||||
" and the XTC"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
Ok((
|
||||
format!(
|
||||
"rebuilt the {date} issue: {} EPUBs{xtc}",
|
||||
published.epubs.len()
|
||||
),
|
||||
None,
|
||||
))
|
||||
}
|
||||
jobs::Job::DryRun => generate_job(config, db, None, true).await,
|
||||
jobs::Job::ProfileRebuild => Ok((cmd_profile_rebuild(config, db).await?, None)),
|
||||
jobs::Job::FeaturesBackfill => {
|
||||
@@ -1008,6 +1247,7 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
|
||||
None,
|
||||
)),
|
||||
jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)),
|
||||
jobs::Job::InterestsCategorize => Ok((interests::categorize(config, db).await?, None)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1151,6 +1391,10 @@ mod tests {
|
||||
lock_holder(&parse(&["backfill-social"])),
|
||||
Some("backfill-social")
|
||||
);
|
||||
assert_eq!(
|
||||
lock_holder(&parse(&["interests", "backfill"])),
|
||||
Some("interests backfill")
|
||||
);
|
||||
for args in [
|
||||
vec!["serve"],
|
||||
vec!["explain", "--date", "2026-09-02", "--near-misses"],
|
||||
@@ -1159,7 +1403,9 @@ mod tests {
|
||||
vec!["db", "migrate"],
|
||||
vec!["features", "prune"],
|
||||
vec!["config", "check"],
|
||||
vec!["interests", "import"],
|
||||
vec!["job", "run", "features-prune"],
|
||||
vec!["job", "run", "interests-categorize"],
|
||||
vec!["job", "run", "not-a-job"],
|
||||
] {
|
||||
assert_eq!(lock_holder(&parse(&args)), None, "{args:?}");
|
||||
@@ -1211,6 +1457,34 @@ mod tests {
|
||||
assert!(Cli::try_parse_from(["daily-epub", "job"]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_interests_commands() {
|
||||
match Cli::try_parse_from([
|
||||
"daily-epub",
|
||||
"interests",
|
||||
"import",
|
||||
"--opml",
|
||||
"/tmp/interests.opml",
|
||||
"--profile",
|
||||
"/tmp/profile.md",
|
||||
])
|
||||
.unwrap()
|
||||
.command
|
||||
{
|
||||
Command::Interests(InterestsCommand::Import(args)) => {
|
||||
assert_eq!(args.opml, Some(PathBuf::from("/tmp/interests.opml")));
|
||||
assert_eq!(args.profile, Some(PathBuf::from("/tmp/profile.md")));
|
||||
}
|
||||
other => panic!("expected interests import, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
Cli::try_parse_from(["daily-epub", "interests", "backfill"])
|
||||
.unwrap()
|
||||
.command,
|
||||
Command::Interests(InterestsCommand::Backfill)
|
||||
));
|
||||
}
|
||||
|
||||
/// Dashboard plan §17 "Jobs": `job run` flips the dashboard's `requested`
|
||||
/// row to `running` and then `ok` with the command's message, in-process
|
||||
/// and without systemd.
|
||||
@@ -1242,7 +1516,7 @@ mod tests {
|
||||
row.message
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.starts_with("pruned 0 embeddings"),
|
||||
.starts_with("kept every embedding (retention 0)"),
|
||||
"{row:?}"
|
||||
);
|
||||
assert_eq!(row.run_id, None);
|
||||
@@ -1432,6 +1706,15 @@ mod tests {
|
||||
.command,
|
||||
Command::Features(FeaturesCommand::Prune)
|
||||
));
|
||||
match Cli::try_parse_from(["daily-epub", "features", "related", "--date", "2026-09-01"])
|
||||
.unwrap()
|
||||
.command
|
||||
{
|
||||
Command::Features(FeaturesCommand::Related(args)) => {
|
||||
assert_eq!(args.date, Some("2026-09-01".parse().unwrap()));
|
||||
}
|
||||
other => panic!("expected features related, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1497,4 +1780,149 @@ mod tests {
|
||||
assert_eq!(rows[1].get::<f64, _>("value"), 0.0);
|
||||
assert!(db.current_ratings(36500).await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interests_import_is_idempotent_and_preserves_theme_groups() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("interests.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
let opml_path = dir.path().join("interests.opml");
|
||||
let profile_path = dir.path().join("profile.md");
|
||||
std::fs::write(
|
||||
&opml_path,
|
||||
r#"<opml><body><outline text="Rust"/><outline text="Flibbertigibbet"/></body></opml>"#,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
&profile_path,
|
||||
"# Reader\n\n## Interests\n- rust\n- Postgres query plans\n",
|
||||
)
|
||||
.unwrap();
|
||||
let config = Config {
|
||||
profile_path: profile_path.clone(),
|
||||
..Config::default()
|
||||
};
|
||||
|
||||
let args = || InterestsImportArgs {
|
||||
opml: Some(opml_path.clone()),
|
||||
profile: None,
|
||||
};
|
||||
assert_eq!(
|
||||
cmd_interests_import(&config, &db, args()).await.unwrap(),
|
||||
"imported 3, skipped 0 existing, 1 left for the categorizer"
|
||||
);
|
||||
assert_eq!(
|
||||
cmd_interests_import(&config, &db, args()).await.unwrap(),
|
||||
"imported 0, skipped 3 existing, 0 left for the categorizer"
|
||||
);
|
||||
|
||||
let rows = interests::list(&db).await.unwrap();
|
||||
assert_eq!(rows.len(), 3);
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.find(|interest| interest.name == "Rust")
|
||||
.and_then(|interest| interest.category.as_deref()),
|
||||
Some("Systems & languages")
|
||||
);
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.find(|interest| interest.name == "Flibbertigibbet")
|
||||
.and_then(|interest| interest.category.as_deref()),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interests_backfill_inserts_matches_without_replacing_run_rows() {
|
||||
use daily_epub::curate::embedding::encode_blob;
|
||||
use daily_epub::curate::signals::TopInterest;
|
||||
use sqlx::Row as _;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("interests.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||
(1, 'https://example.com/1', 'One', '2026-09-12T00:00:00Z'),
|
||||
(2, 'https://example.com/2', 'Two', '2026-09-12T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let now = jiff::Timestamp::now();
|
||||
let interests::AddOutcome::Added(one_id) =
|
||||
interests::add(&db, "First axis", None, now).await.unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
let interests::AddOutcome::Added(two_id) =
|
||||
interests::add(&db, "Second axis", None, now).await.unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
for (article_id, vector) in [(1, [1.0_f32, 0.0]), (2, [0.0, 1.0])] {
|
||||
sqlx::query(
|
||||
"INSERT INTO article_embeddings
|
||||
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||
VALUES (?, 'test-model', 2, 'hash', ?, '2026-09-12T00:00:00Z')",
|
||||
)
|
||||
.bind(article_id)
|
||||
.bind(encode_blob(&vector).unwrap())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
for (name, vector) in [("First axis", [1.0_f32, 0.0]), ("Second axis", [0.0, 1.0])] {
|
||||
sqlx::query(
|
||||
"INSERT INTO interest_embeddings
|
||||
(interest, model, dimension, embedding, created_at)
|
||||
VALUES (?, 'test-model', 2, ?, '2026-09-12T00:00:00Z')",
|
||||
)
|
||||
.bind(name)
|
||||
.bind(encode_blob(&vector).unwrap())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
interests::replace_matches(
|
||||
&db,
|
||||
Some(77),
|
||||
&[(
|
||||
1,
|
||||
vec![TopInterest {
|
||||
name: "First axis".into(),
|
||||
cos: 0.75,
|
||||
z: 1.5,
|
||||
}],
|
||||
)],
|
||||
&HashMap::from([("First axis".to_string(), one_id)]),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut config = Config::default();
|
||||
config.voyage.enabled = false;
|
||||
config.voyage.model = "test-model".into();
|
||||
config.voyage.output_dimension = 2;
|
||||
|
||||
let message = cmd_interests_backfill(&config, &db).await.unwrap();
|
||||
assert_eq!(
|
||||
message,
|
||||
"no Voyage client; used cached interest vectors and wrote 1 interest match rows"
|
||||
);
|
||||
let rows = sqlx::query(
|
||||
"SELECT article_id, interest_id, run_id, cos, z
|
||||
FROM article_interests ORDER BY article_id",
|
||||
)
|
||||
.fetch_all(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(rows[0].get::<i64, _>("interest_id"), one_id);
|
||||
assert_eq!(rows[0].get::<Option<i64>, _>("run_id"), Some(77));
|
||||
assert_eq!(rows[0].get::<f64, _>("cos"), 0.75);
|
||||
assert_eq!(rows[1].get::<i64, _>("interest_id"), two_id);
|
||||
assert_eq!(rows[1].get::<Option<i64>, _>("run_id"), None);
|
||||
}
|
||||
}
|
||||
+216
-29
@@ -46,7 +46,7 @@ use crate::types::{
|
||||
Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta,
|
||||
Lineup, Models, TokenUsage, reading_minutes,
|
||||
};
|
||||
use crate::{comments, dedupe, discovery, epub, http, miniflux, publish, social, world};
|
||||
use crate::{comments, dedupe, discovery, epub, http, interests, miniflux, publish, social, world};
|
||||
|
||||
/// One `generate` invocation's inputs — the CLI flags, already parsed (§2).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -268,6 +268,7 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
||||
db.finish_run(run_id, &report).await?;
|
||||
if stages.published.is_some() {
|
||||
prune_retention(config, db).await;
|
||||
refresh_related(config, db, date).await;
|
||||
}
|
||||
// The issue row is written before the report is costed, so stamp the finished
|
||||
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
||||
@@ -321,6 +322,15 @@ async fn prune_retention(config: &Config, db: &Db) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Store the issue's related-article links (best effort: a failure only
|
||||
/// leaves the web article pages without them until `features related`).
|
||||
async fn refresh_related(config: &Config, db: &Db, date: Date) {
|
||||
match crate::related::refresh(db, &config.voyage, &[date]).await {
|
||||
Ok(refreshed) => tracing::info!(links = refreshed.links, "related articles stored"),
|
||||
Err(error) => tracing::warn!(%error, "related articles failed; continuing"),
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
||||
#[derive(Debug)]
|
||||
struct StageOutput {
|
||||
@@ -648,27 +658,19 @@ async fn run_stages(
|
||||
.copied()
|
||||
.filter(|id| !selected_set.contains(id))
|
||||
.collect::<Vec<_>>();
|
||||
// The editor's one-line `why` (§13) lands in `candidate_runs.editor_why` so
|
||||
// `explain` can quote it; heuristic picks leave it NULL.
|
||||
let selected_with_why = lineup
|
||||
.picks
|
||||
.iter()
|
||||
.map(|pick| (pick.article.id, pick.why.as_deref()))
|
||||
.collect::<Vec<_>>();
|
||||
set_candidate_stage(
|
||||
&mut personalized,
|
||||
¬_selected,
|
||||
"shortlisted",
|
||||
Some("not_selected"),
|
||||
);
|
||||
let why = selected_with_why.into_iter().collect::<HashMap<_, _>>();
|
||||
for candidate in &mut personalized {
|
||||
if selected_set.contains(&candidate.article.id) {
|
||||
candidate.stage = "selected".into();
|
||||
candidate.excluded_reason = None;
|
||||
}
|
||||
}
|
||||
record_candidates_with_why(ctx, &personalized, &why)
|
||||
record_candidates(ctx, &personalized)
|
||||
.await
|
||||
.context("recording selection telemetry")?;
|
||||
report.counts.exploration_selected = personalized
|
||||
@@ -943,15 +945,22 @@ async fn prepare_features(
|
||||
}
|
||||
};
|
||||
report.counts.embedded = article_embeddings.len() as i64;
|
||||
let interests =
|
||||
match profile::load_standing_interests(&config.interests_opml, &config.profile_path) {
|
||||
let interest_rows = match interests::list(db).await {
|
||||
Ok(interests) => interests,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "could not load standing interests for embeddings");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let interest_embeddings = match service.interests(&interests).await {
|
||||
let interest_names = interest_rows
|
||||
.iter()
|
||||
.map(|interest| interest.name.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let interest_ids = interest_rows
|
||||
.into_iter()
|
||||
.map(|interest| (interest.name, interest.id))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let interest_embeddings = match service.interests(&interest_names).await {
|
||||
Ok(embeddings) => embeddings,
|
||||
Err(error) => {
|
||||
report.warn(format!("interest embedding stage degraded: {error}"));
|
||||
@@ -1009,6 +1018,21 @@ async fn prepare_features(
|
||||
.remove(&candidate.article.id)
|
||||
.unwrap_or_else(|| signals::Signals::baseline(&candidate.article));
|
||||
}
|
||||
let matches = candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.top_interests.is_empty())
|
||||
.map(|candidate| {
|
||||
(
|
||||
candidate.article.id,
|
||||
candidate.signals.top_interests.clone(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if let Err(error) =
|
||||
interests::replace_matches(db, Some(ctx.run_id), &matches, &interest_ids).await
|
||||
{
|
||||
report.warn(format!("could not record interest matches: {error}"));
|
||||
}
|
||||
if let Err(error) = record_candidates(ctx, candidates).await {
|
||||
report.warn(format!("could not record eligible candidates: {error}"));
|
||||
}
|
||||
@@ -1017,14 +1041,6 @@ async fn prepare_features(
|
||||
}
|
||||
|
||||
async fn record_candidates(ctx: &StageContext<'_>, candidates: &[Candidate]) -> Result<()> {
|
||||
record_candidates_with_why(ctx, candidates, &HashMap::new()).await
|
||||
}
|
||||
|
||||
async fn record_candidates_with_why(
|
||||
ctx: &StageContext<'_>,
|
||||
candidates: &[Candidate],
|
||||
editor_why: &HashMap<ArticleId, Option<&str>>,
|
||||
) -> Result<()> {
|
||||
for candidate in candidates {
|
||||
let json = telemetry::serialize_candidate(candidate);
|
||||
let admitted_by = if candidate.admitted_by.is_empty() {
|
||||
@@ -1045,7 +1061,6 @@ async fn record_candidates_with_why(
|
||||
rank_utility: candidate.rank_utility,
|
||||
cluster_id: candidate.cluster,
|
||||
cluster_rank: candidate.cluster_rank,
|
||||
editor_why: editor_why.get(&candidate.article.id).copied().flatten(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1090,6 +1105,51 @@ async fn persist_articles(db: &Db, articles: &mut [Article]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rebuild and republish a published issue's EPUBs (and XTC) from its stored
|
||||
/// snapshot, for when rendering changed after the issue went out.
|
||||
///
|
||||
/// The content is exactly the issue's: the `issues.issue_json` snapshot with
|
||||
/// each article's body restored from its row (the snapshot drops bodies).
|
||||
/// Unlike the web reader it does not refresh anything else from the live rows,
|
||||
/// so social lines and metadata stay as published. Nothing is curated or
|
||||
/// fetched except the images, which are downloaded again as in a run.
|
||||
pub async fn rebuild(config: &Config, db: &Db, date: Date) -> Result<Published> {
|
||||
let row = db
|
||||
.issue_by_date(date)
|
||||
.await?
|
||||
.with_context(|| format!("no issue for {date}"))?;
|
||||
let raw = row
|
||||
.issue_json
|
||||
.with_context(|| format!("the {date} issue predates issue snapshots"))?;
|
||||
let mut issue: Issue = serde_json::from_str(&raw).context("decoding issues.issue_json")?;
|
||||
let ids: Vec<ArticleId> = issue.lineup.picks.iter().map(|p| p.article.id).collect();
|
||||
let mut articles = db.get_articles(&ids).await?;
|
||||
for pick in &mut issue.lineup.picks {
|
||||
let article = articles
|
||||
.remove(&pick.article.id)
|
||||
.with_context(|| format!("article {} is gone", pick.article.id))?;
|
||||
pick.article.content_html = article.content_html;
|
||||
}
|
||||
|
||||
let (artifacts, _) = epub::build_all(&issue, config, &config.out_dir)
|
||||
.await
|
||||
.context("building the EPUB editions")?;
|
||||
let x4 = artifacts.iter().find(|a| a.edition == Edition::X4);
|
||||
let xtc = match x4 {
|
||||
Some(artifact) if config.xtc.enabled => {
|
||||
Some(epub::x4::convert(&config.xtc, &artifact.path, &config.out_dir).await?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let published = publish::publish_issue(config, &issue, &artifacts, xtc.as_deref())
|
||||
.await
|
||||
.context("publishing the issue")?;
|
||||
record_issue(db, &issue, &published)
|
||||
.await
|
||||
.context("recording the issue")?;
|
||||
Ok(published)
|
||||
}
|
||||
|
||||
/// Write the `issues` row and replace `issue_articles` for the date (notes §12).
|
||||
async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<()> {
|
||||
let path_for = |edition: Edition| {
|
||||
@@ -1138,7 +1198,6 @@ async fn build_llms(
|
||||
) -> Llms {
|
||||
let profile = match profile::load_or_build(
|
||||
ctx.db,
|
||||
&ctx.config.interests_opml,
|
||||
&ctx.config.profile_path,
|
||||
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
@@ -1161,6 +1220,39 @@ async fn build_llms(
|
||||
let make_clients = |prompt: String| Llms::from_config(ctx.config, prompt, meters);
|
||||
|
||||
let mut llms = make_clients(profile.text);
|
||||
// Categorizing regroups the prompt's standing interests, so the prompt is
|
||||
// reloaded and the clients remade before anything uses them.
|
||||
if !llms.is_empty() {
|
||||
match interests::uncategorized(ctx.db).await {
|
||||
Ok(pending) if !pending.is_empty() => {
|
||||
match interests::categorize(ctx.config, ctx.db).await {
|
||||
Ok(message) => {
|
||||
tracing::info!(%message);
|
||||
match profile::load_or_build(
|
||||
ctx.db,
|
||||
&ctx.config.profile_path,
|
||||
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(regrouped) => {
|
||||
report.counts.verdicts_in_prompt = regrouped.verdicts as i64;
|
||||
llms = make_clients(regrouped.text);
|
||||
}
|
||||
Err(error) => report.warn(format!(
|
||||
"could not rebuild the taste prompt after categorizing: {error:#}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
Err(error) => report.warn(format!("interest categorization failed: {error:#}")),
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => report.warn(format!(
|
||||
"could not check for uncategorized interests: {error:#}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
let Some(rebuild_client) = llms.editor_or_bulk() else {
|
||||
report.warn("no LLM provider is available; curating heuristically");
|
||||
return llms;
|
||||
@@ -1168,7 +1260,6 @@ async fn build_llms(
|
||||
match profile::weekly_rebuild_if_due(
|
||||
ctx.db,
|
||||
rebuild_client,
|
||||
&ctx.config.interests_opml,
|
||||
&ctx.config.profile_path,
|
||||
ctx.config.curation.feedback.verdicts_in_prompt,
|
||||
)
|
||||
@@ -1310,6 +1401,91 @@ fn elapsed_ms(since: Timestamp) -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Read one entry of a zip archive as text.
|
||||
fn zip_entry(path: &std::path::Path, name: &str) -> String {
|
||||
use std::io::Read;
|
||||
let mut archive = zip::ZipArchive::new(std::fs::File::open(path).unwrap()).unwrap();
|
||||
let mut text = String::new();
|
||||
archive
|
||||
.by_name(name)
|
||||
.unwrap()
|
||||
.read_to_string(&mut text)
|
||||
.unwrap();
|
||||
text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebuild_republishes_from_the_snapshot_without_touching_it() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut issue = crate::epub::fixtures::issue();
|
||||
for pick in &mut issue.lineup.picks {
|
||||
// No images: the rebuild must not need the network here.
|
||||
pick.article.content_html =
|
||||
"<p>Before.</p><pre data-lang=\"rust\"><code>fn main() {}</code></pre>".into();
|
||||
pick.article.image_count = 0;
|
||||
let article = &pick.article;
|
||||
db.upsert_entry(&crate::types::Entry {
|
||||
id: article.best_entry_id,
|
||||
feed_id: article.feed_id,
|
||||
feed_title: Some(article.feed_title.clone()),
|
||||
category: article.category.clone(),
|
||||
title: article.title.clone(),
|
||||
url: article.url.clone(),
|
||||
canonical_url: Some(article.canonical_url.clone()),
|
||||
author: article.author.clone(),
|
||||
published_at: article.published_at,
|
||||
comments_url: article.comments_url.clone(),
|
||||
raw_content: article.content_html.clone(),
|
||||
fetched_at: article.first_seen,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
pick.article.id = db.upsert_article(article).await.unwrap();
|
||||
}
|
||||
let mut snapshot = issue.clone();
|
||||
for pick in &mut snapshot.lineup.picks {
|
||||
pick.article.content_html.clear();
|
||||
}
|
||||
let issue_json = serde_json::to_string(&snapshot).unwrap();
|
||||
db.upsert_issue(
|
||||
issue.meta.date,
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&issue.editorial.front_page_html),
|
||||
None,
|
||||
Some(&issue_json),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut config = Config {
|
||||
out_dir: dir.path().join("out"),
|
||||
..Config::default()
|
||||
};
|
||||
config.publish.epub_dir = dir.path().join("epubs");
|
||||
config.publish.xtc_dir = dir.path().join("xtc");
|
||||
config.xtc.enabled = false;
|
||||
let published = rebuild(&config, &db, issue.meta.date).await.unwrap();
|
||||
|
||||
assert_eq!(published.epubs.len(), 2);
|
||||
let row = db.issue_by_date(issue.meta.date).await.unwrap().unwrap();
|
||||
assert_eq!(row.issue_json.as_deref(), Some(issue_json.as_str()));
|
||||
let standard = &published.epubs[0];
|
||||
assert_eq!(
|
||||
row.epub_path.as_deref(),
|
||||
Some(standard.path.to_str().unwrap())
|
||||
);
|
||||
let chapter = zip_entry(&standard.path, "OEBPS/art-1001.xhtml");
|
||||
assert!(chapter.contains("<p>Before.</p>"), "body restored");
|
||||
assert!(chapter.contains("<span class=\"hl-keyword\">fn</span>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_date_matches_the_masthead_format() {
|
||||
let date: Date = "2026-08-15".parse().unwrap();
|
||||
@@ -1577,11 +1753,8 @@ mod tests {
|
||||
config.curation.blocked_domains = vec!["blocked.example".into()];
|
||||
config.voyage.output_dimension = 4;
|
||||
config.target_article_count = 1;
|
||||
config.interests_opml = dir.path().join("interests.opml");
|
||||
std::fs::write(
|
||||
&config.interests_opml,
|
||||
"<opml><body><outline text=\"Writerdeck\"/></body></opml>",
|
||||
)
|
||||
interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
config.profile_path = dir.path().join("profile.md");
|
||||
std::fs::write(&config.profile_path, "# Reader profile\n").unwrap();
|
||||
@@ -1708,6 +1881,7 @@ mod tests {
|
||||
assert!(report.voyage_tokens > 0);
|
||||
// One batch for the two articles, one for the interest.
|
||||
assert_eq!(backend.calls(), 2);
|
||||
assert_eq!(backend.requests()[1].input, ["Writerdeck"]);
|
||||
let signals = &features
|
||||
.iter()
|
||||
.find(|candidate| candidate.article.id == a)
|
||||
@@ -1724,6 +1898,19 @@ mod tests {
|
||||
);
|
||||
assert!(signals.preliminary.is_some());
|
||||
|
||||
let expected_matches = features
|
||||
.iter()
|
||||
.map(|candidate| candidate.signals.top_interests.len() as i64)
|
||||
.sum::<i64>();
|
||||
let stored_matches: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM article_interests WHERE run_id = ?")
|
||||
.bind(h.run_id)
|
||||
.fetch_one(h.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(expected_matches > 0);
|
||||
assert_eq!(stored_matches, expected_matches);
|
||||
|
||||
let rows = stage_rows(&h.db, h.run_id).await;
|
||||
assert_eq!(rows.len(), 4, "one row per considered article");
|
||||
assert_eq!(rows[&blocked].0, "excluded");
|
||||
|
||||
+446
@@ -0,0 +1,446 @@
|
||||
//! Related articles under each web issue article.
|
||||
//!
|
||||
//! For every pick, the nearest embedded articles across the whole archive
|
||||
//! (`article_embeddings`, kept forever by default). Finding them is a scan of
|
||||
//! every cached vector, so it runs once when an issue is published and once
|
||||
//! per issue from `features related`, and the result lives in `issue_related`;
|
||||
//! reader pages only read that table.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use anyhow::Result;
|
||||
use jiff::civil::Date;
|
||||
use sqlx::Row as _;
|
||||
|
||||
use crate::config::VoyageConfig;
|
||||
use crate::curate::embedding::{decode_blob, dot, load_article_embeddings};
|
||||
use crate::db::Db;
|
||||
use crate::types::ArticleId;
|
||||
|
||||
/// Links shown under each article.
|
||||
const PER_ARTICLE: usize = 5;
|
||||
/// Below this cosine a neighbour is noise, not a related article (the same
|
||||
/// cut the curation prompts use for "closest rated").
|
||||
const MIN_COSINE: f64 = 0.55;
|
||||
/// Embedding rows decoded per statement, so memory stays flat as the archive grows.
|
||||
const SCAN_PAGE: i64 = 1000;
|
||||
/// Candidates kept per pick between pages: enough headroom for the
|
||||
/// same-story filter at the end to still fill [`PER_ARTICLE`].
|
||||
const KEEP: usize = PER_ARTICLE * 4;
|
||||
/// Share of title words two articles from one site must have in common to be
|
||||
/// one story re-published under a new URL ("…after AI accusations" and
|
||||
/// "…after AI allegations").
|
||||
const SAME_SITE_TITLE_OVERLAP: f64 = 0.8;
|
||||
|
||||
/// What [`refresh`] wrote.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
pub struct Refreshed {
|
||||
pub issues: usize,
|
||||
pub links: usize,
|
||||
}
|
||||
|
||||
/// Recompute `issue_related` for the given issues (every issue when `dates` is
|
||||
/// empty) in one pass over the embedding cache.
|
||||
///
|
||||
/// Excluded: the pick itself, the rest of its issue, articles that only have a
|
||||
/// feed excerpt (their page would be near-empty), and articles that are the
|
||||
/// pick's or a closer link's story again (see [`Story::same_as`]).
|
||||
pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<Refreshed> {
|
||||
let dates = if dates.is_empty() {
|
||||
sqlx::query_scalar::<_, String>("SELECT date FROM issues ORDER BY date")
|
||||
.fetch_all(db.pool())
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter_map(|raw| raw.parse().ok())
|
||||
.collect()
|
||||
} else {
|
||||
dates.to_vec()
|
||||
};
|
||||
|
||||
// Every pick of every requested issue, with its story key for de-duplication.
|
||||
let mut groups: Vec<Group> = Vec::with_capacity(dates.len());
|
||||
for date in &dates {
|
||||
let rows = sqlx::query(
|
||||
"SELECT ia.article_id, a.title, a.canonical_url FROM issue_articles ia
|
||||
JOIN articles a ON a.id = ia.article_id
|
||||
WHERE ia.issue_date = ? ORDER BY ia.section, ia.position",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
groups.push(Group {
|
||||
date: *date,
|
||||
members: rows.iter().map(|row| row.get("article_id")).collect(),
|
||||
picks: rows
|
||||
.iter()
|
||||
.map(|row| Target {
|
||||
article_id: row.get("article_id"),
|
||||
story: Story::from_row(row),
|
||||
vector: Vec::new(),
|
||||
best: Vec::new(),
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
let ids: Vec<ArticleId> = groups
|
||||
.iter()
|
||||
.flat_map(|group| group.picks.iter().map(|pick| pick.article_id))
|
||||
.collect();
|
||||
let mut vectors = load_article_embeddings(db, voyage, &ids).await?;
|
||||
for group in &mut groups {
|
||||
// A pick without a cached vector simply gets no related list.
|
||||
group
|
||||
.picks
|
||||
.retain_mut(|pick| match vectors.remove(&pick.article_id) {
|
||||
Some(vector) => {
|
||||
pick.vector = vector;
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
});
|
||||
}
|
||||
|
||||
let mut after = i64::MIN;
|
||||
loop {
|
||||
let rows = sqlx::query(
|
||||
"SELECT ae.article_id, ae.embedding, a.title, a.canonical_url
|
||||
FROM article_embeddings ae JOIN articles a ON a.id = ae.article_id
|
||||
WHERE ae.model = ? AND ae.dimension = ? AND a.excerpt_only = 0
|
||||
AND ae.article_id > ?
|
||||
ORDER BY ae.article_id LIMIT ?",
|
||||
)
|
||||
.bind(&voyage.model)
|
||||
.bind(voyage.output_dimension as i64)
|
||||
.bind(after)
|
||||
.bind(SCAN_PAGE)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
let Some(last) = rows.last() else { break };
|
||||
after = last.get("article_id");
|
||||
for row in &rows {
|
||||
let candidate: ArticleId = row.get("article_id");
|
||||
let vector = match decode_blob(
|
||||
&row.get::<Vec<u8>, _>("embedding"),
|
||||
voyage.output_dimension,
|
||||
) {
|
||||
Ok(vector) => vector,
|
||||
Err(error) => {
|
||||
tracing::warn!(article_id = candidate, %error, "ignoring a malformed embedding");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let story = Story::from_row(row);
|
||||
for group in &mut groups {
|
||||
if group.members.contains(&candidate) {
|
||||
continue;
|
||||
}
|
||||
for pick in &mut group.picks {
|
||||
let cosine = dot(&pick.vector, &vector)?;
|
||||
if cosine.is_finite() && cosine >= MIN_COSINE {
|
||||
pick.best.push((candidate, cosine, story.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for group in &mut groups {
|
||||
for pick in &mut group.picks {
|
||||
// Highest cosine first; ties by id so reruns write the same rows.
|
||||
pick.best.sort_by(|left, right| {
|
||||
right
|
||||
.1
|
||||
.total_cmp(&left.1)
|
||||
.then_with(|| left.0.cmp(&right.0))
|
||||
});
|
||||
pick.best.truncate(KEEP);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut refreshed = Refreshed::default();
|
||||
let mut tx = db.pool().begin().await?;
|
||||
for group in &groups {
|
||||
let date = group.date.to_string();
|
||||
sqlx::query("DELETE FROM issue_related WHERE issue_date = ?")
|
||||
.bind(&date)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for pick in &group.picks {
|
||||
let mut seen = vec![&pick.story];
|
||||
let mut chosen = Vec::new();
|
||||
for (id, cosine, story) in &pick.best {
|
||||
if chosen.len() == PER_ARTICLE {
|
||||
break;
|
||||
}
|
||||
if !seen.iter().any(|earlier| earlier.same_as(story)) {
|
||||
seen.push(story);
|
||||
chosen.push((id, cosine));
|
||||
}
|
||||
}
|
||||
for (position, (related_id, cosine)) in chosen.into_iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind(&date)
|
||||
.bind(pick.article_id)
|
||||
.bind(position as i64)
|
||||
.bind(related_id)
|
||||
.bind(cosine)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
refreshed.links += 1;
|
||||
}
|
||||
}
|
||||
refreshed.issues += 1;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(refreshed)
|
||||
}
|
||||
|
||||
/// The stored related links for one pick, closest first.
|
||||
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<ArticleId>> {
|
||||
Ok(sqlx::query_scalar(
|
||||
"SELECT related_id FROM issue_related
|
||||
WHERE issue_date = ? AND article_id = ? ORDER BY position",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.bind(article_id)
|
||||
.fetch_all(db.pool())
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// The most recent issue each of `ids` was published in, for linking a related
|
||||
/// article to its issue page rather than the standalone one.
|
||||
pub async fn latest_issues(db: &Db, ids: &[ArticleId]) -> Result<HashMap<ArticleId, Date>> {
|
||||
let mut issues = HashMap::new();
|
||||
for id in ids {
|
||||
let date: Option<String> =
|
||||
sqlx::query_scalar("SELECT MAX(issue_date) FROM issue_articles WHERE article_id = ?")
|
||||
.bind(id)
|
||||
.fetch_one(db.pool())
|
||||
.await?;
|
||||
if let Some(date) = date.and_then(|raw| raw.parse().ok()) {
|
||||
issues.insert(*id, date);
|
||||
}
|
||||
}
|
||||
Ok(issues)
|
||||
}
|
||||
|
||||
struct Group {
|
||||
date: Date,
|
||||
/// Every pick of the issue, embedded or not: none of them is "related".
|
||||
members: HashSet<ArticleId>,
|
||||
picks: Vec<Target>,
|
||||
}
|
||||
|
||||
struct Target {
|
||||
article_id: ArticleId,
|
||||
story: Story,
|
||||
vector: Vec<f32>,
|
||||
/// (candidate, cosine, story), closest first after each scan page.
|
||||
best: Vec<(ArticleId, f64, Story)>,
|
||||
}
|
||||
|
||||
/// What decides whether two articles are one story told twice. The ingest
|
||||
/// dedupe only merges within one run, so a story re-published on another day
|
||||
/// under a new URL reaches the archive as a second article.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Story {
|
||||
/// Canonical URL host without `www.`.
|
||||
site: String,
|
||||
/// Lowercased title words.
|
||||
words: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Story {
|
||||
fn from_row(row: &sqlx::sqlite::SqliteRow) -> Self {
|
||||
// Re-canonicalized so legacy rows still wrapped in a Scour redirect
|
||||
// count as the outlet's site, not Scour's.
|
||||
let raw: String = row.get("canonical_url");
|
||||
let site = crate::dedupe::canonical_url(&raw)
|
||||
.and_then(|canonical| url::Url::parse(&canonical).ok())
|
||||
.and_then(|url| {
|
||||
url.host_str()
|
||||
.map(|host| host.trim_start_matches("www.").to_string())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let words = row
|
||||
.get::<Option<String>, _>("title")
|
||||
.unwrap_or_default()
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|word| !word.is_empty())
|
||||
.map(str::to_lowercase)
|
||||
.collect();
|
||||
Self { site, words }
|
||||
}
|
||||
|
||||
/// The same title anywhere (syndication), or mostly the same title on the
|
||||
/// same site (a re-slugged republish).
|
||||
fn same_as(&self, other: &Story) -> bool {
|
||||
if self.words.is_empty() || other.words.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if self.words == other.words {
|
||||
return true;
|
||||
}
|
||||
let shared = self.words.intersection(&other.words).count() as f64;
|
||||
let union = self.words.union(&other.words).count() as f64;
|
||||
!self.site.is_empty()
|
||||
&& self.site == other.site
|
||||
&& shared / union >= SAME_SITE_TITLE_OVERLAP
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::curate::embedding::encode_blob;
|
||||
|
||||
fn voyage() -> VoyageConfig {
|
||||
VoyageConfig {
|
||||
model: "voyage-4-lite".into(),
|
||||
output_dimension: 2,
|
||||
..VoyageConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn harness() -> (tempfile::TempDir, Db) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("related.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
async fn article(db: &Db, id: ArticleId, title: &str, vector: &[f32], excerpt_only: bool) {
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, excerpt_only, first_seen)
|
||||
VALUES (?, ?, ?, ?, '2026-08-15T00:00:00Z')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(format!("https://example.com/{id}"))
|
||||
.bind(title)
|
||||
.bind(excerpt_only)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||
let unit: Vec<f32> = vector.iter().map(|v| v / norm).collect();
|
||||
sqlx::query(
|
||||
"INSERT INTO article_embeddings
|
||||
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||
VALUES (?, 'voyage-4-lite', 2, 'h', ?, '2026-08-15T00:00:00Z')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(encode_blob(&unit).unwrap())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn issue(db: &Db, date: &str, picks: &[ArticleId]) {
|
||||
sqlx::query("INSERT INTO issues (date, issue_number, generated_at) VALUES (?, 1, ?)")
|
||||
.bind(date)
|
||||
.bind("2026-09-01T00:00:00Z")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
for (position, id) in picks.iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position)
|
||||
VALUES (?, ?, 'Top Stories', ?)",
|
||||
)
|
||||
.bind(date)
|
||||
.bind(id)
|
||||
.bind(position as i64)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn day(raw: &str) -> Date {
|
||||
raw.parse().unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn links_skip_the_issue_excerpts_duplicates_and_far_articles() {
|
||||
let (_dir, db) = harness().await;
|
||||
article(&db, 1, "Rust borrow checker", &[1.0, 0.0], false).await;
|
||||
article(&db, 2, "Same issue neighbour", &[1.0, 0.01], false).await;
|
||||
article(&db, 3, "Lifetimes explained", &[1.0, 0.1], false).await;
|
||||
article(&db, 4, "Paywalled", &[1.0, 0.05], true).await;
|
||||
article(&db, 5, "Rust Borrow-Checker!", &[1.0, 0.02], false).await;
|
||||
article(&db, 6, "Ownership deep dive", &[1.0, 0.3], false).await;
|
||||
article(&db, 7, "Ownership: deep dive", &[1.0, 0.31], false).await;
|
||||
article(&db, 8, "Gardening", &[0.0, 1.0], false).await;
|
||||
issue(&db, "2026-09-01", &[1, 2]).await;
|
||||
|
||||
let refreshed = refresh(&db, &voyage(), &[day("2026-09-01")]).await.unwrap();
|
||||
let one = for_article(&db, day("2026-09-01"), 1).await.unwrap();
|
||||
assert_eq!(one, vec![3, 6], "closest first, one copy of each story");
|
||||
// 5 only repeats article 1's title, so it is fair game for article 2.
|
||||
let two = for_article(&db, day("2026-09-01"), 2).await.unwrap();
|
||||
assert_eq!(two, vec![5, 3, 6]);
|
||||
assert_eq!(
|
||||
refreshed,
|
||||
Refreshed {
|
||||
issues: 1,
|
||||
links: 5
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshing_every_issue_replaces_rows_and_links_earlier_picks() {
|
||||
let (_dir, db) = harness().await;
|
||||
article(&db, 1, "One", &[1.0, 0.0], false).await;
|
||||
article(&db, 2, "Two", &[1.0, 0.1], false).await;
|
||||
article(&db, 3, "Three", &[1.0, 0.3], false).await;
|
||||
issue(&db, "2026-09-01", &[1]).await;
|
||||
issue(&db, "2026-09-02", &[2]).await;
|
||||
|
||||
assert_eq!(refresh(&db, &voyage(), &[]).await.unwrap().issues, 2);
|
||||
// A rerun rewrites rather than duplicates.
|
||||
refresh(&db, &voyage(), &[]).await.unwrap();
|
||||
let ids = for_article(&db, day("2026-09-02"), 2).await.unwrap();
|
||||
assert_eq!(ids, vec![1, 3]);
|
||||
let issues = latest_issues(&db, &[1, 3]).await.unwrap();
|
||||
assert_eq!(issues.get(&1), Some(&day("2026-09-01")));
|
||||
assert_eq!(issues.get(&3), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stories_match_on_identical_titles_or_close_titles_from_one_site() {
|
||||
let story = |site: &str, title: &str| Story {
|
||||
site: site.into(),
|
||||
words: title
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|word| !word.is_empty())
|
||||
.map(str::to_lowercase)
|
||||
.collect(),
|
||||
};
|
||||
let accusations = story(
|
||||
"the-independent.com",
|
||||
"The bestselling novel pulled from prestigious literary prize after AI accusations",
|
||||
);
|
||||
let allegations = story(
|
||||
"the-independent.com",
|
||||
"The bestselling novel pulled from prestigious literary prize after AI allegations",
|
||||
);
|
||||
let conversation = story(
|
||||
"theconversation.com",
|
||||
"A bestselling novel was pulled from a French literary prize after AI accusations – what next?",
|
||||
);
|
||||
assert!(accusations.same_as(&allegations), "a re-slugged republish");
|
||||
assert!(
|
||||
!accusations.same_as(&conversation),
|
||||
"other outlets' coverage stays"
|
||||
);
|
||||
assert!(
|
||||
story("a.com", "Foo: Bar").same_as(&story("b.com", "foo — bar")),
|
||||
"syndicated copies match across sites"
|
||||
);
|
||||
assert!(!story("a.com", "").same_as(&story("a.com", "")));
|
||||
}
|
||||
}
|
||||
@@ -372,8 +372,6 @@ pub struct Pick {
|
||||
/// Order within the section, ascending.
|
||||
pub position: i64,
|
||||
pub is_lead: bool,
|
||||
/// Editor-written reason, at most 14 words (§13).
|
||||
pub why: Option<String>,
|
||||
/// Newspaper-abstract summary from stage C; `None` until editorial runs.
|
||||
pub summary: Option<String>,
|
||||
pub llm: Option<Deep>,
|
||||
|
||||
+180
-20
@@ -53,6 +53,7 @@ pub fn routes() -> Router<AppState> {
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub struct ArticlesQuery {
|
||||
pub q: Option<String>,
|
||||
pub interest: Option<String>,
|
||||
pub feed: Option<String>,
|
||||
pub stage: Option<String>,
|
||||
pub reason: Option<String>,
|
||||
@@ -68,8 +69,9 @@ pub struct ArticlesQuery {
|
||||
const RATED: [&str; 7] = ["any", "loved", "good", "down", "slop", "cleared", "none"];
|
||||
const PUBLISHED: [&str; 2] = ["yes", "no"];
|
||||
|
||||
const ARTICLE_SORTS: [(&str, &str); 7] = [
|
||||
const ARTICLE_SORTS: [(&str, &str); 8] = [
|
||||
("first_seen", "x.first_seen DESC, x.id DESC"),
|
||||
("match", "x.match_cos DESC, x.id DESC"),
|
||||
("utility", "x.utility DESC, x.id DESC"),
|
||||
("quality", "x.quality DESC, x.id DESC"),
|
||||
("fit", "x.fit DESC, x.id DESC"),
|
||||
@@ -82,6 +84,7 @@ const ARTICLE_SORTS: [(&str, &str); 7] = [
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ArticleFilters {
|
||||
pub q: Option<String>,
|
||||
pub interest: Option<InterestFilter>,
|
||||
pub feed: Option<i64>,
|
||||
pub stage: Option<String>,
|
||||
pub reason: Option<String>,
|
||||
@@ -93,8 +96,14 @@ pub struct ArticleFilters {
|
||||
pub sort: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct InterestFilter {
|
||||
pub id: Option<i64>,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
impl ArticleFilters {
|
||||
pub fn from_query(query: &ArticlesQuery) -> Self {
|
||||
pub fn from_query(query: &ArticlesQuery, interest: Option<InterestFilter>) -> Self {
|
||||
let owned = |value: Option<&str>| value.map(str::to_string);
|
||||
let date = |value: Option<&str>| {
|
||||
non_empty(value)
|
||||
@@ -103,8 +112,19 @@ impl ArticleFilters {
|
||||
};
|
||||
let mut kinds: Vec<&str> = TRIAGE_KINDS.to_vec();
|
||||
kinds.push(PROVIDER_REJECTED);
|
||||
let sort = ARTICLE_SORTS
|
||||
.iter()
|
||||
.find(|(name, _)| Some(*name) == query.sort.as_deref())
|
||||
.map(|(name, _)| *name)
|
||||
.filter(|sort| *sort != "match" || interest.is_some())
|
||||
.unwrap_or(if interest.is_some() {
|
||||
"match"
|
||||
} else {
|
||||
ARTICLE_SORTS[0].0
|
||||
});
|
||||
Self {
|
||||
q: owned(non_empty(query.q.as_deref())),
|
||||
interest,
|
||||
feed: non_empty(query.feed.as_deref()).and_then(|feed| feed.parse::<i64>().ok()),
|
||||
stage: owned(allow_listed(query.stage.as_deref(), &STAGES)),
|
||||
reason: owned(allow_listed(query.reason.as_deref(), &REASONS)),
|
||||
@@ -113,11 +133,7 @@ impl ArticleFilters {
|
||||
from: date(query.from.as_deref()),
|
||||
to: date(query.to.as_deref()),
|
||||
kind: owned(allow_listed(query.kind.as_deref(), &kinds)),
|
||||
sort: ARTICLE_SORTS
|
||||
.iter()
|
||||
.find(|(name, _)| Some(*name) == query.sort.as_deref())
|
||||
.map(|(name, _)| *name)
|
||||
.unwrap_or(ARTICLE_SORTS[0].0),
|
||||
sort,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,6 +198,10 @@ impl ArticleFilters {
|
||||
fn params(&self) -> Vec<(&'static str, Option<String>)> {
|
||||
vec![
|
||||
("q", self.q.clone()),
|
||||
(
|
||||
"interest",
|
||||
self.interest.as_ref().map(|interest| interest.name.clone()),
|
||||
),
|
||||
("feed", self.feed.map(|feed| feed.to_string())),
|
||||
("stage", self.stage.clone()),
|
||||
("reason", self.reason.clone()),
|
||||
@@ -219,6 +239,13 @@ pub struct ArticleListRow {
|
||||
pub rating: Option<String>,
|
||||
pub rating_class: &'static str,
|
||||
pub published: Option<String>,
|
||||
pub interests: Vec<ArticleInterest>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ArticleInterest {
|
||||
pub name: String,
|
||||
pub href: String,
|
||||
}
|
||||
|
||||
const ARTICLE_INNER: &str =
|
||||
@@ -226,12 +253,14 @@ const ARTICLE_INNER: &str =
|
||||
a.word_count, e.feed_id, COALESCE(e.feed_title, '') AS feed_title,
|
||||
l.run_id, l.stage, l.excluded_reason, l.utility, r.date AS run_date,
|
||||
t.score AS triage, t.kind AS triage_kind, d.score AS quality, d.fit AS fit,
|
||||
{match_cos} AS match_cos,
|
||||
(SELECT re.label FROM rating_events re
|
||||
WHERE re.article_id = a.id AND re.kind = 'explicit'
|
||||
ORDER BY re.event_at DESC, re.id DESC LIMIT 1) AS rating,
|
||||
(SELECT ia.issue_date FROM issue_articles ia
|
||||
WHERE ia.article_id = a.id ORDER BY ia.issue_date DESC LIMIT 1) AS published
|
||||
FROM articles a
|
||||
{interest_join}
|
||||
LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||
LEFT JOIN candidate_runs l ON l.article_id = a.id
|
||||
AND l.run_id = (SELECT MAX(c2.run_id) FROM candidate_runs c2
|
||||
@@ -245,10 +274,16 @@ pub async fn list_articles(
|
||||
config: &Config,
|
||||
filters: &ArticleFilters,
|
||||
page: u32,
|
||||
) -> Result<(Vec<ArticleListRow>, Pagination), sqlx::Error> {
|
||||
) -> anyhow::Result<(Vec<ArticleListRow>, Pagination)> {
|
||||
let (clauses, binds) = filters.where_clauses();
|
||||
let count_sql = format!("SELECT COUNT(*) FROM ({ARTICLE_INNER}) x WHERE 1 = 1{clauses}");
|
||||
let total: i64 = bind_all(dynamic_query(count_sql), &binds)
|
||||
let (inner, interest_binds) = article_inner(filters);
|
||||
let all_binds = interest_binds
|
||||
.iter()
|
||||
.cloned()
|
||||
.chain(binds.iter().cloned())
|
||||
.collect::<Vec<_>>();
|
||||
let count_sql = format!("SELECT COUNT(*) FROM ({inner}) x WHERE 1 = 1{clauses}");
|
||||
let total: i64 = bind_all(dynamic_query(count_sql), &all_binds)
|
||||
.fetch_one(db.pool())
|
||||
.await?
|
||||
.get(0);
|
||||
@@ -258,15 +293,15 @@ pub async fn list_articles(
|
||||
total,
|
||||
};
|
||||
let select_sql = format!(
|
||||
"SELECT * FROM ({ARTICLE_INNER}) x WHERE 1 = 1{clauses} ORDER BY {} LIMIT ? OFFSET ?",
|
||||
"SELECT * FROM ({inner}) x WHERE 1 = 1{clauses} ORDER BY {} LIMIT ? OFFSET ?",
|
||||
filters.order_by()
|
||||
);
|
||||
let rows = bind_all(dynamic_query(select_sql), &binds)
|
||||
let rows = bind_all(dynamic_query(select_sql), &all_binds)
|
||||
.bind(i64::from(ARTICLES_PER_PAGE))
|
||||
.bind(pagination.offset())
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
let rows = rows
|
||||
let mut rows: Vec<ArticleListRow> = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let id: ArticleId = row.get("id");
|
||||
@@ -290,12 +325,45 @@ pub async fn list_articles(
|
||||
rating_class: widget_label(rating.as_deref()),
|
||||
rating,
|
||||
published: row.get("published"),
|
||||
interests: Vec::new(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let ids = rows.iter().map(|row| row.id).collect::<Vec<_>>();
|
||||
let mut by_article: HashMap<ArticleId, Vec<ArticleInterest>> = HashMap::new();
|
||||
for matched in crate::interests::matches_for_articles(db, &ids).await? {
|
||||
by_article
|
||||
.entry(matched.article_id)
|
||||
.or_default()
|
||||
.push(ArticleInterest {
|
||||
href: crate::interests::articles_href(&matched.name),
|
||||
name: matched.name,
|
||||
});
|
||||
}
|
||||
for row in &mut rows {
|
||||
row.interests = by_article.remove(&row.id).unwrap_or_default();
|
||||
}
|
||||
Ok((rows, pagination))
|
||||
}
|
||||
|
||||
fn article_inner(filters: &ArticleFilters) -> (String, Vec<Bind>) {
|
||||
match &filters.interest {
|
||||
Some(interest) => (
|
||||
ARTICLE_INNER.replace("{match_cos}", "ai.cos").replace(
|
||||
"{interest_join}",
|
||||
"JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?",
|
||||
),
|
||||
vec![Bind::Int(interest.id.unwrap_or(-1))],
|
||||
),
|
||||
None => (
|
||||
ARTICLE_INNER
|
||||
.replace("{match_cos}", "NULL")
|
||||
.replace("{interest_join}", ""),
|
||||
Vec::new(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Template)]
|
||||
#[template(path = "dashboard/articles.html")]
|
||||
struct ArticlesTemplate {
|
||||
@@ -308,6 +376,7 @@ struct ArticlesTemplate {
|
||||
rated: Vec<&'static str>,
|
||||
kinds: Vec<&'static str>,
|
||||
sorts: Vec<&'static str>,
|
||||
interest_options: Vec<crate::interests::Interest>,
|
||||
pager: Pager,
|
||||
}
|
||||
|
||||
@@ -319,11 +388,24 @@ async fn list(
|
||||
) -> Result<Response, WebError> {
|
||||
let viewer = auth.user().await.map(Viewer::from);
|
||||
let config = state.config();
|
||||
let filters = ArticleFilters::from_query(&query);
|
||||
let interest_options = crate::interests::list(&state.db)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
let requested_interest = non_empty(query.interest.as_deref()).map(str::to_string);
|
||||
let interest = requested_interest.map(|name| {
|
||||
let found = interest_options
|
||||
.iter()
|
||||
.find(|interest| interest.name.eq_ignore_ascii_case(&name));
|
||||
InterestFilter {
|
||||
id: found.map(|interest| interest.id),
|
||||
name: found.map(|interest| interest.name.clone()).unwrap_or(name),
|
||||
}
|
||||
});
|
||||
let filters = ArticleFilters::from_query(&query, interest);
|
||||
let page_no = page_number(query.page);
|
||||
let (articles, pagination) = list_articles(&state.db, &config, &filters, page_no)
|
||||
.await
|
||||
.map_err(db_err)?;
|
||||
.map_err(WebError::Internal)?;
|
||||
let pager = Pager::new(pagination, "/dashboard/articles", &filters.params());
|
||||
let mut kinds: Vec<&'static str> = TRIAGE_KINDS.to_vec();
|
||||
kinds.push(PROVIDER_REJECTED);
|
||||
@@ -342,6 +424,7 @@ async fn list(
|
||||
rated: RATED.to_vec(),
|
||||
kinds,
|
||||
sorts: ARTICLE_SORTS.iter().map(|(name, _)| *name).collect(),
|
||||
interest_options,
|
||||
pager,
|
||||
})
|
||||
.into_response())
|
||||
@@ -389,7 +472,6 @@ pub struct HistoryRow {
|
||||
pub utility: String,
|
||||
pub rank: String,
|
||||
pub cluster: String,
|
||||
pub editor_why: Option<String>,
|
||||
pub signals: SignalsView,
|
||||
}
|
||||
|
||||
@@ -557,8 +639,7 @@ pub async fn assessments(
|
||||
pub async fn run_history(db: &Db, article_id: ArticleId) -> Result<Vec<HistoryRow>, sqlx::Error> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT cr.run_id, r.date, r.status, cr.stage, cr.excluded_reason, cr.admitted_by,
|
||||
cr.signals_json, cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank,
|
||||
cr.editor_why
|
||||
cr.signals_json, cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank
|
||||
FROM candidate_runs cr JOIN runs r ON r.id = cr.run_id
|
||||
WHERE cr.article_id = ? ORDER BY cr.run_id DESC",
|
||||
)
|
||||
@@ -590,7 +671,6 @@ pub async fn run_history(db: &Db, article_id: ArticleId) -> Result<Vec<HistoryRo
|
||||
(Some(id), None) => id.to_string(),
|
||||
_ => "—".into(),
|
||||
},
|
||||
editor_why: row.get("editor_why"),
|
||||
signals: SignalsView::from_json(&row.get::<String, _>("signals_json")),
|
||||
}
|
||||
})
|
||||
@@ -845,7 +925,7 @@ mod tests {
|
||||
fn query(f: impl FnOnce(&mut ArticlesQuery)) -> ArticleFilters {
|
||||
let mut query = ArticlesQuery::default();
|
||||
f(&mut query);
|
||||
ArticleFilters::from_query(&query)
|
||||
ArticleFilters::from_query(&query, None)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -939,6 +1019,86 @@ mod tests {
|
||||
assert_eq!(rows[0].id, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_match_sort_needs_an_interest_and_round_trips_in_the_pager_links() {
|
||||
let without = query(|q| q.sort = Some("match".into()));
|
||||
assert_eq!(without.sort, "first_seen");
|
||||
assert!(without.params().contains(&("sort", None)));
|
||||
|
||||
let with = ArticlesQuery {
|
||||
interest: Some("Rust".into()),
|
||||
..ArticlesQuery::default()
|
||||
};
|
||||
let with = ArticleFilters::from_query(
|
||||
&with,
|
||||
Some(InterestFilter {
|
||||
id: Some(7),
|
||||
name: "Rust".into(),
|
||||
}),
|
||||
);
|
||||
assert_eq!(with.sort, "match", "the interest filter sorts by cosine");
|
||||
let params = with.params();
|
||||
assert!(params.contains(&("interest", Some("Rust".into()))));
|
||||
assert!(params.contains(&("sort", Some("match".into()))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interest_filter_finds_matches_unknown_is_empty_and_badges_link() {
|
||||
let seed = seed().await;
|
||||
let crate::interests::AddOutcome::Added(interest_id) = crate::interests::add(
|
||||
&seed.db,
|
||||
"Rust & Systems",
|
||||
Some("Software"),
|
||||
"2026-09-12T12:00:00Z".parse().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap() else {
|
||||
unreachable!();
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||
VALUES (1, ?, 0.82, 2.1, ?)",
|
||||
)
|
||||
.bind(interest_id)
|
||||
.bind(seed.run_id)
|
||||
.execute(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let app = app_with_users(&seed.db).await;
|
||||
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||
let filtered = get(
|
||||
&app,
|
||||
"/dashboard/articles?interest=rust+%26+systems",
|
||||
Some(&admin),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(filtered.status(), axum::http::StatusCode::OK);
|
||||
let filtered = crate::web::dashboard::tests::response_text(filtered).await;
|
||||
assert!(filtered.contains("Article 1 about prose"), "{filtered}");
|
||||
assert!(!filtered.contains("Article 2 about graphs"), "{filtered}");
|
||||
assert!(filtered.contains("value=\"match\" selected"), "{filtered}");
|
||||
assert!(
|
||||
filtered.contains(
|
||||
"href=\"/dashboard/articles?interest=Rust+%26+Systems\">Rust & Systems</a>"
|
||||
),
|
||||
"{filtered}"
|
||||
);
|
||||
|
||||
let unknown = get(
|
||||
&app,
|
||||
"/dashboard/articles?interest=not-a-real-interest",
|
||||
Some(&admin),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(unknown.status(), axum::http::StatusCode::OK);
|
||||
let unknown = crate::web::dashboard::tests::response_text(unknown).await;
|
||||
assert!(
|
||||
unknown.contains("No articles match this filter."),
|
||||
"{unknown}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn article_detail_shows_assessments_run_history_and_rating_events() {
|
||||
let seed = seed().await;
|
||||
|
||||
@@ -69,6 +69,12 @@ struct WhyArticle {
|
||||
title: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct WhyInterest {
|
||||
name: String,
|
||||
href: String,
|
||||
}
|
||||
|
||||
/// One table row.
|
||||
#[derive(Debug)]
|
||||
struct FeedRow {
|
||||
@@ -79,7 +85,7 @@ struct FeedRow {
|
||||
/// The candidate's title, or its feed URL when it has none.
|
||||
label: String,
|
||||
host: String,
|
||||
interests: Vec<String>,
|
||||
interests: Vec<WhyInterest>,
|
||||
articles: Vec<WhyArticle>,
|
||||
article_count: usize,
|
||||
first_seen: String,
|
||||
@@ -383,7 +389,13 @@ fn row(
|
||||
) -> FeedRow {
|
||||
let mut scored = evidence_of(candidate, evidence);
|
||||
let score = discovery::score(&scored);
|
||||
let interests = discovery::why(&scored);
|
||||
let interests = discovery::why(&scored)
|
||||
.into_iter()
|
||||
.map(|name| WhyInterest {
|
||||
href: crate::interests::articles_href(&name),
|
||||
name,
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Best evidence first; the articles nothing is known about come last, in
|
||||
// title order.
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
//! Dashboard: standing interests and their rating-derived weights.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use askama::Template;
|
||||
use axum::Router;
|
||||
use axum::extract::{Extension, Form, Path, Query, State};
|
||||
use axum::response::{IntoResponse, Redirect, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum_login::tower_sessions::Session;
|
||||
use jiff::Timestamp;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::jobs::set_flash;
|
||||
use crate::curate::signals;
|
||||
use crate::interests as interest_store;
|
||||
use crate::server::AppState;
|
||||
use crate::types::ArticleId;
|
||||
use crate::web::session::{AuthSession, Viewer};
|
||||
use crate::web::{Html, Page, WebError, encode_component, take_flash};
|
||||
|
||||
const PATH: &str = "/dashboard/interests";
|
||||
const SORTS: [&str; 4] = ["weight", "name", "matches", "added"];
|
||||
|
||||
/// Routes contributed by the interests page.
|
||||
pub fn routes() -> Router<AppState> {
|
||||
Router::new()
|
||||
.route(PATH, get(index).post(add))
|
||||
.route("/dashboard/interests/{id}/category", post(set_category))
|
||||
.route("/dashboard/interests/{id}/delete", post(delete))
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct InterestsQuery {
|
||||
category: Option<String>,
|
||||
sort: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct AddForm {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
category: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CategoryForm {
|
||||
#[serde(default)]
|
||||
category: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CategoryOption {
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct InterestRow {
|
||||
id: i64,
|
||||
name: String,
|
||||
href: String,
|
||||
category: Option<String>,
|
||||
category_href: String,
|
||||
weight: String,
|
||||
weight_value: f64,
|
||||
up: String,
|
||||
down: String,
|
||||
rated_matches: usize,
|
||||
matched_articles: i64,
|
||||
added: String,
|
||||
created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Template)]
|
||||
#[template(path = "dashboard/interests.html")]
|
||||
struct InterestsTemplate {
|
||||
page: Page,
|
||||
rows: Vec<InterestRow>,
|
||||
categories: Vec<CategoryOption>,
|
||||
selected_category: String,
|
||||
selected_sort: String,
|
||||
sorts: Vec<&'static str>,
|
||||
total: usize,
|
||||
uncategorized: usize,
|
||||
lookback_days: i64,
|
||||
half_life_days: String,
|
||||
affinity_gate: String,
|
||||
attributable: usize,
|
||||
affinity_full: usize,
|
||||
jobs_enabled: bool,
|
||||
}
|
||||
|
||||
async fn index(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthSession,
|
||||
Extension(session): Extension<Session>,
|
||||
Query(query): Query<InterestsQuery>,
|
||||
) -> Result<Response, WebError> {
|
||||
let viewer = auth
|
||||
.user()
|
||||
.await
|
||||
.map(Viewer::from)
|
||||
.ok_or_else(|| WebError::Unauthenticated { next: PATH.into() })?;
|
||||
let config = state.config();
|
||||
let ranking = &config.curation.ranking;
|
||||
let db = &state.db;
|
||||
let now = Timestamp::now();
|
||||
let interests = interest_store::list(db).await.map_err(WebError::Internal)?;
|
||||
let total = interests.len();
|
||||
let uncategorized = interests
|
||||
.iter()
|
||||
.filter(|interest| interest.category.is_none())
|
||||
.count();
|
||||
|
||||
let mut category_names = interests
|
||||
.iter()
|
||||
.filter_map(|interest| interest.category.clone())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
category_names.sort_by(|left, right| {
|
||||
left.to_lowercase()
|
||||
.cmp(&right.to_lowercase())
|
||||
.then_with(|| left.cmp(right))
|
||||
});
|
||||
let categories = category_names
|
||||
.iter()
|
||||
.map(|name| CategoryOption { name: name.clone() })
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let selected_category = query
|
||||
.category
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|category| !category.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_default();
|
||||
let selected_sort = query
|
||||
.sort
|
||||
.as_deref()
|
||||
.filter(|sort| SORTS.contains(sort))
|
||||
.unwrap_or("weight")
|
||||
.to_string();
|
||||
|
||||
let ratings = db.current_ratings(ranking.rating_lookback_days).await?;
|
||||
let rated_ids = ratings
|
||||
.iter()
|
||||
.map(|rating| rating.article_id)
|
||||
.collect::<Vec<_>>();
|
||||
let matches = interest_store::matches_for_articles(db, &rated_ids)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
let rated = ratings
|
||||
.iter()
|
||||
.map(|rating| {
|
||||
let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0;
|
||||
(
|
||||
rating.article_id,
|
||||
rating.value,
|
||||
signals::decay(age_days, ranking.rating_half_life_days),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<(ArticleId, f64, f64)>>();
|
||||
let matched = matches
|
||||
.iter()
|
||||
.map(|row| (row.article_id, row.interest_id, row.z))
|
||||
.collect::<Vec<_>>();
|
||||
let rates = interest_store::rates(&rated, &matched);
|
||||
let counts = interest_store::match_counts(db)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
|
||||
let mut rows = interests
|
||||
.into_iter()
|
||||
.filter(|interest| match selected_category.as_str() {
|
||||
"" => true,
|
||||
"uncategorized" => interest.category.is_none(),
|
||||
category => interest
|
||||
.category
|
||||
.as_deref()
|
||||
.is_some_and(|current| current.eq_ignore_ascii_case(category)),
|
||||
})
|
||||
.map(|interest| {
|
||||
let rate = rates
|
||||
.by_interest
|
||||
.get(&interest.id)
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let category_href = interest
|
||||
.category
|
||||
.as_deref()
|
||||
.map(|category| format!("{PATH}?category={}", encode_component(category)))
|
||||
.unwrap_or_else(|| format!("{PATH}?category=uncategorized"));
|
||||
InterestRow {
|
||||
id: interest.id,
|
||||
href: interest_store::articles_href(&interest.name),
|
||||
name: interest.name,
|
||||
category: interest.category,
|
||||
category_href,
|
||||
weight: if rate.n > 0 {
|
||||
format!("{:.2}", rate.weight())
|
||||
} else {
|
||||
"—".into()
|
||||
},
|
||||
weight_value: rate.weight(),
|
||||
up: format!("{:.2}", rate.up),
|
||||
down: format!("{:.2}", rate.down),
|
||||
rated_matches: rate.n,
|
||||
matched_articles: counts.get(&interest.id).copied().unwrap_or(0),
|
||||
added: super::fmt_stored_time(Some(&interest.created_at), &config),
|
||||
created_at: interest.created_at,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
sort_rows(&mut rows, &selected_sort);
|
||||
|
||||
let mut page = Page::new("Interests", Some(viewer), "interests");
|
||||
page.flash = take_flash(&session).await?;
|
||||
Ok(Html(InterestsTemplate {
|
||||
page,
|
||||
rows,
|
||||
categories,
|
||||
selected_category,
|
||||
selected_sort,
|
||||
sorts: SORTS.to_vec(),
|
||||
total,
|
||||
uncategorized,
|
||||
lookback_days: ranking.rating_lookback_days,
|
||||
half_life_days: format!("{}", ranking.rating_half_life_days),
|
||||
affinity_gate: format!(
|
||||
"{:.2}",
|
||||
signals::gate(
|
||||
rates.attributable,
|
||||
ranking.affinity_floor,
|
||||
ranking.affinity_full
|
||||
)
|
||||
),
|
||||
attributable: rates.attributable,
|
||||
affinity_full: ranking.affinity_full,
|
||||
jobs_enabled: config.server.jobs_enabled,
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
fn sort_rows(rows: &mut [InterestRow], sort: &str) {
|
||||
rows.sort_by(|left, right| {
|
||||
let selected = match sort {
|
||||
"name" => Ordering::Equal,
|
||||
"matches" => right.matched_articles.cmp(&left.matched_articles),
|
||||
"added" => right.created_at.cmp(&left.created_at),
|
||||
_ => right
|
||||
.weight_value
|
||||
.partial_cmp(&left.weight_value)
|
||||
.unwrap_or(Ordering::Equal),
|
||||
};
|
||||
selected.then_with(|| {
|
||||
left.name
|
||||
.to_lowercase()
|
||||
.cmp(&right.name.to_lowercase())
|
||||
.then_with(|| left.name.cmp(&right.name))
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn form_category(value: &str) -> Option<&str> {
|
||||
let value = value.trim();
|
||||
(!value.is_empty()).then_some(value)
|
||||
}
|
||||
|
||||
async fn add(
|
||||
State(state): State<AppState>,
|
||||
Extension(session): Extension<Session>,
|
||||
Form(form): Form<AddForm>,
|
||||
) -> Result<Response, WebError> {
|
||||
match interest_store::add(
|
||||
&state.db,
|
||||
&form.name,
|
||||
form_category(&form.category),
|
||||
Timestamp::now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(interest_store::AddOutcome::Added(_)) => {
|
||||
set_flash(&session, "success", format!("Added {}.", form.name.trim())).await?;
|
||||
}
|
||||
Ok(interest_store::AddOutcome::Duplicate) => {
|
||||
set_flash(
|
||||
&session,
|
||||
"error",
|
||||
format!("{} already exists.", form.name.trim()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => set_flash(&session, "error", error.to_string()).await?,
|
||||
}
|
||||
Ok(Redirect::to(PATH).into_response())
|
||||
}
|
||||
|
||||
async fn set_category(
|
||||
State(state): State<AppState>,
|
||||
Extension(session): Extension<Session>,
|
||||
Path(id): Path<i64>,
|
||||
Form(form): Form<CategoryForm>,
|
||||
) -> Result<Response, WebError> {
|
||||
interest_store::set_category(
|
||||
&state.db,
|
||||
id,
|
||||
form_category(&form.category),
|
||||
Timestamp::now(),
|
||||
)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
set_flash(&session, "success", "Category saved.".into()).await?;
|
||||
Ok(Redirect::to(PATH).into_response())
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
State(state): State<AppState>,
|
||||
Extension(session): Extension<Session>,
|
||||
Path(id): Path<i64>,
|
||||
) -> Result<Response, WebError> {
|
||||
let name: Option<String> = sqlx::query_scalar("SELECT name FROM interests WHERE id = ?")
|
||||
.bind(id)
|
||||
.fetch_optional(state.db.pool())
|
||||
.await
|
||||
.map_err(super::db_err)?;
|
||||
interest_store::delete(&state.db, id)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
set_flash(
|
||||
&session,
|
||||
"success",
|
||||
name.map(|name| format!("Deleted {name}."))
|
||||
.unwrap_or_else(|| "Interest already deleted.".into()),
|
||||
)
|
||||
.await?;
|
||||
Ok(Redirect::to(PATH).into_response())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::body::Body;
|
||||
use axum::http::{Method, Request, StatusCode, header};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::*;
|
||||
use crate::interests::AddOutcome;
|
||||
use crate::web::dashboard::tests::{
|
||||
app_with_users, assert_admin_only, login_cookie, response_text, seed,
|
||||
};
|
||||
|
||||
async fn post_form(
|
||||
app: &axum::Router,
|
||||
uri: &str,
|
||||
cookie: Option<&str>,
|
||||
body: &str,
|
||||
) -> Response {
|
||||
let mut request = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(uri)
|
||||
.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.header("sec-fetch-site", "same-origin");
|
||||
if let Some(cookie) = cookie {
|
||||
request = request.header(header::COOKIE, cookie);
|
||||
}
|
||||
app.clone()
|
||||
.oneshot(request.body(Body::from(body.to_string())).unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn page_sorts_rating_weights_descending_and_is_admin_only() {
|
||||
let seed = seed().await;
|
||||
let now = Timestamp::now();
|
||||
let AddOutcome::Added(rust) = interest_store::add(&seed.db, "Rust", Some("Software"), now)
|
||||
.await
|
||||
.unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
interest_store::add(&seed.db, "Cooking", None, now)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO article_interests (article_id, interest_id, cos, z, run_id)
|
||||
VALUES (1, ?, 0.8, 3.0, ?)",
|
||||
)
|
||||
.bind(rust)
|
||||
.bind(seed.run_id)
|
||||
.execute(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let app = app_with_users(&seed.db).await;
|
||||
let body = assert_admin_only(&app, PATH).await;
|
||||
let table = body.split_once("<tbody>").unwrap().1;
|
||||
assert!(table.find(">Rust</a>").unwrap() < table.find(">Cooking</a>").unwrap());
|
||||
assert!(
|
||||
table.contains("/dashboard/articles?interest=Rust"),
|
||||
"{table}"
|
||||
);
|
||||
assert!(body.contains("affinity gate"), "{body}");
|
||||
|
||||
let reader = login_cookie(&app, "reader", "correct horse battery").await;
|
||||
for (uri, body) in [
|
||||
(PATH.to_string(), "name=New&category="),
|
||||
(
|
||||
format!("/dashboard/interests/{rust}/category"),
|
||||
"category=Other",
|
||||
),
|
||||
(format!("/dashboard/interests/{rust}/delete"), ""),
|
||||
] {
|
||||
let forbidden = post_form(&app, &uri, Some(&reader), body).await;
|
||||
assert_eq!(forbidden.status(), StatusCode::FORBIDDEN, "{uri}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_reports_a_case_insensitive_duplicate() {
|
||||
let seed = seed().await;
|
||||
interest_store::add(&seed.db, "Rust", None, Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
let app = app_with_users(&seed.db).await;
|
||||
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||
let response = post_form(&app, PATH, Some(&admin), "name=rust&category=").await;
|
||||
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||
let body =
|
||||
response_text(crate::web::dashboard::tests::get(&app, PATH, Some(&admin)).await).await;
|
||||
assert!(body.contains("rust already exists."), "{body}");
|
||||
assert_eq!(interest_store::list(&seed.db).await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_action_removes_matches_and_the_embedding() {
|
||||
let seed = seed().await;
|
||||
let AddOutcome::Added(id) = interest_store::add(&seed.db, "Rust", None, Timestamp::now())
|
||||
.await
|
||||
.unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO article_interests (article_id, interest_id, cos, z)
|
||||
VALUES (1, ?, 0.8, 2.0)",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO interest_embeddings
|
||||
(interest, model, dimension, embedding, created_at)
|
||||
VALUES ('Rust', 'test', 1, X'00000000', '2026-09-12T00:00:00Z')",
|
||||
)
|
||||
.execute(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let app = app_with_users(&seed.db).await;
|
||||
let admin = login_cookie(&app, "admin", "correct horse battery").await;
|
||||
let response = post_form(
|
||||
&app,
|
||||
&format!("/dashboard/interests/{id}/delete"),
|
||||
Some(&admin),
|
||||
"",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::SEE_OTHER);
|
||||
let interests: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interests")
|
||||
.fetch_one(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let matches: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_interests")
|
||||
.fetch_one(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let embeddings: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interest_embeddings")
|
||||
.fetch_one(seed.db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((interests, matches, embeddings), (0, 0, 0));
|
||||
}
|
||||
}
|
||||
+25
-20
@@ -11,6 +11,7 @@
|
||||
|
||||
pub mod articles;
|
||||
pub mod feeds;
|
||||
pub mod interests;
|
||||
pub mod jobs;
|
||||
pub mod profile;
|
||||
pub mod ratings;
|
||||
@@ -45,6 +46,7 @@ pub fn router() -> Router<AppState> {
|
||||
.merge(runs::routes())
|
||||
.merge(articles::routes())
|
||||
.merge(ratings::routes())
|
||||
.merge(interests::routes())
|
||||
.merge(feeds::routes())
|
||||
.merge(profile::routes())
|
||||
.merge(settings::routes())
|
||||
@@ -58,10 +60,11 @@ pub fn router() -> Router<AppState> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Signal names in the order of curation plan §7.5.
|
||||
pub const SIGNAL_NAMES: [&str; 8] = [
|
||||
pub const SIGNAL_NAMES: [&str; 9] = [
|
||||
"interest",
|
||||
"knn",
|
||||
"feed",
|
||||
"affinity",
|
||||
"social",
|
||||
"heuristic",
|
||||
"triage",
|
||||
@@ -276,6 +279,7 @@ pub struct SignalLine {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct InterestLine {
|
||||
pub name: String,
|
||||
pub href: String,
|
||||
pub z: String,
|
||||
pub cos: String,
|
||||
}
|
||||
@@ -343,6 +347,7 @@ impl SignalsView {
|
||||
.iter()
|
||||
.map(|interest| InterestLine {
|
||||
name: interest.name.clone(),
|
||||
href: crate::interests::articles_href(&interest.name),
|
||||
z: format!("{:.2}", interest.z),
|
||||
cos: format!("{:.3}", interest.cos),
|
||||
})
|
||||
@@ -428,6 +433,8 @@ struct OverviewTemplate {
|
||||
ratings_total: i64,
|
||||
access_requests: i64,
|
||||
feed_candidates: i64,
|
||||
interests_total: usize,
|
||||
uncategorized_interests: usize,
|
||||
unrated: Vec<UnratedPick>,
|
||||
active_jobs: Vec<JobLine>,
|
||||
finished_jobs: Vec<JobLine>,
|
||||
@@ -455,6 +462,14 @@ async fn overview(
|
||||
.await
|
||||
.map_err(db_err)?;
|
||||
let feed_candidates = crate::discovery::count(db, "candidate").await?;
|
||||
let interests = crate::interests::list(db)
|
||||
.await
|
||||
.map_err(WebError::Internal)?;
|
||||
let interests_total = interests.len();
|
||||
let uncategorized_interests = interests
|
||||
.iter()
|
||||
.filter(|interest| interest.category.is_none())
|
||||
.count();
|
||||
let unrated = unrated_picks(db).await?;
|
||||
let (active_jobs, finished_jobs) = jobs_summary(db, &config).await?;
|
||||
let sparklines = overview_sparklines(db).await?;
|
||||
@@ -474,6 +489,8 @@ async fn overview(
|
||||
ratings_total,
|
||||
access_requests,
|
||||
feed_candidates,
|
||||
interests_total,
|
||||
uncategorized_interests,
|
||||
unrated,
|
||||
active_jobs,
|
||||
finished_jobs,
|
||||
@@ -848,17 +865,8 @@ pub(crate) mod tests {
|
||||
Some("[\"triage\",\"interest\"]"),
|
||||
88.0,
|
||||
1,
|
||||
Some("Why one"),
|
||||
),
|
||||
(
|
||||
2,
|
||||
"selected",
|
||||
None,
|
||||
Some("[\"blend\"]"),
|
||||
80.0,
|
||||
2,
|
||||
Some("Why two"),
|
||||
),
|
||||
(2, "selected", None, Some("[\"blend\"]"), 80.0, 2),
|
||||
(
|
||||
3,
|
||||
"shortlisted",
|
||||
@@ -866,7 +874,6 @@ pub(crate) mod tests {
|
||||
Some("[\"triage\"]"),
|
||||
75.0,
|
||||
3,
|
||||
None,
|
||||
),
|
||||
(
|
||||
4,
|
||||
@@ -875,10 +882,9 @@ pub(crate) mod tests {
|
||||
Some("[\"triage\"]"),
|
||||
60.0,
|
||||
4,
|
||||
None,
|
||||
),
|
||||
];
|
||||
for (article_id, stage, reason, admitted_by, utility, rank, why) in rows {
|
||||
for (article_id, stage, reason, admitted_by, utility, rank) in rows {
|
||||
let json = signals(utility / 10.0, article_id == 2);
|
||||
telemetry::write(
|
||||
&db,
|
||||
@@ -893,7 +899,6 @@ pub(crate) mod tests {
|
||||
rank_utility: Some(rank),
|
||||
cluster_id: Some(1),
|
||||
cluster_rank: Some(rank),
|
||||
editor_why: why,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -914,7 +919,6 @@ pub(crate) mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -941,7 +945,6 @@ pub(crate) mod tests {
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -991,9 +994,9 @@ pub(crate) mod tests {
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles
|
||||
(issue_date, article_id, section, position, is_lead, summary, why)
|
||||
VALUES (?, 1, 'Top Stories', 0, 1, 'Summary one', 'Why one'),
|
||||
(?, 2, 'Top Stories', 1, 0, 'Summary two', 'Why two')",
|
||||
(issue_date, article_id, section, position, is_lead, summary)
|
||||
VALUES (?, 1, 'Top Stories', 0, 1, 'Summary one'),
|
||||
(?, 2, 'Top Stories', 1, 0, 'Summary two')",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.bind(date.to_string())
|
||||
@@ -1225,6 +1228,8 @@ pub(crate) mod tests {
|
||||
"/dashboard/articles/1".to_string(),
|
||||
"/dashboard/ratings".to_string(),
|
||||
"/dashboard/ratings?tab=events".to_string(),
|
||||
"/dashboard/interests".to_string(),
|
||||
"/dashboard/interests?category=uncategorized&sort=name".to_string(),
|
||||
"/dashboard/profile".to_string(),
|
||||
"/dashboard/stats?days=14".to_string(),
|
||||
"/dashboard/settings".to_string(),
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
//! Dashboard: the profile page (`/dashboard/profile`, web plan §11).
|
||||
//!
|
||||
//! Edits `profile.md` with version history, shows what the loader parses out
|
||||
//! of it, the standing OPML interests by theme, the stored system prompt and
|
||||
//! the weekly learned adjustments, and offers the `profile-rebuild` job.
|
||||
//! Edits `profile.md` with version history and shows the stored interests,
|
||||
//! system prompt, and weekly learned adjustments.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
@@ -18,6 +17,7 @@ use sqlx::Row;
|
||||
|
||||
use crate::curate::profile::{self, KV_LEARNED_ADJUSTMENTS, ProfileFile, REBUILD_INTERVAL_DAYS};
|
||||
use crate::db::{Db, DbError, KV_TASTE_PROFILE};
|
||||
use crate::interests;
|
||||
use crate::server::AppState;
|
||||
use crate::web::session::{AuthSession, Viewer};
|
||||
use crate::web::{Flash, Html, Page, WebError, format_time, take_flash};
|
||||
@@ -104,8 +104,7 @@ fn read_profile(path: &Path) -> anyhow::Result<Option<String>> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The live preview of what the loader extracts (§11): the passthrough body
|
||||
/// and the `## Interests` lines.
|
||||
/// The live preview of the prose that reaches the prompt.
|
||||
pub fn preview(content: &str) -> ProfileFile {
|
||||
profile::parse_profile_str(content)
|
||||
}
|
||||
@@ -191,7 +190,7 @@ async fn versions(db: &Db, config: &crate::config::Config) -> Result<Vec<Version
|
||||
// Page
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct ThemeView {
|
||||
struct CategoryView {
|
||||
name: String,
|
||||
members: String,
|
||||
count: usize,
|
||||
@@ -207,12 +206,10 @@ struct ProfileTemplate {
|
||||
bytes: usize,
|
||||
max_bytes: usize,
|
||||
preview_body: String,
|
||||
preview_interests: Vec<String>,
|
||||
versions: Vec<VersionView>,
|
||||
opml_path: String,
|
||||
opml_count: usize,
|
||||
opml_error: String,
|
||||
themes: Vec<ThemeView>,
|
||||
interest_count: usize,
|
||||
category_count: usize,
|
||||
categories: Vec<CategoryView>,
|
||||
prompt: String,
|
||||
prompt_chars: usize,
|
||||
prompt_version: String,
|
||||
@@ -254,20 +251,17 @@ async fn show(
|
||||
let content = stored.unwrap_or_default();
|
||||
let parsed = preview(&content);
|
||||
|
||||
let (opml_count, opml_error, themes) = match profile::parse_interests(&config.interests_opml) {
|
||||
Ok(interests) => {
|
||||
let themes = profile::group_into_themes(&interests)
|
||||
let grouped = interests::grouped(db).await.map_err(WebError::Internal)?;
|
||||
let interest_count = grouped.iter().map(|(_, members)| members.len()).sum();
|
||||
let category_count = grouped.len();
|
||||
let categories = grouped
|
||||
.into_iter()
|
||||
.map(|(name, members)| ThemeView {
|
||||
.map(|(name, members)| CategoryView {
|
||||
name,
|
||||
count: members.len(),
|
||||
members: members.join(", "),
|
||||
})
|
||||
.collect();
|
||||
(interests.len(), String::new(), themes)
|
||||
}
|
||||
Err(error) => (0, format!("{error:#}"), Vec::new()),
|
||||
};
|
||||
|
||||
let prompt = db.kv_get(KV_TASTE_PROFILE).await?.unwrap_or_default();
|
||||
let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default();
|
||||
@@ -297,12 +291,10 @@ async fn show(
|
||||
max_bytes: MAX_PROFILE_BYTES,
|
||||
content,
|
||||
preview_body: parsed.body,
|
||||
preview_interests: parsed.interests,
|
||||
versions: versions(db, &config).await?,
|
||||
opml_path: config.interests_opml.display().to_string(),
|
||||
opml_count,
|
||||
opml_error,
|
||||
themes,
|
||||
interest_count,
|
||||
category_count,
|
||||
categories,
|
||||
prompt_chars: prompt.len(),
|
||||
prompt_verdicts: count_verdict_lines(&prompt),
|
||||
prompt,
|
||||
@@ -516,14 +508,14 @@ mod tests {
|
||||
.unwrap();
|
||||
let config = Config {
|
||||
profile_path: dir.path().join("profile.md"),
|
||||
interests_opml: dir.path().join("interests.opml"),
|
||||
..Config::default()
|
||||
};
|
||||
std::fs::write(&config.profile_path, "# Original\n\nProse.\n").unwrap();
|
||||
std::fs::write(
|
||||
&config.interests_opml,
|
||||
r#"<outline text="Rust"/><outline text="Boston"/>"#,
|
||||
)
|
||||
interests::add(&db, "Rust", Some("Software"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
interests::add(&db, "Boston", Some("Places"), Timestamp::now())
|
||||
.await
|
||||
.unwrap();
|
||||
db.kv_set(
|
||||
KV_TASTE_PROFILE,
|
||||
@@ -617,7 +609,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_page_shows_editor_preview_interests_prompt_and_rebuild_form() {
|
||||
async fn profile_page_shows_editor_standing_interests_prompt_and_rebuild_form() {
|
||||
let (_dir, _state, app, cookie) = setup().await;
|
||||
let response = get(&app, Some(&cookie)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
@@ -625,7 +617,8 @@ mod tests {
|
||||
assert!(body.contains("# Original"));
|
||||
assert!(body.contains("Prose."));
|
||||
assert!(body.contains("Rust, Boston") || body.contains("Rust") && body.contains("Boston"));
|
||||
assert!(body.contains("2 interests"));
|
||||
assert!(body.contains("2 standing interests in 2 categories"));
|
||||
assert!(body.contains("Interests page"));
|
||||
assert!(body.contains("system prompt text"));
|
||||
assert!(body.contains("Rank depth higher."));
|
||||
assert!(body.contains("never built"));
|
||||
@@ -666,7 +659,7 @@ mod tests {
|
||||
|
||||
let page = text(get(&app, Some(&cookie)).await).await;
|
||||
assert!(page.contains("Saved; the next run rebuilds the system prompt."));
|
||||
assert!(page.contains("Writerdeck"));
|
||||
assert!(page.contains("section is ignored"));
|
||||
assert!(page.contains("# Original"));
|
||||
assert!(page.contains(">tyler<"));
|
||||
|
||||
|
||||
@@ -506,7 +506,6 @@ pub struct CandidateView {
|
||||
pub fit: String,
|
||||
pub exploration: bool,
|
||||
pub auto_include: bool,
|
||||
pub editor_why: Option<String>,
|
||||
pub signals: SignalsView,
|
||||
}
|
||||
|
||||
@@ -541,7 +540,6 @@ impl CandidateView {
|
||||
fit: fmt_opt(row.get("fit"), 1),
|
||||
exploration: signals.exploration,
|
||||
auto_include: signals.auto_include,
|
||||
editor_why: row.get("editor_why"),
|
||||
signals,
|
||||
}
|
||||
}
|
||||
@@ -575,7 +573,7 @@ pub async fn candidates(
|
||||
"SELECT cr.article_id, COALESCE(a.title, '') AS title,
|
||||
COALESCE(e.feed_title, '') AS feed_title, a.word_count,
|
||||
cr.stage, cr.excluded_reason, cr.admitted_by, cr.signals_json, cr.utility,
|
||||
cr.rank_utility, cr.cluster_id, cr.cluster_rank, cr.editor_why,
|
||||
cr.rank_utility, cr.cluster_id, cr.cluster_rank,
|
||||
t.score AS triage, d.score AS quality, d.fit AS fit
|
||||
{CANDIDATE_FROM}{clauses}
|
||||
ORDER BY {}
|
||||
@@ -964,8 +962,6 @@ mod tests {
|
||||
assert_eq!(rows[0].article_id, 1);
|
||||
assert_eq!(rows[0].admitted_first.as_deref(), Some("triage"));
|
||||
assert_eq!(rows[0].admitted_rest, "interest");
|
||||
assert_eq!(rows[0].editor_why.as_deref(), Some("Why one"));
|
||||
|
||||
let by_rank = CandidateFilters::from_query(&query(None, Some("rank")));
|
||||
let (rows, _) = candidates(db, seed.run_id, &by_rank, 1).await.unwrap();
|
||||
assert_eq!(rows[0].rank, "1");
|
||||
@@ -1099,7 +1095,6 @@ mod tests {
|
||||
"near miss: {detail}"
|
||||
);
|
||||
assert!(detail.contains("Gaussian Splatting"), "signals: {detail}");
|
||||
assert!(detail.contains("Why one"), "{detail}");
|
||||
assert!(detail.contains("Alpha Blog"), "{detail}");
|
||||
assert!(detail.contains("class=\"funnel\""), "{detail}");
|
||||
assert!(
|
||||
|
||||
@@ -246,7 +246,6 @@ const PATH_KEYS: &[&str] = &[
|
||||
"database_path",
|
||||
"out_dir",
|
||||
"profile_path",
|
||||
"interests_opml",
|
||||
"publish.epub_dir",
|
||||
"publish.xtc_dir",
|
||||
"xtc.settings",
|
||||
@@ -295,7 +294,6 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
||||
("database_path", "SQLite file; parent directories are created on demand."),
|
||||
("out_dir", "Where generate writes artifacts before publishing (overridden by --out)."),
|
||||
("profile_path", "Hand-maintained reader profile, loaded every run."),
|
||||
("interests_opml", "Scour interests OPML merged with the profile interests."),
|
||||
("miniflux.base_url", "Miniflux root (no /v1)."),
|
||||
("miniflux.public_url", "Browser-facing Miniflux web UI URL for dashboard links. Defaults to miniflux.base_url."),
|
||||
("miniflux.api_key", "X-Auth-Token for Miniflux. Required; environment only."),
|
||||
@@ -351,21 +349,25 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
||||
("curation.ranking.knn_full", "Rated articles at which the knn signal reaches full weight. Must be > knn_floor."),
|
||||
("curation.ranking.feed_floor", "Attributable ratings before the feed-affinity signal starts to count."),
|
||||
("curation.ranking.feed_full", "Attributable ratings at which feed affinity reaches full weight. Must be > feed_floor."),
|
||||
("curation.ranking.affinity_floor", "Interest-attributable ratings before the affinity signal starts to count."),
|
||||
("curation.ranking.affinity_full", "Interest-attributable ratings at which affinity reaches full weight. Must be > affinity_floor."),
|
||||
("curation.ranking.semantic_min_words", "Bodies shorter than this are not embedded."),
|
||||
("curation.ranking.exploration_slots", "Shortlist slots reserved for exploration picks."),
|
||||
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted."),
|
||||
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted; 0 keeps them forever."),
|
||||
("curation.ranking.telemetry_retention_days", "features prune: candidate_runs and article_assessments older than this are deleted."),
|
||||
("curation.ranking.quotas.triage", "Deep-set slots filled by triage score."),
|
||||
("curation.ranking.quotas.interest", "Deep-set slots filled by interest similarity."),
|
||||
("curation.ranking.quotas.knn", "Deep-set slots filled by rated-neighbour preference."),
|
||||
("curation.ranking.weights.preliminary.interest", "Interest similarity in the preliminary blend."),
|
||||
("curation.ranking.weights.preliminary.knn", "Rated-neighbour preference in the preliminary blend."),
|
||||
("curation.ranking.weights.preliminary.affinity", "Rating-derived interest affinity in the preliminary blend."),
|
||||
("curation.ranking.weights.preliminary.heuristic", "Heuristic score in the preliminary blend."),
|
||||
("curation.ranking.weights.preliminary.feed", "Feed affinity in the preliminary blend."),
|
||||
("curation.ranking.weights.preliminary.social", "Social signal in the preliminary blend."),
|
||||
("curation.ranking.weights.utility.quality", "Deep-assessment quality in the utility score."),
|
||||
("curation.ranking.weights.utility.fit", "Deep-assessment fit in the utility score."),
|
||||
("curation.ranking.weights.utility.knn", "Rated-neighbour preference in the utility score."),
|
||||
("curation.ranking.weights.utility.affinity", "Rating-derived interest affinity in the utility score."),
|
||||
("curation.ranking.weights.utility.interest", "Interest similarity in the utility score."),
|
||||
("curation.ranking.weights.utility.feed", "Feed affinity in the utility score."),
|
||||
("curation.ranking.weights.utility.triage", "Triage score in the utility score."),
|
||||
|
||||
+315
-39
@@ -119,7 +119,7 @@ pub async fn load(
|
||||
(issue, true)
|
||||
} else {
|
||||
let rows = sqlx::query(
|
||||
"SELECT article_id, section, position, is_lead, summary, why
|
||||
"SELECT article_id, section, position, is_lead, summary
|
||||
FROM issue_articles WHERE issue_date = ? ORDER BY section, position",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
@@ -148,7 +148,6 @@ pub async fn load(
|
||||
section,
|
||||
position: pick_row.get("position"),
|
||||
is_lead: pick_row.get("is_lead"),
|
||||
why: pick_row.get("why"),
|
||||
summary,
|
||||
llm: None,
|
||||
top_interests: Vec::new(),
|
||||
@@ -881,7 +880,6 @@ struct FullEntry {
|
||||
reading_minutes: i64,
|
||||
is_lead: bool,
|
||||
summary: String,
|
||||
why: Option<String>,
|
||||
understanding: chapters::Understanding,
|
||||
rating: Option<RatingWidget>,
|
||||
}
|
||||
@@ -945,24 +943,74 @@ struct ArticleLink {
|
||||
struct ArticleTemplate {
|
||||
page: Page,
|
||||
toc: Toc,
|
||||
head: ArticleHead,
|
||||
discussion_html: Option<String>,
|
||||
rating: Option<RatingWidget>,
|
||||
previous: Option<ArticleLink>,
|
||||
next: Option<ArticleLink>,
|
||||
issue_href: String,
|
||||
related: Vec<RelatedView>,
|
||||
}
|
||||
|
||||
/// An archive article that never ran in an issue, reached from a related list:
|
||||
/// the stored body with no summary, rating or issue navigation.
|
||||
#[derive(Template)]
|
||||
#[template(path = "article_standalone.html")]
|
||||
struct StandaloneArticleTemplate {
|
||||
page: Page,
|
||||
head: ArticleHead,
|
||||
}
|
||||
|
||||
/// What `_article_head.html` renders (header and body) plus the footer links
|
||||
/// both article pages share.
|
||||
struct ArticleHead {
|
||||
title: String,
|
||||
source_url: String,
|
||||
byline: Option<String>,
|
||||
source: Source,
|
||||
meta_line: String,
|
||||
why: Option<String>,
|
||||
social_line: Option<String>,
|
||||
/// Issue-only: empty and `None` on the standalone page.
|
||||
understanding: chapters::Understanding,
|
||||
summary: Option<String>,
|
||||
excerpt_only: bool,
|
||||
body_html: String,
|
||||
discussion_html: Option<String>,
|
||||
read_online_url: String,
|
||||
dashboard_href: String,
|
||||
rating: Option<RatingWidget>,
|
||||
previous: Option<ArticleLink>,
|
||||
next: Option<ArticleLink>,
|
||||
issue_href: String,
|
||||
}
|
||||
|
||||
impl ArticleHead {
|
||||
fn new(article: &Article, config: &Config, is_admin: bool) -> Self {
|
||||
Self {
|
||||
title: article.title.clone(),
|
||||
source_url: article.canonical_url.clone(),
|
||||
byline: article.author.as_ref().map(|author| format!("By {author}")),
|
||||
source: Source::new(article, config, is_admin),
|
||||
meta_line: format!(
|
||||
"{} words · ~{} min read",
|
||||
thousands(article.word_count),
|
||||
article.reading_minutes()
|
||||
),
|
||||
social_line: chapters::social_line(&article.social),
|
||||
understanding: chapters::Understanding::default(),
|
||||
summary: None,
|
||||
excerpt_only: article.excerpt_only,
|
||||
body_html: prepare_body(&article.content_html),
|
||||
read_online_url: article.url.clone(),
|
||||
dashboard_href: format!("/dashboard/articles/{}", article.id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of the "Related" list under an issue article.
|
||||
#[derive(Debug)]
|
||||
struct RelatedView {
|
||||
title: String,
|
||||
href: String,
|
||||
author: Option<String>,
|
||||
source: Source,
|
||||
reading_minutes: i64,
|
||||
date: String,
|
||||
}
|
||||
|
||||
#[derive(Template)]
|
||||
@@ -1029,7 +1077,6 @@ pub async fn render_full(
|
||||
summary: summary_for(&view.issue, pick)
|
||||
.unwrap_or_default()
|
||||
.to_string(),
|
||||
why: pick.why.clone(),
|
||||
understanding: chapters::understanding(pick),
|
||||
rating: is_admin.then(|| {
|
||||
RatingWidget::for_issue(
|
||||
@@ -1118,32 +1165,21 @@ pub async fn article(
|
||||
let is_admin = viewer.role == crate::web::users::Role::Admin;
|
||||
let config = state.config();
|
||||
let active_nav = if view.is_latest { "latest" } else { "archive" };
|
||||
let related = related_views(&state.db, &config, is_admin, date, article_id).await?;
|
||||
let mut page = Page::new(article.title.clone(), Some(viewer.clone()), active_nav);
|
||||
page.flash = take_flash(&session).await?;
|
||||
Ok(Html(ArticleTemplate {
|
||||
page,
|
||||
toc,
|
||||
title: article.title.clone(),
|
||||
source_url: article.canonical_url.clone(),
|
||||
byline: article.author.as_ref().map(|author| format!("By {author}")),
|
||||
source: Source::new(article, &config, is_admin),
|
||||
meta_line: format!(
|
||||
"{} words · ~{} min read",
|
||||
thousands(article.word_count),
|
||||
article.reading_minutes()
|
||||
),
|
||||
why: pick.why.clone(),
|
||||
social_line: chapters::social_line(&article.social),
|
||||
head: ArticleHead {
|
||||
understanding: chapters::understanding(pick),
|
||||
summary: summary_for(&view.issue, pick).map(str::to_string),
|
||||
excerpt_only: article.excerpt_only,
|
||||
body_html: prepare_body(&article.content_html),
|
||||
..ArticleHead::new(article, &config, is_admin)
|
||||
},
|
||||
discussion_html: pick
|
||||
.discussion
|
||||
.as_ref()
|
||||
.map(|discussion| crate::comments::render_xhtml(discussion, &article.title)),
|
||||
read_online_url: article.url.clone(),
|
||||
dashboard_href: format!("/dashboard/articles/{}", article.id),
|
||||
rating: is_admin.then(|| {
|
||||
RatingWidget::for_issue(
|
||||
article.id,
|
||||
@@ -1155,10 +1191,90 @@ pub async fn article(
|
||||
previous,
|
||||
next,
|
||||
issue_href: issue_href(date),
|
||||
related,
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `/articles/{id}`: any stored article on its own page. One that ran in an
|
||||
/// issue redirects to its (latest) issue page, which carries the summary and
|
||||
/// related list.
|
||||
pub async fn standalone(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthSession,
|
||||
Extension(session): Extension<Session>,
|
||||
Path(article_id): Path<ArticleId>,
|
||||
) -> Result<Response, WebError> {
|
||||
let viewer = auth
|
||||
.user()
|
||||
.await
|
||||
.map(Viewer::from)
|
||||
.ok_or_else(|| WebError::Unauthenticated {
|
||||
next: standalone_href(article_id),
|
||||
})?;
|
||||
let Some(article) = state.db.get_article(article_id).await? else {
|
||||
return Err(WebError::NotFound);
|
||||
};
|
||||
if let Some(date) = crate::related::latest_issues(&state.db, &[article_id])
|
||||
.await?
|
||||
.remove(&article_id)
|
||||
{
|
||||
return Ok(Redirect::to(&article_href(date, article_id)).into_response());
|
||||
}
|
||||
let is_admin = viewer.role == crate::web::users::Role::Admin;
|
||||
let config = state.config();
|
||||
let mut page = Page::new(article.title.clone(), Some(viewer), "archive");
|
||||
page.flash = take_flash(&session).await?;
|
||||
Ok(Html(StandaloneArticleTemplate {
|
||||
page,
|
||||
head: ArticleHead::new(&article, &config, is_admin),
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// The stored related links of one pick, each pointing at the issue page it
|
||||
/// ran on or, failing that, its standalone page.
|
||||
async fn related_views(
|
||||
db: &Db,
|
||||
config: &Config,
|
||||
is_admin: bool,
|
||||
date: Date,
|
||||
article_id: ArticleId,
|
||||
) -> anyhow::Result<Vec<RelatedView>> {
|
||||
let ids = crate::related::for_article(db, date, article_id).await?;
|
||||
if ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let articles = db.get_articles(&ids).await?;
|
||||
let issues = crate::related::latest_issues(db, &ids).await?;
|
||||
Ok(ids
|
||||
.iter()
|
||||
.filter_map(|id| {
|
||||
let article = articles.get(id)?;
|
||||
let href = match issues.get(id) {
|
||||
Some(date) => article_href(*date, article.id),
|
||||
None => standalone_href(article.id),
|
||||
};
|
||||
Some(RelatedView {
|
||||
title: article.title.clone(),
|
||||
href,
|
||||
author: article.author.clone(),
|
||||
source: Source::new(article, config, is_admin),
|
||||
reading_minutes: article.reading_minutes(),
|
||||
date: short_date(article.published_at.unwrap_or(article.first_seen), config),
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// "Aug 15, 2026" in the configured zone.
|
||||
fn short_date(timestamp: jiff::Timestamp, config: &Config) -> String {
|
||||
match config.tz() {
|
||||
Ok(tz) => timestamp.to_zoned(tz).strftime("%b %-d, %Y").to_string(),
|
||||
Err(_) => timestamp.strftime("%b %-d, %Y").to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn world(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthSession,
|
||||
@@ -1340,6 +1456,10 @@ fn article_href(date: Date, article_id: ArticleId) -> String {
|
||||
format!("/issues/{date}/articles/{article_id}")
|
||||
}
|
||||
|
||||
fn standalone_href(article_id: ArticleId) -> String {
|
||||
format!("/articles/{article_id}")
|
||||
}
|
||||
|
||||
fn issue_href(date: Date) -> String {
|
||||
format!("/issues/{date}")
|
||||
}
|
||||
@@ -1420,7 +1540,7 @@ fn thousands(value: i64) -> String {
|
||||
/// Sanitize article HTML exactly as the EPUB does, then add browser-only image
|
||||
/// loading/referrer attributes without converting the fragment to XHTML.
|
||||
fn prepare_body(html: &str) -> String {
|
||||
let clean = ammonia::clean(html);
|
||||
let clean = crate::highlight::clean_and_highlight(html);
|
||||
let mut output = String::with_capacity(clean.len() + 64);
|
||||
let mut cursor = 0usize;
|
||||
while let Some(relative) = clean[cursor..].find('<') {
|
||||
@@ -1636,7 +1756,6 @@ mod tests {
|
||||
rank_utility: Some(3),
|
||||
cluster_id: Some(1),
|
||||
cluster_rank: Some(2),
|
||||
editor_why: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1844,7 +1963,6 @@ mod tests {
|
||||
assert!(html.contains("The Lead Story"));
|
||||
assert!(html.contains("Hacker News"));
|
||||
assert!(html.contains("What it argues, and why it is worth the time."));
|
||||
assert!(html.contains("Why it's here"));
|
||||
assert!(!html.contains("Two stories today"));
|
||||
assert!(!html.contains("Something happened"));
|
||||
assert!(!html.contains("Body of"));
|
||||
@@ -1895,20 +2013,75 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
let document = roxmltree::Document::parse(&feed).unwrap();
|
||||
assert_eq!(
|
||||
document
|
||||
let atom_entries = document
|
||||
.descendants()
|
||||
.filter(|node| node.tag_name().name() == "entry")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
.count();
|
||||
assert_eq!(atom_entries, 1);
|
||||
assert!(feed.contains("What it argues, and why it is worth the time."));
|
||||
assert!(feed.contains("Why it's here"));
|
||||
assert!(!feed.contains("Something happened"));
|
||||
assert!(!feed.contains("Two stories today"));
|
||||
assert!(!feed.contains("Body of"));
|
||||
assert!(!feed.contains("write path"));
|
||||
|
||||
let json = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/feed.json")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
json.headers().get(header::CONTENT_TYPE).unwrap(),
|
||||
"application/feed+json; charset=utf-8"
|
||||
);
|
||||
assert_eq!(
|
||||
json.headers().get(header::CACHE_CONTROL).unwrap(),
|
||||
"public, max-age=300"
|
||||
);
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&to_bytes(json.into_body(), 1024 * 1024).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(json["version"], "https://jsonfeed.org/version/1.1");
|
||||
let items = json["items"].as_array().unwrap();
|
||||
assert_eq!(items.len(), atom_entries);
|
||||
let item = &items[0];
|
||||
assert!(item["id"].as_str().unwrap().starts_with("tag:"), "{item}");
|
||||
assert!(
|
||||
item["url"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.ends_with(&format!("/issues/{}", source.meta.date)),
|
||||
"{item}"
|
||||
);
|
||||
assert!(
|
||||
item["title"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("The Daily EPUB — "),
|
||||
"{item}"
|
||||
);
|
||||
let content_html = item["content_html"].as_str().unwrap();
|
||||
assert!(content_html.contains("<h2>"), "{content_html}");
|
||||
assert!(
|
||||
content_html.contains("<p class=\"meta\">A. Writer · Example Feed · example.com</p>"),
|
||||
"{content_html}"
|
||||
);
|
||||
assert!(
|
||||
content_html.contains(
|
||||
"<p class=\"links\"><a href=\"https://news.ycombinator.com/item?id=40100000\">Hacker News</a></p>"
|
||||
),
|
||||
"{content_html}"
|
||||
);
|
||||
assert!(!content_html.contains("</a> — "), "{content_html}");
|
||||
assert!(
|
||||
!item["date_published"].as_str().unwrap().is_empty(),
|
||||
"{item}"
|
||||
);
|
||||
|
||||
let robots = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
@@ -2000,10 +2173,15 @@ mod tests {
|
||||
assert!(issue.contains("Two stories today"));
|
||||
assert!(issue.contains("What it argues"));
|
||||
assert!(issue.contains("A short abstract for the second piece"));
|
||||
assert!(issue.contains("Why it"));
|
||||
assert!(issue.contains("Software engineering · Analysis"));
|
||||
assert!(issue.contains("copy-on-write · ZFS"));
|
||||
assert!(issue.contains("Matches: Filesystems · Rust"));
|
||||
assert!(issue.contains("<span class=\"interest-chip\">Filesystems</span>"));
|
||||
assert!(issue.contains("<span class=\"interest-chip\">Rust</span>"));
|
||||
let issue_rubric_position = issue.find("Software engineering · Analysis").unwrap();
|
||||
let issue_matches_position = issue.find("Matched interests: </span>").unwrap();
|
||||
let issue_summary_position = issue.find("What it argues").unwrap();
|
||||
assert!(issue_rubric_position < issue_matches_position);
|
||||
assert!(issue_matches_position < issue_summary_position);
|
||||
assert!(issue.contains("A. Writer · Example Feed"));
|
||||
assert!(!issue.contains("example feed · Example Feed"));
|
||||
assert!(issue.contains("World Briefing"));
|
||||
@@ -2012,6 +2190,8 @@ mod tests {
|
||||
|
||||
let article_id = source.lineup.picks[0].article.id;
|
||||
let dashboard_href = format!("/dashboard/articles/{article_id}");
|
||||
let interest_href = crate::interests::articles_href("Filesystems");
|
||||
assert!(!issue.contains(&interest_href));
|
||||
assert!(!issue.contains(&dashboard_href));
|
||||
let article = app
|
||||
.clone()
|
||||
@@ -2034,13 +2214,15 @@ mod tests {
|
||||
assert!(article.contains("loading=\"lazy\""));
|
||||
assert!(article.contains("referrerpolicy=\"no-referrer\""));
|
||||
let rubric_position = article.find("Software engineering · Analysis").unwrap();
|
||||
let why_position = article.find("Why it's here").unwrap();
|
||||
let matches_position = article.find("Matched interests: </span>").unwrap();
|
||||
assert!(article.contains("<span class=\"interest-chip\">Filesystems</span>"));
|
||||
assert!(article.contains("<span class=\"interest-chip\">Rust</span>"));
|
||||
let summary_position = article
|
||||
.find("What it argues, and why it is worth the time.")
|
||||
.unwrap();
|
||||
let social_position = article.find("342 on HN").unwrap();
|
||||
assert!(rubric_position < why_position);
|
||||
assert!(why_position < summary_position);
|
||||
assert!(rubric_position < matches_position);
|
||||
assert!(matches_position < summary_position);
|
||||
assert!(summary_position < social_position);
|
||||
assert!(article.contains("A Niche Delight"));
|
||||
assert!(article.contains("rel=\"next\""));
|
||||
@@ -2141,6 +2323,98 @@ mod tests {
|
||||
assert!(!reader.contains(&feed_href));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn related_links_reach_issue_pages_and_standalone_archive_articles() {
|
||||
let (_dir, db, source) = seeded_issue(true).await;
|
||||
crate::web::users::add(&db, "reader", "correct horse battery", false)
|
||||
.await
|
||||
.unwrap();
|
||||
let pick = &source.lineup.picks[0].article;
|
||||
let sibling = &source.lineup.picks[1].article;
|
||||
let mut archived = pick.clone();
|
||||
archived.id = 0;
|
||||
archived.canonical_url = "https://example.com/archived-story".into();
|
||||
archived.title = "An archived story nobody picked".into();
|
||||
archived.content_html = "<p>Archived body text.</p>".into();
|
||||
let archived_id = db.upsert_article(&archived).await.unwrap();
|
||||
for (position, related_id) in [archived_id, sibling.id].into_iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
|
||||
VALUES (?, ?, ?, ?, 0.8)",
|
||||
)
|
||||
.bind(source.meta.date.to_string())
|
||||
.bind(pick.id)
|
||||
.bind(position as i64)
|
||||
.bind(related_id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let app = crate::server::router(crate::server::AppState::new(
|
||||
db,
|
||||
crate::config::Config::default(),
|
||||
None,
|
||||
));
|
||||
let cookie = login_cookie(&app, "reader", "correct horse battery").await;
|
||||
let get = |uri: String, cookie: Option<String>| {
|
||||
let app = app.clone();
|
||||
async move {
|
||||
let mut request = Request::builder().uri(uri);
|
||||
if let Some(cookie) = cookie {
|
||||
request = request.header(header::COOKIE, cookie);
|
||||
}
|
||||
app.oneshot(request.body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
|
||||
let page = response_text(
|
||||
get(
|
||||
format!("/issues/{}/articles/{}", source.meta.date, pick.id),
|
||||
Some(cookie.clone()),
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
let related = page
|
||||
.split_once("id=\"related-heading\"")
|
||||
.expect("related list")
|
||||
.1;
|
||||
let archived_at = related
|
||||
.find(&format!("href=\"/articles/{archived_id}\""))
|
||||
.expect("unpicked article links to its standalone page");
|
||||
let sibling_at = related
|
||||
.find(&format!(
|
||||
"href=\"/issues/{}/articles/{}\"",
|
||||
source.meta.date, sibling.id
|
||||
))
|
||||
.expect("a picked article links to its issue page");
|
||||
assert!(archived_at < sibling_at, "stored order is kept");
|
||||
|
||||
let standalone = get(format!("/articles/{archived_id}"), Some(cookie.clone())).await;
|
||||
assert_eq!(standalone.status(), StatusCode::OK);
|
||||
let standalone = response_text(standalone).await;
|
||||
assert!(standalone.contains("An archived story nobody picked"));
|
||||
assert!(standalone.contains("Archived body text."));
|
||||
|
||||
let picked = get(format!("/articles/{}", pick.id), Some(cookie)).await;
|
||||
assert_eq!(picked.status(), StatusCode::SEE_OTHER);
|
||||
assert_eq!(
|
||||
picked.headers()[header::LOCATION],
|
||||
format!("/issues/{}/articles/{}", source.meta.date, pick.id)
|
||||
);
|
||||
|
||||
let anonymous = get(format!("/articles/{archived_id}"), None).await;
|
||||
assert!(anonymous.status().is_redirection());
|
||||
assert!(
|
||||
anonymous.headers()[header::LOCATION]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("/login")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fallback_full_issue_recovers_colophon_world_and_behind() {
|
||||
let (_dir, db, source) = seeded_issue(false).await;
|
||||
@@ -2741,6 +3015,8 @@ mod tests {
|
||||
assert!(admin_issue.contains("Was this a good pick?"));
|
||||
assert!(admin_issue.contains("value=\"loved\" data-label=\"loved\" class=\"active\""));
|
||||
assert!(admin_issue.contains(&dashboard_href));
|
||||
let interest_href = crate::interests::articles_href("Filesystems");
|
||||
assert!(admin_issue.contains(&format!("class=\"interest-chip\" href=\"{interest_href}\"")));
|
||||
|
||||
let admin_article = app
|
||||
.clone()
|
||||
|
||||
@@ -322,6 +322,7 @@ impl Page {
|
||||
| "runs"
|
||||
| "articles"
|
||||
| "ratings"
|
||||
| "interests"
|
||||
| "profile"
|
||||
| "stats"
|
||||
| "jobs"
|
||||
@@ -680,6 +681,7 @@ pub fn router(config: &crate::config::Config) -> axum::Router<crate::server::App
|
||||
.route_layer(from_fn(session::require_password_change));
|
||||
let full_issues = axum::Router::new()
|
||||
.route("/issues/{date}/articles/{article_id}", get(issue::article))
|
||||
.route("/articles/{article_id}", get(issue::standalone))
|
||||
.route("/issues/{date}/world", get(issue::world))
|
||||
.route("/issues/{date}/behind", get(issue::behind))
|
||||
.route("/issues/{date}/read", get(issue::read))
|
||||
@@ -707,6 +709,7 @@ pub fn router(config: &crate::config::Config) -> axum::Router<crate::server::App
|
||||
.route("/issues", get(public::archive))
|
||||
.route("/issues/{date}", get(public::show_issue))
|
||||
.route("/feed.xml", get(public::feed))
|
||||
.route("/feed.json", get(public::feed_json))
|
||||
.route("/robots.txt", get(public::robots))
|
||||
.route("/static/{file}", get(static_asset))
|
||||
.merge(access_routes)
|
||||
|
||||
+138
-21
@@ -4,6 +4,7 @@ use axum::http::{HeaderMap, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum_login::tower_sessions::Session;
|
||||
use jiff::civil::Date;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::server::AppState;
|
||||
use crate::types::{Issue, SocialSource};
|
||||
@@ -53,7 +54,6 @@ pub struct PublicEntry {
|
||||
pub reading_minutes: i64,
|
||||
pub word_count: i64,
|
||||
pub summary: Option<String>,
|
||||
pub why: Option<String>,
|
||||
pub understanding: crate::epub::chapters::Understanding,
|
||||
pub comment_links: Vec<CommentLink>,
|
||||
pub is_lead: bool,
|
||||
@@ -132,7 +132,6 @@ impl From<&Issue> for PublicIssue {
|
||||
.map(str::trim)
|
||||
.filter(|summary| !summary.is_empty())
|
||||
.map(str::to_string),
|
||||
why: pick.why.clone(),
|
||||
understanding: crate::epub::chapters::understanding(pick),
|
||||
comment_links,
|
||||
is_lead: pick.is_lead,
|
||||
@@ -291,10 +290,25 @@ pub async fn archive(
|
||||
Ok(public_cache(response, &headers))
|
||||
}
|
||||
|
||||
pub async fn feed(State(state): State<AppState>) -> Result<Response, WebError> {
|
||||
/// One issue as both public feeds carry it.
|
||||
///
|
||||
/// Atom and JSON Feed differ only in how these five fields are spelled out, so
|
||||
/// the expensive part — loading the issue and rendering `feed_entry.html` — is
|
||||
/// done once, by [`feed_entries`], and shared.
|
||||
struct FeedEntry {
|
||||
id: String,
|
||||
href: String,
|
||||
title: String,
|
||||
generated_at: jiff::Timestamp,
|
||||
content: String,
|
||||
}
|
||||
|
||||
/// The last 30 issues, newest first, plus the newest `generated_at` among them
|
||||
/// (the feed-level `<updated>`).
|
||||
async fn feed_entries(state: &AppState) -> Result<(Vec<FeedEntry>, jiff::Timestamp), WebError> {
|
||||
let config = state.config();
|
||||
let rows = state.db.issue_dates(Some(30)).await?;
|
||||
let mut entries = String::new();
|
||||
let mut entries = Vec::new();
|
||||
let mut updated = jiff::Timestamp::UNIX_EPOCH;
|
||||
for row in rows {
|
||||
let Some(view) = issue::load(&state.db, &config, row.date).await? else {
|
||||
@@ -308,20 +322,38 @@ pub async fn feed(State(state): State<AppState>) -> Result<Response, WebError> {
|
||||
let content = FeedEntryTemplate { issue: &issue }.render();
|
||||
crate::web::timing::record_render(started.elapsed());
|
||||
let content = content.map_err(|error| WebError::Internal(error.into()))?;
|
||||
let href = format!(
|
||||
entries.push(FeedEntry {
|
||||
id: format!(
|
||||
"tag:{},{}:issue/{}",
|
||||
feed_host(&config.server.public_url),
|
||||
issue.date.year(),
|
||||
issue.date
|
||||
),
|
||||
href: format!(
|
||||
"{}/issues/{}",
|
||||
config.server.public_url.trim_end_matches('/'),
|
||||
issue.date
|
||||
);
|
||||
),
|
||||
title: format!("The Daily EPUB — {}", issue.date),
|
||||
generated_at: issue.generated_at,
|
||||
content,
|
||||
});
|
||||
}
|
||||
Ok((entries, updated))
|
||||
}
|
||||
|
||||
pub async fn feed(State(state): State<AppState>) -> Result<Response, WebError> {
|
||||
let config = state.config();
|
||||
let (issues, updated) = feed_entries(&state).await?;
|
||||
let mut entries = String::new();
|
||||
for entry in &issues {
|
||||
entries.push_str(&format!(
|
||||
"<entry><id>tag:{},{}:issue/{}</id><title>The Daily EPUB — {}</title><updated>{}</updated><link rel=\"alternate\" href=\"{}\"/><content type=\"html\">{}</content></entry>",
|
||||
feed_host(&config.server.public_url),
|
||||
issue.date.year(),
|
||||
issue.date,
|
||||
issue.date,
|
||||
issue.generated_at,
|
||||
xml_escape(&href),
|
||||
xml_escape(&content),
|
||||
"<entry><id>{}</id><title>{}</title><updated>{}</updated><link rel=\"alternate\" href=\"{}\"/><content type=\"html\">{}</content></entry>",
|
||||
entry.id,
|
||||
entry.title,
|
||||
entry.generated_at,
|
||||
xml_escape(&entry.href),
|
||||
xml_escape(&entry.content),
|
||||
));
|
||||
}
|
||||
let home = config.server.public_url.trim_end_matches('/');
|
||||
@@ -343,6 +375,59 @@ pub async fn feed(State(state): State<AppState>) -> Result<Response, WebError> {
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// JSON Feed 1.1 <https://jsonfeed.org/version/1.1>, the same issues `/feed.xml`
|
||||
/// carries.
|
||||
#[derive(Serialize)]
|
||||
struct JsonFeed<'a> {
|
||||
version: &'static str,
|
||||
title: &'static str,
|
||||
home_page_url: &'a str,
|
||||
feed_url: String,
|
||||
items: Vec<JsonFeedItem<'a>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct JsonFeedItem<'a> {
|
||||
id: &'a str,
|
||||
url: &'a str,
|
||||
title: &'a str,
|
||||
/// The rendered `feed_entry.html`, raw: JSON escaping is serde's job.
|
||||
content_html: &'a str,
|
||||
date_published: String,
|
||||
}
|
||||
|
||||
pub async fn feed_json(State(state): State<AppState>) -> Result<Response, WebError> {
|
||||
let config = state.config();
|
||||
let (entries, _updated) = feed_entries(&state).await?;
|
||||
let home = config.server.public_url.trim_end_matches('/');
|
||||
let body = serde_json::to_string(&JsonFeed {
|
||||
version: "https://jsonfeed.org/version/1.1",
|
||||
title: "The Daily EPUB",
|
||||
home_page_url: home,
|
||||
feed_url: format!("{home}/feed.json"),
|
||||
items: entries
|
||||
.iter()
|
||||
.map(|entry| JsonFeedItem {
|
||||
id: &entry.id,
|
||||
url: &entry.href,
|
||||
title: &entry.title,
|
||||
content_html: &entry.content,
|
||||
date_published: entry.generated_at.to_string(),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
.map_err(|error| WebError::Internal(error.into()))?;
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/feed+json; charset=utf-8"),
|
||||
(header::CACHE_CONTROL, PUBLIC_CACHE),
|
||||
],
|
||||
body,
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub async fn robots() -> Response {
|
||||
(
|
||||
StatusCode::OK,
|
||||
@@ -423,7 +508,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn public_issue_shows_summaries_and_why_but_no_bodies() {
|
||||
fn public_issue_shows_summaries_but_no_bodies() {
|
||||
let source = crate::epub::fixtures::issue();
|
||||
let public = PublicIssue::from(&source);
|
||||
assert_eq!(
|
||||
@@ -442,19 +527,20 @@ mod tests {
|
||||
assert!(html.contains("Hacker News"));
|
||||
assert!(html.contains("What it argues, and why it is worth the time."));
|
||||
assert!(html.contains("A short abstract for the second piece."));
|
||||
assert!(html.contains("The systems story with enough operational detail to matter"));
|
||||
assert!(html.contains("A small-scene delight outside the usual technical orbit"));
|
||||
assert!(!html.contains("The systems story with enough operational detail to matter"));
|
||||
assert!(!html.contains("A small-scene delight outside the usual technical orbit"));
|
||||
assert!(html.contains("Software engineering · Analysis"));
|
||||
assert!(html.contains("copy-on-write · ZFS"));
|
||||
assert!(html.contains("Matches: Filesystems · Rust"));
|
||||
assert!(html.contains("<span class=\"interest-chip\">Filesystems</span>"));
|
||||
assert!(html.contains("<span class=\"interest-chip\">Rust</span>"));
|
||||
let rubric_position = html.find("Software engineering · Analysis").unwrap();
|
||||
let why_position = html.find("Why it's here").unwrap();
|
||||
let matches_position = html.find("Matched interests: </span>").unwrap();
|
||||
let summary_position = html
|
||||
.find("What it argues, and why it is worth the time.")
|
||||
.unwrap();
|
||||
let comments_position = html.find(">Hacker News:").unwrap();
|
||||
assert!(rubric_position < why_position);
|
||||
assert!(why_position < summary_position);
|
||||
assert!(rubric_position < matches_position);
|
||||
assert!(matches_position < summary_position);
|
||||
assert!(summary_position < comments_position);
|
||||
for private in [
|
||||
"Two stories today",
|
||||
@@ -467,4 +553,35 @@ mod tests {
|
||||
assert!(!html.contains(private), "leaked {private:?} in {html}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_entry_wraps_byline_and_links_in_paragraphs() {
|
||||
let mut source = crate::epub::fixtures::issue();
|
||||
// A second comment link, so the separator between links is exercised.
|
||||
source.lineup.picks[0].article.comments_url =
|
||||
Some("https://example.com/discuss".to_string());
|
||||
let issue = PublicIssue::from(&source);
|
||||
let html = FeedEntryTemplate { issue: &issue }.render().unwrap();
|
||||
|
||||
assert!(html.contains("<h2>Top Stories</h2><ul>"), "{html}");
|
||||
assert!(
|
||||
html.contains(
|
||||
"<li><a href=\"https://example.com/1001\">The Lead Story</a><p class=\"meta\">A. Writer · Example Feed · example.com</p>"
|
||||
),
|
||||
"{html}"
|
||||
);
|
||||
assert!(
|
||||
html.contains("<p class=\"summary\">What it argues, and why it is worth the time.</p>"),
|
||||
"{html}"
|
||||
);
|
||||
assert!(
|
||||
html.contains(
|
||||
"<p class=\"links\"><a href=\"https://news.ycombinator.com/item?id=40100000\">Hacker News</a> · <a href=\"https://example.com/discuss\">Comments</a></p></li>"
|
||||
),
|
||||
"{html}"
|
||||
);
|
||||
// The old run-together byline and adjacent links are gone.
|
||||
assert!(!html.contains("</a> — "), "{html}");
|
||||
assert!(!html.contains("Hacker News</a><a "), "{html}");
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,2 @@
|
||||
/*! tailwindcss v4.3.3 | MIT License | https://tailwindcss.com */
|
||||
@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-duration:initial}}}@layer theme{:root,:host{--font-sans:ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;--font-serif:"Newsreader", "Newsreader Fallback Georgia", "Newsreader Fallback Times", "Newsreader Fallback Noto Serif", "Newsreader Fallback DejaVu Serif", "Iowan Old Style", "Palatino Linotype", Georgia, serif;--font-mono:ui-monospace, SFMono-Regular, Menlo, monospace;--spacing:.25rem;--container-md:28rem;--container-lg:32rem;--container-2xl:42rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-normal:0em;--leading-tight:1.25;--leading-snug:1.375;--leading-normal:1.5;--leading-relaxed:1.625;--radius-sm:.25rem;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;--default-mono-font-family:ui-monospace, SFMono-Regular, Menlo, monospace;--color-paper:var(--paper);--color-paper-2:var(--paper-2);--color-ink:var(--ink)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-vaLine truncated
|
||||
@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-duration:initial}}}@layer theme{:root,:host{--spacing:.25rem;--container-md:28rem;--container-lg:32rem;--container-2xl:42rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-normal:0em;--leading-tight:1.25;--leading-snug:1.375;--leading-normal:1.5;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;--default-mono-font-family:ui-monospace, SFMono-Regular, Menlo, monospace}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{paddiLine truncated
|
||||
+49
-5
@@ -1,4 +1,4 @@
|
||||
@import "tailwindcss";
|
||||
@import "tailwindcss" source(none);
|
||||
@source "../web/templates";
|
||||
@source "./static/app.js";
|
||||
|
||||
@@ -83,6 +83,7 @@
|
||||
--paper:#f6f3ec; --paper-2:#ece7db; --ink:#1c1b18; --ink-2:#4f4b43; --muted:#6b665a;
|
||||
--rule:#d8d2c4; --rule-strong:#1c1b18; --accent:#a3231f; --accent-hover:#7c1a16;
|
||||
--loved:#2f6f46; --good:#2f6a8f; --down:#9c3f36; --warn:#7d631c; color-scheme:light;
|
||||
--hl-keyword:#8c2a22; --hl-string:#3b6246; --hl-number:#7a4f16; --hl-type:#2c5572;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
@@ -90,6 +91,7 @@
|
||||
--paper:#151513; --paper-2:#1e1d1a; --ink:#ebe6da; --ink-2:#bfb9aa; --muted:#8f8a7d;
|
||||
--rule:#3a3833; --rule-strong:#ebe6da; --accent:#ef8c82; --accent-hover:#f7aaa2;
|
||||
--loved:#7fcb92; --good:#7fb8d8; --down:#ea8f84; --warn:#d9b45a; color-scheme:dark;
|
||||
--hl-keyword:#e3a197; --hl-string:#a3c9a9; --hl-number:#d8b77a; --hl-type:#9dbdd3;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +99,7 @@
|
||||
--paper:#151513; --paper-2:#1e1d1a; --ink:#ebe6da; --ink-2:#bfb9aa; --muted:#8f8a7d;
|
||||
--rule:#3a3833; --rule-strong:#ebe6da; --accent:#ef8c82; --accent-hover:#f7aaa2;
|
||||
--loved:#7fcb92; --good:#7fb8d8; --down:#ea8f84; --warn:#d9b45a; color-scheme:dark;
|
||||
--hl-keyword:#e3a197; --hl-string:#a3c9a9; --hl-number:#d8b77a; --hl-type:#9dbdd3;
|
||||
}
|
||||
|
||||
@theme inline {
|
||||
@@ -159,8 +162,8 @@
|
||||
.kv { @apply grid grid-cols-[minmax(9rem,1fr)_2fr] gap-x-5 gap-y-1.5; }
|
||||
.kv dt { @apply text-sm text-muted; }
|
||||
.kv dd { @apply m-0 min-w-0 text-sm; overflow-wrap:anywhere; }
|
||||
/* Dashboard table layout contract: docs/dashboard-tables.md. */
|
||||
.scroll-x { @apply max-w-full overflow-x-auto; overscroll-behavior-inline:contain; }
|
||||
.scroll-x > table { min-width:max-content; }
|
||||
.scroll-x.tall { @apply max-h-[70vh] overflow-y-auto; overscroll-behavior-block:contain; }
|
||||
|
||||
.reader-page { @apply min-w-0; overflow-wrap:anywhere; }
|
||||
@@ -178,6 +181,17 @@
|
||||
.prose-body th, .prose-body td { overflow-wrap:normal; }
|
||||
.prose-body pre { @apply max-w-full overflow-x-auto rounded-sm bg-paper-2 p-4 text-sm leading-relaxed; }
|
||||
.prose-body :not(pre) > code { @apply rounded-sm bg-paper-2 px-1 py-0.5 text-[0.9em]; }
|
||||
/* Syntax tokens (src/highlight.rs): ink-first, with a few print colours —
|
||||
oxblood keywords, bottle-green strings, umber literals, slate types. All at
|
||||
least 4.5:1 on paper-2 in both themes. Names and variables stay ink. */
|
||||
.hl-keyword, .hl-tag, .hl-diff-delete { color:var(--hl-keyword); }
|
||||
.hl-string, .hl-diff-add { color:var(--hl-string); }
|
||||
.hl-number, .hl-constant, .hl-macro, .hl-attribute, .hl-label { color:var(--hl-number); }
|
||||
.hl-type, .hl-namespace, .hl-constructor { color:var(--hl-type); }
|
||||
.hl-comment { color:var(--muted); font-style:italic; }
|
||||
.hl-punctuation, .hl-operator { color:var(--ink-2); }
|
||||
.hl-title, .hl-strong { font-weight:600; }
|
||||
.hl-emphasis { font-style:italic; }
|
||||
.prose-body blockquote { @apply my-7 border-l-2 border-rule-strong pl-5 text-ink-2; }
|
||||
.prose-body ul { @apply list-disc pl-6; }
|
||||
.prose-body ol { @apply list-decimal pl-6; }
|
||||
@@ -186,6 +200,12 @@
|
||||
|
||||
.reader-page :where(h1, h2, h3, h4), .reader-section-heading { text-wrap:balance; }
|
||||
.reader-section-heading { @apply border-t-[3px] border-double border-rule-strong pt-3 font-serif text-[1.05rem] font-semibold uppercase leading-[1.2] tracking-[0.15em] text-ink; }
|
||||
/* Matched-interest tags: a hairline in the divider colour with squared
|
||||
corners, so the row reads as small print, not as buttons. */
|
||||
.interest-chip { @apply inline-block rounded-xs border border-rule px-2 py-px text-xs leading-5 text-ink-2 no-underline; }
|
||||
a.interest-chip { @apply relative; }
|
||||
a.interest-chip::after { content:""; @apply absolute -inset-y-1.5 inset-x-0; }
|
||||
a.interest-chip:hover { @apply border-accent text-accent; }
|
||||
|
||||
.rating { @apply my-5 flex flex-wrap items-center gap-x-3 font-sans text-xs; }
|
||||
.rating-prompt { @apply font-serif text-[0.8rem] italic text-muted; }
|
||||
@@ -213,7 +233,30 @@
|
||||
ul.error, ul.notice { @apply list-disc pl-8; }
|
||||
|
||||
/* ---------- dashboard shell ---------- */
|
||||
.dashboard { @apply mx-auto my-8 max-w-7xl px-4 pb-6 font-sans text-base leading-normal sm:px-6; }
|
||||
.dashboard {
|
||||
--gutter:1rem;
|
||||
@apply my-8 grid pb-6 font-sans text-base leading-normal;
|
||||
grid-template-columns:
|
||||
[full-start] var(--gutter)
|
||||
[wide-start] 1fr
|
||||
[content-start] minmax(0, calc(80rem - 2 * var(--gutter)))
|
||||
[content-end] 1fr
|
||||
[wide-end] var(--gutter)
|
||||
[full-end];
|
||||
}
|
||||
@media (width >= 40rem) { .dashboard { --gutter:1.5rem; } }
|
||||
/* Grid items never collapse margins, so a child's bottom margin (`.table-filter`, `.rating`) would add to
|
||||
the next child's `mt-4`; the rhythm comes from top margins only. `!important` beats those utilities. */
|
||||
.dashboard > * { grid-column:content; min-width:0; margin-bottom:0 !important; }
|
||||
/* The wrapper shrinks to its table (a centred grid item is sized fit-content), so the
|
||||
flanks beside a narrow table stay page, not scroll container: wheeling there scrolls the page. */
|
||||
.dashboard > .scroll-x {
|
||||
grid-column:wide;
|
||||
justify-self:center;
|
||||
min-width:min(100%, calc(80rem - 2 * var(--gutter)));
|
||||
max-width:100%;
|
||||
}
|
||||
.dashboard > .scroll-x > table { width:auto; min-width:100%; }
|
||||
.dashboard > * + * { @apply mt-4; }
|
||||
.dashboard h1 { @apply text-2xl font-semibold leading-tight tracking-[-0.01em]; }
|
||||
.dashboard h2 { @apply text-lg font-semibold; }
|
||||
@@ -248,8 +291,9 @@
|
||||
|
||||
.muted, .meta { @apply text-muted; }
|
||||
.jobs a, .picks > li > a:first-child { @apply text-ink no-underline hover:text-accent; }
|
||||
.num { @apply whitespace-nowrap text-right tabular-nums; }
|
||||
.cell-wrap { @apply min-w-[10rem] max-w-[24rem] whitespace-normal; overflow-wrap:anywhere; }
|
||||
.num { @apply text-right tabular-nums; }
|
||||
td.num { @apply whitespace-nowrap; }
|
||||
.cell-wrap { @apply min-w-[10rem] max-w-[32rem] whitespace-normal; overflow-wrap:anywhere; }
|
||||
.cell-tight { @apply whitespace-nowrap; }
|
||||
|
||||
.filters { @apply flex flex-wrap items-end gap-x-4 gap-y-3 border-y border-rule py-3 font-sans text-sm; }
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ head.source_url }}">{{ head.title }}</a></h1>{% match head.byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% let source = head.source %}{% include "_source.html" %} · {{ head.meta_line }}</p>{% let understanding = head.understanding %}{% include "_understanding.html" %}{% match head.summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match head.social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if head.excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
|
||||
<div class="prose-body mt-10 border-t border-rule pt-8">{{ head.body_html|safe }}</div>
|
||||
@@ -1,16 +1,15 @@
|
||||
<tr>
|
||||
<td class="cell-wrap"><details class="signals"><summary><a href="{{ candidate.href }}">{{ candidate.title }}</a></summary>{% let signals = candidate.signals %}{% include "_signals_table.html" %}</details></td>
|
||||
<td class="cell-tight text-muted">{{ candidate.feed }}</td>
|
||||
<td class="text-muted">{{ candidate.feed }}</td>
|
||||
<td class="num">{{ candidate.words }}</td>
|
||||
<td class="cell-tight"><span class="badge {{ candidate.stage }}">{{ candidate.stage }}</span></td>
|
||||
<td class="cell-tight">{% if let Some(reason) = candidate.reason %}<span class="badge reason">{{ reason }}</span>{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{% if let Some(first) = candidate.admitted_first %}{{ first }}{% if !candidate.admitted_rest.is_empty() %} <span class="muted text-xs">({{ candidate.admitted_rest }})</span>{% endif %}{% endif %}</td>
|
||||
<td><span class="badge {{ candidate.stage }}">{{ candidate.stage }}</span></td>
|
||||
<td>{% if let Some(reason) = candidate.reason %}<span class="badge reason">{{ reason }}</span>{% endif %}</td>
|
||||
<td class="text-muted">{% if let Some(first) = candidate.admitted_first %}{{ first }}{% if !candidate.admitted_rest.is_empty() %} <span class="muted text-xs">({{ candidate.admitted_rest }})</span>{% endif %}{% endif %}</td>
|
||||
<td class="num">{{ candidate.utility }}</td>
|
||||
<td class="num">{{ candidate.rank }}</td>
|
||||
<td class="num">{{ candidate.cluster }}</td>
|
||||
<td class="num">{{ candidate.triage }}</td>
|
||||
<td class="num">{{ candidate.quality }}</td>
|
||||
<td class="num">{{ candidate.fit }}</td>
|
||||
<td class="cell-tight">{% if candidate.exploration %}<span class="badge">exploration</span>{% endif %}{% if candidate.auto_include %}<span class="badge">auto</span>{% endif %}</td>
|
||||
<td class="cell-wrap text-muted">{% if let Some(why) = candidate.editor_why %}{{ why }}{% endif %}</td>
|
||||
<td>{% if candidate.exploration %}<span class="badge">exploration</span>{% endif %}{% if candidate.auto_include %}<span class="badge">auto</span>{% endif %}</td>
|
||||
</tr>
|
||||
@@ -0,0 +1 @@
|
||||
{% if !related.is_empty() %}<section class="mt-12" aria-labelledby="related-heading"><h2 id="related-heading" class="reader-section-heading">Related</h2><ul class="m-0 list-none p-0">{% for item in related %}<li class="border-b border-rule py-5"><h3 class="font-serif text-xl font-semibold leading-[1.15] tracking-[-0.01em]"><a class="text-ink no-underline hover:text-accent" href="{{ item.href }}">{{ item.title }}</a></h3><p class="mt-1.5 font-sans text-sm text-muted">{% match item.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{% let source = item.source %}{% include "_source.html" %} · {{ item.reading_minutes }} min read · {{ item.date }}</p></li>{% endfor %}</ul></section>{% endif %}
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="scroll-x"><table class="signals"><thead><tr><th>signal</th><th class="num">raw</th><th class="num">norm</th><th class="num">weight</th><th>present</th></tr></thead>
|
||||
<tbody>{% for line in signals.lines %}<tr{% if !line.present %} class="muted"{% endif %}><td>{{ line.name }}</td><td class="num">{{ line.raw }}</td><td class="num">{{ line.norm }}</td><td class="num">{{ line.weight }}</td><td>{% if line.present %}yes{% else %}<span class="muted">absent</span>{% endif %}</td></tr>{% endfor %}</tbody></table></div>
|
||||
<p class="muted">Preliminary blend <span class="tabular-nums text-ink">{{ signals.blend }}</span>{% if let Some(cos) = signals.top1_cos %} · interest top-1 cosine <span class="tabular-nums">{{ cos }}</span>{% endif %}{% if signals.exploration %} · <span class="badge">exploration</span>{% endif %}{% if signals.auto_include %} · <span class="badge">auto-include</span>{% endif %}{% if signals.slop_author %} · <span class="badge down">slop author</span>{% endif %}</p>
|
||||
{% if !signals.top_interests.is_empty() %}<p class="page-eyebrow">Top interests</p><ul>{% for interest in signals.top_interests %}<li>{{ interest.name }} <span class="muted">· z {{ interest.z }} · cos {{ interest.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
||||
{% if !signals.top_interests.is_empty() %}<p class="page-eyebrow">Top interests</p><ul>{% for interest in signals.top_interests %}<li><a href="{{ interest.href }}">{{ interest.name }}</a> <span class="muted">· z {{ interest.z }} · cos {{ interest.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
||||
{% if !signals.neighbours.is_empty() %}<p class="page-eyebrow">Nearest rated neighbours</p><ul>{% for neighbour in signals.neighbours %}<li><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span> <a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a> <span class="muted">· cos {{ neighbour.cos }}</span></li>{% endfor %}</ul>{% endif %}
|
||||
{% if !signals.notes.is_empty() %}<ul class="muted">{% for note in signals.notes %}<li>{{ note }}</li>{% endfor %}</ul>{% endif %}
|
||||
</div>{% endif %}
|
||||
@@ -1 +1 @@
|
||||
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}<p class="mt-3 font-sans text-sm text-muted">{% if let Some(kicker) = understanding.kicker %}<span class="text-[0.72rem] uppercase tracking-[0.12em]">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}<span class="ml-3">{{ topics }}</span>{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}<span>{{ topics }}</span>{% endif %}{% endif %}</p>{% endif %}{% if why.is_some() || understanding.interests.is_some() %}<div class="mt-4 border-l-2 border-accent pl-3">{% if let Some(why) = why %}<p class="italic text-ink-2">Why it's here: {{ why }}</p>{% endif %}{% if let Some(interests) = understanding.interests %}<p class="mt-1 font-sans text-sm text-muted">Matches: {{ interests }}</p>{% endif %}</div>{% endif %}
|
||||
{% if understanding.kicker.is_some() || understanding.topics.is_some() %}<p class="mt-3 font-sans text-sm text-muted">{% if let Some(kicker) = understanding.kicker %}<span class="text-[0.72rem] uppercase tracking-[0.12em]">{{ kicker }}</span>{% if let Some(topics) = understanding.topics %}<span class="ml-3">{{ topics }}</span>{% endif %}{% else %}{% if let Some(topics) = understanding.topics %}<span>{{ topics }}</span>{% endif %}{% endif %}</p>{% endif %}{% if !understanding.interests.is_empty() %}<p class="mt-2.5 flex flex-wrap items-center gap-1.5 font-sans"><span class="sr-only">Matched interests: </span>{% for interest in understanding.interests %}{% if page.is_admin() %}<a class="interest-chip" href="{{ interest.href }}">{{ interest.name }}</a>{% else %}<span class="interest-chip">{{ interest.name }}</span>{% endif %}{% endfor %}</p>{% endif %}
|
||||
@@ -1,8 +1,8 @@
|
||||
{% extends "layout.html" %}{% block ears %}<span class="sm:hidden">{{ toc.short_date }}</span><span class="hidden sm:inline">{{ toc.display_date }} · No. {{ toc.issue_number }}</span>{% endblock %}{% block content %}<div class="mx-auto max-w-7xl lg:grid lg:grid-cols-[15rem_minmax(0,1fr)] lg:gap-x-8 lg:px-6 xl:grid-cols-[18rem_minmax(0,1fr)] xl:gap-x-14">{% include "_toc.html" %}
|
||||
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0" data-toc-scroll>
|
||||
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ source_url }}">{{ title }}</a></h1>{% match byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% include "_source.html" %} · {{ meta_line }}</p>{% include "_understanding.html" %}{% match summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
|
||||
<div class="prose-body mt-10 border-t border-rule pt-8">{{ body_html|safe }}</div>
|
||||
{% include "_article_head.html" %}
|
||||
{% match discussion_html %}{% when Some with (discussion) %}<section class="discussion prose-body mt-12 border-t border-rule pt-6 [&_.comment-meta]:font-sans [&_.comment-meta]:text-sm [&_.comment-meta]:text-muted [&_blockquote.reply]:ml-6"><h2 class="mb-6 border-0 pt-0 text-2xl">Discussion</h2>{{ discussion|safe }}</section>{% when None %}{% endmatch %}
|
||||
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
|
||||
{% include "_related.html" %}
|
||||
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
|
||||
</article>
|
||||
</div>{% endblock %}
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends "layout.html" %}{% block ears %}Article{% endblock %}{% block content %}
|
||||
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0">
|
||||
{% include "_article_head.html" %}
|
||||
<footer class="mt-12 border-t border-rule pt-6"><p class="flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p></footer>
|
||||
</article>{% endblock %}
|
||||
@@ -43,22 +43,21 @@
|
||||
|
||||
<h2>Run history</h2>
|
||||
{% if history.is_empty() %}<p class="muted text-sm">Never considered by a run.</p>{% else %}<div class="scroll-x"><table class="history">
|
||||
<thead><tr><th>run</th><th>date</th><th>stage</th><th>reason</th><th>admitted by</th><th class="num">utility</th><th class="num">rank</th><th class="num">cluster</th><th>editor</th></tr></thead>
|
||||
<thead><tr><th>run</th><th>date</th><th>stage</th><th>reason</th><th>admitted by</th><th class="num">utility</th><th class="num">rank</th><th class="num">cluster</th></tr></thead>
|
||||
<tbody>{% for row in history %}<tr>
|
||||
<td class="cell-wrap"><details class="signals"><summary><a href="{{ row.run_href }}">run {{ row.run_id }}</a> <span class="badge {{ row.status }}">{{ row.status }}</span></summary>{% let signals = row.signals %}{% include "_signals_table.html" %}</details></td>
|
||||
<td class="cell-tight text-muted">{{ row.date }}</td>
|
||||
<td class="cell-tight"><span class="badge {{ row.stage }}">{{ row.stage }}</span></td>
|
||||
<td class="cell-tight text-muted">{% if let Some(reason) = row.reason %}{{ reason }}{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{% if let Some(first) = row.admitted_first %}{{ first }}{% if !row.admitted_rest.is_empty() %} <span class="muted text-xs">({{ row.admitted_rest }})</span>{% endif %}{% endif %}</td>
|
||||
<td><span class="badge {{ row.stage }}">{{ row.stage }}</span></td>
|
||||
<td class="text-muted">{% if let Some(reason) = row.reason %}{{ reason }}{% endif %}</td>
|
||||
<td class="text-muted">{% if let Some(first) = row.admitted_first %}{{ first }}{% if !row.admitted_rest.is_empty() %} <span class="muted text-xs">({{ row.admitted_rest }})</span>{% endif %}{% endif %}</td>
|
||||
<td class="num">{{ row.utility }}</td>
|
||||
<td class="num">{{ row.rank }}</td>
|
||||
<td class="num">{{ row.cluster }}</td>
|
||||
<td class="cell-wrap text-muted">{% if let Some(why) = row.editor_why %}{{ why }}{% endif %}</td>
|
||||
</tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
|
||||
<h2>Neighbours and interests</h2>
|
||||
{% if let Some(signals) = latest_signals %}{% if signals.top_interests.is_empty() && signals.neighbours.is_empty() %}<p class="muted text-sm">The latest run recorded no interests or neighbours for this article.</p>{% else %}<div class="cards">
|
||||
<section class="card"><h3>Top interests</h3>{% if signals.top_interests.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>interest</th><th class="num">z</th><th class="num">cos</th></tr></thead><tbody>{% for interest in signals.top_interests %}<tr><td>{{ interest.name }}</td><td class="num">{{ interest.z }}</td><td class="num">{{ interest.cos }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
||||
<section class="card"><h3>Top interests</h3>{% if signals.top_interests.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>interest</th><th class="num">z</th><th class="num">cos</th></tr></thead><tbody>{% for interest in signals.top_interests %}<tr><td class="cell-wrap"><a href="{{ interest.href }}">{{ interest.name }}</a></td><td class="num">{{ interest.z }}</td><td class="num">{{ interest.cos }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
||||
<section class="card"><h3>Nearest rated neighbours</h3>{% if signals.neighbours.is_empty() %}<p class="muted">None.</p>{% else %}<div class="scroll-x"><table><thead><tr><th>label</th><th class="num">cos</th><th>article</th></tr></thead><tbody>{% for neighbour in signals.neighbours %}<tr><td><span class="badge {{ neighbour.label }}">{{ neighbour.label }}</span></td><td class="num">{{ neighbour.cos }}</td><td class="cell-wrap"><a href="/dashboard/articles/{{ neighbour.article_id }}">{{ neighbour.title }}</a></td></tr>{% endfor %}</tbody></table></div>{% endif %}</section>
|
||||
</div>{% endif %}{% else %}<p class="muted text-sm">No signals recorded.</p>{% endif %}
|
||||
|
||||
@@ -68,9 +67,9 @@
|
||||
<tbody>{% for article in articles %}<tr>
|
||||
<td class="num">{{ article.cosine }}</td>
|
||||
<td class="cell-wrap"><a href="/dashboard/articles/{{ article.id }}">{{ article.title }}</a></td>
|
||||
<td class="cell-tight">{{ article.feed }}</td>
|
||||
<td>{{ article.feed }}</td>
|
||||
<td class="cell-tight text-muted">{{ article.first_seen }}</td>
|
||||
<td class="cell-tight">{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% else %}<span class="muted">unrated</span>{% endif %}</td>
|
||||
<td>{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% else %}<span class="muted">unrated</span>{% endif %}</td>
|
||||
</tr>{% endfor %}</tbody></table></div>{% endif %}{% else %}<p class="muted text-sm">No embedding stored — run features-backfill.</p>{% endif %}
|
||||
|
||||
<h2>Embedding</h2>
|
||||
@@ -81,12 +80,12 @@
|
||||
<thead><tr><th>when</th><th>label</th><th class="num">value</th><th>kind</th><th>source</th><th>user</th><th>issue</th><th>note</th></tr></thead>
|
||||
<tbody>{% for event in events %}<tr>
|
||||
<td class="cell-tight text-muted">{{ event.event_at }}</td>
|
||||
<td class="cell-tight"><span class="badge {{ event.label_class }}">{{ event.label }}</span></td>
|
||||
<td><span class="badge {{ event.label_class }}">{{ event.label }}</span></td>
|
||||
<td class="num">{{ event.value }}</td>
|
||||
<td class="cell-tight text-muted">{{ event.kind }}</td>
|
||||
<td class="cell-tight text-muted">{{ event.source }}</td>
|
||||
<td class="cell-tight">{% if let Some(user) = event.user %}{{ user }}{% else %}<span class="muted">—</span>{% endif %}</td>
|
||||
<td class="text-muted">{{ event.kind }}</td>
|
||||
<td class="text-muted">{{ event.source }}</td>
|
||||
<td>{% if let Some(user) = event.user %}{{ user }}{% else %}<span class="muted">—</span>{% endif %}</td>
|
||||
<td class="cell-tight">{% if let Some(date) = event.issue_date %}<a href="/issues/{{ date }}">{{ date }}</a>{% endif %}</td>
|
||||
<td class="cell-wrap text-muted">{% if let Some(note) = event.note %}{{ note }}{% endif %}</td>
|
||||
<td class="cell-wrap text-muted">{% if let Some(note) = event.note %}<div class="line-clamp-3" title="{{ note }}">{{ note }}</div>{% endif %}</td>
|
||||
</tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
</section>{% endblock %}
|
||||
@@ -6,6 +6,7 @@
|
||||
<form class="filters" method="get" action="/dashboard/articles">
|
||||
<label>Title or URL <input type="search" name="q" value="{% if let Some(q) = filters.q %}{{ q }}{% endif %}" placeholder="contains…"></label>
|
||||
<label>Feed id <input type="number" name="feed" value="{{ feed_value }}" min="1"></label>
|
||||
<label>Interest <select name="interest"><option value="">any</option>{% for interest in interest_options %}<option value="{{ interest.name }}"{% if let Some(current) = filters.interest %}{% if current.name.as_str() == interest.name.as_str() %} selected{% endif %}{% endif %}>{{ interest.name }}</option>{% endfor %}</select></label>
|
||||
<label>Stage <select name="stage"><option value="">any</option>{% for name in stages %}<option value="{{ name }}"{% if let Some(current) = filters.stage %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||
<label>Reason <select name="reason"><option value="">any</option>{% for name in reasons %}<option value="{{ name }}"{% if let Some(current) = filters.reason %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||
<label>Rated <select name="rated"><option value="">all</option>{% for name in rated %}<option value="{{ name }}"{% if let Some(current) = filters.rated %}{% if current.as_str() == *name %} selected{% endif %}{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||
@@ -17,21 +18,22 @@
|
||||
<div class="filter-actions"><button class="btn" type="submit">Apply</button> <a class="btn-quiet" href="/dashboard/articles">Reset</a></div>
|
||||
</form>
|
||||
{% include "dashboard/_pager.html" %}
|
||||
{% if articles.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table class="articles" data-filter>
|
||||
<thead><tr><th>first seen</th><th>title</th><th>feed</th><th class="num">words</th><th>last stage</th><th>reason</th><th class="num">utility</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>rating</th><th>published</th></tr></thead>
|
||||
{% if articles.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table class="articles" data-filter>
|
||||
<thead><tr><th>first seen</th><th>title</th><th>feed</th><th>interests</th><th class="num">words</th><th>last stage</th><th>reason</th><th class="num">utility</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>rating</th><th>published</th></tr></thead>
|
||||
<tbody>{% for article in articles %}<tr>
|
||||
<td class="cell-tight text-muted">{{ article.first_seen }}</td>
|
||||
<td class="cell-wrap"><a href="{{ article.href }}">{{ article.title }}</a></td>
|
||||
<td class="cell-tight">{% if let Some(feed_id) = article.feed_id %}<a href="/dashboard/articles?feed={{ feed_id }}">{{ article.feed }}</a>{% else %}{{ article.feed }}{% endif %}</td>
|
||||
<td>{% if let Some(feed_id) = article.feed_id %}<a href="/dashboard/articles?feed={{ feed_id }}">{{ article.feed }}</a>{% else %}{{ article.feed }}{% endif %}</td>
|
||||
<td>{% for interest in article.interests %}<a class="badge" href="{{ interest.href }}">{{ interest.name }}</a>{% if !loop.last %} {% endif %}{% endfor %}</td>
|
||||
<td class="num">{{ article.words }}</td>
|
||||
<td class="cell-tight">{% if let Some(stage) = article.stage %}<span class="badge {{ stage }}">{{ stage }}</span>{% if let Some(run_id) = article.run_id %} <a class="muted text-xs" href="/dashboard/runs/{{ run_id }}">{% if let Some(date) = article.run_date %}{{ date }}{% else %}run {{ run_id }}{% endif %}</a>{% endif %}{% else %}<span class="muted text-xs">never considered</span>{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{% if let Some(reason) = article.reason %}{{ reason }}{% endif %}</td>
|
||||
<td>{% if let Some(stage) = article.stage %}<span class="badge {{ stage }}">{{ stage }}</span>{% if let Some(run_id) = article.run_id %} <a class="muted text-xs" href="/dashboard/runs/{{ run_id }}">{% if let Some(date) = article.run_date %}{{ date }}{% else %}run {{ run_id }}{% endif %}</a>{% endif %}{% else %}<span class="muted text-xs">never considered</span>{% endif %}</td>
|
||||
<td class="text-muted">{% if let Some(reason) = article.reason %}{{ reason }}{% endif %}</td>
|
||||
<td class="num">{{ article.utility }}</td>
|
||||
<td class="num">{{ article.triage }}</td>
|
||||
<td class="num">{{ article.quality }}</td>
|
||||
<td class="num">{{ article.fit }}</td>
|
||||
<td class="cell-tight">{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% endif %}</td>
|
||||
<td>{% if let Some(rating) = article.rating %}<span class="badge {{ article.rating_class }}">{{ rating }}</span>{% endif %}</td>
|
||||
<td class="cell-tight">{% if let Some(date) = article.published %}<a href="/issues/{{ date }}">{{ date }}</a>{% endif %}</td>
|
||||
</tr>{% endfor %}{% if articles.is_empty() %}<tr><td colspan="12" class="text-muted">No articles match this filter.</td></tr>{% endif %}</tbody></table></div>
|
||||
</tr>{% endfor %}{% if articles.is_empty() %}<tr><td colspan="13" class="text-muted">No articles match this filter.</td></tr>{% endif %}</tbody></table></div>
|
||||
{% include "dashboard/_pager.html" %}
|
||||
</section>{% endblock %}
|
||||
@@ -6,12 +6,12 @@
|
||||
<nav class="tabs" aria-label="Candidate status"><a href="/dashboard/feeds"{% if status == "candidate" %} class="active" aria-current="page"{% endif %}>Candidates ({{ candidate_count }})</a> <a href="/dashboard/feeds?status=added"{% if status == "added" %} class="active" aria-current="page"{% endif %}>Added ({{ added_count }})</a> <a href="/dashboard/feeds?status=dismissed"{% if status == "dismissed" %} class="active" aria-current="page"{% endif %}>Dismissed ({{ dismissed_count }})</a></nav>
|
||||
{% if let Some(error) = categories_error %}<p class="notice">{{ error }}</p>{% endif %}
|
||||
{% include "dashboard/_pager.html" %}
|
||||
<div class="scroll-x"><table class="table-fixed">
|
||||
<div class="scroll-x"><table>
|
||||
<thead><tr><th class="num w-14">Score</th><th>Feed</th><th>Why</th><th class="num w-20">Articles</th><th class="w-20">Seen</th><th class="w-60 lg:w-84"><span class="sr-only">Actions</span></th></tr></thead>
|
||||
<tbody>{% for row in rows %}<tr>
|
||||
<td class="num tabular-nums">{{ row.score }}</td>
|
||||
<td class="cell-wrap"><a class="line-clamp-2" href="{{ row.feed_url }}" rel="noopener" target="_blank" title="{{ row.feed_url }}">{{ row.label }}</a><span class="muted block truncate text-xs" title="{{ row.host }}">{{ row.host }}</span></td>
|
||||
<td class="cell-wrap">{% for interest in row.interests %}<span class="badge">{{ interest }}</span> {% endfor %}{% for article in row.articles %}<div class="line-clamp-1 text-xs"><a href="/dashboard/articles/{{ article.id }}" title="{{ article.title }}">{{ article.title }}</a></div>{% endfor %}</td>
|
||||
<td class="cell-wrap">{% for interest in row.interests %}<a class="badge" href="{{ interest.href }}">{{ interest.name }}</a> {% endfor %}{% for article in row.articles %}<div class="line-clamp-1 text-xs"><a href="/dashboard/articles/{{ article.id }}" title="{{ article.title }}">{{ article.title }}</a></div>{% endfor %}</td>
|
||||
<td class="num">{{ row.article_count }}</td>
|
||||
<td class="cell-tight text-xs text-muted"><span title="First seen {{ row.first_seen }} · last seen {{ row.last_seen }}">{{ row.seen }}</span></td>
|
||||
<td>{% if status == "candidate" %}<div class="flex flex-wrap items-start gap-2"><form class="form-inline flex-nowrap" method="post" action="/dashboard/feeds/{{ row.id }}/add"><select class="min-w-0 max-w-36 truncate" name="category_id" aria-label="Category for {{ row.label }}"{% if categories.is_empty() %} disabled{% endif %}>{% for category in categories %}<option value="{{ category.id }}"{% if category.id == selected_category %} selected{% endif %}>{{ category.title }}</option>{% endfor %}</select><button class="btn-primary" type="submit"{% if categories.is_empty() %} disabled title="Miniflux categories are unavailable"{% endif %}>Add</button></form><form method="post" action="/dashboard/feeds/{{ row.id }}/dismiss"><button class="btn" type="submit">Dismiss</button></form></div>{% else %}<span class="muted text-xs">{{ row.decided_at }}{% if let Some(href) = row.miniflux_href %} · <a href="{{ href }}" rel="noopener" target="_blank">in Miniflux</a>{% endif %}</span>{% endif %}</td>
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{% extends "layout.html" %}{% block content %}<section class="dashboard interests">
|
||||
<header class="page-head"><div>
|
||||
<h1>Interests</h1>
|
||||
<p class="page-desc">{{ total }} standing interests. Weights use the last {{ lookback_days }} days of ratings with a {{ half_life_days }}-day half-life · affinity gate {{ affinity_gate }} · {{ attributable }} of {{ affinity_full }} attributable ratings.</p>
|
||||
</div><div class="page-actions"><span class="badge">{{ uncategorized }} uncategorized</span><form method="post" action="/dashboard/jobs/interests-categorize" class="form-inline"><button class="btn" type="submit"{% if !jobs_enabled %} disabled{% endif %}>Categorize now</button>{% if !jobs_enabled %} <span class="meta text-xs">Jobs are disabled on this server (<code>server.jobs_enabled = false</code>); run <code>daily-epub job run interests-categorize</code> instead.</span>{% endif %}</form></div></header>
|
||||
|
||||
<section class="card"><h2>Add an interest</h2><form class="filters" method="post" action="/dashboard/interests">
|
||||
<label>Interest <input name="name" type="text" minlength="1" maxlength="80" required placeholder="e.g. Rust macros"></label>
|
||||
<label>Category <select name="category"><option value="">let the categorizer decide</option>{% for category in categories %}<option value="{{ category.name }}">{{ category.name }}</option>{% endfor %}</select></label>
|
||||
<div class="filter-actions"><button class="btn btn-primary" type="submit">Add</button></div>
|
||||
</form></section>
|
||||
|
||||
<form class="filters" method="get" action="/dashboard/interests">
|
||||
<label>Category <select name="category"><option value="">all</option><option value="uncategorized"{% if selected_category == "uncategorized" %} selected{% endif %}>uncategorized</option>{% for category in categories %}<option value="{{ category.name }}"{% if selected_category == category.name.as_str() %} selected{% endif %}>{{ category.name }}</option>{% endfor %}</select></label>
|
||||
<label>Sort <select name="sort">{% for sort in sorts %}<option value="{{ sort }}"{% if selected_sort == *sort %} selected{% endif %}>{{ sort }}</option>{% endfor %}</select></label>
|
||||
<div class="filter-actions"><button class="btn" type="submit">Apply</button> <a class="btn-quiet" href="/dashboard/interests">Reset</a></div>
|
||||
</form>
|
||||
{% if rows.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter interests" aria-label="Filter interests" data-table-filter>{% endif %}
|
||||
<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th>interest</th><th>category</th><th class="num">weight</th><th class="num">up</th><th class="num">down</th><th class="num">rated matches</th><th class="num">matched articles</th><th>added</th><th>actions</th></tr></thead>
|
||||
<tbody>{% for row in rows %}<tr>
|
||||
<td class="cell-wrap"><a href="{{ row.href }}">{{ row.name }}</a></td>
|
||||
<td>{% if let Some(category) = row.category %}<a href="{{ row.category_href }}">{{ category }}</a>{% else %}<a class="muted" href="{{ row.category_href }}">uncategorized</a>{% endif %}</td>
|
||||
<td class="num tabular-nums">{{ row.weight }}</td>
|
||||
<td class="num tabular-nums">{{ row.up }}</td>
|
||||
<td class="num tabular-nums">{{ row.down }}</td>
|
||||
<td class="num tabular-nums">{{ row.rated_matches }}</td>
|
||||
<td class="num tabular-nums">{{ row.matched_articles }}</td>
|
||||
<td class="cell-tight text-muted">{{ row.added }}</td>
|
||||
<td><div class="flex flex-wrap items-start gap-2"><form class="form-inline flex-nowrap" method="post" action="/dashboard/interests/{{ row.id }}/category"><select class="min-w-0 max-w-44 truncate" name="category" aria-label="Category for {{ row.name }}"><option value="">uncategorized</option>{% for category in categories %}<option value="{{ category.name }}"{% if let Some(current) = row.category %}{% if current.as_str() == category.name.as_str() %} selected{% endif %}{% endif %}>{{ category.name }}</option>{% endfor %}</select><button class="btn" type="submit">Save</button></form><form method="post" action="/dashboard/interests/{{ row.id }}/delete" onsubmit="return confirm('Delete this interest? Its article matches and cached embedding will also be removed.')"><button class="btn" type="submit">Delete</button></form></div></td>
|
||||
</tr>{% endfor %}{% if rows.is_empty() %}<tr><td colspan="9" class="text-muted">No interests match this filter.</td></tr>{% endif %}</tbody>
|
||||
</table></div>
|
||||
</section>{% endblock %}
|
||||
@@ -17,18 +17,18 @@
|
||||
</form>
|
||||
</section>{% endfor %}</div>
|
||||
<h2>History</h2>
|
||||
{% if jobs.is_empty() %}<p class="muted text-sm">No jobs recorded yet.</p>{% else %}{% if jobs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table data-filter>
|
||||
{% if jobs.is_empty() %}<p class="muted text-sm">No jobs recorded yet.</p>{% else %}{% if jobs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th class="num">job</th><th>name</th><th>requested by</th><th>requested</th><th>started</th><th>finished</th><th class="num">duration</th><th>status</th><th>message</th><th class="num">run</th></tr></thead>
|
||||
<tbody>{% for job in jobs %}<tr>
|
||||
<td class="num"><a href="/dashboard/jobs/{{ job.id }}">{{ job.id }}</a></td>
|
||||
<td class="cell-tight font-mono text-xs">{{ job.name }}</td>
|
||||
<td class="cell-tight text-muted">{{ job.requested_by }}</td>
|
||||
<td class="font-mono text-xs">{{ job.name }}</td>
|
||||
<td class="text-muted">{{ job.requested_by }}</td>
|
||||
<td class="cell-tight text-muted">{{ job.requested }}</td>
|
||||
<td class="cell-tight text-muted">{{ job.started }}</td>
|
||||
<td class="cell-tight text-muted">{{ job.finished }}</td>
|
||||
<td class="num">{{ job.duration }}</td>
|
||||
<td class="cell-tight"><span class="badge {{ job.status }}">{{ job.status }}</span></td>
|
||||
<td class="message cell-wrap text-muted">{% if let Some(message) = job.message %}{{ message }}{% endif %}</td>
|
||||
<td><span class="badge {{ job.status }}">{{ job.status }}</span></td>
|
||||
<td class="message cell-wrap text-muted">{% if let Some(message) = job.message %}<div class="line-clamp-3" title="{{ message }}">{{ message }}</div>{% endif %}</td>
|
||||
<td class="num">{% if let Some(run_id) = job.run_id %}<a href="/dashboard/runs/{{ run_id }}">{{ run_id }}</a>{% endif %}</td>
|
||||
</tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
</section>{% endblock %}
|
||||
@@ -11,6 +11,7 @@
|
||||
<div class="tile"><span class="tile-label">Unrated picks</span><span class="tile-num">{{ unrated.len() }}</span><span class="tile-delta">from the last three issues</span></div>
|
||||
<div class="tile"><span class="tile-label">Active jobs</span><span class="tile-num">{{ active_jobs.len() }}</span><span class="tile-delta"><a href="/dashboard/jobs">all jobs</a></span></div>
|
||||
<div class="tile"><span class="tile-label">Feed candidates</span><span class="tile-num">{{ feed_candidates }}</span><span class="tile-delta"><a href="/dashboard/feeds">review feeds</a></span></div>
|
||||
<div class="tile"><span class="tile-label">Interests</span><span class="tile-num">{{ interests_total }}</span><span class="tile-delta"><a href="/dashboard/interests?category=uncategorized">{{ uncategorized_interests }} uncategorized</a></span></div>
|
||||
<div class="tile"><span class="tile-label">Access requests</span><span class="tile-num">{{ access_requests }}</span><span class="tile-delta"><a href="/dashboard/users">review requests</a></span></div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{% extends "layout.html" %}{% block content %}<section class="dashboard profile">
|
||||
<header class="page-head"><div>
|
||||
<h1>Profile</h1>
|
||||
<p class="page-desc">The standing taste file the curator reads before every run: what you like, what the OPML declares, and what the editor model has learned from your verdicts.</p>
|
||||
<p class="page-desc">The standing taste file the curator reads before every run, alongside stored interests and what the editor model has learned from your verdicts.</p>
|
||||
</div><div class="page-actions"><a class="btn" href="/dashboard/ratings">Ratings</a><a class="btn" href="/dashboard/settings#curation.feedback">Feedback settings</a></div></header>
|
||||
|
||||
<div class="profile-grid grid gap-6 lg:grid-cols-2">
|
||||
<div class="profile-editor min-w-0">
|
||||
<h3 class="font-mono text-base">profile.md</h3>
|
||||
<p class="meta mt-1 text-xs"><code>{{ path }}</code>{% if !exists %} — <strong class="text-down">missing</strong>; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any <code>## Interests</code> section is parsed one interest per line; everything else goes into the system prompt verbatim.</p>
|
||||
<p class="meta mt-1 text-xs"><code>{{ path }}</code>{% if !exists %} — <strong class="text-down">missing</strong>; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any <code>## Interests</code> section is ignored; everything else goes into the system prompt verbatim.</p>
|
||||
<form method="post" action="/dashboard/profile" class="profile-form mt-3">
|
||||
<textarea name="content" rows="24" spellcheck="true" required class="w-full font-mono text-sm">{{ content }}</textarea>
|
||||
<div class="mt-3 flex flex-wrap items-center gap-3"><button class="btn-primary" type="submit">Save</button> <span class="meta text-xs">The next run rebuilds the system prompt from the saved file.</span></div>
|
||||
@@ -19,8 +19,6 @@
|
||||
<p class="meta mt-1 text-xs">A live read of the text on the left, exactly as <code>curate::profile</code> splits it.</p>
|
||||
<h4 class="mt-4 page-eyebrow">Passthrough sections</h4>
|
||||
{% if preview_body.trim().is_empty() %}<p class="meta mt-1 text-sm">Nothing passes through — the file is empty or only has an Interests section.</p>{% else %}<pre class="preview mt-2">{{ preview_body }}</pre>{% endif %}
|
||||
<h4 class="mt-5 page-eyebrow">Extracted <code>## Interests</code> lines</h4>
|
||||
{% if preview_interests.is_empty() %}<p class="meta mt-1 text-sm">None. The prompt uses the OPML interests alone.</p>{% else %}<ul class="mt-2 list-disc space-y-1 pl-5 text-sm marker:text-muted">{% for interest in preview_interests %}<li>{{ interest }}</li>{% endfor %}</ul>{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -32,17 +30,16 @@
|
||||
<tbody>{% for version in versions %}<tr>
|
||||
<td class="num">#{{ version.id }}</td>
|
||||
<td class="cell-tight text-muted">{{ version.saved_at }}</td>
|
||||
<td class="cell-tight">{{ version.saved_by }}</td>
|
||||
<td>{{ version.saved_by }}</td>
|
||||
<td class="num text-muted">{{ version.bytes }} bytes</td>
|
||||
<td><pre class="preview">{{ version.preview }}</pre></td>
|
||||
<td class="cell-tight"><form method="post" action="/dashboard/profile/restore" data-confirm="Restore version #{{ version.id }}? The current file is kept as a new version."><input type="hidden" name="version_id" value="{{ version.id }}"><button class="btn" type="submit">Restore</button></form></td>
|
||||
<td class="cell-wrap"><pre class="preview">{{ version.preview }}</pre></td>
|
||||
<td><form method="post" action="/dashboard/profile/restore" data-confirm="Restore version #{{ version.id }}? The current file is kept as a new version."><input type="hidden" name="version_id" value="{{ version.id }}"><button class="btn" type="submit">Restore</button></form></td>
|
||||
</tr>{% endfor %}</tbody></table></div>
|
||||
{% endif %}
|
||||
|
||||
<h2>Standing interests</h2>
|
||||
<p class="meta text-sm"><code>{{ opml_path }}</code> · {{ opml_count }} interests, grouped the way the system prompt lists them. The union of these and the <code>## Interests</code> lines above is what the prompt uses; edit the OPML file to change them.</p>
|
||||
{% if !opml_error.is_empty() %}<p class="error">{{ opml_error }}</p>{% endif %}
|
||||
{% if !themes.is_empty() %}<dl class="kv themes">{% for theme in themes %}<dt class="text-ink">{{ theme.name }} <span class="meta">({{ theme.count }})</span></dt><dd class="text-muted">{{ theme.members }}</dd>{% endfor %}</dl>{% endif %}
|
||||
<p class="meta text-sm">{{ interest_count }} standing interests in {{ category_count }} categories — manage them on the <a href="/dashboard/interests">Interests page</a>.</p>
|
||||
{% if !categories.is_empty() %}<dl class="kv themes">{% for category in categories %}<dt class="text-ink">{{ category.name }} <span class="meta">({{ category.count }})</span></dt><dd class="text-muted">{{ category.members }}</dd>{% endfor %}</dl>{% endif %}
|
||||
|
||||
<h2>Learned adjustments</h2>
|
||||
<p class="meta text-sm">Rebuilt weekly from ratings by the editor model (prompt version {{ prompt_version }}, built {{ prompt_built_at }}, {{ learned_age }}). {% if rebuild_due %}<strong class="text-warn">A rebuild is due</strong> — the next run performs it, or start it now.{% else %}The next scheduled rebuild is at least {{ rebuild_interval_days }} days after the last one; the next run performs it when due.{% endif %}</p>
|
||||
|
||||
@@ -19,9 +19,9 @@
|
||||
<tbody>{% for row in imports %}<tr>
|
||||
<td class="cell-tight text-muted">{{ row.requested }}</td>
|
||||
<td class="cell-wrap"><a href="{{ row.url }}" rel="noopener" target="_blank">{{ row.url }}</a></td>
|
||||
<td class="cell-tight"><span class="badge {{ row.label }}">{{ row.verdict }}</span></td>
|
||||
<td class="cell-tight"><span class="badge {{ row.status }}">{{ row.status }}</span></td>
|
||||
<td class="cell-wrap text-muted">{{ row.message }}</td>
|
||||
<td><span class="badge {{ row.label }}">{{ row.verdict }}</span></td>
|
||||
<td><span class="badge {{ row.status }}">{{ row.status }}</span></td>
|
||||
<td class="cell-wrap text-muted"><div class="line-clamp-3" title="{{ row.message }}">{{ row.message }}</div></td>
|
||||
<td class="num">{% if let Some(article_id) = row.article_id %}<a href="/dashboard/articles/{{ article_id }}">{{ article_id }}</a>{% endif %}</td>
|
||||
</tr>{% endfor %}
|
||||
{% if imports.is_empty() %}<tr><td colspan="6" class="text-muted">No rating imports yet.</td></tr>{% endif %}</tbody>
|
||||
@@ -55,19 +55,19 @@
|
||||
<div class="filter-actions"><button class="btn" type="submit">Filter</button> <a class="btn-quiet" href="{{ events_href }}">Reset</a></div>
|
||||
</form>
|
||||
<p class="meta text-sm">{{ pagination.total }} events, newest first. History is append-only; a later explicit event for the same article marks the earlier one <em>superseded</em>. Use the clear verdict to retract.</p>
|
||||
<div class="scroll-x tall"><table>
|
||||
<div class="scroll-x"><table>
|
||||
<thead><tr><th class="num">Event</th><th>Verdict</th><th class="num">Value</th><th>Article</th><th>Issue</th><th>When</th><th>Source</th><th>By</th><th>Note</th></tr></thead>
|
||||
<tbody>
|
||||
{% for event in events %}<tr{% if event.superseded %} class="superseded"{% endif %}>
|
||||
<td class="num">#{{ event.id }}{% if event.kind != "explicit" %} <span class="badge">{{ event.kind }}</span>{% endif %}{% if event.superseded %} <span class="badge cleared">superseded</span>{% endif %}</td>
|
||||
<td class="cell-tight"><span class="badge {{ event.badge }}">{{ event.verdict }}</span></td>
|
||||
<td><span class="badge {{ event.badge }}">{{ event.verdict }}</span></td>
|
||||
<td class="num">{{ event.value }}</td>
|
||||
<td class="cell-wrap"><a href="/dashboard/articles/{{ event.article_id }}">{{ event.title }}</a></td>
|
||||
<td class="cell-tight">{% if !event.issue_date.is_empty() %}<a href="/issues/{{ event.issue_date }}">{{ event.issue_date }}</a>{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{{ event.when }}</td>
|
||||
<td class="cell-tight text-muted">{{ event.source }}</td>
|
||||
<td class="cell-tight text-muted">{{ event.username }}</td>
|
||||
<td class="cell-wrap text-muted">{{ event.note }}</td>
|
||||
<td class="text-muted">{{ event.source }}</td>
|
||||
<td class="text-muted">{{ event.username }}</td>
|
||||
<td class="cell-wrap text-muted"><div class="line-clamp-3" title="{{ event.note }}">{{ event.note }}</div></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% if events.is_empty() %}<tr><td colspan="9" class="text-muted">No rating events match.</td></tr>{% endif %}
|
||||
@@ -83,20 +83,20 @@
|
||||
<div class="filter-actions"><button class="btn" type="submit">Filter</button> <a class="btn-quiet" href="{{ current_href }}">Reset</a></div>
|
||||
</form>
|
||||
<p class="meta text-sm">One row per rated article (its latest explicit event), newest first. {% match last_run %}{% when Some with (run) %}"Used last run" counts the candidates of run #{{ run.id }} ({{ run.date }}, {{ run.status }}) that listed the article among their nearest rated neighbours, and how many of those were selected.{% when None %}No run has happened yet, so "Used last run" is empty.{% endmatch %}</p>
|
||||
<div class="scroll-x tall"><table class="contributions">
|
||||
<div class="scroll-x"><table class="contributions">
|
||||
<thead><tr><th>Verdict</th><th>Article</th><th>When · by</th><th>Note</th><th>Age → decay</th><th class="num">Neighbour weight</th><th>Feed credit</th><th>In prompt</th><th>In rebuild</th><th>Used last run</th></tr></thead>
|
||||
<tbody>
|
||||
{% for row in current %}<tr>
|
||||
<td class="rating-cell min-w-[13rem]">{% let widget = row.widget %}{% include "_rating_widget.html" %}</td>
|
||||
<td class="cell-wrap"><a href="/dashboard/articles/{{ row.article_id }}">{{ row.title }}</a><br><span class="meta text-xs">{{ row.feed_title }}{% if !row.issue_date.is_empty() %} · <a href="/issues/{{ row.issue_date }}">{{ row.issue_date }}</a>{% endif %}</span></td>
|
||||
<td class="cell-tight">{{ row.when }}<br><span class="meta text-xs">{{ row.source }} · <span class="by">{{ row.username }}</span></span></td>
|
||||
<td class="cell-wrap text-muted">{{ row.note }}</td>
|
||||
<td class="cell-wrap text-muted"><div class="line-clamp-3" title="{{ row.note }}">{{ row.note }}</div></td>
|
||||
<td class="cell-tight tabular-nums">{{ row.age_days }} d → {{ row.decay }}</td>
|
||||
<td class="num">{% if row.has_embedding && !row.neighbour_weight.is_empty() %}{{ row.neighbour_weight }}{% if row.beyond_lookback %} <span class="meta text-xs">(beyond lookback)</span>{% endif %}{% else if row.badge == "cleared" %}<span class="meta">—</span>{% else %}<span class="meta text-xs">no embedding</span>{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{% for credit in row.feed_credits %}{{ credit.title }} <span class="tabular-nums text-ink">{{ credit.credit }}</span>{% if !loop.last %}<br>{% endif %}{% endfor %}</td>
|
||||
<td class="cell-wrap text-muted">{% for credit in row.feed_credits %}{{ credit.title }} <span class="tabular-nums text-ink">{{ credit.credit }}</span>{% if !loop.last %}<br>{% endif %}{% endfor %}</td>
|
||||
<td class="text-center">{% if row.in_prompt %}<span class="text-loved">✓</span>{% endif %}</td>
|
||||
<td class="text-center">{% if row.in_rebuild %}<span class="text-loved">✓</span>{% endif %}</td>
|
||||
<td class="cell-tight tabular-nums">{% if row.used_total > 0 %}{{ row.used_total }} ({{ row.used_selected }} selected){% endif %}</td>
|
||||
<td class="tabular-nums">{% if row.used_total > 0 %}{{ row.used_total }} ({{ row.used_selected }} selected){% endif %}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% if current.is_empty() %}<tr><td colspan="10" class="text-muted">No current verdicts match.</td></tr>{% endif %}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<div class="scroll-x"><table class="funnel-table">
|
||||
<thead><tr><th>stage</th><th>share</th><th class="num">reached</th><th class="num">stopped here</th><th>why</th></tr></thead>
|
||||
<tbody>{% for stage in funnel %}<tr>
|
||||
<td class="cell-tight align-middle"><span class="badge {{ stage.stage }}">{{ stage.stage }}</span></td>
|
||||
<td class="align-middle"><span class="badge {{ stage.stage }}">{{ stage.stage }}</span></td>
|
||||
<td class="funnel-cell"><div class="funnel"><svg class="h-3" width="{{ stage.percent }}%" role="img" aria-label="{{ stage.reached }} reached"></svg></div></td>
|
||||
<td class="num align-middle">{{ stage.reached }}</td>
|
||||
<td class="num align-middle text-muted">{{ stage.stopped }}</td>
|
||||
@@ -45,7 +45,7 @@
|
||||
<h2>Near misses</h2>
|
||||
{% if near_misses.is_empty() %}<p class="muted text-sm">None recorded.</p>{% else %}<div class="scroll-x"><table>
|
||||
<thead><tr><th class="num">#</th><th class="num">score</th><th>title</th><th>feed</th><th class="num">quality</th><th class="num">fit</th><th>stage</th><th>reason</th></tr></thead>
|
||||
<tbody>{% for miss in near_misses %}<tr><td class="num text-muted">{{ loop.index }}</td><td class="num">{{ miss.score }}</td><td class="cell-wrap"><a href="{{ miss.href }}">{{ miss.title }}</a></td><td class="cell-tight text-muted">{{ miss.feed }}</td><td class="num">{{ miss.quality }}</td><td class="num">{{ miss.fit }}</td><td class="cell-tight"><span class="badge {{ miss.stage }}">{{ miss.stage }}</span></td><td class="cell-wrap text-muted">{% if let Some(reason) = miss.reason %}{{ reason }}{% endif %}</td></tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
<tbody>{% for miss in near_misses %}<tr><td class="num text-muted">{{ loop.index }}</td><td class="num">{{ miss.score }}</td><td class="cell-wrap"><a href="{{ miss.href }}">{{ miss.title }}</a></td><td class="text-muted">{{ miss.feed }}</td><td class="num">{{ miss.quality }}</td><td class="num">{{ miss.fit }}</td><td><span class="badge {{ miss.stage }}">{{ miss.stage }}</span></td><td class="cell-wrap text-muted">{% if let Some(reason) = miss.reason %}{{ reason }}{% endif %}</td></tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
|
||||
<h2>Candidates</h2>
|
||||
<form class="filters" method="get" action="/dashboard/runs/{{ run.id }}">
|
||||
@@ -58,8 +58,8 @@
|
||||
<div class="filter-actions"><button class="btn" type="submit">Apply</button> <a class="btn-quiet" href="/dashboard/runs/{{ run.id }}">Reset</a></div>
|
||||
</form>
|
||||
{% include "dashboard/_pager.html" %}
|
||||
{% if candidates.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table class="candidates" data-filter>
|
||||
<thead><tr><th>title</th><th>feed</th><th class="num">words</th><th>stage</th><th>reason</th><th>admitted by</th><th class="num">utility</th><th class="num">rank</th><th class="num">cluster</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>flags</th><th>editor</th></tr></thead>
|
||||
<tbody>{% for candidate in candidates %}{% include "_candidate_row.html" %}{% endfor %}{% if candidates.is_empty() %}<tr><td colspan="14" class="text-muted">No candidates match this filter.</td></tr>{% endif %}</tbody></table></div>
|
||||
{% if candidates.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table class="candidates" data-filter>
|
||||
<thead><tr><th>title</th><th>feed</th><th class="num">words</th><th>stage</th><th>reason</th><th>admitted by</th><th class="num">utility</th><th class="num">rank</th><th class="num">cluster</th><th class="num">triage</th><th class="num">quality</th><th class="num">fit</th><th>flags</th></tr></thead>
|
||||
<tbody>{% for candidate in candidates %}{% include "_candidate_row.html" %}{% endfor %}{% if candidates.is_empty() %}<tr><td colspan="13" class="text-muted">No candidates match this filter.</td></tr>{% endif %}</tbody></table></div>
|
||||
{% include "dashboard/_pager.html" %}
|
||||
</section>{% endblock %}
|
||||
@@ -7,12 +7,12 @@
|
||||
<label>Status <select name="status"><option value="">any</option>{% for name in statuses %}<option value="{{ name }}"{% if status == *name %} selected{% endif %}>{{ name }}</option>{% endfor %}</select></label>
|
||||
<div class="filter-actions"><button class="btn" type="submit">Filter</button>{% if !status.is_empty() %}<a class="btn-quiet" href="/dashboard/runs">Reset</a>{% endif %}</div>
|
||||
</form>
|
||||
{% if runs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table data-filter>
|
||||
{% if runs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th>run</th><th>date</th><th>status</th><th>started</th><th>duration</th><th>considered → eligible → triaged → assessed → shortlisted → selected</th><th>cost</th><th class="num">total</th><th class="num">warnings</th></tr></thead>
|
||||
<tbody>{% for run in runs %}<tr>
|
||||
<td class="num"><a href="/dashboard/runs/{{ run.id }}">{{ run.id }}</a></td>
|
||||
<td class="cell-tight"><a href="/issues/{{ run.date }}">{{ run.date }}</a></td>
|
||||
<td class="cell-tight"><span class="badge {{ run.status }}">{{ run.status }}</span>{% if run.dry_run %} <span class="muted text-xs">dry</span>{% endif %}</td>
|
||||
<td><span class="badge {{ run.status }}">{{ run.status }}</span>{% if run.dry_run %} <span class="muted text-xs">dry</span>{% endif %}</td>
|
||||
<td class="cell-tight text-muted">{{ run.started }}</td>
|
||||
<td class="num text-muted">{{ run.duration }}</td>
|
||||
<td class="cell-tight tabular-nums">{{ run.funnel }}</td>
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
<h1>Settings history</h1>
|
||||
<p class="page-desc">Every settings change made through the dashboard, newest first. Hand edits to <code>config.toml</code> on disk are not recorded here.</p>
|
||||
</div><div class="page-actions"><a class="btn" href="/dashboard/settings">Back to settings</a></div></header>
|
||||
{% if changes.is_empty() %}<p class="muted text-sm">No settings have been changed through the dashboard yet.</p>{% else %}{% if changes.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table class="history" data-filter>
|
||||
{% if changes.is_empty() %}<p class="muted text-sm">No settings have been changed through the dashboard yet.</p>{% else %}{% if changes.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table class="history" data-filter>
|
||||
<thead><tr><th>When</th><th>Who</th><th>Key</th><th>Before</th><th>After</th></tr></thead>
|
||||
<tbody>{% for change in changes %}<tr>
|
||||
<td class="cell-tight text-muted">{{ change.changed_at }}</td>
|
||||
<td class="cell-tight">{% match change.username %}{% when Some with (name) %}{{ name }}{% when None %}<em class="text-muted">removed user</em>{% endmatch %}</td>
|
||||
<td>{% match change.username %}{% when Some with (name) %}{{ name }}{% when None %}<em class="text-muted">removed user</em>{% endmatch %}</td>
|
||||
<td><code>{{ change.key }}</code></td>
|
||||
<td class="cell-wrap">{% match change.old_value %}{% when Some with (value) %}<pre class="preview">{{ value }}</pre>{% when None %}<em class="text-muted">absent</em>{% endmatch %}</td>
|
||||
<td class="cell-wrap">{% match change.new_value %}{% when Some with (value) %}<pre class="preview">{{ value }}</pre>{% when None %}<em class="text-muted">removed</em>{% endmatch %}</td>
|
||||
|
||||
@@ -27,14 +27,14 @@
|
||||
</div>
|
||||
|
||||
<h2>Spend by day</h2>
|
||||
{% if cost_rows.is_empty() %}<p class="muted text-sm">No provider costs recorded in the window.</p>{% else %}{% if cost_rows.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table data-filter>
|
||||
{% if cost_rows.is_empty() %}<p class="muted text-sm">No provider costs recorded in the window.</p>{% else %}{% if cost_rows.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th>day (UTC)</th>{% for provider in providers %}<th class="num">{{ provider }}</th>{% endfor %}<th class="num">total</th></tr></thead>
|
||||
<tbody>{% for row in cost_rows %}<tr><td class="cell-tight">{{ row.date }}</td>{% for cell in row.cells %}<td class="num text-muted">{{ cell }}</td>{% endfor %}<td class="num">{{ row.total }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
|
||||
<h2>Runs in the window</h2>
|
||||
{% if runs.is_empty() %}<p class="muted text-sm">No finished runs in the window.</p>{% else %}{% if runs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table data-filter>
|
||||
{% if runs.is_empty() %}<p class="muted text-sm">No finished runs in the window.</p>{% else %}{% if runs.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th class="num">run</th><th>date</th><th>status</th><th class="num">cost</th><th class="num">selected</th><th class="num">duration</th></tr></thead>
|
||||
<tbody>{% for run in runs %}<tr><td class="num"><a href="/dashboard/runs/{{ run.run_id }}">{{ run.run_id }}</a></td><td class="cell-tight"><a href="/issues/{{ run.date }}">{{ run.date }}</a></td><td class="cell-tight"><span class="badge {{ run.status }}">{{ run.status }}</span></td><td class="num">{{ run.cost }}</td><td class="num">{{ run.selected }}</td><td class="num text-muted">{{ run.duration }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
<tbody>{% for run in runs %}<tr><td class="num"><a href="/dashboard/runs/{{ run.run_id }}">{{ run.run_id }}</a></td><td class="cell-tight"><a href="/issues/{{ run.date }}">{{ run.date }}</a></td><td><span class="badge {{ run.status }}">{{ run.status }}</span></td><td class="num">{{ run.cost }}</td><td class="num">{{ run.selected }}</td><td class="num text-muted">{{ run.duration }}</td></tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
|
||||
<details class="explain disclosure" id="stats-text"><summary>As text (<code>daily-epub stats --days {{ days }}</code>)</summary><div class="disclosure-body"><pre class="preview">{{ text }}</pre></div></details>
|
||||
</section>{% endblock %}
|
||||
@@ -15,7 +15,7 @@
|
||||
<td><div class="flex flex-wrap gap-2"><form class="form-inline" method="post" action="/dashboard/users/requests/{{ request.id }}/approve"><input type="text" name="username" value="{{ request.suggested_username }}" autocomplete="username" maxlength="32" required pattern="[A-Za-z0-9._-]+" aria-label="Username for {{ request.email }}"><button class="btn-primary" type="submit"{% if !mail_configured %} disabled title="Email is not configured"{% endif %}>Approve</button></form><form method="post" action="/dashboard/users/requests/{{ request.id }}/done"><button class="btn" type="submit">Mark done</button></form></div></td>
|
||||
</tr>{% endfor %}</tbody></table></div>{% endif %}
|
||||
</section>
|
||||
{% if users.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x tall"><table data-filter>
|
||||
{% if users.len() > 1 %}<input type="search" class="table-filter" placeholder="Filter rows on this page" aria-label="Filter rows on this page" data-table-filter>{% endif %}<div class="scroll-x"><table data-filter>
|
||||
<thead><tr><th>Username</th><th>Role</th><th>Status</th><th>Created</th><th>Last login</th><th class="num">Open sessions</th></tr></thead>
|
||||
<tbody>{% for user in users %}<tr>
|
||||
<td class="font-medium text-ink">{{ user.username }}</td>
|
||||
|
||||
@@ -1 +1 @@
|
||||
{% for section in issue.sections %}<h2>{{ section.name }}</h2><ul>{% for entry in section.entries %}<li><a href="{{ entry.url }}">{{ entry.title }}</a> — {% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{{ entry.source }}{% match entry.publication %}{% when Some with (publication) %} · {{ publication }}{% when None %}{% endmatch %}{% match entry.summary %}{% when Some with (summary) %}<p class="summary">{{ summary }}</p>{% when None %}{% endmatch %}{% match entry.why %}{% when Some with (why) %}<p class="why"><em>Why it's here: {{ why }}</em></p>{% when None %}{% endmatch %}{% if !entry.comment_links.is_empty() %} · {% for link in entry.comment_links %}<a href="{{ link.url }}">{{ link.label }}</a>{% endfor %}{% endif %}</li>{% endfor %}</ul>{% endfor %}
|
||||
{% for section in issue.sections %}<h2>{{ section.name }}</h2><ul>{% for entry in section.entries %}<li><a href="{{ entry.url }}">{{ entry.title }}</a><p class="meta">{% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{{ entry.source }}{% match entry.publication %}{% when Some with (publication) %} · {{ publication }}{% when None %}{% endmatch %}</p>{% match entry.summary %}{% when Some with (summary) %}<p class="summary">{{ summary }}</p>{% when None %}{% endmatch %}{% if !entry.comment_links.is_empty() %}<p class="links">{% for link in entry.comment_links %}<a href="{{ link.url }}">{{ link.label }}</a>{% if !loop.last %} · {% endif %}{% endfor %}</p>{% endif %}</li>{% endfor %}</ul>{% endfor %}
|
||||
@@ -5,7 +5,7 @@
|
||||
<section aria-labelledby="brief-heading" data-toc-entry="/issues/{{ date }}"><h1 id="brief-heading" class="reader-section-heading">The Brief</h1><div class="editorial prose-body mt-5 [&>p:first-child]:first-letter:float-left [&>p:first-child]:first-letter:mr-2 [&>p:first-child]:first-letter:mt-1 [&>p:first-child]:first-letter:font-serif [&>p:first-child]:first-letter:text-[4.6rem] [&>p:first-child]:first-letter:font-semibold [&>p:first-child]:first-letter:leading-[0.72]">{{ front_page_html|safe }}</div></section>
|
||||
{% if (page.is_admin() && read_href.is_some()) || downloads.is_some() %}<div class="my-8 flex flex-wrap gap-2 border-y border-rule py-4">{% if page.is_admin() %}{% match read_href %}{% when Some with (href) %}<a class="btn" href="{{ href }}" target="_blank" rel="noopener">Read in BookOrbit</a>{% when None %}{% endmatch %}{% endif %}{% match downloads %}{% when Some with (downloads) %}{% if downloads.others.is_empty() %}<a class="btn" href="{{ downloads.primary.href }}">Download {{ downloads.primary.label }}</a>{% else %}<div class="relative inline-flex"><a class="btn rounded-r-none" href="{{ downloads.primary.href }}">Download {{ downloads.primary.label }}</a><details class="-ml-px"><summary class="btn list-none rounded-l-none px-2 [&::-webkit-details-marker]:hidden" aria-label="Choose download format"><span aria-hidden="true">▾</span></summary><div class="absolute left-0 top-full z-20 mt-1 min-w-52 border border-rule bg-paper p-1 shadow-[0_18px_32px_-26px_rgb(0_0_0_/_0.6)]"><a class="btn group w-full justify-between border-0" href="{{ downloads.primary.href }}"><span>{% if downloads.primary.label == "EPUB" %}Standard {% endif %}{{ downloads.primary.label }}</span><span class="ml-4 text-muted group-hover:text-paper">{{ downloads.primary.size }}</span></a>{% for download in downloads.others %}<a class="btn group w-full justify-between border-0" href="{{ download.href }}"><span>{{ download.label }}</span><span class="ml-4 text-muted group-hover:text-paper">{{ download.size }}</span></a>{% endfor %}</div></details></div>{% endif %}{% when None %}{% endmatch %}</div>{% endif %}
|
||||
<section class="mt-12" aria-labelledby="index-heading"><h2 id="index-heading" class="text-center text-3xl font-semibold leading-[1.1] tracking-[-0.01em]">In This Issue</h2>
|
||||
{% for section in sections %}<section class="mt-14"><h3 class="reader-section-heading">{{ section.name }}</h3><ul class="m-0 list-none p-0">{% for entry in section.entries %}<li class="border-b border-rule py-6" data-toc-entry="{{ entry.href }}"><h4 class="font-serif font-semibold leading-[1.1] tracking-[-0.01em] {% if entry.is_lead %}text-3xl{% else %}text-2xl{% endif %}"><a class="text-ink no-underline hover:text-accent" href="{{ entry.href }}">{{ entry.title }}</a></h4><p class="mt-2 font-sans text-sm text-muted">{% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{% let source = entry.source %}{% include "_source.html" %} · {{ entry.reading_minutes }} min read{% if page.is_admin() %} · <a class="text-muted hover:text-accent" href="{{ entry.dashboard_href }}">dashboard</a>{% endif %}</p>{% let understanding = entry.understanding %}{% let why = entry.why %}{% include "_understanding.html" %}{% if !entry.summary.is_empty() %}<p class="index-summary mt-4">{{ entry.summary }}</p>{% endif %}{% match entry.rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}</li>{% endfor %}</ul></section>{% endfor %}
|
||||
{% for section in sections %}<section class="mt-14"><h3 class="reader-section-heading">{{ section.name }}</h3><ul class="m-0 list-none p-0">{% for entry in section.entries %}<li class="border-b border-rule py-6" data-toc-entry="{{ entry.href }}"><h4 class="font-serif font-semibold leading-[1.1] tracking-[-0.01em] {% if entry.is_lead %}text-3xl{% else %}text-2xl{% endif %}"><a class="text-ink no-underline hover:text-accent" href="{{ entry.href }}">{{ entry.title }}</a></h4><p class="mt-2 font-sans text-sm text-muted">{% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{% let source = entry.source %}{% include "_source.html" %} · {{ entry.reading_minutes }} min read{% if page.is_admin() %} · <a class="text-muted hover:text-accent" href="{{ entry.dashboard_href }}">dashboard</a>{% endif %}</p>{% let understanding = entry.understanding %}{% include "_understanding.html" %}{% if !entry.summary.is_empty() %}<p class="index-summary mt-4">{{ entry.summary }}</p>{% endif %}{% match entry.rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}</li>{% endfor %}</ul></section>{% endfor %}
|
||||
</section>
|
||||
{% if has_world || has_behind %}<nav class="my-12 flex flex-wrap items-center justify-center gap-x-3 gap-y-1 border-y border-rule py-4 text-center font-sans text-sm uppercase tracking-[0.08em]" aria-label="Issue chapters">{% if has_world %}<a class="text-ink no-underline hover:text-accent" href="/issues/{{ date }}/world">World Briefing</a>{% endif %}{% if has_world && has_behind %}<span class="text-muted" aria-hidden="true">·</span>{% endif %}{% if has_behind %}<a class="text-ink no-underline hover:text-accent" href="/issues/{{ date }}/behind">Behind the paper</a>{% endif %}</nav>{% endif %}
|
||||
<footer id="colophon" class="mt-14 scroll-mt-6 font-sans text-sm leading-relaxed" data-toc-entry="/issues/{{ date }}#colophon"><h2 class="reader-section-heading">Colophon</h2><p class="my-4 text-ink-2"><em>The Daily EPUB</em> is assembled every morning from a personal feed reader.</p><dl class="kv border-y border-rule py-4 text-xs sm:text-sm"><dt>Generated</dt><dd>{{ colophon.generated_at }}</dd><dt>Bulk model</dt><dd>{{ colophon.bulk_model }}</dd><dt>Editor model</dt><dd>{{ colophon.editor_model }}</dd><dt>Summaries model</dt><dd>{{ colophon.summaries_model }}</dd><dt>Entries considered</dt><dd>{% match colophon.entries_fetched %}{% when Some with (entries) %}{{ entries }}{% match colophon.feeds_seen %}{% when Some with (feeds) %} from {{ feeds }} feeds{% when None %}{% endmatch %}{% when None %}n/a{% endmatch %}</dd><dt>Candidates scored</dt><dd>{% match colophon.candidates %}{% when Some with (candidates) %}{{ candidates }}{% when None %}n/a{% endmatch %}</dd><dt>Articles selected</dt><dd>{{ colophon.article_count }} across {{ colophon.section_count }} sections</dd><dt>Words</dt><dd>{{ colophon.total_words }} · ~{{ colophon.reading_minutes }} min read</dd>{% for cost in colophon.provider_costs %}<dt>{{ cost.provider }} cost</dt><dd>{{ cost.cost }}</dd>{% endfor %}<dt>Total token cost</dt><dd>{% match colophon.cost_usd %}{% when Some with (cost) %}{{ cost }}{% when None %}n/a{% endmatch %}</dd><dt>Generator</dt><dd>{{ colophon.generator_version }}</dd></dl></footer>
|
||||
|
||||
@@ -4,6 +4,6 @@
|
||||
<header class="mb-10 text-center"><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">{{ issue.display_date }} · No. {{ issue.issue_number }}</p><p class="mt-2 font-sans text-sm text-muted">{{ issue.stats_line }}</p><p class="mx-auto mt-5 max-w-[58ch] text-lg italic leading-relaxed text-ink-2">A personal morning paper, assembled daily; the selection is the reader's, the words are the authors'.</p></header>
|
||||
<p class="notice mb-8"><a href="/login">Sign in</a> to read every article's full text and download the editions, or <a href="/request-access">request access</a>.</p>
|
||||
{% if !downloads.is_empty() %}<div class="mb-8 flex flex-wrap justify-center gap-2">{% for download in downloads %}<a class="btn" href="{{ download.href }}">{{ download.label }} <span class="ml-1 text-muted">{{ download.size }}</span></a>{% endfor %}</div>{% endif %}
|
||||
{% for section in issue.sections %}<section class="mt-14"><h2 class="reader-section-heading">{{ section.name }}</h2><div>{% for entry in section.entries %}<article class="border-b border-rule py-6{% if entry.is_lead %} pt-5{% endif %}"><h3 class="font-semibold leading-[1.1] tracking-[-0.01em] {% if entry.is_lead %}text-3xl{% else %}text-2xl{% endif %}"><a class="text-ink no-underline hover:text-accent" href="{{ entry.url }}">{{ entry.title }}</a></h3><p class="mt-2 font-sans text-sm leading-relaxed text-muted">{% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{{ entry.source }}{% match entry.publication %}{% when Some with (publication) %} · {{ publication }}{% when None %}{% endmatch %} · {{ entry.reading_minutes }} min</p>{% let understanding = entry.understanding %}{% let why = entry.why %}{% include "_understanding.html" %}{% match entry.summary %}{% when Some with (summary) %}<p class="index-summary mt-4">{{ summary }}</p>{% when None %}{% endmatch %}{% if !entry.comment_links.is_empty() %}<p class="mt-3 flex flex-wrap gap-x-4 gap-y-1 font-sans text-sm text-muted">{% for link in entry.comment_links %}<a class="text-muted hover:text-accent" rel="noopener" target="_blank" href="{{ link.url }}">{{ link.label }}{% if !link.meta.is_empty() %}: {{ link.meta }}{% endif %}</a>{% endfor %}</p>{% endif %}</article>{% endfor %}</div></section>{% endfor %}
|
||||
{% for section in issue.sections %}<section class="mt-14"><h2 class="reader-section-heading">{{ section.name }}</h2><div>{% for entry in section.entries %}<article class="border-b border-rule py-6{% if entry.is_lead %} pt-5{% endif %}"><h3 class="font-semibold leading-[1.1] tracking-[-0.01em] {% if entry.is_lead %}text-3xl{% else %}text-2xl{% endif %}"><a class="text-ink no-underline hover:text-accent" href="{{ entry.url }}">{{ entry.title }}</a></h3><p class="mt-2 font-sans text-sm leading-relaxed text-muted">{% match entry.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{{ entry.source }}{% match entry.publication %}{% when Some with (publication) %} · {{ publication }}{% when None %}{% endmatch %} · {{ entry.reading_minutes }} min</p>{% let understanding = entry.understanding %}{% include "_understanding.html" %}{% match entry.summary %}{% when Some with (summary) %}<p class="index-summary mt-4">{{ summary }}</p>{% when None %}{% endmatch %}{% if !entry.comment_links.is_empty() %}<p class="mt-3 flex flex-wrap gap-x-4 gap-y-1 font-sans text-sm text-muted">{% for link in entry.comment_links %}<a class="text-muted hover:text-accent" rel="noopener" target="_blank" href="{{ link.url }}">{{ link.label }}{% if !link.meta.is_empty() %}: {{ link.meta }}{% endif %}</a>{% endfor %}</p>{% endif %}</article>{% endfor %}</div></section>{% endfor %}
|
||||
<p class="mt-10 text-center font-sans text-sm"><a href="/issues">Browse the archive →</a></p>
|
||||
{% endif %}</article>{% endblock %}
|
||||
@@ -47,6 +47,7 @@
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "runs" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/runs"{% if page.active_nav == "runs" %} aria-current="page"{% endif %}>Runs</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "articles" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/articles"{% if page.active_nav == "articles" %} aria-current="page"{% endif %}>Articles</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "ratings" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/ratings"{% if page.active_nav == "ratings" %} aria-current="page"{% endif %}>Ratings</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "interests" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/interests"{% if page.active_nav == "interests" %} aria-current="page"{% endif %}>Interests</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "feeds" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/feeds"{% if page.active_nav == "feeds" %} aria-current="page"{% endif %}>Feeds</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "profile" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/profile"{% if page.active_nav == "profile" %} aria-current="page"{% endif %}>Profile</a>
|
||||
<a class="flex min-h-10 min-w-10 items-center justify-center border-b-2 no-underline hover:text-ink {% if page.active_nav == "stats" %}border-accent text-ink{% else %}border-transparent text-muted{% endif %}" href="/dashboard/stats"{% if page.active_nav == "stats" %} aria-current="page"{% endif %}>Stats</a>
|
||||
|
||||
@@ -35,7 +35,6 @@ fn config_check_prints_the_facts_and_exits_zero_without_keys() {
|
||||
"config: ",
|
||||
"database_path: /var/lib/daily-epub/daily-epub.db",
|
||||
"profile_path: data/profile.md",
|
||||
"interests_opml: data/scour-interests.opml",
|
||||
"llm.bulk: deepseek · openai · deepseek-v4-flash",
|
||||
"key MISSING (set DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY)",
|
||||
"llm.editor: anthropic · anthropic · claude-opus-5 · effort high · max_daily_usd $3.00",
|
||||
@@ -66,4 +65,8 @@ fn config_check_exits_non_zero_on_an_invalid_config() {
|
||||
let (code, _, stderr) = run("[deepseek]\nmodel = \"x\"\n");
|
||||
assert_ne!(code, 0);
|
||||
assert!(stderr.contains("[providers.deepseek]"), "{stderr}");
|
||||
|
||||
let (code, _, stderr) = run("interests_opml = \"data/scour-interests.opml\"\n");
|
||||
assert_ne!(code, 0);
|
||||
assert!(stderr.contains("interests_opml"), "{stderr}");
|
||||
}
|
||||
@@ -619,11 +619,6 @@ async fn llm_pipeline_runs_against_a_mock_backend() {
|
||||
"the model's summaries were used, not excerpts"
|
||||
);
|
||||
assert!(editorial_doc.front_page_html.contains("storage internals"));
|
||||
assert!(
|
||||
lineup.picks.iter().all(|p| p.why.is_none()),
|
||||
"the scripted editor gave no why lines"
|
||||
);
|
||||
|
||||
// Every scripted response was consumed, and the meter priced them (§3.6).
|
||||
assert_eq!(backend.calls(), 1 + 1 + 5 + 1);
|
||||
let total = meter.total();
|
||||
|
||||
Vendored
+5
-5
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"picks": [
|
||||
{ "id": 101, "section": "Top Stories", "position": 1, "lead_story": true, "why": "The migration post-mortem with the invoices still attached" },
|
||||
{ "id": 102, "section": "Tech & Engineering", "position": 1, "lead_story": false, "why": "A failover story you'd argue with over coffee" },
|
||||
{ "id": 103, "section": "Tech & Engineering", "position": 2, "lead_story": false, "why": "Rare first-hand detail on a tool you use daily" },
|
||||
{ "id": 104, "section": "AI & Machine Learning", "position": 1, "lead_story": false, "why": "The one benchmark piece this week that shows its work" },
|
||||
{ "id": 105, "section": "Boston & Local", "position": 1, "lead_story": false, "why": "MBTA slow zones charted by a rider, not a press office" },
|
||||
{ "id": 101, "section": "Top Stories", "position": 1, "lead_story": true },
|
||||
{ "id": 102, "section": "Tech & Engineering", "position": 1, "lead_story": false },
|
||||
{ "id": 103, "section": "Tech & Engineering", "position": 2, "lead_story": false },
|
||||
{ "id": 104, "section": "AI & Machine Learning", "position": 1, "lead_story": false },
|
||||
{ "id": 105, "section": "Boston & Local", "position": 1, "lead_story": false },
|
||||
{ "id": 106, "section": "Culture & Essays", "position": 1 },
|
||||
{ "section": "Niche Corner", "position": 2, "lead_story": false },
|
||||
"the model sometimes trails off like this"
|
||||
|
||||
+10
-7
@@ -21,6 +21,7 @@ use daily_epub::curate::assess::parse_deep_response;
|
||||
use daily_epub::curate::editor::parse_selection_response;
|
||||
use daily_epub::curate::editorial::BriefResponse;
|
||||
use daily_epub::curate::profile;
|
||||
use daily_epub::interests;
|
||||
|
||||
fn repo(rel: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join(rel)
|
||||
@@ -183,13 +184,14 @@ fn stage_c_fixture_parses_into_the_brief() {
|
||||
assert!(value.get("section_intros").is_none());
|
||||
}
|
||||
|
||||
/// The taste profile is seeded from this file; a broken export would silently
|
||||
/// gut the system prompt (§3.6a).
|
||||
/// The importer relies on this file; a broken export would silently lose
|
||||
/// standing interests (§3.6a).
|
||||
#[test]
|
||||
fn scour_opml_still_yields_the_interest_list() {
|
||||
let interests = profile::parse_interests(&repo("data/scour-interests.opml"))
|
||||
.expect("the shipped OPML must parse");
|
||||
let unique: BTreeSet<String> = interests.iter().map(|n| n.to_lowercase()).collect();
|
||||
let raw = std::fs::read_to_string(repo("data/scour-interests.opml"))
|
||||
.expect("the shipped OPML must be readable");
|
||||
let names = interests::parse_opml(&raw);
|
||||
let unique: BTreeSet<String> = names.iter().map(|n| n.to_lowercase()).collect();
|
||||
|
||||
assert!(
|
||||
unique.len() > 180,
|
||||
@@ -206,7 +208,7 @@ fn scour_opml_still_yields_the_interest_list() {
|
||||
assert!(unique.contains(expected), "{expected} disappeared");
|
||||
}
|
||||
assert!(
|
||||
!interests.iter().any(|n| n.contains("token=")),
|
||||
!names.iter().any(|n| n.contains("token=")),
|
||||
"interest names must not leak the Scour token"
|
||||
);
|
||||
|
||||
@@ -217,9 +219,10 @@ fn scour_opml_still_yields_the_interest_list() {
|
||||
"/data/profile.md"
|
||||
)))
|
||||
.expect("profile file");
|
||||
let grouped = vec![("Imported interests".to_string(), names)];
|
||||
let document = profile::build(
|
||||
&profile_file.body,
|
||||
&interests,
|
||||
&grouped,
|
||||
profile::NO_LEARNED_ADJUSTMENTS,
|
||||
&[],
|
||||
60,
|
||||
|
||||
Reference in new issue
Block a user