Compare commits
159
Commits
d0560afa1a
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddc98788c6 | ||
|
|
682fc4a6fb | ||
|
|
08b15c42b4 | ||
|
|
03f0a454be | ||
|
|
bf6a8d0129 | ||
|
|
7851d8e231 | ||
|
|
5abf062c24 | ||
|
|
9e521ddaef | ||
|
|
50551114cd | ||
|
|
13c81440ef | ||
|
|
248038ec97 | ||
|
|
78fc38e99d | ||
|
|
8d0516bedc | ||
|
|
e5ca78ba41 | ||
|
|
3c4f8de3a4 | ||
|
|
dd91b83c27 | ||
|
|
815e7307d1 | ||
|
|
9af6fedbe0 | ||
|
|
6016745297 | ||
|
|
8a355b637f | ||
|
|
8889a4e351 | ||
|
|
0f17480be7 | ||
|
|
525ddfd88d | ||
|
|
f1b1e7fd98 | ||
|
|
ce9b629115 | ||
|
|
36424784af | ||
|
|
cd2bbe8840 | ||
|
|
8b56a65bad | ||
|
|
1089c29179 | ||
|
|
7063b835db | ||
|
|
57ba47a5a0 | ||
|
|
861ae6cb5e | ||
|
|
af70da1493 | ||
|
|
055b1df769 | ||
|
|
22cfb34788 | ||
|
|
154731a003 | ||
|
|
a78e44f56e | ||
|
|
2d857e3e10 | ||
|
|
f0c0927ab8 | ||
|
|
e82f7b0d13 | ||
|
|
4338dc1a99 | ||
|
|
37ddddf867 | ||
|
|
71079a93f5 | ||
|
|
31d2690701 | ||
|
|
1a009a2113 | ||
|
|
c538afa540 | ||
|
|
f2a44507d4 | ||
|
|
87f3a5dbf1 | ||
|
|
f71a4c2799 | ||
|
|
8a8d5d1f16 | ||
|
|
c8df55e91a | ||
|
|
c707f432f2 | ||
|
|
b02e1c7b8c | ||
|
|
c6a9a98a0d | ||
|
|
bedbcb218f | ||
|
|
7ce8c65939 | ||
|
|
41c144bf20 | ||
|
|
8365b7093e | ||
|
|
b66a8eefa5 | ||
|
|
a52f4123dd | ||
|
|
20991fedfc | ||
|
|
0212e407e5 | ||
|
|
72b32dc69b | ||
|
|
56d2d74a97 | ||
|
|
81cbbb407f | ||
|
|
c0f086e0da | ||
|
|
2bbb4809b7 | ||
|
|
c971fd9317 | ||
|
|
4a2fc6d1c4 | ||
|
|
e69940fac8 | ||
|
|
78d9ef5959 | ||
|
|
5079d9c008 | ||
|
|
e56acfb9dc | ||
|
|
a3a041dc06 | ||
|
|
49a2de14ae | ||
|
|
502ba38607 | ||
|
|
df76c81b59 | ||
|
|
feb887a8af | ||
|
|
87ff3d92c8 | ||
|
|
8c821527e8 | ||
|
|
f0529d2d01 | ||
|
|
af8850e948 | ||
|
|
d94cefcbd7 | ||
|
|
7036fd347e | ||
|
|
71665c293c | ||
|
|
142a8d9905 | ||
|
|
9aff78ebf1 | ||
|
|
e370fcaeb2 | ||
|
|
0f45c34fdb | ||
|
|
a72f036bd1 | ||
|
|
0851d56459 | ||
|
|
6c2d74c734 | ||
|
|
17a88043ff | ||
|
|
9c4964ae27 | ||
|
|
1e362fee3e | ||
|
|
f6b2d3ea9a | ||
|
|
e727d53b85 | ||
|
|
bc40773964 | ||
|
|
f634f06bf1 | ||
|
|
de332cd4b3 | ||
|
|
e616638c36 | ||
|
|
7d12f53314 | ||
|
|
41fe61a691 | ||
|
|
2388d7bd02 | ||
|
|
62803d73a6 | ||
|
|
4c8eeda147 | ||
|
|
04bf14f710 | ||
|
|
239125a8e5 | ||
|
|
1047cf4633 | ||
|
|
9acd1b0800 | ||
|
|
b299b4ba05 | ||
|
|
4248e48a8b | ||
|
|
bcbe1583af | ||
|
|
0cdc302f84 | ||
|
|
da24e7c7c5 | ||
|
|
5cc4ac4bd3 | ||
|
|
fafb737e53 | ||
|
|
ad2e99eeb9 | ||
|
|
560bbc13af | ||
|
|
78e7e4aa2c | ||
|
|
6b0fa51e87 | ||
|
|
e719e031cf | ||
|
|
ca910d99e5 | ||
|
|
16244a6a70 | ||
|
|
a4c338a9f9 | ||
|
|
29c66da865 | ||
|
|
12134f3596 | ||
|
|
f410df47c0 | ||
|
|
61ed37fc20 | ||
|
|
9b1c137c80 | ||
|
|
eb960f1b57 | ||
|
|
10cbe9ed79 | ||
|
|
f6e317a9fe | ||
|
|
aa3de51d9c | ||
|
|
d404946c78 | ||
|
|
073a85f3c7 | ||
|
|
572056ce3b | ||
|
|
d36f203e4c | ||
|
|
63eee6bbd3 | ||
|
|
da065437c0 | ||
|
|
849231e49a | ||
|
|
4995a8bd5d | ||
|
|
69cd736455 | ||
|
|
a70de4af06 | ||
|
|
bf2ab8441e | ||
|
|
89423f01fa | ||
|
|
af2a3ceb49 | ||
|
|
7a842b61b6 | ||
|
|
99d1338890 | ||
|
|
5edbeb509f | ||
|
|
d403c51edf | ||
|
|
d261cd485d | ||
|
|
05a74a0dcf | ||
|
|
10f4afd091 | ||
|
|
76527e3821 | ||
|
|
8b51454c20 | ||
|
|
ea3b141373 | ||
|
|
57efbb49b4 | ||
|
|
3a9f4b99e0 |
No files matched your search
@@ -3,3 +3,5 @@ target/
|
||||
*.sqlite
|
||||
out/
|
||||
.env
|
||||
dev/
|
||||
node_modules/
|
||||
Generated
+1541
-22
File diff suppressed because it is too large.
Load diff
+34
-4
@@ -1,14 +1,18 @@
|
||||
[package]
|
||||
name = "daily-epub"
|
||||
version = "0.1.0"
|
||||
version = "0.3.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
ammonia = "4.1.4"
|
||||
anyhow = "1.0.104"
|
||||
arborium = { version = "2.18.2", default-features = false, features = ["lang-asm", "lang-awk", "lang-bash", "lang-c", "lang-c-sharp", "lang-clojure", "lang-cmake", "lang-commonlisp", "lang-cpp", "lang-css", "lang-d", "lang-dart", "lang-diff", "lang-dockerfile", "lang-elisp", "lang-elixir", "lang-elm", "lang-erlang", "lang-fish", "lang-fsharp", "lang-gleam", "lang-glsl", "lang-go", "lang-graphql", "lang-groovy", "lang-haskell", "lang-hcl", "lang-html", "lang-ini", "lang-java", "lang-javascript", "lang-jq", "lang-json", "lang-julia", "lang-kotlin", "lang-lean", "lang-lua", "lang-make", "lang-matlab", "lang-nix", "lang-objc", "lang-ocaml", "lang-odin", "lang-perl", "lang-php", "lang-powershell", "lang-prolog", "lang-proto", "lang-python", "lang-r", "lang-ruby", "lang-rust", "lang-scala", "lang-scheme", "lang-scss", "lang-solidity", "lang-sql", "lang-starlark", "lang-swift", "lang-toml", "lang-tsx", "lang-typescript", "lang-verilog", "lang-vhdl", "lang-vim", "lang-x86asm", "lang-xml", "lang-yaml", "lang-zig", "lang-zsh"] }
|
||||
askama = "0.16.0"
|
||||
async-trait = "0.1.92"
|
||||
axum = "0.8.9"
|
||||
axum-login = { git = "https://github.com/maxcountryman/axum-login.git", rev = "151c72d7a1b4646830f86b4332e6bd6e34d719a7", version = "0.18.0" }
|
||||
base64 = "0.23.1"
|
||||
betlang = "0.1.1"
|
||||
clap = { version = "4.6.6", features = ["derive"] }
|
||||
dom_smoothie = "0.18.0"
|
||||
epub-builder = "0.8.3"
|
||||
@@ -18,9 +22,22 @@ hex = "0.4.3"
|
||||
hmac = "0.13.0"
|
||||
image = "0.25.10"
|
||||
jiff = { version = "0.2.35", features = ["serde"] }
|
||||
lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "hostname"] }
|
||||
libc = "0.2.189"
|
||||
password-auth = "1.0.0"
|
||||
percent-encoding = "2.3.2"
|
||||
rand = "0.10.2"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "gzip", "json", "charset", "http2", "query", "system-proxy"] }
|
||||
reqwest = { version = "0.13.4", default-features = false, features = [
|
||||
"rustls",
|
||||
"gzip",
|
||||
"json",
|
||||
"charset",
|
||||
"http2",
|
||||
"query",
|
||||
"system-proxy",
|
||||
] }
|
||||
resvg = "0.48.1"
|
||||
rpassword = "7.5.4"
|
||||
scraper = "0.27.0"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
serde_json = "1.0.151"
|
||||
@@ -35,15 +52,28 @@ sqlx = { version = "0.9.0", default-features = false, features = [
|
||||
"tls-rustls-ring",
|
||||
] }
|
||||
thiserror = "2.0.20"
|
||||
time = "0.3.55"
|
||||
tiny-skia = "0.12.0"
|
||||
tokio = { version = "1.53.1", features = ["full"] }
|
||||
tokio-util = { version = "0.7.19", features = ["io"] }
|
||||
tower-http = { version = "0.7.0", features = ["trace", "fs"] }
|
||||
toml = "1.1.4"
|
||||
toml_edit = "0.25"
|
||||
tower-http = { version = "0.7.0", features = ["trace", "fs", "request-id"] }
|
||||
tower_governor = { version = "0.8.0", features = ["axum"] }
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
zip = { version = "6", default-features = false, features = ["deflate"] }
|
||||
|
||||
[profile.release]
|
||||
# Per-request cost in the web server is JSON decode and template render, both
|
||||
# CPU-bound and spread across crates; whole-program LTO and one codegen unit
|
||||
# buy a slice of that for a longer release build (tests use the dev profile).
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
|
||||
[dev-dependencies]
|
||||
figment = { version = "0.10.19", features = ["test", "toml", "env"] }
|
||||
roxmltree = "0.21.1"
|
||||
tempfile = "3.27.0"
|
||||
zip = { version = "6", default-features = false, features = ["deflate"] }
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
@@ -4,19 +4,35 @@ A personalized daily newspaper, delivered as an EPUB.
|
||||
|
||||
Every morning a systemd timer wakes one Rust binary. It pulls the last ~26 hours
|
||||
from a self-hosted [Miniflux](https://miniflux.app), deduplicates and extracts
|
||||
the articles, enriches them with HackerNews/Lobsters/Reddit social proof, filters
|
||||
300–500 candidates down to ~120 with cheap heuristics, and asks DeepSeek to score,
|
||||
select and introduce 15–25 of them. It assembles two EPUB editions (a standard one
|
||||
the articles, enriches them with HackerNews/Lobsters/Reddit social proof, has
|
||||
DeepSeek triage every eligible opening and closely assess a 120-article union,
|
||||
then utility-ranks and diversity-caps a 60-item shortlist for Claude Opus 5 — the editor —
|
||||
which assembles the issue
|
||||
(no minimum size, a hard ceiling), then writes the summaries and *The Brief*.
|
||||
It assembles two EPUB editions (a standard one
|
||||
and one tuned for the Xteink X4 e-ink reader), converts the X4 edition to XTC, and
|
||||
publishes the lot over its own OPDS catalog — which doubles as a
|
||||
[BookOrbit](https://github.com/thallada/bookorbit) watched folder if you run one.
|
||||
Each article chapter ends with 👍/👎 links that feed back into tomorrow's curation.
|
||||
Each article chapter ends with a one-line note of what the pipeline made of the
|
||||
piece (topic, format, depth, and the reader interests it matched) and Loved it /
|
||||
Good / Not for me / AI slop links that feed back into tomorrow's curation (an AI
|
||||
slop report also cuts every future article by that author); a short *Behind
|
||||
the paper* chapter before the colophon says what the run considered, how the
|
||||
deep set was admitted, whether the learned signals were active, the ten
|
||||
highest-utility near misses, and what it all cost.
|
||||
|
||||
Steady-state cost is roughly **$0.05–0.30/day** in DeepSeek tokens, hard-capped by
|
||||
`max_daily_usd`.
|
||||
Steady-state cost is roughly **$1/day**: $0.05–0.30 in DeepSeek tokens plus
|
||||
~$0.50–0.80 for the Claude editor and a few cents of Voyage AI embeddings, each
|
||||
provider with its own per-UTC-day ceiling (`providers.<name>.max_daily_usd` and
|
||||
`voyage.max_daily_usd`). Those ceilings are runaway guards, not accounting — set
|
||||
hard spend limits in the providers' dashboards as the real backstop. The two
|
||||
LLM roles — *bulk* (triage, assessment, fallbacks) and *editor* — are assigned
|
||||
by name to entries of a provider registry, so swapping DeepSeek or Claude for
|
||||
Gemini (or anything OpenAI-compatible) is a config line plus an API key.
|
||||
|
||||
- Full design: [`docs/plans/2026-08-15-the-daily-epub.md`](docs/plans/2026-08-15-the-daily-epub.md)
|
||||
- Implementation decisions: [`docs/plans/2026-08-15-implementation-notes.md`](docs/plans/2026-08-15-implementation-notes.md)
|
||||
- Web dashboard rollout: [`docs/runbooks/web-dashboard-rollout.md`](docs/runbooks/web-dashboard-rollout.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -24,20 +40,48 @@ Steady-state cost is roughly **$0.05–0.30/day** in DeepSeek tokens, hard-cappe
|
||||
|
||||
```
|
||||
Miniflux ingest ─▶ dedupe ─▶ extraction ─▶ persist ─▶ social enrichment
|
||||
─▶ pre-filter ─▶ LLM scoring ─▶ selection ─▶ comments ─▶ editorial
|
||||
─▶ hygiene ─▶ embeddings (Voyage) + cheap signals ─▶ triage (bulk LLM)
|
||||
─▶ union admission ─▶ deep assessment (bulk LLM) ─▶ utility + diversity
|
||||
─▶ editor (editor LLM) ─▶ comments ─▶ editorial (editor LLM)
|
||||
─▶ world briefing ─▶ EPUB (standard + X4) ─▶ XTC ─▶ publish ─▶ report
|
||||
```
|
||||
|
||||
Every stage writes to SQLite, so a run is idempotent per date: re-running
|
||||
`generate --date 2026-08-15` replaces that issue rather than duplicating it.
|
||||
Only one writer runs at a time: `generate`, `profile rebuild`, `features
|
||||
backfill` and `backfill-social` take an advisory `flock` on
|
||||
`<database_path>.lock`, and a second invocation exits with `generate is already
|
||||
running` (naming whichever command holds it). `serve`, `explain`, `stats`,
|
||||
`ratings`, `features prune` and `db migrate` never wait on it, and the kernel
|
||||
releases the lock when the holder exits, however it exits.
|
||||
|
||||
Every run ends with a four-line summary in the log and on stdout:
|
||||
|
||||
```
|
||||
curation: 412 considered → 398 eligible → 398 triaged → 120 assessed → 60 shortlisted → 17 selected
|
||||
admission: triage 60 · interest 20 · knn 12 · exploration 5 · blend 23 · auto 0
|
||||
preference: 14 rated w/ embeddings → knn 0.35 · feed off · 41 verdicts in prompt
|
||||
providers: anthropic $0.62 · deepseek $0.11 · voyage $0.02 · total $0.75 · 23m12s
|
||||
```
|
||||
|
||||
The full report (per-stage counts and timings — `embed`, `signals`, `triage`,
|
||||
`admit`, `assess`, `rank`, `editor`, `summaries`, `brief` among them — and
|
||||
per-provider usage) is stored on the `runs` row and in `issues.report_json`.
|
||||
|
||||
**Failure policy.** Miniflux ingest, SQLite writes, EPUB assembly and publishing
|
||||
are fatal — without them there is no issue, and the `runs` row records why.
|
||||
Social lookups, comment fetching, the world briefing, images and the XTC
|
||||
conversion are best-effort: they log, add a warning (run status `degraded`) and
|
||||
the run continues. Every DeepSeek stage *degrades*: a missing key, a dead API or
|
||||
a tripped budget turns the run into the `--skip-llm` shape (prefilter order
|
||||
selects, feed excerpts stand in for summaries) instead of losing the day's issue.
|
||||
the run continues. Every LLM stage *degrades*: an editor call that fails, is
|
||||
refused, or is over its provider's daily ceiling is retried with the same
|
||||
prompt on the bulk provider; if the bulk provider is missing, dead or over
|
||||
budget too, the run takes the `--skip-llm` shape (admission uses cheap signals
|
||||
and feed excerpts stand in for summaries) instead of losing the day's issue.
|
||||
Which provider plays which role is the `[llm]` table (`bulk = "deepseek"`,
|
||||
`editor = "anthropic"` by default); a role whose key is absent is simply
|
||||
unavailable, and `daily-epub config check` shows the resolved assignment before
|
||||
a run. Anthropic's server-side refusal fallback (`fallbacks = "default"`) is
|
||||
enabled on every request to an `anthropic`-kind provider.
|
||||
|
||||
---
|
||||
|
||||
@@ -47,14 +91,15 @@ selects, feed excerpts stand in for summaries) instead of losing the day's issue
|
||||
|---|---|---|
|
||||
| Rust (2024 edition toolchain) | building | `cargo build --release` |
|
||||
| **Miniflux** with an API key | the only content source | Settings → API Keys. The client is read-only and never mutates read state. |
|
||||
| **DeepSeek API key** | curation + editorial | <https://platform.deepseek.com>. Optional: `--skip-llm` runs the whole pipeline without it. |
|
||||
| A 32+ byte random secret | signs the 👍/👎 rating links | `openssl rand -hex 32` |
|
||||
| A key for the **bulk** provider (DeepSeek by default) | triage and deep assessment, and the fallback for every editor call | <https://platform.deepseek.com>. `DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY`. Optional: `--skip-llm` runs the whole pipeline without it. |
|
||||
| A key for the **editor** provider (Anthropic by default) | the editor: selection, summaries, The Brief, the weekly profile rebuild | <https://console.anthropic.com>. `DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY`. Optional: without it every editor call runs on the bulk provider. Set a dashboard spend limit; `providers.anthropic.max_daily_usd` is only a runaway guard. Any other `[providers.*]` entry (Gemini is shipped) can take either role — see *Switching providers*. |
|
||||
| **Voyage AI API key** | article and interest embeddings behind the learned ranking signals | <https://www.voyageai.com>. Optional: without it (or with `--skip-embeddings`) the run uses cached vectors only and the learned signals are absent, never a penalty. |
|
||||
| A 32+ byte random secret | signs the article rating links | `openssl rand -hex 32` |
|
||||
| **BookOrbit** library + watched folder | *optional* — a richer library UI on top of the same folder | Delivery does not need it: `daily-epub serve` has its own OPDS catalog over `publish.epub_dir`. If you do run it, create a dedicated "The Daily EPUB" library, enable *Watch folders*, and point `publish.epub_dir` at it. |
|
||||
| **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node <repo>/cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. |
|
||||
| A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. |
|
||||
|
||||
`data/scour-interests.opml` (the ~220 Scour interests the taste profile is seeded
|
||||
from) must be readable at the path in `interests_opml`.
|
||||
`data/profile.md` is the hand-maintained reader profile; its path is configurable.
|
||||
|
||||
---
|
||||
|
||||
@@ -67,21 +112,203 @@ cargo test # everything is offline; no keys needed
|
||||
sudo install -m0755 target/release/daily-epub /usr/local/bin/
|
||||
```
|
||||
|
||||
The web UI uses Tailwind CSS v4, but Node is only a development dependency: the
|
||||
compiled stylesheet is committed and embedded in the Rust binary. After editing
|
||||
`src/web/tailwind.css`, a web template, or `src/web/static/app.js`, run
|
||||
`npm install` once and then `npm run css`. Commit both the source changes and
|
||||
`src/web/static/app.css`; `npm run css:check` verifies that the committed output
|
||||
is current.
|
||||
|
||||
### Commands
|
||||
|
||||
```
|
||||
daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm]
|
||||
daily-epub serve # rating endpoints + OPDS catalog + downloads
|
||||
daily-epub profile rebuild # regenerate the taste profile from ratings (weekly inside generate)
|
||||
daily-epub backfill-social # re-poll social scores for recent articles
|
||||
daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] [--skip-embeddings] [--rescore]
|
||||
daily-epub serve # public site, private dashboard, OPDS, ratings, downloads
|
||||
daily-epub profile rebuild # regenerate learned profile adjustments
|
||||
daily-epub ratings list [--days 90] [--label loved|good|down|slop|cleared]
|
||||
daily-epub ratings set --article 42 --label loved --note "excellent"
|
||||
daily-epub ratings clear --url https://example.com/article
|
||||
daily-epub explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N]
|
||||
daily-epub explain --date YYYY-MM-DD --near-misses [N]
|
||||
daily-epub stats [--days 14] # the evaluation framework, one fact per line
|
||||
daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes]
|
||||
daily-epub features prune # stale embeddings, old telemetry and assessments
|
||||
daily-epub backfill-social [--days 7] # re-poll social scores for recent articles
|
||||
daily-epub feeds discover [--days 14] [--limit 50] # seed feed candidates from recent aggregator articles
|
||||
daily-epub interests import [--opml PATH] [--profile PATH] # one-time standing-interest import
|
||||
daily-epub interests backfill # match compatible cached article embeddings to interests
|
||||
daily-epub db migrate # run migrations (also automatic on every start)
|
||||
daily-epub config check # validate the config, print the resolved roles, keys and paths
|
||||
daily-epub users add USER [--admin] [--password-stdin]
|
||||
daily-epub users passwd USER [--password-stdin]
|
||||
daily-epub users role USER user|admin
|
||||
daily-epub users disable USER
|
||||
daily-epub users enable USER
|
||||
daily-epub users list
|
||||
daily-epub users logout USER # revoke all of USER's sessions
|
||||
daily-epub job run NAME # systemd job-unit entry point; normally not run by hand
|
||||
```
|
||||
|
||||
On the server, run these through the `justfile` (`just --list`): every server
|
||||
recipe wraps the binary in `systemd-run` as the `daily-epub` user with
|
||||
`/etc/daily-epub/env` loaded, exactly as the units do, so API keys resolve.
|
||||
`just de <subcommand...>` runs anything; `just feeds-discover 30`,
|
||||
`just config-check`, `just deploy` and `just logs-generate` cover the common cases.
|
||||
Plain `sudo -u daily-epub daily-epub ...` does **not** read the env file.
|
||||
|
||||
`--dry-run` does everything except deliver: it still ingests, persists entries and
|
||||
articles, curates and **builds both EPUBs into `--out`**, but it does not copy to
|
||||
BookOrbit, does not run the retention sweep, does not write the `issues` row and
|
||||
does not advance the ingest watermark. It prints the lineup and the cost report.
|
||||
|
||||
`--skip-embeddings` reads the embedding cache but makes zero Voyage calls.
|
||||
`--rescore` ignores reusable triage/deep assessments for this run.
|
||||
`--max-articles N` is a ceiling, never a target: the hard maximum becomes
|
||||
`min(curation.max_article_count, N)` and the soft target is lowered to fit.
|
||||
|
||||
`explain` answers "why was this (not) in the paper" from the `candidate_runs`
|
||||
row the run persisted for every considered article: the stage it reached and the
|
||||
reason it stopped, every raw and normalized signal with its presence and
|
||||
effective weight, the top interests, the nearest rated neighbours, the triage
|
||||
and deep assessments (quality, fit, category, rationale, facets), utility and
|
||||
rank, the cluster it landed in and what suppressed it, and the editor's reason
|
||||
for a pick. `--url` canonicalizes the address; an article that is not in the
|
||||
database at all is reported as never ingested (a feed problem, not a ranking
|
||||
one). `--near-misses` lists the highest-utility articles that were not selected
|
||||
(by preliminary blend for articles the ranker never reached).
|
||||
|
||||
`stats` is the whole evaluation framework, on purpose: for the last `--days`
|
||||
(14) it prints the issues and articles published, the mean issue size, explicit
|
||||
ratings by label and per issue, the up/down ratio of rated picks per admitting
|
||||
retriever (`admitted_by[0]` — triage, interest, knn, exploration, blend,
|
||||
auto_include), the exploration yield (admitted, selected, rated positively),
|
||||
cost per day per provider from `runs.provider_costs_json`, and the mean
|
||||
generation time. Plain text, one fact per line. Tune from it and from reading
|
||||
the paper; anything more waits for more ratings.
|
||||
|
||||
`features backfill` embeds the rated and published articles first (the learned
|
||||
set), then the standing interests, then — only with `--all` — every other
|
||||
article first seen in the window. It prints an estimate and asks before spending
|
||||
more than 5M tokens unless `--yes`; a warm cache makes zero calls. `features
|
||||
prune` drops embeddings of articles neither rated nor published that are older
|
||||
than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows and
|
||||
`article_assessments` older than `curation.ranking.telemetry_retention_days`.
|
||||
`generate` runs the same sweep once after publishing, best effort.
|
||||
|
||||
`config check` loads and validates the configuration exactly as `generate`
|
||||
would and prints one fact per line: the config path, the database, profile,
|
||||
interests and publish paths with `exists`/`MISSING`, each `[llm]` role with its
|
||||
provider name, kind, model, effort, ceiling and whether its key is present, the
|
||||
Voyage line likewise, and `editorial.summary_model`. Lines that need attention
|
||||
start with `!`. It exits non-zero only on a validation error — a missing key or
|
||||
file is a warning, since the run degrades rather than fails — and never opens
|
||||
the database or takes the lock, so it is safe to run next to a live `generate`.
|
||||
|
||||
## Web site and dashboard
|
||||
|
||||
The server is both the public newspaper index and the private operator UI. An
|
||||
anonymous visitor sees titles, authors, sources, metadata, AI summaries and
|
||||
outbound comment links; article bodies, the Brief, the World Briefing
|
||||
and comments stay private. A signed-in `user`
|
||||
sees complete issues and article chapters and can download available formats
|
||||
from a single download menu. An
|
||||
`admin` can additionally follow direct dashboard links from issue entries and
|
||||
article chapters, rate articles, and use every `/dashboard/*` page, including
|
||||
settings and jobs. `/dashboard/interests` lists the standing interests with
|
||||
their LLM-assigned category and a rating-derived weight, adds or deletes them,
|
||||
and links every interest name in the UI to the articles it matched.
|
||||
Personalization is shared across accounts for now.
|
||||
|
||||
Visitors can request an account with their preferred username at
|
||||
`/request-access`; admins review open requests on `/dashboard/users`. Approving
|
||||
a request creates a `user` account,
|
||||
emails a random temporary password to the requester, and marks the request
|
||||
done; the new user must choose a new password on first sign-in. Approval is
|
||||
available only when `[mail]` is active, so an account is never created with a
|
||||
password that cannot be delivered. Admins can instead create an account with
|
||||
`daily-epub users add <username>` on the server and mark the request done.
|
||||
Usernames are case-insensitive and passwords must be 12–1024 characters.
|
||||
Bootstrap with `daily-epub users add <username> --admin`; use `users passwd`, `role`,
|
||||
`disable`/`enable`, `list`, and `logout` for later administration. Password
|
||||
changes and disabling a user revoke that user's sessions. `/dashboard/users`
|
||||
also shows roles, status, login times, and open sessions.
|
||||
When `[mail]` is active and `mail.notify_to` is set, each stored request queues
|
||||
a plain-text SMTP notification with a direct dashboard review link; SMTP runs
|
||||
in the background and never delays the visitor's response. Mail settings are
|
||||
loaded at server startup, so changes require a service restart.
|
||||
|
||||
The Jobs page starts only the fixed job catalogue as
|
||||
`daily-epub-job@<name>.service`; the web server never runs the pipeline inside
|
||||
its own process. Install `systemd/daily-epub-job@.service` and the narrowly
|
||||
scoped `systemd/50-daily-epub.rules` polkit rule, and put the server user in
|
||||
`systemd-journal` so the page can show status and its configured journal tail.
|
||||
Set `server.jobs_enabled = false` to make starts unavailable.
|
||||
|
||||
| Job name | Action |
|
||||
|---|---|
|
||||
| `generate` / `generate-YYYY-MM-DD` | Build and publish an issue. |
|
||||
| `dry-run` | Run the pipeline without publishing or recording an issue. |
|
||||
| `profile-rebuild` | Rebuild learned taste adjustments from ratings. |
|
||||
| `features-backfill` | Embed rated and recently published articles and interests. |
|
||||
| `backfill-social` | Refresh recent social scores. |
|
||||
| `features-prune` | Remove stale embeddings and curation telemetry. |
|
||||
| `import-ratings` | Fetch, embed, and rate URLs queued from the Ratings page. |
|
||||
| `interests-categorize` | File uncategorized interests under categories with the bulk model. |
|
||||
|
||||
The Ratings page accepts up to 500 historical article URLs at a time with one
|
||||
verdict and optional note. Imports run in the background and show per-URL
|
||||
status in the page plus live logs on the job page. If dashboard jobs are
|
||||
disabled, queueing still works; run `daily-epub job run import-ratings` by hand.
|
||||
|
||||
The Feeds page lists feeds discovered behind articles that reached the paper
|
||||
only through an aggregator (Hacker News, Lobsters, Reddit, Scour), ranked by
|
||||
the same per-article signals the paper selects on, so the strongest leads sit
|
||||
at the top. Each row offers **Add**, which subscribes the feed in a chosen
|
||||
Miniflux category, and **Dismiss**. Discovery runs as a best-effort stage
|
||||
during `generate`; `daily-epub feeds discover --days 14 --limit 50` seeds the
|
||||
page from articles already in the database.
|
||||
|
||||
The Settings page derives its fields from `Config`, rewrites `config.toml` in
|
||||
place with `toml_edit`, preserves comments/order and file permissions, validates
|
||||
before an atomic rename, and records attributed history. It re-reads hand edits
|
||||
on the next view. `DAILY_EPUB_*` overrides appear locked, and secrets are shown
|
||||
only as present or absent. Shipped providers (`deepseek`, `anthropic`,
|
||||
`gemini`) cannot be removed because defaults would restore them; leave one
|
||||
unreferenced or edit it. Custom providers can be removed after no `[llm]` role
|
||||
references them. The service needs `/etc/daily-epub` in `ReadWritePaths` for
|
||||
these writes.
|
||||
|
||||
An article detail page shows the ten closest articles across all stored,
|
||||
compatible embeddings, independent of runs or ratings, alongside the rated
|
||||
neighbours captured by the latest run. Its triage kind and deep-assessment
|
||||
format also distinguish repositories, documentation, product pages,
|
||||
discussions, papers, interviews, media, and fiction instead of forcing those
|
||||
pages into essay or report labels. Run `features backfill --all` when recent
|
||||
articles do not yet have embeddings to compare.
|
||||
|
||||
### Web routes
|
||||
|
||||
| Route | Access | Purpose |
|
||||
|---|---|---|
|
||||
| `GET /`, `/issues`, `/issues/{date}`, `/feed.xml`, `/feed.json` | Public | Latest issue, archive, stripped issue index, and the equivalent Atom and JSON Feed 1.1 feeds. Signed-in issue views expand to the complete issue. |
|
||||
| `GET /issues/{date}/articles/{id}`, `/world`, `/behind` | User or admin | Private article, World Briefing, and Behind the paper chapters. |
|
||||
| `GET /issues/{date}/read` | User or admin | Open the Standard edition in BookOrbit's web reader when the integration is enabled. |
|
||||
| `GET /robots.txt`, `/static/{file}` | Public | Crawler policy and embedded CSS, JavaScript, and favicon. |
|
||||
| `GET/POST /login`, `POST /logout` | Public/session | Sign in and out; login POSTs share a per-client-IP throttle budget with access requests. |
|
||||
| `GET/POST /request-access` | Public | Request a reader account; POSTs share the login throttle, while GET stays unlimited. Requests are reviewed by an admin. |
|
||||
| `GET /account`, `POST /account/password`, `/account/logout-all` | User or admin | Change the current password or revoke sessions; temporary-password users must change it before opening protected pages. |
|
||||
| `POST /rate` | Admin | Append an attributed dashboard rating event. |
|
||||
| `GET /dashboard` | Admin | Run, budget, rating, job, and config overview. |
|
||||
| `GET /dashboard/runs[/{id}]`, `/articles[/{id}]`, `/ratings`, `/stats` | Admin | Pipeline history, article explanations, rating contributions/history, historical URL imports, and evaluation stats. |
|
||||
| `POST /dashboard/ratings/import` | Admin | Queue historical URLs with a verdict and start the background import job. |
|
||||
| `GET/POST /dashboard/profile`, `POST /dashboard/profile/restore` | Admin | Edit `profile.md`, inspect prompts/adjustments, and restore a version. |
|
||||
| `GET/POST /dashboard/settings`, `POST /dashboard/settings/providers`, `GET /dashboard/settings/history` | Admin | Edit validated configuration and inspect its audit log. |
|
||||
| `GET /dashboard/jobs`, `GET /dashboard/jobs/{id}`, `POST /dashboard/jobs/{name}` | Admin | Start fixed systemd jobs and inspect status and logs. |
|
||||
| `GET /dashboard/users`, `POST /dashboard/users/requests/{id}/approve`, `POST /dashboard/users/requests/{id}/done` | Admin | Approve and email access requests, mark requests handled another way, and view users/open sessions; other account edits use the CLI. |
|
||||
| `GET /files/epub/{name}`, `/files/xtc/{name}` | Public if Basic auth is unset; otherwise session or Basic auth | Published downloads. Keeping them public when Basic auth is absent preserves existing OPDS acquisition links. |
|
||||
| `GET /opds`, `/opds/`, `/opds/daily.xml` | Existing optional Basic auth | OPDS acquisition feed. |
|
||||
| `GET /r/...`, `/healthz`, `/issues.json` | Existing policy | HMAC rating links, health, and issue reports. |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
@@ -93,12 +320,43 @@ Start from [`config.example.toml`](config.example.toml). Load order, later wins:
|
||||
3. `DAILY_EPUB_*` environment variables
|
||||
|
||||
Nested keys use a **double underscore**: `[miniflux] api_key` becomes
|
||||
`DAILY_EPUB_MINIFLUX__API_KEY`. Top-level keys are just uppercased:
|
||||
`DAILY_EPUB_LOOKBACK_HOURS=30`. As a convenience, plain **`DAILY_EPUB_SECRET`**
|
||||
is accepted as an alias for `server.hmac_secret` (the explicit key wins if both
|
||||
are set).
|
||||
`DAILY_EPUB_MINIFLUX__API_KEY`, and a provider's key is
|
||||
`DAILY_EPUB_PROVIDERS__<NAME>__API_KEY` with the `[providers.<name>]` table name
|
||||
upper-cased — the shipped registry reads `DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY`,
|
||||
`DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY` and
|
||||
`DAILY_EPUB_PROVIDERS__GEMINI__API_KEY` (provider names are therefore lowercase
|
||||
`a-z0-9_`). Top-level keys are just uppercased: `DAILY_EPUB_LOOKBACK_HOURS=30`.
|
||||
As a convenience, plain **`DAILY_EPUB_SECRET`** is accepted as an alias for
|
||||
`server.hmac_secret` (the explicit key wins if both are set).
|
||||
|
||||
Secrets belong in the environment file, never in the TOML.
|
||||
Secrets belong in the environment file, never in the TOML. Stale configuration
|
||||
fails at startup rather than silently curating without a provider: a
|
||||
`[deepseek]` or `[anthropic]` table, a top-level `max_daily_usd`, a batch-size
|
||||
or temperature key outside `[llm]`, or a `DAILY_EPUB_DEEPSEEK__*` /
|
||||
`DAILY_EPUB_ANTHROPIC__*` environment variable is an error naming the new key.
|
||||
|
||||
### Switching providers
|
||||
|
||||
The roles are names, the providers are tables. To run the editor on Gemini:
|
||||
|
||||
```toml
|
||||
[llm]
|
||||
editor = "gemini" # [providers.gemini] is already declared in config.example.toml
|
||||
```
|
||||
|
||||
and put `DAILY_EPUB_PROVIDERS__GEMINI__API_KEY=…` in the env file. A one-off
|
||||
A/B without touching the file, since every key is also an env var:
|
||||
|
||||
```sh
|
||||
DAILY_EPUB_LLM__EDITOR=gemini daily-epub generate --dry-run --date 2026-09-02
|
||||
```
|
||||
|
||||
The same works for `bulk`. Both roles may name one provider (they then share
|
||||
one client and one `max_daily_usd`), `editor = ""` runs everything on bulk, and
|
||||
a new endpoint is a new `[providers.<name>]` table: `kind = "openai"` for any
|
||||
OpenAI-compatible chat-completions API (DeepSeek, Gemini, OpenAI, a local
|
||||
server), `kind = "anthropic"` for the Messages API. `daily-epub config check`
|
||||
prints what resolved.
|
||||
|
||||
### Reference
|
||||
|
||||
@@ -106,33 +364,81 @@ Secrets belong in the environment file, never in the TOML.
|
||||
|---|---|---|
|
||||
| `timezone` | `America/New_York` | Day boundaries and `--date` interpretation. |
|
||||
| `lookback_hours` | `26` | Size of the ingest window ending at the issue day's end (clamped to now). |
|
||||
| `target_article_count` | `20` | Lineup size the selector aims for. `--max-articles` overrides it. |
|
||||
| `prefilter_keep` | `120` | Candidates surviving the heuristic pre-filter. Must be ≥ `target_article_count`. |
|
||||
| `target_article_count` | `20` | Soft target the editor aims for. There is no minimum: a nine-pick issue is published as nine. |
|
||||
| `retention_days` | `21` | EPUBs older than this are deleted from `publish.epub_dir`. SQLite history is kept forever. |
|
||||
| `xtc_retention_count` | `5` | How many XTC issues to keep in `publish.xtc_dir`. Counted, not dated: each `.xtch` is ~80–100 MB, so the binding constraint is disk, not age. |
|
||||
| `max_daily_usd` | `2.0` | Hard ceiling on DeepSeek spend **per day**, not per run — a re-run inherits what earlier runs for that date already spent. Tripping it skips remaining LLM work and degrades to excerpts. |
|
||||
| `world_briefing` | `true` | Include the Wikipedia Current Events section. |
|
||||
| `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. |
|
||||
| `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. |
|
||||
| `interests_opml` | `data/scour-interests.opml` | Scour interest export used to seed the taste profile. |
|
||||
| `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. |
|
||||
| `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). |
|
||||
| `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. |
|
||||
| `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. |
|
||||
| `deepseek.base_url` | `https://api.deepseek.com/v1` | OpenAI-compatible endpoint. |
|
||||
| `deepseek.model` | `deepseek-v4-flash` | Verified 2026-08-15 (DeepSeek-V4-Flash-0731). |
|
||||
| `deepseek.api_key` | — | **`DAILY_EPUB_DEEPSEEK__API_KEY`**. Absent ⇒ the run curates heuristically. |
|
||||
| `deepseek.score_batch_size` | `12` | Articles per stage-A scoring request. |
|
||||
| `deepseek.score_temperature` | `0.3` | Scoring/selection temperature. |
|
||||
| `deepseek.editorial_temperature` | `0.8` | Summaries, intros, front page. |
|
||||
| `deepseek.price_input_per_mtok` | `0.14` | USD per 1M cache-miss input tokens (cost guardrail arithmetic). |
|
||||
| `deepseek.price_cached_input_per_mtok` | `0.0028` | USD per 1M prefix-cache-hit input tokens. |
|
||||
| `deepseek.price_output_per_mtok` | `0.28` | USD per 1M output tokens. |
|
||||
| `llm.bulk` | `deepseek` | The `[providers.*]` name that runs triage, deep assessment and every fallback. `""` ⇒ no bulk provider (those stages are skipped). |
|
||||
| `llm.editor` | `anthropic` | The provider that assembles the lineup, writes the summaries and The Brief and rebuilds the profile. `""` or absent ⇒ everything runs on `bulk`. Naming the same provider as `bulk` shares one client and one ceiling. |
|
||||
| `llm.triage_batch_size` | `25` | Articles per first-pass triage request. |
|
||||
| `llm.deep_batch_size` | `8` | Articles per close-reading assessment request. The removed `score_batch_size` key is a startup error. |
|
||||
| `llm.score_temperature` | `0.3` | Scoring temperature, sent only to `openai`-kind providers. |
|
||||
| `llm.editorial_temperature` | `0.8` | Summaries and The Brief on an `openai`-kind provider. |
|
||||
| `providers.<name>.kind` | — | `openai` (chat completions at `{base_url}/chat/completions`, bearer key) or `anthropic` (Messages API: `output_config.effort`, a cached system block, `fallbacks = "default"` with the `server-side-fallback-2026-07-01` beta). Shipped entries: `deepseek`, `anthropic`, `gemini`. |
|
||||
| `providers.<name>.base_url` | — | Endpoint root. DeepSeek `https://api.deepseek.com/v1`; Anthropic `https://api.anthropic.com`; Gemini `https://generativelanguage.googleapis.com/v1beta/openai`. |
|
||||
| `providers.<name>.model` | — | `deepseek-v4-flash` (verified 2026-08-15), `claude-opus-5`, `gemini-3.8-flash` (verified 2026-09-02). |
|
||||
| `providers.<name>.api_key` | — | **`DAILY_EPUB_PROVIDERS__<NAME>__API_KEY`**, environment only. Absent ⇒ that role is unavailable and degrades (bulk ⇒ heuristic curation, editor ⇒ bulk). |
|
||||
| `providers.<name>.effort` | `high` (anthropic, gemini) | `anthropic`: `low`, `medium`, `high`, `xhigh` or `max` → `output_config.effort`. `openai`: passed through as `reasoning_effort` (Gemini takes `minimal`–`high`); omit it for models without one (DeepSeek). |
|
||||
| `providers.<name>.max_daily_usd` | `2.0` / `3.0` / `3.0` | That provider's ceiling per **UTC day** of the run's start, not per run — a re-run inherits what earlier runs that day already spent on it (`runs.provider_costs_json`). Tripping it skips that provider's remaining calls; in-flight requests finish and the paper still publishes. `0` disables the guard. |
|
||||
| `providers.<name>.max_concurrent_requests` | `4` | Triage and deep-assessment batches in flight on the bulk provider; summaries in flight on the summary provider. |
|
||||
| `providers.<name>.price_input_per_mtok` | `0.14` / `5.0` / `0.75` | USD per 1M cache-miss input tokens (cost guardrail arithmetic). |
|
||||
| `providers.<name>.price_cache_read_per_mtok` | `0.0028` / `0.5` / `0.075` | USD per 1M cache-hit input tokens. |
|
||||
| `providers.<name>.price_cache_write_per_mtok` | `0.0` / `6.25` / `0.0` | USD per 1M tokens written to the prompt cache (implicit caches charge nothing). |
|
||||
| `providers.<name>.price_output_per_mtok` | `0.28` / `25.0` / `3.75` | USD per 1M output tokens, thinking tokens included where the provider bills them as output. |
|
||||
| `voyage.enabled` | `true` | Embed articles and interests with Voyage AI. `false` ⇒ cached vectors only. |
|
||||
| `voyage.base_url` | `https://api.voyageai.com/v1` | `POST {base_url}/embeddings`. |
|
||||
| `voyage.model` | `voyage-4-lite` | Embedding model; changing it invalidates the cache. |
|
||||
| `voyage.api_key` | — | **`DAILY_EPUB_VOYAGE__API_KEY`**. Absent ⇒ cached vectors only. |
|
||||
| `voyage.output_dimension` | `512` | One of 256, 512, 1024, 2048. |
|
||||
| `voyage.batch_size` | `32` | Texts per request. |
|
||||
| `voyage.max_concurrent_requests` | `4` | Requests in flight. |
|
||||
| `voyage.max_input_chars` | `60000` | Per-article cut, on a char boundary. |
|
||||
| `voyage.max_daily_usd` | `0.50` | Runaway guard at $0.02/M tokens. |
|
||||
| `curation.max_article_count` | `28` | Hard ceiling on issue size. `--max-articles N` lowers it to `min(28, N)` and drags the soft target down with it. Must be ≥ `target_article_count`. |
|
||||
| `curation.always_include_feeds` | `[]` | Miniflux feed ids or URL substrings that can never be dropped. |
|
||||
| `curation.blocked_domains` | `[]` | Hosts excluded outright. |
|
||||
| `curation.paywall_domains` | `[]` | Extra paywalled hosts, merged with the built-in list (nytimes, wsj, ft, economist, …). |
|
||||
| `curation.sections` | 8 sections | The **only** section names the model may use. `World Briefing` is reserved and never offered. |
|
||||
| `curation.feedback.loved_value` | `1.0` | Weight for a Loved it verdict. |
|
||||
| `curation.feedback.good_value` | `0.35` | Weight for a Good verdict. |
|
||||
| `curation.feedback.not_for_me_value` | `-1.0` | Weight for a Not for me verdict. |
|
||||
| `curation.feedback.slop_value` | `-1.0` | Weight for an AI slop verdict; the author penalty is `curation.ranking.slop_author_penalty`. |
|
||||
| `curation.feedback.verdicts_in_prompt` | `60` | Recent explicit verdicts included in the system prompt. |
|
||||
| `curation.recent_rejection_days` | `7` | Churn window for recent low triage/deep assessments. |
|
||||
| `curation.recent_rejection_floor` | `3.0` | Scores below this floor are excluded during the churn window (except auto-includes). |
|
||||
| `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. |
|
||||
| `curation.ranking.affinity_floor` / `affinity_full` | `15` / `40` | Interest-attributable ratings where affinity starts and reaches full weight. |
|
||||
| `curation.ranking.weights.preliminary.affinity` | `0.10` | Rating-derived interest affinity in the preliminary blend. |
|
||||
| `curation.ranking.weights.preliminary.interest` | `0.30` | Interest similarity in the preliminary blend. |
|
||||
| `curation.ranking.weights.preliminary.social` | `0.05` | Social signal in the preliminary blend. |
|
||||
| `curation.ranking.weights.utility.affinity` | `0.05` | Rating-derived interest affinity in the utility score. |
|
||||
| `curation.ranking.weights.utility.knn` | `0.10` | Rated-neighbour preference in the utility score. |
|
||||
| `editorial.summary_model` | `editor` | Which `[llm]` role writes the per-article summaries: `editor` (with per-article bulk fallback) or `bulk`. |
|
||||
| `editorial.summary_input_tokens` | `3000` | Article text offered to the summary prompt. |
|
||||
| `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. |
|
||||
| `publish.xtc_dir` | `/var/lib/daily-epub/xtc` | XTC artifacts. **Not** listed in the OPDS feed — CrossPoint cannot acquire them — but downloadable at `/files/xtc/<name>` for sideloading. |
|
||||
| `bookorbit.enabled` | `false` | Enable the admin-only **Read in BookOrbit** integration when both OPDS credentials are set. |
|
||||
| `bookorbit.public_url` | `https://bookorbit.hallada.net` | Browser-facing BookOrbit base URL. |
|
||||
| `bookorbit.api_url` | `http://127.0.0.1:3498` | Server-facing BookOrbit base URL used for OPDS lookups. |
|
||||
| `bookorbit.opds_user` | unset | Dedicated OPDS user created in BookOrbit's Settings → OPDS. |
|
||||
| `bookorbit.opds_pass` | — | **`DAILY_EPUB_BOOKORBIT__OPDS_PASS`**, environment only. |
|
||||
| `mail.enabled` | `false` | Enable outbound SMTP when the relay, sender, username, and password are configured. Mail settings require a server restart. |
|
||||
| `mail.smtp_host` | `""` | SMTP relay hostname, such as `email-smtp.us-east-1.amazonaws.com`. |
|
||||
| `mail.smtp_port` | `587` | SMTP relay port. Use 587 with STARTTLS or commonly 465 with implicit TLS. |
|
||||
| `mail.smtp_starttls` | `true` | `true` uses STARTTLS; `false` uses implicit TLS. |
|
||||
| `mail.smtp_user` | unset | SMTP username. It may be supplied as `DAILY_EPUB_MAIL__SMTP_USER`. |
|
||||
| `mail.smtp_pass` | — | **`DAILY_EPUB_MAIL__SMTP_PASS`**, environment only. |
|
||||
| `mail.from` | `""` | Sender mailbox, either a bare address or `Name <address>`. |
|
||||
| `mail.notify_to` | unset | Recipient for access-request notifications. |
|
||||
| `discovery.enabled` | `true` | Run the feed discovery stage during `generate`. |
|
||||
| `discovery.max_lookups_per_run` | `30` | Hosts one run may look up in Miniflux. Each host is re-checked at most every 90 days. |
|
||||
| `discovery.skip_hosts` | aggregators, code hosts, social networks | Hosts never looked up. A host matches itself or any subdomain of it. |
|
||||
| `xtc.enabled` | `true` | Set `false` to skip the converter entirely. |
|
||||
| `xtc.command` | `node` | Converter executable. |
|
||||
| `xtc.args` | `["/opt/epub-to-xtc-converter/cli/index.js", "convert"]` | Prefix; the code appends `<input.epub> -o <output> -f <format>` (plus `-c <settings>`). |
|
||||
@@ -141,7 +447,40 @@ Secrets belong in the environment file, never in the TOML.
|
||||
| `server.bind` | `127.0.0.1:3499` | Listen address. |
|
||||
| `server.public_url` | `https://daily.hallada.net` | Base URL the rating links inside the EPUB are built from. |
|
||||
| `server.hmac_secret` | — | **`DAILY_EPUB_SERVER__HMAC_SECRET`** (or `DAILY_EPUB_SECRET`). Without it, generated links are rejected with 403. |
|
||||
| `server.basic_auth_user` / `_pass` | unset | Optional Basic auth for `/opds/*` and `/files/*`. |
|
||||
| `server.basic_auth_user` / `_pass` | unset | Optional Basic auth for `/opds/*` and `/files/*`; signed-in web users may download from `/files/*` without Basic auth. |
|
||||
| `server.session_days` | `30` | Sliding lifetime for dashboard login sessions. |
|
||||
| `server.login_attempts` | `10` | Shared login and access-request POSTs allowed per IP in one throttle window. |
|
||||
| `server.login_window_minutes` | `15` | Length of the login throttle window. |
|
||||
| `server.jobs_enabled` | `true` | Allow the dashboard to start the fixed systemd job catalogue. |
|
||||
| `server.journal_lines` | `300` | Journal lines shown on a dashboard job page (10–5000). |
|
||||
|
||||
`[curation.ranking]` holds the ranker's tunables. The learned signals are
|
||||
gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to
|
||||
`knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from
|
||||
`feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the
|
||||
signal is absent. `affinity` (rating-derived interest affinity) likewise ramps
|
||||
from `affinity_floor` (15) to `affinity_full` (40) interest-attributable ratings.
|
||||
Ratings decay with `rating_half_life_days` (60) over
|
||||
`rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and
|
||||
`negative_coefficient` (0.75) shape the signal. `slop_author_penalty` (0.75)
|
||||
is the fraction of the blend and utility removed from every candidate whose
|
||||
author currently carries an AI slop verdict, on any feed and with no age
|
||||
limit. `triage_max` (800),
|
||||
`deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3),
|
||||
`semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]`
|
||||
(`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]`
|
||||
(`quality` 0.40 · `fit` 0.20 · `knn` 0.10 · `affinity` 0.05 · `interest` 0.10 · `feed` 0.05 ·
|
||||
`triage` 0.05 · `social` 0.03 · `heuristic` 0.02, over the signals present for
|
||||
each article of the deep set) and `[curation.ranking.diversity]`
|
||||
(`cluster_threshold` 0.85, `per_cluster_cap` 2, `utility_protected` 10) drive
|
||||
the LLM triage, deep assessment, utility ranking and diversification stages.
|
||||
`[curation.ranking.weights.preliminary]` (`interest` 0.30 · `knn` 0.25 ·
|
||||
`affinity` 0.10 · `heuristic` 0.20 · `feed` 0.10 · `social` 0.05) blends the cheap signals; weights
|
||||
are renormalized over the signals present for each article, so they need not sum
|
||||
to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are
|
||||
what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥
|
||||
shortlist_keep ≥ target_article_count`; `*_full > *_floor`; `0 ≤
|
||||
cluster_threshold ≤ 1`; `per_cluster_cap ≥ 1`; Voyage batch size and concurrency ≥ 1.
|
||||
|
||||
---
|
||||
|
||||
@@ -165,10 +504,22 @@ sudo install -m0640 -o daily-epub -g daily-epub config.example.toml /etc/daily-e
|
||||
sudo -e /etc/daily-epub/config.toml # set publish dirs, xtc args, public_url
|
||||
sudo tee /etc/daily-epub/env >/dev/null <<EOF
|
||||
DAILY_EPUB_MINIFLUX__API_KEY=…
|
||||
DAILY_EPUB_DEEPSEEK__API_KEY=…
|
||||
DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY=…
|
||||
DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY=…
|
||||
# DAILY_EPUB_PROVIDERS__GEMINI__API_KEY=… # only if a role names "gemini"
|
||||
DAILY_EPUB_VOYAGE__API_KEY=…
|
||||
DAILY_EPUB_SERVER__HMAC_SECRET=$(openssl rand -hex 32)
|
||||
DAILY_EPUB_MAIL__SMTP_USER=…
|
||||
DAILY_EPUB_MAIL__SMTP_PASS=…
|
||||
EOF
|
||||
sudo chown daily-epub:daily-epub /etc/daily-epub/env && sudo chmod 0600 /etc/daily-epub/env
|
||||
# One key per [providers.<name>] entry a role uses, named after the table
|
||||
# (upper-cased). Any may be left unset: that role is then unavailable and the
|
||||
# run degrades (editor → bulk, bulk → heuristic curation, Voyage → cached
|
||||
# vectors). The pre-registry DAILY_EPUB_DEEPSEEK__API_KEY /
|
||||
# DAILY_EPUB_ANTHROPIC__API_KEY names are a startup error, not a silent no-op.
|
||||
sudo -u daily-epub bash -c 'set -a; . /etc/daily-epub/env; set +a;
|
||||
daily-epub --config /etc/daily-epub/config.toml config check' # roles, keys present?, paths
|
||||
|
||||
# publish dirs must exist and be writable by the service user
|
||||
sudo install -d -o daily-epub -g daily-epub /var/lib/daily-epub/xtc
|
||||
@@ -176,15 +527,21 @@ sudo setfacl -m u:daily-epub:rwx /srv/bookorbit/libraries/daily-epub # or chow
|
||||
|
||||
# units
|
||||
sudo install -m0644 systemd/daily-epub.service systemd/daily-epub-generate.service \
|
||||
systemd/daily-epub-generate.timer /etc/systemd/system/
|
||||
systemd/daily-epub-generate.timer systemd/daily-epub-job@.service \
|
||||
/etc/systemd/system/
|
||||
sudo install -m0644 systemd/50-daily-epub.rules /etc/polkit-1/rules.d/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now daily-epub.service daily-epub-generate.timer
|
||||
```
|
||||
|
||||
> **Keep `ReadWritePaths` in sync.** Both units run under `ProtectSystem=strict`
|
||||
> and list the publish directories explicitly:
|
||||
> **Keep `ReadWritePaths` in sync.** The units run under `ProtectSystem=strict`
|
||||
> and list the publish directories explicitly; the server additionally lists
|
||||
> `/etc/daily-epub` so Settings can replace `config.toml`:
|
||||
> ```
|
||||
> # generate and job units
|
||||
> ReadWritePaths=/home/thallada/bookorbit/books/daily-epub /var/lib/daily-epub/xtc
|
||||
> # server unit
|
||||
> ReadWritePaths=/home/thallada/bookorbit/books/daily-epub /var/lib/daily-epub/xtc /etc/daily-epub
|
||||
> ```
|
||||
> If you change `publish.epub_dir` or `publish.xtc_dir` in the config, change
|
||||
> these lines too and `systemctl daemon-reload`, or publishing fails with
|
||||
@@ -198,25 +555,95 @@ Node for the XTC converter, whose JIT needs W+X pages.
|
||||
The rating links baked into every article chapter point at
|
||||
`server.public_url`, so `daily.hallada.net` must resolve and serve TLS from the
|
||||
internet (e-readers tap these links). The OPDS catalog rides on the same host.
|
||||
With an existing certificate, a minimal nginx site is:
|
||||
|
||||
The login throttle's `SmartIpKeyExtractor` keys on the first entry of
|
||||
`X-Forwarded-For`, so nginx must *set* that header from the connection's peer
|
||||
address rather than appending to whatever the client sent — otherwise anyone
|
||||
can mint a fresh throttle bucket per attempt by sending their own header. (The
|
||||
original config appended with `$proxy_add_x_forwarded_for`; that was the
|
||||
spoofable version.) The whole arrangement is only safe because the configured
|
||||
bind address is loopback and nginx is the only process that can reach it.
|
||||
|
||||
Nothing sits in front of nginx. The site ran behind the Cloudflare proxy for one
|
||||
day (2026-09-04 to 2026-09-05) and was taken back out: measured from Boston, the
|
||||
proxied signed-in page took 70 ms after the TLS handshake against 27 ms direct,
|
||||
and at this traffic the edge cache is cold for anonymous readers anyway. The zone
|
||||
still lives on Cloudflare's nameservers, so re-proxying is a one-click toggle;
|
||||
`docs/runbooks/cdn-rollout.md` keeps the zone settings, cache rules and the
|
||||
nginx additions (the `set_real_ip_from` snippet) that the proxied setup needs.
|
||||
|
||||
```nginx
|
||||
# One pool of idle connections to the app, so a request does not pay a fresh
|
||||
# loopback TCP connect (and leave a TIME-WAIT socket) every time.
|
||||
upstream daily_epub {
|
||||
server 127.0.0.1:3499;
|
||||
keepalive 8;
|
||||
}
|
||||
|
||||
server {
|
||||
# HTTP/3 (nginx ≥ 1.25 built --with-http_v3_module; nginx.org packages
|
||||
# are). QUIC folds the TCP and TLS handshakes into one round trip. UDP 443
|
||||
# must be open in the firewall. `reuseport` goes on exactly one quic
|
||||
# listener per address; other server blocks on this host share it.
|
||||
listen 443 quic reuseport;
|
||||
listen [::]:443 quic reuseport;
|
||||
listen 443 ssl;
|
||||
listen [::]:443 ssl;
|
||||
http2 on;
|
||||
http3 on;
|
||||
server_name daily.hallada.net;
|
||||
# Tell an HTTP/2 client that HTTP/3 is available on the same port; the
|
||||
# browser switches on its next connection and remembers for a day.
|
||||
add_header Alt-Svc 'h3=":443"; ma=86400' always;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/daily.hallada.net/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/daily.hallada.net/privkey.pem;
|
||||
ssl_trusted_certificate /etc/letsencrypt/live/daily.hallada.net/fullchain.pem;
|
||||
|
||||
# TLS 1.3 0-RTT: a returning browser sends its first request inside the
|
||||
# handshake and saves a round trip. Safe only because the app answers 425
|
||||
# to any non-GET that arrives as early data (the Early-Data header below).
|
||||
# Applies to the TCP listener; over QUIC, nginx needs OpenSSL ≥ 3.5.1 for
|
||||
# early data, and the nginx.org build links the distro's OpenSSL.
|
||||
ssl_early_data on;
|
||||
|
||||
include /etc/nginx/snippets/security-headers.conf;
|
||||
|
||||
# The global `gzip on` only covers text/html. The stylesheet is ~12 KB of
|
||||
# plain CSS again — the Newsreader faces are separate .woff2 URLs, not
|
||||
# base64 inside it (see the doc comment on `web::APP_CSS`) — so this list is
|
||||
# about the JSON, XML and SVG responses. woff2 is already compressed and is
|
||||
# deliberately absent.
|
||||
gzip_types text/css application/javascript application/json
|
||||
application/speculationrules+json image/svg+xml
|
||||
application/atom+xml application/xml text/plain;
|
||||
gzip_min_length 1024;
|
||||
gzip_vary on;
|
||||
gzip_proxied any;
|
||||
|
||||
# Keep a browser's connection open longer than the 75 s default so the
|
||||
# first click after a pause reuses it instead of paying a new TLS handshake.
|
||||
keepalive_timeout 300s;
|
||||
|
||||
# XTCH files can be tens of MB; don't buffer them through nginx memory.
|
||||
proxy_max_temp_file_size 0;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:3499;
|
||||
proxy_pass http://daily_epub;
|
||||
# Keep-alive to the upstream needs HTTP/1.1 and an empty Connection
|
||||
# header (the default would forward "close").
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
# SET, not append. The login throttle keys on the first X-Forwarded-For
|
||||
# entry, so a client-supplied header must never survive into the app.
|
||||
# $remote_addr is the connection's peer, which is the visitor itself
|
||||
# now that no proxy sits in front of nginx.
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
# "1" while the request arrived as 0-RTT data (RFC 8470); the app
|
||||
# rejects non-safe methods sent that way.
|
||||
proxy_set_header Early-Data $ssl_early_data;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,12 +655,88 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
Enable it (`ln -s` into `sites-enabled`, `nginx -t`, `systemctl reload nginx`),
|
||||
then `curl https://daily.hallada.net/healthz` should return `ok`. No auth is
|
||||
needed at the proxy layer: rating links are self-authenticating (HMAC tokens)
|
||||
and the OPDS/file routes use the app-level Basic auth from
|
||||
**Only while the record is proxied through Cloudflare:** the connection's peer
|
||||
is then Cloudflare, so `real_ip` has to rewrite `$remote_addr` from
|
||||
`CF-Connecting-IP` before the `X-Forwarded-For` line runs. Add
|
||||
`include /etc/nginx/snippets/cloudflare-real-ip.conf;` to the server block,
|
||||
above `location /`. Leave it out when the record is DNS-only: it would be inert
|
||||
for ordinary visitors, but anything connecting *from* a Cloudflare address (a
|
||||
Worker, say) could then name its own address and dodge the login throttle.
|
||||
`/etc/nginx/snippets/cloudflare-real-ip.conf` is one `set_real_ip_from` line per
|
||||
published Cloudflare range plus the header to read. Cloudflare adds ranges from
|
||||
time to time, so **regenerate it from the source rather than copying this list**
|
||||
(<https://www.cloudflare.com/ips-v4>, <https://www.cloudflare.com/ips-v6>):
|
||||
|
||||
```sh
|
||||
{ curl -s https://www.cloudflare.com/ips-v4; echo; \
|
||||
curl -s https://www.cloudflare.com/ips-v6; echo; } \
|
||||
| awk 'NF {print "set_real_ip_from " $0 ";"} END {print "real_ip_header CF-Connecting-IP;"}' \
|
||||
| sudo tee /etc/nginx/snippets/cloudflare-real-ip.conf
|
||||
```
|
||||
|
||||
As of this writing that produces:
|
||||
|
||||
```nginx
|
||||
set_real_ip_from 173.245.48.0/20;
|
||||
set_real_ip_from 103.21.244.0/22;
|
||||
set_real_ip_from 103.22.200.0/22;
|
||||
set_real_ip_from 103.31.4.0/22;
|
||||
set_real_ip_from 141.101.64.0/18;
|
||||
set_real_ip_from 108.162.192.0/18;
|
||||
set_real_ip_from 190.93.240.0/20;
|
||||
set_real_ip_from 188.114.96.0/20;
|
||||
set_real_ip_from 197.234.240.0/22;
|
||||
set_real_ip_from 198.41.128.0/17;
|
||||
set_real_ip_from 162.158.0.0/15;
|
||||
set_real_ip_from 104.16.0.0/13;
|
||||
set_real_ip_from 104.24.0.0/14;
|
||||
set_real_ip_from 172.64.0.0/13;
|
||||
set_real_ip_from 131.0.72.0/22;
|
||||
set_real_ip_from 2400:cb00::/32;
|
||||
set_real_ip_from 2606:4700::/32;
|
||||
set_real_ip_from 2803:f800::/32;
|
||||
set_real_ip_from 2405:b500::/32;
|
||||
set_real_ip_from 2405:8100::/32;
|
||||
set_real_ip_from 2a06:98c0::/29;
|
||||
set_real_ip_from 2c0f:f248::/32;
|
||||
real_ip_header CF-Connecting-IP;
|
||||
```
|
||||
|
||||
Enable the site (`ln -s` into `sites-enabled`, `nginx -t`, `systemctl reload
|
||||
nginx`), then `curl https://daily.hallada.net/healthz` should return `ok`.
|
||||
Every response carries `Server-Timing: app;dur=<ms>`, the time the app spent
|
||||
on the request; the browser's DevTools network panel shows it next to the
|
||||
network timings, and `curl -sI` prints it, so origin work and the network can
|
||||
be told apart without touching the logs. No
|
||||
auth is needed at the proxy layer: rating links are self-authenticating (HMAC
|
||||
tokens) and the OPDS/file routes use the app-level Basic auth from
|
||||
`server.basic_auth_user`/`_pass` if you set them.
|
||||
|
||||
#### HTTP caching
|
||||
|
||||
The origin says what may be cached and for how long. No shared cache sits in
|
||||
front of it today (the Cloudflare proxy was retired on 2026-09-05), but every
|
||||
response still carries an explicit policy so the proxy can come back with a DNS
|
||||
toggle and nothing else. The whole matrix:
|
||||
|
||||
| route | `Cache-Control` |
|
||||
|---|---|
|
||||
| `/`, `/issues`, `/issues/{date}`, `/feed.xml`, `/feed.json`, `/issues.json` (anonymous) | `public, max-age=300` |
|
||||
| the same pages with a `daily_session=` cookie | `private, no-store` |
|
||||
| `/robots.txt` | `public, max-age=86400` |
|
||||
| `/static/*?v=<hash>` | `public, max-age=31536000, immutable` |
|
||||
| `/static/*` with no `?v=` | `public, max-age=3600` |
|
||||
| `/files/epub/*`, `/files/xtc/*`, `/opds*` (every status) | `private, no-store` |
|
||||
| `/dashboard*`, `/login`, `/account`, errors, anything else | `no-store` |
|
||||
|
||||
Five minutes is the whole freshness story: an issue changes once a day, a
|
||||
reader's tab or a CDN edge revalidates within five minutes of the new one
|
||||
landing, and signed-in requests never touch a shared cache at all. A longer
|
||||
edge age with an API purge after each publish was considered and dropped as
|
||||
not worth its moving parts. The `no-store` on the last row is a default applied
|
||||
by the `security_headers` middleware to any response that set no policy of its
|
||||
own, so a route added later cannot silently inherit the CDN's default TTL.
|
||||
|
||||
### Installing the XTC converter
|
||||
|
||||
```sh
|
||||
@@ -287,6 +790,16 @@ This is deliberately independent of BookOrbit: it needs only the directory, so
|
||||
BookOrbit is optional, and it puts the day's issue one screen from the X4's home
|
||||
instead of several clicks down a library tree.
|
||||
|
||||
For desktop reading, the issue page can show admins a **Read in BookOrbit**
|
||||
button that opens the Standard edition in BookOrbit's web reader. Create an OPDS
|
||||
user in BookOrbit under Settings → OPDS, put its name in `config.toml`, put
|
||||
`DAILY_EPUB_BOOKORBIT__OPDS_PASS` in `/etc/daily-epub/env`, set
|
||||
`bookorbit.enabled = true`, and restart `daily-epub.service`; no systemd change
|
||||
is needed because the unit already allows loopback HTTP. Book and file ids are
|
||||
looked up lazily on the first click and cached on the `issues` row; if BookOrbit
|
||||
re-indexes a book, `/issues/<date>/read?refresh=1` clears the cache and resolves
|
||||
the ids again.
|
||||
|
||||
### Why XTC is not in the feed
|
||||
|
||||
XTC files are still generated and still land in `publish.xtc_dir` — they are just
|
||||
@@ -327,6 +840,7 @@ Condensed from spec §5. Run it in this order the first time.
|
||||
|
||||
```sh
|
||||
# 1. Config and schema
|
||||
daily-epub --config /etc/daily-epub/config.toml config check # roles → providers, keys present?
|
||||
daily-epub --config /etc/daily-epub/config.toml db migrate
|
||||
|
||||
# 2. Ingest only, no keys spent: does Miniflux answer, and with how much?
|
||||
@@ -336,12 +850,13 @@ DAILY_EPUB_OUT_DIR=./out daily-epub generate --dry-run --skip-llm --max-articles
|
||||
# 3. Inspect the artifacts
|
||||
ls -la ./out # two .epub files
|
||||
epubcheck "./out/The Daily EPUB - $(date +%F).epub" # expect zero errors
|
||||
# open the standard edition in Calibre / KOReader: cover, From the Editor,
|
||||
# In This Issue, sections, discussions, colophon; TOC depth 2
|
||||
# open the standard edition in Calibre / KOReader: cover, The Brief,
|
||||
# In This Issue, sections, discussions, Behind the paper, colophon; TOC depth 2
|
||||
|
||||
# 4. Now with DeepSeek, still not publishing
|
||||
# 4. Now with the bulk and editor providers, still not publishing
|
||||
daily-epub generate --dry-run --out ./out --max-articles 6
|
||||
# → check the lineup is sane and the printed cost is well under $0.50
|
||||
# → check the lineup is sane (at most 6 picks) and the
|
||||
# printed per-provider cost is well under $1
|
||||
|
||||
# 5. Full live run
|
||||
sudo systemctl start daily-epub-generate
|
||||
@@ -356,17 +871,17 @@ curl -s https://daily.hallada.net/healthz
|
||||
curl -s https://daily.hallada.net/opds/daily.xml | head
|
||||
curl -s https://daily.hallada.net/issues.json | jq '.[0]'
|
||||
|
||||
# 7. Feedback loop: tap 👍 in KOReader, then
|
||||
sqlite3 /var/lib/daily-epub/daily-epub.db 'select * from ratings;'
|
||||
sqlite3 /var/lib/daily-epub/daily-epub.db 'select * from feed_priors;'
|
||||
# 7. Feedback loop: tap Loved it / Good / Not for me / AI slop in KOReader, then
|
||||
sqlite3 /var/lib/daily-epub/daily-epub.db 'select * from rating_events order by event_at desc;'
|
||||
|
||||
# 8. Watch cost and quality for a week
|
||||
sqlite3 /var/lib/daily-epub/daily-epub.db \
|
||||
'select date, status, entries_fetched, candidates, selected, cost_usd from runs order by id desc limit 7;'
|
||||
daily-epub stats --days 7
|
||||
daily-epub explain --date $(date +%F) --near-misses
|
||||
```
|
||||
|
||||
Tune `prefilter_keep`, `target_article_count` and `curation.always_include_feeds`
|
||||
from what you see in step 8.
|
||||
Tune `curation.ranking.deep_keep`, `target_article_count` and `curation.always_include_feeds`
|
||||
from what you see in step 8, and read the paper's *Behind the paper* chapter
|
||||
each morning: it is the same numbers, on the device.
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
@@ -395,9 +910,12 @@ cargo fmt
|
||||
|
||||
The crate is a library plus a thin binary, so tests drive the pipeline directly.
|
||||
`tests/e2e_pipeline.rs` is the capstone: synthetic entries → dedupe → offline
|
||||
extraction → prefilter → selection (both the `--skip-llm` route and a
|
||||
extraction → signals → triage → admission → selection (both the `--skip-llm` route and a
|
||||
`MockBackend` DeepSeek route) → editorial → both EPUB editions → publish → OPDS
|
||||
and database rows, with no network access anywhere.
|
||||
and database rows, with no network access anywhere. `tests/m4_epub.rs` covers
|
||||
the rendered chapters, including *Behind the paper* in both editions; the
|
||||
`stats` and lock tests live next to their modules (`curate/telemetry.rs`,
|
||||
`lock.rs`).
|
||||
|
||||
### Layout
|
||||
|
||||
@@ -406,8 +924,9 @@ and database rows, with no network access anywhere.
|
||||
server. The stages themselves:
|
||||
|
||||
```text
|
||||
miniflux.rs ingest curate/ scoring and selection
|
||||
dedupe.rs clustering prefilter, llm, score, select, editorial
|
||||
miniflux.rs ingest curate/ triage, assessment and selection
|
||||
dedupe.rs clustering prefilter, llm, triage, admit, assess, rank,
|
||||
editor, editorial, embedding, signals, telemetry
|
||||
extract.rs body text profile/ the reader's taste profile
|
||||
images/ article images comments.rs discussion chapters
|
||||
normalize usable <img> world.rs the world briefing
|
||||
@@ -416,6 +935,8 @@ images/ article images comments.rs discussion chapters
|
||||
encode re-encode publish.rs BookOrbit + XTC
|
||||
embed into the page server.rs ratings, OPDS
|
||||
html.rs markup helpers db.rs SQLite
|
||||
interests.rs standing interests and stored article matches
|
||||
lock.rs one writer at a time (flock on <database_path>.lock)
|
||||
```
|
||||
|
||||
Two modules are worth knowing about before you go looking for their contents.
|
||||
@@ -478,8 +999,35 @@ From spec §7, plus what implementation turned up:
|
||||
stack, so `curate/llm.rs` speaks the OpenAI-compatible wire protocol over the
|
||||
shared `reqwest` client instead, behind a `ChatBackend` trait. The dependency
|
||||
was removed.
|
||||
- **Only DeepSeek is wired.** Another provider means another `ChatBackend` impl.
|
||||
- **No embedding-based personal ranker yet** (spec §3.9 future work); the schema
|
||||
is ready for it once ~200 ratings exist.
|
||||
- **One reader, one issue per day.** There is no multi-user support and no
|
||||
weekly/retrospective edition (spec §6).
|
||||
- **Two wire protocols are implemented**, `openai` (chat completions) and
|
||||
`anthropic` (Messages API), each a `ChatBackend` impl. Providers are config
|
||||
entries over those two kinds, each with its own `UsageMeter`, price table and
|
||||
ceiling; a protocol that is neither means another impl. Voyage AI embeddings
|
||||
sit behind the analogous `EmbeddingBackend` trait in `curate/embedding.rs`.
|
||||
- **Triage and union admission replace the heuristic gate.** Every eligible
|
||||
article gets interest, rated-neighbour, feed-affinity, interest-affinity,
|
||||
social and heuristic signals, then DeepSeek reads its opening (up to
|
||||
`triage_max`). The deep set is
|
||||
the union of triage, interest, neighbour, exploration, blend and auto-include
|
||||
retrievers. `explain` shows the assessment and `admitted_by`. Learned signals
|
||||
stay absent until their gates open (8, 15, and 15 ratings respectively).
|
||||
- **Deep assessment and diversity are live.** DeepSeek reads a representative
|
||||
beginning/middle/end sample, separates editorial quality from reader fit, and
|
||||
records descriptive facets. Utility is normalized over the deep set; embedding
|
||||
leader clusters cap near-duplicates before the 60-item editor shortlist.
|
||||
- **DeepSeek's content filter rejects whole batches.** A `400 Content Exists
|
||||
Risk` refuses the entire request when any one article in it trips the input
|
||||
filter, without saying which. Triage and deep assessment therefore bisect a
|
||||
rejected batch (halves, then quarters, down to single articles) so the other
|
||||
articles keep their assessment. A single article the bulk provider still
|
||||
refuses is retried once on the editor provider with the same prompt when that
|
||||
is a different provider; if that also fails, an `article_assessments` row with
|
||||
`kind = 'provider_rejected'` and a NULL score is written so the article is not
|
||||
sent again for `assessment_reuse_days`, and it is ranked on its other signals.
|
||||
See them with `explain` (`triage: rejected by provider — deepseek: …`), the
|
||||
`triage:` / `assess:` log lines (`… 3 rejected (2 recovered on gemini)`), the
|
||||
` · N rejected` suffix on the `curation:` line, or
|
||||
`sqlite3 /var/lib/daily-epub/daily-epub.db "select stage, count(*) from article_assessments where kind = 'provider_rejected' group by stage"`.
|
||||
- **One shared reader profile, one issue per day.** Multiple login accounts and
|
||||
`user`/`admin` roles are supported, but they share one personalization model;
|
||||
there is no per-user issue or weekly/retrospective edition (spec §6).
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
[general]
|
||||
# EPUB chapter templates live next to the builder that renders them
|
||||
# (implementation notes §11).
|
||||
dirs = ["src/epub/templates"]
|
||||
dirs = ["src/epub/templates", "src/web/templates"]
|
||||
+190
-15
@@ -3,17 +3,19 @@
|
||||
# Load order (later wins): built-in defaults ← this file ← `DAILY_EPUB_*` env vars.
|
||||
# Nested keys use a double underscore in env vars, e.g.
|
||||
# DAILY_EPUB_MINIFLUX__API_KEY=...
|
||||
# DAILY_EPUB_DEEPSEEK__API_KEY=...
|
||||
# DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY=...
|
||||
# DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY=...
|
||||
# DAILY_EPUB_PROVIDERS__GEMINI__API_KEY=...
|
||||
# DAILY_EPUB_VOYAGE__API_KEY=...
|
||||
# DAILY_EPUB_SERVER__HMAC_SECRET=...
|
||||
# DAILY_EPUB_LOOKBACK_HOURS=30
|
||||
# DAILY_EPUB_LLM__EDITOR=gemini # one-off role override, no file edit
|
||||
|
||||
timezone = "America/New_York"
|
||||
lookback_hours = 26
|
||||
target_article_count = 20
|
||||
prefilter_keep = 120
|
||||
retention_days = 21 # EPUBs, by age
|
||||
xtc_retention_count = 5 # XTC issues, by count (~80-100 MB each)
|
||||
max_daily_usd = 2.0
|
||||
world_briefing = true
|
||||
|
||||
# SQLite database file. Parent directories are created on demand.
|
||||
@@ -22,27 +24,95 @@ database_path = "/var/lib/daily-epub/daily-epub.db"
|
||||
# Default output directory for generated artifacts (overridden by `--out`).
|
||||
out_dir = "/var/lib/daily-epub/out"
|
||||
|
||||
# Path to the Scour interests OPML used to seed the taste profile (§3.6).
|
||||
interests_opml = "data/scour-interests.opml"
|
||||
# Hand-maintained reader profile loaded into the system prompt.
|
||||
profile_path = "data/profile.md"
|
||||
|
||||
[miniflux]
|
||||
base_url = "http://127.0.0.1:8082"
|
||||
# Browser-facing Miniflux web UI URL; defaults to base_url.
|
||||
# public_url = "https://miniflux.example.com"
|
||||
# api_key via DAILY_EPUB_MINIFLUX__API_KEY env
|
||||
page_limit = 250
|
||||
|
||||
[deepseek]
|
||||
# The two LLM roles, each assigned to a provider declared in [providers.*]
|
||||
# below. Switching the editor to Gemini is `editor = "gemini"` plus its key in
|
||||
# the env file; nothing else changes. An empty editor ("") runs everything on
|
||||
# the bulk provider. Every editor call degrades to bulk when the editor's key
|
||||
# is missing, its daily ceiling is hit, or the API refuses/fails.
|
||||
[llm]
|
||||
bulk = "deepseek" # triage, deep assessment, and every fallback
|
||||
editor = "anthropic" # lineup, summaries, The Brief, the weekly profile rebuild
|
||||
triage_batch_size = 25 # articles per first-pass triage request
|
||||
deep_batch_size = 8 # articles per close-reading assessment request
|
||||
score_temperature = 0.3 # sent only by providers that take a temperature (kind = "openai")
|
||||
editorial_temperature = 0.8 # summaries and The Brief on an openai-kind provider
|
||||
|
||||
# The provider registry. Any number of entries; a role above names one by its
|
||||
# table name. Keys never live here: DAILY_EPUB_PROVIDERS__<NAME>__API_KEY.
|
||||
# `max_daily_usd` is a per-provider runaway guard per UTC day (0 = none), not
|
||||
# accounting — set hard spend limits in each provider's dashboard as well.
|
||||
# Prices are USD per 1M tokens and only feed the guard's arithmetic.
|
||||
[providers.deepseek]
|
||||
kind = "openai" # openai | anthropic
|
||||
base_url = "https://api.deepseek.com/v1"
|
||||
model = "deepseek-v4-flash" # DeepSeek-V4-Flash-0731 (confirmed 2026-08-15)
|
||||
# api_key via DAILY_EPUB_DEEPSEEK__API_KEY env
|
||||
score_batch_size = 12
|
||||
score_temperature = 0.3
|
||||
editorial_temperature = 0.8
|
||||
# USD per 1M tokens, used for the cost guardrail.
|
||||
# api_key via DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY env
|
||||
max_daily_usd = 2.0
|
||||
max_concurrent_requests = 4 # triage and deep-assessment batches in flight
|
||||
price_input_per_mtok = 0.14
|
||||
price_cached_input_per_mtok = 0.0028
|
||||
price_cache_read_per_mtok = 0.0028 # prefix-cache hits
|
||||
price_cache_write_per_mtok = 0.0 # DeepSeek caches implicitly, no write charge
|
||||
price_output_per_mtok = 0.28
|
||||
|
||||
# Claude Opus 5 over the Messages API. Requests carry `output_config.effort`,
|
||||
# a cached system block, and `fallbacks = "default"` so a classifier refusal is
|
||||
# re-routed server-side; a refusal that still comes back degrades to bulk.
|
||||
[providers.anthropic]
|
||||
kind = "anthropic"
|
||||
base_url = "https://api.anthropic.com"
|
||||
model = "claude-opus-5"
|
||||
# api_key via DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY env
|
||||
effort = "high" # low | medium | high | xhigh | max → output_config.effort
|
||||
max_daily_usd = 3.0
|
||||
max_concurrent_requests = 4 # summaries in flight when this is the summary provider
|
||||
price_input_per_mtok = 5.0
|
||||
price_cache_read_per_mtok = 0.5
|
||||
price_cache_write_per_mtok = 6.25
|
||||
price_output_per_mtok = 25.0
|
||||
|
||||
# Gemini 3.8 Flash through Google's OpenAI-compatible endpoint (beta, verified
|
||||
# 2026-09-02). Declared but unreferenced until a role names it.
|
||||
[providers.gemini]
|
||||
kind = "openai"
|
||||
base_url = "https://generativelanguage.googleapis.com/v1beta/openai"
|
||||
model = "gemini-3.8-flash"
|
||||
# api_key via DAILY_EPUB_PROVIDERS__GEMINI__API_KEY env
|
||||
effort = "high" # minimal | low | medium | high → reasoning_effort
|
||||
max_daily_usd = 3.0
|
||||
max_concurrent_requests = 4
|
||||
price_input_per_mtok = 0.75 # promotional through 2026-12-31; $1.50 from 2027-01-01
|
||||
price_cache_read_per_mtok = 0.075 # implicit cache hits; $0.15 from 2027-01-01
|
||||
price_cache_write_per_mtok = 0.0
|
||||
price_output_per_mtok = 3.75 # includes thinking tokens; $7.50 from 2027-01-01
|
||||
|
||||
# Voyage AI embeddings behind the interest and rated-neighbour signals. Set
|
||||
# `enabled = false` (or leave the key unset) and the paper still builds: the
|
||||
# learned signals are simply absent, never a penalty.
|
||||
[voyage]
|
||||
enabled = true
|
||||
base_url = "https://api.voyageai.com/v1"
|
||||
model = "voyage-4-lite"
|
||||
# api_key via DAILY_EPUB_VOYAGE__API_KEY env
|
||||
output_dimension = 512 # 256 | 512 | 1024 | 2048
|
||||
batch_size = 32
|
||||
max_concurrent_requests = 4
|
||||
max_input_chars = 60000 # per article, cut on a char boundary
|
||||
max_daily_usd = 0.50 # runaway guard ($0.02 / M tokens)
|
||||
|
||||
[curation]
|
||||
max_article_count = 28 # hard ceiling; there is no minimum (§13)
|
||||
recent_rejection_days = 7
|
||||
recent_rejection_floor = 3.0
|
||||
always_include_feeds = [] # miniflux feed ids or site urls
|
||||
blocked_domains = []
|
||||
# Extra paywalled hosts, merged with the built-in list (nytimes, wsj, ft, …).
|
||||
@@ -59,6 +129,71 @@ sections = [
|
||||
"From the Blogroll",
|
||||
]
|
||||
|
||||
[curation.feedback]
|
||||
loved_value = 1.0
|
||||
good_value = 0.35
|
||||
not_for_me_value = -1.0
|
||||
slop_value = -1.0 # AI slop: a full negative; the author penalty is below
|
||||
verdicts_in_prompt = 60
|
||||
|
||||
# Every weight, quota, gate and threshold of the personalized ranker. The
|
||||
# learned signals (`knn`, `feed`, `affinity`) contribute nothing until their gates open:
|
||||
# the weight ramps linearly from `*_floor` to `*_full` rated articles.
|
||||
[curation.ranking]
|
||||
triage_max = 800 # eligible articles the triage LLM reads
|
||||
deep_keep = 120 # deep-assessment set
|
||||
shortlist_keep = 60 # what the editor sees
|
||||
assessment_reuse_days = 3
|
||||
rating_lookback_days = 180
|
||||
rating_half_life_days = 60
|
||||
neighbour_k = 5
|
||||
negative_coefficient = 0.75
|
||||
knn_floor = 8
|
||||
knn_full = 25
|
||||
feed_floor = 15
|
||||
feed_full = 40
|
||||
affinity_floor = 15
|
||||
affinity_full = 40
|
||||
slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict
|
||||
semantic_min_words = 300
|
||||
exploration_slots = 5
|
||||
embedding_retention_days = 0 # `features prune`: unrated, unpublished vectors; 0 = keep forever
|
||||
telemetry_retention_days = 180 # `features prune`: candidate_runs and article_assessments
|
||||
|
||||
[curation.ranking.quotas]
|
||||
triage = 60
|
||||
interest = 20
|
||||
knn = 20
|
||||
|
||||
# Weights need not sum to 1; they are renormalized over the present signals.
|
||||
[curation.ranking.weights.preliminary]
|
||||
interest = 0.30
|
||||
knn = 0.25
|
||||
affinity = 0.10
|
||||
heuristic = 0.20
|
||||
feed = 0.10
|
||||
social = 0.05
|
||||
|
||||
[curation.ranking.weights.utility]
|
||||
quality = 0.40
|
||||
fit = 0.20
|
||||
knn = 0.10
|
||||
affinity = 0.05
|
||||
interest = 0.10
|
||||
feed = 0.05
|
||||
triage = 0.05
|
||||
social = 0.03
|
||||
heuristic = 0.02
|
||||
|
||||
[curation.ranking.diversity]
|
||||
cluster_threshold = 0.85
|
||||
per_cluster_cap = 2
|
||||
utility_protected = 10
|
||||
|
||||
[editorial]
|
||||
summary_model = "editor" # editor | bulk — which [llm] role writes the summaries
|
||||
summary_input_tokens = 3000 # article text offered per summary
|
||||
|
||||
[publish]
|
||||
# Where both EPUB editions land, and what the OPDS feed lists. BookOrbit is
|
||||
# optional — it just watches this folder if you run it.
|
||||
@@ -80,9 +215,49 @@ format = "xtch" # xtc (1-bit) | xtch (grayscale)
|
||||
settings = "/etc/daily-epub/xtc-settings.json"
|
||||
|
||||
[server]
|
||||
bind = "127.0.0.1:3499"
|
||||
public_url = "https://daily.hallada.net"
|
||||
bind = "127.0.0.1:3499" # listen address; keep loopback when trusting proxy IP headers
|
||||
public_url = "https://daily.hallada.net" # base URL for EPUB rating links and same-origin checks
|
||||
session_days = 30 # sliding lifetime for web login sessions
|
||||
login_attempts = 10 # shared login/access-request POSTs per client IP per window
|
||||
login_window_minutes = 15 # length of the login throttle window
|
||||
jobs_enabled = true # let the dashboard start daily-epub-job@<name>.service
|
||||
journal_lines = 300 # job-page journal tail; valid range 10..=5000
|
||||
# hmac_secret via DAILY_EPUB_SERVER__HMAC_SECRET env (32+ random bytes)
|
||||
# Optional Basic auth for the XTC OPDS feed and file downloads:
|
||||
# Optional Basic auth for /opds/* and /files/*; without it those existing
|
||||
# routes remain public. A signed-in web user can download without Basic auth.
|
||||
# basic_auth_user = "daily"
|
||||
# basic_auth_pass = "..."
|
||||
|
||||
[bookorbit]
|
||||
enabled = false
|
||||
public_url = "https://bookorbit.hallada.net" # what the browser opens
|
||||
api_url = "http://127.0.0.1:3498" # where the server talks OPDS; same host
|
||||
opds_user = "" # an OPDS user from BookOrbit → Settings → OPDS
|
||||
# opds_pass: environment only (DAILY_EPUB_BOOKORBIT__OPDS_PASS)
|
||||
|
||||
[mail]
|
||||
enabled = false
|
||||
smtp_host = "" # e.g. email-smtp.us-east-1.amazonaws.com
|
||||
smtp_port = 587 # 587 with STARTTLS; commonly 465 with implicit TLS
|
||||
smtp_starttls = true # false selects implicit TLS
|
||||
smtp_user = ""
|
||||
# smtp_pass: environment only (DAILY_EPUB_MAIL__SMTP_PASS)
|
||||
from = "" # e.g. The Daily EPUB <daily@hallada.net>
|
||||
# notify_to = "operator@example.com" # recipient for access-request notifications
|
||||
|
||||
[discovery]
|
||||
enabled = true # run the feed discovery stage during generate
|
||||
max_lookups_per_run = 30 # hosts looked up in Miniflux per run; each host is re-checked at most every 90 days
|
||||
skip_hosts = [ # never looked up: the host itself or any subdomain of it
|
||||
"news.ycombinator.com",
|
||||
"lobste.rs",
|
||||
"reddit.com",
|
||||
"github.com",
|
||||
"gist.github.com",
|
||||
"x.com",
|
||||
"twitter.com",
|
||||
"youtube.com",
|
||||
"en.wikipedia.org",
|
||||
"arxiv.org",
|
||||
"docs.google.com",
|
||||
]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Reader profile
|
||||
|
||||
## Who he is
|
||||
A software engineer in the Boston area who reads on e-ink in the morning. He would
|
||||
rather read six excellent long pieces than thirty adequate short ones. He reads across
|
||||
an unusually wide range of subjects and does not need a topic to be professionally
|
||||
useful to enjoy it.
|
||||
|
||||
## What he wants
|
||||
- Long-form and high-effort above all: essays, deep dives, post-mortems, field notes,
|
||||
annotated experiments, thorough explainers, personal narratives with real specificity.
|
||||
Length is a proxy, not the goal.
|
||||
- Any topic, if the writing is excellent.
|
||||
- Social proof is evidence a critical audience read it, not a verdict.
|
||||
- Boston and New England local news: city government, transit, universities, civic stories.
|
||||
- Ultra-niche community news: small scenes with their own vocabulary.
|
||||
- World and US news kept light and neutral (the World Briefing covers it separately).
|
||||
|
||||
## What he does not want
|
||||
Press releases and funding announcements dressed as news; SEO listicles; link roundups;
|
||||
changelogs without analysis; sponsored content; crypto and engagement bait; rewrites of a
|
||||
story he can read at the source; culture-war outrage; one paragraph stretched to five.
|
||||
|
||||
## How to judge
|
||||
Would he still be glad he read this an hour later? Reward specificity, first-hand
|
||||
experience, honest uncertainty, and prose with a human behind it. Penalize padding,
|
||||
unsourced confidence, and summaries of other people's work.
|
||||
|
||||
## Interests
|
||||
(optional: one per line; merged with data/scour-interests.opml)
|
||||
@@ -0,0 +1,79 @@
|
||||
# Dashboard tables
|
||||
|
||||
How every data table on the dashboard is laid out, and the rules a new table
|
||||
must follow so it never grows a horizontal scrollbar on a desktop screen.
|
||||
Everything here is plain CSS in `src/web/tailwind.css` plus four cell classes;
|
||||
there is no table library and no JavaScript involved.
|
||||
|
||||
## The problem this solves
|
||||
|
||||
A table used to be `min-width: max-content` inside a `.scroll-x` wrapper, and
|
||||
most cells were `white-space: nowrap`. Any table whose natural width exceeded
|
||||
the page column (80rem, ~1355px at the site's 110% scale) became a horizontal
|
||||
scroller, even on a 2560px monitor, and the columns that mattered were hidden
|
||||
behind the scrollbar. Wrapping columns were squeezed to their minimum while the
|
||||
nowrap ones kept everything.
|
||||
|
||||
## The three rules
|
||||
|
||||
1. **Tables get the whole viewport, everything else keeps the page column.**
|
||||
`.dashboard` is a CSS grid with two named column spans: `content` (at most
|
||||
80rem minus the page gutters, exactly what the header uses) and `wide` (the
|
||||
viewport minus the same gutters). Every direct child sits in `content`; a
|
||||
direct-child `.scroll-x` sits in `wide`. The table inside a wide wrapper is
|
||||
`width: auto` (shrink-to-fit) with `min-width` equal to the content column
|
||||
and `max-width: 100%`, centred. A table that fits the page column therefore
|
||||
looks exactly as before; a table that needs more grows, centred, up to the
|
||||
viewport edge; the page itself never scrolls sideways.
|
||||
|
||||
2. **Columns wrap before the table scrolls.** The `min-width: max-content` rule
|
||||
is gone. A table shrinks by wrapping its text columns down to their floors,
|
||||
and only when the floors alone no longer fit (phones, mostly) does the
|
||||
`.scroll-x` wrapper scroll. Headers always wrap.
|
||||
|
||||
3. **A cell declares what it holds, and nothing else decides its width.**
|
||||
There are exactly four cell classes:
|
||||
|
||||
| class | use it for | behaviour |
|
||||
| ------------ | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
|
||||
| *(none)* | short prose, names, tokens, badges and badge lists | wraps at spaces (a `.badge` is nowrap on its own, so lists wrap between badges) |
|
||||
| `num` | numbers, money, counts | right-aligned tabular figures; `td.num` never wraps, `th.num` may |
|
||||
| `cell-tight` | timestamps, dates, ids, a single short token that must not split | `white-space: nowrap`. Nothing longer than ~20 characters |
|
||||
| `cell-wrap` | titles, URLs, notes, messages, anything free-form | floor 10rem, ceiling 32rem, `overflow-wrap: anywhere` |
|
||||
|
||||
Anything long that must not wrap gets an inner block with a Tailwind clamp
|
||||
(`<div class="line-clamp-2" title="…">`) inside a `cell-wrap` cell; the cell
|
||||
ceiling bounds the column, the clamp bounds the row height. Do not put
|
||||
`truncate`/`line-clamp-*` on a `td` itself (a table cell cannot be a
|
||||
`-webkit-box`), and do not use `table-fixed` on a page-level table (fixed
|
||||
layout needs a definite width and is silently ignored with `width: auto`).
|
||||
|
||||
## Checklist for a new table
|
||||
|
||||
- Wrap it in `<div class="scroll-x">` and make that wrapper a **direct child** of `<section class="dashboard">` if the
|
||||
table has more than a handful of columns. Tables inside `.card`, `.cards` or
|
||||
`details` stay inside their box and just wrap/scroll there.
|
||||
- Let the page scroll. A table that is the page's feature (paginated lists,
|
||||
anything that is the last thing on the page) shows every row at full height;
|
||||
a scrolling island inside a scrolling page just fights the wheel. Add `tall`
|
||||
(70vh cap, own scrollbar) only when the table is a side feature with real
|
||||
content below it, as the profile page's version history is.
|
||||
- Give every `td` one of the four classes above according to its content.
|
||||
When in doubt, leave it unclassed: wrapping is the safe default.
|
||||
- Never `cell-tight` a name, a title, a reason sentence or a free-text field.
|
||||
- Check the page at 390, 1280 and 2560px. At 1440 and above the wrapper's
|
||||
`scrollWidth` must equal its `clientWidth`; at 390 a wide table is allowed to
|
||||
scroll. A table's floor is the sum of its nowrap columns plus 10rem per
|
||||
`cell-wrap` column: the 13-column articles table bottoms out near 1270px, so
|
||||
it still scrolls a little on a 1280px viewport. Tables inside a `.card`
|
||||
cannot grow past the card (20rem minimum), so a card table with more than
|
||||
four numeric columns (run page: provider usage) scrolls inside its card at
|
||||
1280px; move such a table to page level if that matters.
|
||||
|
||||
## Measuring
|
||||
|
||||
`docs/plans/briefs/dashboard-tables/measure.mjs` logs in as the dev-seed admin,
|
||||
visits every dashboard page at several widths and prints, per page, whether the
|
||||
document overflows and which table wrappers scroll. Run it against a seeded
|
||||
dev server (`cargo run --example seed_dev_db -- ./dev`) before and after a CSS
|
||||
change; the "scrolling" count for widths ≥ 1280 should be zero.
|
||||
@@ -1,6 +1,8 @@
|
||||
# Implementation notes (shared brief for all implementation agents)
|
||||
|
||||
Authoritative spec: `docs/plans/2026-08-15-the-daily-epub.md`. Read it fully before writing code.
|
||||
For curation (§3.5, §3.6 and §3.9 of that spec) the authority is now
|
||||
`docs/plans/2026-09-02-personalized-curation-v2.md`; see "Curation v2" below.
|
||||
This file records implementation-time decisions and verified external facts. Follow both.
|
||||
|
||||
## Verified external facts (2026-08-15)
|
||||
@@ -68,6 +70,47 @@ This file records implementation-time decisions and verified external facts. Fol
|
||||
walks back up to `MAX_LOOKBACK_DAYS` and the section is datelined with the day it actually
|
||||
covers, not the masthead date.
|
||||
|
||||
## Verified external facts (2026-09-02, curation v2)
|
||||
|
||||
- **Anthropic Messages API** (verified 2026-09-02 against the bundled Claude API reference):
|
||||
`POST https://api.anthropic.com/v1/messages` with headers `x-api-key`,
|
||||
`anthropic-version: 2023-06-01`, `content-type: application/json` and
|
||||
`anthropic-beta: server-side-fallback-2026-07-01`. Model id `claude-opus-5`; pricing
|
||||
**$5.00 / M input, $25.00 / M output**, cache reads 0.1× input ($0.50/M), cache writes
|
||||
1.25× ($6.25/M); the minimum cacheable prefix is 512 tokens. **No sampling parameters**
|
||||
(`temperature`, `top_p`, `top_k` are a 400) and no `thinking` block — adaptive thinking is on
|
||||
by default and depth is set with `output_config: {"effort": "high"}` (`low | medium | high |
|
||||
xhigh | max`). The system prompt goes in `system: [{type: "text", text, cache_control:
|
||||
{type: "ephemeral"}}]`; no assistant prefill, JSON is asked for in the prompt and parsed
|
||||
tolerantly. `"fallbacks": "default"` (with the beta header) routes a request the safety
|
||||
classifiers would refuse to a fallback model server-side; a response can still end with
|
||||
`stop_reason: "refusal"` on HTTP 200, which the code treats as an error that degrades the
|
||||
call to DeepSeek. Usage fields: `input_tokens` (uncached remainder),
|
||||
`cache_creation_input_tokens`, `cache_read_input_tokens`, `output_tokens`. Timeout 300 s;
|
||||
retry 429/5xx/network, never 400. Key only from `DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY`
|
||||
(the provider registry below; the pre-registry `DAILY_EPUB_ANTHROPIC__API_KEY` is a startup
|
||||
error).
|
||||
- **Gemini 3.8 Flash over the OpenAI-compatible endpoint** (beta, verified 2026-09-02):
|
||||
`POST https://generativelanguage.googleapis.com/v1beta/openai/chat/completions` with
|
||||
`Authorization: Bearer <key>`, the standard `messages` / `temperature` /
|
||||
`response_format: {"type": "json_object"}` body. Model id `gemini-3.8-flash`. Reasoning depth
|
||||
is the OpenAI `reasoning_effort` field, which Google maps onto Gemini 3.x's `thinking_level`
|
||||
(`minimal | low | medium | high`; `none` is not accepted by 3.x models). Usage: implicit
|
||||
cache hits are reported in `prompt_tokens_details.cached_tokens` (the same field DeepSeek
|
||||
now fills), and `completion_tokens` already includes the thinking tokens that
|
||||
`completion_tokens_details.reasoning_tokens` breaks out — so output is priced from
|
||||
`completion_tokens` alone, never the sum. Prices per 1M tokens (promotional through
|
||||
2026-12-31): **$0.75 input, $0.075 cache read, $3.75 output** (thinking included); from
|
||||
2027-01-01 **$1.50 / $0.15 / $7.50**. No cache-write charge. Key only from
|
||||
`DAILY_EPUB_PROVIDERS__GEMINI__API_KEY`. Shipped as `[providers.gemini]`, unreferenced until
|
||||
a role names it.
|
||||
- **Voyage AI embeddings** (verified 2026-09-02): `POST https://api.voyageai.com/v1/embeddings`
|
||||
with `Authorization: Bearer <key>`; body `{input: [...], model: "voyage-4-lite", input_type:
|
||||
"document" | "query", truncation: true, output_dimension: 512, output_dtype: "float"}`. Up
|
||||
to 1,000 inputs and 1M tokens per request, 32k tokens per input. Vectors are
|
||||
unit-normalized, so dot product = cosine. **$0.02 / M tokens** after a 200M-token free
|
||||
allocation. Key only from `DAILY_EPUB_VOYAGE__API_KEY`.
|
||||
|
||||
## Cross-cutting implementation decisions
|
||||
|
||||
1. **sqlx usage**: use *runtime* queries (`sqlx::query(...).bind(...)`) and manual row mapping
|
||||
@@ -80,12 +123,32 @@ This file records implementation-time decisions and verified external facts. Fol
|
||||
Pipeline stages are best-effort where the spec says so (social, XTC, world briefing, images).
|
||||
4. **HTTP**: one shared `reqwest::Client` (rustls, gzip, no cookies, 10s timeouts, UA
|
||||
`the-daily-epub/1.0 (personal rss digest; contact tyler@hallada.net)`), passed by clone.
|
||||
5. **LLM**: `async-openai` with custom base URL. All LLM calls go through `curate/llm.rs`
|
||||
`LlmClient` which tracks token usage into a shared `UsageMeter` (input/cached/output tokens,
|
||||
cost usd) and enforces `max_daily_usd`.
|
||||
5. **LLM**: a hand-rolled `reqwest` client, not `async-openai` (the published crate exposes
|
||||
neither `Client` nor `CreateChatCompletionRequest` at the pinned version). Every LLM call
|
||||
goes through `curate/llm.rs`: `LlmClient { provider, system_prompt, model, effort,
|
||||
max_concurrent_requests, meter, backend, retry }` over the `ChatBackend` trait, with two
|
||||
wire protocols — `OpenAiCompatibleBackend` (`{base_url}/chat/completions`, bearer key,
|
||||
`response_format: json_object`, `reasoning_effort` when the provider has an `effort`) and
|
||||
`AnthropicBackend` (Messages API, facts above). **Providers are config, not code**: the
|
||||
`[providers.<name>]` registry (`kind = openai | anthropic`, `base_url`, `model`, `effort`,
|
||||
`max_daily_usd`, `max_concurrent_requests`, `price_*`) is a `BTreeMap<String,
|
||||
ProviderConfig>`, and `[llm] bulk = "<name>"` / `editor = "<name>"` assign the two roles by
|
||||
name (`editor = ""` means everything runs on bulk; both roles on one provider share one
|
||||
client and one ceiling). `LlmClient::for_provider(name, &cfg, ..)` dispatches on `kind`;
|
||||
`Llms::from_config(&config, prompt, &meters)` builds the roles; `editor_or_bulk()` degrades
|
||||
to bulk when the editor client is missing or its meter is tripped. One `UsageMeter` per
|
||||
*referenced* provider (`llm::provider_meters`), keyed by provider name — the same key used
|
||||
for `runs.provider_costs_json`, the UTC-day spend preload and the log lines — plus Voyage's
|
||||
own. `LlmClient.provider` is the config name, never the kind. The system prompt is sent
|
||||
first and byte-identical within a run so every provider's prefix cache hits. Keys come only
|
||||
from `DAILY_EPUB_PROVIDERS__<NAME>__API_KEY` (figment lower-cases the path, so provider
|
||||
names are `[a-z0-9_]+`); `daily-epub config check` prints the resolved roles without
|
||||
opening the database.
|
||||
6. **Testing**: unit tests inline per module; integration tests in `tests/` over fixture JSON in
|
||||
`tests/fixtures/`. Never hit the network in tests. LLM stage mockable via `--skip-llm`
|
||||
(prefilter order used for selection, feed excerpts as summaries).
|
||||
`tests/fixtures/`. Never hit the network in tests: `MockBackend` (`ChatBackend`) and the
|
||||
embedding mock (`EmbeddingBackend`) stand in for all three providers. `--skip-llm` makes
|
||||
zero LLM calls (admission by cheap signals, `select_without_llm` by utility, excerpt
|
||||
summaries); `--skip-embeddings` makes zero Voyage calls.
|
||||
7. **Style**: rustfmt defaults, `cargo clippy` clean-ish, no `unwrap()` outside tests, tracing
|
||||
spans per pipeline stage.
|
||||
8. **File ownership**: waves of agents work in parallel on disjoint files. Do not edit files
|
||||
@@ -99,3 +162,118 @@ This file records implementation-time decisions and verified external facts. Fol
|
||||
`style-x4.css`). Askama 0.12+ configured via `askama.toml` if needed.
|
||||
12. **Determinism**: chapter ids `art-{entry_id}`, stable filenames, issue regeneration for the
|
||||
same date replaces prior rows/files (idempotent upsert everywhere).
|
||||
|
||||
## Curation v2 (2026-09-02)
|
||||
|
||||
The personalized ranker is specified in `docs/plans/2026-09-02-personalized-curation-v2.md`
|
||||
(§0 settled decisions, §3 target pipeline, §19 configuration, §21 the seven landed steps);
|
||||
`docs/plans/2026-09-02-curation-v2-progress.md` records per-step deviations. Facts an
|
||||
implementer needs that are easy to get wrong:
|
||||
|
||||
- **Tables** (`migrations/0002_curation_v2.sql`, `0003_drop_scores.sql`; never edit
|
||||
`0001_init.sql`): `rating_events` (append-only; the current verdict is the latest
|
||||
`explicit` event), `article_embeddings` and `interest_embeddings` (f32 little-endian BLOBs,
|
||||
`input_hash` = sha256 of the embedded text), `article_assessments` (`stage IN ('triage',
|
||||
'deep')`, reused while `model` and `prompt_version` match and `assessed_at` is within
|
||||
`assessment_reuse_days`; `--rescore` ignores the cache), `candidate_runs` (one row per
|
||||
considered article per run, upserted with every column set on each stage transition),
|
||||
`runs.config_json` / `runs.provider_costs_json`, `issue_articles.why`. `ratings`,
|
||||
`feed_priors` and `scores` are dropped; `kv` keeps `ingest_watermark`, `taste_profile`,
|
||||
`taste_profile_learned`, `profile_version`.
|
||||
- **Feedback**: `Vote` is `loved | good | down` (`NotForMe`); the HMAC message stays
|
||||
`{issue_date}/{article_id}/{vote}`. `Vote::parse("up")` → `Loved` and `auth::verify_token`
|
||||
still accepts tokens signed over the literal `up` segment because published issues carry
|
||||
those links. Keep both.
|
||||
- **Budget day**: each provider's `UsageMeter` is preloaded with the spend of earlier runs on
|
||||
the **UTC date of the run's `started_at`**, summed from `runs.provider_costs_json`
|
||||
(`db::spend_for_date` by nominal issue date is gone). A tripped meter skips that provider's
|
||||
remaining calls; the paper always publishes.
|
||||
- **Lock**: `src/lock.rs` takes `libc::flock(LOCK_EX | LOCK_NB)` on `<database_path>.lock`
|
||||
for `generate`, `profile rebuild`, `features backfill` and `backfill-social`; a second
|
||||
writer exits with "<command> is already running". `serve`, `explain`, `stats`, `ratings`,
|
||||
`features prune` and `db migrate` never take it.
|
||||
- **Retention**: `telemetry::prune` removes `article_embeddings` of unrated, unpublished
|
||||
articles older than `embedding_retention_days` (120) and `candidate_runs` rows plus
|
||||
`article_assessments` older than `telemetry_retention_days` (180). `features prune` runs it
|
||||
on demand; `generate` runs it once after publishing, best effort.
|
||||
- **Keys**: `DAILY_EPUB_PROVIDERS__<NAME>__API_KEY` and `DAILY_EPUB_VOYAGE__API_KEY` map onto
|
||||
`ProviderConfig.api_key` / `VoyageConfig.api_key` through figment; the fields exist only
|
||||
for that mapping and are never documented in TOML, logged, or stored
|
||||
(`Config::providers_redacted()` is what reaches `runs.config_json`).
|
||||
- **Provider registry** (2026-09-02, after step 7): the `[deepseek]` and `[anthropic]` tables
|
||||
and the top-level `max_daily_usd` are gone. `[llm]` holds the role names and the role-level
|
||||
knobs (`triage_batch_size`, `deep_batch_size`, `score_temperature`,
|
||||
`editorial_temperature`); `[providers.deepseek]`, `[providers.anthropic]` and
|
||||
`[providers.gemini]` ship in `config.example.toml` and are `Config::default()` key for key.
|
||||
Stale shapes fail at load, naming the new key: a `[deepseek]`/`[anthropic]` header, a
|
||||
top-level `max_daily_usd`, any of the four role keys outside `[llm]`, or a
|
||||
`DAILY_EPUB_DEEPSEEK__*` / `DAILY_EPUB_ANTHROPIC__*` environment variable. Batching
|
||||
concurrency (`triage`, `assess`) is the bulk provider's `max_concurrent_requests`; the
|
||||
summaries fan out at the summary provider's. `Models { bulk, editor, summaries }` in the
|
||||
colophon and Behind the paper stay model ids taken from the built clients.
|
||||
- **Provider rejections** (2026-09-02): `curate::batch` bisects a triage or deep batch that
|
||||
fails with a non-transient error (`Api`, `Refusal`, `EmptyResponse`, or a response that
|
||||
parses to zero items) down to single articles; a rejected single is retried once on the
|
||||
editor client when it is another provider. An article both refuse gets an
|
||||
`article_assessments` row with `kind = 'provider_rejected'`, `score`/`fit` NULL,
|
||||
`rationale = "<provider>: <message ≤ 200 chars>"`, the bulk `model` and the stage's
|
||||
`prompt_version`; the cache loaders skip it (leaving the assessment absent) while it is within
|
||||
`assessment_reuse_days`, `--rescore` ignores it, and `admit::hygiene` never treats it as a low
|
||||
score. Because a recovered article's row carries the editor's model, reuse accepts rows whose
|
||||
`model` is either configured model (`triage::reusable_models`).
|
||||
|
||||
## Web dashboard (2026-09-03)
|
||||
|
||||
Verified on the production host and against the implemented dependency graph on 2026-09-03:
|
||||
|
||||
- **Host authorization and units:** polkit 124 supports JavaScript rules. The installed rule must
|
||||
grant user `daily-epub` only `org.freedesktop.systemd1.manage-units`, verb `start`, for units
|
||||
matching `^daily-epub-job@[a-z0-9-]+\.service$`. `daily-epub.service` now needs
|
||||
`SupplementaryGroups=systemd-journal` to read job logs and `/etc/daily-epub` in
|
||||
`ReadWritePaths` to atomically replace `config.toml`. The job template deliberately omits
|
||||
`MemoryDenyWriteExecute` because generate jobs can launch Node's JIT; the server retains it.
|
||||
- **Settings writes:** direct `toml_edit 0.25` performs typed, comment/order-preserving updates.
|
||||
A candidate file is loaded through `Config::load` before its permissions are copied and it is
|
||||
renamed over the original. The process therefore needs write access to the containing
|
||||
directory, not only the file. Every changed dotted key is written to `config_changes`.
|
||||
- **Authentication stack:** `axum-login` is pinned to git revision
|
||||
`151c72d7a1b4646830f86b4332e6bd6e34d719a7`, whose graph contains `tower-sessions 0.15`.
|
||||
The local `SqliteSessionStore` uses this crate's existing sqlx 0.9 pool because the published
|
||||
tower SQLx store is incompatible. `password-auth 1.0` supplies Argon2id PHC hashes;
|
||||
`tower_governor 0.8` throttles login POSTs. Its smart IP extractor trusts forwarded headers,
|
||||
so production must bind to loopback and accept traffic only from the configured reverse proxy.
|
||||
- **Test environment:** router tests use `tower::ServiceExt::oneshot`, temp SQLite databases and
|
||||
`MockRunner`; they do not bind or invoke systemd. This sandbox forbids loopback listeners, so
|
||||
the four `curate::llm::tests::anthropic_*` tests, the three OpenAI tests using the same fake
|
||||
listener, `extract::tests::relative_urls_resolve_against_the_url_we_landed_on`, the five
|
||||
listener-based `server::tests::*`, and `tests/m7_server.rs` are filtered only for sandbox runs.
|
||||
The complete suite is expected to run outside the sandbox.
|
||||
|
||||
Implementation-time decisions recorded while landing dashboard steps 1–7:
|
||||
|
||||
- `/files/*` remains public when Basic auth is not configured, preserving existing OPDS
|
||||
acquisition behavior. With Basic auth configured, either valid Basic credentials or any valid
|
||||
web session authorizes a download. This intentionally resolves the plan's conflicting request
|
||||
to redirect unauthenticated downloads in favor of its compatibility acceptance criterion.
|
||||
- Final reports are attached to an issue after `finish_run`, when publish timing and status are
|
||||
complete. Issue snapshots omit article bodies and rehydrate them from `articles`; old rows use
|
||||
the reduced fallback renderer. The CLI password prompt uses `rpassword` after it became
|
||||
available to the orchestrator.
|
||||
- Flash handlers extract the exact tower session installed by the auth layer through
|
||||
`Extension<Session>`. Dashboard `down` forms persist the established `not_for_me` label.
|
||||
Ratings-page feed credit includes rating decay because that is what `signals::feed_rates`
|
||||
actually uses; the stored prompt verdict count is inferred from the prompt text because
|
||||
`TasteProfile.verdicts` is not persisted.
|
||||
- Dynamic list SQL is assembled only from fixed fragments and allow-listed sort/filter names,
|
||||
wrapped in sqlx 0.9's `AssertSqlSafe`; all user values remain bound parameters. Funnel bars
|
||||
count rows that reached each stage because a row stores the stage where it stopped. SVG/meter
|
||||
attributes replace inline styles under the CSP. Article pages omit the nominal extract method
|
||||
because database reconstruction currently hard-codes it and would display misleading data.
|
||||
- Shipped providers cannot be removed: deleting one would cause `Config::default()` to restore
|
||||
it. They remain editable and may be unreferenced; custom providers are removable. An absent
|
||||
setting already equal to its submitted default stays absent. Settings that are captured while
|
||||
building the server, session, or throttle layers carry restart notices.
|
||||
- A job start reloads a hand-edited config before inserting and starting the unit, retaining the
|
||||
last-good config if reload fails. The offline lifecycle test uses `features-prune`; generate
|
||||
run-id linkage is covered separately. CLI duration statistics retain finished dry runs for
|
||||
byte-identical output, while dashboard run series and overview sparklines exclude them.
|
||||
@@ -0,0 +1,139 @@
|
||||
# Personalized Curation v2 — implementation progress and handoff
|
||||
|
||||
**Updated:** 2026-09-02 (end of session 2)
|
||||
**Plan:** `docs/plans/2026-09-02-personalized-curation-v2.md` (§21 is the step sequence)
|
||||
**Branch:** `curation-v2` (branched from `main` at `a599745`; not merged, not pushed)
|
||||
**Briefs:** `docs/plans/curation-v2-briefs/` — `00-preamble.md` + one `stepN.md` per step. Each
|
||||
brief was handed to an implementation agent as `cat 00-preamble.md stepN.md`.
|
||||
|
||||
## Where things stand
|
||||
|
||||
| Step (§21) | Status | Commit |
|
||||
|---|---|---|
|
||||
| 1. Feedback and profile | **done**, reviewed (codex review: no actionable issues) | `3a9f4b9` |
|
||||
| 2. Claude editor and editorial | **done** | `57efbb4` |
|
||||
| 3. Embeddings, signals, telemetry | **done** (branch `curation-v2-step3`, merged) | `ea3b141` + merge commit |
|
||||
| 4. Triage replaces the gate | **done** (Codex plugin, high effort), reviewed | `10f4afd` |
|
||||
| 5. Deep assessment, utility, diversity | **done** (Codex started, cut off by quota; Claude agent finished), reviewed | `05a74a0` |
|
||||
| 6. Paper telemetry, stats, lock | **done** (Claude agent), reviewed | `d261cd4` |
|
||||
| 7. Cleanup + implementation notes | **done** (Claude agent), reviewed | `d403c51` |
|
||||
|
||||
**Post-plan addition (same day):** a provider-agnostic LLM registry — `[llm]` roles (`bulk`,
|
||||
`editor`) over named `[providers.*]` entries of `kind = openai | anthropic`, keys from
|
||||
`DAILY_EPUB_PROVIDERS__<NAME>__API_KEY`, a `gemini` entry (Gemini 3.8 Flash via Google's
|
||||
OpenAI-compatible endpoint) declared but unreferenced, and a `daily-epub config check` subcommand.
|
||||
The old `[deepseek]`/`[anthropic]` tables, top-level `max_daily_usd` and the old key env vars
|
||||
fail loudly. The server upgrade is written up in `docs/runbooks/curation-v2-migration.md`.
|
||||
|
||||
All seven plan steps are implemented. `cargo fmt --check`, `cargo clippy --all-targets`
|
||||
(including `-W dead_code`) and `cargo test` (329 lib tests + 6 bin tests + the 7 integration suites) are green at HEAD.
|
||||
|
||||
Nothing has been run against the production database or the real providers. No local
|
||||
`config.toml`, database, or API keys exist on the dev box, so verification so far is tests only.
|
||||
|
||||
## Decisions and deviations made while implementing
|
||||
|
||||
Steps 1–3 (session 1):
|
||||
|
||||
- **Legacy `up` links.** `Vote::parse("up")` → `Loved`, and `auth::verify_token` also accepts tokens
|
||||
signed over the literal `up` segment so already-published issues keep working. Keep both.
|
||||
- **Same-date regeneration** no longer excludes its own picks (`published_before` uses issue dates
|
||||
strictly before the run date), per plan §8.1.
|
||||
- **`VoyageConfig.api_key`** exists as a field (figment maps `DAILY_EPUB_VOYAGE__API_KEY` into it;
|
||||
`deny_unknown_fields` would otherwise reject the env var). Never document it in TOML.
|
||||
- **`anthropic.max_concurrent_requests`** is validated but not consumed; summary concurrency is
|
||||
the constant `SUMMARY_CONCURRENCY = 4` in `editorial.rs`.
|
||||
- `RATINGS_LOOKBACK_DAYS` in `profile/mod.rs` is effectively unbounded (36,500) for the prompt
|
||||
verdict block and the weekly rebuild; the knn/feed preference state uses
|
||||
`curation.ranking.rating_lookback_days` (180) as the plan says.
|
||||
- Footer CSS: `.rating` has no `white-space: nowrap` (it would clip on narrow e-ink screens).
|
||||
|
||||
Step 4:
|
||||
|
||||
- Hygiene order is blocked → published_before → recently_rejected, all skipped for auto-includes.
|
||||
- `admitted_by` records every retriever whose own top-N would have taken the article (first entry
|
||||
is the admitting one); exploration is flagged only when `exploration` admitted it.
|
||||
- Under the pool cap (`triage_max`) the surplus gets `stage='eligible'`, `excluded_reason='not_admitted'`.
|
||||
- Migration `0003_drop_scores.sql` drops `scores`; the churn rule reads `article_assessments`.
|
||||
|
||||
Step 5:
|
||||
|
||||
- `Candidate` is the only flow type; `ScoredArticle`, `combined_score`, `score.rs`, `select.rs` are gone.
|
||||
- The editor omits the `facets:` line when every facet is unknown (plan renders it unconditionally).
|
||||
- Articles cut at the shortlist keep the stage they reached (`admitted` if never deep-assessed,
|
||||
else `assessed`) with `excluded_reason` `cluster_suppressed` (hit the base cap) or `shortlist_cap`.
|
||||
- `Curator::assess` is a no-op under `--skip-llm`; cached deep rows are still reused when DeepSeek
|
||||
is merely down.
|
||||
- `--near-misses` and the Behind-the-paper list order by utility, falling back to the preliminary
|
||||
blend per row.
|
||||
- Fixtures: `tests/fixtures/deepseek_deep_batch{,_messy}.json` replace the old score fixtures;
|
||||
`deepseek_triage_batch.json` added in step 4.
|
||||
|
||||
Step 6:
|
||||
|
||||
- `BehindThePaper` hangs off `Issue.behind` (filled after `build_issue`), not a `build_issue` argument.
|
||||
Generation time and cost shown in the chapter are measured at issue assembly (they exclude the
|
||||
EPUB build and publish, a few seconds); the cost includes Voyage.
|
||||
- Voyage spend is recorded under `provider_costs["voyage"]`; `finish()` counts it once and keeps its
|
||||
tokens out of the LLM aggregate. Voyage spend is not preloaded into its meter for the UTC day.
|
||||
- `src/lock.rs` uses `libc::flock` (`libc` was already in `Cargo.lock`; now a direct dependency).
|
||||
The holder's command name is written into the lock file for the "X is already running" message.
|
||||
- The mid-run `admission:` line is `debug`; the four-line §15.4 block is logged once after
|
||||
`finish()` and printed by `print_report`. Timings `summaries` and `brief` replace `editorial`.
|
||||
|
||||
Step 7:
|
||||
|
||||
- `features prune` sweeps `article_embeddings`, `candidate_runs` and now `article_assessments`;
|
||||
`generate` runs it once after a published (non-dry) issue, best effort.
|
||||
- The Brief chapter's TOC/title string is now "The Brief" (was still "From the Editor").
|
||||
- Kept on purpose: `UsageMeter::new(&DeepseekConfig, ..)` (a convenience over `with_prices`, 12
|
||||
call sites); `StageCounts.candidates` (feeds `runs.candidates` and the colophon); the
|
||||
`prefilter_keep`/`score_batch_size` removed-key startup errors (operator guards); the SQL comment
|
||||
"filled in by step 4/5" inside the applied migration `0002` (sqlx checksums); six v1-era
|
||||
unreferenced `pub` items (`enrich_one`, `issues_before`, `today_in_tz`, `to_json_pretty`,
|
||||
`with_retry_policy`, `db::issues_before`) as pre-v2 API surface.
|
||||
- `config::tests::shipped_example_config_matches_the_defaults_key_for_key` compares the example
|
||||
against `Config::default()` in both directions (numbers within 1e-6).
|
||||
|
||||
## Operator to-dos before the first real run
|
||||
|
||||
1. Set `DAILY_EPUB_ANTHROPIC__API_KEY` and `DAILY_EPUB_VOYAGE__API_KEY` in the systemd env file.
|
||||
2. Set hard spend limits in the DeepSeek, Anthropic and Voyage dashboards (the meters are runaway
|
||||
guards, not accounting).
|
||||
3. Copy `data/profile.md` to wherever `profile_path` points on the server (default is relative to
|
||||
`WorkingDirectory=/var/lib/daily-epub`, like `data/scour-interests.opml`).
|
||||
4. Back up the DB, then `daily-epub db migrate` (0002 drops `ratings`/`feed_priors`, 0003 drops `scores`).
|
||||
5. `daily-epub features backfill --rated-only` then `--days 30` to warm the embedding cache.
|
||||
6. `daily-epub generate --dry-run`, read the paper (including the new Behind-the-paper chapter),
|
||||
then `explain --date … --near-misses` and `stats`.
|
||||
7. Watch the first few real runs' `providers:` log line against the ~$0.80/day estimate (plan §3).
|
||||
|
||||
## How the work was run (so the next session can repeat it)
|
||||
|
||||
- Orchestrator: Claude Code (this repo), one implementation agent per step, review + commit by the
|
||||
orchestrator after independent `cargo fmt/clippy/test`.
|
||||
- **Codex via the `openai-codex` Claude Code plugin (v1.0.6) works on this host as of session 2**
|
||||
(the operator fixed bubblewrap). Launch:
|
||||
`node ~/.claude/plugins/cache/openai-codex/codex/1.0.6/scripts/codex-companion.mjs task --background --write --effort high "$(cat brief.md)"`,
|
||||
poll `… status <job-id> --json` (`.job.status`), read `… result <job-id>`. Inside its sandbox
|
||||
`bind()` on 127.0.0.1 is forbidden, so exactly ten pre-existing loopback tests fail there
|
||||
(`curate::llm::tests::anthropic_*` ×4, `extract::tests::relative_urls_resolve_against_the_url_we_landed_on`,
|
||||
`server::tests::*` ×5, plus the two `m7_server` integration tests); the brief must tell the agent
|
||||
to ignore those and the orchestrator runs the full suite outside. The old "host quirk" paragraph
|
||||
in `00-preamble.md` is obsolete.
|
||||
- Codex usage budget is roughly one large step per ChatGPT usage window: in session 2 it finished
|
||||
step 4 (~26 min) and was cut off 21 minutes into step 5 ("try again at 7:44 PM"). The job status
|
||||
becomes `failed` with the partial work left in the tree; a Claude `general-purpose` agent with a
|
||||
"finish, don't restart" brief (known failures, scope audit checklist) completed it. Steps 6 and 7
|
||||
ran on Claude agents directly.
|
||||
- Steps 4 → 5 → 6 → 7 were sequential, each reviewed against the plan sections named in its brief
|
||||
before committing.
|
||||
|
||||
## Next session: what remains
|
||||
|
||||
1. `git checkout curation-v2 && cargo test` (expect green). Follow
|
||||
`docs/runbooks/curation-v2-migration.md` on the server.
|
||||
2. Optional: a Codex `review --background --scope branch --base main` pass over the whole branch.
|
||||
3. `git merge --no-ff curation-v2` into `main`, build, deploy, and do the operator to-dos above.
|
||||
4. After a week of real runs: read `stats`, tune `[curation.ranking]` from what `explain` shows,
|
||||
and revisit the deferred items in plan §23.
|
||||
@@ -497,7 +497,7 @@ Computed in `src/curate/signals.rs` after embeddings. Every signal is `Option<f6
|
||||
|---|---|---|
|
||||
| `interest` | z-scored standing-interest match (§9.1) | no embedding, or fewer than 30 eligible articles have embeddings (then use raw top-1 cosine and log it) |
|
||||
| `knn` | signed rated-neighbour preference (§9.2) | no embedding, or gate closed |
|
||||
| `feed` | mean Beta-smoothed rating rate over the article's distinct direct feeds (§9.3) | no direct feed with any rating, or gate closed |
|
||||
| `feed` | mean Beta-smoothed rating rate over the article's distinct direct feeds and author (§9.3) | no direct feed or author with any rating, or gate closed |
|
||||
| `social` | existing `composite_social_score` | no `social` rows |
|
||||
| `heuristic` | `longform_points(word_count)` − excerpt-only penalty − roundup penalty, from `prefilter.rs` with the social, Scour/HN, multi-source, and feed-prior terms **removed** | never |
|
||||
|
||||
@@ -531,7 +531,9 @@ preference: 14 rated articles with embeddings → knn gate 0.35; feed gate 0.0 (
|
||||
|
||||
### 9.3 Feed affinity
|
||||
|
||||
From the same rating set. Credit each rating's `value` to the article's distinct direct feeds (`SourceKind::Feed`), split evenly; if there are none, to `best_entry_id`'s feed. Per feed: `rate = (up + 1) / (up + down + 2)` where `up = Σ max(value, 0)` and `down = Σ max(−value, 0)`, decayed as in §9.2. A candidate's `feed` signal is the **mean** over its distinct direct feeds that have any rating (never the max). Gate: `feed_floor = 15`, `feed_full = 40` attributable ratings.
|
||||
From the same rating set. Credit each rating's decayed `value` to the article's distinct direct feeds (`SourceKind::Feed`), split evenly; if there are none, use `best_entry_id`'s feed. For an aggregator-only article, that fallback aggregator feed receives only `0.25 ×` the feed credit. When an article has an author, its normalized author key (trimmed, internal whitespace collapsed, lowercased) separately receives the full credit, so future articles by that author carry the history across any feed.
|
||||
|
||||
Per feed and author: `rate = (up + 1) / (up + down + 2)` where `up = Σ max(value × decay, 0)` and `down = Σ max(−value × decay, 0)`. A candidate's `feed` signal is the **mean** over its distinct direct feeds and author that have any rating (never the max). A rating is attributable if it credits at least one feed or author. Gate: `feed_floor = 15`, `feed_full = 40` attributable ratings.
|
||||
|
||||
---
|
||||
|
||||
@@ -987,7 +989,7 @@ No test touches the network. Mock backends for all three providers, following th
|
||||
- **Rating events**: latest explicit event wins; `cleared` removes an article from the learned set; migration copies old rows with the right labels and values; the CLI `set`/`clear` append rows with `source = 'cli'`.
|
||||
- **Embeddings**: BLOB round trip; wrong length and non-finite rejected; cache hit on same hash, miss on changed text/model/dimension; response mapped by index and length-checked; a failed batch does not abort the others; embedded text contains no feed name or author.
|
||||
- **Interest z-scores**: a broad interest with uniformly high cosine does not dominate; a specific interest with one strong match does; raw fallback under 30 articles.
|
||||
- **Preference**: one loved article gives a positive `knn` to a near neighbour; two unrelated loved clusters both score high (the anti-centroid test); `good` moves the signal 0.35× as much as `loved`; decay halves at the half-life; gate is 0 below `knn_floor`, 1 at `knn_full`, linear between; feed credit sums to 1 across direct feeds; feed affinity uses the mean.
|
||||
- **Preference**: one loved article gives a positive `knn` to a near neighbour; two unrelated loved clusters both score high (the anti-centroid test); `good` moves the signal 0.35× as much as `loved`; decay halves at the half-life; gate is 0 below `knn_floor`, 1 at `knn_full`, linear between; ordinary feed credit sums to 1 across direct feeds; aggregator-only credit is 0.25× to its feed and 1× to its author; feed affinity uses the mean of rated feeds and author.
|
||||
- **Normalization**: a constant signal normalizes to 0.5 for everyone; ties get equal percentiles (400 identical zeros → all 0.5, no id ramp); absent values do not shift others; effective weights sum to 1; a candidate missing a signal is scored on the rest.
|
||||
- **Triage and deep parsing**: realistic fixtures; malformed items do not sink a batch; unknown facet tokens degrade to `None`; every enum token in both prompts round-trips; cached assessments are reused within `assessment_reuse_days` and ignored with `--rescore`.
|
||||
- **Admission**: a strong-interest, weak-heuristic, no-social article reaches the deep set; a 60-word stub with high interest similarity is not admitted by `interest`/`knn`; quotas honoured; inactive retrievers release quota; exploration deterministic per date; auto-includes always admitted; excluded articles get thin rows with the right reason.
|
||||
|
||||
@@ -0,0 +1,815 @@
|
||||
# Web site and operator dashboard
|
||||
|
||||
**Date:** 2026-09-03
|
||||
**Repository:** `thallada/the-daily-epub`
|
||||
**Status:** implementation plan, ready to execute
|
||||
**Builds on:** `docs/plans/2026-09-02-personalized-curation-v2.md` (the ranker this dashboard inspects; its §23 deferred a "web dashboard for ratings and knobs" — this is that plan)
|
||||
|
||||
This plan is written so that a fresh implementation agent can execute it end to end. It records the decisions made in the 2026-09-03 brainstorming session with the operator, the verified facts about the current code and the production host, the target design in enough detail to code from, and the order in which to land it.
|
||||
|
||||
---
|
||||
|
||||
## 0. Decisions recorded from the brainstorm (2026-09-03)
|
||||
|
||||
These are settled. Do not reopen them during implementation.
|
||||
|
||||
| Topic | Decision |
|
||||
|---|---|
|
||||
| Purpose | Publicly, `https://daily.hallada.net` is an aggregator: every issue as a stripped HTML page of titles linking out, authors, sources and comment links. Privately, after login, it is the operator's dashboard: every article ever considered and why it was or was not in the paper, the full rating history with edits, every issue in full HTML with downloads, all config knobs, and pipeline actions. |
|
||||
| Frontend | Server-rendered **askama** templates plus one hand-written CSS file and a few hundred lines of plain JavaScript. No node build, no SPA, no framework; assets are embedded in the binary with `include_str!`. |
|
||||
| Auth | Username + password, sessions in a SQLite table, one `HttpOnly; Secure; SameSite=Lax` cookie. Users are created from the CLI (`daily-epub users add`). No third-party identity, no passkeys. **Use the ecosystem, not hand-rolled code**: `axum-login` over `tower-sessions` for login/logout/session lifecycle and route guards, `password-auth` for argon2 hashing, `tower_governor` for the login throttle. axum-login is taken from **git at the pinned rev `151c72d7a1b4646830f86b4332e6bd6e34d719a7`** (`main`, 2026-05-07: tower-sessions 0.15 and the finalized `Require` API), not from the 0.18.0 crates.io release; §2 records why and what was verified. The only auth code we write is a ~60-line `SessionStore` over our own sqlx pool (the official store pins sqlx 0.8; we are on 0.9) and the `AuthnBackend` glue (§6). |
|
||||
| Roles | `user` and `admin`. **Signed-in users of either role** see every issue in full HTML (Brief, summaries, article bodies, discussions, World Briefing, Behind the paper) and can download the EPUB/XTC files. **Admins** additionally rate, browse the dashboard, edit settings and the profile, and start jobs. |
|
||||
| Copyright boundary | Anonymous visitors see titles, metadata, AI summaries and `why` lines, but never the Brief, article bodies, comments or World Briefing. The public renderer takes a dedicated `PublicIssue` type that cannot carry those private fields. |
|
||||
| Personalization | One shared algorithm for now. `rating_events` gains a nullable `user_id` so a per-user algorithm is possible later; nothing else is per user. HMAC links from the EPUB stay unattributed (`user_id NULL`, `source = 'epub'`). |
|
||||
| Settings | Every key of `config.toml`, grouped by table, editable from the UI and written back **in place with comments preserved** (`toml_edit`). A key overridden by a `DAILY_EPUB_*` environment variable is shown locked. API keys and the HMAC secret are shown as present/absent only. The page re-reads the file on every view, so hand edits show up. `data/profile.md` gets an editor with version history. |
|
||||
| Actions | A Jobs page starts pipeline work through **systemd**: `systemctl start daily-epub-job@<name>.service`, permitted by a polkit rule for the `daily-epub` user. The server never runs the pipeline in-process (its unit has `MemoryDenyWriteExecute=yes`, which the XTC converter's Node JIT cannot live with). |
|
||||
| Feed | A public Atom feed at `/feed.xml` with one entry per issue, carrying the same stripped list as the public page. |
|
||||
| Existing routes | `/r/…`, `/opds…`, `/files/…`, `/healthz`, `/issues.json` keep working unchanged. `/files/*` additionally accepts a session cookie. |
|
||||
| Retention | HTML issue pages are built from the database, so they outlive the EPUB files (`retention_days`); download links appear only while the file exists. The operator should raise `curation.ranking.telemetry_retention_days` if they want candidate history beyond 180 days; this plan does not change the default. |
|
||||
| Design | Newspaper masthead in a serif, dashboard tables in the system sans stack, no external fonts or CDNs, light and dark via `prefers-color-scheme`, every wide table scrolls inside its own container. |
|
||||
|
||||
---
|
||||
|
||||
## 1. Read these first
|
||||
|
||||
- `docs/plans/2026-08-15-implementation-notes.md` — conventions (runtime sqlx queries, `jiff`, error style, askama, no network in tests). All of it applies here.
|
||||
- `docs/plans/2026-09-02-personalized-curation-v2.md` §6 (feedback), §7 (tables), §9–§13 (signals, admission, utility, editor — the numbers the dashboard explains), §15 (explain/stats), §19 (config).
|
||||
- `src/server.rs` — the existing axum router, `AppState`, `handle_rating`, Basic auth, `safe_join`, the tiny e-ink pages. This plan extends this file's router and moves nothing.
|
||||
- `src/auth.rs` — HMAC rating links (unchanged).
|
||||
- `src/config.rs` — `Config`, `Config::load`, `Config::resolve_path`, `validate`, `check_report`, `providers_redacted`, `stale_toml_error`, `ENV_PREFIX`/`ENV_SPLIT`.
|
||||
- `src/db.rs` — `Db`, `current_ratings`, `append_rating_event`, `upsert_issue`, `replace_issue_articles`, `start_run`/`finish_run`, `provider_spend_for_utc_day`, `get_article`.
|
||||
- `src/curate/telemetry.rs` — `ExplainRow`, `resolve_run`, `explain_row`, `near_misses`, `render_explain`, `stats`, `SignalsJson`. The dashboard renders the same data as HTML.
|
||||
- `src/curate/signals.rs` — `decay`, `gate`, `direct_feeds`, `PreferenceState::load/summary`, used by the ratings page.
|
||||
- `src/curate/profile/mod.rs` — `parse_profile_str`, `load_profile`, `load_or_build`, `rebuild`, `KV_LEARNED_ADJUSTMENTS`.
|
||||
- `src/epub/chapters.rs`, `src/epub/templates/` — what an issue contains and how it is rendered; `prepare_body`, `social_line`, `behind_*_line`, `render_in_this_issue`.
|
||||
- `src/epub/fixtures.rs` — `fixtures::issue()` builds a full `Issue` for tests; the web tests use it.
|
||||
- `src/main.rs` — CLI layout (`clap` derive), `lock_holder`, `cmd_*` helpers.
|
||||
- `systemd/`, `README.md` §"Reverse proxy", `docs/runbooks/curation-v2-migration.md` — how the host is set up.
|
||||
|
||||
---
|
||||
|
||||
## 2. Verified facts (2026-09-03)
|
||||
|
||||
Host and deployment (checked on the production box, which is also the dev checkout):
|
||||
|
||||
- nginx site `daily` proxies `/` → `http://127.0.0.1:3499` with `Host`, `X-Forwarded-For` and `X-Forwarded-Proto` set, TLS from Let's Encrypt, HTTP→HTTPS redirect. `snippets/security-headers.conf` adds `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY`, `Referrer-Policy: strict-origin-when-cross-origin`, a `Permissions-Policy`, and HSTS. **No Content-Security-Policy**, so inline `<script>`/`<style>` would work, but this plan serves CSS/JS from `/static/` anyway and adds its own CSP header from the app (§16).
|
||||
- `daily-epub.service` and `daily-epub-generate.timer` are active. Both units run as `daily-epub` (uid 995, no supplementary groups) with `ProtectSystem=strict`, `ProtectHome=read-only`, `ReadWritePaths=/home/thallada/bookorbit/books/daily-epub /var/lib/daily-epub/xtc`, `StateDirectory=daily-epub`. **`/etc/daily-epub` is therefore read-only inside the server** — the settings writer needs `ReadWritePaths=/etc/daily-epub` (§18). The server unit alone has `MemoryDenyWriteExecute=yes`.
|
||||
- `/etc/daily-epub/config.toml` is `0640 daily-epub:daily-epub` (per the unit header and runbook); `/etc/daily-epub/env` holds the API keys and `DAILY_EPUB_SERVER__HMAC_SECRET`.
|
||||
- polkit is version 124 (JavaScript rules in `/etc/polkit-1/rules.d/` are supported). `journalctl -u <unit>` as `daily-epub` requires membership in `systemd-journal` (gid 999).
|
||||
- `systemctl show -p ActiveState,SubState,Result,ExecMainStatus,ExecMainStartTimestamp,ExecMainExitTimestamp <unit>` works unprivileged and is the status source for jobs.
|
||||
|
||||
Code:
|
||||
|
||||
- `issues.report_json` is **never written**: `pipeline::record_issue` passes `None` to `upsert_issue`, and nothing else sets it, so `GET /issues.json` returns `"report": null` for every issue. `runs` keeps only counters. This plan fixes both (§5.3).
|
||||
- The World Briefing and the discussion chapters are built at run time and never persisted; `Issue` (with `Lineup`, `Editorial`, `WorldBriefing`, `Colophon`, `BehindThePaper`, `Pick.discussion`) derives `Serialize`/`Deserialize`. Article bodies live in `articles.content_html` for every article ever ingested; nothing prunes `articles`.
|
||||
- `Config::load(explicit)` reads `--config` or `./config.toml`, then `DAILY_EPUB_*` env (`ENV_PREFIX = "DAILY_EPUB_"`, `ENV_SPLIT = "__"`), accepts `DAILY_EPUB_SECRET` as an alias for `server.hmac_secret`, then `validate()`. `Config::resolve_path` returns the path `load` used. `Config` and every sub-struct derive `Serialize`/`Deserialize` with `deny_unknown_fields`; `Config::default()` is the shipped example key for key (a test enforces it).
|
||||
- `main::serve` receives `(config, db)` only; the config path is not passed through. `AppState { db, config }` is cloned per request.
|
||||
- Crates already present: `axum 0.8`, `askama 0.16`, `tower-http 0.7` (`trace`, `fs`), `sqlx 0.9` (sqlite, runtime queries), `jiff`, `serde_json`, `sha2`, `hex`, `hmac`, `base64`, `rand 0.10`, `ammonia`, `figment`, `libc`. `toml_edit` and `toml` are in `Cargo.lock` transitively (via figment) but not direct dependencies; `tower` is transitive only.
|
||||
- Current crates.io versions (checked with `cargo search` on 2026-09-03): `toml_edit 0.25.13`, `tower 0.5.3`, `askama 0.16.0`. Add with `cargo add` and let Cargo resolve; do not hand-pin older versions.
|
||||
|
||||
Auth ecosystem (crates.io dependency metadata and docs.rs, checked 2026-09-03):
|
||||
|
||||
- **`axum-login` 0.18.0** (crates.io, released 2025-07-20, ~1M downloads): `axum ^0.8.1`, `tower-sessions ^0.14` (re-exported as `axum_login::tower_sessions`), `tower-cookies ^0.11`. Exports `AuthUser` (`id()`, `session_auth_hash()`), `AuthnBackend` (`authenticate`, `get_user`), `AuthzBackend` (`get_user_permissions`), the `AuthSession` extractor, `AuthManagerLayerBuilder::new(backend, session_layer).build()`, and the `login_required!`, `permission_required!`, `predicate_required!` route-layer macros. Login cycles the session id (fixation defence) and stores the user's `session_auth_hash`, so changing a password invalidates every session of that user.
|
||||
- **`axum-login` `main` at `151c72d7a1b4646830f86b4332e6bd6e34d719a7`** (2026-05-07, "Upgrade tower-sessions to 0.15.0 (#315)"; 13 commits past `v0.18.0`, none since; the default branch is `main`, there is no `master`). What changed since 0.18.0 (CHANGELOG "Unreleased" + diff): `tower-sessions ^0.15`; the new builder-based `require` module (`Require::<Backend>::builder().decision(PermissionsPredicate).unauthenticated(RedirectHandler::new().login_url(..).redirect_field(..)).build()`) with the macros rewritten as thin wrappers over it; `permission_required!` answers 401 when unauthenticated (302 to `login_url` when one is given) and 403 only when authenticated without the permission; `AuthSession` is now `Clone` over an `Arc<Mutex<_>>` with `login(&self, &user)`, `logout(&self)`, `authenticate(&self, creds)`, **`user().await -> Option<User>` (a method, no longer a public field)** and `backend()`; the `session` field is no longer public (use tower-sessions' own `Session` extractor for flashes); the `async-trait` dependency is gone (native async fn in traits). The user id is stored in the session under the key `"axum-login.data"` (configurable with `AuthManagerLayerBuilder::with_data_key`).
|
||||
**Verified locally on 2026-09-03** (nightly 1.100, 2026-09-01) at that rev: `cargo clippy --lib --all-features` clean; `cargo test --doc` 15/15; `cargo test --lib` 79/79 once the unit-test modules use `MemoryStore` instead of `tower_sessions_sqlx_store::SqliteStore`; `tests/feature_gates.rs` passes. A scratch consumer crate depending on the git rev together with `tower-sessions 0.15`, `sqlx 0.9` (sqlite, single copy, single `libsqlite3-sys 0.37`), `axum 0.8.9`, `tower_governor 0.8`, `password-auth 1.0` and the §6.2 store implementation compiles (`cargo check`, `cargo tree -d` shows no second sqlx or sqlite).
|
||||
**Why the repository's CI is red** (the push run for that commit fails at the `check` job's clippy step and skips the tests): the crate's dev-dependency and its examples use `tower-sessions-sqlx-store 0.15.0`, which depends on `tower-sessions-core 0.14`; against `tower-sessions 0.15` that store no longer implements the (0.15) `SessionStore` trait, so the test targets and the examples fail to compile and the integration tests (which spawn the example binaries) fail. That is a harness/examples problem in the stores repository lagging behind, not a library defect; our own store is written against whatever `tower-sessions` axum-login re-exports, so it is unaffected either way.
|
||||
**Maintenance status** (discussion #330, opened 2026-07-12 by the author): "I don't have the time I would like to maintain axum-login"; three volunteers (one collaborator offering co-maintenance, one asking about updating the stores for 0.15 on 2026-08-06) and no release or new commit since 2026-06. Expect a 0.19 eventually; pinning the rev means the plan's API is the one 0.19 will ship, and a later switch to the crates.io release is a one-line `Cargo.toml` change.
|
||||
**Fallback** if the git dependency is ever unwanted: `axum-login = "0.18"` from crates.io with `tower-sessions 0.14`; the only source differences are `auth.user` (field) instead of `auth.user().await`, `&mut auth` for `login`/`logout`, and `pub session` on `AuthSession`.
|
||||
- **`tower-sessions` 0.15.0** (`SessionManagerLayer::new(store).with_secure(..).with_same_site(..).with_name(..).with_expiry(Expiry::OnInactivity(time::Duration))`): lazy — no cookie is set unless a handler touches the session, so anonymous pages stay cookie-free. `SessionStore` is a trait with `save`, `load`, `delete` (and a provided `create`) over `Record { id: Id, data: HashMap<String, serde_json::Value>, expiry_date: time::OffsetDateTime }`; the trait is unchanged from 0.14. 0.15 over 0.14 is only the `rand 0.9` bump and a memory-ordering fix in the session's modified-flag handling (#254), which is the other reason to take the git rev of axum-login rather than 0.18.0.
|
||||
- **`tower-sessions-sqlx-store` 0.15.0** pins **`sqlx ^0.8`** (and `tower-sessions-core 0.14`). This crate is on `sqlx 0.9`; two sqlx versions would each bundle SQLite and collide on the `links = "sqlite3"` native key, so the official store **cannot** be used. A store over our own pool is ~60 lines (§6.2).
|
||||
- **`password-auth` 1.0.0** (`generate_hash(password) -> String` PHC/argon2id, `verify_password(password, hash) -> Result<(), VerifyError>`; `argon2 ^0.5` inside). The crate axum-login's own SQLite example uses.
|
||||
- **`tower_governor` 0.8.0** (`governor 0.10`, `axum` feature for `axum ^0.8`): `GovernorConfigBuilder::default().per_second(n).burst_size(n).key_extractor(SmartIpKeyExtractor)`, `GovernorLayer::new(config)`, a periodic `limiter.retain_recent()` cleanup, and `into_make_service_with_connect_info::<SocketAddr>()` on the server. `SmartIpKeyExtractor` reads `x-forwarded-for`, `x-real-ip`, `forwarded`, then the peer address.
|
||||
- **`axum_csrf` 0.11.0** (`axum-core ^0.5`, `cookie 0.18`): a signed-cookie token with a `CsrfToken` extractor (`authenticity_token()`, `verify()`). Compatible, but it is a second cookie plus per-form plumbing that `SameSite=Lax` and a fetch-metadata origin check already cover; kept as the fallback option (§6.4).
|
||||
- **`axum_session` 0.21 / `axum_session_sqlx` 0.11 / `axum_session_auth` 0.21** (released 2026-08-10): match `sqlx ^0.9` and `axum ^0.8.9` exactly and would need no custom store, but they add `chrono`, `dashmap` and `async-recursion`, create their own table outside our migrations, cache users in memory (`cache_clear_user` after a role change), and set a cookie for every visitor unless the session mode is changed. Rejected in favour of the larger and lazier `axum-login`/`tower-sessions` pair; recorded here so the choice is not re-litigated.
|
||||
- `askama.toml` has `dirs = ["src/epub/templates"]`; the EPUB templates are `.xhtml` with `escape = "html"` declared per template. `.html` templates are HTML-escaped by default.
|
||||
- `telemetry::resolve_run` treats the latest run with `status != 'dry_run'` as the run of a date; `RunStatus` is `running | ok | degraded | failed | dry_run`.
|
||||
- `db::current_ratings_including_cleared(lookback_days)` returns the latest explicit event per article joined with title, feed, latest summary and deep facets. `profile::MAX_RATINGS_IN_REBUILD` (200) and `RATINGS_LOOKBACK_DAYS` (36,500) are private constants; make the first `pub`.
|
||||
- `signals::decay(age_days, half_life)`, `signals::gate(n, floor, full)`, `signals::direct_feeds(&article)` and `PreferenceState::load(..)` + `summary()` exist and are `pub`.
|
||||
- Rating attribution today: `RatingEvent { source: "epub" | "cli" | "migration", .. }`; `"dashboard"` is already reserved in the plan's `source` comment.
|
||||
- `publish::issue_filename(date, edition, "epub")` gives the EPUB names the OPDS feed and `/files/epub/{name}` use; `issues.epub_path/x4_path/xtc_path` hold full paths.
|
||||
- `server.rs` unit tests call handlers and helpers directly (12 tests); `tests/m7_server.rs` drives the real binary over TCP with a hand-rolled HTTP client and an env-only configuration. Both patterns continue.
|
||||
|
||||
---
|
||||
|
||||
## 3. Site map and access levels
|
||||
|
||||
| Route | Anonymous | Signed in (user) | Admin |
|
||||
|---|---|---|---|
|
||||
| `GET /` | latest issue, public rendering | latest issue, full rendering | same + rating buttons |
|
||||
| `GET /issues` | archive by month | same | same |
|
||||
| `GET /issues/{date}` | public issue page | full issue page + downloads | same + rating buttons |
|
||||
| `GET /issues/{date}/articles/{article_id}` | 302 → `/login?next=` | article chapter (body, discussion) | same + rating buttons |
|
||||
| `GET /issues/{date}/world`, `/behind` | 302 → login | the chapter | same |
|
||||
| `GET /feed.xml` | Atom, public content | same | same |
|
||||
| `GET /robots.txt` | allow `/`, `/issues`; disallow `/dashboard`, `/login`, `/files`, `/r`, `/opds` | | |
|
||||
| `GET /static/{file}` | `app.css`, `app.js`, `favicon.svg` | | |
|
||||
| `GET /login`, `POST /login`, `POST /logout` | login form | | |
|
||||
| `GET /account`, `POST /account/password` | 302 → login | change own password, sign out everywhere | same |
|
||||
| `GET /files/epub/{name}`, `/files/xtc/{name}` | Basic auth (existing) | session cookie **or** Basic auth | same |
|
||||
| `POST /rate` | 403 | 403 | append a rating event |
|
||||
| `GET /dashboard` | 302 → login | 403 page | overview |
|
||||
| `GET /dashboard/runs`, `/dashboard/runs/{id}` | | 403 | runs list, run detail with candidate table |
|
||||
| `GET /dashboard/articles`, `/dashboard/articles/{id}` | | 403 | every article, article detail |
|
||||
| `GET /dashboard/ratings`, `POST /rate` | | 403 | rating history, contributions, edit |
|
||||
| `GET /dashboard/profile`, `POST /dashboard/profile`, `POST /dashboard/profile/restore` | | 403 | `profile.md` editor, prompt, history |
|
||||
| `GET /dashboard/stats` | | 403 | the `stats` numbers as tables + sparklines |
|
||||
| `GET /dashboard/settings`, `POST /dashboard/settings`, `POST /dashboard/settings/providers`, `GET /dashboard/settings/history` | | 403 | every config key |
|
||||
| `GET /dashboard/jobs`, `POST /dashboard/jobs/{name}`, `GET /dashboard/jobs/{id}` | | 403 | start and watch jobs |
|
||||
| `GET /dashboard/users` | | 403 | list users and sessions (read-only; CLI edits) |
|
||||
| `GET /r/{date}/{article_id}/{vote}?t=` | HMAC (existing) | | |
|
||||
| `GET /opds…`, `/healthz`, `/issues.json` | existing behaviour | | |
|
||||
|
||||
Every `/dashboard/*` page is `Cache-Control: no-store`. Every HTML response carries `Vary: Cookie`.
|
||||
|
||||
Dates in paths are `YYYY-MM-DD` issue dates; ids are database ids. An unknown date or id is a 404 page in the site layout, not a bare string.
|
||||
|
||||
---
|
||||
|
||||
## 4. Architecture
|
||||
|
||||
### 4.1 Module layout
|
||||
|
||||
```text
|
||||
src/server.rs existing router + handlers; `router()` now merges `web::router()`
|
||||
src/web/
|
||||
├── mod.rs WebState, `Html<T>` response helper, WebError, layout context, pagination, time formatting
|
||||
├── session.rs `SqliteSessionStore` (tower-sessions `SessionStore` over our pool), `Backend` (axum-login `AuthnBackend` + `AuthzBackend`), `Viewer` helper, origin check middleware
|
||||
├── users.rs User/Role types, password-auth hash/verify wrappers, `daily-epub users …` implementation
|
||||
├── public.rs `/`, `/issues`, `/issues/{date}` (public branch), `/feed.xml`, `/robots.txt`, `PublicIssue`
|
||||
├── issue.rs IssueView loader (issue_json → fallback), full issue page, article/world/behind pages, downloads gate
|
||||
├── rate.rs `POST /rate` (form + JSON), rating widget data
|
||||
├── dashboard/
|
||||
│ ├── mod.rs overview
|
||||
│ ├── runs.rs runs list, run detail, candidate table, config diff
|
||||
│ ├── articles.rs articles list, article detail
|
||||
│ ├── ratings.rs ratings page, contributions
|
||||
│ ├── profile.rs profile editor, history, prompt view
|
||||
│ ├── stats.rs stats page (renders `telemetry::StatsData`)
|
||||
│ ├── settings.rs settings schema, form, toml_edit writer, providers, history
|
||||
│ ├── jobs.rs jobs page, JobRunner trait, systemctl/journalctl runner
|
||||
│ └── users.rs users list page
|
||||
├── static/
|
||||
│ ├── app.css
|
||||
│ ├── app.js
|
||||
│ └── favicon.svg
|
||||
└── templates/ askama `.html` templates (§4.3)
|
||||
src/jobs.rs job catalogue (name → CLI equivalent), `daily-epub job run`, jobs table lifecycle
|
||||
migrations/0004_web.sql
|
||||
systemd/daily-epub-job@.service
|
||||
systemd/50-daily-epub.rules polkit rule
|
||||
docs/runbooks/web-dashboard-rollout.md
|
||||
```
|
||||
|
||||
`askama.toml` becomes `dirs = ["src/epub/templates", "src/web/templates"]`. Web templates end in `.html`; there is no name collision with the `.xhtml` EPUB templates.
|
||||
|
||||
### 4.2 State
|
||||
|
||||
```rust
|
||||
pub struct AppState {
|
||||
pub db: Db,
|
||||
pub config: Arc<RwLock<Arc<Config>>>, // swapped after a settings save or when the file's mtime changes
|
||||
pub config_path: Option<PathBuf>, // Config::resolve_path(cli.config); None ⇒ settings read-only
|
||||
pub web: Arc<WebState>,
|
||||
}
|
||||
pub struct WebState {
|
||||
pub jobs: Arc<dyn JobRunner>, // SystemdRunner in production, MockRunner in tests
|
||||
pub started_at: Timestamp,
|
||||
pub config_mtime: Mutex<Option<SystemTime>>,
|
||||
}
|
||||
impl AppState { pub fn config(&self) -> Arc<Config> } // read-lock + clone the Arc
|
||||
```
|
||||
|
||||
`main::serve` gains the config path: `server::serve(config, config_path, db)`. Handlers that today read `state.config.server.*` call `state.config()` instead (mechanical change; keep the tests).
|
||||
|
||||
**Config reload.** `WebState::reload_if_changed(&AppState)` compares the file's mtime with the cached one on every `/dashboard/settings` GET and every job start, re-runs `Config::load(config_path)` and swaps the Arc when it changed and validates. A file that fails validation is reported on the settings page (`! config.toml on disk does not load: <error>`) and the previous config stays live.
|
||||
|
||||
### 4.3 Templates and rendering
|
||||
|
||||
- `layout.html`: `<head>` (charset, viewport, `<title>`, `/static/app.css?v={VERSION}`, Atom `<link rel="alternate">`, favicon), masthead ("The Daily EPUB", serif), primary nav (Latest · Archive · Feed · Sign in / account name), admin nav bar when the viewer is an admin (Overview · Runs · Articles · Ratings · Profile · Stats · Jobs · Settings · Users), `<main>`, footer (`daily-epub {VERSION}`), `/static/app.js` at the end.
|
||||
- Every page template `{% extends "layout.html" %}` and receives a `Page` context: `title`, `viewer: Option<Viewer { username, role }>`, `flash: Option<Flash>`, `active_nav`.
|
||||
- `Html<T: Template>` implements `IntoResponse`: renders, sets `text/html; charset=utf-8`, and on a render error logs and returns the 500 page.
|
||||
- `WebError { NotFound, Forbidden, Unauthenticated { next }, BadRequest(String), Csrf, Db(sqlx::Error), Internal(anyhow::Error) }` → `IntoResponse`. `Unauthenticated` redirects HTML requests to `/login?next=<path>` and returns 401 JSON to `Accept: application/json`.
|
||||
- Flash messages live in the tower-sessions session (`session.insert("flash", Flash { kind, text })`, removed when the next page reads it), so POST-redirect-GET can confirm "Saved 3 settings" or "Rated: Loved it". Only signed-in flows set flashes, so anonymous visitors never get a session cookie from this.
|
||||
- Templates (all under `src/web/templates/`): `layout.html`, `error.html`, `login.html`, `account.html`, `issue_public.html`, `issue_full.html`, `issue_list.html`, `article.html`, `world.html`, `behind.html`, `dashboard/overview.html`, `dashboard/runs.html`, `dashboard/run.html`, `dashboard/articles.html`, `dashboard/article.html`, `dashboard/ratings.html`, `dashboard/profile.html`, `dashboard/stats.html`, `dashboard/settings.html`, `dashboard/settings_history.html`, `dashboard/jobs.html`, `dashboard/job.html`, `dashboard/users.html`, plus partials `_rating_widget.html`, `_pagination.html`, `_signals_table.html`, `_candidate_row.html`, `_sparkline.html`.
|
||||
- Static assets: `include_str!` the three files; `GET /static/{file}` sets `Content-Type`, `Cache-Control: public, max-age=86400`, and an `ETag` of the sha256 (respond 304 to `If-None-Match`). The `?v={VERSION}` query busts caches across releases.
|
||||
|
||||
### 4.4 Design system (the one CSS file)
|
||||
|
||||
- Tokens: `--bg`, `--fg`, `--muted`, `--rule`, `--accent`, `--loved`, `--good`, `--down`, redefined under `@media (prefers-color-scheme: dark)`.
|
||||
- Masthead and issue pages: Georgia/serif headings, `max-width: 72ch` reading column, hairline rules like the EPUB front page.
|
||||
- Dashboard: system sans, `max-width: 1200px`, dense tables (`font-size: .9rem`, sticky `<thead>`), every `<table>` wrapped in `.scroll-x { overflow-x: auto }`.
|
||||
- Components: `.badge` per stage/reason/label (`selected`, `shortlisted`, `assessed`, `admitted`, `triaged`, `eligible`, `excluded`, `loved`, `good`, `down`, `cleared`), `.kv` definition lists, `.funnel` (a horizontal bar per stage, widths proportional to counts), `.spark` (inline SVG polyline, server-rendered), `.rating` button group with the active verdict filled.
|
||||
- No layout JavaScript. `app.js` does exactly: intercept `.rating` forms and `POST` with `fetch` + `Accept: application/json`, then update the button state; confirm dialogs on `data-confirm` forms (jobs, provider removal, clear rating); auto-refresh `dashboard/job.html` every 5 s while the status is `requested`/`running`; a "filter as you type" on tables with `data-filter` (client-side, current page only); collapse/expand for `<details>` state kept in `localStorage` (guard with try/catch).
|
||||
|
||||
---
|
||||
|
||||
## 5. Data model — migration `0004_web.sql`
|
||||
|
||||
Do not edit earlier migrations. Plain SQL, no Rust bootstrap.
|
||||
|
||||
### 5.1 Tables
|
||||
|
||||
```sql
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
||||
password_hash TEXT NOT NULL, -- PHC string from argon2
|
||||
role TEXT NOT NULL CHECK (role IN ('user', 'admin')),
|
||||
disabled INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
last_login_at TEXT
|
||||
);
|
||||
|
||||
-- tower-sessions records (§6.2). `id` is the session Id's string form (a random
|
||||
-- 128-bit value the cookie carries), `data` the JSON-encoded record map.
|
||||
-- `expiry` is unix seconds because tower-sessions hands us a `time::OffsetDateTime`
|
||||
-- and the store compares it in SQL; `user_id` is denormalized from the
|
||||
-- axum-login entry in `data` so the Users page and `users logout` can find a
|
||||
-- user's sessions without decoding every row.
|
||||
CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
expiry INTEGER NOT NULL,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_sessions_user ON sessions(user_id);
|
||||
CREATE INDEX idx_sessions_expiry ON sessions(expiry);
|
||||
|
||||
ALTER TABLE rating_events ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE TABLE config_changes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
key TEXT NOT NULL, -- dotted path, e.g. curation.ranking.deep_keep
|
||||
old_value TEXT, -- TOML literal as it was in the file (NULL = absent)
|
||||
new_value TEXT, -- NULL = removed
|
||||
changed_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_config_changes_at ON config_changes(changed_at);
|
||||
|
||||
CREATE TABLE profile_versions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
content TEXT NOT NULL, -- the profile.md text *before* the save (and one row for the first save's original)
|
||||
saved_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
saved_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL, -- catalogue name, e.g. generate, dry-run, generate-2026-09-01
|
||||
unit TEXT NOT NULL, -- daily-epub-job@<name>.service
|
||||
requested_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
requested_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('requested', 'running', 'ok', 'failed')),
|
||||
message TEXT, -- error text or one-line summary
|
||||
run_id INTEGER REFERENCES runs(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX idx_jobs_requested_at ON jobs(requested_at);
|
||||
|
||||
ALTER TABLE runs ADD COLUMN report_json TEXT;
|
||||
ALTER TABLE issues ADD COLUMN issue_json TEXT;
|
||||
```
|
||||
|
||||
### 5.2 `issues.issue_json`
|
||||
|
||||
The full `Issue` the pipeline assembled, serialized with `serde_json`, **with every `pick.article.content_html` replaced by an empty string** (bodies are already in `articles.content_html`; the discussion threads and the World Briefing are the parts that exist nowhere else). Written by `pipeline::record_issue` alongside the lineup. A regenerated date overwrites it. Expected size: 100–300 KB per issue.
|
||||
|
||||
The web loader (`web::issue::load`) rehydrates each pick's body from `articles` by id. Issues published before this change have `issue_json IS NULL`; the loader then builds a reduced `Issue` from `issues` (`front_page_html` as the Brief, `issue_number`, `generated_at`), `issue_articles` (section, position, `is_lead`, `summary`, `why`) and `articles` — with `world_briefing = None`, `discussion = None`, `behind = Default`. Section order for the fallback is `config.curation.sections` order, then any unknown section names in first-seen order.
|
||||
|
||||
### 5.3 `runs.report_json` and `issues.report_json`
|
||||
|
||||
`db::finish_run` writes `serde_json::to_string(report)` into `runs.report_json`. `pipeline::record_issue` passes the same string as `report_json` to `upsert_issue` (fixing the never-written column; `GET /issues.json` starts returning real reports). `RunReport` already serializes; nothing in it is secret (`config_json` is `providers_redacted()`).
|
||||
|
||||
### 5.4 Job attribution and rating attribution
|
||||
|
||||
- `POST /rate` appends `RatingEvent { source: "dashboard", user_id: Some(viewer.id), .. }`. Extend `RatingEvent` with `pub user_id: Option<i64>` (default `None`; the `epub`/`cli` writers leave it `None`) and `db::append_rating_event` binds it. `db::current_ratings*` and `rated_article_from_row` gain the column so the ratings page can show who rated.
|
||||
- `jobs.run_id` is set by `daily-epub job run` for the `generate*` jobs from `GenerateOutcome` (add `pub run_id: i64` to `GenerateOutcome`; `pipeline::generate` already has it as `run_id`).
|
||||
|
||||
---
|
||||
|
||||
## 6. Authentication and sessions
|
||||
|
||||
Sessions, login, logout and route guards come from **`axum-login` at the pinned git rev** (§2) over **`tower-sessions` 0.15** (the version it re-exports; use `axum_login::tower_sessions`, do not add `tower-sessions` separately). Passwords go through **`password-auth` 1.0**. The login throttle is **`tower_governor` 0.8**. What we write ourselves is limited to: the session store over our sqlx pool (§6.2), the `AuthnBackend`/`AuthzBackend` impl over the `users` table (§6.3), the origin check (§6.4), and the CLI.
|
||||
|
||||
### 6.1 Users and passwords (`src/web/users.rs`)
|
||||
|
||||
```rust
|
||||
pub enum Role { User, Admin } // as_str: "user" | "admin"; parse; Display
|
||||
#[derive(Clone)]
|
||||
pub struct User { id: i64, username: String, password_hash: String, role: Role, disabled: bool, created_at, last_login_at }
|
||||
impl std::fmt::Debug for User { /* redact password_hash */ }
|
||||
pub fn hash_password(plain: &str) -> String // password_auth::generate_hash (argon2id, random salt, PHC string)
|
||||
pub fn verify_password(hash: &str, plain: &str) -> bool // password_auth::verify_password(..).is_ok()
|
||||
```
|
||||
|
||||
Rules: usernames are 1–32 chars of `[A-Za-z0-9._-]`, compared case-insensitively (`COLLATE NOCASE`); passwords are at least 12 characters and at most 1,024. `password_auth::verify_password` is a blocking argon2 computation; call it inside `tokio::task::spawn_blocking` from the backend.
|
||||
|
||||
CLI (`src/main.rs` + `web::users`), none of which take the run lock:
|
||||
|
||||
```text
|
||||
daily-epub users add <username> [--admin] [--password-stdin] # hidden double prompt (rpassword), or one line from stdin
|
||||
daily-epub users passwd <username> [--password-stdin] # also deletes the user's sessions (session_auth_hash changes anyway)
|
||||
daily-epub users role <username> user|admin
|
||||
daily-epub users disable <username> | enable <username> # disable also deletes the user's sessions
|
||||
daily-epub users list # username · role · created · last login · open sessions
|
||||
daily-epub users logout <username> # delete that user's sessions
|
||||
```
|
||||
|
||||
`users add` refuses a duplicate name; `--admin` on the very first user is the intended bootstrap (§18).
|
||||
|
||||
### 6.2 Session store (`src/web/session.rs`)
|
||||
|
||||
```rust
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SqliteSessionStore { pool: SqlitePool }
|
||||
|
||||
#[async_trait]
|
||||
impl tower_sessions::SessionStore for SqliteSessionStore {
|
||||
async fn save(&self, record: &Record) -> session_store::Result<()>; // INSERT … ON CONFLICT(id) DO UPDATE SET data, expiry, user_id, updated_at
|
||||
async fn load(&self, id: &Id) -> session_store::Result<Option<Record>>; // SELECT data FROM sessions WHERE id = ? AND expiry > strftime('%s','now')
|
||||
async fn delete(&self, id: &Id) -> session_store::Result<()>;
|
||||
}
|
||||
impl SqliteSessionStore {
|
||||
pub async fn delete_expired(&self) -> Result<u64>; // on server start and once an hour from a tokio task
|
||||
pub async fn delete_for_user(&self, user_id: i64) -> Result<u64>; // users passwd/disable/logout, "sign out everywhere"
|
||||
}
|
||||
```
|
||||
|
||||
`data` is `serde_json::to_string(&record.data)`; `expiry` is `record.expiry_date.unix_timestamp()`. `user_id` is read best-effort from `record.data["axum-login.data"]["user_id"]` (the key axum-login uses) and stored alongside, so the Users page and the CLI can list and delete a user's sessions without decoding every row; a record without that key (an anonymous session that set flash data) stores `NULL`. Map every sqlx error to `session_store::Error::Backend(e.to_string())`. This is the whole store: ~60 lines plus tests. It mirrors what `tower-sessions-sqlx-store` does, which we cannot link because it pins sqlx 0.8 (§2).
|
||||
|
||||
Layer, built in `server::router`:
|
||||
|
||||
```rust
|
||||
let session_layer = SessionManagerLayer::new(SqliteSessionStore::new(db.pool().clone()))
|
||||
.with_name("daily_session")
|
||||
.with_http_only(true)
|
||||
.with_same_site(SameSite::Lax)
|
||||
.with_secure(config.server.public_url.starts_with("https://"))
|
||||
.with_expiry(Expiry::OnInactivity(time::Duration::days(config.server.session_days as i64)));
|
||||
let auth_layer = AuthManagerLayerBuilder::new(Backend::new(db.clone()), session_layer).build();
|
||||
```
|
||||
|
||||
`Expiry::OnInactivity` is the sliding lifetime. tower-sessions only writes a cookie when a handler touches the session, so anonymous page views create no rows and no cookie. Flash messages (§4.3) use tower-sessions' own `Session` extractor (`session.insert("flash", ..).await` / `session.remove(..).await`), since `AuthSession` no longer exposes its inner session.
|
||||
|
||||
### 6.3 Backend (`src/web/session.rs`)
|
||||
|
||||
```rust
|
||||
#[derive(Clone)] pub struct Backend { db: Db }
|
||||
pub struct Credentials { username: String, password: String, next: Option<String> }
|
||||
|
||||
impl AuthUser for User {
|
||||
type Id = i64;
|
||||
fn id(&self) -> i64 { self.id }
|
||||
fn session_auth_hash(&self) -> &[u8] { self.password_hash.as_bytes() } // password change ⇒ every session of the user is invalid
|
||||
}
|
||||
impl AuthnBackend for Backend { // native async fn; no async-trait at this rev
|
||||
type User = User; type Credentials = Credentials; type Error = BackendError;
|
||||
async fn authenticate(&self, creds) -> Result<Option<User>> {
|
||||
// SELECT by username COLLATE NOCASE; verify against the row's hash, or against
|
||||
// a fixed dummy hash when the row is missing (equal timing); None when disabled.
|
||||
}
|
||||
async fn get_user(&self, id: &i64) -> Result<Option<User>> { /* None when disabled */ }
|
||||
}
|
||||
impl AuthzBackend for Backend {
|
||||
type Permission = Role; // Hash + Eq + Clone
|
||||
async fn get_user_permissions(&self, user: &User) -> Result<HashSet<Role>> {
|
||||
Ok(match user.role { Role::Admin => [Role::User, Role::Admin].into(), Role::User => [Role::User].into() })
|
||||
}
|
||||
}
|
||||
pub type AuthSession = axum_login::AuthSession<Backend>;
|
||||
```
|
||||
|
||||
Guards, applied as `route_layer` on the sub-routers (§3):
|
||||
|
||||
```rust
|
||||
// any signed-in user: full issues, article/world/behind pages, /account
|
||||
.route_layer(login_required!(Backend, login_url = "/login", redirect_field = "next"))
|
||||
// admins: everything under /dashboard and POST /rate
|
||||
.route_layer(permission_required!(Backend, login_url = "/login", redirect_field = "next", Role::Admin))
|
||||
```
|
||||
|
||||
`permission_required!` redirects anonymous requests to `/login?next=…` and answers 403 to a signed-in user without the permission; the 403 is turned into the site's error page by a small `map_response` on the dashboard router. Pages that render differently for anonymous and signed-in viewers (`/`, `/issues/{date}`) take `AuthSession` and branch on `auth.user().await`. `/files/*` (§8) checks `auth.user().await.is_some()` before falling back to Basic auth.
|
||||
|
||||
`Viewer { user: User }` is derived from `auth.user().await` for templates; the layout shows the username and role from it.
|
||||
|
||||
Login: `GET /login?next=` renders the form. `POST /login` calls `auth.authenticate(creds)`; `Ok(Some(user))` → `auth.login(&user).await` (axum-login cycles the session id), `UPDATE users SET last_login_at`, redirect to `next` when it is a same-site absolute path (starts with `/`, not `//`), else `/`; `Ok(None)` → the form again with "invalid username or password" (status 401); `Err` → 500 page. `POST /logout` → `auth.logout().await`, redirect `/`. `/account` shows the viewer and offers "change password" (verify current, `hash_password`, update the row; the new `session_auth_hash` logs out every other session automatically) and "sign out everywhere" (`delete_for_user`).
|
||||
|
||||
### 6.4 CSRF: SameSite + origin check
|
||||
|
||||
The session cookie is `SameSite=Lax`, so browsers do not attach it to cross-site `POST`s at all. On top of that, one middleware (`web::session::require_same_origin`, applied to every `POST` route except `/r/*` which has no session) rejects with 403 when: `Sec-Fetch-Site` is present and is neither `same-origin` nor `none`; or, when it is absent, the `Origin` header (or the origin of `Referer`) does not equal the origin of `server.public_url` or of the request `Host`. That is the OWASP "fetch metadata + origin verification" defence and is enough for cookie-authenticated forms; there is no per-form token to render or check. `app.js` `fetch` calls send `credentials: "same-origin"` and get `Sec-Fetch-Site: same-origin` from the browser automatically.
|
||||
|
||||
If a per-form token is ever wanted (e.g. for a very old browser without fetch metadata), add `axum_csrf` 0.11 as a second layer; nothing in this plan precludes it.
|
||||
|
||||
### 6.5 Login throttle
|
||||
|
||||
`tower_governor` on the `POST /login` route only:
|
||||
|
||||
```rust
|
||||
let governor = Arc::new(GovernorConfigBuilder::default()
|
||||
.per_second(config.server.login_window_minutes * 60 / config.server.login_attempts) // e.g. 90 s per token
|
||||
.burst_size(config.server.login_attempts) // e.g. 10
|
||||
.key_extractor(SmartIpKeyExtractor)
|
||||
.finish().expect("governor config"));
|
||||
// route_layer(GovernorLayer::new(governor.clone())) on POST /login; a tokio task calls
|
||||
// governor.limiter().retain_recent() every 60 s.
|
||||
```
|
||||
|
||||
`SmartIpKeyExtractor` trusts `X-Forwarded-For`; that is acceptable because `server.bind` is loopback and nginx is the only peer (README §"Reverse proxy" says so; add a sentence). `server::serve` binds with `into_make_service_with_connect_info::<SocketAddr>()` so the peer address is available as the fallback key. Over the limit the layer answers 429; the login page template is not involved. Note the limiter counts attempts, not failures — ten login POSTs per fifteen minutes per IP is plenty for one operator.
|
||||
|
||||
## 7. Public site
|
||||
|
||||
### 7.1 `PublicIssue` — the only thing the public templates receive
|
||||
|
||||
```rust
|
||||
pub struct PublicIssue {
|
||||
pub date: Date, pub issue_number: i64, pub display_date: String,
|
||||
pub article_count: i64, pub reading_minutes: i64,
|
||||
pub sections: Vec<PublicSection>,
|
||||
pub generated_at: Timestamp,
|
||||
}
|
||||
pub struct PublicSection { pub name: String, pub entries: Vec<PublicEntry> }
|
||||
pub struct PublicEntry {
|
||||
pub title: String, pub url: String, // canonical_url; the link target
|
||||
pub author: Option<String>,
|
||||
pub source: String, // feed_title
|
||||
pub domain: String, // host of `url`, "www." stripped
|
||||
pub reading_minutes: i64, pub word_count: i64,
|
||||
pub summary: Option<String>, pub why: Option<String>,
|
||||
pub comment_links: Vec<CommentLink>, // {label, url, meta}
|
||||
pub is_lead: bool,
|
||||
}
|
||||
```
|
||||
|
||||
`PublicIssue::from(&Issue)` is the only constructor, and it copies exactly these fields. There is deliberately no `body`, `brief`, or `world` field, so the template cannot render them. Test `public_issue_shows_summaries_and_why_but_no_bodies` (§17) renders the fixture issue publicly and asserts that summaries and `why` lines are present while the Brief, every article body sentence and every comment string are absent from the HTML.
|
||||
|
||||
**2026-09-03:** the operator chose to publish summaries and why lines; bodies/Brief/World/comments remain private.
|
||||
|
||||
`comment_links`: one per `SocialRef` with an `item_url` — label `Hacker News` / `Lobsters` / `Reddit`, meta `"342 points · 210 comments"` — plus `Comments` for `entries.comments_url` when it is set and differs from the article URL. Links carry `rel="noopener"` and `target="_blank"`; article title links are plain `<a href>` (the aggregator's purpose is to send readers to the source, so no `nofollow`).
|
||||
|
||||
### 7.2 Pages
|
||||
|
||||
- `GET /` — the latest issue by date, rendered as `issue_public.html` (or `issue_full.html` when signed in, §8), with a strap line ("A personal morning paper, assembled daily from N feeds; the selection is the reader's, the words are the authors'") and a link to the archive. When no issue exists: an empty-state page.
|
||||
- `GET /issues` — every issue date, newest first, grouped by month: date, weekday, `issue_number`, article count. Anonymous and signed-in alike.
|
||||
- `GET /issues/{date}` — `issue_public.html`: masthead, dateline `Friday, September 3, 2026 · No. 20`, `stats_line` (articles · read time · sections), then sections in `Lineup.section_order` with entries: title → source link, `author · source (domain) · 8 min`, comment links. The World Briefing is **not** listed publicly (its entries are Wikipedia text). The lead story is marked with `.lead`.
|
||||
- `GET /feed.xml` — Atom 1.0, last 30 issues: `<id>tag:daily.hallada.net,2026:issue/{date}</id>` (host from `server.public_url`), `<title>The Daily EPUB — {date}</title>`, `<updated>` = `generated_at`, `<link rel="alternate" href="{public_url}/issues/{date}">`, `<content type="html">` = the public section list as escaped HTML (the same `PublicIssue`, rendered by `feed_entry.html`). `Content-Type: application/atom+xml; charset=utf-8`, `Cache-Control: public, max-age=300`.
|
||||
- `GET /robots.txt` — as in §3.
|
||||
|
||||
Public pages send `Cache-Control: public, max-age=300` only when no session cookie is present; with a cookie they are `private, no-store`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Full issue views for signed-in users
|
||||
|
||||
`web::issue::load(state, date) -> Result<Option<IssueView>>` (§5.2). `IssueView` carries the reconstructed `Issue`, downloads, snapshot provenance, and an optional trusted World Briefing HTML fragment recovered from a legacy EPUB. `Download { label, href, size_bytes }` lists the standard EPUB, the X4 EPUB and the XTC file **only when the file at `issues.*_path` (or `publish::issue_filename` in `epub_dir`) exists**; `href` is the existing `/files/epub/{name}` or `/files/xtc/{name}`.
|
||||
|
||||
- `GET /issues/{date}` (signed in) — `issue_full.html`: masthead and dateline; **The Brief** (`editorial.front_page_html`, already sanitized XHTML, rendered `|safe`); download buttons; the index exactly like the EPUB's In-this-issue page: per section, title (→ `/issues/{date}/articles/{id}`), `source · N min read`, summary, `Why it's here`, the rating widget for admins; then links to World Briefing and Behind the paper; the colophon facts (models, cost, counts) in a footer block.
|
||||
- `GET /issues/{date}/articles/{article_id}` — `article.html`: `article-header` (title linking to the source, byline, meta line `feed · 1,850 words · ~8 min`, `Why it's here`, social line via `chapters::social_line`, summary), the body (`ammonia::clean(articles.content_html)` — reuse the same ammonia configuration the EPUB uses; do **not** run `to_xhtml`; `<img>` tags keep their remote `src` and get `loading="lazy" referrerpolicy="no-referrer"`), the discussion (`comments::render_xhtml(discussion, title)` when `pick.discussion` is present; it is XHTML and renders fine as HTML), prev/next links in issue order, "Read online ↗", and the rating widget for admins. 404 when the article is not in that issue.
|
||||
- `GET /issues/{date}/world` — `world::render_xhtml` in the layout for snapshots, or trusted `OEBPS/world.xhtml` body content recovered from our own sanitized legacy EPUB; 404 when neither source has a briefing.
|
||||
- `GET /issues/{date}/behind` — the same lines as the EPUB chapter (`chapters::behind_*_line`, near misses linking to `/dashboard/articles/{id}` for admins), reconstructed from the stored report/run for legacy issues.
|
||||
|
||||
`/files/epub/{name}` and `/files/xtc/{name}`: `serve_file` accepts either a valid session (any role) **or** the existing Basic auth; when neither is present and Basic auth is configured, it challenges as today; when Basic auth is not configured it redirects HTML clients to `/login?next=` and returns 401 to others. OPDS clients are unaffected.
|
||||
|
||||
The HMAC confirmation page (`server::confirmation_page`) gains one line: `<a href="{public_url}/issues/{date}">Open this issue on the site</a>`.
|
||||
|
||||
### 8.1 Rating widget (`_rating_widget.html`, `web::rate`)
|
||||
|
||||
A `<form class="rating" method="post" action="/rate">` with hidden `article_id`, `issue_date`, `next`, and four buttons `label=loved|good|down|cleared` (the last styled as a link "clear"); the current verdict button is filled. An optional `note` text field appears on the dashboard variants (`/dashboard/articles/{id}`, `/dashboard/ratings`), not on issue pages. `POST /rate`:
|
||||
|
||||
1. Admin route layer (§6.3) and the origin check (§6.4) have already passed; read the viewer from `AuthSession`.
|
||||
2. Validate `article_id` exists; `issue_date` optional (issue pages pass it; the dashboard passes the latest `issue_articles` date for the article, if any, like the CLI does).
|
||||
3. Append `RatingEvent { kind: "explicit", source: "dashboard", label, value: vote.value(&cfg.curation.feedback) (0.0 for cleared), note, user_id, event_at: now }`.
|
||||
4. `Accept: application/json` → `200 {"article_id", "label", "event_id"}`; otherwise set a flash and `303 See Other` to `next` (validated same-site path).
|
||||
|
||||
---
|
||||
|
||||
## 9. Dashboard: overview, runs, articles
|
||||
|
||||
All dashboard queries live in `src/web/dashboard/*.rs` as runtime `sqlx` queries with manual row mapping. Page size 50 (articles) / 100 (candidates), `?page=N`, `_pagination.html`. Sort and filter parameters are query-string, validated against allow-lists (never interpolated into SQL).
|
||||
|
||||
### 9.1 Overview (`/dashboard`)
|
||||
|
||||
- **Last run** card: date, status badge, started/duration, the four-line block (`curation:`, `admission:`, `preference:`, `providers:`) built from `runs.report_json` (fall back to the counters when NULL), warnings count with a link to the run.
|
||||
- **Budget today** card: per provider (from `config.referenced_providers()` + voyage): spent today (`db::provider_spend_for_utc_day(now)` + the last run if it is today) vs `max_daily_usd`, as a bar.
|
||||
- **Sparklines** (server-rendered SVG, last 30 non-dry runs): cost per run, selected per run, generation seconds.
|
||||
- **Ratings this week**: counts by label, and **"Unrated picks"** — the last three issues' picks with no explicit event, title + feed + inline rating widget. (The operator forgets to rate; this is the nudge.)
|
||||
- **Jobs**: any `requested`/`running` job, and the last five finished.
|
||||
- **Config on disk**: the `Config::check_report` lines that start with `! ` (missing keys/files), if any.
|
||||
|
||||
### 9.2 Runs (`/dashboard/runs`, `/dashboard/runs/{id}`)
|
||||
|
||||
List: one row per `runs` row, newest first, 50 per page: id, date (→ issue page), status badge, started (config tz), duration, `considered → eligible → triaged → assessed → shortlisted → selected` from `report_json.counts` (or `candidates`/`selected` when NULL), cost per provider, total, dry-run marker, warnings count. Filter `?status=`.
|
||||
|
||||
Detail:
|
||||
|
||||
1. Header: date, status, started/finished/duration, link to the issue, link to the previous and next run of that date.
|
||||
2. **Funnel**: `SELECT stage, excluded_reason, COUNT(*) FROM candidate_runs WHERE run_id = ? GROUP BY 1, 2` rendered as bars in pipeline order (`excluded` → `eligible` → `triaged` → `admitted` → `assessed` → `shortlisted` → `selected`) with the reason breakdown under each.
|
||||
3. **Admission mix** (`report_json.counts.admitted_by`), **preference state** (`rated_with_embeddings`, `knn_gate`, `feed_gate`, `verdicts_in_prompt`), **timings** table (`report_json.timings`), **provider usage** (tokens in/cached/write/out, cost), **warnings** list, **per-feed counts** top 20.
|
||||
4. **Config diff**: flatten `runs.config_json` of this run and of the previous non-dry run into dotted keys; show keys whose values differ (`key · before → after`). This is how a tuning change is tied to its first paper.
|
||||
5. **Near misses**: `telemetry::near_misses(db, run_id, 10)`.
|
||||
6. **Candidates** table: every `candidate_runs` row joined to `articles`/`entries`/`article_assessments`: title (→ article detail), feed, words, stage badge, reason, `admitted_by`, utility, rank, cluster (`id · rank`), triage `interest`, deep `quality`/`fit`, exploration/auto-include flags, `editor_why`. Filters: `stage`, `reason`, `admitted_by` (prefix match on the JSON array's first element via `json_extract(admitted_by, '$[0]')`), `q` (title LIKE), `flag=exploration|auto`. Sort: utility desc (default), rank, triage, quality, fit, title. A `<details>` per row expands `_signals_table.html` (raw · norm · weight · present for each signal, top interests, neighbours, notes) from `signals_json`.
|
||||
|
||||
### 9.3 Articles (`/dashboard/articles`, `/dashboard/articles/{id}`)
|
||||
|
||||
List: `articles` LEFT JOIN `entries` (best entry) LEFT JOIN the article's **latest** `candidate_runs` row (`MAX(run_id)` subquery) LEFT JOIN triage/deep `article_assessments` LEFT JOIN the current rating (latest explicit event) LEFT JOIN the latest `issue_articles` row. Columns: first seen, title, feed, words, last stage/reason (with run date), utility, triage, quality/fit, rating badge, published date. Filters: `q` (title or canonical_url LIKE), `feed` (feed_id), `stage`, `reason`, `rated=any|loved|good|down|cleared|none`, `published=yes|no`, `from`/`to` (first_seen dates), `kind` (triage kind). Sort: first_seen desc (default), utility, quality, fit, triage, words, title. Add index `CREATE INDEX idx_candidate_runs_article_run ON candidate_runs(article_id, run_id DESC)` in the migration so the latest-row join stays fast.
|
||||
|
||||
Detail — everything the system knows, in this order:
|
||||
|
||||
1. **Article**: title → source, canonical URL, feed (+ category), author, published, first seen, words, excerpt-only, image count, extract method (from `sources_json`? no — from the best entry; show what `Article` carries), sources list (`SourceRef` kind + feed), social refs with links and scores, "in issues" (date · section · position · lead), current rating badge, the rating widget with note field, "Explain (text)" `<details>` containing `telemetry::render_explain` for the latest run row, verbatim in `<pre>`.
|
||||
2. **Assessments**: triage (interest, kind, why, model, prompt version, assessed_at) and deep (quality, fit, category, rationale, paywalled guess, facets as a table, model, assessed_at); a `provider_rejected` row is shown as such.
|
||||
3. **Run history**: every `candidate_runs` row for the article, newest first: run (→ run detail), date, stage, reason, admitted_by, utility, rank, cluster, editor why. Clicking a row expands its `_signals_table.html`.
|
||||
4. **Neighbours and interests** from the latest row's `signals_json`: top interests (z, cos) and nearest rated neighbours (label, cos, → their article page).
|
||||
5. **Embedding**: model, dimension, created_at, input hash — never the vector.
|
||||
6. **Rating events**: all events for this article (including superseded ones), with source, user, note, value.
|
||||
|
||||
---
|
||||
|
||||
## 10. Ratings page (`/dashboard/ratings`)
|
||||
|
||||
Two tabs (query `?tab=current|events`).
|
||||
|
||||
**Header** (from `PreferenceState::load(db, &config.curation.ranking, now).summary()` — reuse whatever the pipeline calls): `N rated articles with embeddings → neighbour signal at {knn_gate:.0%} (floor {knn_floor}, full {knn_full}) · {attributable_feed_ratings} attributable feed ratings → feed affinity at {feed_gate:.0%} · {verdicts_in_prompt} verdicts in the prompt · {n} in the weekly rebuild set`. When a rated article has no embedding, a notice: "M rated articles have no embedding and cannot act as neighbours — run the `features backfill --rated-only` job."
|
||||
|
||||
**Current** tab: `db::current_ratings_including_cleared(36_500)`, one row per article, newest first, filter by label/source/feed/q. Columns:
|
||||
|
||||
| Column | Definition |
|
||||
|---|---|
|
||||
| Verdict | badge; the rating widget inline (with note) to change it |
|
||||
| Article | title → article detail, feed, issue date if any |
|
||||
| When · by | `event_at` (config tz), `source`, username |
|
||||
| Note | `note` |
|
||||
| Age → decay | `age_days` and `decay = 0.5^(age/half_life)` (`signals::decay`) |
|
||||
| Neighbour weight | `value × decay` when the article has an embedding; "no embedding" otherwise. This is exactly the `w_i` of plan §9.2 |
|
||||
| Feed credit | each direct feed (`signals::direct_feeds`, else the best entry's feed) and `value / n` credited to it (plan §9.3) |
|
||||
| In prompt | ✓ when its rank among current non-cleared verdicts (newest first) is `< curation.feedback.verdicts_in_prompt` |
|
||||
| In rebuild | ✓ when that rank is `< profile::MAX_RATINGS_IN_REBUILD` |
|
||||
| Used last run | from the latest non-dry run: the number of candidates whose `signals_json.neighbours` include this article id (`WHERE signals_json LIKE '%"article_id":<id>,%'` then parsed to confirm), and how many of those were selected |
|
||||
|
||||
A "How ratings enter the algorithm" `<details>` block at the top explains the four paths in prose (prompt verdict block, weekly learned adjustments, neighbour signal with decay and gate, feed affinity with gate) with the current config values inlined, and links to the settings groups that tune them.
|
||||
|
||||
**Events** tab: every `rating_events` row, newest first, 100 per page, with `superseded` marked when a later explicit event exists for the same article; filters by label, source, user, date range. Nothing here is editable — history is append-only, and the "clear" verdict is the way to retract.
|
||||
|
||||
---
|
||||
|
||||
## 11. Profile page (`/dashboard/profile`)
|
||||
|
||||
- **profile.md editor**: a monospace `<textarea>` with the current file (`config.profile_path`), a Save button (an origin-checked POST, §6.4), and a live preview of what the loader parses: the sections that pass through verbatim and the `## Interests` lines it extracted (`profile::parse_profile_str`). Saving: validate the text parses (it always does; reject only empty or > 64 KB), insert the **previous** content into `profile_versions`, write atomically (`<path>.tmp` + rename, preserve mode), flash "Saved; the next run rebuilds the system prompt". The server unit needs the profile's directory writable (it is under `/var/lib/daily-epub`, the `StateDirectory`; §18 confirms).
|
||||
- **History**: `profile_versions` newest first with a diff-less preview (first 200 chars) and a "Restore" button (writes that content back, recording the current one as a new version).
|
||||
- **Standing interests**: the OPML interests (`profile::parse_interests`) grouped by `themes::group_into_themes` — read-only, with the file path and count; a note that the union with `## Interests` is what the prompt uses.
|
||||
- **System prompt**: `kv.taste_profile` (`TasteProfile { text, version, built_at, verdicts }`) in a collapsed `<pre>`, the learned adjustments block (`kv.taste_profile_learned`) shown separately with its age and whether a weekly rebuild is due (`profile::is_stale`), and a "Rebuild profile now" button that starts the `profile-rebuild` job (§14).
|
||||
|
||||
---
|
||||
|
||||
## 12. Stats page (`/dashboard/stats`)
|
||||
|
||||
Refactor `telemetry::stats(db, days, now) -> String` into `telemetry::stats_data(db, days, now) -> StatsData` plus `telemetry::render_stats_text(&StatsData) -> String`; the CLI output is byte-identical (a test pins it against the current fixture). `StatsData` carries: issues, articles published, ratings by label, ratings per issue, per-retriever up/down, exploration yield, mean issue size, cost per day per provider (as a date → provider → usd map), mean generation time, and the per-run series used by the sparklines.
|
||||
|
||||
The page: `?days=14|30|90`, the same figures as tables, three sparklines (cost per day stacked per provider as bars, selected per issue, ratings per week by label), and the retriever yield as a small table with the up/down ratio.
|
||||
|
||||
---
|
||||
|
||||
## 13. Settings (`/dashboard/settings`)
|
||||
|
||||
### 13.1 The schema is derived, not hand-listed
|
||||
|
||||
`settings::schema(config: &Config, file: Option<&toml_edit::DocumentMut>) -> Vec<SettingGroup>`:
|
||||
|
||||
1. `let defaults = toml::Value::try_from(Config::default())` and `let current = toml::Value::try_from(config)`; walk both in parallel, depth-first, producing one `SettingField` per leaf:
|
||||
|
||||
```rust
|
||||
pub struct SettingField {
|
||||
pub path: String, // "curation.ranking.deep_keep"
|
||||
pub group: String, // "curation.ranking" (the enclosing table)
|
||||
pub kind: FieldKind, // Bool | Integer | Float | Text | TextList | Enum(&'static [&'static str]) | Secret | Path
|
||||
pub current: String, // rendered value (TextList: one per line)
|
||||
pub default: String,
|
||||
pub source: Source, // Default | File | Env(String /* the var name */)
|
||||
pub help: Option<&'static str>,
|
||||
pub restart_required: bool, // server.bind, database_path
|
||||
}
|
||||
```
|
||||
|
||||
2. `source`: `Env` when `std::env::var_os(env_name(path))` is set, where `env_name` = `ENV_PREFIX + path.to_uppercase().replace('.', ENV_SPLIT)` (`DAILY_EPUB_SECRET` also counts for `server.hmac_secret`); else `File` when the key exists in the parsed document; else `Default`.
|
||||
3. `kind`: from the default's TOML type; `TextList` for arrays of strings; `Enum` from a static table keyed by path suffix — `providers.*.kind` → `openai|anthropic`, `providers.*.effort` → `low|medium|high|xhigh|max` (anthropic) or free text (openai), `editorial.summary_model` → `editor|bulk`, `xtc.format` → `xtc|xtch`, `llm.bulk`/`llm.editor` → the provider names (plus `""` for editor); `Secret` for paths ending in `api_key`, `hmac_secret`, `basic_auth_pass`; `Path` for `database_path`, `out_dir`, `profile_path`, `interests_opml`, `publish.*_dir`, `xtc.settings`.
|
||||
4. `help`: a static `SETTINGS_HELP: &[(&str, &str)]` seeded from the README's configuration table and the comments in `config.example.toml`. Every key in the shipped example must have an entry (test).
|
||||
5. Groups render in this order: top level, `llm`, `providers.<name>` (one card each), `voyage`, `curation`, `curation.feedback`, `curation.ranking`, `curation.ranking.quotas`, `curation.ranking.weights.preliminary`, `curation.ranking.weights.utility`, `curation.ranking.diversity`, `editorial`, `publish`, `xtc`, `server`, `miniflux`.
|
||||
|
||||
`Secret` fields render as "set (from `DAILY_EPUB_…`)" / "not set" with no input. `Env` fields render disabled with "locked by `DAILY_EPUB_…` in the env file". Everything else is an input of its kind, showing the default beside it and a "reset to default" affordance. Weights render with a note that they are renormalized.
|
||||
|
||||
### 13.2 Saving
|
||||
|
||||
`POST /dashboard/settings` (admin, origin-checked) carries every editable field. The handler:
|
||||
|
||||
1. `reload_if_changed`; load `config_path` into `toml_edit::DocumentMut` (a missing file starts from an empty document; `config_path == None` → 400 "no config file is configured; start the server with --config").
|
||||
2. For each posted field whose parsed value differs from the current **file** value (compare TOML values; a field equal to the default but absent from the file is still written when the user submitted a change to it — "explicit beats implicit"; a field unchanged from what the file holds is skipped): set it in the document at its path, creating intermediate tables as implicit tables; typed by `FieldKind` (integers → `i64`, floats → `f64` formatted with at most 6 decimals, bools, strings, `TextList` → a multi-line array with one item per line, trailing comma). Parsing failures (letters in a number, unknown enum value) collect into a field-error list and abort before writing.
|
||||
3. Render the document to a string, write to `<path>.tmp.<pid>`, and validate by `Config::load(Some(&tmp))` (the same loader the CLI uses, env included). On error: remove the temp file, re-render the form with the message. On success: copy the original file's permissions onto the temp file, `rename` it over the original, and swap `state.config`.
|
||||
4. Insert one `config_changes` row per changed key with the TOML literal before/after (`NULL` when absent/removed) and the viewer's id; flash "Saved N settings — they apply to the next run" (plus "restart the server for: …" when a `restart_required` key changed).
|
||||
|
||||
Because `generate` runs as its own process and reads `config.toml` at start, a saved setting reaches the next paper with no further plumbing. `runs.config_json` continues to snapshot what each run actually used, and the run page's config diff (§9.2) shows it.
|
||||
|
||||
### 13.3 Providers
|
||||
|
||||
The `providers.*` cards each have a "Remove" button (`POST /dashboard/settings/providers` with `action=remove&name=`; refused when an `[llm]` role names it) and the page has an "Add provider" form (`action=add&name=&kind=`; name validated as `[a-z0-9_]+`, must not exist). Adding inserts a `[providers.<name>]` table whose keys are `ProviderConfig::default()` with the chosen `kind` and a `base_url`/`model` placeholder, via `toml::to_string` → parsed into `toml_edit::Item`. Both go through the same validate-and-rename path and log `config_changes` rows (`providers.<name>` with the whole table as the literal).
|
||||
|
||||
### 13.4 History
|
||||
|
||||
`GET /dashboard/settings/history`: `config_changes` newest first, 100 per page — when, who, key, before → after. Linked from the settings page and from the run page's config diff.
|
||||
|
||||
---
|
||||
|
||||
## 14. Jobs
|
||||
|
||||
### 14.1 Catalogue (`src/jobs.rs`)
|
||||
|
||||
```rust
|
||||
pub enum Job {
|
||||
Generate { date: Option<Date> }, // "generate" | "generate-YYYY-MM-DD"
|
||||
DryRun, // "dry-run" → generate --dry-run
|
||||
ProfileRebuild, // "profile-rebuild" → profile rebuild
|
||||
FeaturesBackfill, // "features-backfill" → features backfill --days 30 --yes
|
||||
BackfillSocial, // "backfill-social" → backfill-social --days 7
|
||||
FeaturesPrune, // "features-prune" → features prune
|
||||
}
|
||||
impl Job {
|
||||
pub fn parse(name: &str) -> Option<Job>; // names match ^[a-z0-9-]+$ and the forms above
|
||||
pub fn name(&self) -> String;
|
||||
pub fn unit(&self) -> String; // format!("daily-epub-job@{}.service", name)
|
||||
pub fn description(&self) -> &'static str; // shown on the Jobs page
|
||||
pub fn takes_lock(&self) -> Option<&'static str>; // same names lock_holder uses
|
||||
pub fn dangerous(&self) -> bool; // Generate (republishes an issue) → confirm dialog
|
||||
}
|
||||
```
|
||||
|
||||
### 14.2 `daily-epub job run <name>`
|
||||
|
||||
A new subcommand in `main.rs`. It:
|
||||
|
||||
1. Parses the name (unknown → exit 2 with the catalogue).
|
||||
2. Takes the run lock when `takes_lock()` says so (via `lock_holder`, so "generate is already running" behaves as today).
|
||||
3. Finds the newest `jobs` row with this `name` and `status = 'requested'`; if none, inserts one (`requested_by NULL` — the operator started the unit by hand). Sets `running` + `started_at`.
|
||||
4. Runs the mapped command **in-process** by calling the same functions `main` uses (`pipeline::generate`, `cmd_profile_rebuild`, `cmd_features`, `cmd_backfill_social`, `telemetry::prune`), with the same logging.
|
||||
5. On success sets `ok`, `finished_at`, `run_id` (for generate) and a one-line `message` (the `curation:` line for generate; counts otherwise). On error sets `failed` and `message = error`, exits non-zero so systemd records `Result=exit-code`.
|
||||
|
||||
### 14.3 Unit, polkit, groups
|
||||
|
||||
`systemd/daily-epub-job@.service` — a copy of `daily-epub-generate.service` with `Description=The Daily EPUB job %i` and `ExecStart=/usr/local/bin/daily-epub --config /etc/daily-epub/config.toml job run %i`. Same hardening, same `ReadWritePaths`, `TimeoutStartSec=45min`. The timer keeps driving `daily-epub-generate.service` unchanged.
|
||||
|
||||
`systemd/50-daily-epub.rules` (installed to `/etc/polkit-1/rules.d/`):
|
||||
|
||||
```js
|
||||
polkit.addRule(function (action, subject) {
|
||||
if (action.id == "org.freedesktop.systemd1.manage-units" &&
|
||||
subject.user == "daily-epub" &&
|
||||
action.lookup("verb") == "start" &&
|
||||
/^daily-epub-job@[a-z0-9-]+\.service$/.test(action.lookup("unit"))) {
|
||||
return polkit.Result.YES;
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
`systemd/daily-epub.service` gains `SupplementaryGroups=systemd-journal` (journal reads) and `ReadWritePaths=/etc/daily-epub` (settings writes). Nothing else in the hardening changes; `systemctl` talks to systemd over the system bus (`AF_UNIX`, already allowed).
|
||||
|
||||
### 14.4 Runner and page
|
||||
|
||||
```rust
|
||||
#[async_trait] pub trait JobRunner: Send + Sync {
|
||||
async fn start(&self, unit: &str) -> Result<(), String>; // systemctl start --no-block <unit>
|
||||
async fn status(&self, unit: &str) -> Result<UnitStatus, String>; // systemctl show -p ActiveState,SubState,Result,ExecMainStatus,ExecMainStartTimestamp,ExecMainExitTimestamp
|
||||
async fn log(&self, unit: &str, lines: usize) -> Result<String, String>; // journalctl -u <unit> -n <lines> --no-pager -o short-iso
|
||||
}
|
||||
```
|
||||
|
||||
`SystemdRunner` spawns the commands with `tokio::process::Command` (10 s timeout, stderr captured into the error). `MockRunner` records calls and returns scripted statuses for tests. `server.jobs_enabled = false` swaps in a `DisabledRunner` whose page says so.
|
||||
|
||||
`GET /dashboard/jobs`: the catalogue as cards (name, description, Start button with `data-confirm` for `dangerous()`; the `generate-<date>` card has a date input), then the jobs table newest first (name, requested by/at, started, finished, duration, status badge, message, run → run detail).
|
||||
|
||||
`POST /dashboard/jobs/{name}` (admin, origin-checked): parse; refuse when a job with the same unit is `requested`/`running` (409 with a flash); insert `requested`; `runner.start(unit)`; on error mark the row `failed` with the message; redirect to `/dashboard/jobs/{id}`.
|
||||
|
||||
`GET /dashboard/jobs/{id}`: the row, the live `UnitStatus`, and the journal tail (`server.journal_lines`, 300) in a `<pre>`. If the row is still `requested` but the unit is `inactive` with `Result=exit-code`/`failed` 30 s after `requested_at`, mark it `failed` ("unit exited before the job started; see the log"). `app.js` refreshes while `requested`/`running`.
|
||||
|
||||
---
|
||||
|
||||
## 15. Configuration additions
|
||||
|
||||
```toml
|
||||
[server]
|
||||
bind = "127.0.0.1:3499"
|
||||
public_url = "https://daily.hallada.net"
|
||||
session_days = 30 # web login lifetime, sliding
|
||||
login_attempts = 10 # failed logins per IP per window before a 429
|
||||
login_window_minutes = 15
|
||||
jobs_enabled = true # the Jobs page starts daily-epub-job@<name>.service via systemctl
|
||||
journal_lines = 300 # log tail shown on a job page
|
||||
# hmac_secret / basic_auth_* unchanged
|
||||
```
|
||||
|
||||
Validation: `session_days ≥ 1`, `login_attempts ≥ 1`, `login_window_minutes ≥ 1`, `journal_lines ∈ 10..=5000`. Update `config.example.toml`, `Config::default()` (the key-for-key test enforces both), the README table, and `SETTINGS_HELP`. These keys appear on the settings page automatically.
|
||||
|
||||
New dependencies: `axum-login = { git = "https://github.com/maxcountryman/axum-login.git", rev = "151c72d7a1b4646830f86b4332e6bd6e34d719a7" }` (brings `tower-sessions` 0.15 and `tower-cookies`; use the re-exports, do not add `tower-sessions` separately — a second copy would be a different version; the server builds from source with GitHub access, as it does today), `password-auth`, `tower_governor` with the `axum` feature, `time` (the type tower-sessions' `Expiry` and `Record` use; no `formatting` feature needed since the store keeps unix seconds), `async-trait` (only for our `SessionStore` impl — tower-sessions' trait still uses it; axum-login's own traits no longer do), `rpassword`, `toml_edit`, `toml` (already transitive; promote to direct for `toml::Value`), dev: `tower` with `util` (router `oneshot` tests). Do **not** add `argon2` or `tower-sessions-sqlx-store` (§2). Confirm the sqlx SQLite build has `json_extract` (it does; bundled SQLite ships JSON1).
|
||||
|
||||
---
|
||||
|
||||
## 16. Security checklist
|
||||
|
||||
- Password hashes: `password-auth` argon2id PHC strings; `User`'s `Debug` redacts the hash; never logged. Session ids are tower-sessions' random ids stored server-side; the cookie is `HttpOnly`, `Secure` on https, `SameSite=Lax`; login cycles the id; a password change or `disable` invalidates the user's sessions (`session_auth_hash` + `delete_for_user`).
|
||||
- CSRF: `SameSite=Lax` plus the fetch-metadata/origin middleware on every POST (§6.4). Redirect targets validated to same-site paths.
|
||||
- Login: `tower_governor` per-IP limit on `POST /login` (§6.5); equal-timing dummy verification for unknown users; no username enumeration in messages ("invalid username or password").
|
||||
- Authorization: `login_required!`/`permission_required!` route layers on whole sub-routers (§6.3), so a forgotten check in one handler still redirects or 403s; `POST /rate` sits under the admin layer.
|
||||
- Headers on every app response (via a `tower-http` `SetResponseHeader` layer or a small middleware): `Content-Security-Policy: default-src 'self'; img-src * data:; style-src 'self'; script-src 'self'; frame-ancestors 'none'; form-action 'self'`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: strict-origin-when-cross-origin`. (`img-src *` because article pages show remote images for signed-in users.) nginx keeps adding its own.
|
||||
- All user text is escaped by askama; the only `|safe` inputs are `front_page_html` (already sanitized by the pipeline), `ammonia::clean` output, `comments::render_xhtml`, `world::render_xhtml` (built from sanitized comment/portal HTML), a recovered World Briefing body read from our own sanitized EPUB XHTML, and the server-rendered SVG sparklines.
|
||||
- Settings: secrets never rendered, never written; validation through `Config::load` before rename; permissions preserved; every change attributed. `Path` fields are written as given — the operator is the admin, and the config validates paths on load.
|
||||
- Jobs: unit names come only from the fixed catalogue regex; the polkit rule allows `start` only, on that regex only, for that user only.
|
||||
- Files: `safe_join` unchanged. `robots.txt` disallows private paths. Public caching only without a cookie.
|
||||
- Logs: never log passwords, tokens, cookies, or `Authorization`. The `TraceLayer` stays; add a request-id.
|
||||
|
||||
---
|
||||
|
||||
## 17. Tests
|
||||
|
||||
No test touches the network or systemd. Router-level tests use `tower::ServiceExt::oneshot` against `server::router(state)` with a temp database (`Db::open_and_migrate` on a `tempfile` path), a `MockRunner`, and `fixtures::issue()` written through `record_issue`'s helpers.
|
||||
|
||||
- **Migration**: temp DB through `0001`…`0004`; the new tables exist; `rating_events.user_id` present; old rows keep `user_id NULL`.
|
||||
- **Users/passwords**: hash round-trips through `password-auth`; wrong password fails; malformed hash fails safely; `users add` rejects short passwords and duplicate names (case-insensitive); `role`, `disable`, `logout`, `passwd` behave and the last three delete sessions.
|
||||
- **Session store**: `save`/`load`/`delete` round-trip a `Record`; `load` returns `None` past `expiry`; `user_id` is denormalized from the axum-login key and `NULL` otherwise; `delete_expired` and `delete_for_user` count correctly; a store error surfaces as `session_store::Error::Backend`.
|
||||
- **Login flow (router oneshot)**: `POST /login` sets `daily_session` with `HttpOnly`, `SameSite=Lax`, and `Secure` iff the public URL is https; the cookie then reaches `/account`; a disabled user's cookie does not; a password change logs the other session out; logout clears; anonymous `GET /` sets no cookie at all; `next` is honoured only for same-site paths.
|
||||
- **Guards**: anonymous `/dashboard` → 302 `/login?next=/dashboard`; signed-in `user` role → 403 site page; admin → 200; `POST /rate` follows the same three outcomes.
|
||||
- **Origin check**: POST with `Sec-Fetch-Site: cross-site` → 403; absent fetch metadata and a foreign `Origin` → 403; same-origin passes; the HMAC rating route (GET) is untouched.
|
||||
- **Throttle**: with a test governor config of burst 3, the fourth `POST /login` from one address → 429; another address still passes.
|
||||
- **Public rendering**: `public_issue_shows_summaries_and_why_but_no_bodies` (§7.1); comment links built from social refs and `comments_url`; the Brief, article bodies, comments and World Briefing are absent publicly; the feed validates as XML, has one entry per issue, and its content is the public list with summaries and why lines; `robots.txt` content; `Cache-Control` public without a cookie, `no-store` with one.
|
||||
- **Full rendering**: signed-in `/issues/{date}` contains the Brief, summaries and `why`; article page contains the body and the discussion; the fallback loader (no `issue_json`) reconstructs colophon/Behind facts and recovers the World chapter from an available EPUB while discussion links remain unavailable; downloads listed only for files that exist; `/files/epub` accepts a session, still challenges Basic auth when configured and no session.
|
||||
- **issue_json**: `record_issue` writes it with empty bodies; the loader rehydrates bodies; `runs.report_json` and `issues.report_json` written; `/issues.json` returns real reports.
|
||||
- **Rating**: `POST /rate` as admin appends a `dashboard` event with `user_id`; as user → 403; anonymous → redirect/401; `cleared` writes value 0; JSON and form variants; `next` validated.
|
||||
- **Dashboard queries**: funnel counts match a seeded `candidate_runs` set; candidate filters and sorts are allow-listed (an unknown sort falls back, never errors); config diff finds changed dotted keys and ignores unchanged; articles list filters; article detail shows assessments, run history and rating events.
|
||||
- **Ratings contributions**: decay/weight/feed-credit/in-prompt/in-rebuild computed against hand-checked values; "used last run" counts neighbours in `signals_json`.
|
||||
- **Profile**: save writes the file and a `profile_versions` row; restore swaps; oversized rejected; the parsed preview matches `parse_profile_str`.
|
||||
- **Settings schema**: every leaf of `Config::default()` appears exactly once; every shipped key has help text; secrets are `Secret` and never carry a value; env detection uses the derived name; enum options match `validate()`'s accepted values (a round-trip test over each option).
|
||||
- **Settings writer**: starting from `config.example.toml`, changing three keys preserves every comment and the original key order (string compare of untouched lines); a new key in an absent table creates the table; `TextList` writes a multi-line array; an invalid value (e.g. `deep_keep < shortlist_keep`) is rejected by `Config::load` and the file is untouched; permissions preserved; `config_changes` rows written; provider add/remove; removing a referenced provider refused.
|
||||
- **Stats refactor**: `render_stats_text(stats_data(..))` equals the previous `stats(..)` output for a seeded DB.
|
||||
- **Jobs**: `Job::parse` accepts the catalogue and dated form, rejects `../x`, uppercase, and unknown names; `POST /dashboard/jobs/{name}` inserts a row and calls `MockRunner::start` with the right unit; a running duplicate is refused; a failed start marks the row; `job run` (in-process, `--skip-llm`-style config, mocked providers) flips `requested → running → ok` and sets `run_id`; the polkit rule file is present in the repo and matches the unit regex (string test).
|
||||
- **Binary** (`tests/m7_server.rs` style): the shipped binary serves `/`, `/issues`, `/feed.xml`, `/login` from env-only config; `/dashboard` redirects to `/login`; `users add` then a real login over TCP works.
|
||||
|
||||
---
|
||||
|
||||
## 18. Deployment — `docs/runbooks/web-dashboard-rollout.md`
|
||||
|
||||
Write this runbook as part of step 7; its content:
|
||||
|
||||
1. Build and install the binary; `daily-epub --help` shows `users`, `job`.
|
||||
2. `sudo install -m0644 systemd/daily-epub-job@.service /etc/systemd/system/`; `sudo install -m0644 systemd/50-daily-epub.rules /etc/polkit-1/rules.d/`; update `/etc/systemd/system/daily-epub.service` with `SupplementaryGroups=systemd-journal` and `ReadWritePaths=… /etc/daily-epub`; `daemon-reload`.
|
||||
3. Confirm `/etc/daily-epub` and `config.toml` are owned by `daily-epub` (settings writes rename into that directory) and that `profile.md` lives under `/var/lib/daily-epub` (writable `StateDirectory`).
|
||||
4. Add the new `[server]` keys to `config.toml` if overriding defaults (optional).
|
||||
5. Restart `daily-epub.service`; the migration `0004` applies on start. Verify `curl -s https://daily.hallada.net/` shows the latest issue and `/feed.xml` parses.
|
||||
6. `sudo -u daily-epub daily-epub --config /etc/daily-epub/config.toml users add tyler --admin` (hidden prompt). Log in, open `/dashboard`.
|
||||
7. Smoke test jobs: start `features-prune` from the Jobs page; watch the log tail; `systemctl status daily-epub-job@features-prune.service`. If polkit refuses, `journalctl -u polkit` shows the denied action.
|
||||
8. Smoke test settings: change `curation.ranking.utility_protected` by one, confirm `config.toml` kept its comments, read `/dashboard/settings/history`, change it back.
|
||||
9. Optional: raise `curation.ranking.telemetry_retention_days` for longer article history.
|
||||
|
||||
---
|
||||
|
||||
## 19. Implementation sequence
|
||||
|
||||
Each step is a shippable commit or small series; `cargo fmt --check`, `cargo clippy --all-targets`, `cargo test` green at each. Do not combine steps.
|
||||
|
||||
1. **Foundation.** Migration `0004`, `RatingEvent.user_id`, `runs.report_json` + `issues.report_json` writes, `issues.issue_json` write and loader, `GenerateOutcome.run_id`. `web/` skeleton: `WebState`, `AppState.config()` refactor (config path passed into `serve`), layout, error pages, static assets with ETag, security headers layer. `SqliteSessionStore`, `Backend`, the axum-login/tower-sessions/tower_governor layers, CLI `users …`, `/login`, `/logout`, `/account`, the origin-check middleware. Public site: `/`, `/issues`, `/issues/{date}` (public branch), `/feed.xml`, `/robots.txt`, `PublicIssue` with its no-leak test. `/files/*` accepts sessions. README route table updated.
|
||||
2. **Full issues.** `issue_full.html`, article/world/behind pages, downloads, prev/next, the rating widget and `POST /rate`, the HMAC confirmation link. `app.js` rating enhancement.
|
||||
3. **Dashboard reads.** Overview, runs list/detail (funnel, admission, timings, providers, warnings, config diff, near misses, candidate table with filters and the signals partial), articles list/detail (with the run history and the text explain). Index `idx_candidate_runs_article_run`.
|
||||
4. **Ratings and profile.** Ratings page with contributions and the events tab; profile editor, versions, prompt view. `MAX_RATINGS_IN_REBUILD` made `pub`.
|
||||
5. **Settings.** Schema derivation, help table, form rendering, `toml_edit` writer with validate-and-rename, env locks, providers add/remove, history, config reload-on-mtime. Unit file `ReadWritePaths` change in `systemd/`.
|
||||
6. **Jobs and stats.** `jobs.rs` catalogue, `job run` subcommand, `JobRunner` + `SystemdRunner`/`MockRunner`, unit template, polkit rule, `SupplementaryGroups`, the Jobs pages; the `stats_data` refactor and the stats page; overview sparklines.
|
||||
7. **Docs and polish.** README (site, roles, users CLI, jobs, settings), `config.example.toml`, `docs/plans/2026-08-15-implementation-notes.md` (new verified facts with dates: polkit, unit changes, toml_edit, axum-login/tower-sessions, password-auth, tower_governor), the rollout runbook (§18), users page, `robots.txt`/feed links in the layout, a final pass over dark mode and narrow screens.
|
||||
|
||||
---
|
||||
|
||||
## 20. Acceptance criteria
|
||||
|
||||
1. Anonymous `GET /issues/{date}` shows every pick's title (linking to the source), author, source, comment links, section and reading time — and contains **no** Brief, summary, `why`, body, comment text or World Briefing; a test enforces it. `/feed.xml` is a valid Atom feed with the same content.
|
||||
2. A signed-in user sees the complete issue in HTML — Brief, summaries, `why` lines, article bodies, discussions, World Briefing, Behind the paper — and can download the EPUB, X4 and XTC files while they exist. Issues from before `issue_json` still render (without world/discussion).
|
||||
3. Only admins can rate; a dashboard rating appends a `rating_events` row with `source = 'dashboard'` and the admin's `user_id`; the EPUB's HMAC links keep working unchanged.
|
||||
4. Every article the pipeline ever recorded in `candidate_runs` is reachable from `/dashboard/articles` with its stage, reason, signals, assessments, facets, utility, cluster, admission path, editor `why`, rating history and issue appearances — the same facts `explain` prints, plus history across runs.
|
||||
5. Every run is listed with its funnel, admission mix, timings, provider costs, warnings, near misses, and the config keys that changed since the previous run.
|
||||
6. The ratings page shows every current verdict with its decayed neighbour weight, feed credit, prompt/rebuild membership and last-run neighbour usage, and lets the admin change, clear or annotate it.
|
||||
7. The settings page lists every key in `config.toml` (derived from `Config`, so a new key appears without UI work); saving writes the file with comments preserved, validates through `Config::load`, records the change, and a hand edit on disk shows up on the next page view. Secrets are never displayed; env-overridden keys are locked.
|
||||
8. `profile.md` is editable with version history; the prompt and learned adjustments are viewable.
|
||||
9. The Jobs page starts `daily-epub-job@<name>.service` through systemd + polkit, shows status and the journal tail, and links a generate job to its run; the server never runs the pipeline in-process.
|
||||
10. Sessions come from axum-login/tower-sessions with password-auth hashes, `SameSite=Lax` cookies plus an origin check on every POST, a tower_governor login throttle, and a CSP; no password, token or API key is ever logged, rendered or stored in plain form.
|
||||
11. Existing routes (`/r`, `/opds`, `/files`, `/healthz`, `/issues.json`) behave as before, and `/issues.json` now carries real reports.
|
||||
12. `cargo test` covers all of §17 without network or systemd.
|
||||
|
||||
---
|
||||
|
||||
## 21. Deferred, with triggers
|
||||
|
||||
| Item | Trigger |
|
||||
|---|---|
|
||||
| Per-user personalization (per-user `rating_events`, profiles, issues) | A second real reader. `user_id` on `rating_events` is the hook; nothing else is prepared on purpose. |
|
||||
| Editing `data/scour-interests.opml` from the UI | The operator asks; `## Interests` in `profile.md` already covers additions. |
|
||||
| Search over article bodies (FTS5) | Title/URL `LIKE` feels slow or insufficient. |
|
||||
| Live log streaming (SSE) on the job page | The 5-second refresh feels slow. |
|
||||
| Passkeys / WebAuthn | A second admin or a phishing concern. |
|
||||
| Backfilling `issue_json` for old issues from the EPUB files on disk | Only the last `retention_days` of EPUBs exist; the fallback renderer covers the rest. |
|
||||
| Charts beyond sparklines | `stats` grows a question the tables cannot answer. |
|
||||
|
||||
**2026-09-03:** the operator chose to publish summaries and why lines; bodies/Brief/World/comments remain private. Public per-article why lines are therefore no longer deferred.
|
||||
@@ -0,0 +1,106 @@
|
||||
# "Read in BookOrbit" link on the issue page
|
||||
|
||||
**Date:** 2026-09-05
|
||||
**Repository:** `thallada/the-daily-epub`
|
||||
**Status:** small implementation plan, ready to execute
|
||||
**Builds on:** `docs/plans/2026-09-03-web-dashboard.md` (the signed-in issue page this adds a button to)
|
||||
|
||||
Written for a fresh implementation agent. Every fact about BookOrbit below was verified on 2026-09-05 against its source (`github.com/bookorbit/bookorbit`, `HEAD`) and against the live instance on this host. Do not re-derive them; do re-check anything marked *assumption*.
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
On the signed-in issue page (`src/web/templates/issue_full.html`), next to the existing **Download EPUB / X4 EPUB / XTC** buttons, add a **Read in BookOrbit** button that opens the *normal* (Standard) edition of that issue in BookOrbit's in-browser EPUB reader at `https://bookorbit.hallada.net`. The operator reads on desktop through this link instead of downloading.
|
||||
|
||||
Out of scope: the X4 edition, the public (anonymous) issue page, the OPDS feed, the EPUB itself.
|
||||
|
||||
## 2. Verified facts
|
||||
|
||||
### About this repo
|
||||
|
||||
- The pipeline does **not** upload to BookOrbit. `publish::publish_issue` atomically copies both EPUB editions into `publish.epub_dir` (`/home/thallada/bookorbit/books/daily-epub` on the host); BookOrbit's *watched folder* scanner picks them up later. Nothing in the run ever learns a BookOrbit id.
|
||||
- BookOrbit runs on the same host as Docker (`~/bookorbit/docker-compose.yml`): app container `bookorbit-app` published on `127.0.0.1:3498`, Postgres in `bookorbit-db` with **no published port**, and the `daily-epub` service user has no Docker socket access. Reading BookOrbit's database directly is therefore not an option.
|
||||
- Issue titles: `Issue::title()` is `The Daily EPUB — <date>` (em dash), and `title_for(Edition::Standard)` is that exact string while `title_for(Edition::X4)` appends ` (X4)` (`src/types.rs` ~531–544). Filenames use a hyphen instead: `publish::issue_filename(date, edition, "epub")` → `The Daily EPUB - <date>.epub` / `... (X4).epub`.
|
||||
- The signed-in issue view is built in `src/web/issue.rs`: `IssueView` (line ~33) carries `downloads: Vec<Download>`, assembled around line 258 from the `issues` row (`row.epub_path`, `row.x4_path`, `row.xtc_path`). The template renders them at `src/web/templates/issue_full.html:6` as `<a class="btn" href="{{ download.href }}">Download …</a>`.
|
||||
- Signed-in issue routes live in the `full_issues` router in `src/web/mod.rs` (~line 659: `/issues/{date}/articles/{article_id}`, `/issues/{date}/world`, `/issues/{date}/behind`, behind `login_required!` and `map_forbidden`); public routes (`/issues`, `/issues/{date}`) are on the root router ~line 682. Put the new route in **`full_issues`**.
|
||||
- `issues` table (`migrations/0001_init.sql:70`): `date TEXT PK, issue_number, generated_at, epub_path, x4_path, xtc_path, front_page_html, report_json` (+ `issue_json` added later). Migrations run via `sqlx::migrate!("./migrations")` (`src/db.rs:24`); latest file is `0004_web.sql`, so the new one is `0005_bookorbit.sql`. `Db::issue_by_date(date) -> Option<IssueRow>` is at `src/db.rs:537`; `upsert_issue` at `:496` (do not touch it, ids are resolved lazily, see §3).
|
||||
- Config: sections are `#[serde(deny_unknown_fields, default)]` structs hung off `Config` (`src/config.rs:82–85`, e.g. `pub server: ServerConfig`), with `Default` impls. Env overrides are `DAILY_EPUB_<SECTION>__<KEY>`. `config.example.toml` documents every key; the README has a configuration table (~line 350).
|
||||
- The settings dashboard (`src/web/dashboard/settings.rs`) has a **hardcoded section registry** (`const` list around line 248–259 ending `"publish", "xtc", "server", "miniflux"`), a secret-field list (`("server.basic_auth_pass", FieldKind::Secret)` ~line 219, plus `SECRET_SUFFIXES` at ~223), per-key help text (~line 360), and a test asserting the section list (~line 1683). A new section must be added to all of these or the settings page/test breaks.
|
||||
- Shared HTTP client: `crate::http::build_client(timeout)` (`src/http.rs:14`), rustls + gzip, no cookies. reqwest 0.13 with the `query` and `json` features is already a dependency; **no XML parser is in the tree** — the OPDS response is tiny and regular, so either add `quick-xml` or match with a small hand parser (see §4).
|
||||
|
||||
### About BookOrbit
|
||||
|
||||
- Web reader route (client router, `client/src/router/index.ts`): **`/read/:bookId/:fileId`**. Both ids are required numeric integers. Book detail page is `/book/:bookId`.
|
||||
- An unauthenticated visit to `/read/...` serves the SPA shell (200) and the client auth guard redirects to `/login?redirect=<fullPath>`, returning to the reader after sign-in. So the link can point straight at the reader.
|
||||
- OPDS (server `server/src/modules/opds/opds.service.ts`, `BASE = '/api/v1/opds'`):
|
||||
- Auth is **HTTP Basic** against a dedicated *OPDS user* created in BookOrbit's settings (separate static credentials, not the web login). Verified live: `GET http://127.0.0.1:3498/api/v1/opds/catalog?q=Daily` → `401`, `www-authenticate: Basic realm="bookorbit OPDS"`.
|
||||
- Search: `GET /api/v1/opds/catalog?q=<terms>` (Atom acquisition feed; server-side `ILIKE` on title, accent-insensitive). Paged with `?page=N`.
|
||||
- Each `<entry>` contains `<title>…</title>`, `<id>urn:bookorbit:book:<bookId></id>`, and one acquisition link per file:
|
||||
`<link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/<bookId>/download?fileId=<fileId>" type="application/epub+zip" title="EPUB"/>`.
|
||||
**That href carries both numbers the reader URL needs.** Each daily issue edition is its own book with a single EPUB file.
|
||||
- The JSON API (`/api/v1/books/search?q=`, `/api/v1/books/:id` with `files[].id`) exists too but needs a JWT from `/auth/login` (5/min throttle, refresh cookie). Not worth it; OPDS is enough.
|
||||
- Book titles come from the EPUB `dc:title`, so the Standard edition is titled `The Daily EPUB — 2026-09-05` and the X4 one `The Daily EPUB — 2026-09-05 (X4)`. *Assumption:* BookOrbit's metadata fetcher has not renamed them; the matcher in §4 tolerates a fallback to the filename form.
|
||||
|
||||
## 3. Design decisions (settled)
|
||||
|
||||
| Topic | Decision | Why |
|
||||
|---|---|---|
|
||||
| Where ids are resolved | **Lazily, on click**, by a new redirect route `GET /issues/{date}/read` in the signed-in router. | BookOrbit indexes the folder some time after the run ends, so the id does not exist at publish time. Rendering the issue page must stay free of cross-service calls (see perf work in `main` 17a8804 and earlier). |
|
||||
| Caching | Two nullable columns on `issues`: `bookorbit_book_id INTEGER`, `bookorbit_file_id INTEGER`. Filled on first successful lookup; later clicks redirect from the cache with no network call. | One OPDS call per issue, ever. |
|
||||
| Stale cache | If BookOrbit answers the *reader page*, we can't tell from a redirect. Instead: `GET /issues/{date}/read?refresh=1` clears the cached ids and re-resolves. Expose it as a tiny "wrong book?" link only if trivial; otherwise document the query param in the README. | Books can be deleted/re-added in BookOrbit, changing ids. |
|
||||
| Not indexed yet | Route returns **503** with the existing error template (`src/web/templates/error.html`), message "BookOrbit has not indexed this issue yet. Try again in a minute." Nothing cached. | Honest, cheap, retry-friendly. |
|
||||
| Which edition | Standard only. The entry whose `<title>` equals `issue.title_for(Edition::Standard)` **exactly**; entries ending in ` (X4)` are never chosen. | The whole point is desktop reading. |
|
||||
| Auth to BookOrbit | HTTP Basic with an OPDS user's credentials from config. | Static creds, no token lifecycle; same mechanism KOReader already uses. |
|
||||
| Feature flag | Whole feature off unless `[bookorbit] enabled = true` **and** the three credentials/URL keys are set. Button and route are absent when off. | Repo stays usable without BookOrbit, as today. |
|
||||
| Link target | `<public_url>/read/<bookId>/<fileId>`, `public_url` from config (`https://bookorbit.hallada.net`), opened with `target="_blank" rel="noopener"`. | BookOrbit's own login redirect handles the signed-out case. |
|
||||
|
||||
## 4. Implementation steps
|
||||
|
||||
1. **Config** (`src/config.rs`, `config.example.toml`, README table):
|
||||
```toml
|
||||
[bookorbit]
|
||||
enabled = false
|
||||
public_url = "https://bookorbit.hallada.net" # what the browser opens
|
||||
api_url = "http://127.0.0.1:3498" # where the server talks OPDS; same host
|
||||
opds_user = "" # an OPDS user from BookOrbit → Settings → OPDS
|
||||
# opds_pass: environment only (DAILY_EPUB_BOOKORBIT__OPDS_PASS)
|
||||
```
|
||||
`BookorbitConfig { enabled: bool, public_url: String, api_url: String, opds_user: Option<String>, opds_pass: Option<String> }` with `deny_unknown_fields, default`. Add `pub bookorbit: BookorbitConfig` to `Config` and its `Default`. Add a `fn is_active(&self) -> bool` (enabled + user + pass non-empty). Validate URLs have no trailing slash (or trim).
|
||||
|
||||
2. **Settings dashboard** (`src/web/dashboard/settings.rs`): add `"bookorbit"` to the section list, `("bookorbit.opds_pass", FieldKind::Secret)` (and `"opds_pass"` to `SECRET_SUFFIXES`), help text for the five keys, update the section-list test. Copy the pattern used for `server.basic_auth_pass`.
|
||||
|
||||
3. **Migration** `migrations/0005_bookorbit.sql`:
|
||||
```sql
|
||||
ALTER TABLE issues ADD COLUMN bookorbit_book_id INTEGER;
|
||||
ALTER TABLE issues ADD COLUMN bookorbit_file_id INTEGER;
|
||||
```
|
||||
Extend `IssueRow` and `issue_by_date`'s `SELECT` with the two columns; add `Db::set_bookorbit_ids(date, Option<(i64, i64)>)` (`None` clears).
|
||||
|
||||
4. **OPDS client** — new module `src/bookorbit.rs` (~100 lines):
|
||||
- `pub async fn find_issue(client: &reqwest::Client, cfg: &BookorbitConfig, issue_title: &str, date: Date) -> Result<Option<(i64, i64)>>`.
|
||||
- `GET {api_url}/api/v1/opds/catalog?q={date}` with `.basic_auth(user, Some(pass))` and `Accept: application/atom+xml`. Searching by the ISO date string avoids em-dash/hyphen and ILIKE-escaping questions, and returns at most the two editions.
|
||||
- Parse entries: split on `<entry>`…`</entry>`, take `<title>` (XML-unescape `&` etc.), and the first acquisition `href` matching `^/api/v1/opds/(\d+)/download\?fileId=(\d+)$`. Choose the entry whose title equals `issue_title` exactly; if none, accept the title equal to the filename stem `The Daily EPUB - <date>` (hyphen form); never accept a title ending in `(X4)`. Prefer `quick-xml` if you'd rather not hand-roll; either is fine, keep it small and unit-tested against a fixture built from the real XML shape in §2.
|
||||
- Map `401/403` to a clear error ("BookOrbit rejected the OPDS credentials"), connection errors to "BookOrbit unreachable". Timeout 5 s via `http::build_client`.
|
||||
- Log at `info` on first resolution with both ids.
|
||||
|
||||
5. **Route** `GET /issues/{date}/read` (`src/web/issue.rs`, registered in `src/web/mod.rs` next to `/issues/{date}/behind`, same session guard as the other signed-in issue routes):
|
||||
- 404 if the feature is inactive or the issue row is missing.
|
||||
- If `refresh=1`, clear cached ids first.
|
||||
- Cached ids → `303 See Other` to `{public_url}/read/{book}/{file}`.
|
||||
- Else call `bookorbit::find_issue`; on `Some`, store and redirect; on `None`, 503 with the "not indexed yet" message; on `Err`, 502 with the error text (operator-only page, so the text can be specific).
|
||||
- Hold a `reqwest::Client` in `WebState` (`src/web/mod.rs:144`) or build one per call; per-call is acceptable at this frequency.
|
||||
|
||||
6. **Template**: add `read_href: Option<String>` to `IssueView` (set to `/issues/{date}/read` when the feature is active and `row.epub_path` is present, i.e. only alongside a real Standard EPUB). In `issue_full.html:6`, render `<a class="btn" href="{{ href }}" target="_blank" rel="noopener">Read in BookOrbit</a>` **before** the download buttons, inside the same flex row, and keep the row visible when it is the only button.
|
||||
|
||||
7. **Tests**: unit tests for the XML matcher (exact title, X4 skipped, hyphen fallback, no match, malformed href); a `src/web` test that the route is 404 when inactive and 303 to the right URL when ids are cached (no network). Follow `downloads_are_listed_only_while_the_files_exist` (`src/web/issue.rs:~2059`) for the fixture style.
|
||||
|
||||
8. **Docs/ops**: README configuration table rows for `[bookorbit]`; a note under Delivery: create an OPDS user in BookOrbit (Settings → OPDS), put its name in `config.toml` and the password in `/etc/daily-epub/env` as `DAILY_EPUB_BOOKORBIT__OPDS_PASS`, restart `daily-epub.service`. No systemd change is needed: `systemd/daily-epub.service` sets only `RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX` and no `IPAddressDeny`, so loopback HTTP to port 3498 is allowed.
|
||||
|
||||
## 5. Acceptance
|
||||
|
||||
- With the section absent or `enabled = false`: no button, `/issues/<date>/read` → 404, all existing tests pass.
|
||||
- With valid OPDS credentials: first click on today's issue → 303 to `https://bookorbit.hallada.net/read/<n>/<m>` and the `issues` row now has both ids; second click makes no request to BookOrbit (check the journal).
|
||||
- Clicking before BookOrbit has scanned the new file → 503 "not indexed yet", nothing cached, later click succeeds.
|
||||
- The X4 edition is never the target even though it matches the same search.
|
||||
- `cargo fmt`, `cargo clippy`, `cargo test` clean.
|
||||
@@ -0,0 +1,157 @@
|
||||
# Feed discovery: propose new Miniflux subscriptions from aggregator hits
|
||||
|
||||
**Date:** 2026-09-07
|
||||
**Repository:** `thallada/the-daily-epub`
|
||||
**Status:** small implementation plan, ready to execute
|
||||
**Builds on:** `docs/plans/2026-09-03-web-dashboard.md` (the admin dashboard this adds a page to), `docs/plans/2026-09-02-personalized-curation-v2.md` (the per-article signals this reuses)
|
||||
|
||||
Written for a fresh implementation agent. Facts about this repo were checked against `main` (`e69940f`) on 2026-09-07. Facts about the Miniflux API were checked against `https://miniflux.app/docs/api.html` the same day; anything marked *assumption* must be re-checked against the live instance before relying on it.
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Most of the paper's articles arrive through aggregators (Hacker News, Lobsters, Reddit, Scour), not through the author's own feed. Every such article is a lead on a feed the operator does not subscribe to yet. This feature:
|
||||
|
||||
1. During `generate`, after articles are persisted, asks Miniflux to discover the feed(s) behind each aggregator-only article and records the ones the operator is not already subscribed to as **feed candidates**.
|
||||
2. Adds an admin dashboard page, **Feeds** (`/dashboard/feeds`), listing candidates ranked by how likely the operator is to enjoy the feed, computed **algorithmically from telemetry the pipeline already persists** (utility / preliminary blend, stage reached, explicit ratings). No LLM calls, no new embeddings.
|
||||
3. Each row has **Add** (with a Miniflux category picker; calls `POST /v1/feeds`) and **Dismiss**.
|
||||
|
||||
Over time the subscribed set grows and the pool of direct-feed articles each issue draws from grows with it.
|
||||
|
||||
Out of scope (deliberately, see §7): auto-subscribing, feed previews/health checks, Scour integration, OPML, per-feed LLM judgement, un-dismissing from the UI.
|
||||
|
||||
## 2. Verified facts
|
||||
|
||||
### About this repo
|
||||
|
||||
- **Pipeline order** (`src/pipeline.rs:354` `run_stages`): ingest → dedupe → extraction → `persist_articles` (stage 4, ~line 443, mints real article ids) → social enrichment (stage 5, best effort, ~line 448) → hygiene/embeddings/signals (stage 6) → … Failure policy is documented at the top of the file: *best effort* stages log, push a warning onto `report` (status `degraded`) and continue. The new stage is best effort.
|
||||
- **How an article arrived** is stored per article in `articles.sources_json` as `Vec<SourceRef>` (`src/types.rs:64`), each with a `SourceKind` (`Scour | HnFrontpage | Lobsters | Reddit | Feed`, `src/types.rs:49`). Classification is `dedupe::classify_source_with_feed` (`src/dedupe.rs:177`). `Article::came_via(kind)` exists (`src/types.rs:147`). "Aggregator-only" therefore means `!article.sources.iter().any(|s| s.kind == SourceKind::Feed)`.
|
||||
- **Ingest already loads the whole subscription list once per run**: `MinifluxClient::ingest_window` returns `(Vec<Entry>, HashMap<FeedId, FeedMeta>)` where `FeedMeta` has `feed_url` and `site_url` (`src/miniflux.rs:105`). That map is the "already subscribed" set; no extra call needed in the pipeline.
|
||||
- **Miniflux client** (`src/miniflux.rs`) is currently GET-only: one private `get_json` helper with retry (`RetryPolicy`, retry on network/5xx/429), `X-Auth-Token` auth, `MinifluxError::{Http, Status, Decode, MissingApiKey}`. Module doc says "Reads only … read state is never mutated" — still true for the pipeline; the dashboard **Add** action is the only writer and only creates feeds.
|
||||
- **Per-article ranking evidence is persisted** in `candidate_runs` (`migrations/0002_curation_v2.sql:59`): `(run_id, article_id) PK, stage, excluded_reason, signals_json, utility REAL, rank_utility, …`. `signals_json` decodes to `curate::telemetry::SignalsJson` (`src/curate/telemetry.rs:114`) whose `blend()` returns the preliminary blend on 0–100 and whose `top_interests: Vec<TopInterest{name, z, cos}>` names the matched interests. `utility` (0–100) is set only for articles that reached the deep set (`admitted`/`assessed`/`shortlisted`/`selected`). Stage names: `dashboard::STAGES` (`src/web/dashboard/mod.rs`). `runs.status` distinguishes finished runs; dry runs are not flagged in `runs` (a dry run still writes telemetry, which is fine to use).
|
||||
- **Ratings**: `Db::current_ratings(lookback_days) -> Vec<RatedArticle>` (`src/db.rs:708`) returns one current verdict per article with `value` (`loved` 1.0, `good` 0.35, `not_for_me` −1.0) — the same numbers `signals::RatedExample` uses.
|
||||
- **Dashboard conventions**: one submodule per page group under `src/web/dashboard/`, each exposing `routes() -> Router<AppState>` merged in `dashboard::router()` (`src/web/dashboard/mod.rs:41`). Admin gating is applied by the caller. POST-then-redirect with a session flash is the established pattern (`users::approve`, `src/web/dashboard/users.rs:122`, uses `set_flash(&session, "error"|"ok", text)` then `Redirect::to`). Nav tabs are hardcoded in `src/web/templates/layout.html:47–54` keyed on `page.active_nav`. Overview tiles live in `src/web/templates/dashboard/overview.html`. Templates are askama; shared pager is `dashboard/_pager.html`.
|
||||
- **`AppState`** (`src/server.rs:79`) holds `db`, `config: Arc<RwLock<Arc<Config>>>` (read via `state.config()`), `web: Arc<WebState>`, `mailer`. Handlers construct outbound clients from config as needed (`bookorbit.rs` does this for OPDS); reuse whatever shared `reqwest::Client` `WebState` already carries, otherwise `http::build_client(http::DEFAULT_TIMEOUT)`.
|
||||
- **Config**: sections are `#[serde(deny_unknown_fields, default)]` structs on `Config` with `Default` impls; env override is `DAILY_EPUB_<SECTION>__<KEY>`. The settings dashboard has a **hardcoded section list**, help text and a test asserting the list (`src/web/dashboard/settings.rs`, see the BookOrbit plan §2 for the exact spots) — a new section must be added there or the settings page/test breaks. `config.example.toml` and the README config table document every key.
|
||||
- **Migrations**: `sqlx::migrate!("./migrations")`; latest is `0009_request_username.sql`, so the new file is `0010_feed_discovery.sql`.
|
||||
- **Jobs / CLI**: catalogue jobs are in `src/jobs.rs` and run through `main.rs` (`Job::ImportRatings => imports::run(...)`). A one-off backfill only needs a CLI subcommand, not a job (§4 step 6).
|
||||
|
||||
### About Miniflux (docs, 2026-09-07)
|
||||
|
||||
- **Discover**: `POST /v1/discover` with body `{"url": "<page or site url>"}` → `[{"url": "<feed url>", "title": "…", "type": "rss|atom|json"}]`. Optional body fields (`username`, `password`, `user_agent`, `fetch_via_proxy`) are not needed. **Observed live on 2026-09-07** (operator ran it against several hosts):
|
||||
- When the page has no `<link rel="alternate">`, Miniflux falls back to probing well-known paths on the site root (`/atom.xml`, `/feed.atom`, `/feed.xml`, `/feed/`, `/index.rss`, `/index.xml`, `/rss.xml`, `/rss/`, `/rss/feed.xml`) and **returns every path that answered 200 without checking that the body is a feed**. A site that serves its SPA shell for any path (zombo.com, immich, BookOrbit) yields up to nine bogus results, each with `title == url`. **Discover results are leads, not facts; every one must be validated before it is stored (§3, §4 step 3).**
|
||||
- When the target site fails, the response is a 4xx/5xx with `{"error_message": "fetcher: bad gateway (502 status code)"}`, `"fetcher: unexpected status code (412 status code)"`, or `"resource not found"` (nothing found / 404). Treat all of these as "host checked, zero candidates" and move on; never retry within the run.
|
||||
- Real `<link rel="alternate">` hits work from an **article** URL. Observed for `https://blog.philz.dev/blog/language-server-db/`:
|
||||
`[{"title":"blog.philz.dev","url":"https://blog.philz.dev/feed/feed.xml","type":"atom"},{"title":"blog.philz.dev","url":"https://blog.philz.dev/feed/feed.json","type":"json"}]`.
|
||||
The same content is offered twice (Atom and JSON Feed) under one title, so link-tag hits need deduping by title too (§3). Miniflux substitutes the URL for the title when the tag has none, so `title == url` marks a result as a well-known-path guess in practice.
|
||||
- **Create feed duplicate**: observed `HTTP 400` with `{"error_message":"This feed already exists."}`. The flash shows `error_message` verbatim; on that exact message the row is also flipped to `added` (without a `miniflux_feed_id`), since the operator subscribed by other means.
|
||||
- **Create feed**: `POST /v1/feeds` with `{"feed_url": "…", "category_id": <int>}` → `{"feed_id": <int>}`. Errors come back as 4xx/5xx with a JSON `error_message`; a duplicate subscription is a 4xx (*assumption:* 400 with "This feed already exists"). Surface `error_message` in the flash rather than special-casing.
|
||||
- **Categories**: `GET /v1/categories` → `[{"id", "user_id", "title", "hide_globally"}]`.
|
||||
- **Feeds**: `GET /v1/feeds` elements carry `feed_url` and `site_url` (already modelled as `MinifluxFeed`).
|
||||
- API keys are full-access; the existing key can create feeds. Nothing to provision.
|
||||
|
||||
## 3. Design decisions (settled)
|
||||
|
||||
| Topic | Decision | Why |
|
||||
|---|---|---|
|
||||
| Where discovery runs | A best-effort **pipeline stage right after social enrichment** (needs real article ids, before hygiene). Plus a CLI backfill (`daily-epub feeds discover --days N`) that runs the same function over recent `articles`, for seeding. | The user asked for "a step in article processing". The backfill is the same function behind a flag; without it the page is empty until tomorrow. |
|
||||
| Which articles | Aggregator-only articles: no `SourceKind::Feed` in `sources`. Skip when the canonical URL's host is in `discovery.skip_hosts` or matches the host of any subscribed feed's `site_url`/`feed_url`. | Articles that already came through a direct feed prove nothing. Host match against subscriptions catches "same site, different feed URL" cheaply. |
|
||||
| Discovery mechanism | **Miniflux `POST /v1/discover`** with the article URL, followed by **our own validation fetch** of each returned URL. No new crate. | Miniflux already does `<link rel=alternate>` + well-known paths + YouTube etc. and runs on the same host, but its well-known-path guesses are unverified (§2). Validating is ~40 lines; replacing discover with a home-built finder is far more. |
|
||||
| Validation | `GET` each discovered URL with the shared client (10 s timeout, read at most 256 KB). Accept only if the body, after an optional BOM, whitespace and `<?xml …?>` declaration, starts with `<rss`, `<feed`, or `<rdf:RDF`, or is JSON containing `"https://jsonfeed.org/version/`. Take the feed's own `<title>` from that body (first `<title>…</title>` in the buffer, unescaped, trimmed) as the candidate title when Miniflux gave `title == url`. Any other body, a non-2xx status or an error rejects the URL. | Content-type alone is unreliable (SPAs return `text/html` for everything, some real feeds return `text/xml` or `application/octet-stream`). Sniffing the root element is cheap and decisive. |
|
||||
| Duplicate results | After validation: group link-tag hits (`title != url`) by `title`, keeping one per group and preferring `type` `rss`/`atom` over `json`; then at most **one** `title == url` guess (the first in Miniflux's order); at most 3 per host overall. | blog.philz.dev offers the same feed as Atom and JSON Feed under one title; WordPress answers `/feed/` and `/rss/`, Hugo `/index.xml` and `/feed.xml`, all with the same content. One is enough. Real tag/category feeds arrive with distinct link-tag titles and are kept. |
|
||||
| Per-host memo | Table `feed_discovery_hosts(host PK, checked_at, candidates)`. A host is looked up **once**; re-checked only after `RECHECK_DAYS = 90` (const). New articles on an already-checked host are just **linked** to that host's existing candidates (no network). | Keeps per-run lookups bounded and lets a candidate accumulate evidence for free. |
|
||||
| Per-run cap | `discovery.max_lookups_per_run` (default 30), concurrency 4, 10 s timeout per lookup. Order: articles with the highest social score first. | A run already takes ~12 min; discovery must not add more than ~1 min. |
|
||||
| Multiple feeds per host | Drop results whose URL or title contains `comment` before validating. Each kept result is one candidate keyed by canonicalized `feed_url` (unique). | Comment feeds are noise; tag/category feeds are a real choice the operator can make on the page. |
|
||||
| Ranking | Pure function over the candidate's linked articles (§4 step 4): per-article evidence `s_i ∈ [−1, 1]` = explicit rating value if rated; else `0.85` if `stage = selected`; else `utility/100` if present; else `blend()/100` if present; else no evidence. Feed score = shrunk mean `(Σ s_i + prior·k) / (n + k)` with `prior = 0.3`, `k = 2`, shown as 0–100. Sort: score desc, then `n` desc, then `last_seen` desc. | Reuses exactly the signals the paper selects on (interest, knn, feed, social, heuristic, triage, quality, fit) with zero new cost. Shrinkage stops a single 95-utility article outranking three 80s. Constants live in one place; not configurable (YAGNI). |
|
||||
| "Why" column | The three most frequent `top_interests[].name` across the linked articles, plus the linked article titles (top 3 by `s_i`) linking to `/dashboard/articles/{id}`. | Enough to decide at a glance; all read from `signals_json`, no extra work. |
|
||||
| Add | Row form: `<select name="category_id">` + **Add** → `POST /dashboard/feeds/{id}/add` → Miniflux `POST /v1/feeds` → set `status = 'added'`, `miniflux_feed_id`, `decided_at`; flash "Added *title* to *category*". The select defaults to the last category used (`kv` key `feed_discovery_last_category`). | The operator will click Add many times in a row; remembering the category saves a click each time. |
|
||||
| Categories | `GET /v1/categories` **on page render**, one call. If it fails: flash the error and render the table with Add disabled. | Categories change rarely but caching them is more code than a call. |
|
||||
| Dismiss | `POST /dashboard/feeds/{id}/dismiss` → `status = 'dismissed'`, `decided_at`. No restore button. | Dismissals are cheap to undo with SQL if ever needed. |
|
||||
| Reconciliation | At each run's discovery stage, any `candidate` whose `feed_url` (or host) now appears in the subscribed feed map becomes `added` (with `miniflux_feed_id`). | Feeds added directly in Miniflux disappear from the list without manual bookkeeping. |
|
||||
| Page filter | `?status=candidate|added|dismissed` (default `candidate`), allow-listed; paginated with `_pager.html`, 50 rows. | One query param buys an audit view of what was added/dismissed. |
|
||||
| Config | `[discovery] enabled = true, max_lookups_per_run = 30, skip_hosts = [...]`. | Exactly the knobs that need turning: kill switch, run-time bound, noise list. |
|
||||
|
||||
## 4. Implementation steps
|
||||
|
||||
1. **Migration** `migrations/0010_feed_discovery.sql`:
|
||||
```sql
|
||||
CREATE TABLE feed_candidates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
host TEXT NOT NULL,
|
||||
title TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('candidate', 'added', 'dismissed')),
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
miniflux_feed_id INTEGER,
|
||||
decided_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_feed_candidates_status_host ON feed_candidates(status, host);
|
||||
|
||||
CREATE TABLE feed_candidate_articles (
|
||||
candidate_id INTEGER NOT NULL REFERENCES feed_candidates(id) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (candidate_id, article_id)
|
||||
);
|
||||
|
||||
CREATE TABLE feed_discovery_hosts (
|
||||
host TEXT PRIMARY KEY,
|
||||
checked_at TEXT NOT NULL,
|
||||
candidates INTEGER NOT NULL
|
||||
);
|
||||
```
|
||||
`Db` methods (in a new `src/discovery.rs`, not `db.rs`, following `imports.rs`): `hosts_checked(hosts) -> HashMap<host, (checked_at, candidates)>`, `mark_host_checked`, `upsert_candidate(feed_url, host, title, now) -> id` (bumps `last_seen`, never demotes `added`/`dismissed`), `link_article(candidate_id, article_id)`, `candidates_for_host(host)`, `list(status, page)`, `set_status(id, status, miniflux_feed_id, now)`, `reconcile_added(subscribed: &[(feed_url, site_host)], now)`, `count(status)`.
|
||||
|
||||
2. **Miniflux client** (`src/miniflux.rs`): add a private `post_json<B: Serialize, T: DeserializeOwned>(path, body)` mirroring `get_json` (same retry predicate; **do not retry** `create_feed` on 5xx — a retried create can double-subscribe; pass `RetryPolicy::none()` or a flag). Wire types `Discovered { url, title, type }`, `Category { id, title }`, `CreatedFeed { feed_id }`. Methods: `discover(url) -> Vec<Discovered>`, `categories() -> Vec<Category>`, `create_feed(feed_url, category_id) -> i64`. Make `MinifluxError::Status` keep the `error_message` string when the body is JSON so the flash reads "This feed already exists" rather than raw JSON. Update the module doc's "reads only" sentence. Tests: deserialize fixtures for all three.
|
||||
|
||||
3. **Discovery stage** (`src/discovery.rs` + wiring in `src/pipeline.rs` after stage 5, timing key `"discovery"`, report count `feed_candidates_new`, warning on error):
|
||||
```text
|
||||
select aggregator-only articles → drop skip_hosts / subscribed hosts
|
||||
→ reconcile_added(subscribed)
|
||||
→ for each distinct host: if checked within RECHECK_DAYS → link article to its candidates
|
||||
else queue (cap max_lookups_per_run, highest social score first)
|
||||
→ discover() ×4 concurrent (an error_message response = host checked, 0 candidates)
|
||||
→ drop "comment" results and already-subscribed feed_urls
|
||||
→ validate each remaining URL (GET, ≤256 KB, feed-root sniff, pull <title>)
|
||||
→ one link-tag hit per title (xml over json) + at most one guessed URL, max 3
|
||||
→ upsert_candidate + link_article + mark_host_checked
|
||||
```
|
||||
The function signature is `pub async fn run(db, miniflux: &MinifluxClient, http: &reqwest::Client, cfg: &DiscoveryConfig, articles: &[Article], subscribed: &HashMap<FeedId, FeedMeta>, now) -> Result<Summary>` so the pipeline and the CLI share it. Host = `url::Url::host_str()` lowercased with a leading `www.` stripped; compare subscriptions the same way. Budget: one discover call plus up to nine validation GETs per host, so with the 30-host cap the stage is bounded at ~300 small requests, 4 in flight; log the count. Pure helpers (`aggregator_only`, `host_of`, `keep_result`, `sniff_feed(&[u8]) -> Option<FeedKind>`, `feed_title(&[u8])`, `dedupe_guesses`) get unit tests, including the observed zombo.com response (nine `title == url` guesses that all fail the sniff → zero candidates), the observed blog.philz.dev response (Atom + JSON under one title → exactly the Atom candidate), and an SPA shell body (`<!doctype html>…`) rejected by `sniff_feed`.
|
||||
|
||||
4. **Ranking** (`src/discovery.rs`, pure): `pub fn score(evidence: &[ArticleEvidence]) -> f64` and `pub fn why(evidence) -> Vec<String>` per §3. Evidence loading is one query per page render: for the page's candidates, `SELECT article_id, stage, utility, signals_json FROM candidate_runs WHERE article_id IN (…) ORDER BY run_id DESC` keeping the latest row per article, joined with `current_ratings`. The page holds ≤ 50 candidates × a handful of articles each, so ranking happens in Rust after loading **all** `candidate`-status rows (sorting must be global, not per page). If the candidate count ever exceeds ~2 000 this needs a stored score; not now.
|
||||
Unit tests: rated article dominates; shrinkage (one 0.95 article scores below three 0.8s); no evidence → prior; `why` picks the modal interests.
|
||||
|
||||
5. **Dashboard page** `src/web/dashboard/feeds.rs` + `src/web/templates/dashboard/feeds.html`:
|
||||
- Routes: `GET /dashboard/feeds`, `POST /dashboard/feeds/{id}/add`, `POST /dashboard/feeds/{id}/dismiss`. Merge into `dashboard::router()`. Nav tab **Feeds** in `layout.html` (`active_nav = "feeds"`). Overview tile "Feed candidates → review" linking to the page.
|
||||
- Table columns: Score (0–100, tabular numerals), Feed (title → `feed_url`, host beneath), Why (interest names + up to 3 article titles linking to the article page), Articles (n), Seen (first/last via `format_time`), Actions (category select + Add; Dismiss). `?status=` filter links at the top. Added rows show the category and a link to the Miniflux feed page (`{base_url}/feeds/{miniflux_feed_id}`); dismissed rows show only the date.
|
||||
- Add handler: read `category_id` from the form and pass it through (Miniflux validates it), call `create_feed`. On `Ok` set status + `kv` last category + flash ok. On a 400 whose `error_message` is `This feed already exists.` set status `added` and flash that message as info. On any other `Err` flash `error_message` (or the transport error) and leave the row as `candidate`. Requires `state.config().miniflux.api_key`; if the client fails to build, flash "Miniflux API key is not configured".
|
||||
- Server test in the existing dashboard test harness: page renders with an empty table; add/dismiss flip `status` (point `miniflux.base_url` at a `tokio` listener that answers the create call, as the BookOrbit tests do for OPDS; note the loopback-bind caveat in `memory/codex-sandbox-quirk.md`).
|
||||
|
||||
6. **CLI backfill** (`src/main.rs`): `daily-epub feeds discover [--days 14] [--limit 200]` — loads `articles` with `first_seen` in the window, fetches the feed map (`MinifluxClient::feed_map`), calls `discovery::run` with `max_lookups_per_run = limit`, prints the summary. Takes the pipeline run lock (it writes the same tables the run does). No job-catalogue entry.
|
||||
|
||||
7. **Config + docs**: `DiscoveryConfig { enabled: bool, max_lookups_per_run: usize, skip_hosts: Vec<String> }` with defaults `true, 30, ["news.ycombinator.com", "lobste.rs", "reddit.com", "github.com", "gist.github.com", "x.com", "twitter.com", "youtube.com", "en.wikipedia.org", "arxiv.org", "docs.google.com"]`. Register the section in the settings dashboard (list, help text, test). Document in `config.example.toml` and the README table. Add a short "Feeds" paragraph to the dashboard section of the README.
|
||||
|
||||
Order matters only for 1 → 2 → 3 → 4 → 5; 6 and 7 can be done alongside. Expect roughly 900 lines including tests.
|
||||
|
||||
## 5. Verification
|
||||
|
||||
- `cargo test` green; new pure-function tests in `discovery.rs`, wire-type tests in `miniflux.rs`, one handler test for the page.
|
||||
- Live, on the server after deploy: run `daily-epub feeds discover --days 14 --limit 50`, open `/dashboard/feeds`, confirm the top rows look like feeds the operator would actually want, click **Add** on one, confirm it appears in Miniflux under the chosen category and the row flips to *added*. Run `generate --dry-run` and check the report has a `discovery` timing under ~60 s and no warning.
|
||||
- Check the `journalctl` line for how many hosts were skipped as already subscribed; if it is near zero the `www.`/host normalisation is wrong.
|
||||
- Check the line for discover results rejected by validation. A healthy day should reject a good share of them (SPA catch-alls); zero rejections means the sniff is too lenient, and near-total rejection means it is too strict (look for feeds served with a leading comment or DOCTYPE).
|
||||
|
||||
## 6. Rollout notes
|
||||
|
||||
- Migrations run automatically on the first `daily-epub` command after deploy.
|
||||
- No new secrets: the existing `DAILY_EPUB_MINIFLUX__API_KEY` can create feeds.
|
||||
- Seed the page once with the CLI backfill so it is useful on day one.
|
||||
- Every Miniflux behaviour the design depends on was observed live on 2026-09-07 (§2); the recorded responses are the test fixtures. Nothing remains to confirm before implementation.
|
||||
|
||||
## 7. Non-goals (do not add)
|
||||
|
||||
- Auto-subscribing without a click, or any LLM judgement of feeds.
|
||||
- Fetching or previewing feed contents, feed health checks, unread counts.
|
||||
- Un-dismiss / undo buttons, per-row notes, bulk actions, sorting controls (the list is already sorted by the only order that matters).
|
||||
- A stored/cached score column, a scheduled job, OPML export, Scour "recommended feeds" integration (ideas.md item, separate).
|
||||
- Configurable ranking constants.
|
||||
@@ -0,0 +1,247 @@
|
||||
# First-class interests: a table, a page, categories, links, and a rating-driven weight
|
||||
|
||||
**Date:** 2026-09-12
|
||||
**Repository:** `thallada/the-daily-epub`
|
||||
**Status:** implementation plan, ready to execute
|
||||
**Builds on:** `docs/plans/2026-09-02-personalized-curation-v2.md` (§8 profile and interests, §9 cheap signals, §12 blend and utility), `docs/plans/2026-09-03-web-dashboard.md` (page conventions), `docs/plans/2026-09-07-feed-discovery.md` (the most recent page + job + CLI addition; copy its shapes)
|
||||
|
||||
Written for a fresh implementation agent. Facts about this repo were checked against `main` (`e82f7b0`) on 2026-09-12. Nothing here depends on an external service beyond the LLM providers and Voyage that the pipeline already uses.
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Today the reader's ~230 standing interests live in a Scour OPML export (`data/scour-interests.opml`) plus an optional `## Interests` section of `profile.md`. They are parsed on every run, embedded, matched against each day's articles (`signals::interest_matches`), grouped for the system prompt by a hand-written keyword table (`profile/themes.rs`), and — since the last commit — shown to the reader as the **Matches:** line under every article header. They cannot be added from the dashboard, ratings never touch them, and nothing links from an interest to the articles that matched it.
|
||||
|
||||
This plan makes interests a first-class record:
|
||||
|
||||
1. **A database table is the only source of interests.** The OPML and the profile's `## Interests` section become one-time import inputs.
|
||||
2. **An Interests dashboard page** lists them with category, weight, and match counts; adds new ones; filters by category; changes a category by hand; deletes.
|
||||
3. **Categories** are kept (the grouped view the operator likes) and assigned by an LLM in one daily batch, only when uncategorized interests exist, with a manual **Categorize now** button.
|
||||
4. **Every interest name in the web UI is a link** to the Articles page filtered (and sorted) to the articles that matched it, with a cut-off so unrelated articles never appear. The Articles page gains an interest filter and an interests column.
|
||||
5. **Ratings nudge interests.** Each explicit rating credits the interests the article matched, scaled by how strongly it matched; the resulting per-interest weight is shown on the Interests page (default sort: highest first) and feeds curation as one bounded signal, so diversity is preserved.
|
||||
6. **Backfill** is two CLI commands the operator runs once.
|
||||
|
||||
## 2. Verified facts
|
||||
|
||||
### Interests today
|
||||
|
||||
- `profile::parse_interests(opml)` reads `<outline text="…">` names; `parse_profile_str` strips `## Interests` from `profile.md` and returns its lines; `union_interests` dedupes case-insensitively (`src/curate/profile/mod.rs:36–150`). `load_standing_interests(opml, profile)` is called from `pipeline::prepare_features` (`src/pipeline.rs:946`), `embedding::plan_backfill` (`src/curate/embedding.rs:901`), and `profile::load_or_build`/`rebuild` (through `prompt_inputs`). `config.interests_opml` is referenced in 20 places in `src/` (mostly tests that write a temp OPML), `config.example.toml:29`, README (lines 103 and 369), and `docs/runbooks/curation-v2-migration.md`.
|
||||
- The system prompt's "Standing interests" section groups names with `profile::group_into_themes` (`src/curate/profile/themes.rs`): twelve keyword themes plus "Other standing interests", deterministic, sorted. The profile dashboard page renders the same grouping (`src/web/dashboard/profile.rs:249`, template `dashboard/profile.html:42`).
|
||||
- `interest_embeddings` (migration 0002) is keyed by the interest **name**; `EmbeddingService::interests(&[String])` returns cached vectors and fetches misses (`src/curate/embedding.rs:743`). A new name is embedded automatically the next time it is passed.
|
||||
- `signals::interest_matches(article_embeddings, interest_embeddings)` (`src/curate/signals.rs:492`) computes cosine per interest × article, z-scores each interest across the day's embedded articles (std floored at 1e-3; raw top-1 cosine fallback under 30 articles), keeps the top `RECORDED_TOP = 3` per article as `TopInterest { name, z, cos }`, and scores `0.7·z₁ + 0.3·mean(top-3 z)`. `signals.top_interests` is serialized into `candidate_runs.signals_json` (`telemetry::serialize_signals`), copied into `Pick.top_interests` as bare names by the editor (`src/curate/editor.rs:598`), and rendered as the Matches line by `chapters::understanding` → `_understanding.html` (web) and `chapter.xhtml` / `in_this_issue.xhtml` (EPUB). Prompts list interests with `z ≥ 1.5` as "matches" (`triage.rs:135`, `assess.rs:192`, `editor.rs:159`).
|
||||
- Interest names also appear on: the dashboard article detail (`dashboard/article.html:61`, "Top interests" table), `_signals_table.html:5` (runs and article history), and the Feeds page "Why" badges (`dashboard/feeds.html:14`, from `discovery::why`).
|
||||
- Daily volume: ~365 entries → a few hundred articles per run; 512-dim `voyage-4-lite` vectors; `embedding_retention_days = 120`, so the embedding cache holds on the order of 30–50 k vectors (≈2 KB each). Scanning the cache per web request is not an option; a per-run cosine pass is milliseconds (§9.1 of the curation plan).
|
||||
|
||||
### Ratings and the learned signals
|
||||
|
||||
- `rating_events` is append-only; `Db::current_ratings(lookback_days) -> Vec<RatedArticle>` returns the latest explicit verdict per article with `value` (`loved` 1.0, `good` 0.35, `not_for_me` −1.0, `slop` per `feedback.slop_value`) and `event_at` (`src/db.rs:713`). Ratings are written by `web::rate::post` (dashboard and issue pages), `rate::record_explicit` (CLI), and `imports::run`.
|
||||
- `signals::PreferenceState::load` (`src/curate/signals.rs:238`) builds the run's learned state from current ratings: decayed weights `value × 0.5^(age/half_life)`, kNN over rated embeddings, and **feed/author affinity** as Beta-smoothed rates `(up+1)/(up+down+2)` with a gate `gate(n, feed_floor=15, feed_full=40)`. Every cheap signal is `Option<f64>`, percentile-normalized over the eligible set (`normalize`, `PERCENTILE_SIGNALS`), blended with renormalized weights (`preliminary_blend`; `rank::calculate_utility_for` for the deep set). Absent is never zero. The slop-author factor scales both blends. Diversity is enforced downstream by `rank::shortlist` (cluster threshold 0.85, `per_cluster_cap = 2`, `utility_protected = 10`) and by the editor prompt.
|
||||
- Signal names are enumerated in: `signals::PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend`'s candidate list, `rank::calculate_utility_for`'s weighted list, `telemetry::serialize_signals` (raw list) and `RENDERED_SIGNALS`, `dashboard::SIGNAL_NAMES`, `config::{PreliminaryWeights, UtilityWeights}`, and `settings::SETTINGS_HELP`. A new signal touches all of them.
|
||||
- The Ratings dashboard page computes each verdict's feed credit on the fly with a pure function (`ratings::contribution`) rather than storing it. Follow that precedent.
|
||||
|
||||
### Dashboard, jobs, CLI, config
|
||||
|
||||
- One submodule per page group under `src/web/dashboard/`, each with `routes()`, merged in `dashboard::router()`; admin gating is applied by the caller. Nav tabs are hard-coded in `src/web/templates/layout.html` (keyed on `page.active_nav`); overview tiles in `dashboard/overview.html`. POST → flash → redirect via `jobs::set_flash`. `Page::is_admin()` is available in every template. Pager partial `dashboard/_pager.html`; client-side row filter `data-table-filter`.
|
||||
- The Articles list (`src/web/dashboard/articles.rs`) builds `ARTICLE_INNER` (articles ⨝ best entry ⨝ latest `candidate_runs` row ⨝ assessments, with correlated subqueries for rating and publication) and wraps it in `SELECT * FROM (…) x WHERE 1=1 {clauses} ORDER BY {sort}`. Filters are allow-listed; sorts come from `ARTICLE_SORTS`; `Pager::new(pagination, path, &filters.params())` round-trips them.
|
||||
- Jobs are a fixed catalogue (`jobs::Job`, `CATALOGUE`, `parse`, `name`, `description`, `takes_lock`, `dangerous`) run as `daily-epub-job@<name>.service` and dispatched in `main::run_job`. The profile page starts `profile-rebuild` with a plain form posting to `/dashboard/jobs/profile-rebuild`. The only daily entry point is the `daily-epub-generate.timer` (05:30 America/New_York).
|
||||
- `pipeline::build_llms` (`src/pipeline.rs:1133`) builds the taste-profile prompt, the clients, and runs the weekly learned-adjustments rebuild, then rebuilds the clients with the new prompt. An LLM step that must precede the prompt goes here.
|
||||
- Config sections are `#[serde(deny_unknown_fields, default)]`; an unknown top-level key makes `Config` fail to load. The settings page has a hard-coded `GROUP_ORDER`, `PATH_KEYS`, and `SETTINGS_HELP`, with a test over the section list. Migrations: `sqlx::migrate!("./migrations")`; latest is `0012_article_publication.sql`, so the new file is `0013_interests.sql`. `db.rs` has a migration test asserting a table list (`src/db.rs:1720`).
|
||||
|
||||
## 3. Options considered
|
||||
|
||||
### 3.1 Where "articles matching interest X" comes from
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. Compute on request | Load the interest vector and every cached article embedding, dot, sort. | **Rejected.** 30–50 k × 2 KB per request; the dashboard is 1–7 ms today and should stay there. |
|
||||
| B. Query `candidate_runs.signals_json` with `json_each` | The top-3 names are already persisted per run. | **Rejected.** A JSON scan over every telemetry row (hundreds of thousands, pruned at 180 days) per request, no index, and rows vanish with telemetry retention. |
|
||||
| C. **A junction table written by the signals stage** | `article_interests(article_id, interest_id, cos, z)`: the same top-3 the run already computes, one row each, indexed by interest. The filter is an indexed join. | **Chosen.** ~1 k rows/day, zero extra computation, and the backfill is one pass over cached embeddings. |
|
||||
|
||||
### 3.2 How an interest weight is computed and stored
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. Stored counters updated on every rating event | Add `up/down` columns to `interests`; the rating handlers, CLI, and importer bump them; write a backfill migration script. | **Rejected.** Four write paths to keep in sync, decay cannot be stored (it is a function of *now*), and it duplicates the ratings history that already exists. |
|
||||
| B. **Derived on the fly from current ratings × matches** | One pure function over `current_ratings` and their `article_interests` rows, exactly like feed affinity. The run computes it in `PreferenceState::load`; the Interests page computes it on render. | **Chosen.** No new write path, always current, decay and lookback for free, backfill is "make sure rated articles have match rows". ≤ a few hundred ratings × 3 rows: microseconds. |
|
||||
| C. Ask the weekly learned-adjustments rebuild to write per-interest weights | An LLM judges the rating history per interest. | **Rejected.** Non-deterministic, weekly, and the numbers would not be explainable. The existing prose rebuild already sees the ratings. |
|
||||
|
||||
### 3.3 How the weight enters curation
|
||||
|
||||
| Option | How | Verdict |
|
||||
|---|---|---|
|
||||
| A. **A new bounded cheap signal, `affinity`** | Per article: match-strength-weighted mean of its matched interests' weights; percentile-normalized; gated on rating count; small configured weight in the preliminary blend and the utility. | **Chosen.** Fits the existing design (absent ≠ zero, renormalized weights, `explain` shows it), and its influence is capped at its weight share, so one runaway interest cannot dominate. Diversity machinery downstream is untouched. |
|
||||
| B. Multiply each interest's z by its weight before the top-3 is taken | Changes which interests appear as matches. | **Rejected.** Entangles "what does this article match" with "what does he like", and the Matches line would drift with ratings. |
|
||||
| C. Annotate the prompt's Standing interests with ↑/↓ | Cheap and the LLM would use it. | **Deferred** (§9). Worth adding once the weights have a few weeks of ratings behind them; it is a five-line change on top of this plan. |
|
||||
| D. A new deep-set admission retriever by affinity | Like the `interest` and `knn` quotas. | **Rejected.** More slots for the same signal; the blend fill already admits high-affinity articles. |
|
||||
|
||||
### 3.4 Categories
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| A. **`interests.category TEXT NULL`; the category set is the distinct values** | **Chosen.** No FK, no second page, renaming is an `UPDATE`. |
|
||||
| B. A separate `interest_categories` table with FK | Rejected: a table with one meaningful column. |
|
||||
| C. Keep the keyword table in `themes.rs` | Rejected as the source of truth (the user wants DB-tracked interests and LLM categorization), but **kept for the one-time import** so the current grouping survives unchanged. |
|
||||
|
||||
### 3.5 When the categorizer runs
|
||||
|
||||
| Option | Verdict |
|
||||
|---|---|
|
||||
| A. **Inside `generate`, before the prompt is built, only when uncategorized interests exist; plus a catalogue job for the button** | **Chosen.** The morning timer is the only daily trigger that exists; a run without new interests spends nothing. |
|
||||
| B. Its own systemd timer | Rejected: another unit to install for a call that takes seconds. |
|
||||
| C. Synchronously in the Add handler | Rejected: an LLM call in a request path, and the user asked for a daily batch. |
|
||||
|
||||
## 4. Design decisions (settled)
|
||||
|
||||
| Topic | Decision |
|
||||
|---|---|
|
||||
| Match rule (one definition everywhere) | An interest **matches** an article when it is among the article's top three interests by z **and** `z ≥ MATCH_MIN_Z = 1.0`. `interest_matches` applies this when it truncates, so `signals.top_interests`, the Matches line, the stored rows, the Articles filter, and the interest weights all agree. The score formula is unchanged (computed before the cut). The prompts keep their stricter `z ≥ 1.5` for "matches interests". |
|
||||
| Stored rows | `article_interests(article_id, interest_id, cos, z, run_id)`, upserted per eligible article per run (the same article can be eligible on consecutive days; the latest run wins; cosine is stable, z is that day's). Rows are never pruned (≈40 bytes each, ~1 k/day). |
|
||||
| Link target and filter key | `/dashboard/articles?interest=<name>`: names are unique (case-insensitive), human-readable, and `Pick.top_interests` already carries names, so no id has to travel through `issue_json`. The handler resolves the name to an id; an unknown name yields an empty list, never an error. When `interest` is set and `sort` is absent, the sort defaults to `match` (cosine descending). |
|
||||
| Who sees links | The dashboard is admin-only, so the Matches line links only for admin viewers (`page.is_admin()`); readers and anonymous visitors see plain text as today. The EPUB never links. |
|
||||
| Interest weight | Beta-smoothed rate over decayed, strength-scaled credits (§5.2): `(up + 1) / (up + down + 2)`, in (0, 1), 0.5 = no information. Shown with two decimals plus `up`, `down`, and the number of rated matches. |
|
||||
| Match strength | `s = clamp(z / 3, 0, 1)`: a z of 3 credits the full rating, a bare match (z = 1) a third. |
|
||||
| Curation signal | `affinity` (§5.3): signed, centred on zero, gated on the number of ratings that credited at least one interest (`affinity_floor = 15`, `affinity_full = 40`, same shape as the feed gate). Default weights: preliminary 0.10 (taken from `interest` 0.35→0.30 and `social` 0.10→0.05), utility 0.05 (taken from `knn` 0.15→0.10). |
|
||||
| Diversity | Guaranteed by construction: the signal's share of the blend is its configured weight (≤ 10 % / 5 %), percentile normalization caps a favourite interest's articles at percentile 1.0 of that one signal, Beta smoothing means a single *loved* moves a weight from 0.50 to at most 0.67, ratings decay with the 60-day half-life, and `rank::shortlist`'s per-cluster cap and the editor's diversity instructions are unchanged. No new cap is needed. |
|
||||
| Source of truth | The `interests` table. `interests_opml` is **removed** from config; `profile.md`'s `## Interests` section is still stripped by the parser but no longer read (the importer consumes it once; the profile page says so). |
|
||||
| Prompt grouping | "Standing interests" groups by `interests.category` (sorted by category name, members sorted case-insensitively); `NULL` renders under the existing label "Other standing interests". `themes.rs` is used only by the importer. |
|
||||
| Categorizer | One JSON call on the bulk provider over every uncategorized interest, given the existing category list; may create a category only when none fits. Unassigned names stay `NULL` and are retried the next day. Also the catalogue job `interests-categorize` behind the **Categorize now** button. |
|
||||
| Interests page actions | Add (name + optional category), change category (per-row select + Save), Delete (confirm; cascades match rows, deletes the cached embedding). No rename (delete + add). |
|
||||
| Backfill | `daily-epub interests import` (OPML + profile section → rows, categorized by the keyword table) and `daily-epub interests backfill` (match rows for every cached embedding). Weights need no backfill: they are derived. |
|
||||
| Config | No new section. `[curation.ranking]` gains `affinity_floor`, `affinity_full`; the two weight tables gain `affinity`. |
|
||||
|
||||
## 5. The numbers
|
||||
|
||||
### 5.1 Match rows
|
||||
|
||||
In `prepare_features` (`src/pipeline.rs`), after `signals::compute_all` and before the candidates are handed on: for every candidate with non-empty `signals.top_interests`, upsert `(article_id, interest_id, cos, z, run_id)`. One transaction, `INSERT … ON CONFLICT(article_id, interest_id) DO UPDATE SET cos, z, run_id`. Names map to ids through the `interests` rows loaded at the top of the stage (the same rows whose names go to `service.interests`). Best effort: a failure is a report warning, never a failed run.
|
||||
|
||||
### 5.2 Interest weight (pure, `interests::rates`)
|
||||
|
||||
Inputs: `current_ratings(rating_lookback_days)` and the `article_interests` rows of those articles.
|
||||
|
||||
```text
|
||||
for each current rating r on article a (value v_r, decay d_r = 0.5^(age_days / half_life_days))
|
||||
for each match row (a, i, z):
|
||||
s = clamp(z / 3, 0, 1)
|
||||
credit = v_r × d_r × s
|
||||
up_i += max(credit, 0)
|
||||
down_i += max(−credit, 0)
|
||||
n_i += 1
|
||||
weight_i = (up_i + 1) / (up_i + down_i + 2) # 0.5 when n_i = 0
|
||||
```
|
||||
|
||||
`cleared` verdicts are already excluded by `current_ratings`; `slop` carries `feedback.slop_value` like everywhere else. A rating is *attributable to interests* when it credits at least one interest; that count drives the gate.
|
||||
|
||||
### 5.3 The `affinity` signal (pure, in `PreferenceState`)
|
||||
|
||||
```text
|
||||
matched = the article's top_interests with n_i > 0
|
||||
affinity = Σ s_i × (weight_i − 0.5) / Σ s_i absent when matched is empty or the gate is 0
|
||||
gate = gate(attributable_interest_ratings, affinity_floor, affinity_full)
|
||||
```
|
||||
|
||||
The raw value lives in [−0.5, 0.5]; it is percentile-normalized with the other cheap signals, weighted by `weights.preliminary.affinity × gate` in the blend and `weights.utility.affinity × gate` in the utility. `explain` and `_signals_table.html` show it like any other signal. The once-per-run preference log line gains `affinity gate … (n=…)`.
|
||||
|
||||
## 6. Data model — `migrations/0013_interests.sql`
|
||||
|
||||
```sql
|
||||
CREATE TABLE interests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
category TEXT, -- NULL until categorized
|
||||
created_at TEXT NOT NULL,
|
||||
categorized_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_interests_category ON interests(category);
|
||||
|
||||
CREATE TABLE article_interests (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||
cos REAL NOT NULL,
|
||||
z REAL NOT NULL,
|
||||
run_id INTEGER, -- NULL for backfilled rows
|
||||
PRIMARY KEY (article_id, interest_id)
|
||||
);
|
||||
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||
```
|
||||
|
||||
`interest_embeddings` stays keyed by name (no migration): a deleted interest also deletes its embedding row; a renamed one is a new interest.
|
||||
|
||||
## 7. Implementation steps
|
||||
|
||||
Sizes are rough line counts including tests. Steps 1–3 must be in order; 4–8 can proceed in parallel after 3; 9 last.
|
||||
|
||||
1. **Migration + `src/interests.rs`** (~350). The table above; the db-list test in `db.rs` gains both tables. Module (following `discovery.rs`: data access, pure functions, and the LLM step in one file):
|
||||
- `Interest { id, name, category, created_at, categorized_at }`; `list(db) -> Vec<Interest>` (ordered by name), `add(db, name, category, now) -> Result<id, Duplicate>` (trimmed, 1–80 chars, unique case-insensitive), `set_category(db, id, Option<category>, now)`, `delete(db, id)` (also `DELETE FROM interest_embeddings WHERE interest = name`), `names(db) -> Vec<String>`, `grouped(db) -> Vec<(String, Vec<String>)>` (for the prompt), `uncategorized(db)`.
|
||||
- `replace_matches(db, run_id, &[(ArticleId, &[TopInterest])], names→ids)`; `matches_for_articles(db, ids) -> Vec<MatchRow { article_id, interest_id, name, cos, z }>`; `match_counts(db) -> HashMap<interest_id, i64>`.
|
||||
- `pub fn rates(ratings: &[(ArticleId, value, decay)], rows: &[(ArticleId, interest_id, z)]) -> (HashMap<interest_id, Rate { up, down, n }>, attributable)` per §5.2, and `Rate::weight()`.
|
||||
- Tests: uniqueness is case-insensitive; `rates` matches hand-checked numbers (a loved z=3 → up 1.0; a good z=1.5 → up 0.175; a not-for-me z=0.9 → nothing, below the cut); delete cascades; `grouped` puts `NULL` last under "Other standing interests".
|
||||
|
||||
2. **Signals** (~250). In `signals.rs`: `MATCH_MIN_Z = 1.0` applied inside `interest_matches` after truncation; `Signals.affinity: Option<f64>`; `"affinity"` in `PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend` (gate `affinity_gate`); `PreferenceState` gains `interest_rates: HashMap<String, Rate>` (keyed by name — `TopInterest` carries names), `affinity_gate`, `attributable_interest_ratings`, `fn affinity(&self, top: &[TopInterest]) -> Option<f64>`, loaded in `load` from `interests::matches_for_articles(rated ids)` and exposed on `PreferenceSummary` and in `log`. `rank::calculate_utility_for` adds `("affinity", configured.affinity, candidate.signals.affinity_gate)`. `telemetry`: `serialize_signals` raw list and `RENDERED_SIGNALS`; `render_explain` prints it with the others. `dashboard::SIGNAL_NAMES` and `_signals_table.html` pick it up automatically once the name is in the list. Config: `RankingConfig { affinity_floor: 15, affinity_full: 40 }`, `PreliminaryWeights { affinity: 0.10, interest: 0.30, social: 0.05 }`, `UtilityWeights { affinity: 0.05, knn: 0.10 }`, `SETTINGS_HELP` lines, `config.example.toml`, README table.
|
||||
Tests: the top-3 cut drops a z=0.4 third interest; affinity is absent under the gate and with no rated interests; a candidate matching a 0.8-weight interest outranks an otherwise identical one matching a 0.3-weight interest in the blend; blend weights still renormalize to 1.
|
||||
|
||||
3. **Replace the OPML/profile plumbing** (~300, mostly deletions and test edits). `profile::load_standing_interests`, `parse_interests`, `union_interests`, and `prompt_inputs`' OPML argument go; `build`, `load_or_build`, `rebuild`, and `weekly_rebuild_if_due` take `grouped: Vec<(String, Vec<String>)>` from `interests::grouped(db)` instead of `interests: &[String]` + `group_into_themes`. `pipeline::prepare_features` and `embedding::plan_backfill` take names from `interests::names(db)`. `themes.rs` stays but is only referenced by the importer (step 8); `pub use themes::group_into_themes` is removed from the profile module. Remove `Config.interests_opml` (struct, `Default`, `config check` line, the `deny_unknown_fields` implication is a rollout note in §10), `settings::PATH_KEYS`/`SETTINGS_HELP`, `config.example.toml`, README (lines 103, 369), and the runbook mention. Every test that writes a temp OPML instead inserts rows with `interests::add`. The profile page loses the OPML card and the "Extracted `## Interests` lines" preview; in their place one line: "N standing interests in M categories — manage them on the Interests page", and the editor note says the `## Interests` section is ignored.
|
||||
|
||||
4. **Match rows in the run** (~80). `prepare_features` writes them per §5.1 (timing folded into the existing `signals` timing; count `interest_matches` on the report counts is optional — skip unless free).
|
||||
|
||||
5. **Articles page** (~200). `ArticlesQuery.interest: Option<String>` → `ArticleFilters.interest: Option<(i64, String)>` resolved by name (case-insensitive) in `from_query`'s caller (it needs the db; resolve in `list` before building filters, or make `from_query` async — pick the former). When set: `ARTICLE_INNER` gains `JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?` (inside the inner query so `idx_article_interests_interest` drives it; the join is a `{interest_join}` placeholder that is empty otherwise) and exposes `ai.cos AS match_cos`; `ARTICLE_SORTS` gains `("match", "x.match_cos DESC, x.id DESC")`, which is the default when `interest` is set and `sort` is absent, and is ignored (falls back to `first_seen`) when it is not. Filter UI: a `<select name="interest">` over all interest names (blank = any), populated from `interests::list`. New column **interests**: for the page's ≤ 50 rows, one query through `matches_for_articles`, grouped per article, rendered as badges linking to `?interest=<name>` (ordered by z desc). `params()` round-trips `interest` and `sort=match`. Test: a seeded match row is found by the filter, an unknown name yields an empty table (200), and the column links.
|
||||
|
||||
6. **Interests page** (~450). `src/web/dashboard/interests.rs` + `dashboard/interests.html`; routes `GET /dashboard/interests`, `POST /dashboard/interests` (add), `POST /dashboard/interests/{id}/category`, `POST /dashboard/interests/{id}/delete`. Nav tab **Interests** after Ratings; overview tile "N interests · M uncategorized". Page: add form (name, category select with a blank "let the categorizer decide" option), category filter tabs or select (`?category=<name>` and `?category=uncategorized`), sort select (`weight` default, `name`, `matches`, `added`), the `data-table-filter` row filter, no pager (≤ a few hundred rows). Columns: interest (link → `/dashboard/articles?interest=<name>`), category (link → `?category=`), weight (2 decimals, tabular; `—` when n = 0), up, down, rated matches n, matched articles (count), added (`format_time`), actions (category select + Save; Delete with `confirm`). Weights come from `interests::rates` over `current_ratings(rating_lookback_days)` computed on render; the header says which lookback and half-life it uses and shows the affinity gate state ("affinity gate 0.35 · 22 of 40 attributable ratings"), mirroring the Ratings page. **Categorize now** button posts to `/dashboard/jobs/interests-categorize` (the existing generic job route), shown enabled when `jobs_enabled` and disabled with the manual command otherwise, with the uncategorized count next to it. Flash messages for add/duplicate/deleted. Tests: page renders with weights sorted descending; add rejects a duplicate differing only in case; delete removes match rows and the embedding row; routes are admin-only.
|
||||
|
||||
7. **Links everywhere** (~120). `chapters::Understanding.interests` becomes `Vec<InterestRef { name, href }>` (`href = "/dashboard/articles?interest=" + encode_component(name)`) with `interests_line()` for the EPUB templates; `_understanding.html` renders `{% for %}…{% if page.is_admin() %}<a href>…` joined by ` · `. Dashboard article detail "Top interests" table, `_signals_table.html`, and the Feeds page "Why" badges link the same way. Existing tests asserting `Matches: Filesystems · Rust` keep passing for non-admin views; add one admin assertion.
|
||||
|
||||
8. **Categorizer + job + CLI** (~350).
|
||||
- `interests::categorize(config, db) -> Result<String>`: load uncategorized; if none, return "nothing to categorize"; build `Llms::from_config` with the current taste prompt (as `cmd_profile_rebuild` does) and take `bulk` (fall back to `editor_or_bulk`); one `complete_json` at temperature 0.2 with:
|
||||
```text
|
||||
TASK: file each new standing interest under one of the reader's interest categories.
|
||||
Existing categories (reuse these names verbatim): <list>
|
||||
Create a new category only when none of the existing ones fits; a new category must be broad enough to hold several interests and named like the existing ones (two to five words, sentence case). Every interest gets exactly one category.
|
||||
New interests: <one per line>
|
||||
Return JSON exactly: {"assignments":[{"interest":"…","category":"…"}]}
|
||||
```
|
||||
Validate: interest must be one of the batch (case-insensitive), category trimmed, 1–60 chars; unknown names are ignored; anything unassigned stays `NULL`. Log "categorized N (M new categories: …)". Return that as the job message.
|
||||
- `pipeline::build_llms`: after the clients exist, if `interests::uncategorized(db)` is non-empty and a client is available, run `categorize` (warning on error), then rebuild the prompt and the clients exactly as the weekly rebuild path does (the grouping changed). Both the categorizer and the weekly rebuild share that "prompt changed, remake clients" tail.
|
||||
- `jobs::Job::InterestsCategorize` (`interests-categorize`, description "File uncategorized interests under categories with the bulk model, creating new ones only when needed.", `takes_lock` None, not dangerous); `main::run_job` dispatch; catalogue tests.
|
||||
- CLI `daily-epub interests <import|backfill>` in `main.rs`:
|
||||
- `import [--opml PATH] [--profile PATH]` (defaults `data/scour-interests.opml` and `config.profile_path`): parse the OPML with the old `parse_interests` (moved into `interests.rs` as a private helper) and the profile's `## Interests` lines, insert with `add` skipping duplicates, assign each new row's category with `themes::group_into_themes` mapped back per name ("Other standing interests" → `NULL` so the LLM gets a go at them). Print "imported N, skipped M existing, K left for the categorizer".
|
||||
- `backfill`: load every `article_embeddings` row compatible with the configured model/dimension in chunks, load interest vectors from `interest_embeddings` (fetching misses through the service if Voyage is enabled), run `interest_matches` over the whole set (z over the whole cached set stands in for the per-day cohort; say so in the docstring), and `INSERT OR IGNORE` the rows with `run_id NULL` so real-run rows win. Print the row count. Takes the run lock (it writes what the run writes).
|
||||
- Tests: the categorizer with `MockBackend` assigns two interests, creates one new category, ignores a name not in the batch; `import` is idempotent; `backfill` on a two-article fixture writes the expected rows and leaves an existing run-written row alone.
|
||||
|
||||
9. **Docs** (~40). README: the "Web site and dashboard" paragraph gets a sentence on Interests; the Commands section gets `interests import` / `interests backfill`; the configuration table gains the three keys and loses `interests_opml`; the Layout section lists `src/interests.rs`. `docs/ideas.md`: strike the "what interests most match" item.
|
||||
|
||||
Expect roughly 2 100 lines including tests.
|
||||
|
||||
## 8. Verification
|
||||
|
||||
- `cargo test`, `cargo clippy`, `cargo fmt` green.
|
||||
- Dev seed (`dev/`): `interests import --opml data/scour-interests.opml`, then `interests backfill` (no embeddings in the seed → "0 rows", exits 0), open `/dashboard/interests`: 230 rows, every one categorized except a handful under Uncategorized, weight `—` everywhere; add "Rust macros" → appears uncategorized; **Categorize now** with jobs disabled shows the manual command.
|
||||
- After deploy, on the server: run the two commands (§10), open the page, click **Categorize now**, watch the job page, refresh: no Uncategorized left, new categories (if any) look sane. Rate one article from the issue page; its matched interests' weights move on the next refresh with the expected magnitude (`0.5 → 0.67` for a full-strength *loved*).
|
||||
- Next morning's run: `explain` on a selected article shows `affinity` present with a weight, or absent with the gate at 0 if fewer than 15 ratings credited an interest (expected at first; the log line says which). The Articles page filtered by an interest the reader knows well returns articles that obviously belong, sorted by cosine; nothing unrelated appears at the bottom of the first page. Dashboard timing for that page stays single-digit ms (`Server-Timing` header).
|
||||
- The system prompt on the profile page groups Standing interests by the DB categories with the same members as before the change (byte-stable apart from category names that the LLM added).
|
||||
|
||||
## 9. Deferred, with triggers
|
||||
|
||||
- **Prompt annotation** of Standing interests with the strongest and weakest weights (e.g. `Rust ↑`, `Coffee ↓`) — once the affinity gate is open and the weights look right for a couple of weeks.
|
||||
- **Renaming** an interest in place, and merging duplicates — when the operator actually needs it (delete + add loses nothing but the match rows, which the next runs rebuild).
|
||||
- **Match-row retention** — if `article_interests` ever exceeds a few million rows; today it grows ~1 k/day.
|
||||
- **Implicit ratings** feeding the weights — already designed for in the curation plan §6.4; nothing here blocks it.
|
||||
- **Scour** — the Scour feeds in Miniflux keep working as sources; nothing here syncs interests back to Scour.
|
||||
|
||||
## 10. Rollout notes
|
||||
|
||||
1. Deploy the binary. **Before restarting the service, delete `interests_opml` from `/etc/daily-epub/config.toml`** (the key is gone and `Config` rejects unknown keys). Migrations run on the first command.
|
||||
2. `daily-epub interests import --opml /var/lib/daily-epub/data/scour-interests.opml` (the profile's `## Interests` section is read from `profile_path`; it has been empty so far). Then delete that section from `profile.md`, or leave it — it is ignored either way.
|
||||
3. `daily-epub features backfill --rated-only` if any rated article lacks an embedding (it prints the count), then `daily-epub interests backfill`.
|
||||
4. Open `/dashboard/interests`, press **Categorize now** for the leftovers, and read the weights: they are live from that moment because they are derived from the existing ratings.
|
||||
5. Optional: `daily-epub generate --dry-run` to see `affinity` in a run before the morning.
|
||||
|
||||
## 11. Non-goals (do not add)
|
||||
|
||||
- A category editor page, category colours, or ordering beyond alphabetical.
|
||||
- Per-interest notes, pinning, or manual weight overrides — the weight is derived from ratings only, so the operator changes it by rating.
|
||||
- Interest suggestions from articles ("you seem to like X") — a separate feature with its own plan if wanted.
|
||||
- Syncing interests to or from Scour, or OPML export.
|
||||
- A stored weight column, incremental counters, or a scheduled recompute.
|
||||
- Configurable `MATCH_MIN_Z` or match strength curve — constants until there is a reason.
|
||||
- Public (anonymous or reader-role) links from the Matches line; the dashboard is admin-only.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Shared brief — dashboard tables (2026-09-20)
|
||||
|
||||
The operator sees horizontal scrollbars on most dashboard tables, even on an
|
||||
ultrawide monitor, because every table is capped at the 80rem page column and
|
||||
`min-width: max-content` forbids wrapping. The fix is a small, reusable table
|
||||
framework described in **`docs/dashboard-tables.md` — read it first; it is the
|
||||
spec.** Two tasks run in parallel in separate git worktrees:
|
||||
|
||||
- `01-css.md` — the CSS side (`src/web/tailwind.css`, rebuilt `app.css`).
|
||||
- `02-templates.md` — the template audit (every `td` gets the right cell class).
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Work only in the worktree named in your brief. **Never touch the main checkout**
|
||||
at `/home/thallada/workspace/the-daily-epub`. **Do not `git commit`** (the sandbox
|
||||
cannot write `.git/worktrees`); leave the work in the tree, the orchestrator
|
||||
commits.
|
||||
- Styling lives in **`src/web/tailwind.css`** (Tailwind v4, `@apply` inside
|
||||
`@layer components`, semantic colour tokens). After every CSS change run
|
||||
`npm run css` (writes `src/web/static/app.css`, which the binary embeds with
|
||||
`include_str!`). `npm run css:check` must print no diff at the end.
|
||||
- CSP forbids inline `<style>`, `<script>` and `style=""` attributes. No new JS,
|
||||
no new dependencies, no table library: this is plain CSS + four cell classes.
|
||||
- Keep every hook `app.js` relies on: `table[data-filter]`, `.table-filter`,
|
||||
`input[data-table-filter]`, `.scroll-x` (the filter script does
|
||||
`table.closest(".scroll-x")`), `form[data-confirm]`, `details[id]`, `[data-refresh]`.
|
||||
- Keep `cargo fmt`, `cargo clippy --all-targets -- -D warnings` and the tests
|
||||
green (`cargo test --lib web::` while iterating, one full `cargo test` at the
|
||||
end). **Sandbox note:** inside the Codex sandbox `bind()` on 127.0.0.1 is
|
||||
forbidden; ignore every test that fails with `PermissionDenied`/bind errors
|
||||
(about 17 pre-existing ones). Do not "fix" them.
|
||||
- 4 cores / 7 GB shared with another agent building Rust at the same time: one
|
||||
cargo process at a time, be patient.
|
||||
- KISS / YAGNI: no config knobs, no per-table width tables, no JS resizing, no
|
||||
responsive card-ification of rows. Where the brief and the code disagree,
|
||||
follow the code and note the deviation in your handoff.
|
||||
- Finish by writing `docs/plans/briefs/dashboard-tables/handoff-<task>.md`:
|
||||
what changed (file by file), deviations, what you verified and how, anything
|
||||
left open.
|
||||
|
||||
## Previewing (optional — the sandbox usually cannot bind a port)
|
||||
|
||||
Seed a throwaway database with `cargo run --example seed_dev_db -- ./dev`
|
||||
(git-ignored) and serve it with
|
||||
`DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) DAILY_EPUB_SERVER__BIND=127.0.0.1:<port> cargo run -- --config ./dev/config.toml serve`.
|
||||
`measure.mjs` in this directory (needs `playwright` on the module path; a copy
|
||||
lives at `~/.npm/_npx/705bc6b22212b352/node_modules`) prints per page and width
|
||||
whether the document overflows and which table wrappers scroll. If you cannot
|
||||
bind, rely on the router tests and say so; the orchestrator screenshots every
|
||||
page at 390/768/1280/1920/2560 after merging.
|
||||
@@ -0,0 +1,85 @@
|
||||
# Task 01 — table framework CSS
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-tables-css` (branch `tables-css`).
|
||||
Read `00-shared.md` and `docs/dashboard-tables.md` first. Only `src/web/tailwind.css`
|
||||
and the rebuilt `src/web/static/app.css` change in this task (plus your handoff).
|
||||
The template audit happens in a sibling worktree; do not edit templates.
|
||||
|
||||
## 1. `.dashboard` becomes a content grid
|
||||
|
||||
Replace `.dashboard { @apply mx-auto my-8 max-w-7xl px-4 pb-6 … sm:px-6; }` with a
|
||||
grid that keeps the page column pixel-identical to the header
|
||||
(`mx-auto max-w-7xl px-4 sm:px-6` = content width `80rem - 2 * gutter`, centred):
|
||||
|
||||
```css
|
||||
.dashboard {
|
||||
--gutter: 1rem;
|
||||
@apply my-8 grid pb-6 font-sans text-base leading-normal;
|
||||
grid-template-columns:
|
||||
[full-start] var(--gutter)
|
||||
[wide-start] 1fr
|
||||
[content-start] minmax(0, calc(80rem - 2 * var(--gutter)))
|
||||
[content-end] 1fr
|
||||
[wide-end] var(--gutter)
|
||||
[full-end];
|
||||
}
|
||||
@media (width >= 40rem) { .dashboard { --gutter: 1.5rem; } } /* Tailwind `sm` */
|
||||
.dashboard > * { grid-column: content; min-width: 0; }
|
||||
.dashboard > .scroll-x { grid-column: wide; }
|
||||
```
|
||||
|
||||
Keep `.dashboard > * + * { mt-4 }` and every other `.dashboard …` rule as they
|
||||
are (margins on grid items work). Verify in the templates that nothing relied
|
||||
on the old padding: `.save-bar` uses `-mx-4 sm:-mx-6` to bleed to the edges;
|
||||
under the grid it still overflows its `content` area into the gutter columns
|
||||
by the same amount, so leave it unless you see a problem. The `sm` breakpoint
|
||||
is `40rem` in Tailwind v4 and `@media` rem is always 16px-based, so the query
|
||||
above matches `sm:` exactly.
|
||||
|
||||
Prototype measured in Chromium and Firefox at 390/1280/2560: `h1` left edge and
|
||||
width identical to today, no document overflow, table centred and wider than
|
||||
the column only when its content needs it.
|
||||
|
||||
## 2. Tables shrink by wrapping; page-level tables may grow past the column
|
||||
|
||||
- Delete `.scroll-x > table { min-width:max-content; }`.
|
||||
- Add, for the wide wrapper only:
|
||||
|
||||
```css
|
||||
.dashboard > .scroll-x > table {
|
||||
width: auto; /* shrink-to-fit */
|
||||
min-width: min(100%, calc(80rem - 2 * var(--gutter)));
|
||||
max-width: 100%;
|
||||
margin-inline: auto;
|
||||
}
|
||||
```
|
||||
|
||||
A table narrower than the page column fills the column as today; a wider one
|
||||
grows, centred, to the viewport edge; the wrapper scrolls only past that.
|
||||
Tables that are not direct children (cards, disclosures, signal tables inside
|
||||
cells) keep `w-full` from the base rule and simply wrap, then scroll.
|
||||
|
||||
## 3. Cell vocabulary (exactly four classes, see the doc's table)
|
||||
|
||||
- `.num` → `@apply text-right tabular-nums;` and `td.num { @apply whitespace-nowrap; }`.
|
||||
Header cells with `.num` may wrap ("cache write", "matched articles").
|
||||
- `.cell-tight` unchanged (`whitespace-nowrap`).
|
||||
- `.cell-wrap` → `@apply min-w-[10rem] max-w-[32rem] whitespace-normal; overflow-wrap:anywhere;`
|
||||
(ceiling raised from 24rem so titles get a single line when the screen has
|
||||
room; `max-width` on a cell is honoured by Chromium and Firefox for the
|
||||
column's max-content contribution — measured).
|
||||
- `th` keeps wrapping (no nowrap); keep the sticky `thead`.
|
||||
- `td > pre.preview { max-w-md }` and `.rating-cell` rules stay.
|
||||
|
||||
## 4. Small things to keep honest
|
||||
|
||||
- `.prose-body table` (reader pages) keeps its own rules; do not touch.
|
||||
- Put a short comment above the table rules pointing at `docs/dashboard-tables.md`.
|
||||
- `npm run css`, then `cargo build` (the binary embeds app.css) and
|
||||
`cargo test --lib web::`. Router tests assert on markup, not on these rules,
|
||||
so nothing should break; if something does, say what and why in the handoff.
|
||||
- If you can run the dev server + `measure.mjs`, do it and paste the ≥1280 lines
|
||||
for `/dashboard/articles`, `/dashboard/runs` and `/dashboard/runs/1` into the
|
||||
handoff. With unchanged templates some tables will still scroll at 1280 —
|
||||
that is expected until the template audit lands — but at 2560 the articles
|
||||
and candidates tables must fit without scrolling.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Task 02 — template audit: every cell says what it holds
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-tables-tpl` (branch `tables-tpl`).
|
||||
Read `00-shared.md` and `docs/dashboard-tables.md` first. Only files under
|
||||
`src/web/templates/` change in this task (plus your handoff). The CSS lands in a
|
||||
sibling worktree; assume the four cell classes behave exactly as the doc's
|
||||
table says (`cell-wrap` floor 10rem / ceiling 32rem with `overflow-wrap:anywhere`,
|
||||
`cell-tight` nowrap, `td.num` nowrap right-aligned, unclassed cells wrap at
|
||||
spaces, `th` always wraps, page-level tables grow past the page column when
|
||||
their content needs it).
|
||||
|
||||
Go through every `<table>` in `src/web/templates/dashboard/*.html`,
|
||||
`src/web/templates/_candidate_row.html` and `src/web/templates/_signals_table.html`
|
||||
and apply these rules to each `td`:
|
||||
|
||||
1. **`cell-tight` only on timestamps, dates, ids and single tokens ≤ ~20
|
||||
characters** (`first_seen`, `when`, `started`, `finished`, `requested`,
|
||||
`created`, `last_login`, `saved_at`, `changed_at`, `event_at`, `issue_date`,
|
||||
`published`, `run.date`, `run.funnel` "310 → 200 → …", `age → decay`,
|
||||
`#id`). **Remove it** from feed names/titles (`article.feed`, `candidate.feed`,
|
||||
`miss.feed`, `row.feed_credits`, nearest-article `feed`), reasons
|
||||
(`article.reason`, `row.reason`, `candidate.reason`), "admitted by" cells,
|
||||
usernames / `requested_by` / `saved_by`, roles, sources, kinds, job names,
|
||||
and from any cell that combines a badge with more text (stage + run date).
|
||||
A `.badge` is already `whitespace-nowrap` on its own, so a badge-only cell
|
||||
needs no class.
|
||||
2. **`cell-wrap` on every free-form column**: titles, URLs, notes, messages,
|
||||
"reason or comment", facet values, badge lists (interests), config diff
|
||||
before/after, anything a user typed. Keep it where it already is.
|
||||
3. **`num` on numbers only** (it already is; just make sure no text column
|
||||
uses it because it wanted right alignment — `#id` cells may keep `num`).
|
||||
4. **Long text that should not stretch a row**: if a column can hold hundreds of
|
||||
characters (job `message`, rating-import `message`, `note`), keep
|
||||
`cell-wrap` and wrap the text in `<div class="line-clamp-3" title="{{ … }}">`
|
||||
**only** if the full text is reachable somewhere else on the page or via the
|
||||
`title`; otherwise let it wrap. Never put `truncate`/`line-clamp-*` on a `td`.
|
||||
5. **`dashboard/feeds.html`**: drop `table-fixed` (fixed layout is ignored once
|
||||
the page-level table is `width:auto`). Keep the `w-*` hints on the `th`s
|
||||
(they act as minimums under auto layout), keep the `line-clamp-*`/`truncate`
|
||||
inner elements — they already sit on block children, which is the pattern —
|
||||
and put `cell-wrap` on the Feed and Why cells if it is not there. The
|
||||
actions cell keeps its forms; give it `cell-tight` only if the buttons wrap
|
||||
badly, otherwise leave it unclassed.
|
||||
6. **`dashboard/run.html` Feeds card** (`<table class="table-fixed">` inside
|
||||
`.card`): this table is `w-full` inside a card, so `table-fixed` still works
|
||||
there; leave it, but it is the only allowed `table-fixed`.
|
||||
7. **Header cells**: remove nothing, but make sure no `th` has `whitespace-nowrap`
|
||||
or `cell-tight`; long headers such as
|
||||
"considered → eligible → triaged → assessed → shortlisted → selected" may wrap.
|
||||
8. **Wrappers**: every page-level table (more than a handful of columns:
|
||||
runs, candidates, articles, ratings current/events, interests, feeds, users,
|
||||
jobs, stats runs/costs, settings history, profile versions, near misses,
|
||||
funnel) must sit in a `<div class="scroll-x …">` that is a **direct child**
|
||||
of `<section class="dashboard …">` — check that no such wrapper is nested in
|
||||
a stray `<div>`; tables inside `.card`/`.cards`/`details` stay where they are.
|
||||
Keep `tall` where it is today.
|
||||
|
||||
Do not change the columns themselves, their order, the text, or any
|
||||
`data-*`/`id`/`name`/`action`/`aria-*` attribute. Do not touch reader/public
|
||||
templates (`issue_*.html`, `article.html`, `world.html`, `behind.html`, `_toc.html`).
|
||||
|
||||
Verify: `cargo test --lib web::` (router tests assert on text and some class
|
||||
names; if one breaks only because a class moved, update the test and say so),
|
||||
then `cargo fmt`/`clippy`. List in the handoff, per template, which cells lost
|
||||
`cell-tight`, which gained `cell-wrap`, and any clamp you added.
|
||||
@@ -0,0 +1,75 @@
|
||||
# Dashboard tables CSS handoff
|
||||
|
||||
## What changed
|
||||
|
||||
- `src/web/tailwind.css`
|
||||
- Disabled Tailwind v4 automatic repository scanning with
|
||||
`@import "tailwindcss" source(none);`; retained the explicit template and
|
||||
`app.js` sources.
|
||||
- Replaced the padded/max-width dashboard container with the specified named
|
||||
`content`/`wide` grid and 1rem/1.5rem gutter variable.
|
||||
- Let direct-child `.scroll-x` wrappers occupy the wide track and made their
|
||||
tables shrink-to-fit between the content-column floor and wrapper ceiling.
|
||||
- Removed the global `.scroll-x > table { min-width:max-content; }` rule.
|
||||
- Kept `.num` alignment/tabular figures on headers and cells, while limiting
|
||||
nowrap to `td.num`.
|
||||
- Raised `.cell-wrap`'s maximum width from 24rem to 32rem.
|
||||
- Added a pointer from the table rules to `docs/dashboard-tables.md`.
|
||||
- `src/web/static/app.css`
|
||||
- Rebuilt from the Tailwind source and checked for reproducibility.
|
||||
|
||||
No templates, JavaScript, reader table rules, preview rules, or rating-cell
|
||||
rules changed.
|
||||
|
||||
## Generated CSS diff audit
|
||||
|
||||
Compared committed and rebuilt CSS one closing brace per line with:
|
||||
|
||||
```sh
|
||||
diff -U0 \
|
||||
<(git show HEAD:src/web/static/app.css | tr '}' '\n') \
|
||||
<(tr '}' '\n' < src/web/static/app.css)
|
||||
```
|
||||
|
||||
The 177-line diff contained 111 removed and 12 added content lines (excluding
|
||||
the two diff file headers). The component-rule changes were exactly the table
|
||||
framework changes above: remove `.scroll-x > table`, replace `.dashboard`, add
|
||||
the three direct-child dashboard selectors, split `td.num` nowrap from `.num`,
|
||||
and change `.cell-wrap` from 24rem to 32rem. The remainder was removal of
|
||||
utilities, unused theme symbols, and supporting custom properties that had
|
||||
previously been emitted by repository-wide automatic source discovery. There
|
||||
were no unrelated added selectors.
|
||||
|
||||
The rebuilt file shrank from 77,721 to 70,697 bytes. The documentation-only
|
||||
utility selectors `.line-clamp-3`, `.max-w-[32rem]`, `.min-w-[10rem]`, and
|
||||
`.pb-6` are absent. The semantic `.cell-wrap` component still contains
|
||||
`min-width:10rem` and `max-width:32rem`, and `.dashboard` still contains its
|
||||
requested `padding-bottom` declaration.
|
||||
|
||||
## Verification
|
||||
|
||||
- `npm run css` — passed.
|
||||
- `npm run css:check` — passed with no diff.
|
||||
- `cargo build` — passed after the final CSS build.
|
||||
- `cargo test --lib web::` — 135 passed; the two failures were
|
||||
`web::dashboard::feeds::tests::a_duplicate_subscription_still_decides_the_row`
|
||||
and `web::dashboard::feeds::tests::add_subscribes_and_records_the_miniflux_feed_id`.
|
||||
Both failed at the test bind setup with the documented
|
||||
`PermissionDenied` / `Operation not permitted` sandbox error.
|
||||
- `cargo fmt` — passed without changes.
|
||||
- `cargo clippy --all-targets -- -D warnings` — passed.
|
||||
- `git diff --check` — passed.
|
||||
|
||||
An extra full `cargo test` required by the shared brief was attempted, but the
|
||||
filesystem ran out of space while creating another test archive before tests
|
||||
started. I removed only this package's generated artifacts with
|
||||
`cargo clean -p daily-epub` (2.0 GiB), then reran the complete user-requested
|
||||
command sequence above successfully. The full suite remains unverified; the
|
||||
focused web suite is verified subject only to the two allowed bind failures.
|
||||
|
||||
## Deviations and open items
|
||||
|
||||
There are no implementation deviations. I did not run the browser measurement
|
||||
script because the sandbox cannot bind a preview server, as independently
|
||||
confirmed by the focused test failures. Visual viewport measurement remains for
|
||||
the post-merge environment described in the shared brief.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Dashboard tables template handoff
|
||||
|
||||
Implemented `02-templates.md` against `00-shared.md` and
|
||||
`docs/dashboard-tables.md`. No columns, content, hooks, or attributes were
|
||||
changed. The existing page-level `.scroll-x` wrappers already had the required
|
||||
direct-child topology, so none moved.
|
||||
|
||||
## Cell audit by template
|
||||
|
||||
| Template | Cells that lost `cell-tight` | Cells that gained `cell-wrap` | Clamp added |
|
||||
| --- | --- | --- | --- |
|
||||
| `_candidate_row.html` | Feed, stage, reason, admitted by, flags | Flags (badge list) | None |
|
||||
| `_signals_table.html` | None | None | None |
|
||||
| `dashboard/article.html` | Run history: stage, reason, admitted by; nearest articles: feed, rating; rating events: label, kind, source, user | Top-interests interest name | Rating-event note: inner `line-clamp-3` with the full note in `title` |
|
||||
| `dashboard/articles.html` | Feed, last stage (badge plus run date), reason, rating | None | None |
|
||||
| `dashboard/feeds.html` | None | None (Feed and Why already had it) | None (existing Feed/Why inner clamps retained) |
|
||||
| `dashboard/interests.html` | Category | None | None |
|
||||
| `dashboard/jobs.html` | Job name, requested by, status | None (message already had it) | Message: inner `line-clamp-3` with the full message in `title` |
|
||||
| `dashboard/profile.html` | Saved by, restore action | Preview | None |
|
||||
| `dashboard/ratings.html` | Imports: verdict, status; events: verdict, source, by; current: when/by, feed credit, used last run | Current feed credit | Import message, event note, and current note: inner `line-clamp-3` with full text in `title` |
|
||||
| `dashboard/run.html` | Funnel stage; near misses: feed, stage | None | None |
|
||||
| `dashboard/runs.html` | Status (badge plus dry-run text) | None | None |
|
||||
| `dashboard/settings_history.html` | Who | None | None |
|
||||
| `dashboard/stats.html` | Runs status | None | None |
|
||||
| `dashboard/users.html` | None | None | None |
|
||||
|
||||
`dashboard/feeds.html` also lost page-level `table-fixed`; its existing header
|
||||
width hints and inner Feed/Why clamps remain. The card-local `table-fixed` feeds
|
||||
table in `dashboard/run.html` remains as specified.
|
||||
|
||||
## Deviations
|
||||
|
||||
None. Templates whose cells were already compliant were left unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo test --lib web::`: 135 passed; two Miniflux mock-server tests failed
|
||||
only because sandbox `bind()` returned `PermissionDenied` at
|
||||
`src/web/dashboard/feeds.rs:497`, as permitted by the brief.
|
||||
- `cargo fmt`: passed.
|
||||
- `cargo clippy --all-targets -- -D warnings`: passed.
|
||||
- `git diff --check`: passed.
|
||||
|
||||
The first clippy attempt exhausted the filesystem while writing generated Rust
|
||||
metadata. Removing the disposable `target/debug/incremental` cache freed enough
|
||||
space; the exact clippy command then completed successfully.
|
||||
@@ -0,0 +1,42 @@
|
||||
// usage: node measure.mjs <port> <outdir> [widths]
|
||||
import { chromium } from "playwright";
|
||||
import fs from "node:fs";
|
||||
const [port, outdir, widthsArg] = process.argv.slice(2);
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
const widths = (widthsArg || "390,768,1280,1920,2560").split(",").map(Number);
|
||||
const pages = ["/dashboard","/dashboard/runs","/dashboard/runs/1","/dashboard/runs/2","/dashboard/articles","/dashboard/articles/1","/dashboard/ratings","/dashboard/ratings?tab=events","/dashboard/interests","/dashboard/feeds","/dashboard/feeds?status=added","/dashboard/profile","/dashboard/stats","/dashboard/jobs","/dashboard/settings","/dashboard/settings/history","/dashboard/users","/","/issues","/account"];
|
||||
fs.mkdirSync(outdir, { recursive: true });
|
||||
const browser = await chromium.launch();
|
||||
const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } });
|
||||
const page = await ctx.newPage();
|
||||
await page.goto(`${base}/login`);
|
||||
await page.fill('input[name="username"]', "admin");
|
||||
await page.fill('input[name="password"]', "adminpassword123");
|
||||
await page.click('button[type="submit"]');
|
||||
await page.waitForLoadState("networkidle");
|
||||
const report = [];
|
||||
for (const w of widths) {
|
||||
await page.setViewportSize({ width: w, height: 1000 });
|
||||
for (const p of pages) {
|
||||
const resp = await page.goto(`${base}${p}`, { waitUntil: "networkidle" });
|
||||
const status = resp?.status();
|
||||
const info = await page.evaluate(() => {
|
||||
const de = document.documentElement;
|
||||
const container = document.querySelector(".dashboard, main > section, main > div, main");
|
||||
const tables = [...document.querySelectorAll("table")].map((t) => {
|
||||
const host = t.closest(".scroll-x") || t.parentElement;
|
||||
const r = t.getBoundingClientRect();
|
||||
return { cls: t.className, ths: t.querySelectorAll("thead th").length, rows: t.querySelectorAll("tbody tr").length, tableW: Math.round(r.width), hostW: Math.round(host.getBoundingClientRect().width), hostScroll: host.scrollWidth, hostClient: host.clientWidth, overflow: host.scrollWidth > host.clientWidth + 1 };
|
||||
});
|
||||
const culprits = [...document.querySelectorAll("body *")].filter((el) => el.scrollWidth > el.clientWidth + 1 && getComputedStyle(el).overflowX === "visible" && !el.closest("table")).slice(0, 5).map((el) => `${el.tagName.toLowerCase()}.${String(el.className).split(" ").slice(0,2).join(".")} ${el.scrollWidth}>${el.clientWidth}`);
|
||||
return { pageOverflow: de.scrollWidth > de.clientWidth, docW: de.clientWidth, containerW: container ? Math.round(container.getBoundingClientRect().width) : null, tables, culprits };
|
||||
});
|
||||
const name = p.replace(/[\/?=]/g, "_").replace(/^_/, "") || "root";
|
||||
if (w === 1280 || w === 2560 || w === 390) await page.screenshot({ path: `${outdir}/${w}-${name}.png`, fullPage: true });
|
||||
report.push({ w, p, status, ...info });
|
||||
const bad = info.tables.filter((t) => t.overflow);
|
||||
console.log(`${w} ${p} status=${status} page=${info.pageOverflow ? "OVERFLOW" : "ok"} container=${info.containerW} tables=${info.tables.length} scrolling=${bad.length}${bad.length ? " [" + bad.map((t) => `${t.cls || "-"}:${t.hostScroll}>${t.hostClient}`).join(", ") + "]" : ""}${info.culprits.length ? " culprits=" + info.culprits.join(" | ") : ""}`);
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(`${outdir}/report.json`, JSON.stringify(report, null, 1));
|
||||
await browser.close();
|
||||
@@ -0,0 +1,93 @@
|
||||
# Shared brief — UI polish pass (2026-09-04)
|
||||
|
||||
The operator asked for a design/bug-fix pass over the web dashboard v2 (Tailwind
|
||||
redesign, merged to `main` at `e719e03`+). The work is split into three tasks
|
||||
(`01-toc.md`, `02-rating.md`, `03-shell.md`) that run in parallel in separate git
|
||||
worktrees. Read `docs/plans/briefs/web-dashboard-v2/00-shared.md` for the design
|
||||
language (quiet morning paper: warm paper, near-black ink, one red accent,
|
||||
hairline rules, real typographic hierarchy; reader pages serif, dashboard sans)
|
||||
and the Tailwind v4 conventions. This brief overrides it where they disagree.
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Work only in the worktree named in your brief. **Never touch the main checkout**
|
||||
at `/home/thallada/workspace/the-daily-epub`. Do not `git commit` (the sandbox
|
||||
cannot write `.git/worktrees`); leave the work in the tree and the orchestrator
|
||||
commits. If you can commit, one commit on the worktree branch is fine.
|
||||
- Styling lives in **`src/web/tailwind.css`** (Tailwind v4, `@apply` in
|
||||
`@layer components`, semantic colour tokens `paper/paper-2/ink/ink-2/muted/rule/
|
||||
rule-strong/accent/loved/good/down/warn`) and in Tailwind utility classes in the
|
||||
askama templates under `src/web/templates/`. **Rebuild the committed CSS** after
|
||||
every change with `npm run css` (writes `src/web/static/app.css`; the binary
|
||||
embeds it with `include_str!`, so rebuild Rust after the CSS). `npm run css:check`
|
||||
must print no diff at the end.
|
||||
- CSP is `default-src 'self'; style-src 'self'; script-src 'self'`: **no inline
|
||||
`<style>`/`<script>`, no `style=""` attributes, no CDNs.** Setting CSS custom
|
||||
properties or `element.style.*` from `app.js` (CSSOM) is allowed by CSP but
|
||||
prefer class/attribute toggles styled from `tailwind.css`.
|
||||
- JS is plain `src/web/static/app.js` (runs once at the end of `<body>`) and the
|
||||
tiny pre-paint `src/web/static/theme.js`. Keep every hook app.js relies on:
|
||||
`form.rating`, `button[data-label]`, `.active` + `aria-pressed` on rating
|
||||
buttons, `button.clear`, `form[data-confirm]`, `details[id]`, `table[data-filter]`,
|
||||
`.table-filter`, `button[data-reset]`/`data-default`, `[data-refresh]`,
|
||||
`[data-toc-panel]`, `[data-toc-toggle]`, `[data-toc-progress]`, `[data-toc-scroll]`,
|
||||
`[data-theme-toggle]`. Keep `id`/`name`/`action`/`aria-*` attributes used by
|
||||
handlers or tests. Router tests assert on text and some class names; if a test
|
||||
breaks only because a cosmetic class changed, update the test.
|
||||
- Don't change routes, auth, migrations or `web::security_headers`.
|
||||
- Prefer CSS transitions (interruptible) over keyframes; name exact transition
|
||||
properties (never `transition: all`); respect the existing
|
||||
`prefers-reduced-motion` rule. Touch targets ≥ 44 px on phones (extend with a
|
||||
pseudo-element if the visible control is smaller); ≥ 40 px in the dense
|
||||
dashboard. Tabular numerals for changing numbers. Hairline rules, not boxes,
|
||||
for structure. No decorative motion on frequent interactions.
|
||||
- Keep `cargo fmt`, `cargo clippy --all-targets -- -D warnings` and the tests
|
||||
green. Iterate with narrow runs (`cargo test --lib web::`), one `cargo test` at
|
||||
the end. **Sandbox note:** inside the Codex sandbox `bind()` on 127.0.0.1 is
|
||||
forbidden, so these pre-existing tests fail there and must be ignored:
|
||||
`curate::llm::tests::anthropic_*` (4), `extract::tests::relative_urls_resolve_against_the_url_we_landed_on`,
|
||||
`server::tests::*` (5) and `tests/m7_server.rs`. Do not "fix" them.
|
||||
- This host has 4 cores / 7 GB and other agents build Rust at the same time; be
|
||||
patient with `cargo` and don't run several builds concurrently yourself.
|
||||
- Don't gold-plate and don't widen scope beyond your brief's numbered items plus
|
||||
the polish items it lists. Where the brief and the code disagree, follow the
|
||||
code and note the deviation.
|
||||
- Finish by writing `docs/plans/briefs/ui-pass/handoff-<task>.md`: what changed
|
||||
(file by file, briefly), design decisions, deviations, what you verified and
|
||||
how (commands, screenshots), anything left open.
|
||||
|
||||
## Previewing
|
||||
|
||||
A seeded dev database exists in the main checkout at
|
||||
`/home/thallada/workspace/the-daily-epub/dev` (two issues: No. 18 on 2026-09-01,
|
||||
legacy; No. 19 on 2026-09-02, full snapshot; accounts `admin`/`adminpassword123`
|
||||
and `reader`/`readerpassword123`). Copy it if you need your own:
|
||||
`cp -r /home/thallada/workspace/the-daily-epub/dev ./dev` (git-ignored). Run your
|
||||
worktree's server on **your own port** (given in your brief; 3499 is production,
|
||||
3599 is the orchestrator's):
|
||||
|
||||
```text
|
||||
DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) \
|
||||
DAILY_EPUB_SERVER__BIND=127.0.0.1:<port> \
|
||||
cargo run -- --config ./dev/config.toml serve
|
||||
```
|
||||
|
||||
(the listen address is `server.bind` in `./dev/config.toml`, overridable by the
|
||||
env var above; your worktree's `dev/config.toml` is already set to your port. If
|
||||
the sandbox forbids binding, rely on router tests + `cargo test` and say so in
|
||||
the handoff).
|
||||
|
||||
Screenshots: `node shoot.mjs <outdir> [--user admin|reader|none] [--theme light|dark] [--mobile] [--full] /path …`
|
||||
with `BASE=http://127.0.0.1:<port>`, from
|
||||
`/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/553c7d77-d8f4-44f8-a379-73cedea23814/scratchpad/shoot/`
|
||||
(Playwright with a headless Chromium already installed under `~/.cache/ms-playwright`).
|
||||
If that directory is unreadable, the bare browser at
|
||||
`~/.cache/ms-playwright/chromium_headless_shell-1208/chrome-headless-shell-linux64/chrome-headless-shell --headless --no-sandbox --screenshot=out.png --window-size=1280,1400 URL`
|
||||
works for anonymous pages. **Look at your screenshots** (read the PNGs) before
|
||||
calling a design change done, in light and dark, desktop and 390 px phone.
|
||||
|
||||
Useful dev URLs: `/` (latest issue, No. 19), `/issues/2026-09-01` (archived
|
||||
issue), `/issues/2026-09-02/articles/<id>` (chapter; pick ids from the TOC),
|
||||
`/issues/2026-09-02/world`, `/issues/2026-09-02/behind`, `/dashboard`,
|
||||
`/dashboard/runs`, `/dashboard/runs/<id>`, `/dashboard/ratings`,
|
||||
`/dashboard/settings`, `/dashboard/articles`.
|
||||
@@ -0,0 +1,88 @@
|
||||
# Task 1 — table-of-contents sidebar: mobile height, scroll sync, scrollbar
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-ui1` (branch `ui-toc`). Dev port 3601.
|
||||
Files you own: `src/web/templates/_toc.html`, the TOC parts of
|
||||
`src/web/static/app.js` (the `[data-toc-*]` blocks), the "table-of-contents"
|
||||
section of `src/web/tailwind.css`, and the index entries in
|
||||
`src/web/templates/issue_full.html` (only to add data attributes / ids). Backend:
|
||||
`src/web/issue.rs` `Toc`/`TocItem` if a field helps. Read the shared brief first.
|
||||
|
||||
The signed-in issue pages (`issue_full.html`, `article.html`, `world.html`,
|
||||
`behind.html`) share `_toc.html`: a sticky bar with a hamburger below `lg`, a
|
||||
sticky sidebar at `lg+`. Chapters are separate pages; the issue front page lists
|
||||
every chapter in "In This Issue".
|
||||
|
||||
## 1. The open hamburger panel must fill the screen (operator issue 1)
|
||||
|
||||
Today the panel is `absolute top-full max-h-[72vh]`, so on a phone it stops
|
||||
partway down with page content visible beneath it — it looks broken. Make the
|
||||
open panel extend from the bottom of the sticky bar to the **bottom of the
|
||||
viewport** (use `100dvh`, not `100vh`, so mobile browser chrome is accounted
|
||||
for), with the chapter list scrolling inside it and the bar staying fixed above
|
||||
it. Handle both states: the bar already stuck at the top of the viewport, and
|
||||
the bar still in flow below the masthead on first open. Prefer a pure-CSS
|
||||
solution; if the bar's on-screen position must be measured, set one CSS custom
|
||||
property from app.js on open/resize and consume it in `tailwind.css`. Avoid
|
||||
jumps of the underlying page when the panel opens or closes. Keep Escape, the
|
||||
outside-click close, the tap-on-chapter close, and the `lg` breakpoint reset.
|
||||
Content behind the panel must not scroll while it is open (e.g. toggle a class
|
||||
on `<html>` that sets `overflow:hidden`, cleared on close).
|
||||
|
||||
## 2. Scroll-synced current chapter (operator issue 2)
|
||||
|
||||
On the issue front page the reader scrolls through "In This Issue"; each index
|
||||
entry corresponds to a TOC chapter. As the reader scrolls, the TOC's current
|
||||
marker must follow: when entry 5's card reaches the top region of the viewport,
|
||||
chapters 1–4 have visibly "passed" and 5 is current. Implement a scroll-spy in
|
||||
app.js (IntersectionObserver or a rAF-throttled scroll listener; either is fine,
|
||||
but it must be cheap and passive):
|
||||
|
||||
- Mark each index entry `<li>` in `issue_full.html` with the chapter it
|
||||
represents (e.g. `data-toc-entry="{{ entry.href }}"`, matching the TOC link's
|
||||
`href`); "The Brief" section maps to the Brief item, the colophon to the
|
||||
Colophon item.
|
||||
- The current entry is the last one whose top edge is above a threshold
|
||||
(roughly the sticky bar height + ~1/3 of the viewport). Before the first entry
|
||||
the Brief is current.
|
||||
- Move the current styling to a single hook so JS only toggles one attribute:
|
||||
give TOC links a class (say `toc-link`) and style `.toc-link[aria-current]`
|
||||
in `tailwind.css` (keep `aria-current="page"` for the server-rendered chapter
|
||||
on chapter pages; the scroll-spy may use `aria-current="location"`). Passed
|
||||
chapters may stay ink-coloured while unread ones are `ink-2`; keep it subtle.
|
||||
- Update the sticky mobile bar's label and "N / total" counter and both
|
||||
`[data-toc-progress]` bars from the same source of truth (the front page
|
||||
currently leaves the progress bar at 0; it should reflect the current chapter
|
||||
as the reader passes entries). The desktop "Chapter N of M / Front page" line
|
||||
should update too. Chapter pages keep today's progress-within-chapter logic.
|
||||
- When the current item changes, **auto-scroll the TOC panel** so the current
|
||||
link is comfortably in view (there is already a `revealCurrent()`; reuse it,
|
||||
make it use `scrollTo({behavior:"smooth"})` unless `prefers-reduced-motion`,
|
||||
and make sure it works for the desktop sticky sidebar as well as the open
|
||||
mobile panel). Never scroll the window itself from this code.
|
||||
- Chapter pages already mark the chapter current; make sure the sidebar reveals
|
||||
it on load (desktop and mobile) — verify on a long TOC.
|
||||
|
||||
## 3. Scrollbar must not overlap the TOC text (operator issue 11)
|
||||
|
||||
In the sidebar the scrollbar is painted over the right edge of the chapter
|
||||
labels. Reserve the gutter (`scrollbar-gutter: stable`, plus right padding on
|
||||
the panel so the list ends before the gutter) so the scrollbar sits to the
|
||||
right of the content on every platform, including overlay-scrollbar macOS.
|
||||
Keep `scrollbar-width: thin` and the rule-coloured thumb.
|
||||
|
||||
## Polish while you are there
|
||||
|
||||
- Chapter links: ≥ 44 px tall tap targets on phones (they are `py-2.5` today,
|
||||
check), hover/focus-visible states consistent with the nav.
|
||||
- The hamburger icon should animate to a close icon while open using the
|
||||
cross-fade recipe (both SVGs in the DOM, one absolutely positioned; opacity
|
||||
`0→1`, scale `0.25→1`, blur `4px→0`, ~150 ms, `cubic-bezier(0.2,0,0,1)`), and
|
||||
the `aria-label` should flip to "Close contents".
|
||||
|
||||
## Verification
|
||||
|
||||
Screenshot the front page and one chapter, desktop + phone, light + dark, with
|
||||
the panel open on the phone; script a scroll on the front page (Playwright
|
||||
`page.mouse.wheel` or `window.scrollTo`) and screenshot the sidebar at two
|
||||
scroll positions to prove the marker moves and the panel auto-scrolls. Run the
|
||||
web router tests. Write `handoff-toc.md`.
|
||||
@@ -0,0 +1,97 @@
|
||||
# Task 2 — rating widget, issue section headers, ratings-table verdict column
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-ui2` (branch `ui-rating`). Dev port 3602.
|
||||
Files you own: `src/web/templates/_rating_widget.html`, the `.rating*`,
|
||||
`td .rating`, `.prose-body` and reader-typography parts of `src/web/tailwind.css`,
|
||||
`src/web/templates/issue_full.html` and `issue_public.html` (section headers),
|
||||
`src/web/templates/article.html` (rating placement only),
|
||||
`src/web/templates/dashboard/ratings.html`, and the rating `submit` handler in
|
||||
`src/web/static/app.js` if the markup change needs it. Read the shared brief first.
|
||||
|
||||
## 1. Redesign the rating widget (operator issue 3)
|
||||
|
||||
`_rating_widget.html` renders "Was this a good pick? [Loved it][Good][Not for me]
|
||||
clear" (+ a Note input on dashboard pages). The buttons are 40 px tall,
|
||||
full-border segmented, solid-ink when active — too heavy next to the serif index
|
||||
cards on the issue front page and the article footer; they read as a form, not as
|
||||
part of the paper. Redesign it to be cohesive with the article display:
|
||||
|
||||
- Quieter and smaller: sans, `text-xs`/`text-[0.8rem]`, compact height (~30–32 px
|
||||
visible; extend the hit area to 44 px on phones with a pseudo-element), hairline
|
||||
`rule` borders or a single underlined-text style, `rounded-sm`.
|
||||
- The active state should use the semantic colour of the verdict (`loved`, `good`,
|
||||
`down`) as a tint, like `.badge.loved` etc. (`color-mix(in oklab, var(--loved)
|
||||
14%, transparent)` background + coloured text/border), not solid ink — so the
|
||||
chosen verdict is legible at a glance and matches the badges used elsewhere.
|
||||
- The prompt "Was this a good pick?" becomes a small serif-italic or muted-sans
|
||||
lead-in that sits on the same line as the buttons at desktop widths and above
|
||||
them on phones; "clear" stays a quiet text link, visible only when a verdict is
|
||||
active (still in the DOM for the handler; use `[hidden]` or an opacity class).
|
||||
- Keep the markup contract: `form.rating`, `button[type=submit][name=label][value][data-label]`,
|
||||
`.active`, `aria-pressed`, `button.clear`, `label.rating-note > input[name=note]`,
|
||||
the three hidden inputs, `widget.show_note`, and the JSON fetch in app.js (it
|
||||
toggles `.active`/`aria-pressed`; if you add a per-verdict tint class, derive it
|
||||
from `data-label` in CSS — `.rating button.active[data-label="loved"]` — so JS
|
||||
needs no change).
|
||||
- Press feedback: `active:scale-[0.96]` with `transition-property: transform,
|
||||
background-color, color, border-color` on the buttons.
|
||||
|
||||
Check both placements: the index card on `/` (after "Why it's here") and the
|
||||
article footer on a chapter page; also `/dashboard/articles/<id>` which shows the
|
||||
widget with the note field.
|
||||
|
||||
## 2. Section headers in the issue (operator issue 4)
|
||||
|
||||
On the issue front page ("In This Issue") and the public issue page, each section
|
||||
name (e.g. *Technology*) is a 0.72 rem uppercase muted label above a hairline —
|
||||
the same style as tiny eyebrows elsewhere — so readers scroll past it without
|
||||
noticing the section changed. Make section headers read as headers: serif,
|
||||
`text-2xl` or so, weight 600, `leading-[1.1]`, `tracking-[-0.01em]`,
|
||||
`text-wrap:balance`, sitting on a **stronger rule** (`border-t-2 border-rule-strong`
|
||||
or a double rule like the masthead) with generous space above, and a small
|
||||
uppercase eyebrow ("Section" or the article count, optional) if it helps
|
||||
hierarchy. Headlines inside the section stay `text-2xl`/lead `text-3xl`, so the
|
||||
section header must be distinguishable from them by the rule + spacing + smaller
|
||||
size rather than by being bigger; consider `text-xl` uppercase-tracked serif
|
||||
small caps if that reads better — screenshot both and pick one. Apply the same
|
||||
treatment to "The Brief" and "Colophon" headers on the front page so the page has
|
||||
one heading system, and to the section names on `issue_public.html`. Leave the
|
||||
TOC sidebar's section labels alone (task 1 owns that file).
|
||||
|
||||
Also check article bodies (`.prose-body h2/h3` on a chapter page): they should be
|
||||
clearly headings against `1.0625rem` body text; adjust only if they are weak.
|
||||
|
||||
## 3. Ratings table verdict column (operator issue 7)
|
||||
|
||||
`dashboard/ratings.html`, tab "Current": the first column stacks a verdict badge,
|
||||
the three rating buttons, a Note input and "clear" — busy and janky. Make that
|
||||
cell one calm unit:
|
||||
|
||||
- Drop the separate badge: the tinted active button (from item 1) *is* the verdict.
|
||||
If a row is `cleared` show a muted "cleared" state in the group instead.
|
||||
- A compact segmented group of the three verdicts on one line (fixed width, e.g.
|
||||
`w-[13rem]`), the "clear" link tucked at the end of that line, and the note
|
||||
field beneath as a quiet single-line input (`placeholder="Add a note…"`,
|
||||
bottom-hairline only, no box) that only shows a border on focus. Everything
|
||||
left-aligned on one grid so rows line up; no wrapping inside the cell at the
|
||||
table's minimum width.
|
||||
- Vertical rhythm: the cell content top-aligned with the row's article title.
|
||||
- Keep the note column of the table as is (it shows the stored note).
|
||||
|
||||
## Polish while you are there (reader pages)
|
||||
|
||||
- `text-wrap: balance` on headlines (`h1`–`h4` in reader pages, TOC excluded);
|
||||
`text-wrap: pretty` on `.prose-body p` and index summaries.
|
||||
- `.prose-body img`: add the depth outline `outline: 1px solid oklch(0 0 0 / 0.1)`
|
||||
in light and `oklch(1 0 0 / 0.1)` in dark (pure black/white only, use the
|
||||
existing `dark` variant), `outline-offset: -1px`.
|
||||
- Buttons `.btn`, `.btn-primary`, `.btn-danger`: `active:scale-[0.96]` with named
|
||||
transition properties (transform + colours), keeping the reduced-motion rule.
|
||||
|
||||
## Verification
|
||||
|
||||
Screenshot `/` (signed in as reader), a chapter page footer, `/dashboard/ratings`
|
||||
(admin, tab Current) and `/dashboard/articles/<id>`, light + dark, desktop + phone.
|
||||
Exercise a rating click on the dev server (the JSON handler) to confirm the tint
|
||||
switches. Run `cargo test --lib web::` and the ratings/issue router tests. Write
|
||||
`handoff-rating.md`.
|
||||
@@ -0,0 +1,110 @@
|
||||
# Task 3 — site shell: nav state, ears, runs feeds table, FOUC, faster navigation
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-ui3` (branch `ui-shell`). Dev port 3603.
|
||||
Files you own: `src/web/templates/layout.html`, `src/web/mod.rs` (static assets,
|
||||
headers), `src/web/public.rs`, `src/web/issue.rs` (page/nav plumbing only, not
|
||||
the `Toc`), `src/web/templates/dashboard/run.html`, the "dashboard shell" and
|
||||
`@layer base` parts of `src/web/tailwind.css`, and `src/web/static/theme.js`.
|
||||
Read the shared brief first.
|
||||
|
||||
## 1. "Latest" is highlighted on archived issues (operator issue 5)
|
||||
|
||||
Every issue page (`public::show_issue`, `issue::render_full`, `issue::article`,
|
||||
`world`, `behind`) passes `"latest"` as `active_nav`, so viewing `/issues/2026-09-01`
|
||||
from the archive highlights *Latest* in the site nav. Highlight *Archive*
|
||||
instead whenever the issue is not the newest one. Cleanest: `issue::load` (which
|
||||
has the db) records `is_latest` on `IssueView` via `db.latest_issue_date()`, and
|
||||
each renderer chooses `"latest"`/`"archive"` from it; the public branch in
|
||||
`show_issue` needs the same. Add a router test that fetches an older issue and
|
||||
asserts `aria-current="page"` is on the Archive link and not on Latest (and the
|
||||
reverse for the newest issue).
|
||||
|
||||
## 2. "Morning edition" ears on non-issue pages (operator issue 10)
|
||||
|
||||
`layout.html`'s default `{% block ears %}` prints "Morning edition" — it shows on
|
||||
settings and every other dashboard/account page where it is meaningless. Make the
|
||||
default empty and let issue pages keep overriding it with their dateline. On
|
||||
dashboard pages show a short "Dashboard" label there instead (only when
|
||||
`page.is_admin()` and the active nav is a dashboard section) so the row does not
|
||||
look orphaned; anything else stays empty. Check the row keeps its height and the
|
||||
theme toggle stays right-aligned when the left cell is empty. `issue_public.html`'s
|
||||
empty-state hero may keep its own "Morning edition" eyebrow.
|
||||
|
||||
## 3. Runs page "Feeds (top 20)" overflow (operator issue 6)
|
||||
|
||||
`dashboard/run.html`: the Feeds card wraps its table in `.scroll-x`, whose
|
||||
`> table { min-width: max-content }` lets long feed titles push the *entries*
|
||||
column out of the card until the reader scrolls sideways. That table must fit the
|
||||
card: `table-fixed w-full`, the feed cell `truncate` (nowrap + ellipsis) with the
|
||||
full name in a `title` attribute, entries column a fixed narrow width, no
|
||||
horizontal scroll container (or a `.scroll-x` variant that allows shrinking —
|
||||
your call, but don't break the other tables that rely on `max-content`). Check
|
||||
the Timings and Provider usage cards next to it still look right, and the
|
||||
dashboard overview if it has a similar box.
|
||||
|
||||
## 4. Flash of unstyled content (operator issue 8)
|
||||
|
||||
The operator sees a brief flash of unstyled content when navigating between
|
||||
pages, "or maybe just sometimes". Reproduce before fixing: with the dev server,
|
||||
use Playwright to capture the first frames of a navigation with the cache
|
||||
disabled and a throttled network (there are older experiments `flash.mjs` /
|
||||
`flash2.mjs` in the screenshot directory named in the shared brief you may crib
|
||||
from) and note what actually flashes: the fallback serif before Newsreader loads
|
||||
(`font-display: swap`), a theme flip, the TOC panel, restored `details` state,
|
||||
or something else. Then fix the causes you confirm. Expected fixes:
|
||||
|
||||
- Static assets are already versioned by content hash (`?v={{ page.asset_version }}`),
|
||||
so serve `/static/*` with `Cache-Control: public, max-age=31536000, immutable`
|
||||
(update `static_assets_use_content_hash_etags`); keep the ETag.
|
||||
- Fonts: preload both Newsreader files, and change `font-display` from `swap` to
|
||||
`block` (no visible fallback; with the preload and immutable cache the block
|
||||
period is only ever paid once) — or `optional` if you can show it behaves
|
||||
better here. Explain the choice in the handoff.
|
||||
- Anything initialised by app.js at the end of `<body>` that changes layout
|
||||
(e.g. `details[id]` open state restored from localStorage) should either be
|
||||
decided pre-paint in `theme.js` or made not to shift layout.
|
||||
|
||||
## 5. Faster, SPA-like navigation without a client router (operator issue 9)
|
||||
|
||||
The operator wants link clicks to feel like an instant swap rather than a full
|
||||
reload, but wants to decide on HTMX (boost mode) in a separate effort. **Do not**
|
||||
write a fetch-and-swap router and do not add HTMX. Deliver the standards-based
|
||||
progressive enhancements that get most of the way there:
|
||||
|
||||
- **Cross-document view transitions**: `@view-transition { navigation: auto; }`
|
||||
in `tailwind.css` with a short (~120–150 ms) cross-fade of the page and the
|
||||
header kept stable (`view-transition-name` on the site header so the masthead
|
||||
and nav do not fade; the reader's sticky TOC bar may also stay put). Respect
|
||||
reduced motion (`@media (prefers-reduced-motion: reduce)` → no animation).
|
||||
- **Speculation rules** for prerender/prefetch on hover: inline
|
||||
`<script type="speculationrules">` is blocked by the CSP, so serve a JSON rule
|
||||
set as a static asset (`/static/speculation.json`, embedded like the others)
|
||||
and send the `Speculation-Rules: "/static/speculation.json"` response header
|
||||
on HTML responses (add it in `web::security_headers` or the same layer).
|
||||
Rules: `prerender` with `eagerness: "moderate"` for same-origin reader pages
|
||||
(`/`, `/issues/*`); `prefetch` with `eagerness: "moderate"` for `/dashboard/*`
|
||||
(heavy queries — don't prerender them); exclude `/logout`, `/rate`, `/static/*`
|
||||
and anything with a query string that mutates. Both are Chromium-only today and
|
||||
no-ops elsewhere; say so in the handoff.
|
||||
- Make sure `Cache-Control` on HTML stays as is (`public_cache` / no-store for
|
||||
signed-in) so prerendered pages are not stale for signed-in readers, and that
|
||||
the `[data-refresh]` job-page reload cannot run in a prerendered page
|
||||
(`document.prerendering` guard).
|
||||
|
||||
## Polish while you are there (shell)
|
||||
|
||||
- Site nav and dashboard nav links: consistent focus-visible ring (they inherit
|
||||
the base `a` rule — check it is not clipped by `overflow`), and the dashboard
|
||||
nav's active link should get the same 2 px accent underline treatment as the
|
||||
site nav rather than just a colour change.
|
||||
- Theme toggle and the Sign-in/username link: ≥ 40 px tap targets.
|
||||
- `.tile-num`, `.pager`, and anything that shows changing counts already use
|
||||
tabular numerals — confirm; add to `.badge` when it contains a number.
|
||||
|
||||
## Verification
|
||||
|
||||
`cargo test` (full, minus the sandbox-only failures), `npm run css:check`, and
|
||||
screenshots of `/issues/2026-09-01` (archive highlighting), `/dashboard/settings`
|
||||
(ears), `/dashboard/runs/<id>` (feeds card at desktop and phone width), plus your
|
||||
flash reproduction before/after. Curl the headers of `/` and `/static/app.css`
|
||||
and paste them in `handoff-shell.md`.
|
||||
@@ -0,0 +1,167 @@
|
||||
# Task 2 handoff — rating widget, issue section headers, ratings-table verdict column
|
||||
|
||||
Branch `ui-rating` in `/home/thallada/workspace/the-daily-epub-ui2`, dev port 3602.
|
||||
A previous agent (Codex) landed a partial pass as `0cdc302` before hitting a quota
|
||||
limit and never saw the rendered result; `main` was merged in afterwards
|
||||
(`bcbe158`). This pass kept the parts of that work that held up on screen, fixed
|
||||
the parts that did not, and finished the remaining items.
|
||||
|
||||
## What changed
|
||||
|
||||
### `src/web/tailwind.css`
|
||||
|
||||
- **`.reader-section-heading`** — replaced Codex's sentence-case `text-xl` serif
|
||||
heading with a serif **uppercase, letter-spaced section mark** on a double rule:
|
||||
`border-t-[3px] border-double border-rule-strong pt-3 font-serif text-[1.05rem]
|
||||
font-semibold uppercase leading-[1.2] tracking-[0.15em]`. `text-wrap: balance`
|
||||
stays. (Rationale under *Design decisions*.)
|
||||
- **Rating buttons** — inactive labels are now `text-ink-2` (were `text-ink`) and
|
||||
go to `text-ink` on hover, so the group recedes next to serif body copy.
|
||||
- **Rating button hit area** — the `::after` overlay went from `-inset-y-1` /
|
||||
`-inset-y-1.5` (phone) to `-inset-y-1.5` / `-inset-y-2`. The pseudo is
|
||||
positioned against the *padding* box, so the old values gave 37 px desktop /
|
||||
41 px phone; the new values measure 41 px and 45 px (verified with
|
||||
`elementFromPoint`), clearing the ≥ 40 px / ≥ 44 px rules in the shared brief.
|
||||
- **Rating focus ring** — `.rating button[data-label]:focus-visible { z-20
|
||||
ring-offset-0 }`. In a segmented group the inherited `ring-offset-2` painted a
|
||||
detached red box over the neighbouring buttons; hugging the segment reads as
|
||||
one control.
|
||||
- **`.rating-note input`** — now a quiet bottom-hairline field everywhere
|
||||
(`rounded-none border-x-0 border-t-0 border-b border-rule bg-transparent px-0
|
||||
py-1 shadow-none focus:border-ink`) instead of a boxed input on reader/dashboard
|
||||
pages and a *transparent* bottom border in tables.
|
||||
- **Table/list overrides** — `td .rating` / `.picks .rating` lost the
|
||||
`max-w-[15rem]` cap (the cell now sets its own width) and gained `gap-y-1`;
|
||||
table buttons use `px-1.5` so the group fits; `.picks .rating` gets `mt-2` so
|
||||
the widget is not welded to the pick title.
|
||||
- **New `.rating-cell` block** — the ratings-table verdict cell is a one-column
|
||||
grid, `w-[15rem]`, `gap-y-1.5`: row 1 is the segmented group with `clear`
|
||||
pushed to the right edge (`ml-auto`), row 2 is the note field spanning the same
|
||||
width. `min-h-0` on `.rating-actions` inside the cell top-aligns the group with
|
||||
the row's article title (measured: 1 px apart).
|
||||
|
||||
### `src/web/dashboard/ratings.rs`
|
||||
|
||||
- Removed the now-dead `CurrentRow::verdict` field and its assignment (Codex
|
||||
dropped `{{ row.verdict }}` from the template but left the field, which fails
|
||||
`clippy -D warnings`). `badge` stays — it is still read by the "no embedding" /
|
||||
"—" branch in the template.
|
||||
|
||||
### `src/web/templates/world.html`, `src/web/templates/behind.html`
|
||||
|
||||
- Added `reader-page` to the `<article>` so the polish item's
|
||||
`text-wrap: balance` on reader headlines covers the World Briefing and Behind
|
||||
the paper chapters too. (Deviation — see below.)
|
||||
|
||||
### Kept from Codex's partial commit (verified on screen, unchanged)
|
||||
|
||||
`_rating_widget.html` (`.rating-actions` wrapper, `clear`/`cleared` label spans,
|
||||
`placeholder="Add a note…"`), `article.html` / `issue_full.html` /
|
||||
`issue_public.html` (`reader-page`, `reader-section-heading`, `index-summary`,
|
||||
`mt-14` section spacing, colophon and The Brief on the same heading system),
|
||||
`dashboard/ratings.html` (badge dropped from the verdict cell, `rating-cell`
|
||||
class), the per-verdict tints, `.prose-body img` outline, `.prose-body p` /
|
||||
`.index-summary` `text-wrap: pretty`, and `active:scale-[0.96]` with named
|
||||
transition properties on `.btn` / `.btn-primary` / `.btn-danger`.
|
||||
|
||||
### `src/web/static/app.css`
|
||||
|
||||
Rebuilt from `tailwind.css` (Tailwind 4.3.3).
|
||||
|
||||
## Design decisions
|
||||
|
||||
- **Section headers: uppercase mark, not a bigger headline.** The brief asked for
|
||||
both options to be screenshotted. A sentence-case serif `text-2xl` header
|
||||
(`after-rating/sec/C.png`) is the same size as the article headlines directly
|
||||
beneath it and reads as another story, which is the exact failure being fixed.
|
||||
The uppercase, letter-spaced serif mark on a double rule
|
||||
(`after-rating/sec/B.png`, `B2-*.png`) is unmistakably a section marker at a
|
||||
glance and never competes with a headline — the newspaper convention the rest
|
||||
of the design language is borrowing. It is also visibly *not* the old 0.72 rem
|
||||
sans eyebrow: serif, ink-coloured, ~50 % larger, and sitting on the same double
|
||||
rule as the masthead. No extra eyebrow was added; the header is already
|
||||
uppercase, so a second uppercase line above it would be noise.
|
||||
- **`clear` stays `invisible`, not `[hidden]`.** It keeps its layout box, so
|
||||
choosing a verdict does not shift the group sideways. `.rating:has(button
|
||||
.active) button.clear` reveals it; the cleared state swaps the label to a muted
|
||||
"cleared" with no underline, which doubles as the brief's "muted cleared state
|
||||
in the group" for the ratings table.
|
||||
- **Verdict cell is `w-[15rem]`, not `w-[13rem]`.** 13 rem is 208 px; the three
|
||||
labels plus `clear` measure ~219 px at `text-xs`/`px-1.5`, and the wider
|
||||
"cleared" label needs ~237 px. 15 rem (240 px) fits both states with the
|
||||
`clear`/`cleared` link flush right in each, so rows stay aligned whichever state
|
||||
they are in. Shrinking the type to ~0.7 rem would have hit 13 rem but is too
|
||||
small for a dense table that is otherwise 14 px.
|
||||
- **Note field shows a faint rule at rest.** The brief said "bottom-hairline only,
|
||||
no box" *and* "only shows a border on focus"; Codex read that as a transparent
|
||||
border, which left "Add a note…" floating with no affordance and no shared
|
||||
baseline across rows. A `--rule` hairline that goes to `--ink` on focus keeps
|
||||
the cell calm, gives the unit a base line, and still reads as quiet.
|
||||
- **`.prose-body h2`/`h3` left alone.** Checked against 1.0625 rem body copy
|
||||
(`prose-heading-light.png`): `text-2xl` semibold over a hairline is already a
|
||||
clear heading. The brief said to adjust only if weak.
|
||||
|
||||
## Deviations
|
||||
|
||||
- `world.html` and `behind.html` are not in this task's file list, but the polish
|
||||
item asks for `text-wrap: balance` on reader-page headlines and those two are
|
||||
reader pages. The change is one class per file and touches nothing else.
|
||||
- Verdict cell width 15 rem rather than the brief's example 13 rem (reason above).
|
||||
- Section header is `text-[1.05rem]` uppercase rather than `text-2xl`/`text-xl`
|
||||
sentence case (reason above).
|
||||
|
||||
## Verification
|
||||
|
||||
Dev server on `127.0.0.1:3602` against this worktree's own copy of the dev DB
|
||||
(`dev/config.toml` was repointed from the main checkout's `dev/` to
|
||||
`the-daily-epub-ui2/dev/` so rating clicks could not touch the shared database;
|
||||
`dev/` is git-ignored).
|
||||
|
||||
- `cargo fmt -- --check` — clean.
|
||||
- `cargo clippy --all-targets -- -D warnings` — clean.
|
||||
- `cargo test --lib web::` — 84 passed, 0 failed.
|
||||
- `cargo test` — 445 lib tests + all integration tests passed, 0 failed. (One
|
||||
earlier run had a single flaky failure in
|
||||
`config::tests::registry_validation_rejects_bad_roles_kinds_and_efforts`; it
|
||||
passes in isolation and on re-run, and is unrelated to this change — the host
|
||||
was building for three agents at once.)
|
||||
- `npm run css:check` — no diff (Tailwind 4.3.3 from the adjacent checkout's
|
||||
`node_modules/.bin` via `PATH`; only read, never modified).
|
||||
- **Live JS handler** (`interact.mjs`): clicked loved → good → not-for-me → clear
|
||||
→ loved on `/issues/2026-09-02/articles/4`. `.active` and `aria-pressed` follow
|
||||
every click, the tint switches, **0 page navigations**, console clean (no errors
|
||||
or warnings). Also typed into the note field and confirmed it round-trips into
|
||||
the ratings table's Note column.
|
||||
- **Hit areas** measured with `elementFromPoint`: 41 px desktop, 45 px phone.
|
||||
- **Top alignment** measured: verdict group top is 1 px from the article title top.
|
||||
|
||||
### Screenshots
|
||||
|
||||
Under `/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/553c7d77-d8f4-44f8-a379-73cedea23814/scratchpad/after-rating/`:
|
||||
|
||||
- `sec/B.png`, `sec/C.png`, `sec/B2-brief.png`, `sec/B2-colophon.png` — the
|
||||
section-header variant comparison behind the decision above.
|
||||
- `wip/*` — Codex's partial state before this pass (the overflowing verdict cell
|
||||
is `wip/ratings-cell-light.png`).
|
||||
- `final/` — the shipped result:
|
||||
- `/` index card + section: `index-card-light`, `index-card-mobile`,
|
||||
`index-card-mobile-dark`, `index-section-light`, `index-section-dark`,
|
||||
`index-section-mobile`
|
||||
- front-page heading system: `brief-light`, `brief-dark`, `colophon-light`,
|
||||
`colophon-dark`
|
||||
- anonymous issue page: `public-issue-light`, `public-issue-dark`
|
||||
- chapter footer: `article-footer-light`, `article-footer-mobile`
|
||||
- ratings table: `ratings-light`, `ratings-dark`, `ratings-mobile`,
|
||||
`ratings-mobile-dark`, `ratings-cleared-light`, `ratings-cleared-dark`
|
||||
- dashboard article widget: `dash-article-light`, `dash-article-dark`,
|
||||
`dash-article-mobile`
|
||||
- states: `rating-focus`, `rating-hover`, `overview-picks`
|
||||
- polish checks: `prose-heading-light`, `prose-img-light`, `prose-img-dark`
|
||||
|
||||
## Left open
|
||||
|
||||
- Nothing from this brief. The dev server is left running on 3602 against
|
||||
`the-daily-epub-ui2/dev/daily-epub.db`, which now holds 5 current verdicts
|
||||
(loved / good / not-for-me / good / loved, one with a note) plus the extra
|
||||
rating events those clicks appended — `rating_events` is append-only, so the
|
||||
history is longer than the seeded baseline.
|
||||
@@ -0,0 +1,159 @@
|
||||
# Task 3 handoff — site shell
|
||||
|
||||
## What changed
|
||||
|
||||
- `src/web/issue.rs`
|
||||
- `IssueView` now records `is_latest`, computed by `issue::load` from
|
||||
`Db::latest_issue_date()`.
|
||||
- The full issue, article, World Briefing, and Behind the paper renderers now
|
||||
select `latest` or `archive` navigation state from that flag.
|
||||
- Added a router test with two issue dates proving that only the newest issue
|
||||
marks Latest `aria-current="page"` and that an older issue marks Archive.
|
||||
- `src/web/public.rs`
|
||||
- Anonymous issue rendering uses the same `IssueView::is_latest` navigation
|
||||
state as signed-in issue rendering.
|
||||
- `src/web/templates/layout.html`
|
||||
- The default ears block is empty except on an admin dashboard route, where it
|
||||
says `Dashboard`. Issue/account/login/archive/error templates retain their
|
||||
explicit ears; the public empty issue retains its own `Morning edition`.
|
||||
- Added the italic Newsreader preload alongside the upright preload.
|
||||
- Named the site header for cross-document view transitions.
|
||||
- Made the ears-row theme and account/sign-in controls at least 40 px tall;
|
||||
the account/sign-in link also has a 40 px minimum width.
|
||||
- Dashboard navigation links now have a 40 × 40 px minimum target and use the same 2 px accent
|
||||
active underline, ink hover, and inherited focus-visible ring as site nav.
|
||||
- `src/web/templates/dashboard/run.html`
|
||||
- The Feeds table no longer uses `.scroll-x`. It is fixed-layout and full
|
||||
width, truncates feed names with the full value in `title`, and reserves a
|
||||
narrow fixed entries column. Timings and Provider usage retain their
|
||||
horizontal-scroll behavior.
|
||||
- `src/web/tailwind.css`
|
||||
- Both Newsreader faces use `font-display: block`.
|
||||
- Added automatic cross-document view transitions with a 140 ms root
|
||||
cross-fade. The named site header does not fade, and reduced-motion disables
|
||||
transition animation.
|
||||
- `.badge`, `.pager`, and `.pagination` use tabular numerals. (`.tile-num`
|
||||
already did; the brief said `.pager` did too, but the code did not, so it was
|
||||
corrected.)
|
||||
- `src/web/static/app.css`
|
||||
- Rebuilt from `tailwind.css` with Tailwind 4.3.3.
|
||||
- `src/web/static/speculation.json`
|
||||
- Added moderate same-origin prerender document rules for `/` and
|
||||
`/issues/*`, and moderate prefetch rules for `/dashboard` and
|
||||
`/dashboard/*`.
|
||||
- Logout, rating, static, and all query-string links are excluded. Excluding
|
||||
every query is deliberately conservative and therefore also excludes any
|
||||
query that could mutate state.
|
||||
- `src/web/mod.rs`
|
||||
- Serves the speculation rules with
|
||||
`application/speculationrules+json` and retains content-hash ETags.
|
||||
- All `/static/*` responses, including 304 responses, now send
|
||||
`Cache-Control: public, max-age=31536000, immutable`.
|
||||
- HTML responses send `Speculation-Rules: "/static/speculation.json"` while
|
||||
retaining their existing public/private/no-store cache behavior.
|
||||
- Added assertions for immutable cache headers, rules MIME type, both font
|
||||
preloads, the HTML speculation header, dashboard ears, and dashboard active
|
||||
underline.
|
||||
- `src/web/static/app.js`
|
||||
- A `[data-refresh]` job page now waits for `prerenderingchange` before
|
||||
scheduling reload, so a prerendered page cannot reload in the background.
|
||||
- `src/web/static/theme.js`
|
||||
- Continues to apply the saved theme and `has-js` in the head before first
|
||||
paint.
|
||||
- A short-lived mutation observer now restores persisted `details[id]` state
|
||||
as the parser creates disclosures, then disconnects at `DOMContentLoaded`.
|
||||
This prevents a cold/delayed `app.js` request from exposing the default
|
||||
disclosure state before restoration; `app.js` still owns toggle persistence.
|
||||
|
||||
## Design and implementation decisions
|
||||
|
||||
- Chose `font-display: block` rather than `optional`. This prevents a visible
|
||||
fallback-to-Newsreader swap and preserves the publication typeface. Preloading
|
||||
both faces plus immutable content-hashed assets limits the block cost to an
|
||||
uncached first visit.
|
||||
- Kept the 140 ms transition restrained and limited to document navigation.
|
||||
The masthead/nav swap immediately instead of fading, so navigation context
|
||||
stays visually anchored.
|
||||
- Added only platform progressive enhancements. There is no fetch/swap router,
|
||||
HTMX, or route/auth change.
|
||||
- Speculation Rules and cross-document view transitions are progressive:
|
||||
Chromium uses them; unsupported browsers ignore them and retain normal
|
||||
navigation.
|
||||
|
||||
## Flash reproduction and visual verification
|
||||
|
||||
The requested dev server could not start inside this sandbox:
|
||||
|
||||
```text
|
||||
$ DAILY_EPUB_SERVER__BIND=127.0.0.1:3603 cargo run -- --config ./dev/config.toml serve
|
||||
Error: could not bind 127.0.0.1:3603: Operation not permitted (os error 1)
|
||||
```
|
||||
|
||||
Therefore the throttled Playwright before/after capture and the requested light,
|
||||
dark, desktop, and 390 px screenshots could not be produced. The code audit
|
||||
found that theme and mobile TOC state were already applied by `theme.js` before
|
||||
paint. Persisted `details` state has now also moved into that pre-paint path.
|
||||
The other visible-swap candidate was Newsreader's `font-display: swap`; the
|
||||
preload/cache/block changes address that path.
|
||||
|
||||
The requested curl calls likewise could not connect:
|
||||
|
||||
```text
|
||||
$ curl -sS -D - -o /dev/null http://127.0.0.1:3603/
|
||||
curl: (7) Failed to connect to 127.0.0.1 port 3603 after 0 ms: Couldn't connect to server
|
||||
|
||||
$ curl -sS -D - -o /dev/null http://127.0.0.1:3603/static/app.css
|
||||
curl: (7) Failed to connect to 127.0.0.1 port 3603 after 0 ms: Couldn't connect to server
|
||||
```
|
||||
|
||||
Router tests verify the material headers that those calls should show:
|
||||
|
||||
```text
|
||||
GET /
|
||||
content-type: text/html; charset=utf-8
|
||||
cache-control: public, max-age=300
|
||||
speculation-rules: "/static/speculation.json"
|
||||
vary: Cookie
|
||||
|
||||
GET /static/app.css
|
||||
content-type: text/css; charset=utf-8
|
||||
cache-control: public, max-age=31536000, immutable
|
||||
etag: "0b00513092d50175d3615cfd6cdfc5fed6e416f2aea118c3b0f0110a66089036"
|
||||
```
|
||||
|
||||
Signed-in issue responses remain `private, no-store`; dashboard responses
|
||||
remain `no-store`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt -- --check` — passed.
|
||||
- `cargo clippy --all-targets -- -D warnings` — passed.
|
||||
- `npm run css:check` — passed with no diff using the repository's pinned
|
||||
Tailwind 4.3.3 binary. (`npm ci` could not finish under the restricted npm
|
||||
environment, so the already-installed sibling-worktree binary was placed on
|
||||
`PATH`; that worktree was only read, never modified.)
|
||||
- `python3 -m json.tool src/web/static/speculation.json` — passed.
|
||||
- `node --check src/web/static/app.js` and `node --check
|
||||
src/web/static/theme.js` — passed.
|
||||
- `cargo test --lib web::` — 84 passed, 0 failed.
|
||||
- Dedicated newest/archive navigation test — passed.
|
||||
- Dedicated dashboard settings shell/ears/active-nav test — passed.
|
||||
- `cargo test` — 432 library tests passed; 13 tests failed solely because their
|
||||
mock HTTP servers could not bind loopback (`Operation not permitted`). Ten are
|
||||
the sandbox failures named in the shared brief; three newer OpenAI mock-server
|
||||
tests fail at the same listener helper for the same reason.
|
||||
- `cargo test --lib` with those exact 13 bind-dependent tests skipped — 432
|
||||
passed, 0 failed, 13 filtered out.
|
||||
- Non-server integration tests (`config_check`, `e2e_pipeline`, `m2_pipeline`,
|
||||
`m3_curation`, `m4_epub`) — 25 passed, 0 failed.
|
||||
- `cargo test --doc` — passed (0 doctests).
|
||||
- `git diff --check` — passed.
|
||||
|
||||
## Left open
|
||||
|
||||
- Run the four requested screenshot sets and throttled first-frame comparison
|
||||
outside the sandbox, where port 3603 can bind. This is also the remaining
|
||||
browser-level validation for view-transition behavior and Chrome's
|
||||
Speculation Rules panel.
|
||||
- `tests/m7_server.rs` was not run because it is explicitly excluded in this
|
||||
loopback-restricted sandbox.
|
||||
@@ -0,0 +1,89 @@
|
||||
# Task 1 handoff — table-of-contents sidebar
|
||||
|
||||
## What changed
|
||||
|
||||
- `src/web/templates/_toc.html`
|
||||
- Reworked the narrow-screen panel to use a measured viewport offset and a
|
||||
`100dvh`-based height instead of the old `72vh` cap.
|
||||
- Added live label, count, desktop-status, link-label, and per-link progress
|
||||
hooks used by the shared TOC state updater.
|
||||
- Moved current-link presentation onto `.toc-link[aria-current]` and kept the
|
||||
server-rendered `aria-current="page"` state for chapter pages.
|
||||
- Added both hamburger and close SVGs for the open-state cross-fade and raised
|
||||
the mobile toggle/link hit areas to at least 44 px.
|
||||
- Added explicit right padding around the scrollable list.
|
||||
- `src/web/templates/issue_full.html`
|
||||
- Marked The Brief, every article index item, and the colophon with matching
|
||||
`data-toc-entry` values for the front-page scroll-spy.
|
||||
- `src/web/static/app.js`
|
||||
- Added mobile panel sizing from the sticky bar's current viewport position,
|
||||
recalculated on open and layout/visual-viewport resize.
|
||||
- Added mobile document scroll lock and retained toggle, Escape,
|
||||
outside-click, link-click, and `lg` breakpoint close behavior.
|
||||
- Added a passive, requestAnimationFrame-throttled front-page scroll-spy. A
|
||||
single current-link update now drives `aria-current="location"`, the mobile
|
||||
title/count, desktop chapter status, both progress bars, and TOC reveal.
|
||||
- Updated current-link reveal to use `scrollTo`, smooth unless reduced motion
|
||||
is requested, without scrolling the page.
|
||||
- Preserved progress-within-chapter behavior on article pages.
|
||||
- `src/web/tailwind.css`
|
||||
- Added the dynamic mobile panel height and scroll-lock rules.
|
||||
- Added stable scrollbar gutter, right-side spacing, thin/rule-colored
|
||||
cross-browser scrollbar styling, and overscroll containment.
|
||||
- Added the centralized current/hover/focus link styles and exact 150 ms
|
||||
opacity/scale/blur hamburger-to-close transition.
|
||||
- `src/web/static/app.css`
|
||||
- Rebuilt from the Tailwind source.
|
||||
- `src/web/issue.rs`
|
||||
- Added each TOC item's semantic progress position (including back matter) and
|
||||
coverage for those positions and the rendered scroll-spy hooks.
|
||||
- Made the current-link router assertion insensitive to HTML attribute order.
|
||||
|
||||
## Design decisions
|
||||
|
||||
- The open panel remains positioned directly below the shared sticky bar. JS
|
||||
measures only the bar's on-screen bottom edge into `--toc-panel-top`; CSS owns
|
||||
the final `calc(100dvh - var(--toc-panel-top))` height. This handles both the
|
||||
bar still below the masthead and the bar already stuck at the top.
|
||||
- The Brief uses progress position 0. Article, World, and Behind entries use
|
||||
their chapter position. The colophon uses the issue total, so reaching it
|
||||
communicates completion even though World and Behind are separate pages.
|
||||
- Front-page position changes only toggle `aria-current` for current styling;
|
||||
the other text/progress surfaces are derived from that same link's label and
|
||||
progress metadata.
|
||||
- Scrollbar gutter and padding are both used: the gutter reserves space for
|
||||
classic scrollbars, while padding protects labels on overlay-scrollbar
|
||||
platforms.
|
||||
|
||||
## Deviations and open items
|
||||
|
||||
- No live screenshots could be captured. The sandbox rejected the assigned
|
||||
`127.0.0.1:3601` bind with `Operation not permitted`, exactly as anticipated
|
||||
by the shared brief. Consequently desktop/mobile, light/dark, open-panel, and
|
||||
scripted-scroll visual states remain to be checked outside the sandbox.
|
||||
- The full unit test run also found three OpenAI mock-server tests blocked by
|
||||
loopback binding in addition to the bind-dependent tests listed in the shared
|
||||
brief. They fail at the same listener creation point and are unrelated to this
|
||||
change.
|
||||
|
||||
## Verification
|
||||
|
||||
- `node --check src/web/static/app.js` — passed.
|
||||
- `cargo fmt --check` — passed.
|
||||
- `npm run css` — passed with Tailwind 4.3.3; `src/web/static/app.css` rebuilt.
|
||||
- `npm run css:check` — passed with no diff. The worktree lacked its own CLI
|
||||
shim, so both npm commands used the already-installed adjacent checkout's
|
||||
`node_modules/.bin` via `PATH`; all inputs and outputs stayed in this worktree.
|
||||
- `cargo test --lib web::` — 83 passed, 0 failed.
|
||||
- `cargo clippy --all-targets -- -D warnings` — passed.
|
||||
- `cargo test` — library phase: 431 passed, 13 bind-dependent failures. The
|
||||
failures were 7 LLM mock-server tests, the documented relative-URL extraction
|
||||
test, and the documented 5 server tests; each failed with `Operation not
|
||||
permitted` while creating a loopback listener.
|
||||
- `cargo test --test config_check --test e2e_pipeline --test m2_pipeline --test m3_curation --test m4_epub`
|
||||
— 25 passed, 0 failed. `m7_server` was excluded because it requires the same
|
||||
forbidden loopback bind.
|
||||
- Attempted the configured dev-server command on port 3601 — build passed; bind
|
||||
failed with `Operation not permitted`, so screenshots were not possible.
|
||||
- `git diff --check` — passed.
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# Shared brief — web dashboard v2 (design pass + fixes)
|
||||
|
||||
The web site and operator dashboard planned in `docs/plans/2026-09-03-web-dashboard.md`
|
||||
is implemented and deployed (see `docs/plans/briefs/web-dashboard/handoff-step*.md`).
|
||||
This second pass, requested by the operator on 2026-09-03, does six things:
|
||||
|
||||
1. Replaces the hand-written `src/web/static/app.css` with a **Tailwind CSS v4**
|
||||
setup and a fresh, more refined design that keeps the simple newspaper feel.
|
||||
2. Adds a **light/dark theme** that follows the system by default and can be
|
||||
overridden by the reader (persisted in `localStorage`).
|
||||
3. **Public** issue pages now show each article's AI summary and its
|
||||
"Why it's here" line (a deliberate reversal of plan §7.1's "no generated text";
|
||||
article bodies, The Brief, the World Briefing and comments stay private).
|
||||
4. Signed-in issue pages get a **table-of-contents sidebar** (chapters = articles)
|
||||
that collapses into a hamburger menu on mobile and shows where the reader is.
|
||||
5. Signed-in issue pages always offer the **World Briefing** and **Behind the paper**
|
||||
chapters, including for issues published before `issues.issue_json` existed.
|
||||
6. The **Colophon** on those legacy issues shows real numbers instead of zeros.
|
||||
|
||||
Items 5–6 have one root cause: `web::issue::load` falls back to a reduced
|
||||
`Issue` (default `Colophon`, no `world_briefing`, default `BehindThePaper`,
|
||||
`from_json == false`) whenever `issues.issue_json` is NULL — which is every
|
||||
issue published before migration `0004_web.sql` ran (2026-09-03).
|
||||
|
||||
Each step below has its own brief (`01`…`04`). Read the original plan's §4, §7,
|
||||
§8 and §16 and `docs/plans/2026-08-15-implementation-notes.md` for conventions:
|
||||
runtime sqlx queries with manual row mapping, `jiff` timestamps, `thiserror`/`anyhow`
|
||||
errors, askama 0.16 templates, no network in tests, rustfmt defaults, no `unwrap()`
|
||||
outside tests.
|
||||
|
||||
## Ground rules
|
||||
|
||||
- You work in the git worktree named in your brief. **Never touch the main
|
||||
checkout** at `/home/thallada/workspace/the-daily-epub`. Finish by committing
|
||||
**one** commit on your worktree's branch (message prefix `Web dashboard v2 step N:`).
|
||||
- Do not edit migrations. Do not change routes, auth, or the security headers
|
||||
(`web::security_headers`) — in particular the CSP stays
|
||||
`default-src 'self'; img-src * data:; style-src 'self'; script-src 'self'; …`,
|
||||
so **no inline `<style>` or `<script>`, no `style=""` attributes, no CDN**.
|
||||
Fonts and scripts are served from `/static/…` (`include_str!`/`include_bytes!`).
|
||||
- Every `|safe` in a template must be one of the sanitized inputs listed in plan
|
||||
§16 (plus, after step 1, the world chapter body recovered from our own EPUB).
|
||||
- Keep every hook `src/web/static/app.js` relies on: `form.rating`,
|
||||
`button[data-label]`, the `.active` class and `aria-pressed` on rating
|
||||
buttons, `form[data-confirm]`, `details[id]`, `table[data-filter]`,
|
||||
`.table-filter`, `button[data-reset]`/`data-default`, `[data-refresh]`,
|
||||
`.scroll-x`. Keep all `id`, `name`, `action` and `aria-*` attributes that
|
||||
handlers or tests use. Router tests assert on text and some class names; when
|
||||
a test breaks only because a purely cosmetic class was renamed, update the test.
|
||||
- Tests: unit tests inline; router tests with `tower::ServiceExt::oneshot`
|
||||
against `server::router(...)` over a temp DB, never binding a socket.
|
||||
- Keep `cargo fmt`, `cargo clippy --all-targets -- -D warnings` and `cargo test`
|
||||
green. Finish with those three commands and report their output.
|
||||
- **Sandbox note:** inside the Codex sandbox `bind()` on 127.0.0.1 is forbidden,
|
||||
so exactly these pre-existing tests fail there and must be ignored:
|
||||
`curate::llm::tests::anthropic_*` (4), `extract::tests::relative_urls_resolve_against_the_url_we_landed_on`,
|
||||
`server::tests::*` (5 that bind a listener), and `tests/m7_server.rs`. Do not
|
||||
"fix" them. The orchestrator runs the full suite outside the sandbox. If you
|
||||
cannot run the dev server for the same reason, rely on router tests and on
|
||||
rendering templates in tests; say so in the handoff.
|
||||
- This host has 4 cores and 7 GB; another agent builds Rust at the same time.
|
||||
Prefer `cargo test --lib web::` style narrow runs while iterating and one full
|
||||
`cargo test` at the end.
|
||||
- Don't gold-plate. Where this brief and the code disagree, follow the code and
|
||||
note the deviation in your handoff.
|
||||
- When you finish, write `docs/plans/briefs/web-dashboard-v2/handoff-step<N>.md`:
|
||||
what landed, deviations and why, anything left for later, test counts, and
|
||||
(for design steps) which pages you screenshotted.
|
||||
|
||||
## Previewing the site locally
|
||||
|
||||
`examples/seed_dev_db.rs` builds a throwaway database with two issues (No. 18 on
|
||||
2026-09-01 is a *legacy* issue without `issue_json`; No. 19 on 2026-09-02 has the
|
||||
full snapshot), nine articles across four sections, an `admin` and a `reader`:
|
||||
|
||||
```text
|
||||
cargo run --example seed_dev_db -- ./dev # ./dev is git-ignored
|
||||
DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) \
|
||||
cargo run -- --config ./dev/config.toml serve # http://127.0.0.1:3599
|
||||
```
|
||||
|
||||
Accounts: `admin` / `adminpassword123` (dashboard, ratings), `reader` /
|
||||
`readerpassword123` (full issues only). Anonymous visitors get the public pages.
|
||||
Port 3499 belongs to the production server on this host — never use it.
|
||||
|
||||
Screenshots: a Playwright script that signs in, sets the theme and shoots a
|
||||
list of paths lives at
|
||||
`/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/e963db53-510f-4312-ac87-460291a92781/scratchpad/shoot/shoot.mjs`
|
||||
(`node shoot.mjs <outdir> [--user admin|reader|none] [--theme light|dark] [--mobile] [--full] /path …`,
|
||||
`BASE=http://127.0.0.1:3599`). If that path is unreadable from your sandbox, the
|
||||
bare headless Chromium at
|
||||
`~/.cache/ms-playwright/chromium_headless_shell-1208/chrome-headless-shell-linux64/chrome-headless-shell --headless --no-sandbox --screenshot=out.png --window-size=1280,1400 URL`
|
||||
works for anonymous pages. **Look at your screenshots** (the Read tool renders
|
||||
PNGs) before calling a design step done.
|
||||
|
||||
## Design language (steps 2, 3 and 4 must follow this)
|
||||
|
||||
The look: a quiet, well-set morning paper. Warm off-white paper, near-black ink,
|
||||
one red accent, hairline rules instead of boxes, generous whitespace, real
|
||||
typographic hierarchy. Reader pages are serif; the dashboard is a sans-serif
|
||||
instrument panel that shares the same paper, ink, rules and accent so the two
|
||||
halves feel like one publication.
|
||||
|
||||
**Tailwind v4 conventions.** One source file `src/web/tailwind.css`:
|
||||
`@import "tailwindcss"; @source "../web/templates"; @source "./static/app.js";`
|
||||
(paths relative to the file — check they resolve), design tokens in `@theme`,
|
||||
the dark variant as `@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *));`.
|
||||
Colours are **semantic CSS variables swapped per theme**, exposed to Tailwind
|
||||
through `@theme inline`, so templates write `bg-paper text-ink border-rule`
|
||||
once and both themes work without `dark:` on every element:
|
||||
|
||||
```css
|
||||
:root { --paper:#f6f3ec; --paper-2:#ece7db; --ink:#1c1b18; --ink-2:#4f4b43; --muted:#7a7568;
|
||||
--rule:#d8d2c4; --rule-strong:#1c1b18; --accent:#a3231f; --accent-hover:#7c1a16;
|
||||
--loved:#2f6f46; --good:#2f6a8f; --down:#9c3f36; --warn:#8a6d1f; color-scheme:light; }
|
||||
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) { /* dark values */ } }
|
||||
:root[data-theme="dark"] { --paper:#151513; --paper-2:#1e1d1a; --ink:#ebe6da; --ink-2:#bfb9aa; --muted:#8f8a7d;
|
||||
--rule:#3a3833; --rule-strong:#ebe6da; --accent:#ef8c82; --accent-hover:#f7aaa2;
|
||||
--loved:#7fcb92; --good:#7fb8d8; --down:#ea8f84; --warn:#d9b45a; color-scheme:dark; }
|
||||
@theme inline { --color-paper: var(--paper); --color-paper-2: var(--paper-2); --color-ink: var(--ink); … }
|
||||
```
|
||||
|
||||
Put the dark values in **both** the media block and the attribute block (a
|
||||
small `@mixin`-free duplication is fine). Tinted badge backgrounds use
|
||||
`color-mix(in oklab, var(--loved) 14%, transparent)`.
|
||||
|
||||
**Type.** Reader pages: *Newsreader* (variable, optical size axis, self-hosted
|
||||
from `src/web/static/fonts/Newsreader.woff2` + `Newsreader-italic.woff2`,
|
||||
`font-display: swap`, fallback `"Iowan Old Style", "Palatino Linotype", Georgia, serif`,
|
||||
`font-optical-sizing: auto`). Body `1.0625rem/1.65`, max measure `max-w-[68ch]`.
|
||||
Headlines weight 600, `leading-[1.1]`, `tracking-[-0.01em]`: article h1
|
||||
`text-4xl md:text-5xl`; index headlines `text-2xl`, lead story `text-3xl`.
|
||||
UI chrome (nav, datelines, bylines, meta lines, labels, buttons, forms, the
|
||||
whole dashboard): the system sans stack `font-sans` (`ui-sans-serif, system-ui,
|
||||
-apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif`), small
|
||||
labels `text-[0.72rem] uppercase tracking-[0.12em] text-muted`. Code:
|
||||
`ui-monospace, SFMono-Regular, Menlo, monospace`. Tabular numbers in tables.
|
||||
|
||||
**Masthead and chrome.** Centred masthead "The Daily EPUB" in Newsreader 600,
|
||||
`text-5xl` (`text-3xl` on phones), `tracking-[-0.02em]`, a 1px rule above and a
|
||||
3px double rule below. An "ears" row above it (sans, tiny): the dateline /
|
||||
issue number on the left, the theme toggle and sign-in/account on the right.
|
||||
Primary nav below the masthead as a rule-bounded row of uppercase tracked
|
||||
links; the active link (`aria-current="page"`) carries a 2px accent underline.
|
||||
The admin nav is a second, denser row in the same style. Footer: hairline rule,
|
||||
tiny muted sans. The theme toggle is one button cycling *System → Light → Dark*
|
||||
(icon + short label, `aria-label`), no dropdown.
|
||||
|
||||
**Reader pages.** Section headings are tracked uppercase sans labels sitting on
|
||||
a rule, not big serif headings. The Brief gets a drop cap on its first paragraph
|
||||
(`first-letter:` utilities, Newsreader 600, ~3 lines tall) — and nothing else
|
||||
gets a drop cap. Each index entry: headline (link, no underline, hover accent),
|
||||
`source · N min` meta in sans, the summary in serif at body size, then the
|
||||
"Why it's here" line in italic with a 2px accent bar on its left (`border-l-2
|
||||
border-accent pl-3`), then the rating control for admins. Entries are separated
|
||||
by hairlines; the lead story is set larger. Article pages: h1 linking to the
|
||||
source, byline/meta in sans, the summary as an italic standfirst, the body with
|
||||
comfortable paragraph spacing, images `rounded-sm` with a caption-style alt
|
||||
fallback, code blocks on `bg-paper-2`, blockquotes with a rule on the left,
|
||||
`prev/next` as two bordered cards with "Previous"/"Next" labels. Download links
|
||||
are `.btn` buttons. Links inside prose: ink text, `underline decoration-rule
|
||||
underline-offset-4 hover:decoration-accent`.
|
||||
|
||||
**Controls.** `.btn` (sans `text-sm`, 1px `border-rule-strong`, `px-3 py-1.5`,
|
||||
hover fills ink/paper, `focus-visible:ring-2 ring-accent ring-offset-2
|
||||
ring-offset-paper`), `.btn-primary` filled ink. The rating widget is a
|
||||
segmented control: buttons joined edge to edge, the active verdict filled ink
|
||||
on paper, "clear" a muted text button. Inputs: `bg-paper border-rule rounded-sm
|
||||
px-2.5 py-1.5 focus:border-ink`. Flash: `border-l-4 border-accent bg-paper-2`.
|
||||
Badges: pill, tinted background + coloured text per state. Transitions 150ms on
|
||||
colour only; honour `prefers-reduced-motion`.
|
||||
|
||||
**Dashboard.** `max-w-7xl`, sans throughout, a page header (title `text-2xl
|
||||
font-semibold`, one-line description, actions on the right), stat tiles (big
|
||||
tabular number, small label, optional muted delta), tables `text-sm` with sticky
|
||||
`thead` on `bg-paper`, hairline row rules, `hover:bg-paper-2`, right-aligned
|
||||
`tabular-nums` numerics, every table inside `.scroll-x`; filters as one inline
|
||||
form of labelled controls; funnels and sparklines keep their SVG classes.
|
||||
Settings keep their sticky save bar. Nothing boxed unless it is a card with a
|
||||
purpose (stat tiles, job cards, settings groups).
|
||||
|
||||
**Layout components shared by reader and dashboard** live in
|
||||
`@layer components` in `tailwind.css`: `.btn`, `.btn-primary`, `.badge` and its
|
||||
state colours, `.kv` (definition-list grid), `.scroll-x`, `.prose-body`
|
||||
(article/world/discussion body styling), base `table`/`th`/`td`, form controls.
|
||||
Prefer utilities in templates; use a component class only when the same
|
||||
composite repeats in three or more places or when server-rendered HTML we do
|
||||
not control (article bodies, discussions, world chapter) needs styling.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Step 1 — backend data: public summaries, legacy issue fallback
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-v2a` (branch `v2-backend`).
|
||||
Read `00-shared.md` first. This step is Rust plus tiny template edits; a design
|
||||
step runs in parallel in another worktree and will restyle every template, so
|
||||
keep your template changes minimal and semantic (no new CSS, no new classes
|
||||
beyond what you need for tests).
|
||||
|
||||
## A. Public issue pages show summary and "why" (shared brief item 3)
|
||||
|
||||
- `src/web/public.rs`: add `summary: Option<String>` and `why: Option<String>`
|
||||
to `PublicEntry`, filled by `PublicIssue::from(&Issue)` the same way the
|
||||
signed-in index does (`web::issue::summary_for`-style: `pick.summary`, else
|
||||
`editorial.summaries[article_id]`, trimmed, empty → `None`; `why = pick.why`).
|
||||
- `issue_public.html`: after the byline, render `<p class="summary">…</p>` when
|
||||
present and `<p class="why"><em>Why it's here: …</em></p>` when present.
|
||||
`feed_entry.html` (the Atom `<content>`): include the summary and the why line
|
||||
per entry too, so feed readers see the same public presentation.
|
||||
- Tests: rewrite `public_issue_carries_no_generated_text` (and any sibling
|
||||
asserting summaries/why are absent, including in the feed tests) into
|
||||
`public_issue_shows_summaries_and_why_but_no_bodies`: summaries and why lines
|
||||
present; The Brief text, every article body sentence, the World Briefing text
|
||||
and every comment string still absent. Keep the existing assertions that
|
||||
titles, authors, sources and comment links are present.
|
||||
- Docs: in `docs/plans/2026-09-03-web-dashboard.md` §7.1 and §21 add a dated
|
||||
note ("2026-09-03: the operator chose to publish summaries and why lines;
|
||||
bodies/Brief/World/comments remain private"). Update `README.md` if it
|
||||
describes the public page.
|
||||
|
||||
## B. Legacy issues: rebuild colophon, Behind the paper and the World Briefing (items 5–6)
|
||||
|
||||
Context: `web::issue::load` (src/web/issue.rs) takes the fallback branch when
|
||||
`issues.issue_json` is NULL. Production has such issues (everything before
|
||||
2026-09-03). What *does* exist for them:
|
||||
|
||||
- `issues.report_json` (column since `0001_init.sql`) — the run's
|
||||
`report::RunReport` (counts, `provider_costs`, `cost_usd`, `voyage_cost_usd`,
|
||||
`config_json` from `pipeline::resolved_run_config`, `started_at`/`finished_at`,
|
||||
timings). Verify which model names `resolved_run_config` records and read
|
||||
them from there; if it lacks a name, fall back to the current `Config`
|
||||
(`state.config`) the way `pipeline.rs` builds `Models` (look at how
|
||||
`curator.llms.bulk/editor` and `summary_model` are resolved) and say so in a
|
||||
`generator_version`-style note only if unavoidable.
|
||||
- `runs` row(s) for the date (`entries_fetched`, `candidates`, `selected`,
|
||||
tokens, `cost_usd`, `status`, `provider_costs_json`, `config_json`) — use the
|
||||
latest finished run for the date when `issues.report_json` is NULL.
|
||||
- `candidate_runs` rows for that run (curation v2) — `curate::telemetry::paper_near_misses(db, run_id, 10)`
|
||||
gives the near misses exactly as the pipeline does (`pipeline::behind_the_paper`).
|
||||
- The EPUB on disk (`issues.epub_path`, else `publish::issue_filename(date, Edition::Standard, "epub")`
|
||||
under `config.publish.epub_dir`) contains `OEBPS/world.xhtml` when the issue
|
||||
had a World Briefing, and `OEBPS/colophon.xhtml` / `OEBPS/behind.xhtml`.
|
||||
|
||||
Implement, in the fallback branch of `load` (or a helper module
|
||||
`src/web/legacy.rs` called from it):
|
||||
|
||||
1. **Colophon**: `Colophon { provider_costs, models, entries_fetched, feeds_seen,
|
||||
candidates, cost_usd, generator_version }` from the report (or run row). Where
|
||||
a number is genuinely unknown leave 0 but make the template say "n/a" rather
|
||||
than "0 from 0 feeds" — add `Option`s to `ColophonView` where needed and
|
||||
render "n/a". `generated_at` already comes from the `issues` row.
|
||||
2. **Behind the paper**: `BehindThePaper` built like `pipeline::behind_the_paper`
|
||||
from the report counts + `paper_near_misses` + models + total cost +
|
||||
`generation_secs` from `started_at`/`finished_at`. Set `has_behind` in
|
||||
`issue_full` to "a report or run exists", not `from_json`, and make
|
||||
`GET /issues/{date}/behind` render for legacy issues.
|
||||
3. **World Briefing**: when `world_briefing` is `None` and the EPUB exists, open
|
||||
it with the `zip` crate (already a dependency), read `OEBPS/world.xhtml`, and
|
||||
take the inner HTML of `<body>` minus the chapter's own leading `<h1>` (the
|
||||
template supplies the heading). Store it on `IssueView` as
|
||||
`world_html: Option<String>` (or an enum `WorldSource { Briefing(WorldBriefing), Recovered(String) }`)
|
||||
and have the `world` handler render either `world::render_xhtml(&briefing)` or
|
||||
the recovered body `|safe` — it is our own sanitized XHTML, note that in the
|
||||
§16 list in the plan. Set `has_world` accordingly. Missing/corrupt EPUB → no
|
||||
world chapter, never an error (log at debug).
|
||||
4. Keep the JSON branch untouched. Do not backfill `issue_json` in the database.
|
||||
|
||||
Also make the dev seed mirror production: in `examples/seed_dev_db.rs` the
|
||||
legacy issue (2026-09-01) must end with `issues.issue_json = NULL`, an
|
||||
`issues.report_json` present, and — so the world fallback can be exercised — a
|
||||
real EPUB for it written into `dev/epubs/` with the fixture World Briefing
|
||||
(`epub::build_edition`/`build_all` or whatever the builder's public entry is;
|
||||
read `src/epub/mod.rs`) and its path stored in `issues.epub_path`. Add a few
|
||||
`candidate_runs` rows for the legacy run via `db` helpers or SQL so near
|
||||
misses render (look at `web::dashboard::tests::seed()` for the shape of
|
||||
`signals_json`).
|
||||
|
||||
Tests (router tests over a temp DB; see the existing `seeded_issue(false)`
|
||||
harness in `src/web/issue.rs`): fallback colophon shows the report's counts and
|
||||
models and no "0 from 0 feeds"; `/issues/{date}/behind` renders for the legacy
|
||||
issue with the near misses; `/issues/{date}/world` renders the recovered
|
||||
chapter from a small EPUB built in the test with `epub::build` on the fixture
|
||||
issue; `/issues/{date}` links to both chapters; `PublicIssue` never gains a
|
||||
body/brief/world field (extend the existing negative test).
|
||||
|
||||
## Handoff
|
||||
|
||||
`docs/plans/briefs/web-dashboard-v2/handoff-step1.md`. Commit once on
|
||||
`v2-backend`: `Web dashboard v2 step 1: public summaries, legacy issue fallback`.
|
||||
@@ -0,0 +1,100 @@
|
||||
# Step 2 — Tailwind v4 design system, theme toggle, reader pages
|
||||
|
||||
Worktree: `/home/thallada/workspace/the-daily-epub-v2b` (branch `v2-frontend`).
|
||||
Read `00-shared.md` first, especially the design language — it is the spec.
|
||||
You own the design system and the **reader-facing** templates. Do not restyle
|
||||
`src/web/templates/dashboard/*` (step 3 does that, on top of your system); they
|
||||
may look plain until then, which is expected. Do not implement the sidebar
|
||||
table of contents (step 4).
|
||||
|
||||
## 1. Tailwind toolchain (no Node at build or run time)
|
||||
|
||||
- `package.json` at the repo root with `devDependencies`
|
||||
`tailwindcss` and `@tailwindcss/cli` pinned to `4.3.3`, scripts
|
||||
`css` (`tailwindcss -i src/web/tailwind.css -o src/web/static/app.css --minify`),
|
||||
`css:watch`, and `css:check` (build to a temp file and `diff` against the
|
||||
committed `app.css`, non-zero on drift). `package-lock.json` committed;
|
||||
`node_modules/` and `dev/` added to `.gitignore`.
|
||||
- The compiled `src/web/static/app.css` is **committed** and embedded with
|
||||
`include_str!` exactly as today, so `cargo build` and the server never need
|
||||
Node. Document the workflow (edit templates/`tailwind.css` → `npm run css` →
|
||||
commit both) in `README.md` and in `docs/runbooks/web-dashboard-rollout.md`.
|
||||
- Fonts: `src/web/static/fonts/Newsreader.woff2`, `Newsreader-italic.woff2` and
|
||||
`OFL.txt` are already in your worktree. Serve them from
|
||||
`web::static_asset` (`include_bytes!`, `font/woff2`, same ETag/304 logic,
|
||||
long cache) — extend the route so `/static/fonts/Newsreader.woff2` or
|
||||
`/static/Newsreader.woff2` works (pick one, keep `safe` matching: a fixed
|
||||
match arm per file, no path joins). `@font-face` in `tailwind.css` with
|
||||
`font-weight: 200 800; font-stretch: normal; font-display: swap;` and the
|
||||
`unicode-range` for latin from Google's CSS
|
||||
(`U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD`).
|
||||
Add a `<link rel="preload" as="font" type="font/woff2" crossorigin>` for the
|
||||
regular face in the layout.
|
||||
|
||||
## 2. Theme toggle (item 2)
|
||||
|
||||
- `src/web/static/theme.js` (new, served like `app.js`, `include_str!`): a
|
||||
handful of lines run **synchronously in `<head>`** before paint: read
|
||||
`localStorage.theme` (`"light"`/`"dark"`, anything else = system), set
|
||||
`document.documentElement.dataset.theme` accordingly (remove the attribute for
|
||||
system). Wrap storage access in try/catch. No inline script (CSP).
|
||||
- The toggle button in the layout's "ears" row: `<button type="button"
|
||||
data-theme-toggle aria-label="Theme: system">` with an inline SVG icon (sun /
|
||||
moon / auto) and a short visible label. `app.js` handles clicks: cycle
|
||||
system → light → dark → system, persist (`removeItem` for system), update the
|
||||
attribute, icon and label, and re-evaluate on `matchMedia("(prefers-color-scheme: dark)")`
|
||||
changes while in system mode. The three icons can all be in the markup with
|
||||
`hidden` toggled — remember `[hidden]` must win over any `display` utility.
|
||||
- Everything themed goes through the semantic tokens; `color-scheme` follows
|
||||
the theme so native controls and scrollbars match. Check both themes in
|
||||
screenshots: contrast of muted text on paper, accent on dark paper, rating
|
||||
buttons, flash, focus rings, images in dark mode.
|
||||
|
||||
## 3. Design system + layout
|
||||
|
||||
`src/web/tailwind.css` per the shared brief (tokens, dark variant, fonts,
|
||||
`@layer base` for body/typography defaults, `@layer components` for `.btn`,
|
||||
`.btn-primary`, `.badge` + states (`selected loved good down excluded
|
||||
shortlisted admitted eligible cleared assessed triaged reason running requested
|
||||
ok degraded failed dry_run`), `.kv`, `.scroll-x`, `.prose-body`, base tables
|
||||
and form controls — step 3 will consume these). Rewrite `layout.html`: ears
|
||||
row, masthead, primary nav with `aria-current="page"` derived from
|
||||
`page.active_nav` (check what values handlers set), the admin nav row, flash,
|
||||
`<main>`, footer, scripts. Add an accessible "skip to content" link.
|
||||
|
||||
## 4. Reader templates (restyle in place; keep every handler field)
|
||||
|
||||
`issue_public.html`, `issue_full.html`, `issue_list.html`, `article.html`,
|
||||
`world.html`, `behind.html`, `login.html`, `account.html`, `error.html`,
|
||||
`_rating_widget.html`. The public and full issue pages are the showcase — set
|
||||
them like a front page: dateline, stats line, The Brief with its drop cap,
|
||||
download buttons, sections as tracked labels on rules, entries per the spec,
|
||||
lead story larger, the chapter links (World Briefing · Behind the paper) as a
|
||||
centred rule-bounded row, the colophon as a two-column `.kv` in small sans.
|
||||
Note: the backend step adds `summary`/`why` to the public entries and it will
|
||||
merge after you; leave a clear place for them in `issue_public.html` (the same
|
||||
summary/why treatment as the full issue) guarded with `{% if let Some(...) %}`
|
||||
**only if the field exists in your tree** — otherwise just leave a comment
|
||||
`{# step 1 adds summary/why here #}` so the merge is trivial.
|
||||
|
||||
Article page: h1 → source link, byline/meta, "Why it's here" callout, social
|
||||
line, the summary as standfirst, the body in `.prose-body`, discussion with
|
||||
nested comment styling (`blockquote.reply` indent, `.comment-meta` small
|
||||
sans), rating control, "Read online ↗" button, prev/next cards, back link.
|
||||
Login/account: a narrow centred card-free form, labelled inputs, primary button.
|
||||
Archive: months as tracked labels, issues as a hairline list with date, number,
|
||||
article count.
|
||||
|
||||
Phones (390px): masthead scales down, nav wraps as a centred row, everything
|
||||
single-column, tap targets ≥ 40px, tables scroll inside `.scroll-x`.
|
||||
|
||||
## 5. Verify
|
||||
|
||||
Run the seeded dev server (shared brief), screenshot `/`, `/issues`,
|
||||
`/issues/2026-09-02`, `/issues/2026-09-02/articles/1`, `/issues/2026-09-02/world`,
|
||||
`/issues/2026-09-02/behind`, `/login`, `/account` as anonymous, `reader` and
|
||||
`admin`, in light and dark, desktop and mobile, and **look at them**. Fix what
|
||||
looks off. Make sure `npm run css:check` passes on the committed CSS, `cargo
|
||||
fmt`, clippy `-D warnings` and `cargo test` are green (minus the sandbox-only
|
||||
failures), then write `handoff-step2.md` and commit once on `v2-frontend`:
|
||||
`Web dashboard v2 step 2: Tailwind design system, theme toggle, reader pages`.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Step 3 — dashboard restyle on the Tailwind system
|
||||
|
||||
Worktree and branch: given in your launch prompt. Read `00-shared.md`
|
||||
(design language, "Dashboard" and "Controls" paragraphs) and
|
||||
`handoff-step2.md` (what the design system provides: tokens, `.btn`, `.badge`,
|
||||
`.kv`, `.scroll-x`, `.prose-body`, table and form defaults in
|
||||
`src/web/tailwind.css`). Step 2 deliberately left `src/web/templates/dashboard/*`
|
||||
unstyled; you restyle every one of them with Tailwind utilities plus those
|
||||
components, then rebuild `src/web/static/app.css` (`npm run css`) and commit it.
|
||||
|
||||
Templates: `overview`, `runs`, `run`, `articles`, `article`, `ratings`,
|
||||
`profile`, `stats`, `settings`, `settings_history`, `jobs`, `job`, `users`,
|
||||
`_pager`, `_sparkline`, and the shared partials `_candidate_row.html`,
|
||||
`_signals_table.html`, `_pagination.html` (the reader-side `_rating_widget.html`
|
||||
is styled already; the dashboard variant with the note field must look right
|
||||
inside tables and on the article page).
|
||||
|
||||
Rules of the road:
|
||||
|
||||
- Keep every handler field, form `action`/`name`/`id`, `data-*` hook and the
|
||||
SVG classes the sparkline/funnel code emits (`.spark`, `.s0`…`.s5`, `.line`,
|
||||
`.funnel`). Read `src/web/static/app.js` before touching anything it queries.
|
||||
- Old CSS class names that tests assert on (grep `tests/` and the `#[cfg(test)]`
|
||||
modules under `src/web/dashboard/` for `class=` and for names like `badge`,
|
||||
`num`, `muted`, `card`, `superseded`) may stay as harmless extra classes or be
|
||||
updated together with the test — never leave a test asserting a class the
|
||||
template no longer emits.
|
||||
- Page header pattern on every page: title, one-line description, actions
|
||||
(buttons/links) right-aligned, then filters, then content. Stat tiles on the
|
||||
overview and stats pages. Tables per the shared brief; long text cells
|
||||
(`message`, `editor_why`) wrap with `overflow-wrap:anywhere`; numeric columns
|
||||
right-aligned `tabular-nums`; sticky `thead` on `bg-paper`.
|
||||
- Settings: grouped cards with monospace section keys, a two-column setting row
|
||||
(label + help left, control right) collapsing on phones, locked (env-overridden)
|
||||
fields visibly disabled with a small lock note, the sticky save bar, danger
|
||||
actions in `text-down`. Profile: editor/preview split that stacks on phones.
|
||||
Jobs: cards with a status badge, last run, and the start form; the job page's
|
||||
journal in a scrolling monospace block. Users: a simple table.
|
||||
- Both themes, desktop and mobile (390px), every page: screenshot as `admin`
|
||||
with the seeded dev server and **look** before finishing.
|
||||
|
||||
Finish: `npm run css` + `npm run css:check`, `cargo fmt`, clippy `-D warnings`,
|
||||
`cargo test`; write `handoff-step3.md`; commit once:
|
||||
`Web dashboard v2 step 3: dashboard restyle`.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Step 4 — table-of-contents sidebar for signed-in issue pages
|
||||
|
||||
Worktree and branch: given in your launch prompt. Read `00-shared.md`,
|
||||
`handoff-step1.md` (legacy world/behind availability) and `handoff-step2.md`
|
||||
(design system). Item 4 of the shared brief: on every signed-in issue page —
|
||||
`/issues/{date}`, `/issues/{date}/articles/{id}`, `/issues/{date}/world`,
|
||||
`/issues/{date}/behind` — a sidebar lists the issue's chapters, marks where the
|
||||
reader is, and lets them jump anywhere. Public pages get nothing.
|
||||
|
||||
## Data (Rust, `src/web/issue.rs`)
|
||||
|
||||
One helper `issue_toc(&IssueView, current: TocPosition) -> Toc` used by all
|
||||
four handlers, passed to their templates as `toc`:
|
||||
|
||||
```rust
|
||||
struct Toc { date: Date, display_date: String, issue_number: i64,
|
||||
items: Vec<TocItem>, position: usize, total: usize, issue_href: String }
|
||||
enum TocKind { Brief, Section, Chapter, World, Behind, Colophon }
|
||||
struct TocItem { kind: TocKind, label: String, href: String, number: Option<usize>,
|
||||
minutes: Option<i64>, current: bool }
|
||||
```
|
||||
|
||||
Chapters are the picks in issue order (numbered 1..n across sections, section
|
||||
labels interleaved as non-links), followed by World Briefing and Behind the
|
||||
paper when available (`has_world`/`has_behind` as computed after step 1) and a
|
||||
Colophon anchor (`/issues/{date}#colophon`). `position`/`total` count
|
||||
navigable chapters only (articles + world + behind), so the article page can say
|
||||
"Chapter 4 of 11"; on the issue page position is 0 ("Front page").
|
||||
|
||||
## Markup and behaviour
|
||||
|
||||
- Desktop (`lg:` and up): a two-column grid, `aside` on the left `w-64 xl:w-72`,
|
||||
`sticky top-4`, `max-h-[calc(100vh-2rem)] overflow-y-auto`, thin scrollbar.
|
||||
Header: issue number + date (link to the issue), a progress line "Chapter 4
|
||||
of 11" and a 2px progress bar (width = position/total). Then the list:
|
||||
section labels as tracked uppercase sans, chapters as numbered links in
|
||||
Newsreader `text-[0.95rem]` with the read-time in muted sans, the current one
|
||||
bold with a 2px accent bar on the left and `aria-current="page"`; World
|
||||
Briefing / Behind the paper / Colophon after a hairline.
|
||||
- Mobile (below `lg:`): the aside becomes a sticky bar under the site header:
|
||||
a hamburger `<button type="button" aria-controls="toc" aria-expanded="false"
|
||||
aria-label="Contents">` with the current chapter title truncated beside it and
|
||||
"4 / 11" on the right. Tapping toggles a panel (`id="toc"`, `hidden`) that
|
||||
drops down under the bar with the same list; close on link tap, on Escape,
|
||||
and on tap outside. Body scroll stays enabled. Implement in `app.js` (no
|
||||
inline handlers), progressive: without JS the panel is simply open below the
|
||||
bar. Use the same `<nav>` markup for both breakpoints (one element, styled
|
||||
responsively), not two copies.
|
||||
- On article pages `app.js` also updates the progress bar with scroll progress
|
||||
within the chapter (position-1 + scrolled fraction over total), throttled with
|
||||
`requestAnimationFrame`, and honours `prefers-reduced-motion` (no transition).
|
||||
- The reading column keeps its `max-w-[68ch]`; the grid must not push it
|
||||
narrower than ~60ch at `lg`. At `xl` the aside gets its full width.
|
||||
- Sidebar links to the current article do nothing surprising (`href` still
|
||||
set). Keyboard: Tab order is bar → panel → content; the toggle is reachable.
|
||||
|
||||
## Tests
|
||||
|
||||
Router tests: the toc appears for `reader` on all four pages with the right
|
||||
`aria-current` item and "Chapter N of M"; it is absent on public pages and in
|
||||
the anonymous issue page; the legacy issue (no `issue_json`) lists World
|
||||
Briefing and Behind the paper when step 1's fallback finds them and omits
|
||||
World when there is no EPUB. Unit test for `issue_toc` numbering across
|
||||
sections.
|
||||
|
||||
Finish: rebuild CSS (`npm run css` + `css:check`), `cargo fmt`, clippy
|
||||
`-D warnings`, `cargo test`; screenshots of the four pages at desktop and
|
||||
mobile in both themes (panel closed and open); `handoff-step4.md`; one commit:
|
||||
`Web dashboard v2 step 4: table-of-contents sidebar`.
|
||||
@@ -0,0 +1,64 @@
|
||||
# Web dashboard v2 — step 1 handoff
|
||||
|
||||
## What landed
|
||||
|
||||
- Public issue pages and Atom entries now show each pick's trimmed summary and
|
||||
its “Why it's here” line. The public view model still has no article body,
|
||||
Brief, World Briefing, or discussion/comment content, and the public tests
|
||||
assert those fields do not leak.
|
||||
- Legacy `issues.issue_json IS NULL` loads now recover their colophon and
|
||||
Behind the paper data from `issues.report_json`, matching the report to its
|
||||
run by `started_at` for near misses. If the issue report is absent, the loader
|
||||
uses the latest finished run for that issue date. Missing historical values
|
||||
render as `n/a` in the web colophon rather than misleading zeroes.
|
||||
- Historical model names come from the resolved run config. Summary-provider
|
||||
resolution follows `editorial.summary_model`; missing model metadata falls
|
||||
back to the current config and is called out in the generator note.
|
||||
- Legacy World Briefings are recovered from `OEBPS/world.xhtml` in the stored
|
||||
standard EPUB (or its canonical publish path). The EPUB body is used after
|
||||
removing its own leading heading; missing or corrupt files are debug-logged
|
||||
and treated as “no World chapter,” not as request errors.
|
||||
- The development seed's 2026-09-01 issue now has a real report, a real EPUB
|
||||
path with fixture World Briefing content, `issue_json = NULL`, and three
|
||||
`candidate_runs` near misses. The 2026-09-02 issue retains a full snapshot.
|
||||
- README and the dashboard plan now document the operator's 2026-09-03 public
|
||||
summary/why decision and list recovered EPUB XHTML among the trusted `|safe`
|
||||
inputs. `zip` moved from dev-only to runtime dependencies.
|
||||
|
||||
## Tests
|
||||
|
||||
- Focused web issue tests: **10 passed**, including report and run-only
|
||||
colophon fallback, recovered World/Behind routes, missing EPUB behavior, and
|
||||
public/feed privacy assertions.
|
||||
- Public view-model test: **1 passed**.
|
||||
- Development seed smoke test: successfully generated the database and EPUB;
|
||||
SQL verification showed the legacy row has a null snapshot, present report
|
||||
and EPUB path, plus **3** near-miss rows.
|
||||
- `cargo fmt`: clean.
|
||||
- `cargo clippy --all-targets -- -D warnings`: clean.
|
||||
- Unfiltered `cargo test`: library result **427 passed, 13 failed**. Every
|
||||
failure was a pre-existing loopback-listener test rejected by the sandbox.
|
||||
- Filtered sandbox run: **461 passed, 0 failed, 15 filtered** across library,
|
||||
binary, integration, and doc-test targets. The filters were the 13 loopback
|
||||
unit tests and both `tests/m7_server.rs` TCP tests.
|
||||
|
||||
## Deviations and follow-up
|
||||
|
||||
- `00-shared.md` lists four Anthropic listener tests, one extraction listener
|
||||
test, five server listener tests, and `m7_server` as sandbox-only failures.
|
||||
The code also has three pre-existing OpenAI mock-server tests that bind the
|
||||
same forbidden loopback listener; they failed with the identical
|
||||
`PermissionDenied` error. `docs/plans/2026-08-15-implementation-notes.md`
|
||||
already lists those three. They were therefore filtered for the complete
|
||||
sandbox verification and should run normally in the orchestrator.
|
||||
- No migrations, routes, auth behavior, or security headers changed. No
|
||||
database backfill is performed.
|
||||
- This sandbox mounts the worktree's shared Git metadata under
|
||||
`/home/thallada/workspace/the-daily-epub/.git/worktrees/the-daily-epub-v2a`
|
||||
read-only. The required `git add` failed while creating `index.lock`, so the
|
||||
verified changes remain unstaged and the single requested commit still needs
|
||||
to be created by the orchestrator with message `Web dashboard v2 step 1:
|
||||
public summaries, legacy issue fallback`. No source or data in the forbidden
|
||||
main checkout was changed.
|
||||
- Beyond that commit and the orchestrator's unsandboxed full-suite run, nothing
|
||||
remains for step 1. This was not a design step, so no screenshots were taken.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Step 2 handoff — Tailwind design system, theme toggle, reader pages
|
||||
|
||||
Branch `v2-frontend`, worktree `/home/thallada/workspace/the-daily-epub-v2b`, one commit.
|
||||
Step 3 (dashboard restyle) and step 4 (TOC sidebar) build on what is described below.
|
||||
|
||||
## Landed
|
||||
|
||||
### Toolchain
|
||||
|
||||
- `package.json` (root, `private`) pins `tailwindcss` and `@tailwindcss/cli` to `4.3.3` with
|
||||
scripts `css`, `css:watch` and `css:check`. `package-lock.json` is committed.
|
||||
`.gitignore` already carried `node_modules/` and `dev/`.
|
||||
- `src/web/tailwind.css` is the single source; `src/web/static/app.css` is the **committed,
|
||||
minified build** and is still embedded with `include_str!`, so `cargo build`, `cargo test`
|
||||
and the deployed binary never need Node.
|
||||
- `npm run css:check` rebuilds to a temp file and `diff -u`s it against the committed
|
||||
`app.css`, exiting non-zero on drift (verified in both directions).
|
||||
- Workflow documented in `README.md` and `docs/runbooks/web-dashboard-rollout.md`
|
||||
(`npm ci && npm run css:check` added to the rollout build step).
|
||||
|
||||
### Assets
|
||||
|
||||
- `web::static_asset` gained fixed match arms (no path joins) for `theme.js`,
|
||||
`Newsreader.woff2` and `Newsreader-italic.woff2`. The asset tuple is now
|
||||
`(&str, &'static [u8])` so text assets use `include_str!(...).as_bytes()` and the fonts use
|
||||
`include_bytes!`; the SHA-256 ETag / 304 / long-cache path is unchanged and its test still
|
||||
passes. URLs are flat: `/static/Newsreader.woff2`, not `/static/fonts/…`.
|
||||
- `@font-face` for both faces (`font-weight: 200 800`, `font-stretch: normal`,
|
||||
`font-display: swap`, Google's latin `unicode-range`), plus a
|
||||
`<link rel="preload" as="font" type="font/woff2" crossorigin>` for the regular face.
|
||||
`document.fonts.check('600 48px Newsreader')` is true on a loaded page and the server log
|
||||
shows no 404s.
|
||||
|
||||
### Theme
|
||||
|
||||
- `src/web/static/theme.js` runs synchronously in `<head>` (external file, CSP-safe): reads
|
||||
`localStorage.theme`, sets or removes `document.documentElement.dataset.theme` before paint.
|
||||
- `app.js` drives `[data-theme-toggle]`: cycles **system → light → dark → system**, persists
|
||||
(`removeItem` for system), updates the attribute, the SVG icon and the visible label, keeps
|
||||
`aria-label="Theme: <state>"`, and re-renders on `matchMedia` changes while in system mode.
|
||||
- Colours are semantic CSS variables on `:root`, duplicated into
|
||||
`@media (prefers-color-scheme: dark) :root:not([data-theme="light"])` and
|
||||
`:root[data-theme="dark"]`, exposed to Tailwind via `@theme inline`, with
|
||||
`@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *))`.
|
||||
`color-scheme` follows the theme.
|
||||
|
||||
### Reader templates
|
||||
|
||||
`layout.html` (ears row + centred masthead with the 1px rule above and the 3px double rule
|
||||
below + primary nav with `aria-current="page"` + admin nav row + flash + `<main id="content">`
|
||||
+ footer + skip link), `issue_public.html`, `issue_full.html`, `issue_list.html`,
|
||||
`article.html`, `world.html`, `behind.html`, `login.html`, `account.html`, `error.html` and
|
||||
`_rating_widget.html` are all restyled with utilities. Every hook listed in the shared brief
|
||||
(`form.rating`, `button[data-label]`, `.active`, `aria-pressed`, `form[data-confirm]`,
|
||||
`details[id]`, `table[data-filter]`, `.table-filter`, `[data-refresh]`, `.scroll-x`, ids,
|
||||
names, actions) is intact.
|
||||
|
||||
## What step 3 can use
|
||||
|
||||
Tokens (`@theme inline`, so both a CSS variable and a Tailwind colour utility exist):
|
||||
|
||||
| token | utilities |
|
||||
| --- | --- |
|
||||
| `--paper`, `--paper-2` | `bg-paper`, `bg-paper-2` |
|
||||
| `--ink`, `--ink-2`, `--muted` | `text-ink`, `text-ink-2`, `text-muted`, `bg-ink` |
|
||||
| `--rule`, `--rule-strong` | `border-rule`, `border-rule-strong` |
|
||||
| `--accent`, `--accent-hover` | `text-accent`, `border-accent`, `bg-accent`, `hover:text-accent-hover` |
|
||||
| `--loved`, `--good`, `--down`, `--warn` | `text-loved`, `text-good`, `text-down`, `text-warn` |
|
||||
| `--font-serif` / `--font-sans` / `--font-mono` | `font-serif`, `font-sans`, `font-mono` |
|
||||
|
||||
Component classes in `@layer components`: `.btn`, `.btn-primary`, `.badge` + state modifiers
|
||||
(`selected loved good down excluded shortlisted admitted eligible cleared assessed triaged
|
||||
reason running requested ok degraded failed dry_run`), `.kv`, `.scroll-x`, `.prose-body`,
|
||||
`.rating`/`.rating-prompt`/`.rating-note`, `.flash`, `.error`, `.notice`, `.dashboard`,
|
||||
`.cards`/`.card`, `.muted`/`.meta`/`.num`, `.filters`, `.table-filter`, `.pager`/`.run-nav`/
|
||||
`.tabs`, `pre.block`/`pre.preview`/`pre.journal`, `.funnel`, `.spark` (+ `.s0`–`.s5`, `.line`).
|
||||
`@layer base` styles `table`/`thead`/`th`/`td` (sticky head on `bg-paper`, hairline rules,
|
||||
`hover:bg-paper-2`, tabular numerals) and all form controls.
|
||||
|
||||
Repeated inline recipes step 3 will probably want: small label
|
||||
`font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted`; headline
|
||||
`font-semibold leading-[1.1] tracking-[-0.01em]`; reader measure
|
||||
`mx-auto max-w-[68ch] px-4 sm:px-6`; dashboard shell `.dashboard` (`max-w-7xl`, sans).
|
||||
|
||||
## Deviations and fixes beyond the brief
|
||||
|
||||
- **`.block` collided with Tailwind's `block` display utility.** The legacy component was
|
||||
renamed to `pre.block`, which keeps both `<pre class="block">` dashboard usages working and
|
||||
frees `class="block"` for real layout. Before this, the article prev/next cards rendered as
|
||||
grey monospace boxes.
|
||||
- **`icon.hidden = …` does not work on SVG elements** (`hidden` is an `HTMLElement` IDL
|
||||
attribute); the theme icon never changed. `app.js` now uses
|
||||
`icon.toggleAttribute("hidden", …)`. `[hidden] { display:none !important }` stays in
|
||||
`@layer base` so it beats display utilities.
|
||||
- `.prose-body` lists had no markers after Preflight; added `list-disc` / `list-decimal` and a
|
||||
muted `::marker`.
|
||||
- The flash bar was flush to the viewport edge while the header used a gutter; `.flash` lost
|
||||
its own `mx-auto max-w-7xl` and the layout wraps it in `mx-auto max-w-7xl px-4 sm:px-6`.
|
||||
- The article prev/next grid rendered an empty first cell when there is no previous article;
|
||||
the placeholder `<span>` is gone and the next card takes `sm:col-start-2`.
|
||||
- `examples/seed_dev_db.rs` now writes `profile_path` into the throwaway `./dev` directory and
|
||||
seeds it from `data/profile.md`. Without this, saving on `/dashboard/profile` while running
|
||||
the dev server **rewrites the repository's `data/profile.md`** and breaks
|
||||
`curate::profile::tests::shipped_profile_and_opml_parse`. It also fixes a pre-existing
|
||||
`clippy::manual_is_multiple_of` error in that example (`cargo clippy --all-targets
|
||||
-D warnings` did not pass on this branch before).
|
||||
- `Page::new` for the Jobs and Users dashboard pages now passes `"jobs"` / `"users"` instead of
|
||||
`"dashboard"`, so the admin nav highlights the right entry.
|
||||
- `FullEntry` gained `is_lead` so the full issue's lead story is set larger (`text-3xl` vs
|
||||
`text-2xl`), matching what `issue_public.html` already did.
|
||||
- Fonts are served flat from `/static/<name>.woff2` (the brief allowed either shape).
|
||||
- `issue_public.html` carries `{# step 1 adds summary/why here #}` at the point where the
|
||||
backend step's summary/`why` markup should go; use the same treatment as `issue_full.html`
|
||||
(summary as a body-size serif paragraph, then `border-l-2 border-accent pl-3 italic`).
|
||||
- The dashboard templates are deliberately unstyled beyond what the base/component layers give
|
||||
them. They render without errors (all nine routes smoke-tested) but look plain — step 3.
|
||||
- The archive month label is still the raw `YYYY-MM` produced by `web::public`; changing it is
|
||||
a backend concern and was left alone.
|
||||
|
||||
## Verification
|
||||
|
||||
- `npm run css` / `npm run css:check` — clean; drift is detected (exit 1) when `app.css` is
|
||||
edited by hand.
|
||||
- `cargo fmt`, `cargo clippy --all-targets -- -D warnings` — clean.
|
||||
- `cargo test` — **475 passed, 0 failed** (439 lib + 9 + 2 + 3 + 4 + 7 + 9 + 2 integration,
|
||||
one binary with 0 tests). Run outside the Codex sandbox, so the socket-binding tests ran too.
|
||||
No test needed a cosmetic-class update.
|
||||
- Seeded dev server on `127.0.0.1:3599`. All of `/`, `/issues`, `/issues/2026-09-01` (legacy),
|
||||
`/issues/2026-09-02`, `/issues/2026-09-02/articles/1`, `/…/world`, `/…/behind`, `/account`,
|
||||
`/feed.xml` and the nine `/dashboard/*` routes return 200 with **no console errors and no
|
||||
asset 404s**; `/issues/1999-01-01` returns the styled 404 page.
|
||||
- Theme toggle asserted with Playwright: system → light → dark → system, `localStorage.theme`
|
||||
written/removed, `data-theme` set/removed, icon and label follow, state survives a reload.
|
||||
- Focus order checked: the skip link is the first tab stop and becomes visible; the toggle
|
||||
shows the accent focus ring.
|
||||
|
||||
### Screenshots
|
||||
|
||||
`/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/e963db53-510f-4312-ac87-460291a92781/scratchpad/step2-shots/`
|
||||
— 83 PNGs. `/`, `/issues`, `/issues/2026-09-02`, `/issues/2026-09-02/articles/1`, `/…/world`,
|
||||
`/…/behind`, `/account` and `/login` × {anonymous, reader, admin} × {light, dark} ×
|
||||
{1280px desktop, 390px mobile}, full page. Named
|
||||
`<path>-<user>-<theme>[-mobile].png`. Plus `toggle-1..4.png` (the toggle cycle),
|
||||
`flash-light/dark.png`, `rating-active-light.png`, `focus-skiplink.png`, `focus-toggle.png`
|
||||
and `issues_1999-01-01-none-{light,dark}.png` (404 page).
|
||||
|
||||
## Left for later
|
||||
|
||||
- Step 1 adds `summary`/`why` to the public entries (marker comment in place).
|
||||
- Step 3 restyles `src/web/templates/dashboard/*` on top of this system.
|
||||
- Step 4 adds the collapsing table-of-contents sidebar to the signed-in issue pages.
|
||||
@@ -0,0 +1,132 @@
|
||||
# Step 3 handoff — dashboard restyle
|
||||
|
||||
Branch `v2-dashboard`, worktree `/home/thallada/workspace/the-daily-epub-v2c`, one commit
|
||||
on top of the merged `v2-frontend` work (step 1 + step 2).
|
||||
|
||||
## Landed
|
||||
|
||||
Every template under `src/web/templates/dashboard/` plus the shared partials
|
||||
`_candidate_row.html`, `_signals_table.html` and `_pagination.html` is restyled with
|
||||
Tailwind utilities and the component layer step 2 built. `src/web/static/app.css` is
|
||||
rebuilt and committed; `npm run css:check` is clean.
|
||||
|
||||
### New/changed pieces in `src/web/tailwind.css`
|
||||
|
||||
`@layer base` gained: a disabled state for every control, `summary { cursor:pointer }`,
|
||||
and a styled `<meter>` (webkit + moz pseudo-elements, `--rule` track, `--good` value,
|
||||
`--down` when `.over`) for the overview budget bars.
|
||||
|
||||
`@layer components` gained the dashboard vocabulary the templates now use:
|
||||
|
||||
| class | what it is |
|
||||
| --- | --- |
|
||||
| `.page-head`, `.page-desc`, `.page-actions`, `.page-eyebrow` | the page header on all 13 pages: eyebrow/breadcrumb, `h1`, one-line description, right-aligned actions, hairline under it |
|
||||
| `.tiles`, `.tile`, `.tile-label`, `.tile-num`, `.tile-delta` | stat tiles (overview, stats, run detail) |
|
||||
| `.cell-wrap`, `.cell-tight` | long text cells (`message`, `editor_why`, notes, reasons) wrap with `overflow-wrap:anywhere` and a max width; dates/badges stay on one line |
|
||||
| `.scroll-x.tall` | opt-in `max-h-[70vh] overflow-y-auto`, which is what actually makes the base layer's `thead { sticky top-0 bg-paper }` do something. Applied to the long tables (runs, candidates, articles, ratings, jobs history, users, settings history, stats) and deliberately not to the small tables inside cards |
|
||||
| `.filters` / `.filter-actions` / `.form-inline` | one inline filter form per page: rule-bounded row, tiny uppercase labels above each control, buttons grouped at the end |
|
||||
| `.pager`, `.tabs`, `.pagination` | pagination row and the ratings tab strip (accent underline on `.active`) |
|
||||
| `.setting`, `.setting-label`, `.setting-input`, `.help`, `.default`, `.secret`, `button.reset`, `.save-bar` | settings: two-column rows collapsing to one on phones, monospace keys, disabled env-locked fields with a `locked` badge and a lock note, sticky save bar with a soft top shadow |
|
||||
| `.disclosure` (`details.disclosure` + `.disclosure-body`) | the framed `<details>` blocks (ratings "How ratings enter the algorithm", stats "As text", article "Explain") |
|
||||
| `.sparklines`, `.spark-figure`, `.spark-axis`, `.spark-legend` | sparkline cards; the SVG classes (`.spark`, `.s0`…`.s5`, `.line`) are untouched |
|
||||
| `.funnel`, `.funnel-cell` | the funnel bar gets a `bg-paper-2` track; `class="funnel"` is unchanged because a test asserts it verbatim |
|
||||
| `td .rating` / `.picks .rating` overrides | the rating widget compacted for table cells and the overview picks list: prompt hidden, `min-h-8` buttons, small note field, capped width |
|
||||
| `tr.superseded` | muted row (the class stays exactly `superseded`; a test counts it) |
|
||||
| `.badge` default | bare `<span class="badge">` now has a neutral tint, plus `.badge.restart` / `.badge.warn` |
|
||||
|
||||
`.dashboard` gained a vertical rhythm (`> * + * { mt-4 }`), section headings as tracked
|
||||
uppercase labels on a rule (`.dashboard > h2`), muted underlined table links, and
|
||||
`code` chips on `bg-paper-2`.
|
||||
|
||||
### Per page
|
||||
|
||||
- **overview** — five stat tiles (last run, warnings, verdicts this week, unrated picks,
|
||||
active jobs) above the existing cards; budget rows are now provider + spend + a meter bar;
|
||||
jobs/picks lists are hairline-separated with ink links.
|
||||
- **runs** — page header with Stats/Jobs actions, inline status filter with a Reset that
|
||||
only appears when a filter is set, numeric columns right-aligned, empty-state row.
|
||||
- **run** — breadcrumb + status badge header, prev/issue/next as buttons in the header,
|
||||
four tiles (date, duration, cost, warnings), funnel with a track, cards for the report,
|
||||
empty-state row for candidates.
|
||||
- **articles / article** — filter form, wrapped title cells; the article page splits its
|
||||
facts into "Article" and "Provenance" cards, gives the rating widget its own
|
||||
"Your verdict" section and keeps everything else as tables.
|
||||
- **ratings** — description as the page header, tab strip, per-tab filter form, the
|
||||
in-cell rating widget, muted superseded rows.
|
||||
- **profile** — editor/preview split (`lg:grid-cols-2`, stacked on phones), monospace
|
||||
textarea, previews as capped `pre.preview` blocks, history table, `.kv` theme list.
|
||||
- **stats** — window buttons in the header, `summary` rendered as six stat tiles,
|
||||
sparkline cards, cards, two filterable tables, text output in a disclosure.
|
||||
- **jobs / job** — job cards now carry a status badge and a "last started" line linking
|
||||
to that job (see the one backend change below); the job page is breadcrumb + two cards
|
||||
+ a scrolling monospace journal.
|
||||
- **settings / settings history / users** — as described in the table above.
|
||||
|
||||
### Backend changes (three, all small)
|
||||
|
||||
1. `JobCard` gained `last_status` / `last_requested` / `last_id`, filled by a new
|
||||
`attach_last_runs` from the job rows the page already loads (matching `name` or
|
||||
`name-<date>` for the dated `generate` card). No extra query. Unit-tested.
|
||||
2. `stats::stats` passed `"dashboard"` as its nav key, so the admin nav highlighted
|
||||
*Overview* on the Stats page; it now passes `"stats"`.
|
||||
3. `settings.html` dropped the `! ` prefix in front of the "config.toml on disk does not
|
||||
load" banner (the `.error` bar already says it); the assertion in
|
||||
`settings_pages_render_save_and_show_hand_edits` was updated with it. That is the only
|
||||
test touched.
|
||||
|
||||
## Deviations
|
||||
|
||||
- **`class="inline"` was a latent bug**, the same shape as step 2's `.block` collision:
|
||||
Tailwind emits an `.inline { display:inline }` utility, which beats `form.inline`'s
|
||||
`@apply flex` in the components layer, so those forms were laid out inline. The class
|
||||
is renamed to `form-inline` in `jobs.html`, `profile.html` and `settings.html` and in
|
||||
`tailwind.css`. Nothing asserted on it.
|
||||
- **Sticky table heads need a scroll box.** `thead { sticky top-0 }` from step 2 is inert
|
||||
inside a plain `overflow-x:auto` container, so the long tables opt into
|
||||
`.scroll-x.tall` (`max-h-[70vh]`). Small tables inside cards keep the plain `.scroll-x`.
|
||||
- The brief asks for "last run" on the job cards; the data was not in `JobCard`, so it was
|
||||
added (item 1 above) rather than skipped.
|
||||
- `_rating_widget.html` itself is untouched — two tests assert its exact attribute order
|
||||
(`value="loved" data-label="loved" class="active"`), so the table-cell treatment is done
|
||||
purely with `td .rating …` rules.
|
||||
- `.dashboard > h2` renders section headings as small tracked uppercase labels on a rule
|
||||
rather than as large sans headings; `.card h2`/`h3` stay normal-weight `text-base`.
|
||||
|
||||
## Hooks and asserted classes (all preserved)
|
||||
|
||||
`form.rating`, `button[data-label]`, `.active` + `aria-pressed`, `form[data-confirm]`,
|
||||
`details[id]` (`ratings-how`, `profile-prompt`, plus new `stats-text` and `article-explain`),
|
||||
`table[data-filter]`, `.table-filter`, `button[data-reset]`/`data-default`, `[data-refresh]`,
|
||||
`.scroll-x`, `.spark`/`.s0`–`.s5`/`.line`, `class="funnel"`, `class="superseded"`,
|
||||
`badge <state>`, `<h1>Users</h1>`, every form `action`/`name`/`id` and every `aria-*`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `npm run css` + `npm run css:check` — clean.
|
||||
- `cargo fmt`, `cargo clippy --all-targets -- -D warnings` — clean.
|
||||
- `cargo test` — **477 passed, 0 failed** (441 lib + 9 + 2 + 3 + 4 + 7 + 9 + 2 integration,
|
||||
one binary with 0 tests). Run outside the Codex sandbox, so the socket-binding tests ran.
|
||||
- Dev server on `127.0.0.1:3601` (port 3599/3602 belong to other agents, 3499 to
|
||||
production). Every dashboard route returned 200, with **no console errors and no asset
|
||||
404s** in the server log.
|
||||
|
||||
### Screenshots
|
||||
|
||||
`/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/e963db53-510f-4312-ac87-460291a92781/scratchpad/step3-shots/`
|
||||
— 58 PNGs, all signed in as `admin`: `/dashboard`, `/dashboard/runs`, `/dashboard/runs/2`,
|
||||
`/dashboard/articles`, `/dashboard/articles/1`, `/dashboard/ratings`,
|
||||
`/dashboard/ratings?tab=events`, `/dashboard/profile`, `/dashboard/stats`,
|
||||
`/dashboard/jobs`, `/dashboard/jobs/1`, `/dashboard/settings`,
|
||||
`/dashboard/settings/history`, `/dashboard/users` × {light, dark} × {1280px desktop,
|
||||
390px mobile}, full page, plus viewport shots of the settings save bar
|
||||
(`*-top.png`, desktop light/dark and mobile).
|
||||
|
||||
The throwaway `./dev` database was extended by hand for the shoot (four `jobs` rows, eight
|
||||
`rating_events`, four `config_changes`) so the jobs, ratings and settings-history pages had
|
||||
content; `dev/` is git-ignored and `examples/seed_dev_db.rs` was **not** changed. A future
|
||||
design pass may want those rows in the seeder.
|
||||
|
||||
## Left for later
|
||||
|
||||
- Step 4 adds the collapsing table-of-contents sidebar to the signed-in issue pages.
|
||||
- `examples/seed_dev_db.rs` still seeds no jobs, ratings or config changes.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Step 4 handoff — table-of-contents sidebar
|
||||
|
||||
Branch `v2-toc`, worktree `/home/thallada/workspace/the-daily-epub-v2d`, one commit.
|
||||
Builds on step 2's design system (`handoff-step2.md`) and step 1's legacy fallback
|
||||
(`handoff-step1.md`). Also folds in one reader-side fix from the step-2 review: the
|
||||
phone "ears" row.
|
||||
|
||||
## Landed
|
||||
|
||||
### Data (`src/web/issue.rs`)
|
||||
|
||||
- `issue_toc(&IssueView, TocPosition) -> Toc`, one helper feeding all four signed-in
|
||||
handlers (`render_full`, `article`, `world`, `behind`) as the template field `toc`.
|
||||
- `TocPosition` (`FrontPage`, `Article(ArticleId)`, `World`, `Behind`) and `TocKind`
|
||||
(`Brief`, `Section`, `Chapter`, `World`, `Behind`, `Colophon`) as the brief specifies.
|
||||
- Chapters are the picks in `chapters::section_names` × `Lineup::section_picks` order —
|
||||
the same order the issue index and the EPUB use — numbered `1..n` across sections with
|
||||
the section names interleaved as non-links. World Briefing and Behind the paper follow
|
||||
when available (`world_briefing.is_some() || world_html.is_some()`, and `has_behind`,
|
||||
both as computed after step 1), then a `#colophon` anchor on the issue page.
|
||||
- `position` / `total` count navigable chapters only (articles + world + behind), so an
|
||||
article page says "Chapter 4 of 11"; the issue page has `position == 0` and reads
|
||||
"Front page".
|
||||
- New small helpers: `issue_href(date)` (replaces four inline `format!`s) and
|
||||
`crate::pipeline::short_display_date(date)` → `"Wed, Sep 2"`.
|
||||
|
||||
### Markup (`src/web/templates/_toc.html`, included by the four reader templates)
|
||||
|
||||
- One `<nav id="toc" data-toc-panel>` for both breakpoints, wrapped with the article in a
|
||||
`lg:grid lg:grid-cols-[16rem_minmax(0,1fr)] xl:grid-cols-[18rem_minmax(0,1fr)]` shell
|
||||
inside `max-w-7xl`. The reading column keeps `max-w-[68ch]`; at `lg` (1024px viewport)
|
||||
it still gets ~680px, comfortably above the ~60ch floor the brief asks for.
|
||||
- Desktop: `lg:sticky lg:top-6 lg:max-h-[calc(100vh-3rem)] lg:overflow-y-auto`, thin
|
||||
scrollbar. Header (issue number + short date linking to the issue, "Chapter N of M",
|
||||
2px progress bar) is `lg:sticky lg:top-0` **inside** the scrollable nav, so it stays put
|
||||
while the chapter list scrolls under it. Section labels are tracked uppercase sans;
|
||||
chapters are numbered Newsreader `text-[0.95rem]` links with the read time in muted
|
||||
sans; the current one is bold with a 2px accent bar and `aria-current="page"`. World
|
||||
Briefing / Behind the paper / Colophon follow a hairline.
|
||||
- Below `lg`: a sticky bar (`sticky top-0 z-30` on the wrapper, `lg:static`) with the
|
||||
hamburger `<button data-toc-toggle aria-controls="toc" aria-expanded>`, the current
|
||||
chapter title truncated, "4 / 11", and the progress bar as its bottom edge. The same
|
||||
`<nav>` drops under it as an absolutely positioned panel.
|
||||
- No inline `<style>`/`<script>` and no `style=""`; every hook step 2 listed is intact.
|
||||
|
||||
### Behaviour (`src/web/static/app.js`, `theme.js`)
|
||||
|
||||
- The panel toggles by `data-open` on the nav; `@media (width < 64rem) .has-js
|
||||
[data-toc-panel]:not([data-open="true"]) { display:none }` in `tailwind.css`. `theme.js`
|
||||
(synchronous, in `<head>`) adds `has-js` to `<html>`, so **without JS the panel is simply
|
||||
visible** under the bar and **with JS there is no open-then-collapse flash**.
|
||||
- Closes on link tap, on Escape (focus returns to the toggle), on a tap outside, and when
|
||||
crossing to `lg`. Body scroll is never locked.
|
||||
- `revealCurrent()` scrolls the panel — never the page — the minimum needed to bring the
|
||||
current chapter into view, on load and on open. Long issues no longer hide the reader's
|
||||
position behind the fold.
|
||||
- Article pages (`<article data-toc-scroll>`) update every `[data-toc-progress]` with
|
||||
`position - 1 + scrolled fraction`, throttled through `requestAnimationFrame` on a
|
||||
passive scroll listener.
|
||||
|
||||
### Progress bar and CSP
|
||||
|
||||
The bar is a real `<progress class="toc-progress">`: a percentage width would need a
|
||||
`style=""` attribute, which the CSP forbids, and Tailwind cannot generate a class for a
|
||||
runtime number. It is styled in `@layer components` (2px, `--rule` track, `--accent`
|
||||
value) and marked `aria-hidden="true"` because the adjacent text already says
|
||||
"Chapter N of M". Reduced motion: the 150ms transition on the value is dropped both under
|
||||
`prefers-reduced-motion: reduce` and while JS drives the bar (`[data-live]`), so scrolling
|
||||
never animates.
|
||||
|
||||
### Ears row (step-2 review follow-up)
|
||||
|
||||
`layout.html`: the row is `flex-nowrap` with `gap-2 sm:gap-4`; the left cell is
|
||||
`min-w-0 truncate whitespace-nowrap`; the right group and the theme toggle are
|
||||
`shrink-0 whitespace-nowrap`; "Sign in" is `whitespace-nowrap` and a long username is
|
||||
`max-w-32 truncate`. The four issue pages and `issue_public.html` now show
|
||||
`short_display_date` below `sm:` and the full dateline + issue number above it, so at
|
||||
390px the whole row is one line for anonymous, reader and admin.
|
||||
|
||||
## Tests
|
||||
|
||||
- `cargo test` — **478 passed, 0 failed** (442 lib + 9 bin + 2 + 3 + 4 + 7 + 9 + 2
|
||||
integration, 0 doc-tests). Run outside the Codex sandbox, so the socket-binding tests
|
||||
ran too.
|
||||
- New: `web::issue::tests::toc_numbers_chapters_across_sections_and_tracks_the_reader`
|
||||
(unit — full item shape across two sections, `total == 4`, `position` for front page /
|
||||
article / world / behind, the single back-matter divider, `current_label`, short date)
|
||||
and `web::issue::tests::the_sidebar_marks_the_current_chapter_on_every_signed_in_issue_page`
|
||||
(router — all four pages carry the bar and panel, the right "Chapter N of M", the right
|
||||
`aria-current="page"` href and the colophon anchor; the anonymous issue page has none).
|
||||
- Extended: the public-page test now asserts no `data-toc-toggle` / `id="toc"` leaks; the
|
||||
legacy tests assert the sidebar offers World Briefing and Behind the paper, and that it
|
||||
omits World when there is no EPUB to recover it from.
|
||||
- Browser checks (Playwright, `shoot/toc-behaviour.mjs`): 14/14 — collapsed on load,
|
||||
`aria-expanded`/`aria-controls`, open on tap, close on Escape with focus return, close on
|
||||
outside tap, close + navigate on link tap, `aria-current` after navigation, desktop panel
|
||||
always visible with the bar hidden, progress 4 → 5 on scrolling a chapter, no toc on the
|
||||
public page, no console errors.
|
||||
- `cargo fmt`, `cargo clippy --all-targets -- -D warnings`, `npm run css`,
|
||||
`npm run css:check` — all clean.
|
||||
|
||||
## Deviations
|
||||
|
||||
- **`TocItem` has a seventh field, `divider: bool`.** The brief's hairline "after the
|
||||
chapters" cannot be expressed in askama without knowing which entry starts the
|
||||
back-matter group; the helper marks it instead. Everything else matches the brief's
|
||||
shapes. `Toc` also carries `short_date` (for the header and the ears row) and gains one
|
||||
method, `current_label()`, for the mobile bar.
|
||||
- **The toggled element is the `<nav id="toc">` itself**, and the issue header lives inside
|
||||
it (hidden below `lg`, where the sticky bar already shows the same information). There is
|
||||
still exactly one copy of the list, as the brief requires.
|
||||
- **The progress bar is a `<progress>` element**, not a div with a percentage width — see
|
||||
above; a width would need an inline style.
|
||||
- **Visibility is a CSS rule keyed on `.has-js` + `data-open`**, not the `hidden`
|
||||
attribute: `[hidden]` would also hide the desktop sidebar, and a server-rendered `hidden`
|
||||
would break the no-JS case.
|
||||
- **`theme.js` gained one line** (`classList.add("has-js")`). It is the only script that
|
||||
runs before paint, and without it the mobile panel flashes open on every load.
|
||||
- Article top margins were normalised to `mt-10 lg:mt-12` across the four pages so the
|
||||
sidebar and the reading column start on the same line.
|
||||
- `web::public::PublicIssue` gained `short_date` (no private data; the public template
|
||||
needed the phone dateline).
|
||||
|
||||
## Screenshots
|
||||
|
||||
`/tmp/claude-1000/-home-thallada-workspace-the-daily-epub/e963db53-510f-4312-ac87-460291a92781/scratchpad/step4-shots/`
|
||||
— 137 PNGs. Eight paths (`/issues/2026-09-02`, `…/articles/1`, `…/articles/5` — mid-issue,
|
||||
`…/world`, `…/behind`, and the legacy `/issues/2026-09-01` with its `/world` and `/behind`)
|
||||
× {reader, admin} × {light, dark} × {1280, 1024, 390 mobile closed, 390 mobile open}, named
|
||||
`<path>-<user>-<theme>-<width|mobile>[-open].png`. Plus `-390-ears` crops of the phone ears
|
||||
row for anonymous, reader and admin, `-1280-mid` (scrolled, progress bar advanced) and
|
||||
`-1280-tall` (1500px, the whole sidebar including the back-matter group).
|
||||
|
||||
The Playwright helper is `shoot/shoot4.mjs` (adds `--width/--height`, `--click`, `--scroll`
|
||||
and `--tag` to step 2's `shoot.mjs`, and aborts loudly if the sign-in is throttled).
|
||||
|
||||
## Notes for whoever runs the dev server next
|
||||
|
||||
- `src/web/static/app.css`, `app.js`, `theme.js` and the templates are all compiled into
|
||||
the binary (`include_str!` / askama), so **the server must be rebuilt and restarted after
|
||||
every CSS or template edit** — a browser reload alone shows the old assets.
|
||||
- `server.login_attempts` defaults to 10 per 15 minutes per IP. A screenshot sweep signs in
|
||||
once per invocation and trips the throttle after ten runs, which silently yields
|
||||
screenshots of the sign-in page. The throwaway `./dev/config.toml` was given
|
||||
`login_attempts = 2000`; `dev/` is git-ignored and `seed_dev_db` rewrites it.
|
||||
|
||||
## Left for later
|
||||
|
||||
- Nothing from step 4. On a very long issue the sidebar is taller than the viewport at
|
||||
1280×900, so the back-matter group needs a scroll of the sidebar (by design — the brief's
|
||||
`max-h`); `revealCurrent` handles the reader's own position.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Shared brief — web dashboard implementation agents
|
||||
|
||||
You are implementing one step of `docs/plans/2026-09-03-web-dashboard.md` in the
|
||||
`daily-epub` Rust crate. Read that plan **in full** before writing code — every
|
||||
decision in it is settled (§0), and §2 records verified facts about the crates and
|
||||
the host. Also read `docs/plans/2026-08-15-implementation-notes.md` (conventions:
|
||||
runtime sqlx queries with manual row mapping, `jiff` for time with RFC3339 UTC
|
||||
strings in SQLite, `thiserror`/`anyhow` errors, askama templates, no network in
|
||||
tests, rustfmt defaults, no `unwrap()` outside tests, tracing spans).
|
||||
|
||||
## Ground rules
|
||||
|
||||
- Work on the current branch (`web-dashboard`) in place. Do **not** commit; the
|
||||
orchestrator reviews and commits. Do not create branches or stash.
|
||||
- Do not edit earlier migrations (`0001`–`0003`). Step 1 creates
|
||||
`migrations/0004_web.sql`; later steps may append a new migration file only if
|
||||
the plan says so.
|
||||
- Templates for the web live in `src/web/templates/` (`.html`, HTML-escaped by
|
||||
default); `askama.toml` lists both template dirs. EPUB templates in
|
||||
`src/epub/templates/` are untouched.
|
||||
- Every `|safe` in a template must be one of the sanitized inputs listed in plan
|
||||
§16; user text is never `|safe`.
|
||||
- The existing routes (`/r/…`, `/opds…`, `/files/…`, `/healthz`, `/issues.json`)
|
||||
and their tests keep working unchanged.
|
||||
- Tests: unit tests inline per module; router tests with
|
||||
`tower::ServiceExt::oneshot` against `server::router(...)` over a temp DB
|
||||
(`tempfile`), never the network, never systemd. Add the tests the plan's §17
|
||||
lists for your step.
|
||||
- Keep `cargo fmt`, `cargo clippy --all-targets -- -D warnings` and `cargo test`
|
||||
green. Finish with those three commands and report their output.
|
||||
- **Sandbox note:** inside your sandbox `bind()` on 127.0.0.1 is forbidden, so
|
||||
exactly these pre-existing tests fail there and must be ignored:
|
||||
`curate::llm::tests::anthropic_*` (4), `extract::tests::relative_urls_resolve_against_the_url_we_landed_on`,
|
||||
`server::tests::*` (5 that bind a listener), and any test in `tests/m7_server.rs`.
|
||||
Do not "fix" them. Any **new** router test must use `oneshot` and not bind.
|
||||
The orchestrator runs the full suite outside the sandbox.
|
||||
- Don't gold-plate: implement what the plan says for your step and nothing from
|
||||
later steps beyond stubs the plan explicitly asks for. Where the plan's
|
||||
sketch and the real crate API disagree, follow the real API (read the crate
|
||||
source in `~/.cargo/registry` or `~/.cargo/git`) and note the deviation.
|
||||
- When you finish, write a short handoff at
|
||||
`docs/plans/briefs/web-dashboard/handoff-step<N>.md`: what landed, deviations
|
||||
from the plan and why, anything left for the next step, test counts.
|
||||
|
||||
## Parallel steps (added after step 2)
|
||||
|
||||
From step 3 on, steps may run **in parallel in separate git worktrees**, so
|
||||
file ownership matters:
|
||||
|
||||
- `src/web/dashboard/mod.rs` already declares one submodule per page group
|
||||
(`runs`, `articles`, `ratings`, `profile`, `settings`, `jobs`, `stats`,
|
||||
`users`), each exposing `routes() -> Router<AppState>` that the dashboard
|
||||
router merges under the admin layer. Put your routes in **your** submodule's
|
||||
`routes()`; do not touch `web::router` in `src/web/mod.rs` for dashboard
|
||||
routes. Only step 3 edits the `overview` handler in `dashboard/mod.rs`.
|
||||
- You own: your `src/web/dashboard/<group>.rs` files, your templates under
|
||||
`src/web/templates/dashboard/`, partials you introduce, your tests.
|
||||
- Shared files you may touch **additively only** (no reformatting, reordering
|
||||
or renaming of existing code): `src/web/mod.rs` (helpers), `src/db.rs` (new
|
||||
query helpers appended inside `impl Db`), `src/main.rs`, `src/lib.rs`,
|
||||
`Cargo.toml`, `src/web/static/app.css` and `app.js` (append a block at the
|
||||
end under a `/* step N: … */` comment), `src/web/templates/layout.html`.
|
||||
Keep such edits small so merges stay trivial.
|
||||
- If your worktree is on its own branch, finish by committing **one** commit
|
||||
on that branch (message `Web dashboard step N: …`); the orchestrator merges.
|
||||
If you are on `web-dashboard` itself, do not commit.
|
||||
@@ -0,0 +1,113 @@
|
||||
# Step 1 — Foundation
|
||||
|
||||
Read `docs/plans/briefs/web-dashboard/00-shared.md` first, then the plan
|
||||
`docs/plans/2026-09-03-web-dashboard.md`. This step is plan §19 item 1. Sections
|
||||
that govern it: §2 (verified crate facts — read carefully, especially the
|
||||
axum-login git rev API), §3 (routes and access levels), §4 (architecture, state,
|
||||
templates, design system), §5 (migration and data model), §6 (auth, sessions,
|
||||
CSRF, throttle), §7 (public site), §15 (config keys and dependencies), §16
|
||||
(security checklist), §17 (tests for the parts you build).
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. **Dependencies** (§15): `axum-login` from git at rev
|
||||
`151c72d7a1b4646830f86b4332e6bd6e34d719a7`, `password-auth`, `tower_governor`
|
||||
(feature `axum`), `time`, `async-trait`, `rpassword`, `toml_edit`, `toml`
|
||||
(promote to direct), dev `tower` with `util`. Use `cargo add`; do not hand-pin
|
||||
old versions; do not add `tower-sessions` or `argon2` separately. Confirm
|
||||
`cargo tree -d` shows a single sqlx and a single `libsqlite3-sys`.
|
||||
2. **Migration `migrations/0004_web.sql`** exactly as §5.1 (users, sessions +
|
||||
indexes, `rating_events.user_id`, `config_changes`, `profile_versions`,
|
||||
`jobs`, `runs.report_json`, `issues.issue_json`), plus the index
|
||||
`idx_candidate_runs_article_run ON candidate_runs(article_id, run_id DESC)`
|
||||
from §9.3 (put it in this migration so step 3 needs no new file).
|
||||
3. **Data model changes** (§5.2–§5.4): `RatingEvent.user_id: Option<i64>`
|
||||
(bound by `append_rating_event`, read back by `current_ratings*` /
|
||||
`rated_article_from_row`); `db::finish_run` writes `runs.report_json`;
|
||||
`pipeline::record_issue` writes `issues.report_json` (fixing the never-written
|
||||
column) and `issues.issue_json` (the `Issue` with every
|
||||
`pick.article.content_html` emptied); `GenerateOutcome.run_id: i64`. Add the
|
||||
`db` accessors the loader needs (`issue_by_date`, `issue_dates`/listing for
|
||||
the archive, `latest_issue_date`).
|
||||
4. **`src/web/` skeleton** (§4): `mod.rs` (`WebState`, `Html<T>`, `WebError`,
|
||||
`Page` layout context, pagination + time helpers), `session.rs`
|
||||
(`SqliteSessionStore`, `Backend`, `Credentials`, `AuthSession` alias, `Viewer`,
|
||||
`require_same_origin` middleware), `users.rs` (Role, User with redacting
|
||||
Debug, hash/verify wrappers, username/password rules, the CLI operations),
|
||||
`public.rs` (`PublicIssue` + `/`, `/issues`, `/issues/{date}` public branch,
|
||||
`/feed.xml`, `/robots.txt`), `issue.rs` (the `IssueView` loader with
|
||||
`issue_json` → row fallback per §5.2; the *full* page template is step 2 —
|
||||
for now a signed-in viewer on `/` and `/issues/{date}` may see the public
|
||||
rendering plus the downloads list, and the loader must be complete and
|
||||
tested), `static/{app.css,app.js,favicon.svg}` served from `/static/{file}`
|
||||
with sha256 ETag / 304 and `Cache-Control: public, max-age=86400`, and the
|
||||
templates `layout.html`, `error.html`, `login.html`, `account.html`,
|
||||
`issue_public.html`, `issue_list.html`, `feed_entry.html`, `_pagination.html`.
|
||||
`app.css` implements the §4.4 design tokens, masthead, reading column, the
|
||||
dashboard table/badge/kv/funnel/spark/rating classes (so later steps only add
|
||||
to it), light and dark. `app.js` can be minimal now (the `data-confirm` and
|
||||
`<details>` persistence bits); the rating fetch enhancement is step 2.
|
||||
5. **`AppState` refactor** (§4.2): `config: Arc<RwLock<Arc<Config>>>`,
|
||||
`config_path: Option<PathBuf>`, `web: Arc<WebState>`, `AppState::config()`;
|
||||
`server::serve(config, config_path, db)`; `main::serve` passes
|
||||
`Config::resolve_path(cli.config)`. Existing handlers call `state.config()`.
|
||||
`WebState { jobs: Arc<dyn JobRunner>, started_at, config_mtime }` — define a
|
||||
minimal `JobRunner` trait + `DisabledRunner`/`MockRunner` in `src/web/mod.rs`
|
||||
or a small `src/web/dashboard/jobs.rs` stub now so the state shape is final
|
||||
(step 6 fills in `SystemdRunner` and the pages).
|
||||
6. **Auth** (§6): session layer, auth layer, governor on `POST /login`
|
||||
(`into_make_service_with_connect_info::<SocketAddr>` in `serve`; a periodic
|
||||
`retain_recent` task; a periodic `delete_expired` task), `/login` GET/POST,
|
||||
`/logout`, `/account` (change password, sign out everywhere), the origin-check
|
||||
middleware on every POST except `/r/*`, `login_required!` /
|
||||
`permission_required!` route layers on the sub-routers (a `/dashboard` stub
|
||||
router with an admin-only placeholder overview page is fine so the guard
|
||||
tests in §17 can run now), and the 403 → site error page `map_response`.
|
||||
`/files/*` accepts a session or Basic auth per §8's last paragraph.
|
||||
7. **Security headers** (§16) on every app response: CSP, nosniff, referrer
|
||||
policy; `Cache-Control: no-store` on `/dashboard/*`; `Vary: Cookie` on HTML;
|
||||
public pages `public, max-age=300` only without a session cookie.
|
||||
8. **CLI** `daily-epub users add|passwd|role|disable|enable|list|logout` (§6.1),
|
||||
none of which take the run lock; `users add --admin` bootstraps.
|
||||
9. **Config** (§15): the new `[server]` keys with validation, `Config::default()`,
|
||||
`config.example.toml`, README table (the key-for-key test must pass).
|
||||
10. **README**: the route table gains the new public routes and the
|
||||
session-or-Basic note on `/files/*`; a sentence that `SmartIpKeyExtractor`
|
||||
trusts `X-Forwarded-For` because only nginx reaches the bind address.
|
||||
11. **Tests** from §17: migration, users/passwords, session store, login flow
|
||||
(cookie flags, disabled user, password change logs out other sessions,
|
||||
logout, anonymous `/` sets no cookie, `next` validation), guards, origin
|
||||
check, throttle (burst 3 test config), public rendering
|
||||
(`public_issue_carries_no_generated_text`, comment links, no World Briefing,
|
||||
feed is valid XML with one entry per issue, robots.txt, cache headers),
|
||||
issue_json round trip + fallback loader + `runs.report_json` /
|
||||
`issues.report_json` written and `/issues.json` returning real reports, and
|
||||
the `/files/epub` session-or-Basic behaviour. The binary-level test in
|
||||
`tests/m7_server.rs` (serves `/`, `/issues`, `/feed.xml`, `/login`;
|
||||
`/dashboard` redirects; `users add` then a real login over TCP) — add it, but
|
||||
remember it cannot run inside your sandbox.
|
||||
|
||||
## Notes and traps
|
||||
|
||||
- Read the axum-login source at the pinned rev under `~/.cargo/git/checkouts`
|
||||
after `cargo add` to confirm the `Require` builder, `AuthSession::user().await`,
|
||||
`login(&self, &user)`, the macros' signatures, and the session data key
|
||||
`"axum-login.data"`. §2 describes what was verified; trust the source over the
|
||||
plan's sketch if they differ, and record the difference in your handoff.
|
||||
- `tower-sessions` 0.15 `SessionStore` still uses `async_trait`; axum-login's
|
||||
own traits do not.
|
||||
- `password_auth::verify_password` blocks — run it in `spawn_blocking`, and use
|
||||
a fixed dummy hash for unknown usernames so timing does not reveal existence.
|
||||
- The governor's `per_second` takes seconds per token: `login_window_minutes *
|
||||
60 / login_attempts`.
|
||||
- The `sessions.user_id` column is denormalized best-effort from
|
||||
`record.data["axum-login.data"]["user_id"]` — verify the actual shape axum-login
|
||||
stores (it may be a struct with `user_id` and `auth_hash`).
|
||||
- `PublicIssue::from(&Issue)` is the only constructor and carries no generated
|
||||
text; the no-leak test renders `fixtures::issue()` publicly and asserts the
|
||||
Brief, every summary, every `why`, article body sentences and comment strings
|
||||
are absent.
|
||||
- The `serve_file` Basic-auth fallback must keep the existing OPDS client
|
||||
behaviour byte for byte when no session cookie is present.
|
||||
- Keep `server.rs` as the router root: `router()` merges `web::router()`
|
||||
sub-routers; do not move the existing handlers.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Step 2 — Full issue views, rating widget, POST /rate
|
||||
|
||||
Read `00-shared.md`, then the plan `docs/plans/2026-09-03-web-dashboard.md`
|
||||
(§3, §4.3–§4.4, §5.4, §6.3–§6.4, §8 including §8.1, §16, §17), then
|
||||
`handoff-step1.md` for what the foundation actually shipped. This is plan §19
|
||||
item 2. Build on the step 1 code as it exists; do not restructure it.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `issue_full.html` (§8): masthead + dateline, The Brief (`front_page_html`
|
||||
`|safe`), download buttons (from `IssueView.downloads`), the per-section
|
||||
index like the EPUB's In-this-issue page (title → article page,
|
||||
`source · N min read`, summary, Why it's here), the rating widget for admins,
|
||||
links to World Briefing / Behind the paper (only when present), colophon
|
||||
facts footer. `/` and `/issues/{date}` render this for any signed-in viewer.
|
||||
2. `article.html` at `/issues/{date}/articles/{article_id}` (§8): header
|
||||
(title → source, byline, meta line, Why, social line via
|
||||
`chapters::social_line`, summary), body from `ammonia::clean` with the same
|
||||
configuration the EPUB uses (find it in `src/html.rs`/`src/epub`; do not run
|
||||
`to_xhtml`), `<img>` keep remote `src` and get `loading="lazy"
|
||||
referrerpolicy="no-referrer"`, the discussion via `comments::render_xhtml`
|
||||
when present, prev/next in issue order, "Read online ↗", rating widget for
|
||||
admins. 404 (site error page) when the article is not in that issue.
|
||||
3. `world.html` (`/issues/{date}/world`, `world::render_xhtml`, 404 when
|
||||
absent) and `behind.html` (`/issues/{date}/behind`, the same lines as the
|
||||
EPUB chapter via `chapters::behind_*_line`; near misses link to
|
||||
`/dashboard/articles/{id}` for admins).
|
||||
4. These three routes sit under the `login_required!` layer (§3).
|
||||
5. The HMAC confirmation page (`server::confirmation_page`) gains the
|
||||
"Open this issue on the site" link (§8).
|
||||
6. Rating widget partial `_rating_widget.html` + `src/web/rate.rs` (§8.1):
|
||||
`POST /rate` under the admin layer and the origin check; form and JSON
|
||||
variants; `RatingEvent { source: "dashboard", user_id: Some(viewer.id) }`;
|
||||
`cleared` writes value 0.0 with label `cleared`; `issue_date` optional with
|
||||
the dashboard fallback (`db::latest_issue_date_for_article`); JSON →
|
||||
`200 {"article_id","label","event_id"}`, form → flash + 303 to a validated
|
||||
same-site `next`. Check how the CLI (`main.rs` `cmd_ratings`) writes the
|
||||
`cleared` event and match it exactly.
|
||||
7. `app.js`: intercept `.rating` forms, `fetch` with
|
||||
`credentials: "same-origin"` and `Accept: application/json`, update the
|
||||
active button; keep the no-JS form path working.
|
||||
8. Tests (§17 "Full rendering", "Rating"): signed-in `/issues/{date}` contains
|
||||
Brief, summaries and why; article page contains body and discussion; the
|
||||
fallback loader (no `issue_json`) renders without world/discussion links;
|
||||
downloads listed only for files that exist; `POST /rate` as admin appends a
|
||||
`dashboard` event with `user_id`, as user → 403, anonymous → redirect/401,
|
||||
JSON and form variants, `next` validated, `cleared` value 0.
|
||||
@@ -0,0 +1,53 @@
|
||||
# Step 3 — Dashboard reads: overview, runs, articles
|
||||
|
||||
Read `00-shared.md`, the plan (§3, §4, §9 in full, §16, §17 "Dashboard
|
||||
queries"), the curation v2 plan sections it cites
|
||||
(`docs/plans/2026-09-02-personalized-curation-v2.md` §7 tables, §9–§13
|
||||
signals/admission/utility/editor, §15 explain/stats) and the handoffs for
|
||||
steps 1–2. This is plan §19 item 3. Study `src/curate/telemetry.rs`
|
||||
(`ExplainRow`, `resolve_run`, `explain_row`, `near_misses`, `render_explain`,
|
||||
`SignalsJson`) — the dashboard renders the same data as HTML and must reuse
|
||||
those loaders/types where they fit rather than re-deriving them.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `src/web/dashboard/mod.rs` — the overview (§9.1): last-run card from
|
||||
`runs.report_json` (counters fallback), budget-today card per provider
|
||||
(`config.referenced_providers()` + voyage, `db::provider_spend_for_utc_day`),
|
||||
ratings this week by label, "Unrated picks" (last three issues' picks with
|
||||
no explicit event, with the inline rating widget from step 2), jobs summary
|
||||
(the `jobs` table; the pages come in step 6), config-on-disk `! ` lines from
|
||||
`Config::check_report`. Sparklines are step 6 — leave a clearly marked
|
||||
placeholder section or omit.
|
||||
2. `src/web/dashboard/runs.rs` (§9.2): the list with `?status=` filter and
|
||||
pagination; the detail with header + prev/next run, the funnel (from
|
||||
`candidate_runs` grouped by stage/reason, in pipeline order, bars via the
|
||||
`.funnel` CSS), admission mix, preference state, timings, provider usage,
|
||||
warnings, per-feed top 20, the **config diff** against the previous non-dry
|
||||
run (flatten both `config_json` documents to dotted keys), near misses via
|
||||
`telemetry::near_misses`, and the candidates table with the allow-listed
|
||||
filters (`stage`, `reason`, `admitted_by` via
|
||||
`json_extract(admitted_by,'$[0]')`, `q`, `flag`) and sorts (utility default,
|
||||
rank, triage, quality, fit, title), 100 per page, a `<details>` per row with
|
||||
`_signals_table.html` built from `signals_json`.
|
||||
3. `src/web/dashboard/articles.rs` (§9.3): the list with the latest
|
||||
`candidate_runs` row join (the index `idx_candidate_runs_article_run` exists
|
||||
from migration 0004 — confirm), all listed filters and sorts allow-listed,
|
||||
50 per page; the detail with the six blocks in the plan's order (article
|
||||
facts, assessments incl. `provider_rejected`, run history with expandable
|
||||
signals, neighbours/interests, embedding metadata never the vector, rating
|
||||
events). The "Explain (text)" `<details>` wraps `telemetry::render_explain`
|
||||
output in `<pre>`. The rating widget here carries the note field.
|
||||
4. Templates: `dashboard/overview.html`, `dashboard/runs.html`,
|
||||
`dashboard/run.html`, `dashboard/articles.html`, `dashboard/article.html`,
|
||||
partials `_signals_table.html`, `_candidate_row.html`. Every table inside
|
||||
`.scroll-x`; badges per stage/reason/label; sticky thead; the `data-filter`
|
||||
client-side filter from §4.4 in `app.js`.
|
||||
5. Sort/filter parameters are validated against allow-lists and never
|
||||
interpolated into SQL; an unknown sort falls back to the default, never
|
||||
errors.
|
||||
6. Tests (§17 "Dashboard queries"): funnel counts against a seeded
|
||||
`candidate_runs` set; candidate filters/sorts allow-listed; config diff finds
|
||||
changed dotted keys and ignores unchanged; articles list filters; article
|
||||
detail shows assessments, run history and rating events. Router-level tests
|
||||
for the admin guard on each new route.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Step 4 — Ratings page and profile page
|
||||
|
||||
Read `00-shared.md`, the plan (§3, §10, §11, §14.1 for the job names the
|
||||
profile page references, §16, §17 "Ratings contributions" and "Profile"), the
|
||||
curation v2 plan §6, §9.2–§9.3, §11 (profile/learned adjustments), and the
|
||||
handoffs for steps 1–3. This is plan §19 item 4. Reuse `signals::decay`,
|
||||
`signals::gate`, `signals::direct_feeds`, `PreferenceState::load/summary`,
|
||||
`profile::parse_profile_str`, `profile::parse_interests`,
|
||||
`themes::group_into_themes`, `profile::is_stale`; make
|
||||
`profile::MAX_RATINGS_IN_REBUILD` `pub`.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `src/web/dashboard/ratings.rs` + `dashboard/ratings.html` (§10): header from
|
||||
`PreferenceState::summary()` with the "no embedding" notice; the "How
|
||||
ratings enter the algorithm" `<details>` with current config values inlined
|
||||
and links to the settings groups (the settings page is step 5; link to
|
||||
`/dashboard/settings#curation.feedback` etc. — anchors the settings page will
|
||||
provide); the **Current** tab with every column in the plan's table (verdict
|
||||
+ inline widget with note, article, when·by with username, note, age→decay,
|
||||
neighbour weight or "no embedding", feed credit, in prompt, in rebuild, used
|
||||
last run from `signals_json.neighbours` of the latest non-dry run), filters
|
||||
by label/source/feed/q; the **Events** tab (100 per page, `superseded`
|
||||
marking, filters by label/source/user/date range).
|
||||
2. `src/web/dashboard/profile.rs` + `dashboard/profile.html` (§11): the
|
||||
`profile.md` editor (textarea, Save = origin-checked POST; reject empty or
|
||||
> 64 KB; insert the previous content into `profile_versions`; write
|
||||
atomically via `<path>.tmp` + rename preserving mode; flash), the live parsed
|
||||
preview (server-rendered on GET is fine: the passthrough sections and the
|
||||
extracted `## Interests` lines), history with 200-char previews and Restore
|
||||
(`POST /dashboard/profile/restore`), standing OPML interests grouped by
|
||||
theme (read-only, with path and count), the system prompt
|
||||
(`kv.taste_profile` in a collapsed `<pre>`), the learned adjustments block
|
||||
with its age and whether a rebuild is due, and a "Rebuild profile now"
|
||||
button that POSTs to `/dashboard/jobs/profile-rebuild` (step 6 implements
|
||||
the route; render the form now, disabled with a note if `jobs_enabled` is
|
||||
false).
|
||||
3. Tests (§17): decay/weight/feed-credit/in-prompt/in-rebuild against
|
||||
hand-checked values; "used last run" counts neighbours in `signals_json`;
|
||||
profile save writes the file and a `profile_versions` row; restore swaps;
|
||||
oversized rejected; the parsed preview matches `parse_profile_str`; admin
|
||||
guard on every new route.
|
||||
@@ -0,0 +1,49 @@
|
||||
# Step 5 — Settings
|
||||
|
||||
Read `00-shared.md`, the plan (§3, §4.2 "Config reload", §13 in full, §15,
|
||||
§16, §17 "Settings schema" and "Settings writer"), `src/config.rs` in full
|
||||
(every struct, `validate`, `load`, `resolve_path`, `check_report`, `ENV_PREFIX`,
|
||||
`ENV_SPLIT`, the `DAILY_EPUB_SECRET` alias, the stale-key checks), the README
|
||||
configuration table and `config.example.toml`, and the handoffs for steps 1–4.
|
||||
This is plan §19 item 5.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `src/web/dashboard/settings.rs`: `schema(config, file) -> Vec<SettingGroup>`
|
||||
derived by walking `toml::Value::try_from(Config::default())` and the
|
||||
current config in parallel (§13.1) — one `SettingField` per leaf with
|
||||
`path`, `group`, `kind`, `current`, `default`, `source`
|
||||
(Default/File/Env(name)), `help`, `restart_required`. Kinds and the static
|
||||
enum table exactly as §13.1 item 3; `SETTINGS_HELP` seeded from the README
|
||||
table and `config.example.toml` comments, one entry per shipped key (test).
|
||||
Groups render in the §13.1 item 5 order; each group gets an `id` anchor equal
|
||||
to its dotted path (step 4 links to these).
|
||||
2. `GET /dashboard/settings` (§4.2 reload-on-mtime first; report a file that
|
||||
fails to load with the `!` banner and keep the previous config live),
|
||||
`dashboard/settings.html`: `Secret` fields as set/not set, `Env` fields
|
||||
disabled with the lock note, everything else as an input with the default
|
||||
beside it and a reset affordance; weights note. `POST /dashboard/settings`
|
||||
implements §13.2 exactly: `toml_edit::DocumentMut`, only changed fields
|
||||
written ("explicit beats implicit"), typed by kind, `TextList` as a
|
||||
multi-line array with trailing comma, field errors collected before any
|
||||
write, render → `<path>.tmp.<pid>` → validate with `Config::load(Some(tmp))`
|
||||
→ copy permissions → rename → swap `state.config`; one `config_changes` row
|
||||
per changed key; flash with the restart note for `restart_required` keys;
|
||||
`config_path == None` → 400.
|
||||
3. Providers (§13.3): remove (refused when an `[llm]` role names it) and add
|
||||
(`[a-z0-9_]+`, must not exist, `ProviderConfig::default()` with the chosen
|
||||
kind and placeholders) through the same validate-and-rename path, with
|
||||
`config_changes` rows.
|
||||
4. `GET /dashboard/settings/history` (§13.4), `dashboard/settings_history.html`.
|
||||
5. `systemd/daily-epub.service`: add `/etc/daily-epub` to `ReadWritePaths`
|
||||
(the `SupplementaryGroups` line is step 6).
|
||||
6. Tests (§17): every leaf of `Config::default()` appears exactly once; every
|
||||
shipped key has help; secrets are `Secret` with no value; env detection uses
|
||||
the derived name (set the env var inside the test with a unique key and
|
||||
restore it); enum options round-trip through `validate()`; starting from
|
||||
`config.example.toml` changing three keys preserves every comment and the
|
||||
order of untouched lines; a new key in an absent table creates the table;
|
||||
`TextList` multi-line; an invalid value (e.g. `deep_keep < shortlist_keep`)
|
||||
is rejected and the file untouched; permissions preserved; `config_changes`
|
||||
rows; provider add/remove; removing a referenced provider refused; reload
|
||||
on mtime swaps the config.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Step 6 — Jobs and stats
|
||||
|
||||
Read `00-shared.md`, the plan (§2 host facts, §3, §4.2, §9.1 sparklines, §12,
|
||||
§14 in full, §15 `jobs_enabled`/`journal_lines`, §16, §17 "Stats refactor" and
|
||||
"Jobs"), `src/main.rs` (`lock_holder`, `cmd_*` helpers, `GenerateOptions`),
|
||||
`src/curate/telemetry.rs` (`stats`), `systemd/`, and the handoffs for steps
|
||||
1–5. This is plan §19 item 6.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `src/jobs.rs` (§14.1): the `Job` enum, `parse` (regex `^[a-z0-9-]+$` and
|
||||
the listed forms only), `name`, `unit`, `description`, `takes_lock`,
|
||||
`dangerous`. Register in `lib.rs`.
|
||||
2. `daily-epub job run <name>` (§14.2) in `main.rs`: parse (unknown → exit 2
|
||||
with the catalogue), take the lock via `lock_holder` when `takes_lock()`,
|
||||
find-or-insert the `requested` jobs row → `running`, run the mapped command
|
||||
in-process with the same functions `main` uses, set `ok`/`failed`,
|
||||
`finished_at`, `run_id` (generate, from `GenerateOutcome.run_id`), a one-line
|
||||
`message`; non-zero exit on failure. The job lifecycle helpers live in
|
||||
`src/jobs.rs` as db functions or on `Db`.
|
||||
3. `systemd/daily-epub-job@.service` (copy of the generate unit with the `%i`
|
||||
description and `job run %i`), `systemd/50-daily-epub.rules` exactly as
|
||||
§14.3, `SupplementaryGroups=systemd-journal` on `daily-epub.service`.
|
||||
4. `JobRunner` (§14.4): finalize the trait from step 1's stub; `SystemdRunner`
|
||||
with `tokio::process::Command`, 10 s timeout, stderr in the error;
|
||||
`MockRunner` with recorded calls and scripted statuses; `DisabledRunner`
|
||||
when `jobs_enabled = false`. `serve` picks the runner from config.
|
||||
5. `src/web/dashboard/jobs.rs` + `dashboard/jobs.html` + `dashboard/job.html`:
|
||||
the catalogue cards (Start with `data-confirm` for `dangerous()`, a date
|
||||
input on the `generate-<date>` card), the jobs table, `POST
|
||||
/dashboard/jobs/{name}` (409 + flash for a duplicate `requested`/`running`
|
||||
unit; insert; `runner.start`; failed start marks the row), `GET
|
||||
/dashboard/jobs/{id}` (row, live `UnitStatus`, journal tail of
|
||||
`server.journal_lines` in `<pre>`, the 30-second "unit exited before the job
|
||||
started" rule). `reload_if_changed` on every job start (§4.2). `app.js`
|
||||
refreshes the job page every 5 s while `requested`/`running`.
|
||||
6. Stats (§12): refactor `telemetry::stats` into `stats_data -> StatsData` +
|
||||
`render_stats_text(&StatsData) -> String` with byte-identical CLI output (a
|
||||
test pins it against a seeded DB before and after); `GET /dashboard/stats`
|
||||
with `?days=14|30|90`, the tables, three server-rendered SVG sparklines via
|
||||
`_sparkline.html`, the retriever yield table.
|
||||
7. Overview sparklines (§9.1): cost per run, selected per run, generation
|
||||
seconds over the last 30 non-dry runs, using the same partial.
|
||||
8. Tests (§17 "Stats refactor", "Jobs"): `Job::parse` accepts the catalogue and
|
||||
the dated form, rejects `../x`, uppercase and unknown names; `POST
|
||||
/dashboard/jobs/{name}` inserts a row and calls `MockRunner::start` with the
|
||||
right unit; a running duplicate is refused; a failed start marks the row;
|
||||
`job run` in-process (a `--skip-llm`-style config with mocked providers, e.g.
|
||||
`features-prune` or `profile-rebuild` against a temp DB) flips
|
||||
`requested → running → ok`; the polkit rule file is present and its regex
|
||||
matches the unit name format (string test); stats text unchanged.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Step 7 — Docs, users page, polish
|
||||
|
||||
Read `00-shared.md`, the plan (§3, §6.1, §7.2, §18, §19 item 7, §20
|
||||
acceptance criteria), the README, `config.example.toml`,
|
||||
`docs/plans/2026-08-15-implementation-notes.md`, `docs/runbooks/`, and every
|
||||
`handoff-step*.md`. This is plan §19 item 7.
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `src/web/dashboard/users.rs` + `dashboard/users.html`: read-only list of
|
||||
users (username, role, disabled, created, last login, open session count)
|
||||
and a note that edits happen through the `users` CLI.
|
||||
2. README: the site (public vs signed-in vs admin), roles, the `users` CLI,
|
||||
jobs (unit + polkit), settings (in-place writes, env locks), the updated
|
||||
route table, the reverse-proxy note about `X-Forwarded-For`.
|
||||
3. `config.example.toml`: confirm the new `[server]` keys are documented with
|
||||
comments matching the README table.
|
||||
4. `docs/plans/2026-08-15-implementation-notes.md`: a "Web dashboard
|
||||
(2026-09-03)" section recording the verified facts with dates (polkit 124
|
||||
and the rule, unit changes, toml_edit in-place writes, axum-login git rev +
|
||||
tower-sessions 0.15 + our store, password-auth, tower_governor, the
|
||||
sandbox/test caveats) and the implementation-time decisions the handoffs
|
||||
recorded.
|
||||
5. `docs/runbooks/web-dashboard-rollout.md` with the nine steps of plan §18,
|
||||
written for the production host.
|
||||
6. Layout polish: Atom `<link rel="alternate">` and favicon in `<head>`, the
|
||||
Users link in the admin nav, a pass over dark mode and narrow screens in
|
||||
`app.css` (every wide table scrolls in `.scroll-x`; the masthead wraps; nav
|
||||
collapses to wrapping links), and the 404/500 pages in the site layout.
|
||||
7. Walk plan §20's acceptance criteria one by one and fix any gap you can
|
||||
close inside this step; list the rest in your handoff.
|
||||
8. Tests: the users page under the admin guard; a smoke test that every
|
||||
template referenced by a route renders with the fixture data (extend the
|
||||
existing router tests rather than duplicating setup).
|
||||
|
||||
## Follow-ups collected during the orchestrator's reviews of steps 1–6
|
||||
|
||||
- Download buttons on the full issue page show raw byte counts; render them
|
||||
human-readable (KB/MB).
|
||||
- The `/files/*` deviation: with no Basic auth configured the files stay
|
||||
public (see handoff-step1 review notes); document that in the README's route
|
||||
table and the rollout runbook.
|
||||
- The settings page cannot remove a shipped provider (handoff-step5); mention
|
||||
it in the README settings section.
|
||||
- Every step left a handoff in this directory; fold their "left for later"
|
||||
items into the acceptance walk-through and list anything still open.
|
||||
- Two Codex review documents exist under `docs/reviews/` for this plan; leave
|
||||
them as they are.
|
||||
@@ -0,0 +1,72 @@
|
||||
# Step 1 handoff — foundation
|
||||
|
||||
## Landed
|
||||
|
||||
- Added migration `0004_web.sql`: users, SQLite-backed sessions and indexes,
|
||||
rating attribution, config/profile/job history, report/issue snapshots, and
|
||||
the candidate article/run index. `RatingEvent`/`RatedArticle` now carry a
|
||||
nullable user id.
|
||||
- Persisted full `runs.report_json`, final `issues.report_json`, and a compact
|
||||
`issues.issue_json` with article bodies removed. The issue loader rehydrates
|
||||
bodies from `articles` and has a reduced-row fallback for pre-migration
|
||||
issues. `GenerateOutcome` exposes `run_id`.
|
||||
- Added the `web` skeleton, final shared-state shape, Askama layout/error/public
|
||||
templates, embedded CSS/JS/favicon with SHA-256 ETags, security/cache/request
|
||||
headers, and `DisabledRunner`/`MockRunner` stubs.
|
||||
- Added `axum-login` at the required git revision, the local sqlx 0.9 session
|
||||
store, password-auth users/backend, session/auth route layers, role guards,
|
||||
same-origin POST middleware, login governor and cleanup tasks, login/logout,
|
||||
account/password/session revocation, and session-or-Basic file downloads.
|
||||
- Added public latest/archive/issue pages, the stripped `PublicIssue` boundary,
|
||||
Atom feed, and robots policy. Anonymous page loads remain session-cookie-free.
|
||||
- Added all `daily-epub users` commands without the pipeline lock, new server
|
||||
configuration/defaults/validation, example config, and README command/route/
|
||||
reverse-proxy documentation.
|
||||
- Expanded `tests/m7_server.rs` to cover env-only public routes, dashboard
|
||||
redirect, CLI admin bootstrap, and a real TCP login/account request.
|
||||
|
||||
## Deviations and follow-up
|
||||
|
||||
- `rpassword` could not be added: it is absent from the local Cargo cache and
|
||||
this sandbox cannot resolve `index.crates.io` (three retries failed). The CLI
|
||||
currently uses an equivalent Unix `/dev/tty` no-echo double prompt and keeps
|
||||
`--password-stdin`. Replace that helper with `rpassword::prompt_password`
|
||||
after adding the dependency in a network-enabled environment.
|
||||
- Cargo resolved direct `toml_edit` to 0.22.27 rather than the plan's observed
|
||||
0.25.x release; it was selected by `cargo add` for the available toolchain and
|
||||
lock graph, not hand-pinned.
|
||||
- The finished issue report is attached immediately after `finish_run`, rather
|
||||
than during the earlier issue snapshot write, because publish timing and the
|
||||
final run status are not complete at snapshot time. The stored observable
|
||||
value is the same final serialized report.
|
||||
- The pinned axum-login source confirms `AuthSession::user().await`, immutable
|
||||
`login`/`logout`, the macro route layers, and session key
|
||||
`"axum-login.data"`; the implementation follows those real APIs.
|
||||
- Full signed-in issue/article rendering and the real `POST /rate` handler are
|
||||
step 2. Step 1 supplies the protected dashboard overview and admin-only 501
|
||||
rate stub so route-guard tests exercise the final boundary.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt --check`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- `cargo test` with sandbox-bound tests skipped: **389 passed, 0 failed**.
|
||||
Excluded were the four named Anthropic listener tests, the relative-URL
|
||||
listener test, five `server::tests` listener tests, both `m7_server` TCP
|
||||
tests, and three existing OpenAI fake-server tests that also bind loopback.
|
||||
- Focused `cargo test web:: -- --nocapture`: **16 passed, 0 failed**.
|
||||
- `cargo tree -i` shows one `sqlx` version (0.9.0) and one
|
||||
`libsqlite3-sys` version (0.37.0).
|
||||
|
||||
## Orchestrator review (2026-09-03)
|
||||
|
||||
- `rpassword` added (the sandbox had no network); `read_password_hidden` is
|
||||
now `rpassword::prompt_password`. `toml_edit` moved to 0.25 as the plan says.
|
||||
- **Deviation from plan §8 kept on purpose:** `/files/*` with *no* Basic auth
|
||||
configured stays public, exactly as before. The plan wanted a redirect to
|
||||
`/login` there, but the public OPDS feed's acquisition links point at those
|
||||
files and an e-reader cannot log in; acceptance criterion 11 ("`/files`
|
||||
behave as before") and "OPDS clients are unaffected" win. With Basic auth
|
||||
configured, a session cookie of any role bypasses it.
|
||||
- A short new password on `/account/password` is a 400 form error, not a 500.
|
||||
- Full suite outside the sandbox: 370 lib + all integration tests green.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Step 2 handoff — full issues and ratings
|
||||
|
||||
## Landed
|
||||
|
||||
- Signed-in `/` and `/issues/{date}` now render the full issue: Brief, download
|
||||
links for artifacts that still exist, the section/article index with summaries
|
||||
and `why`, admin rating widgets, World/Behind links when their snapshot data is
|
||||
present, and colophon facts.
|
||||
- Added login-protected article, World Briefing, and Behind the paper pages.
|
||||
Article pages use the EPUB's ammonia cleaning path without XHTML conversion,
|
||||
retain remote images with lazy/no-referrer attributes, include rendered
|
||||
discussions, previous/next navigation, source links, and admin ratings.
|
||||
- Replaced the Step 1 `/rate` stub with the admin-only form/JSON handler. Events
|
||||
are append-only `dashboard` events attributed to the viewer; missing issue
|
||||
dates use `latest_issue_date_for_article`; clear events use the CLI's exact
|
||||
`cleared`/`0.0` representation; form redirects validate `next` and carry a
|
||||
flash, while JSON returns the event id.
|
||||
- Added the no-JS rating partial and JavaScript enhancement, active-state
|
||||
updates, issue/article styling, and the signed-in private cache policy.
|
||||
- The e-ink HMAC confirmation page now links to the corresponding site issue.
|
||||
- Added router tests for full/fallback rendering, article discussions and image
|
||||
handling, World/Behind pages, 404s and login protection, artifact gating,
|
||||
both rating representations, role guards, attribution, fallback dates,
|
||||
validated redirects, clear values, admin widget state, and near-miss links.
|
||||
|
||||
## Deviations and notes
|
||||
|
||||
- The pinned `axum-login` dependency disables tower-sessions' `axum-core`
|
||||
feature, so its re-exported `Session` does not implement an Axum extractor in
|
||||
this dependency graph. Handlers use `Extension<Session>` to read the exact
|
||||
session already installed by the auth layer; no second tower-sessions
|
||||
dependency was added.
|
||||
- The web form and JSON response use `down`, as specified for the widget, while
|
||||
the persisted event label is `not_for_me`, matching `cmd_ratings` and the
|
||||
existing learned-rating queries exactly.
|
||||
- No migration was needed. Step 3 can construct `RatingWidget` with
|
||||
`show_note = true` for dashboard article variants.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- Focused `cargo test web:: -- --nocapture`: **22 passed, 0 failed**.
|
||||
- `cargo test` with the documented sandbox listener tests skipped:
|
||||
**395 passed, 0 failed, 15 filtered out**. The filtered tests were the four
|
||||
Anthropic listener tests, three OpenAI fake-server listener tests noted in the
|
||||
Step 1 handoff review, the relative-URL listener test, five `server::tests`
|
||||
listener tests, and both `tests/m7_server.rs` tests.
|
||||
|
||||
## Orchestrator review (2026-09-03)
|
||||
|
||||
- Accepted as is. `Extension<Session>` for flashes is fine (the auth layer's
|
||||
session manager inserts it); no second tower-sessions dependency.
|
||||
- Cosmetic follow-up for step 7: download buttons show raw byte counts;
|
||||
render them human-readable (KB/MB).
|
||||
- Full suite outside the sandbox: 376 lib + all integration tests green.
|
||||
@@ -0,0 +1,100 @@
|
||||
# Step 3 handoff — dashboard reads: overview, runs, articles
|
||||
|
||||
## Landed
|
||||
|
||||
- `src/web/dashboard/mod.rs`: the overview (§9.1) — last-run card built from
|
||||
`runs.report_json` via `RunReport::info_block()` (legacy-counter fallback
|
||||
when the report is missing or unparseable), budget-today `<meter>` per
|
||||
referenced provider plus voyage from `db::provider_spend_for_utc_day(now)`,
|
||||
ratings this week by label, "Unrated picks" (last three issues' picks with
|
||||
no explicit event, each with the step 2 rating widget posting back to
|
||||
`/dashboard`), a jobs summary read straight from the `jobs` table (active
|
||||
rows plus the last five finished; the pages themselves are step 6), and the
|
||||
`! ` lines of `Config::check_report`. Sparklines are a marked HTML comment
|
||||
for step 6. The module also carries the helpers the read pages share:
|
||||
`Pager`, `SignalsView` (the parsed `signals_json` behind
|
||||
`_signals_table.html`), `allow_listed`, `like_pattern`, `Bind`/`bind_all`/
|
||||
`dynamic_query`, formatting helpers, and the `STAGES`/`REASONS`/
|
||||
`RETRIEVERS` allow-lists.
|
||||
- `src/web/dashboard/runs.rs` (§9.2): `/dashboard/runs` with `?status=` and
|
||||
pagination (50), and `/dashboard/runs/{id}` with header (issue link when the
|
||||
issue exists, previous/next run by id), the funnel from `candidate_runs`
|
||||
grouped by stage and reason, admission mix, preference state, timings,
|
||||
provider usage, warnings (`#warnings` anchor used by the overview and the
|
||||
list), top-20 feeds, the config diff (`flatten_json` + `config_diff`)
|
||||
against the previous non-dry run that recorded a config, ten near misses via
|
||||
`telemetry::near_misses`, and the candidates table (100 per page) with the
|
||||
`stage`/`reason`/`admitted_by`/`q`/`flag` filters and `utility` (default)/
|
||||
`rank`/`triage`/`quality`/`fit`/`title` sorts. Each row's title is a
|
||||
`<details>` that expands `_signals_table.html`.
|
||||
- `src/web/dashboard/articles.rs` (§9.3): `/dashboard/articles` (50 per page)
|
||||
joining the best entry, the latest `candidate_runs` row through the
|
||||
`MAX(run_id)` subquery (the index `idx_candidate_runs_article_run` is in
|
||||
migration 0004 — confirmed), both assessments, the current explicit rating
|
||||
and the latest publication; every listed filter and sort;
|
||||
`/dashboard/articles/{id}` with the six blocks in the plan's order,
|
||||
`provider_rejected` rows called out as such, run history rows expanding
|
||||
their signals, neighbours/interests from the latest row, embedding metadata
|
||||
only (never the vector), all rating events with source/user/note/value, the
|
||||
rating widget with `show_note = true`, and "Explain (text)" wrapping
|
||||
`telemetry::render_explain` verbatim in `<pre>`.
|
||||
- Templates: `dashboard/overview.html`, `runs.html`, `run.html`,
|
||||
`articles.html`, `article.html`, partials `_signals_table.html`,
|
||||
`_candidate_row.html` and `dashboard/_pager.html`. Every table sits in
|
||||
`.scroll-x`; badges per stage/reason/label/status; tables carry
|
||||
`data-filter`.
|
||||
- `app.css` / `app.js`: one appended block each (`/* step 3 … */`): cards,
|
||||
filters, funnel bar, pager, signals details, diff colours, and the
|
||||
filter-as-you-type behaviour for `table[data-filter]` (inserts a search
|
||||
input before the table's `.scroll-x` wrapper, hides non-matching rows on
|
||||
the current page only).
|
||||
|
||||
## Deviations and notes
|
||||
|
||||
- **sqlx 0.9 dynamic SQL.** `sqlx::query` only accepts `&'static str` or
|
||||
`AssertSqlSafe`, so the list queries go through `dashboard::dynamic_query`,
|
||||
which wraps the assembled string in `AssertSqlSafe`. The audit holds
|
||||
because the string is built only from constants and allow-listed fragments
|
||||
(`CANDIDATE_SORTS`/`ARTICLE_SORTS` map names to fixed `ORDER BY` text);
|
||||
every user value is bound through `bind_all`. An unknown sort or filter
|
||||
value falls back to the default / is dropped, never an error.
|
||||
- **Funnel semantics.** `stage` records where a row stopped, so the bars
|
||||
show the cumulative "reached" count (rows at this stage or a later one, so
|
||||
the first bar is everything considered) with "stopped here" and the reason
|
||||
breakdown beside it. Widths are `<svg width="N%">` children of `.funnel`
|
||||
because the CSP (`style-src 'self'`) forbids inline `style` attributes;
|
||||
the budget card uses `<meter>` for the same reason.
|
||||
- **Prev/next run** links go to the neighbouring run ids overall (the
|
||||
header already links the date's issue); a rerun of the same date is
|
||||
therefore the immediate neighbour.
|
||||
- **Extract method** is not shown on the article page: `db::article_from_row`
|
||||
hard-codes `ExtractMethod::Miniflux` for every loaded article, so the value
|
||||
would be meaningless. `excerpt_only` is shown instead. Source `kind`s render
|
||||
via their `Debug` names (`Feed`, `Scour`, …).
|
||||
- **`admitted_by` filter** is allow-listed against the six retriever names
|
||||
and still applied as the plan's prefix `LIKE` on
|
||||
`json_extract(admitted_by, '$[0]')`.
|
||||
- The `_pagination.html` partial from step 1 (text only) was left alone; the
|
||||
dashboard uses its own `dashboard/_pager.html` with prev/next links that
|
||||
preserve the other query parameters.
|
||||
- Only additive edits outside my files: two appended blocks in `app.css` and
|
||||
`app.js`. No changes to `web/mod.rs`, `db.rs`, `rate.rs` or `telemetry.rs`
|
||||
(a private `first_retriever` in telemetry was reimplemented as
|
||||
`admitted_by_parts` rather than made `pub`).
|
||||
|
||||
## Left for later steps
|
||||
|
||||
- Step 6: overview sparklines (placeholder comment in `overview.html`), and
|
||||
the `/dashboard/jobs` pages the overview's job links point at.
|
||||
- Step 4: `/dashboard/ratings`, linked from the overview's ratings card.
|
||||
- The overview's ratings-this-week card counts all explicit events in the
|
||||
last seven days by label (a `cleared` is listed but not counted in the
|
||||
total).
|
||||
|
||||
## Verification (outside the sandbox)
|
||||
|
||||
- `cargo fmt`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- `cargo test web::dashboard`: **11 passed, 0 failed**.
|
||||
- `cargo test` (full suite): **387 lib tests passed, 0 failed**, plus every
|
||||
integration test binary green (7, 2, 3, 4, 7, 9, 2), nothing skipped.
|
||||
@@ -0,0 +1,79 @@
|
||||
# Step 4 handoff — ratings page and profile page
|
||||
|
||||
## Landed
|
||||
|
||||
- `src/web/dashboard/ratings.rs` + `dashboard/ratings.html` (`GET /dashboard/ratings`,
|
||||
plan §10). Header from `PreferenceState::load(..).summary()` (the same loader
|
||||
the pipeline uses) plus the "N rated articles have no embedding" notice; the
|
||||
"How ratings enter the algorithm" `<details>` with the live config values
|
||||
inlined and links to `/dashboard/settings#curation.feedback`,
|
||||
`#curation.ranking` and `#curation.ranking.weights.preliminary`.
|
||||
**Current** tab: one row per article from
|
||||
`current_ratings_including_cleared(36_500)` with the verdict badge and the
|
||||
inline rating widget (`show_note = true`), article → `/dashboard/articles/{id}`
|
||||
with feed and issue date, when · source · username, note, age → decay,
|
||||
neighbour weight (`value × decay`, or "no embedding"; a "(beyond lookback)"
|
||||
marker when older than `rating_lookback_days`), feed credit per direct feed,
|
||||
in-prompt / in-rebuild ticks from the rank among non-cleared verdicts, and
|
||||
"used last run" from `signals_json.neighbours` of the latest non-dry run
|
||||
(count of candidates and how many were selected). Filters: label, source,
|
||||
feed, q. **Events** tab: every `rating_events` row, 100 per page,
|
||||
`superseded` marked via an `EXISTS` on a later explicit event; filters label,
|
||||
source, user, from/to date. Filter values are validated (allow-listed labels,
|
||||
`[a-z0-9_]` sources, parsed dates/ids) and only ever bound; the dynamic
|
||||
`WHERE` is assembled from fixed clause strings under `sqlx::AssertSqlSafe`.
|
||||
- `src/web/dashboard/profile.rs` + `dashboard/profile.html` (plan §11).
|
||||
`GET /dashboard/profile`: the `profile.md` textarea, the server-rendered
|
||||
parsed preview (passthrough body in a `<pre>`, extracted `## Interests`
|
||||
lines), history (50 newest `profile_versions` with 200-char previews and a
|
||||
Restore form), OPML standing interests grouped by `group_into_themes` with
|
||||
path and count, the learned adjustments with the prompt version, build time,
|
||||
age and whether `profile::is_stale` says a rebuild is due, the collapsed
|
||||
system prompt (`kv.taste_profile`), and the "Rebuild profile now" form
|
||||
posting to `/dashboard/jobs/profile-rebuild` (disabled with a note when
|
||||
`server.jobs_enabled` is false). `POST /dashboard/profile`: CRLF→LF
|
||||
normalization, reject empty or > 64 KB (400), record the previous file in
|
||||
`profile_versions` with `saved_by`, write `<path>.tmp` + rename preserving
|
||||
the existing mode, flash "Saved; the next run rebuilds the system prompt."
|
||||
Identical content is a no-op flash with no version row.
|
||||
`POST /dashboard/profile/restore` (`version_id`): records the current file
|
||||
as a new version and writes the chosen one back; unknown id → 404.
|
||||
- `src/curate/profile/mod.rs`: `MAX_RATINGS_IN_REBUILD` and `stored_version`
|
||||
are now `pub` (the profile page shows the prompt version and build time).
|
||||
- `src/web/static/app.css`: a `/* step 4 */` block (tabs, filters, contribution
|
||||
table, profile grid, previews).
|
||||
|
||||
## Deviations and notes
|
||||
|
||||
- Feed credit is shown as `value × decay / n`, not the plan §10 table's
|
||||
`value / n`: `signals::feed_rates` credits the **decayed** weight ("decayed as
|
||||
in §9.2", curation plan §9.3), and the page shows what the ranker computes.
|
||||
- "Used last run" scans the run's `candidate_runs` rows once
|
||||
(`signals_json LIKE '%"neighbours":[{%'`) and parses each `SignalsJson`,
|
||||
instead of one `LIKE '%"article_id":<id>,%'` query per rated article; same
|
||||
result, one query.
|
||||
- The verdict block and rebuild set are count-bounded, so the neighbour weight
|
||||
is still shown for ratings older than `rating_lookback_days`, with a
|
||||
"(beyond lookback)" marker, since `PreferenceState::load` would not load them.
|
||||
- `RatedArticle` does not carry the event `source`; the Current tab reads it
|
||||
with one small query per row (ratings are sparse — hundreds at most).
|
||||
- `TasteProfile.verdicts` is not persisted, so the prompt's verdict count is
|
||||
derived by counting lines after `## Recent verdicts` in the stored text.
|
||||
- No new migration, no changes to `db.rs`, `web/mod.rs`, `layout.html` or
|
||||
`app.js`.
|
||||
|
||||
## Left for later steps
|
||||
|
||||
- Step 5 must provide the settings anchors linked from the ratings page:
|
||||
`curation.feedback`, `curation.ranking`, `curation.ranking.weights.preliminary`.
|
||||
- Step 6 implements `POST /dashboard/jobs/profile-rebuild`; the profile page
|
||||
already renders the form (with `data-confirm`).
|
||||
- Step 3's article detail pages are linked as `/dashboard/articles/{id}`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt --check`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- `cargo test` (outside the sandbox, nothing skipped): **390 lib tests passed,
|
||||
0 failed** (14 new in `web::dashboard::{ratings,profile}`), plus every
|
||||
integration suite green (7, 2, 3, 4, 7, 9, 2 tests).
|
||||
@@ -0,0 +1,62 @@
|
||||
# Step 5 handoff — settings
|
||||
|
||||
## Landed
|
||||
|
||||
- `src/web/dashboard/settings.rs`: a schema derived by walking serialized
|
||||
default and live `Config` values, including optional/env-only fields, exact
|
||||
field kinds and enum choices, derived environment names, source tracking,
|
||||
README/example-backed help text, stable group order, and dotted-path anchors.
|
||||
- `/dashboard/settings`: reload-on-mtime with a visible load-error banner and
|
||||
last-good config retention; secret presence-only rendering, environment
|
||||
locks, typed inputs, defaults/reset controls, and weight normalization notes.
|
||||
- Typed `toml_edit` saves that preserve comments/order, collect field errors,
|
||||
write multiline string arrays, create missing tables, validate through
|
||||
`Config::load` in `<path>.tmp.<pid>`, preserve permissions, rename atomically,
|
||||
swap the live config, and record one attributed `config_changes` row per key.
|
||||
- Provider add/remove flows through the same validated writer, including name
|
||||
and kind validation, role-reference refusal, placeholder defaults, audit
|
||||
history, and redaction of a hand-written provider `api_key` from history.
|
||||
- `/dashboard/settings/history`, newest first at 100 rows per page.
|
||||
- A step 5 CSS block and reset helper in `app.js`; `/etc/daily-epub` added to
|
||||
the server unit's `ReadWritePaths`.
|
||||
- Sixteen settings tests covering every schema and writer case listed in §17,
|
||||
including router-level rendering and POST behavior through `oneshot`.
|
||||
|
||||
## Deviations and notes
|
||||
|
||||
- Shipped providers cannot be removed, even when unreferenced. They are members
|
||||
of `Config::default()`, so deleting their TOML table would immediately restore
|
||||
the built-in entry and falsely report success. They can be edited or left
|
||||
unreferenced; custom providers can be removed normally.
|
||||
- `FieldKind::Enum` owns `Vec<String>` rather than the plan sketch's static
|
||||
slice because `llm.bulk` and `llm.editor` must include provider names derived
|
||||
at runtime. The rendered behavior and validation table are unchanged.
|
||||
- Restart notices also cover `server.public_url`, `server.session_days`,
|
||||
`server.login_attempts`, and `server.login_window_minutes`, in addition to
|
||||
the plan sketch's `database_path` and `server.bind`, because those values are
|
||||
captured when the server/session/throttle layers are built.
|
||||
- An absent key whose effective value is already the submitted default remains
|
||||
absent. This reconciles “only changed fields” with the no-JavaScript form
|
||||
posting every editable field; resetting an explicit non-default file value
|
||||
still writes the default explicitly.
|
||||
- Step 6 should call `WebState::reload_if_changed` before starting each job, as
|
||||
§4.2 requires. Step 5 supplies the shared helper; this branch's jobs module is
|
||||
intentionally still the parallel-step stub.
|
||||
- The shared brief's sandbox list omits three pre-existing OpenAI fake-server
|
||||
tests that bind through the same `curate::llm::tests::serve` helper as its four
|
||||
named Anthropic tests. The step 1 handoff records those three additional
|
||||
sandbox failures; they are not settings regressions.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- Focused `cargo test web::dashboard::settings -- --nocapture`: **16 passed,
|
||||
0 failed**.
|
||||
- Unfiltered `cargo test`: **379 library tests passed** before the harness
|
||||
reported the expected loopback-bind failures (the 10 named library tests plus
|
||||
the three OpenAI tests noted above); no settings test failed.
|
||||
- `cargo test` with all pre-existing loopback listener tests excluded:
|
||||
**411 passed, 0 failed, 15 filtered out** (379 library, 7 binary-unit, and 25
|
||||
integration tests passed; 13 library listener tests and both `m7_server`
|
||||
tests were filtered).
|
||||
@@ -0,0 +1,73 @@
|
||||
# Step 6 handoff — jobs and stats
|
||||
|
||||
## Landed
|
||||
|
||||
- Added `src/jobs.rs`: the fixed, regex-constrained job catalogue; jobs-table
|
||||
query/lifecycle helpers; single-line terminal messages; `SystemdRunner` with
|
||||
ten-second command timeouts, captured stderr and systemd status parsing.
|
||||
- Added `daily-epub job run <name>`. It uses the existing command lock policy,
|
||||
claims or creates the requested row, calls the same in-process helpers as the
|
||||
ordinary CLI commands, records `ok`/`failed`, stores the generated run id,
|
||||
and propagates failures to a non-zero process exit. Existing profile,
|
||||
features and social CLI output retains its prior line content.
|
||||
- Finalized `JobRunner`: production `serve` installs `SystemdRunner` when
|
||||
`server.jobs_enabled` is true; disabled servers use `DisabledRunner`;
|
||||
`AppState::new` keeps its existing test behavior; `MockRunner` records calls
|
||||
and scripts starts, statuses and logs.
|
||||
- Added the admin Jobs list/start/detail routes and templates: catalogue cards,
|
||||
dated generation, job history, duplicate conflict handling, start-failure
|
||||
persistence, live unit fields, configured journal tail, 30-second
|
||||
pre-claim failure detection, disabled-state messaging and five-second active
|
||||
refresh.
|
||||
- Added the systemd job template, the §14.3 polkit rule verbatim, and
|
||||
`SupplementaryGroups=systemd-journal` on the server unit.
|
||||
- Split stats into `stats_data` plus `render_stats_text` while pinning the
|
||||
complete previous CLI output. Added the 14/30/90-day stats page, aggregate
|
||||
tables, retriever yield, per-run table, daily provider costs and the three
|
||||
requested SVG charts.
|
||||
- Replaced the overview placeholder with cost-per-run, selected-per-run and
|
||||
generation-time sparklines over the last 30 finished non-dry runs. SVGs use
|
||||
presentation attributes and CSS classes only; no inline styles or template
|
||||
`safe` filters were added.
|
||||
- Added/expanded tests for the catalogue, job lifecycle and run link, CLI lock
|
||||
mapping, in-process job success/failure, polkit/unit files, runner status
|
||||
parsing, every jobs route behavior, exact stats text/data, stats routes and
|
||||
both sparkline forms.
|
||||
|
||||
## Deviations and integration notes
|
||||
|
||||
- Per the parallel-step constraint, job start contains exactly the requested
|
||||
`// TODO(step 5 merge): reload_if_changed` marker and does not duplicate step
|
||||
5's helper. The step-5 merge must replace that marker with its helper call.
|
||||
- The offline in-process job test uses `features-prune`; generate's `run_id`
|
||||
persistence is tested separately through the same lifecycle helper because a
|
||||
full generate would require Miniflux/network fixtures. Production generate
|
||||
and dry-run mappings both store `GenerateOutcome.run_id`.
|
||||
- The server unit intentionally does not add `/etc/daily-epub` to
|
||||
`ReadWritePaths` here; step 5 owns that additive unit change.
|
||||
- `StatsData.durations` retains finished dry runs because that is what the old
|
||||
CLI calculation included and the text must remain byte-identical. Dashboard
|
||||
per-run series and overview sparklines exclude dry runs as specified.
|
||||
|
||||
## Left for later
|
||||
|
||||
- Step-5 merge: call its `reload_if_changed` implementation at the marked job
|
||||
start site and combine its `/etc/daily-epub` unit path change.
|
||||
- Step 7 owns user-facing documentation and deployment/runbook polish.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- Raw `cargo test`: **406 passed, 13 failed** in the library before Cargo
|
||||
stopped; every failure was a sandbox-denied loopback listener (four named
|
||||
Anthropic tests, three pre-existing OpenAI fake-server tests documented in
|
||||
the step-1 handoff, the named extractor test and five named server tests).
|
||||
- `cargo test` with those listener tests and both `tests/m7_server.rs` listener
|
||||
tests skipped: **440 passed, 0 failed, 15 filtered out** across all targets.
|
||||
- Focused `cargo test jobs::`, `cargo test job_run`, and `cargo test stats`:
|
||||
pass.
|
||||
- `node --check` on the polkit JavaScript (via stdin): pass.
|
||||
- `systemd-analyze verify systemd/daily-epub-job@.service`: no diagnostics for
|
||||
the job unit.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# Step 7 handoff — docs, users page, and polish
|
||||
|
||||
## Landed
|
||||
|
||||
- Added the admin-only, read-only `/dashboard/users` page with username, role, enabled/disabled
|
||||
status, created time, last login, and unexpired session count. The page directs edits to the
|
||||
existing `daily-epub users` CLI.
|
||||
- Expanded the README with the public/user/admin boundary, user lifecycle commands, complete
|
||||
route table, systemd+polkit Jobs setup, comment-preserving Settings behavior, environment
|
||||
locks, shipped-provider removal limitation, public `/files/*` behavior without Basic auth,
|
||||
and the `X-Forwarded-For`/loopback trust boundary. Updated `[server]` comments in
|
||||
`config.example.toml` to match.
|
||||
- Added the dated Web dashboard section to the implementation notes: polkit 124 and its narrow
|
||||
rule, service-unit changes, the `toml_edit` write path, the pinned auth/session/password/
|
||||
throttle stack, sandbox test caveats, and the implementation-time decisions from steps 1–6.
|
||||
- Added `docs/runbooks/web-dashboard-rollout.md`, with the plan's nine production rollout steps,
|
||||
commands, permissions, public-files compatibility note, and smoke checks.
|
||||
- Completed layout polish: Atom discovery and favicon remain in `<head>`, Users remains in the
|
||||
admin nav, both navs now wrap cleanly, narrow cards/forms/rating controls collapse, dark-mode
|
||||
form controls use the site colors, focus states are visible, and all wide tables (including
|
||||
the nested signals table and new users table) sit in `.scroll-x`.
|
||||
- Unknown routes and template-render failures now use the site-layout 404/500 pages. Issue
|
||||
download buttons show human-readable B/KB/MB/GB sizes.
|
||||
- Removed stale diff3 ancestor markers from the merged CSS/JavaScript and restored the missing
|
||||
table-filter closure; `node --check src/web/static/app.js` passes.
|
||||
- Added the users guard/content test, file-size tests, site-layout error test, and one
|
||||
fixture-backed router smoke test that renders every dashboard route template.
|
||||
|
||||
## Acceptance criteria walk-through
|
||||
|
||||
1. **Public issue and Atom boundary — met.** `PublicIssue` cannot carry generated/body fields;
|
||||
existing fixture tests assert titles/authors/sources/comment links/metadata are present and
|
||||
private Brief/summary/why/body/comment/World text is absent. The feed parses as XML and uses
|
||||
the same public view.
|
||||
2. **Complete signed-in issue — met.** Existing router tests cover Brief, summaries, why lines,
|
||||
article bodies/discussion, World and Behind pages, reduced pre-snapshot fallback, and
|
||||
existence-gated downloads. All three artifact slots share the same loader; sizes are now
|
||||
human-readable.
|
||||
3. **Admin-only ratings — met.** Whole-router role guards and rating tests cover anonymous,
|
||||
user, and admin outcomes, attributed `source = 'dashboard'` events, clear events, and the
|
||||
unchanged HMAC path.
|
||||
4. **Article history/explainability — met.** The articles list/detail routes expose the latest
|
||||
stage plus full run history, reasons, signals, assessments/facets, utility/cluster/admission,
|
||||
editor why, issue appearances, neighbours, embedding metadata, ratings, and text explain;
|
||||
allow-listed query tests and fixture rendering pass.
|
||||
5. **Run history — met.** Run list/detail renders the cumulative funnel and reasons, admission,
|
||||
preference state, timings, provider costs, warnings, feeds, near misses, config diff, and
|
||||
candidate table; seeded counts, sorts, filters, links, and sparklines are tested.
|
||||
6. **Ratings contributions/history — met.** Current and events tabs cover decay, decayed feed
|
||||
credit, neighbour weight, prompt/rebuild membership, last-run use, append-only edits/clear,
|
||||
annotations, filters, and superseded history with hand-checked tests.
|
||||
7. **Settings — met.** The schema/default/help/env/secret tests cover every config leaf. Writer
|
||||
tests cover typed changes, comments/order/mode preservation, validation before rename,
|
||||
history, providers, and reload-on-mtime. The README records why shipped providers cannot be
|
||||
removed.
|
||||
8. **Profile — met.** Save/restore/version tests cover the atomic editor and parsed preview;
|
||||
the page shows OPML interests, prompt, learned adjustments/staleness, and the rebuild job.
|
||||
9. **Jobs — met.** Fixed-name parsing, polkit/unit files, MockRunner starts/status/logs, duplicate
|
||||
refusal, failure persistence, lifecycle/run linkage, journal tail, reload-before-start, and
|
||||
overview links are present and tested. Pipeline work remains in the separate job unit.
|
||||
10. **Session/security stack — met.** The pinned axum-login/tower-sessions store,
|
||||
password-auth hashes, cookie properties, session invalidation, origin checks, login throttle,
|
||||
CSP/security headers, safe redirect checks, and secret redaction have focused tests.
|
||||
11. **Existing routes — met.** HMAC, OPDS, files, health, and reports remain wired; issue reports
|
||||
are populated. The reviewed compatibility decision keeps files public only when Basic auth
|
||||
is absent, while configured Basic auth or a valid session still gates them.
|
||||
12. **Test coverage — met, with sandbox execution caveat.** All new router tests use `oneshot`;
|
||||
no new test binds or invokes systemd. The filtered full run passed every runnable target.
|
||||
The orchestrator should run the complete suite outside this listener-restricted sandbox.
|
||||
|
||||
## Prior handoff follow-ups
|
||||
|
||||
- Step 1's `rpassword` and direct `toml_edit 0.25` follow-ups were already resolved by the
|
||||
orchestrator; its public-files deviation is now documented.
|
||||
- Step 2's raw download sizes are resolved here.
|
||||
- Step 3's Jobs/ratings targets and overview sparklines, Step 4's settings anchors and profile
|
||||
rebuild action, and Steps 5–6's config reload plus combined unit changes are all present after
|
||||
the merged steps and were included in the smoke/audit pass.
|
||||
- Nothing remains open against §20. Production deployment itself is intentionally left to the
|
||||
operator following the new runbook. Plan §21's product deferrals remain intentional: per-user
|
||||
personalization, OPML editing, body FTS, SSE logs, passkeys, public why lines, old snapshot
|
||||
backfill, and charts beyond the current sparklines.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cargo fmt`: pass.
|
||||
- `cargo clippy --all-targets -- -D warnings`: pass.
|
||||
- `node --check src/web/static/app.js`: pass.
|
||||
- Focused new router/size/error tests: pass.
|
||||
- Raw `cargo test`: **426 passed, 13 failed** before Cargo stopped at the library target; every
|
||||
failure was `PermissionDenied` while binding a loopback listener (the four documented
|
||||
Anthropic tests, three OpenAI tests using the same helper, the extractor test, and five server
|
||||
tests).
|
||||
- `cargo test` with those 13 listener tests plus the two `tests/m7_server.rs` TCP tests filtered:
|
||||
**460 passed, 0 failed, 15 filtered out** across library, binary, integration, and doc targets.
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# Implementation agent brief — The Daily EPUB, "Personalized Curation v2"
|
||||
|
||||
You are implementing ONE step of a multi-step plan in the Rust repository at
|
||||
`/home/thallada/workspace/the-daily-epub` (branch `curation-v2`, already checked out).
|
||||
|
||||
The orchestrator hands you `docs/plans/curation-v2-briefs/00-preamble.md` (this file) followed by one
|
||||
`stepN.md` from the same directory; `docs/plans/2026-09-02-curation-v2-progress.md` records what has
|
||||
landed so far.
|
||||
|
||||
The plan is `docs/plans/2026-09-02-personalized-curation-v2.md`. Read it in full before
|
||||
writing code: §0 records settled decisions (do not reopen them), §1 lists the files to
|
||||
read first, §21 is the implementation sequence. Also read
|
||||
`docs/plans/2026-08-15-implementation-notes.md` §"Cross-cutting implementation decisions"
|
||||
(note: item 5 is stale — the code uses a hand-rolled `reqwest` client in `src/curate/llm.rs`,
|
||||
keep that).
|
||||
|
||||
## Hard rules
|
||||
|
||||
- Implement ONLY the step assigned below. Later steps land separately; do not start them.
|
||||
Where this step needs a type or table that a later step fills in, create it now exactly as
|
||||
the plan specifies and leave it empty/unused with a short `// filled in by step N` comment.
|
||||
- Conventions: sqlx *runtime* queries (`sqlx::query(...).bind(...)`, never the `query!` macros),
|
||||
`jiff` for time, RFC3339 UTC strings in SQLite, `thiserror`/`anyhow` error style, no `unwrap()`
|
||||
outside tests, tracing spans per stage, rustfmt defaults, askama templates.
|
||||
- Tests never touch the network. Use mock backends following `MockBackend` in `src/curate/llm.rs`.
|
||||
- Follow the plan's names for modules, functions, config keys, table and column names, enum
|
||||
strings and prompt constants exactly. If the plan is internally contradictory or impossible
|
||||
for this step, pick the closest behaviour, keep the paper publishable, and describe the
|
||||
deviation in your final report. Do not invent scope beyond the plan.
|
||||
- API keys only from env vars; never in config files, logs, tests or the database.
|
||||
- Prefer existing dependencies in `Cargo.toml`. Add a new crate only if there is no reasonable
|
||||
way without it, and say so in the report.
|
||||
- Do NOT commit and do NOT create branches. Leave all changes in the working tree; the
|
||||
orchestrator reviews and commits. Do not edit anything under `docs/`.
|
||||
- Keep `config.example.toml` and README in sync with any config keys you add or remove
|
||||
(the test `shipped_example_config_parses` must pass).
|
||||
- Before finishing, run and make pass: `cargo fmt`, `cargo clippy --all-targets` (no new
|
||||
warnings), `cargo test`. Fix what you broke; do not delete or `#[ignore]` tests to get green
|
||||
unless the plan removes the feature they cover (then move/replace the tests as §18 says).
|
||||
|
||||
## Final report
|
||||
|
||||
End with a concise report (this is what the orchestrator reads): what you implemented, every
|
||||
deviation from the plan and why, anything from the step you could not finish, and the exact
|
||||
`cargo test` summary line(s). Keep it under 60 lines.
|
||||
|
||||
## Host notes
|
||||
|
||||
- The `openai-codex` Claude Code plugin sandbox works on this host (bubblewrap fixed 2026-09-02):
|
||||
`apply_patch`, the shell, `cargo build/test/clippy` and the warm `~/.cargo/registry` all work.
|
||||
- Inside that sandbox `bind()` on 127.0.0.1 is forbidden, so ten pre-existing tests fail there for
|
||||
environmental reasons: the four `curate::llm::tests::anthropic_*` tests,
|
||||
`extract::tests::relative_urls_resolve_against_the_url_we_landed_on`, and the five
|
||||
`server::tests::*` (the two `tests/m7_server.rs` tests likewise). They are out of scope; do not
|
||||
touch them. Every other test must pass; the orchestrator runs the full suite outside the sandbox.
|
||||
- Cargo's registry is already warm; do not add dependencies that would need a network fetch.
|
||||
- Work autonomously to completion. Do not stop to ask questions; make the closest-to-plan choice
|
||||
and record it in the final report.
|
||||
@@ -0,0 +1,73 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 1 — Feedback and profile (plan §21 step 1)
|
||||
|
||||
Scope (plan §6, §7, §8; tests in §20 under Vote, Rating events, Migration, and the prompt):
|
||||
|
||||
1. **Migration `migrations/0002_curation_v2.sql`** (§7). Create `rating_events`,
|
||||
`article_embeddings`, `interest_embeddings`, `article_assessments`, `candidate_runs` with
|
||||
the exact columns/indexes in §7; add `runs.config_json`, `runs.provider_costs_json`,
|
||||
`issue_articles.why`. Copy `ratings` rows into `rating_events` per §6.2 (`vote=1 → 'loved',1.0`;
|
||||
`vote=-1 → 'not_for_me',-1.0`; `source='migration'`; `kind='explicit'`; `event_at=rated_at`),
|
||||
then `DROP TABLE ratings; DROP TABLE feed_priors;`.
|
||||
**Orchestrator decision:** do NOT drop `scores` in this step. Stage A scoring
|
||||
(`db::upsert_score`, `recently_low_scored_ids`) still writes/reads it until step 4 replaces
|
||||
it with `article_assessments`; step 4 will add a `0003` migration dropping it. Do not edit
|
||||
`0001_init.sql`. Migration is plain SQL, no Rust bootstrap.
|
||||
2. **Three-way `Vote`** (§6.1) in `src/types.rs`: `Loved | Good | NotForMe`, `as_str` →
|
||||
`"loved" | "good" | "down"`, `parse` also accepts legacy `"up"` → `Loved`, `value(&FeedbackConfig)`
|
||||
→ configured 1.0 / 0.35 / −1.0. Add `[curation.feedback]` config
|
||||
(`loved_value`, `good_value`, `not_for_me_value`, `verdicts_in_prompt = 60`). Remove `Vote::as_i64`,
|
||||
`Rating`, `FeedPrior`, and everything that depended on `ratings`/`feed_priors`
|
||||
(`db::upsert_rating`, `ratings_with_feed`, `upsert_feed_prior`, `feed_priors`,
|
||||
`profile::rebuild_feed_priors`, the feed-prior term in `prefilter.rs` and `ScoredArticle.feed_prior`
|
||||
— set the prefilter's feed-prior contribution to nothing; the field can go). HMAC links
|
||||
(`src/auth.rs`) must verify for all three votes; message stays `{issue_date}/{article_id}/{vote}`.
|
||||
3. **Footer + confirmation page** (§6.1): `RatingLinks` in `src/epub/chapters.rs` and
|
||||
`src/epub/templates/chapter.xhtml` render three links on one line sized for e-ink:
|
||||
`Was this a good pick? [ Loved it ] [ Good ] [ Not for me ] Read online ↗`.
|
||||
X4 edition still renders no links. `server::handle_rating` appends one `rating_events` row
|
||||
(`kind='explicit'`, `source='epub'`, label `loved|good|not_for_me`, value from config) and
|
||||
returns "Recorded: Loved it — thanks." (etc.) plus the other two links so a mis-tap can be
|
||||
corrected. No feed-prior rebuild, no provider call.
|
||||
4. **`rating_events` access** (§6.2) in `src/db.rs`: `append_rating_event(...)`,
|
||||
`db::current_ratings(lookback_days) -> Vec<RatedArticle>` implementing the latest-explicit-event
|
||||
rule (`ORDER BY event_at DESC, id DESC`, `cleared` removes the article from the learned set),
|
||||
joining `articles`, the best entry's feed title, and the most recent `issue_articles.summary`.
|
||||
Move `RatedArticle` to `src/types.rs` with `summary: Option<String>`, `facets: Option<Facets>`
|
||||
(define `Facets` per §12.1 now, all `Option`s, unused until step 5), `note: Option<String>`,
|
||||
plus `RatingEvent`. Also define `RatedArticle.value: f64` and `label`.
|
||||
5. **Ratings CLI** (§6.3) in `src/main.rs`: `ratings list [--days 90] [--label loved|good|down|cleared]`,
|
||||
`ratings set --article ID|--url URL --label loved|good|down [--note "..."]`,
|
||||
`ratings clear --article ID|--url URL`. `set`/`clear` append rows with `source='cli'` and
|
||||
`issue_date` from the latest `issue_articles` row for the article if any; `--url` canonicalizes
|
||||
with `dedupe`'s canonical URL function then `db::article_id_for_url`. Print something useful.
|
||||
6. **`data/profile.md`** (§8.2): create the file with the initial content shown in §8.2 verbatim;
|
||||
add `profile_path` config (default `data/profile.md`); loader that parses any `## Interests`
|
||||
section (one per line, leading `- ` stripped) and unions it case-insensitively with the OPML
|
||||
interests, passing everything else through verbatim. Remove `STATED_PREFERENCES` and the
|
||||
reader/wants/does-not-want/how-to-judge parts of `PROFILE_PREAMBLE` from code (they now live
|
||||
in the file); the editor-in-chief framing paragraph stays in code. If the file is missing,
|
||||
warn and continue with the OPML interests only.
|
||||
7. **System prompt** (§8.4): rebuilt every run in this exact order: framing → profile.md verbatim
|
||||
(minus Interests) → standing interests grouped by `profile::themes::group_into_themes` →
|
||||
learned adjustments → **recent verdicts**: up to `verdicts_in_prompt` most recent explicit
|
||||
ratings, newest first, `LOVED | title | feed | one-line summary` (labels `LOVED`/`GOOD`/`NOT FOR ME`),
|
||||
cleared and duplicate articles removed. Byte-identical within a run. `profile_version` bumps only
|
||||
on the weekly rebuild, as today. `kv` keeps `ingest_watermark`, `taste_profile`,
|
||||
`taste_profile_learned`, `profile_version` (§7.8).
|
||||
8. **Weekly rebuild** (§8.3): still on the DeepSeek client this step (the Claude editor client
|
||||
arrives in step 2). Each rated line carries
|
||||
`LOVED | title | feed | summary | facets: format/depth/evidence/technicality/topic_group | note: …`
|
||||
(omit empty parts), up to 200 most recent explicit ratings, cleared excluded; the softened
|
||||
"strong prior, not a rule" instruction; the required "Diversity check:" bullet.
|
||||
9. Pipeline: remove the `rebuild_feed_priors` call and every `feed_priors` reference; the paper
|
||||
must still build. `handle_rating` and `profile rebuild` must not touch dropped tables.
|
||||
|
||||
Tests to add/adapt (§20): Vote parse/serialize incl. `up`→Loved; HMAC for all three; template
|
||||
renders three links standard / none X4; latest explicit event wins; `cleared` removes from the
|
||||
learned set; migration on a temp DB through 0001 then 0002 copies old ratings with the right
|
||||
labels/values and `ratings`/`feed_priors` are gone; CLI `set`/`clear` append `source='cli'` rows
|
||||
(test the db-level function); profile.md Interests parsing and union; system prompt section order
|
||||
and verdict block content. Update `tests/m7_server.rs` and `tests/m4_epub.rs` as needed.
|
||||
@@ -0,0 +1,86 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 2 — Claude editor and editorial (plan §21 step 2)
|
||||
|
||||
Step 1 has landed (three-way votes, `rating_events`, `data/profile.md`, the rebuilt system
|
||||
prompt). Read `git log -3` and the current `src/curate/llm.rs`, `select.rs`, `editorial.rs`,
|
||||
`pipeline.rs`, `config.rs`, `report.rs`, `db.rs`, `src/epub/chapters.rs` + templates before coding.
|
||||
|
||||
Scope (plan §4.1, §4.2, §5, §7.6, §13, §14, §15.1 chapter/in-this-issue `why`, §18 types, §19 config):
|
||||
|
||||
1. **`AnthropicBackend`** in `src/curate/llm.rs` implementing `ChatBackend` exactly per §4.2:
|
||||
`POST {base_url}/v1/messages`; headers `x-api-key`, `anthropic-version: 2023-06-01`,
|
||||
`content-type: application/json`, `anthropic-beta: server-side-fallback-2026-07-01`;
|
||||
body `{model, max_tokens: 16000, system: [{type:"text", text, cache_control:{type:"ephemeral"}}],
|
||||
messages:[{role:"user", content}], output_config:{effort}, fallbacks:"default"}`.
|
||||
**Never** send `temperature`, `top_p`, `top_k`, `thinking`, or an assistant prefill. Ask for JSON
|
||||
in the prompt text and parse tolerantly (reuse `strip_code_fence` + the existing tolerant parsing).
|
||||
Response: concatenate `content[]` blocks with `type == "text"`. Usage: `input_tokens`,
|
||||
`cache_creation_input_tokens`, `cache_read_input_tokens`, `output_tokens`; cost = input×price_input
|
||||
+ cache_creation×price_cache_write + cache_read×price_cache_read + output×price_output (per M).
|
||||
`stop_reason == "refusal"` (HTTP 200) surfaces as a distinct `LlmError` variant that triggers the
|
||||
fallback; 429/5xx/network are `Transient` (retried by the existing `RetryPolicy`); 400 is never
|
||||
retried. Request timeout 300 s. API key only from `DAILY_EPUB_ANTHROPIC__API_KEY`.
|
||||
`ChatRequest` gains `effort: Option<String>`; the Anthropic backend ignores `temperature`/`json`
|
||||
and maps `system` to the cached system block. `TokenUsage` needs a cache-write counter (keep the
|
||||
DeepSeek path's cost identical to today). Generalize `UsageMeter` to take a per-provider price
|
||||
table instead of `&DeepseekConfig`.
|
||||
2. **`Llms { bulk: Option<LlmClient>, editor: Option<LlmClient> }`** with `editor_or_bulk()` (the
|
||||
editor when configured and its meter is not tripped, else bulk). Both clients share the exact same
|
||||
system prompt string (§8.4). `LlmError::MissingApiKey` and friends must name the provider.
|
||||
3. **Config** (§4.2, §19): `[anthropic]` with `enabled`, `base_url`, `model = "claude-opus-5"`,
|
||||
`api_key` (env only), `effort = "high"`, `price_input_per_mtok = 5.0`,
|
||||
`price_cache_write_per_mtok = 6.25`, `price_cache_read_per_mtok = 0.5`, `price_output_per_mtok = 25.0`,
|
||||
`max_daily_usd = 3.0`, `max_concurrent_requests = 4`; `[deepseek].max_concurrent_requests = 4`;
|
||||
`[curation].max_article_count = 28`; `[editorial]` with `summary_model = "editor"` (`editor|bulk`)
|
||||
and `summary_input_tokens = 3000`. Validate `max_article_count >= target_article_count`, effort ∈
|
||||
{low, medium, high, xhigh, max}, batch/concurrency ≥ 1. Startup logs the resolved models and whether
|
||||
each provider is enabled (§19). Update `config.example.toml` and the README (prerequisites, cost
|
||||
line, the note that server-side fallback is enabled, dashboard spend limits as the real backstop).
|
||||
4. **Budget and concurrency** (§5): one `UsageMeter` per provider, each with its own `max_daily_usd`.
|
||||
The budget day is the **UTC date of the run's `started_at`**, preloaded by summing
|
||||
`runs.provider_costs_json` for earlier runs that UTC day; this replaces `db::spend_for_date`
|
||||
(remove it). Write `runs.provider_costs_json` (`{"deepseek": {...usage, cost_usd}, "anthropic": {...}}`)
|
||||
and `runs.config_json` (the resolved `[curation]`, `[editorial]`, model names and prompt versions,
|
||||
as JSON) in `finish_run`. Keep `runs.cost_usd` as the total across providers. Existing Stage A
|
||||
scoring batches run through `futures::stream::iter(...).buffer_unordered(max_concurrent_requests)`
|
||||
with the budget check before each request is spawned; a tripped meter skips remaining calls,
|
||||
lets in-flight finish, records the number of unscored candidates in the report, and continues.
|
||||
5. **The editor** (§13): `EDITOR_INSTRUCTIONS` replaces `SELECT_INSTRUCTIONS` verbatim from the plan
|
||||
with `{soft_target}`/`{hard_max}` substituted. Runs on `editor_or_bulk()`; on refusal/error fall
|
||||
back to the same prompt on the bulk client; if that fails too, `select_without_llm`. Per-item
|
||||
rendering follows §13 with what exists today (Stage A score/category/rationale instead of
|
||||
quality/fit/facets — those arrive in step 5; render `flags: always-include | excerpt only`;
|
||||
`opening:` first 60 words). Do not put the numeric blend in the prompt. `assemble()` keeps section
|
||||
validation, unique lead, auto-include reinsertion, duplicate-id defence, malformed-response
|
||||
fallback and the `hard_max` trim (by today's ranking key, utility arrives in step 5).
|
||||
**Delete the "too few: top up" branch.** `--max-articles N` is a ceiling:
|
||||
`hard_max = min(config.curation.max_article_count, N)`, `soft_target = min(target_article_count, hard_max)`.
|
||||
`select_without_llm` respects `soft_target` as its size. Each pick's `why` (≤14 words) lands on
|
||||
`Pick.why: Option<String>` and in `issue_articles.why` (`replace_issue_articles`).
|
||||
6. **Editorial** (§14): summaries run on the editor client per `editorial.summary_model` with the
|
||||
input budget from config (3,000 tokens), concurrency 4 via `buffer_unordered`, fallback per article:
|
||||
bulk client, then the excerpt. `BRIEF_INSTRUCTIONS` replaces `FRONT_PAGE_INSTRUCTIONS` verbatim
|
||||
from §14.2 (JSON `{"brief": "..."}`); input is the lineup with sections, each pick's title, feed,
|
||||
`why`, summary and the Stage A score. `Editorial.section_intros` is removed (or always empty) and
|
||||
`section.xhtml` renders only the section name; `front_page.xhtml` renders the brief under the
|
||||
masthead; fallback stays `fallback_front_page_html`. The weekly profile rebuild runs on
|
||||
`editor_or_bulk()` (pipeline and the `profile rebuild` CLI).
|
||||
7. **Paper** (§15.1): `chapter.xhtml` gets a small italic `Why it's here: <why>` line under the meta
|
||||
line; the In-this-issue page shows the `why` line under each summary. `Colophon` gains
|
||||
`provider_costs` and `models`; the colophon template prints per-provider cost lines and the models
|
||||
(editor and summaries model, bulk model). `StageCounts`/`RunReport` carry per-provider usage;
|
||||
`print_report` in `main.rs` prints per-provider cost.
|
||||
8. Remove `Vote`-era leftovers you notice only if they are in your files; do not touch embeddings,
|
||||
signals, telemetry or triage (steps 3–5).
|
||||
|
||||
Tests to add/adapt (§20 "Anthropic backend", "Editor", parts of "Pipeline"): request body has the
|
||||
system block with `cache_control`, no `temperature`, `output_config.effort`, `fallbacks`, the beta
|
||||
header; usage fields parsed into cost with cache read/write prices; `stop_reason: refusal` surfaces as
|
||||
the fallback-triggering error; 429 retried, 400 not (use a mock backend or a local `axum`/`tokio`
|
||||
listener — never the real network); a nine-pick response is published as nine; `hard_max` trims;
|
||||
`--max-articles` is a ceiling; auto-includes reinserted; `why` lines land on picks and in
|
||||
`issue_articles.why`; refusal/error on the Anthropic mock falls back to the DeepSeek mock with the
|
||||
same prompt; the brief is parsed and rendered, section intros gone; `provider_costs_json` and
|
||||
`config_json` are written; budget day preload sums by UTC date.
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 3 — Embeddings, signals, telemetry (plan §21 step 3)
|
||||
|
||||
You are working in a git **worktree** on branch `curation-v2-step3`, which branches from the step 1
|
||||
commit. Another agent is implementing step 2 (the Claude editor) in parallel on the main checkout;
|
||||
you will not see its changes and must not need them. Read `git log -3` and the current
|
||||
`src/curate/`, `src/pipeline.rs`, `src/config.rs`, `src/db.rs`, `src/report.rs`, `src/main.rs` first.
|
||||
Do not run `git checkout`, `git stash`, `git worktree` or `git commit`.
|
||||
|
||||
Scope (plan §4.3, §7.1, §7.2, §7.4, §7.5, §9, §12.2 normalization only, §12.4 preliminary blend,
|
||||
§15.2 `explain`, §15.4 log line parts, §16, §17 Voyage row, §19 `[voyage]` + `[curation.ranking]`).
|
||||
The old heuristic prefilter still gates in this step; signals are computed for every article that
|
||||
passes hygiene and persisted, nothing about *selection* changes yet.
|
||||
|
||||
1. **`src/curate/embedding.rs`** (§4.3, §7.1, §7.2): `EmbeddingBackend` trait mirroring `ChatBackend`
|
||||
(with a mock for tests), `VoyageBackend` (`POST {base_url}/embeddings`, `Authorization: Bearer`,
|
||||
body `{input, model, input_type: "document"|"query", truncation: true, output_dimension, output_dtype: "float"}`,
|
||||
response mapped by index and length-checked, usage `total_tokens` metered), batching by `batch_size`
|
||||
with `buffer_unordered(max_concurrent_requests)`, retry 429/5xx via `RetryPolicy`, f32
|
||||
little-endian BLOB encode/decode with length and finiteness checks, `dot(a, b)` with a dimension
|
||||
check, and cache orchestration: `article_embeddings` keyed by `(model, dimension, input_hash = sha256
|
||||
of the embedded text)`, overwritten on change. Embedded text = `"Title: {title}\n\n{plain body}"`
|
||||
via `curate::prompt_text`, whitespace collapsed, cut at `max_input_chars` on a char boundary —
|
||||
**no feed name, author or scores**. Interests go to `interest_embeddings` with `input_type = "query"`
|
||||
and the bare interest string. A failed batch leaves those articles without embeddings; never fatal.
|
||||
A Voyage `UsageMeter` with `price_per_mtok = 0.02` and `max_daily_usd` (0.50) as a runaway guard.
|
||||
Key only from `DAILY_EPUB_VOYAGE__API_KEY`. Raw vectors never reach logs or reports.
|
||||
2. **Config** (§4.3, §19): `[voyage]` (`enabled`, `base_url = "https://api.voyageai.com/v1"`,
|
||||
`model = "voyage-4-lite"`, `output_dimension = 512`, `batch_size = 32`, `max_concurrent_requests = 4`,
|
||||
`max_input_chars = 60000`, `max_daily_usd = 0.50`) and the complete `[curation.ranking]` block from
|
||||
§19 (`triage_max`, `deep_keep`, `shortlist_keep`, `assessment_reuse_days`, `rating_lookback_days`,
|
||||
`rating_half_life_days`, `neighbour_k`, `negative_coefficient`, `knn_floor/full`, `feed_floor/full`,
|
||||
`semantic_min_words`, `exploration_slots`, `embedding_retention_days`, `telemetry_retention_days`,
|
||||
`[curation.ranking.quotas]`, `[curation.ranking.weights.preliminary]`, `[curation.ranking.weights.utility]`,
|
||||
`[curation.ranking.diversity]`) with the plan's defaults and the validation rules in §19 (weights
|
||||
non-negative; `deep_keep ≥ shortlist_keep ≥ target_article_count`; `*_full > *_floor ≥ 0`;
|
||||
`0 ≤ cluster_threshold ≤ 1`; `per_cluster_cap ≥ 1`; batch sizes ≥ 1; dimension ∈ {256,512,1024,2048}).
|
||||
Steps 4–5 will *use* the ranking keys; you only add and validate them. Keep `prefilter_keep` for now.
|
||||
Startup logs whether Voyage is enabled. Update `config.example.toml` and the README.
|
||||
3. **`src/curate/signals.rs`** (§9): a `Signals` type where every signal is `Option<f64>` (absent ≠ 0):
|
||||
`interest` (§9.1 z-scored interest match, top-three interests recorded; raw top-1 cosine fallback
|
||||
under 30 embedded articles, logged), `knn` (§9.2 preference state from `db::current_ratings(rating_lookback_days)`
|
||||
joined to `article_embeddings`, decayed weights `value × 0.5^(age/half_life)`, top-k positives and
|
||||
negatives, `knn = pos − negative_coefficient × neg`, top-three neighbours recorded, gate ramp
|
||||
`clamp((n − knn_floor)/(knn_full − knn_floor), 0, 1)` multiplying the weight, absent at 0),
|
||||
`feed` (§9.3 Beta-smoothed decayed rate credited to distinct direct `SourceKind::Feed` feeds, mean
|
||||
over the article's rated feeds, gate `feed_floor/feed_full`), `social` (existing composite; absent
|
||||
with no rows), `heuristic` (`longform_points(word_count)` − excerpt-only − roundup penalty, from
|
||||
`prefilter.rs` **without** the social, Scour/HN, multi-source terms; expose those pieces as
|
||||
functions). Log once per run:
|
||||
`preference: N rated articles with embeddings → knn gate X; feed gate Y (n=…)`.
|
||||
Also implement the **mid-rank percentile normalizer** of §12.2 and the **preliminary blend** of
|
||||
§12.4 (present-and-active weights renormalized) so `signals_json.norm`/`weights` can be written now.
|
||||
4. **Pipeline**: after social enrichment, for every article that passes hygiene, compute embeddings
|
||||
(cached), signals, and the preliminary blend, then run the old prefilter as today. New stage
|
||||
timings `embed` and `signals`. `--skip-embeddings` uses cached embeddings only (zero Voyage calls);
|
||||
`--skip-llm` unchanged. `StageCounts` gains `eligible`, `embedded`, `rated_with_embeddings`, and
|
||||
the report/`print_report` show them. The paper must still build when Voyage is down or unconfigured
|
||||
(§17): `interest`/`knn` absent, never a penalty.
|
||||
5. **`src/curate/telemetry.rs`** (§7.4, §7.5): the `candidate_runs` writer. One row per considered
|
||||
article per run, upserted with `INSERT … ON CONFLICT(run_id, article_id) DO UPDATE` setting every
|
||||
column. Hygiene-excluded articles get thin rows (`stage='excluded'`, `excluded_reason` ∈
|
||||
`blocked | published_before | recently_rejected`, `signals_json='{}'`). In this step map the old
|
||||
pipeline onto the stage vocabulary: passed hygiene → `eligible`; cut by the prefilter →
|
||||
`eligible` + `excluded_reason='not_admitted'`; prefilter survivors → `admitted`; Stage-A scored →
|
||||
`assessed`; sent to selection → `shortlisted`; picked → `selected` (+ `editor_why` if a `why`
|
||||
exists on the pick, else NULL); unpicked shortlist → `excluded_reason='not_selected'`. `signals_json`
|
||||
follows §7.5 exactly (`v: 1`, `raw`, `norm`, `present`, `weights` = effective preliminary weights,
|
||||
`top_interests`, `neighbours`, `exploration: false`, `auto_include`, `notes`). `utility` and
|
||||
`rank_utility` stay NULL until step 5.
|
||||
6. **CLI** (§15.2, §16) in `src/main.rs`:
|
||||
`explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N]` and
|
||||
`explain --date YYYY-MM-DD --near-misses [N]` printing the persisted `candidate_runs` row for the
|
||||
latest non-dry run of that date (or `--run-id`): stage and reason; every raw/normalized signal with
|
||||
presence and effective weight; top interests with z; nearest rated neighbours; assessments from
|
||||
`article_assessments` when present (empty until step 4); admitted_by; the editor's `why`. `--url`
|
||||
canonicalizes (`dedupe`) and looks the article up; if absent from `articles`, print that it was never
|
||||
ingested. `--near-misses` lists the top N by preliminary blend (utility from step 5 when present)
|
||||
that were not selected. `features backfill [--days 30] [--rated-only] [--all] [--yes]` embeds rated
|
||||
and published articles first, then interests, then other recent articles only under `--all`, prints
|
||||
an estimate and asks for confirmation above 5M tokens unless `--yes`, and is idempotent (warm cache
|
||||
⇒ zero calls). `features prune` removes `article_embeddings` rows for articles neither rated nor
|
||||
published older than `embedding_retention_days` and `candidate_runs` rows whose run started more
|
||||
than `telemetry_retention_days` ago. `features backfill` takes no lock yet (step 6 adds `lock.rs`).
|
||||
7. Types (§18): `Signals`, `Signal` helpers, `RatedArticle` already exists — extend if needed. Do not
|
||||
create `Candidate` yet (step 4/5).
|
||||
|
||||
Tests (§20 "Embeddings", "Interest z-scores", "Preference", "Normalization", plus telemetry/explain):
|
||||
BLOB round trip; wrong length and non-finite rejected; cache hit on same hash, miss on changed
|
||||
text/model/dimension; response mapped by index and length-checked; a failed batch does not abort the
|
||||
others; embedded text contains no feed name or author; a broad interest with uniformly high cosine does
|
||||
not dominate while a specific interest with one strong match does; raw fallback under 30 articles; one
|
||||
loved article gives a positive `knn` to a near neighbour; two unrelated loved clusters both score high
|
||||
(anti-centroid); `good` moves the signal 0.35× as much as `loved`; decay halves at the half-life; gate
|
||||
0 below `knn_floor`, 1 at `knn_full`, linear between; feed credit sums to 1 across direct feeds; feed
|
||||
affinity uses the mean; constant signal → 0.5 for everyone; ties get equal percentiles (400 identical
|
||||
zeros → all 0.5, no id ramp); absent values do not shift others; effective weights sum to 1; a candidate
|
||||
missing a signal is scored on the rest; `candidate_runs` rows written for every considered article with
|
||||
the right stage/reason on a mocked run; `--skip-embeddings` makes zero Voyage calls; a Voyage failure
|
||||
still publishes; `explain` renders from persisted rows and reports "never ingested"; `features prune`
|
||||
respects rated/published.
|
||||
@@ -0,0 +1,89 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 4 — Triage replaces the gate (plan §21 step 4)
|
||||
|
||||
Steps 1–3 have landed: three-way votes and `rating_events`; the Claude editor (`Llms { bulk, editor }`,
|
||||
`AnthropicBackend`, per-provider `UsageMeter`s, `why` lines, the Brief); Voyage embeddings,
|
||||
`signals.rs` (interest/knn/feed/social/heuristic, percentile normalizer, preliminary blend),
|
||||
`telemetry.rs` (`candidate_runs` writer), `explain`, `features backfill|prune`, and the full
|
||||
`[curation.ranking]` config. Read `git log -6`, then the current `src/curate/` (all files),
|
||||
`src/pipeline.rs`, `src/config.rs`, `src/db.rs`, `src/report.rs`, `src/main.rs`, `src/types.rs`.
|
||||
|
||||
Scope (plan §7.3 reuse and churn rules, §8.1 hygiene, §10 triage, §11 admission and exploration,
|
||||
§12.4 preliminary blend already exists, §15.4 log line parts, §17 DeepSeek row, §19 `deep_keep`):
|
||||
|
||||
1. **Migration `migrations/0003_drop_scores.sql`**: `DROP TABLE scores;` (deferred from step 1).
|
||||
Remove `db::upsert_score`, `recently_low_scored_ids` and every `scores` reader. The churn rule
|
||||
now reads `article_assessments`: an article whose latest `triage` `score < recent_rejection_floor`
|
||||
(3.0) or `deep` `score < recent_rejection_floor` within `recent_rejection_days` (7) is
|
||||
`recently_rejected`, unless auto-include (§7.3, §8.1).
|
||||
2. **`src/curate/triage.rs`** (§10): `TRIAGE_INSTRUCTIONS` and `TRIAGE_PROMPT_VERSION = 1` verbatim
|
||||
from the plan; per-article block exactly as §10 (id, title, feed (category), author, length ·
|
||||
excerpt only, opening = first 200 words via `prompt_text` + `truncate_words`, `matches interests:`
|
||||
from `top_interests` with z ≥ 1.5 labelled strong (z ≥ 2.5) / weak, `closest rated:` from
|
||||
neighbours with cosine ≥ 0.55; omit lines with nothing). Batches of `deepseek.triage_batch_size`
|
||||
(25) through `buffer_unordered(max_concurrent_requests)` on the **bulk** client, budget check
|
||||
before each batch. Tolerant parsing generalized from `score.rs::parse_score_response`: a
|
||||
malformed item never sinks the batch, ids not in the batch are dropped, unknown `kind` → `other`,
|
||||
`interest` clamped 0–10. Persist every result to `article_assessments (stage='triage', model,
|
||||
prompt_version, profile_version, score, kind, rationale=why, assessed_at)`.
|
||||
**Cache** (§7.3): skip articles with a reusable `triage` or `deep` assessment (same `model` and
|
||||
`prompt_version`, `assessed_at` within `assessment_reuse_days`); `generate --rescore` ignores the
|
||||
cache. A failed batch leaves `triage` absent for its articles.
|
||||
**Pool cap** (§10): if eligible > `triage_max` (800), triage the union of top `0.7 × triage_max`
|
||||
by preliminary blend, top 100 by `interest`, top 100 by `knn` (if active), all auto-includes,
|
||||
filled to `triage_max` by blend; the rest get `stage='eligible'`, `excluded_reason='not_admitted'`.
|
||||
3. **`src/curate/admit.rs`** (§8.1, §11): hygiene moves here (`blocked`, `published_before` = in
|
||||
`issue_articles` for any issue date before this run's date, `recently_rejected` per item 1;
|
||||
auto-includes never excluded) writing the thin `candidate_runs` rows, and the union admission
|
||||
into `deep_keep` (120) slots in this order, each retriever taking its top-N by its own signal among
|
||||
not-yet-admitted, not-excluded articles, recording every retriever that would have taken an
|
||||
article in `admitted_by` (first = the one that admitted it):
|
||||
`auto_include` (uncapped) → `triage` (quota 60, floor `interest ≥ 5`) → `interest` (20; floors
|
||||
`word_count ≥ semantic_min_words`, not `looks_like_roundup`, triage `interest ≥ 3` if triaged)
|
||||
→ `knn` (20, gate > 0, same floors) → `exploration` (5, §11.1) → `blend` (remaining).
|
||||
Inactive retrievers release their quota to `blend`. Not admitted → `stage='triaged'` (or
|
||||
`'eligible'` if never triaged) + `excluded_reason='not_admitted'`.
|
||||
**Exploration** (§11.1): five slots for articles ranked between `deep_keep` and `deep_keep × 2.5`
|
||||
by the preliminary blend with `word_count ≥ 300`, not roundups, triage `interest ≥ 4`; ordered by
|
||||
`sha256(run_date || article_id)`; flagged `exploration = true` through to the editor prompt.
|
||||
4. **Prefilter reduced** (§18): `prefilter.rs` keeps only hygiene helpers (`is_blocked`,
|
||||
`is_auto_include`, `looks_like_roundup`) and the text heuristic pieces `signals.rs` uses. Delete
|
||||
`prefilter::run`, `score_article`'s social/Scour/HN/multi-source terms, `PrefilterContext`, the
|
||||
`prefilter_keep` config key (replace uses with `curation.ranking.deep_keep`; the config validation
|
||||
`deep_keep ≥ shortlist_keep ≥ target_article_count` already exists), and `Curator::prefilter`.
|
||||
A stale `prefilter_keep` key in a config file must fail loudly with a message naming
|
||||
`curation.ranking.deep_keep` (follow the `bookorbit_dir` precedent in `config.rs`).
|
||||
5. **Pipeline**: hygiene → embeddings → signals → preliminary blend → triage → admission → the
|
||||
*existing* Stage A scoring (`score.rs`, unchanged this step) over the admitted deep set → the
|
||||
editor → editorial, as today. The deep set (`admitted`) is what Stage A scores and what the editor
|
||||
sees; step 5 replaces Stage A with deep assessment and adds the shortlist. Telemetry stages now:
|
||||
`excluded | eligible | triaged | admitted | assessed | shortlisted | selected` with reasons per §7.4;
|
||||
`admitted_by` and `exploration` written; `signals_json.raw/norm/present` gain `triage` (÷10 for norm).
|
||||
Stage timings `triage` and `admit`; `StageCounts` gains `triaged`, `admitted`, `admitted_by` (map),
|
||||
`exploration_admitted`, `exploration_selected`. Log the admission line from §15.4:
|
||||
`admission: triage 60 · interest 20 · knn 12 · exploration 5 · blend 23 · auto 0`.
|
||||
`--skip-llm` skips triage; DeepSeek down or tripped ⇒ no triage, admission by
|
||||
`interest`/`knn`/`blend`, editor still runs (§17). The editor prompt (§13 rendering) shows the
|
||||
triage score and `flags: exploration` where present. `explain` prints the triage assessment
|
||||
(score, kind, why) and `admitted_by`.
|
||||
6. `types.rs` (§18): introduce `Assessment { triage: Option<Triage>, deep: Option<Deep> }` and
|
||||
`Triage { interest, kind, why, model, prompt_version, assessed_at }` (keep `Deep`/`LlmScore`
|
||||
compatible with the existing Stage A output for now), and `Candidate { article, auto_include,
|
||||
exploration, signals, assessment, utility: Option<f64>, cluster: Option<...>, admitted_by: Vec<String>,
|
||||
stage, excluded_reason }` replacing `ScoredArticle` where the pipeline flows through admission.
|
||||
Keep the change mechanical where `score.rs`/`select.rs` still consume the old shape (step 5
|
||||
retires them); an adapter is acceptable.
|
||||
7. `runs.config_json` gains `TRIAGE_PROMPT_VERSION`; `config.example.toml` and README updated
|
||||
(`prefilter_keep` removed, `[deepseek].triage_batch_size`, `--rescore`).
|
||||
|
||||
Tests (§20 "Triage and deep parsing" triage half, "Admission", churn/cache): realistic triage
|
||||
fixture parsed; malformed items do not sink a batch; unknown kind → other; every `kind` token in the
|
||||
prompt round-trips; cached triage reused within `assessment_reuse_days` and ignored with `--rescore`;
|
||||
churn rule excludes a recent low triage score and spares auto-includes; a strong-interest,
|
||||
weak-heuristic, no-social article reaches the deep set; a 60-word stub with high interest similarity
|
||||
is not admitted by `interest`/`knn`; quotas honoured; inactive retrievers release quota; exploration
|
||||
deterministic per date and rotating across dates; auto-includes always admitted; excluded articles get
|
||||
thin rows with the right reason; pool cap path yields `not_admitted` rows; the mocked full pipeline
|
||||
still publishes with DeepSeek failing; migration 0003 drops `scores`.
|
||||
@@ -0,0 +1,69 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 5 — Deep assessment, utility, diversity (plan §21 step 5)
|
||||
|
||||
Steps 1–4 have landed; triage and union admission now gate the deep set and the old prefilter is
|
||||
reduced to hygiene helpers. Read `git log -8`, then all of `src/curate/`, `src/pipeline.rs`,
|
||||
`src/types.rs`, `src/config.rs`, `src/report.rs`, `src/main.rs`, `src/db.rs`.
|
||||
|
||||
Scope (plan §12 in full, §13 rendering with facets/neighbours, §7.3 deep rows, §18 module retirements):
|
||||
|
||||
1. **`src/curate/assess.rs`** replaces `score.rs` (§12.1): `DEEP_INSTRUCTIONS` and
|
||||
`DEEP_PROMPT_VERSION = 1` verbatim from the plan (the section palette is substituted from
|
||||
`curation.sections`). Per article: title, feed, author, length, excerpt-only flag, the triage
|
||||
`why`, the interest and neighbour hint lines from §10, and a **representative sample** of
|
||||
~2,000 tokens: body under ~1,500 words sent whole; otherwise first 600 words, 500 around the
|
||||
midpoint, last 400, with visible `[BEGINNING]`/`[MIDDLE]`/`[END]` markers, split on word
|
||||
boundaries. Batches of `deepseek.deep_batch_size` (8, replaces `score_batch_size`) through
|
||||
`buffer_unordered`, bulk client, budget check per batch. Auto-includes are assessed too.
|
||||
Tolerant parsing: `quality`/`fit` clamped 0–10, `category` validated against the palette (invalid →
|
||||
`None`, the editor decides), `paywalled_guess`, and `facets` where every unknown enum token degrades
|
||||
to `None` and `specific_topics` is capped at 3. **Every enum token in the prompt must round-trip
|
||||
through the parser (test).** Persist to `article_assessments (stage='deep', score=quality, fit,
|
||||
kind=facets.format, facets_json, rationale, category, paywalled_guess, ...)`; reuse per §7.3 with
|
||||
`--rescore` bypass. Facets are shown to the editor, the profile rebuild and `explain`; **not** a
|
||||
numeric signal.
|
||||
2. **`src/curate/rank.rs`** (§12.2–§12.5): normalization (LLM scores ÷ 10; everything else mid-rank
|
||||
percentile over the deep set — reuse/move the step-3 normalizer), the **utility** blend over
|
||||
present signals with `[curation.ranking.weights.utility]` renormalized and learned signals
|
||||
multiplied by their gate ramp first, stored 0–100; the preliminary blend stays for the eligible set.
|
||||
**Diversified shortlist** (§12.5): leader clustering by embedding cosine with `cluster_threshold`
|
||||
(0.85) and `per_cluster_cap` (2): sort by utility desc then article id asc; assign each article to
|
||||
the first existing cluster whose *leader* has cosine ≥ threshold, else make it a new leader;
|
||||
articles without embeddings are singletons; admit in order while the cluster's admitted count is
|
||||
below the cap until `shortlist_keep` (60); the top `utility_protected` (10) by utility and all
|
||||
auto-includes are admitted regardless and still count toward their cluster; exploration picks that
|
||||
reached the deep set get up to 3 reserved slots; if short, relax to cap 3, then uncapped. Persist
|
||||
`cluster_id`, `cluster_rank`, `rank_utility`, `utility`, `excluded_reason ∈ cluster_suppressed |
|
||||
shortlist_cap`, stage `shortlisted`.
|
||||
3. **`src/curate/editor.rs`** replaces `select.rs`: the §13 rendering in full —
|
||||
`quality 8.5 · fit 7.0 · triage 8.0 — <deep rationale>`, `facets: format · depth · evidence ·
|
||||
technicality · topic_group`, `matches:`, `closest rated:`, `flags: exploration | always-include |
|
||||
excerpt only`, `opening:` first 60 words. The editor sees the 60-item shortlist. `assemble()`'s
|
||||
`hard_max` trim and `select_without_llm` now order by **utility**, falling back to the preliminary
|
||||
blend. Delete `ScoredArticle::combined_score()` and `ScoredArticle` itself; `Candidate` is the only
|
||||
flow type. Move the tests from `score.rs`/`select.rs` into `assess.rs`/`editor.rs` and delete the
|
||||
old files (update `curate/mod.rs`, `Curator`).
|
||||
4. **Editorial inputs**: the Brief and the summaries use quality/fit where the plan says so (§14.2
|
||||
input lists quality/fit). The weekly profile rebuild's rated lines now get real facets.
|
||||
5. **Pipeline/report**: stage timings `assess` and `rank`; `StageCounts` gains `assessed`,
|
||||
`shortlisted`, `clusters`; the §15.4 curation line
|
||||
`curation: 412 considered → 398 eligible → 398 triaged → 120 assessed → 60 shortlisted → 17 selected`.
|
||||
`explain` prints the deep assessment (quality, fit, category, rationale, facets), utility and rank,
|
||||
cluster id and what suppressed it; `--near-misses` orders by utility. `runs.config_json` gains
|
||||
`DEEP_PROMPT_VERSION`. `--skip-llm`/DeepSeek down ⇒ no deep assessment, utility over present
|
||||
signals (triage/interest dominate, §12.3), editor still runs on what it has (§17).
|
||||
6. `config.example.toml`/README: `deep_batch_size` replaces `score_batch_size`; a stale
|
||||
`score_batch_size` key fails loudly naming `deep_batch_size`.
|
||||
|
||||
Tests (§20 "Triage and deep parsing" deep half, "Normalization", "Clustering", "Editor" utility
|
||||
parts): realistic deep fixture; malformed items do not sink a batch; unknown facet tokens → `None`;
|
||||
every enum token round-trips; representative sample has the three markers and respects word
|
||||
boundaries, short bodies are sent whole; cached deep rows reused / bypassed with `--rescore`; constant
|
||||
signal → 0.5; ties equal; absent values do not shift others; effective weights sum to 1; a candidate
|
||||
missing a signal is scored on the rest; near-duplicates share a cluster and the third is suppressed;
|
||||
protected top-N survive and count; the bridge case (A~C, B~C, A≁B, utility A>B>C) yields two clusters;
|
||||
articles without embeddings are never suppressed; `hard_max` trims by utility; `select_without_llm`
|
||||
orders by utility; the mocked full pipeline publishes with DeepSeek down and writes `shortlisted`
|
||||
rows with cluster ids.
|
||||
@@ -0,0 +1,62 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 6 — Paper telemetry, stats, lock (plan §21 step 6)
|
||||
|
||||
Steps 1–5 have landed: the full v2 ranking pipeline (hygiene → embeddings → signals → triage →
|
||||
admission → deep assessment → utility → cluster-capped shortlist → Claude editor → editorial),
|
||||
`candidate_runs` telemetry, `explain`, `features`, `ratings`. Read `git log -10`, then
|
||||
`src/pipeline.rs`, `src/report.rs`, `src/main.rs`, `src/curate/telemetry.rs`, `src/epub/chapters.rs`
|
||||
and `src/epub/templates/`, `src/types.rs` (`Colophon`), `src/db.rs`, README, `config.example.toml`.
|
||||
|
||||
Scope (plan §5 lock, §15.1 Behind the paper + In-this-issue, §15.3 `stats`, §15.4 report/log block,
|
||||
§16 lock coverage, §19 startup logging, README/config docs):
|
||||
|
||||
1. **Behind the paper** (§15.1): a new short chapter after the World Briefing and before the colophon
|
||||
(`src/epub/templates/behind.xhtml`, `render_behind_the_paper`) with exactly the content shape in
|
||||
the plan: the considered/eligible/triaged/read-closely/shortlisted/selected line; the
|
||||
`Admitted via:` line; the `Learned signals:` line (rated-with-embeddings count and knn percentage,
|
||||
feed affinity state); `Near misses` — the 10 highest-utility non-selected articles as
|
||||
`<title> — <feed> · quality X · fit Y · <stage/reason>`; the `Models:` line with triage/assessment,
|
||||
editor/summaries and embeddings models; cost and generation time. Both editions (X4 gets the same
|
||||
text, no links). The data comes from the run's `RunReport`/`StageCounts` and the run's
|
||||
`candidate_runs` rows (add a `BehindThePaper` struct to `types.rs` filled by the pipeline; keep
|
||||
templates pure). Chapter ids stable (`behind`).
|
||||
2. **In this issue** page: verify each entry shows the `why` line under the summary (step 2 was asked
|
||||
to do this; complete it if missing). The colophon keeps its fields and shows per-provider cost
|
||||
lines and models (verify; complete if missing).
|
||||
3. **`stats`** (§15.3): `daily-epub stats [--days 14]` printing issues, articles published, explicit
|
||||
ratings by label, ratings per issue, up/down ratio per admitting retriever (`admitted_by[0]` of
|
||||
rated picks, from `candidate_runs` joined to `rating_events` via the latest explicit event),
|
||||
exploration yield (exploration picks selected / admitted, and how many were rated positively),
|
||||
mean issue size, cost per day per provider (from `runs.provider_costs_json`), mean generation time.
|
||||
Plain text, one fact per line, no tables wider than 80 columns.
|
||||
4. **Run report and log block** (§15.4): make sure `StageCounts` has every field listed —
|
||||
`eligible`, `embedded`, `triaged`, `admitted`, `admitted_by` (map), `assessed`, `shortlisted`,
|
||||
`clusters`, `exploration_admitted`, `exploration_selected`, `verdicts_in_prompt`,
|
||||
`rated_with_embeddings`, per-provider usage — and that stage timings include `embed`, `signals`,
|
||||
`triage`, `admit`, `assess`, `rank`, `editor`, `summaries`, `brief`. Emit the four-line info block
|
||||
from §15.4 once per run (`curation:`, `admission:`, `preference:`, `providers:`). `print_report`
|
||||
in `main.rs` prints the same lines.
|
||||
5. **`src/lock.rs`** (§5): `flock(LOCK_EX | LOCK_NB)` on `<database_path>.lock`, taken in `main`
|
||||
for `generate`, `profile rebuild`, `features backfill`, and `backfill-social`; a second invocation
|
||||
exits with `generate is already running` (name the command that holds it if cheap, else the
|
||||
generic message). `serve`, `explain`, `stats`, `ratings`, `db migrate`, `features prune` do not
|
||||
take it. Use `libc`/`rustix` only if already in the dependency tree (check `Cargo.lock`); otherwise
|
||||
`std::fs::File` + the `fs2`-free approach via `libc::flock` is acceptable to add as a tiny dep —
|
||||
say which in the report. Twenty lines, no table, no TTL.
|
||||
6. **Docs**: README rewritten where it describes the pipeline, feedback links, costs, prerequisites
|
||||
(Anthropic and Voyage keys, dashboard spend limits as the real backstop, the server-side fallback
|
||||
note), the CLI (`ratings`, `explain`, `stats`, `features`, `--rescore`, `--skip-embeddings`,
|
||||
`--max-articles` as ceiling), and `data/profile.md`. `config.example.toml` carries every key from
|
||||
plan §19 with comments. Startup logs resolved models and per-provider enabled state (verify).
|
||||
7. `systemd/daily-epub-generate.service`: no change needed unless an env var name changed; note the
|
||||
two new env vars in the README's systemd section.
|
||||
|
||||
Tests (§20 "Lock", "Pipeline" behind-the-paper, stats): two processes/one wins (spawn the binary or
|
||||
use two `File` handles from separate threads with `try_lock` semantics — no sleeping longer than a
|
||||
second), a killed holder frees the lock, `serve` does not take it (parse-level or by checking no lock
|
||||
file is created); the behind-the-paper chapter renders with the counts and near misses on the mocked
|
||||
pipeline run and is parseable XHTML in both editions (extend `tests/m4_epub.rs`); `stats` runs on a
|
||||
temp DB with a couple of runs, issues and rating events and prints every line; the info block appears
|
||||
in the log (capture with `tracing` test subscriber or assert on the formatted strings).
|
||||
@@ -0,0 +1,29 @@
|
||||
|
||||
---
|
||||
|
||||
# YOUR STEP: 7 — Cleanup (plan §21 step 7)
|
||||
|
||||
Steps 1–6 have landed. This step removes what the plan retired and updates the implementation
|
||||
notes. Read `git log -12` and skim every file under `src/` for leftovers.
|
||||
|
||||
1. **Dead code**: anything only the old prefilter/scores/select/feed-prior paths used — unused
|
||||
functions, types (`LlmScore` if nothing reads it, `ScoredArticle`, `FeedPrior`, `Rating`),
|
||||
`kv` keys no longer read, config aliases and `#[serde(alias)]`s introduced as transition aids,
|
||||
`// filled in by step N` comments, stale doc comments that still describe 👍/👎, "From the Editor",
|
||||
section intros, prefilter gating, or `async-openai`. `cargo clippy --all-targets -- -W dead_code`
|
||||
must be clean; do not add `#[allow(dead_code)]`.
|
||||
2. **Prune paths** (§7.1, §7.4): confirm `features prune` covers `article_embeddings`
|
||||
(`embedding_retention_days`) and `candidate_runs` (`telemetry_retention_days`); add
|
||||
`article_assessments` older than `telemetry_retention_days` to it. Make `generate` call the prune
|
||||
once per run after publishing (best effort, logged).
|
||||
3. **Docs**: update `docs/plans/2026-08-15-implementation-notes.md`: fix item 5 (hand-rolled
|
||||
`reqwest` client, not `async-openai`), add the new providers (Anthropic Messages API facts from
|
||||
plan §4.2, Voyage facts from §4.3, each with "verified 2026-09-02"), the new tables, the lock, the
|
||||
budget-day rule, and a short "curation v2" section pointing at
|
||||
`docs/plans/2026-09-02-personalized-curation-v2.md`. This is the one step allowed to edit `docs/`.
|
||||
Do not edit the plan itself.
|
||||
4. **Consistency pass**: `config.example.toml` matches `Config::default()` key for key (write a test
|
||||
that loads the example and compares every `[curation.*]`, `[anthropic]`, `[voyage]`, `[editorial]`
|
||||
value to the defaults, since the defaults are the plan's numbers); README CLI section matches
|
||||
`daily-epub --help` output for every subcommand (spot check by running the binary).
|
||||
5. Run `cargo fmt`, `cargo clippy --all-targets`, `cargo test`; report the summary lines.
|
||||
@@ -0,0 +1,64 @@
|
||||
# Web dashboard step 6 implementation review
|
||||
|
||||
The finished step-6 implementation matches the jobs and stats brief. The fixed
|
||||
job catalogue, CLI lifecycle, systemd runner and files, dashboard routes,
|
||||
stats-data refactor, SVG sparklines, production runner wiring, and required
|
||||
tests are all present. The review found and corrected two faulty test
|
||||
assertions, restored the polkit file to the brief's verbatim rule, made job
|
||||
messages reliably one-line, and strengthened the duplicate-start test to use a
|
||||
genuinely running row. No outstanding correctness finding remains.
|
||||
|
||||
## Critical
|
||||
|
||||
None.
|
||||
|
||||
## High
|
||||
|
||||
None.
|
||||
|
||||
## Medium
|
||||
|
||||
None.
|
||||
|
||||
## Low
|
||||
|
||||
None.
|
||||
|
||||
## Nits
|
||||
|
||||
None.
|
||||
|
||||
## Plan Coverage
|
||||
|
||||
| Requirement | Status | Evidence |
|
||||
|---|---|---|
|
||||
| Fixed job catalogue and validated names | Implemented as planned | `src/jobs.rs`: `Job`, `parse`, `name`, `unit`, `description`, `takes_lock`, `dangerous` and traversal/uppercase/unknown-name tests. |
|
||||
| `daily-epub job run <name>` lifecycle | Implemented as planned | `src/main.rs`: CLI dispatch, existing lock path, in-process command mapping, requested-row claim, terminal update, failure propagation and tests. `src/jobs.rs` owns row lifecycle helpers. |
|
||||
| Unit template, polkit rule and journal group | Implemented as planned | `systemd/daily-epub-job@.service`, verbatim `systemd/50-daily-epub.rules`, and `SupplementaryGroups=systemd-journal` in `systemd/daily-epub.service`; repository-content tests cover their security-sensitive strings. |
|
||||
| Production/test/disabled runners | Implemented as planned | `src/jobs.rs::SystemdRunner` uses bounded `tokio::process::Command`; `src/web/mod.rs` contains scripted `MockRunner` and `DisabledRunner`; `src/server.rs::serve` selects the production runner only when jobs are enabled while `AppState::new` remains test-compatible. |
|
||||
| Jobs list/start/detail pages | Implemented as planned | `src/web/dashboard/jobs.rs` and the two jobs templates implement catalogue cards, date selection, history, duplicate refusal, failed-start persistence, live status, journal tail, the 30-second failure rule, authorization tests and five-second refresh. |
|
||||
| Config reload integration boundary | Partially implemented by design | The required `// TODO(step 5 merge): reload_if_changed` is at the start site. Step 5 owns the helper and merge-time call per the brief's parallel-work constraint. |
|
||||
| Stats data/text split | Implemented as planned | `src/curate/telemetry.rs` exposes `StatsData`, `stats_data`, and `render_stats_text`; seeded fixtures pin the complete legacy output byte-for-byte. |
|
||||
| Stats page | Implemented as planned | `src/web/dashboard/stats.rs` and `dashboard/stats.html` provide the three allowed windows, tables, retriever yield, per-run data and the three requested charts. |
|
||||
| Overview sparklines | Implemented as planned | `src/web/dashboard/mod.rs` loads the last 30 finished non-dry runs and renders cost, selected count and duration through `_sparkline.html`. |
|
||||
| CSP-safe rendering | Implemented as planned | SVG geometry uses presentation attributes and stylesheet classes; changed templates contain no inline `style` attributes and no `safe` filter. |
|
||||
|
||||
## Testing Assessment
|
||||
|
||||
Meaningful tests cover every catalogue form and rejection case, job-row claim
|
||||
and completion (including run-id persistence), real router authorization and
|
||||
POST behavior with `MockRunner`, requested/running duplicate refusal, failed
|
||||
starts, the unit-exited grace rule, disabled jobs, systemd output parsing,
|
||||
polkit/unit contents, in-process job success and failure, exact
|
||||
legacy stats text, stats aggregation, SVG geometry, page output, and overview
|
||||
series.
|
||||
|
||||
The tests deliberately do not execute systemd or the network. The mapped
|
||||
generate branch is therefore covered by its option construction and the
|
||||
separate job run-id lifecycle test rather than a live pipeline invocation;
|
||||
this follows the brief's offline-test rule. No additional step-6 test is needed.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- During the step-5 merge, replace the required job-start TODO with the single
|
||||
`reload_if_changed` call supplied by step 5; do not duplicate that helper.
|
||||
@@ -0,0 +1,92 @@
|
||||
# Web dashboard step 5 implementation review
|
||||
|
||||
The final step 5 working tree implements the settings schema, editor, provider
|
||||
operations, history page, reload-on-mtime behavior, and systemd write access
|
||||
described by the plan. The implementation is well covered by behavior-focused
|
||||
unit and router tests. The review found one security issue in the inherited
|
||||
work—removing a provider could have copied a hand-written `api_key` into
|
||||
`config_changes`—and fixed it with redaction plus a regression assertion.
|
||||
|
||||
## Critical
|
||||
|
||||
None.
|
||||
|
||||
## High
|
||||
|
||||
None. The provider-history secret exposure found during review is fixed in
|
||||
`provider_literal`: `api_key` is removed before a whole-provider literal is
|
||||
stored or rendered by the history page.
|
||||
|
||||
## Medium
|
||||
|
||||
None.
|
||||
|
||||
## Low
|
||||
|
||||
### Shipped provider removal differs from the literal plan wording
|
||||
|
||||
- Evidence: `ProviderCard::removable` and `remove_provider` refuse entries in
|
||||
`default_providers()`, even when no `[llm]` role references them.
|
||||
- Plan difference: §13.3 only explicitly requires refusal when an `[llm]` role
|
||||
names the provider.
|
||||
- Reason and recommendation: deleting a shipped provider's file table cannot
|
||||
remove it from the effective config because `Config::default()` supplies it
|
||||
again; pretending otherwise would reset it while reporting removal. Keep the
|
||||
explicit refusal unless config loading later gains a provider tombstone or
|
||||
replacement-map semantic.
|
||||
|
||||
### Absent defaults are not materialized by an otherwise unchanged form
|
||||
|
||||
- Evidence: `plan_changes` compares an absent file value through its effective
|
||||
default before deciding whether to write.
|
||||
- Plan difference: §13.2's “explicit beats implicit” sentence can be read as
|
||||
requiring an absent default-valued key to be written whenever posted.
|
||||
- Reason and recommendation: the same section says the form carries every
|
||||
editable field and only changed fields are written. Materializing every
|
||||
absent default on any no-JavaScript save would violate that behavior. Keep
|
||||
the effective-value comparison; a reset from an explicit non-default value
|
||||
still writes the default explicitly. Clarify this sentence in a future plan
|
||||
revision if touched-vs-untouched browser state becomes a requirement.
|
||||
|
||||
## Nits
|
||||
|
||||
None.
|
||||
|
||||
## Plan Coverage
|
||||
|
||||
| Requirement | Status | Evidence |
|
||||
|---|---|---|
|
||||
| Derived schema, kinds, sources, help, group order and anchors | Implemented as planned | `schema`, `schema_with_env`, `walk`, `kind_for`, `SETTINGS_HELP` |
|
||||
| Secret redaction and environment locks | Implemented as planned | `source_for`, secret field rendering, provider-literal redaction |
|
||||
| Typed `toml_edit` save with collected errors | Implemented as planned | `plan_changes`, `apply_change` |
|
||||
| Temp-file validation, permission preservation, rename and live swap | Implemented as planned | `write_validated`, `install` |
|
||||
| One attributed history row per changed key | Implemented as planned | `record_changes`, `config_changes` |
|
||||
| Provider add/remove and referenced-provider refusal | Implemented with the shipped-provider qualification above | `add_provider`, `remove_provider` |
|
||||
| Reload on mtime with previous config retained after an error | Implemented as planned | `WebState::reload_if_changed`, settings GET banner |
|
||||
| Settings history page, newest first, 100 per page | Implemented as planned | `history`, `config_changes`, `settings_history.html` |
|
||||
| `/etc/daily-epub` writable in the server unit | Implemented as planned | `systemd/daily-epub.service` |
|
||||
|
||||
## Testing Assessment
|
||||
|
||||
Existing tests are meaningful: they compare the schema with serialized
|
||||
`Config::default()`, compare shipped TOML leaves with the help table, exercise
|
||||
all enum options through deserialization and `validate()`, prove secrets carry
|
||||
no value, mutate and restore a real environment variable, compare untouched
|
||||
configuration lines byte-for-byte, verify multiline arrays and new tables,
|
||||
exercise validation failure and permission preservation, inspect persisted
|
||||
history rows, add/remove/refuse providers, and drive the settings routes through
|
||||
`Router::oneshot`. The provider test also proves a file-sourced API key does not
|
||||
reach the audit table.
|
||||
|
||||
No weak or missing test from the step 5 list remains. The only environmental
|
||||
suite limitation is the repository's pre-existing listener tests, which cannot
|
||||
bind loopback in the sandbox.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Should the plan eventually define a tombstone for removing shipped providers,
|
||||
or should the current “leave built-ins unreferenced” behavior become the
|
||||
documented contract?
|
||||
- The shared brief lists four Anthropic listener tests as sandbox-bound, while
|
||||
the step 1 handoff also identifies three OpenAI tests using the same loopback
|
||||
fake server. The latter fail with the identical sandbox permission error.
|
||||
@@ -0,0 +1,269 @@
|
||||
# Runbook — putting daily.hallada.net behind Cloudflare
|
||||
|
||||
> **Status (2026-09-05): retired.** The `daily` record is **DNS only** again, one
|
||||
> day after this rollout. Measured from Boston with a session cookie (so the edge
|
||||
> bypassed its cache, as it does for every signed-in request): 70 ms after the
|
||||
> TLS handshake through the proxy against 27 ms straight to the origin. The edge
|
||||
> adds a proxy hop and its own overhead on every uncached request, and at this
|
||||
> traffic the cache is cold for anonymous readers too, so the proxy cost the one
|
||||
> signed-in reader more than it gave anyone. The zone stays on Cloudflare's
|
||||
> nameservers; flipping the record back to Proxied re-enables everything below.
|
||||
> When you do, also put the `cloudflare-real-ip.conf` include back into the nginx
|
||||
> server block (README, *Reverse proxy*) — it was removed with the proxy because
|
||||
> a direct connection from a Cloudflare address could otherwise name its own
|
||||
> client IP. The origin's `Cache-Control` matrix (§0) is still in force and is
|
||||
> what makes the toggle safe in either direction.
|
||||
|
||||
**Written:** 2026-09-04 for the production host. Steps 1–3 happen in the Cloudflare
|
||||
and registrar dashboards; steps 4–5 are on the server as an operator with `sudo`.
|
||||
The origin-side changes (the `Cache-Control` matrix below) ship in the same
|
||||
release; deploy the new binary before enabling the proxy.
|
||||
|
||||
Read the whole thing before starting step 1: a nameserver change is the one step
|
||||
that cannot be undone in seconds.
|
||||
|
||||
## 0. What the origin already does
|
||||
|
||||
The app decides what is cacheable; Cloudflare is configured only to obey it.
|
||||
|
||||
| route | `Cache-Control` |
|
||||
|---|---|
|
||||
| `/`, `/issues`, `/issues/{date}`, `/feed.xml`, `/issues.json` (anonymous) | `public, max-age=300` |
|
||||
| the same pages with a `daily_session=` cookie | `private, no-store` |
|
||||
| `/robots.txt` | `public, max-age=86400` |
|
||||
| `/static/*?v=<hash>` | `public, max-age=31536000, immutable` |
|
||||
| `/static/*` with no `?v=` | `public, max-age=3600` |
|
||||
| `/files/epub/*`, `/files/xtc/*`, `/opds*` (every status, 401 and 404 included) | `private, no-store` |
|
||||
| `/dashboard*`, `/login`, `/account`, error pages, anything else | `no-store` |
|
||||
|
||||
Two facts follow from that table and matter for every choice below:
|
||||
|
||||
- A public page is fresh for **five minutes** everywhere: browser and edge alike
|
||||
revalidate within five minutes of a new issue landing, and nothing has to tell
|
||||
the edge that the day changed. (A one-day edge age with an API purge after
|
||||
every publish was tried and dropped: at this traffic an edge rarely keeps a
|
||||
page long enough for the longer age to matter, and the purge was one more
|
||||
token, config section and failure mode.)
|
||||
- Anything gated by a cookie or Basic auth already says `private, no-store`, so
|
||||
even a misconfigured cache-everything rule cannot make a download public. The
|
||||
cookie bypass rule in step 3 is defence in depth, not the only defence.
|
||||
|
||||
## 1. Move DNS to Cloudflare
|
||||
|
||||
The Free plan has no partial (CNAME) setup, so this is a full nameserver move:
|
||||
Cloudflare becomes authoritative for the entire `hallada.net` zone, not just this
|
||||
host. Everything else in the zone must survive the move unchanged.
|
||||
|
||||
1. Export the current zone from Route 53 as BIND:
|
||||
|
||||
```sh
|
||||
ZONE=$(aws route53 list-hosted-zones-by-name --dns-name hallada.net. \
|
||||
--query 'HostedZones[0].Id' --output text | cut -d/ -f3)
|
||||
aws route53 list-resource-record-sets --hosted-zone-id "$ZONE" > /tmp/hallada-net.json
|
||||
```
|
||||
|
||||
The AWS CLI has no BIND exporter; the console's **Export zone file** button on
|
||||
the hosted zone does produce one. Keep both files — the JSON is the rollback
|
||||
reference.
|
||||
|
||||
2. In Cloudflare, **Add a site** → `hallada.net` → Free plan → **Import DNS
|
||||
records** and upload the BIND file. Then read the imported list against the
|
||||
export line by line. Pay attention to `MX`, `TXT` (SPF/DKIM/DMARC), and any
|
||||
`CNAME` used for verification: those must all stay **DNS only** (grey cloud).
|
||||
|
||||
3. Set **only** the `daily` record to **Proxied** (orange cloud). Nothing else in
|
||||
the zone should be proxied — proxying an MX target or a mail-related host
|
||||
breaks it.
|
||||
|
||||
4. Change the nameservers at the registrar to the two Cloudflare assigns.
|
||||
Propagation is usually minutes and can be hours. Cloudflare emails when the
|
||||
zone goes active. **Keep `daily` DNS-only until the zone is Active *and*
|
||||
SSL/TLS → Edge Certificates shows the Universal certificate as Active.** A
|
||||
proxied host with no edge certificate yet fails every TLS handshake
|
||||
(`SSL alert number 40`) while plain HTTP still redirects, which is exactly
|
||||
what happened on the first attempt: readers whose resolvers had picked up
|
||||
the new nameservers got a broken site. Set Full (strict) (step 2) before
|
||||
flipping the record to Proxied.
|
||||
|
||||
5. Before the switch, drop the TTL on the `daily` record (60 s) so a rollback is
|
||||
quick. Verify afterwards:
|
||||
|
||||
```sh
|
||||
dig +short NS hallada.net
|
||||
dig +short daily.hallada.net # Cloudflare anycast addresses once proxied
|
||||
dig +short MX hallada.net # unchanged
|
||||
```
|
||||
|
||||
## 2. Zone settings
|
||||
|
||||
Under the zone's **SSL/TLS**, **Speed** and **Scrape Shield** sections:
|
||||
|
||||
- **SSL/TLS → Overview → Full (strict)**. The origin has a real Let's Encrypt
|
||||
certificate, so there is no reason to accept anything weaker. *Flexible* would
|
||||
make Cloudflare talk plain HTTP to the origin and would break the redirect in
|
||||
the nginx `:80` server block into a loop.
|
||||
- **Caching → Configuration → Browser Cache TTL → Respect Existing Headers.**
|
||||
This one is easy to miss and quietly wrong: the default (**4 hours**) *raises*
|
||||
the `max-age` Cloudflare sends to browsers, so the deliberate 5-minute browser
|
||||
age on the public pages would become 4 hours and a reader's tab would show a
|
||||
stale paper for most of a morning.
|
||||
- **Speed → Optimization → Rocket Loader: OFF.**
|
||||
- **Scrape Shield → Email Address Obfuscation: OFF.**
|
||||
Both inject a Cloudflare-hosted script into the HTML. The site's CSP is
|
||||
`script-src 'self'`, so the browser blocks the injected script and the page
|
||||
either breaks or logs CSP violations on every load. There is nothing to gain:
|
||||
the site has no email addresses in its markup and its own JS is two small
|
||||
files.
|
||||
- **Do not enable Web Analytics with automatic injection** — same reason, it is
|
||||
an injected third-party script. If analytics are wanted later, they have to be
|
||||
first-party and listed in the CSP.
|
||||
- **Always Use HTTPS: ON** is fine and lets the origin's `:80` block stay as a
|
||||
backstop.
|
||||
|
||||
## 3. Cache Rules
|
||||
|
||||
**Caching → Cache Rules.** Order matters, and not the intuitive way: when
|
||||
several rules match one request, **the last matching rule wins** for each
|
||||
setting. So a "bypass" rule placed *above* a "cache" rule is silently overruled
|
||||
for every cookie-bearing request, and signed-in readers get `HIT`s of the
|
||||
anonymous page (this happened on the first rollout). Two safe layouts: put the
|
||||
bypass rule **last**, or, better, write the cookie exclusion into the cache
|
||||
rule's own expression as shown below so order no longer matters.
|
||||
|
||||
**Rule 1 — "Bypass cache for signed-in readers"**
|
||||
|
||||
- When incoming requests match: `http.cookie contains "daily_session="`
|
||||
- Then: **Bypass cache**
|
||||
|
||||
The origin already sends `private, no-store` to a cookie-bearing request, but
|
||||
this makes the bypass a property of the request rather than of the response, so
|
||||
nothing is ever *looked up* in a shared cache for a signed-in reader. If rule 2
|
||||
carries the `not http.cookie contains` clause this rule is belt-and-braces;
|
||||
keep it anyway, and keep it **below** rule 2.
|
||||
|
||||
**Rule 2 — "Cache by origin headers"** *(optional)*
|
||||
|
||||
Without this rule Cloudflare caches only its default static extensions (the
|
||||
CSS, JS, fonts and SVG here) and passes HTML straight through, which already
|
||||
captures most of the benefit: TLS terminates at the edge and the origin is
|
||||
reached over a warm connection. Add the rule if you want the HTML itself
|
||||
served from the edge; it is only safe together with rule 1.
|
||||
|
||||
- When incoming requests match:
|
||||
`http.host eq "daily.hallada.net" and not http.cookie contains "daily_session="`
|
||||
- Then: **Eligible for cache**
|
||||
- **Edge TTL:** *Use cache-control header if present, bypass cache if not*
|
||||
- **Browser TTL:** *Respect origin*
|
||||
|
||||
That Edge TTL mode is the whole point of the origin work: a response with
|
||||
`max-age` is cached for exactly that long, and a response with `no-store` (or
|
||||
one that somehow arrives with no policy at all) is not cached. It is the reason
|
||||
the `security_headers` middleware defaults unknown routes to `no-store` — with
|
||||
this mode, "no header" means "do not cache" rather than "cache for 2 hours".
|
||||
|
||||
Leave **Cache Key** at its default. Do not add "Ignore query string": the
|
||||
`?v=<hash>` on `/static/*` is what makes the immutable one-year `max-age` safe.
|
||||
|
||||
## 4. Lock the origin to Cloudflare
|
||||
|
||||
Once traffic arrives through the edge, direct hits on the origin's port 443
|
||||
should stop. Either is enough; the first is simpler.
|
||||
|
||||
- **Firewall:** allow 443 only from the published Cloudflare ranges.
|
||||
|
||||
```sh
|
||||
{ curl -s https://www.cloudflare.com/ips-v4; echo; \
|
||||
curl -s https://www.cloudflare.com/ips-v6; echo; } \
|
||||
| awk 'NF {print $0}' \
|
||||
| xargs -I{} sudo ufw allow proto tcp from {} to any port 443 comment 'cloudflare'
|
||||
sudo ufw delete allow 443/tcp # remove the open rule last
|
||||
```
|
||||
|
||||
Keep a way in that does not depend on this (SSH from your own address) before
|
||||
removing the open rule.
|
||||
|
||||
- **Authenticated Origin Pulls** (SSL/TLS → Origin Server) instead: install
|
||||
Cloudflare's client CA on the origin and add `ssl_client_certificate` +
|
||||
`ssl_verify_client on` to the nginx server block. Stronger, but one more
|
||||
certificate to keep track of.
|
||||
|
||||
Also apply the nginx changes from the README's *Reverse proxy* section now:
|
||||
the `include /etc/nginx/snippets/cloudflare-real-ip.conf;` (generated from
|
||||
Cloudflare's published ranges, `real_ip_header CF-Connecting-IP;` at the end) and
|
||||
`proxy_set_header X-Forwarded-For $remote_addr;` in place of
|
||||
`$proxy_add_x_forwarded_for`. Without the first, every request looks like it came
|
||||
from Cloudflare and the login throttle becomes global; without the second, a
|
||||
client could forge the throttle key. Then:
|
||||
|
||||
```sh
|
||||
sudo nginx -t && sudo systemctl reload nginx
|
||||
```
|
||||
|
||||
Check a login attempt from a phone on cellular and one from the LAN land in
|
||||
different throttle buckets, and that `journalctl -u daily-epub` shows real client
|
||||
addresses rather than Cloudflare's.
|
||||
|
||||
## 5. Verify
|
||||
|
||||
Anonymous public page — with rule 2, expect `HIT` on the second request (the
|
||||
first fills the edge); without it, `DYNAMIC`. Either way the origin's own
|
||||
`Cache-Control` must come through unchanged:
|
||||
|
||||
```sh
|
||||
curl -sI https://daily.hallada.net/ | grep -iE 'cf-cache-status|cache-control|age'
|
||||
curl -sI https://daily.hallada.net/ | grep -i cf-cache-status # HIT
|
||||
```
|
||||
|
||||
Signed in — the cookie rule must take it out of the cache entirely. A `HIT`
|
||||
here, with the anonymous page's `/login` link in the body, means the bypass rule
|
||||
is being overruled by the cache rule: see the ordering note in step 3.
|
||||
|
||||
```sh
|
||||
curl -sI -H 'Cookie: daily_session=whatever' https://daily.hallada.net/ \
|
||||
| grep -iE 'cf-cache-status|cache-control'
|
||||
# cf-cache-status: BYPASS (DYNAMIC is also acceptable)
|
||||
# cache-control: private, no-store
|
||||
```
|
||||
|
||||
Static assets — versioned URLs should settle on `HIT`:
|
||||
|
||||
```sh
|
||||
curl -s https://daily.hallada.net/ | grep -o '/static/app.css?v=[0-9a-f]*'
|
||||
curl -sI 'https://daily.hallada.net/static/app.css?v=<hash>' \
|
||||
| grep -iE 'cf-cache-status|cache-control'
|
||||
# cf-cache-status: HIT
|
||||
# cache-control: public, max-age=31536000, immutable
|
||||
```
|
||||
|
||||
Downloads and OPDS — never cached, at any status:
|
||||
|
||||
```sh
|
||||
curl -sI https://daily.hallada.net/opds | grep -iE 'cf-cache-status|cache-control'
|
||||
curl -sI https://daily.hallada.net/files/epub/does-not-exist.epub \
|
||||
| grep -iE 'cf-cache-status|cache-control'
|
||||
# cache-control: private, no-store (cf-cache-status: BYPASS or DYNAMIC)
|
||||
```
|
||||
|
||||
The dashboard and the login page:
|
||||
|
||||
```sh
|
||||
curl -sI https://daily.hallada.net/login | grep -iE 'cf-cache-status|cache-control'
|
||||
# cache-control: no-store
|
||||
```
|
||||
|
||||
Finally, the morning after: within five minutes of the `generate` timer firing,
|
||||
`curl -s https://daily.hallada.net/ | grep -o 'issues/[0-9-]*' | head -1` shows
|
||||
the new date from a network that is not signed in.
|
||||
|
||||
## Rollback
|
||||
|
||||
- **Cache misbehaving:** turn on **Development Mode** (Caching → Configuration)
|
||||
for a three-hour edge bypass while you look, or **Purge Everything**. Neither
|
||||
needs a deploy.
|
||||
- **Something worse:** set the `daily` DNS record back to **DNS only** (grey
|
||||
cloud). Traffic goes straight to the origin again within the record's TTL, and
|
||||
nothing about the origin's behaviour depends on Cloudflare being there — the
|
||||
`Cache-Control` headers are correct without it.
|
||||
- **Full retreat:** point the registrar's nameservers back at Route 53. The
|
||||
hosted zone still exists unless it was deleted; do not delete it until the
|
||||
Cloudflare setup has run for a few weeks.
|
||||
@@ -0,0 +1,281 @@
|
||||
# Runbook — upgrading the server from v1 to curation v2
|
||||
|
||||
**Written:** 2026-09-02, for the `curation-v2` branch (all seven plan steps plus the provider
|
||||
registry). Every command below is meant to be run on the server, as `root` via `sudo`, unless it
|
||||
says otherwise. Expect the whole thing to take about an hour, most of it waiting on the embedding
|
||||
backfill and one dry run.
|
||||
|
||||
What changes for the operator, in one paragraph: the binary is replaced; the SQLite schema gains
|
||||
tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first);
|
||||
`config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a
|
||||
`profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained
|
||||
`profile.md` is installed for the hand-maintained reader profile; the systemd units are unchanged.
|
||||
|
||||
## 0. Before touching the server
|
||||
|
||||
On the dev box:
|
||||
|
||||
```sh
|
||||
git checkout curation-v2
|
||||
cargo test # expect green
|
||||
cargo build --release # or build on the server, as you do today
|
||||
```
|
||||
|
||||
The crate uses `edition = "2024"` and let-chains, so the server's toolchain must be current
|
||||
(`rustup update stable`). Read `data/profile.md` once; it is the reader profile you will be
|
||||
editing by hand from now on, and the first thing to tune when the paper feels off.
|
||||
|
||||
## 1. Freeze the timer, back everything up
|
||||
|
||||
```sh
|
||||
sudo systemctl stop daily-epub-generate.timer daily-epub-generate.service
|
||||
sudo systemctl stop daily-epub.service # the server also runs migrations on start
|
||||
|
||||
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/backup
|
||||
sudo -u daily-epub sqlite3 /var/lib/daily-epub/daily-epub.db \
|
||||
".backup '/var/lib/daily-epub/backup/daily-epub-pre-v2-$(date +%F).db'"
|
||||
sudo cp -a /etc/daily-epub/config.toml /etc/daily-epub/config.toml.v1
|
||||
sudo cp -a /etc/daily-epub/env /etc/daily-epub/env.v1
|
||||
sudo cp -a /usr/local/bin/daily-epub /usr/local/bin/daily-epub.v1
|
||||
```
|
||||
|
||||
`.backup` is the safe way to copy a WAL-mode database; a plain `cp` while anything is open can
|
||||
miss the WAL. The migration is one-way (`ratings` is dropped after being copied), so this backup
|
||||
is the rollback path.
|
||||
|
||||
## 2. Install the new binary
|
||||
|
||||
```sh
|
||||
sudo install -m0755 target/release/daily-epub /usr/local/bin/
|
||||
daily-epub --version
|
||||
daily-epub --help # confirm `config`, `stats`, `explain`, `ratings`, `features` exist
|
||||
```
|
||||
|
||||
Do **not** start any service yet: every subcommand except `config check` opens the database and
|
||||
applies pending migrations, and you want the config right first.
|
||||
|
||||
The units in `systemd/` did not change. If you want to be sure your installed copies match:
|
||||
|
||||
```sh
|
||||
diff systemd/daily-epub-generate.service /etc/systemd/system/daily-epub-generate.service
|
||||
diff systemd/daily-epub.service /etc/systemd/system/daily-epub.service
|
||||
```
|
||||
|
||||
## 3. Edit `config.toml` in place
|
||||
|
||||
Everything you do not mention keeps its documented default, so the edit is small. Open it with
|
||||
`sudo -e /etc/daily-epub/config.toml` and apply this checklist.
|
||||
|
||||
**Remove** (each one now fails loudly at startup, naming its replacement):
|
||||
|
||||
| Old key | Why |
|
||||
|---|---|
|
||||
| `interests_opml = …` (top level) | standing interests now live in SQLite; remove this before rollout because unknown keys fail startup |
|
||||
| `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) |
|
||||
| `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` |
|
||||
| the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) |
|
||||
| any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` |
|
||||
|
||||
**Add** near the top:
|
||||
|
||||
```toml
|
||||
profile_path = "/var/lib/daily-epub/data/profile.md"
|
||||
```
|
||||
|
||||
Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which
|
||||
under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an
|
||||
explicit path survives running one-off commands from another directory.
|
||||
|
||||
**Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old
|
||||
`[deepseek]` table if you had changed them; the values shown are the defaults.
|
||||
|
||||
```toml
|
||||
[llm]
|
||||
bulk = "deepseek" # triage, deep assessment, and the fallback for every editor call
|
||||
editor = "anthropic" # lineup, summaries, the Brief, the weekly profile rebuild
|
||||
triage_batch_size = 25
|
||||
deep_batch_size = 8
|
||||
score_temperature = 0.3
|
||||
editorial_temperature = 0.8
|
||||
|
||||
[providers.deepseek]
|
||||
kind = "openai"
|
||||
base_url = "https://api.deepseek.com/v1"
|
||||
model = "deepseek-v4-flash"
|
||||
max_daily_usd = 2.0
|
||||
max_concurrent_requests = 4
|
||||
price_input_per_mtok = 0.14
|
||||
price_cache_read_per_mtok = 0.0028
|
||||
price_cache_write_per_mtok = 0.0
|
||||
price_output_per_mtok = 0.28
|
||||
|
||||
[providers.anthropic]
|
||||
kind = "anthropic"
|
||||
base_url = "https://api.anthropic.com"
|
||||
model = "claude-opus-5"
|
||||
effort = "high"
|
||||
max_daily_usd = 3.0
|
||||
max_concurrent_requests = 4
|
||||
price_input_per_mtok = 5.0
|
||||
price_cache_read_per_mtok = 0.5
|
||||
price_cache_write_per_mtok = 6.25
|
||||
price_output_per_mtok = 25.0
|
||||
```
|
||||
|
||||
Optional, for the Gemini comparison (section 9):
|
||||
|
||||
```toml
|
||||
[providers.gemini]
|
||||
kind = "openai" # Gemini's OpenAI-compatible endpoint
|
||||
base_url = "https://generativelanguage.googleapis.com/v1beta/openai"
|
||||
model = "gemini-3.8-flash"
|
||||
effort = "high"
|
||||
max_daily_usd = 3.0
|
||||
max_concurrent_requests = 4
|
||||
price_input_per_mtok = 0.75 # $1.50 from 2027-01-01
|
||||
price_cache_read_per_mtok = 0.075 # $0.15 from 2027-01-01
|
||||
price_cache_write_per_mtok = 0.0
|
||||
price_output_per_mtok = 3.75 # includes thinking tokens; $7.50 from 2027-01-01
|
||||
```
|
||||
|
||||
**Leave alone** `[miniflux]`, `[server]`, `[publish]`, `[xtc]`, `[world]`, `[curation]
|
||||
sections`, `always_include_feeds`, `blocked_domains`. `[voyage]`, `[editorial]`,
|
||||
`[curation.feedback]` and `[curation.ranking]` all have sensible defaults; copy a section from
|
||||
`config.example.toml` only when you want to change a value in it.
|
||||
|
||||
## 4. Edit the env file
|
||||
|
||||
```sh
|
||||
sudo -e /etc/daily-epub/env
|
||||
```
|
||||
|
||||
| Variable | Action |
|
||||
|---|---|
|
||||
| `DAILY_EPUB_DEEPSEEK__API_KEY` | **rename** to `DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY` (the old name is rejected at startup so it cannot silently disable the bulk model) |
|
||||
| `DAILY_EPUB_PROVIDERS__ANTHROPIC__API_KEY` | add |
|
||||
| `DAILY_EPUB_VOYAGE__API_KEY` | add |
|
||||
| `DAILY_EPUB_PROVIDERS__GEMINI__API_KEY` | add only if you configured `[providers.gemini]` |
|
||||
| `DAILY_EPUB_MINIFLUX__API_KEY`, `DAILY_EPUB_SERVER__HMAC_SECRET` | unchanged |
|
||||
|
||||
Keep it `0600 daily-epub:daily-epub`. Then set hard spend limits in the DeepSeek, Anthropic and
|
||||
Voyage dashboards: the in-app `max_daily_usd` meters are runaway guards, not accounting.
|
||||
|
||||
## 5. Install the reader profile
|
||||
|
||||
```sh
|
||||
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data
|
||||
sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md
|
||||
```
|
||||
|
||||
If the file is missing the run does not fail; it logs a warning and uses empty profile prose,
|
||||
which is a much worse prompt. `config check` in the next step tells you whether it was found.
|
||||
|
||||
## 6. Check the config as the service user
|
||||
|
||||
The units run as `daily-epub` with the env file loaded, so check the same way. This helper runs
|
||||
one command in that identity, with the env file and the working directory the unit uses, and no
|
||||
other hardening:
|
||||
|
||||
```sh
|
||||
de() { sudo systemd-run --quiet --wait --pty --collect \
|
||||
--uid=daily-epub --gid=daily-epub \
|
||||
-p WorkingDirectory=/var/lib/daily-epub -p EnvironmentFile=/etc/daily-epub/env \
|
||||
/usr/local/bin/daily-epub --config /etc/daily-epub/config.toml "$@"; }
|
||||
|
||||
de config check
|
||||
```
|
||||
|
||||
Expected: every line is a fact, no line starts with `!`. Fix anything marked `MISSING` (a key
|
||||
name, a path) before continuing. A stale key in the TOML is reported as an error naming its
|
||||
replacement; go back to section 3.
|
||||
|
||||
## 7. Migrate the schema
|
||||
|
||||
```sh
|
||||
de db migrate
|
||||
sudo -u daily-epub sqlite3 /var/lib/daily-epub/daily-epub.db '.tables'
|
||||
# expect rating_events, article_embeddings, interest_embeddings, article_assessments,
|
||||
# candidate_runs; no ratings, feed_priors or scores
|
||||
sudo -u daily-epub sqlite3 /var/lib/daily-epub/daily-epub.db \
|
||||
"select label, count(*) from rating_events group by label;"
|
||||
# your old votes: up → loved, down → not_for_me, source = 'migration'
|
||||
```
|
||||
|
||||
## 8. Warm the embedding cache, then a dry run
|
||||
|
||||
```sh
|
||||
de features backfill --rated-only # the learned set; prints a token estimate first
|
||||
de features backfill --days 30 # recent articles, so day one is not all cache misses
|
||||
de generate --dry-run --out /var/lib/daily-epub/out-check
|
||||
```
|
||||
|
||||
The dry run makes real DeepSeek, Claude and Voyage calls but publishes nothing and writes no
|
||||
`issues` row. Read the printed lineup and the four report lines (`curation:`, `admission:`,
|
||||
`preference:`, `providers:`); the cost should be well under $1. Then read the EPUB it wrote
|
||||
(Calibre or KOReader): The Brief and the new
|
||||
Behind-the-paper chapter before the colophon. Finally:
|
||||
|
||||
```sh
|
||||
de explain --date "$(date +%F)" --near-misses
|
||||
```
|
||||
|
||||
Remove `/var/lib/daily-epub/out-check` when done.
|
||||
|
||||
## 9. Go live
|
||||
|
||||
```sh
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl start daily-epub.service
|
||||
sudo systemctl enable --now daily-epub-generate.timer
|
||||
sudo systemctl list-timers daily-epub-generate.timer
|
||||
```
|
||||
|
||||
If you want today's paper regenerated by the new pipeline now rather than tomorrow at 05:30:
|
||||
`sudo systemctl start daily-epub-generate`. A same-date rerun replaces today's issue and is a
|
||||
new run id in telemetry.
|
||||
|
||||
Afterwards:
|
||||
|
||||
```sh
|
||||
journalctl -u daily-epub-generate -n 60 --no-pager # the four-line info block near the end
|
||||
de stats --days 14
|
||||
```
|
||||
|
||||
## 10. Comparing editors (Claude Opus 5 vs Gemini 3.8 Flash)
|
||||
|
||||
With `[providers.gemini]` and its key in place, an A/B needs no config edit: environment variables
|
||||
override the TOML, so
|
||||
|
||||
```sh
|
||||
sudo systemd-run --quiet --wait --pty --collect --uid=daily-epub --gid=daily-epub \
|
||||
-p WorkingDirectory=/var/lib/daily-epub -p EnvironmentFile=/etc/daily-epub/env \
|
||||
-E DAILY_EPUB_LLM__EDITOR=gemini \
|
||||
/usr/local/bin/daily-epub --config /etc/daily-epub/config.toml \
|
||||
generate --dry-run --date "$(date +%F)" --out /var/lib/daily-epub/out-gemini
|
||||
```
|
||||
|
||||
produces the same date's paper with Gemini as the editor. Triage and deep assessments are cached
|
||||
per article for three days, so the second run re-requests only articles that have no row yet:
|
||||
ones ingested since the first run, and ones whose earlier batch was rejected by the provider's
|
||||
content filter (those are bisected, retried once on the editor provider, and then recorded as
|
||||
`provider_rejected` so they are not retried daily). The `triage:` and `assess:` log lines say how
|
||||
many were reused versus requested. The bulk-side cost of the second run is therefore small; the
|
||||
editor, summaries and the Brief are the real spend. Compare the two
|
||||
lineups and the Brief side by side, and the `providers:` cost line. To switch
|
||||
for good, set `editor = "gemini"` in `[llm]` (and `summary_model` stays `editor`, so summaries
|
||||
move with it). The same trick works for the bulk role: `DAILY_EPUB_LLM__BULK=gemini`.
|
||||
|
||||
## Rollback
|
||||
|
||||
```sh
|
||||
sudo systemctl stop daily-epub-generate.timer daily-epub.service
|
||||
sudo install -m0755 /usr/local/bin/daily-epub.v1 /usr/local/bin/daily-epub
|
||||
sudo cp -a /etc/daily-epub/config.toml.v1 /etc/daily-epub/config.toml
|
||||
sudo cp -a /etc/daily-epub/env.v1 /etc/daily-epub/env
|
||||
sudo -u daily-epub cp /var/lib/daily-epub/backup/daily-epub-pre-v2-<date>.db /var/lib/daily-epub/daily-epub.db
|
||||
sudo rm -f /var/lib/daily-epub/daily-epub.db-wal /var/lib/daily-epub/daily-epub.db-shm
|
||||
sudo systemctl start daily-epub.service daily-epub-generate.timer
|
||||
```
|
||||
|
||||
The database restore is required, not optional: the v1 binary expects `ratings`, `feed_priors`
|
||||
and `scores`, which v2's migrations drop.
|
||||
@@ -0,0 +1,146 @@
|
||||
# Runbook — web dashboard rollout
|
||||
|
||||
**Written:** 2026-09-03 for the production host. Run these commands from the checked-out
|
||||
`web-dashboard` tree as an operator with `sudo`. The existing curation-v2 migration should
|
||||
already be complete.
|
||||
|
||||
## 1. Build and install
|
||||
|
||||
The deployed binary does not require Node. For UI development, edit
|
||||
`src/web/tailwind.css`, the templates, or `src/web/static/app.js`, then run
|
||||
`npm run css` and commit the regenerated `src/web/static/app.css` alongside the
|
||||
source. The `/static/*.css|js` URLs carry a hash of the embedded assets
|
||||
(`web::ASSET_VERSION`), so a new build busts browser caches on its own — no
|
||||
hard refresh and no version bump needed. CI and release preparation should run `npm run css:check` to detect
|
||||
stylesheet drift.
|
||||
|
||||
```sh
|
||||
npm ci
|
||||
npm run css:check
|
||||
cargo test
|
||||
cargo build --release
|
||||
sudo install -m0755 target/release/daily-epub /usr/local/bin/daily-epub
|
||||
daily-epub --help
|
||||
```
|
||||
|
||||
Confirm the help includes both `users` and `job`. Keep the previous binary available until the
|
||||
smoke tests below pass.
|
||||
|
||||
## 2. Install the job unit and polkit rule
|
||||
|
||||
```sh
|
||||
sudo install -m0644 systemd/daily-epub-job@.service /etc/systemd/system/
|
||||
sudo install -m0644 systemd/50-daily-epub.rules /etc/polkit-1/rules.d/
|
||||
sudo install -m0644 systemd/daily-epub.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
The server unit must contain both:
|
||||
|
||||
```ini
|
||||
SupplementaryGroups=systemd-journal
|
||||
ReadWritePaths=/home/thallada/bookorbit/books/daily-epub /var/lib/daily-epub/xtc /etc/daily-epub
|
||||
```
|
||||
|
||||
Keep the two publish paths aligned with `[publish]`. The polkit rule permits user `daily-epub`
|
||||
to start only `daily-epub-job@[a-z0-9-]+.service`; it does not grant stop, restart, or arbitrary
|
||||
unit management.
|
||||
|
||||
## 3. Verify writable paths and ownership
|
||||
|
||||
Settings saves create a temporary file and rename it into `/etc/daily-epub`, so the directory
|
||||
and config file must both belong to the service account:
|
||||
|
||||
```sh
|
||||
sudo chown daily-epub:daily-epub /etc/daily-epub /etc/daily-epub/config.toml
|
||||
sudo chmod 0750 /etc/daily-epub
|
||||
sudo chmod 0640 /etc/daily-epub/config.toml
|
||||
sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data
|
||||
sudo test -f /var/lib/daily-epub/data/profile.md
|
||||
sudo -u daily-epub test -w /var/lib/daily-epub/data/profile.md
|
||||
```
|
||||
|
||||
Set `profile_path = "/var/lib/daily-epub/data/profile.md"`; `StateDirectory=daily-epub` makes
|
||||
that location writable. Also re-check the configured publish directories as `daily-epub`.
|
||||
|
||||
## 4. Review the new server settings
|
||||
|
||||
All keys have defaults, so additions are optional unless production needs overrides:
|
||||
|
||||
```toml
|
||||
[server]
|
||||
bind = "127.0.0.1:3499"
|
||||
public_url = "https://daily.hallada.net"
|
||||
session_days = 30
|
||||
login_attempts = 10
|
||||
login_window_minutes = 15
|
||||
jobs_enabled = true
|
||||
journal_lines = 300
|
||||
```
|
||||
|
||||
Keep the bind address on loopback: the login throttle trusts `X-Forwarded-For`, so only the
|
||||
reverse proxy should be able to reach the listener. Keep secrets in `/etc/daily-epub/env`, not
|
||||
TOML. If `server.basic_auth_user` and `server.basic_auth_pass` are omitted, the existing
|
||||
`/opds/*` and `/files/*` routes remain public; this is intentional so OPDS acquisition links
|
||||
continue to work. When Basic auth is configured, signed-in users can still download files with
|
||||
their session.
|
||||
|
||||
## 5. Restart, migrate, and verify public pages
|
||||
|
||||
```sh
|
||||
sudo systemctl restart daily-epub.service
|
||||
sudo systemctl --no-pager --full status daily-epub.service
|
||||
curl -fsS https://daily.hallada.net/ | grep -F 'The Daily EPUB'
|
||||
curl -fsS https://daily.hallada.net/feed.xml >/tmp/daily-epub-feed.xml
|
||||
python3 -c 'import xml.etree.ElementTree as E; E.parse("/tmp/daily-epub-feed.xml")'
|
||||
```
|
||||
|
||||
Migration `0004_web.sql` applies automatically at start. Anonymous issue pages and the Atom
|
||||
feed must show the source-link index without summaries, article bodies, comments, the Brief, or
|
||||
World Briefing text.
|
||||
|
||||
## 6. Bootstrap the administrator
|
||||
|
||||
```sh
|
||||
sudo -u daily-epub /usr/local/bin/daily-epub \
|
||||
--config /etc/daily-epub/config.toml users add tyler --admin
|
||||
sudo -u daily-epub /usr/local/bin/daily-epub \
|
||||
--config /etc/daily-epub/config.toml users list
|
||||
```
|
||||
|
||||
The first command prompts twice without echo. Log in at `https://daily.hallada.net/login`, open
|
||||
`/dashboard`, and confirm `/dashboard/users` shows the admin and an open session. A non-admin
|
||||
account should receive the site's 403 page for every dashboard route.
|
||||
|
||||
## 7. Smoke test a job
|
||||
|
||||
Start `features-prune` from `/dashboard/jobs`, open its job detail, and watch the status and log
|
||||
tail. On the host, confirm the same unit:
|
||||
|
||||
```sh
|
||||
systemctl status daily-epub-job@features-prune.service
|
||||
journalctl -u daily-epub-job@features-prune.service -n 100 --no-pager
|
||||
```
|
||||
|
||||
If the page reports a permission failure, inspect `journalctl -u polkit` and verify the installed
|
||||
rule and exact unit name. Do not broaden the rule to arbitrary units.
|
||||
|
||||
## 8. Smoke test an in-place settings write
|
||||
|
||||
In `/dashboard/settings`, change `curation.ranking.utility_protected` by one and save. Confirm
|
||||
the next view shows the new value, comments and ordering remain in the file, and the attributed
|
||||
change appears at `/dashboard/settings/history`:
|
||||
|
||||
```sh
|
||||
sudo -u daily-epub grep -n 'utility_protected' /etc/daily-epub/config.toml
|
||||
```
|
||||
|
||||
Change the value back through the page and confirm the second history row. Environment-overridden
|
||||
fields should be locked, and no API key, HMAC secret, or Basic-auth password should be displayed.
|
||||
|
||||
## 9. Choose the history window
|
||||
|
||||
Candidate and assessment history is pruned after
|
||||
`curation.ranking.telemetry_retention_days` (180 by default). Raise it now if the dashboard
|
||||
should retain a longer article/run history, then reload `/dashboard/settings` and verify the
|
||||
effective value. This changes future pruning only; it cannot restore rows already deleted.
|
||||
@@ -358,6 +358,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
|
||||
first_seen: "2026-08-15T05:30:00Z".parse().unwrap(),
|
||||
url: target.url.clone(),
|
||||
author: None,
|
||||
publication: None,
|
||||
feed_id: 1,
|
||||
feed_title: host(&target.url),
|
||||
category: None,
|
||||
@@ -371,6 +372,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
|
||||
is_lead: false,
|
||||
summary: None,
|
||||
llm: None,
|
||||
top_interests: Vec::new(),
|
||||
discussion: None,
|
||||
}
|
||||
}
|
||||
@@ -412,11 +414,11 @@ fn write_audit_epub(out_dir: &Path, picks: &[Pick], assets: &[ImageAsset]) {
|
||||
Articles are re-extracted live; editorial, discussions and the \
|
||||
world briefing are absent by design.</p>"
|
||||
.into(),
|
||||
section_intros: Default::default(),
|
||||
summaries: Default::default(),
|
||||
},
|
||||
world_briefing: None,
|
||||
colophon: Colophon::default(),
|
||||
behind: Default::default(),
|
||||
};
|
||||
let cfg = daily_epub::config::Config::default();
|
||||
match daily_epub::epub::build_edition_with_images(
|
||||
|
||||
@@ -0,0 +1,501 @@
|
||||
//! Seed a throwaway development database so the web site can be browsed
|
||||
//! locally with realistic data — two issues (one with the full `issue_json`
|
||||
//! snapshot, one "legacy" issue without it), their runs, an admin and a
|
||||
//! reader account. For design work and manual smoke tests only; never run it
|
||||
//! against the production database.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --example seed_dev_db -- ./dev
|
||||
//! DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) \
|
||||
//! cargo run -- --config ./dev/config.toml serve
|
||||
//! ```
|
||||
//!
|
||||
//! Then open http://127.0.0.1:3599 and sign in as `admin` / `adminpassword123`
|
||||
//! or `reader` / `readerpassword123`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use daily_epub::config::Config;
|
||||
use daily_epub::curate::telemetry::{self, CandidateRun};
|
||||
use daily_epub::db::Db;
|
||||
use daily_epub::epub::fixtures;
|
||||
use daily_epub::report::{ProviderUsage, RunReport};
|
||||
use daily_epub::types::{Article, Edition, Entry, Issue, Pick, TokenUsage};
|
||||
use daily_epub::web::users;
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
|
||||
const ADMIN_PASSWORD: &str = "adminpassword123";
|
||||
const READER_PASSWORD: &str = "readerpassword123";
|
||||
|
||||
/// (section, title, feed, summary, words)
|
||||
const STORIES: &[(&str, &str, &str, &str, i64)] = &[
|
||||
(
|
||||
"Top Stories",
|
||||
"How a Forty-Year-Old Filesystem Quietly Rewrote Its Write Path",
|
||||
"Systems Weekly",
|
||||
"A long, careful account of the redesign, with the benchmarks that justified it and the two regressions that nearly sank it.",
|
||||
3100,
|
||||
),
|
||||
(
|
||||
"Top Stories",
|
||||
"The Case Against Feature Flags",
|
||||
"Alice on Software",
|
||||
"Argues that flags outlive their purpose and proposes a retirement discipline, with examples from three codebases.",
|
||||
1850,
|
||||
),
|
||||
(
|
||||
"Top Stories",
|
||||
"What the Latest Battery Chemistry Actually Changes",
|
||||
"Ars Technica",
|
||||
"Separates the press-release claims from the measurable improvements in energy density and cycle life.",
|
||||
2200,
|
||||
),
|
||||
(
|
||||
"Deep Reads",
|
||||
"A Field Guide to Distributed Consensus, Told Through One Outage",
|
||||
"The Morning Paper",
|
||||
"Walks through a real incident to explain leader election, log replication and why the fix was a config change.",
|
||||
4200,
|
||||
),
|
||||
(
|
||||
"Deep Reads",
|
||||
"Why Every Map Is a Lie & How Cartographers Choose Which One to Tell",
|
||||
"Longreads",
|
||||
"A history of projections and the politics behind them, from Mercator to the maps in your phone.",
|
||||
5100,
|
||||
),
|
||||
(
|
||||
"Tools & Craft",
|
||||
"Notes on Writing a Rust Linter That People Actually Enable",
|
||||
"Rust Blog",
|
||||
"Design notes on false-positive budgets, fix suggestions and the social side of shipping a lint.",
|
||||
1600,
|
||||
),
|
||||
(
|
||||
"Tools & Craft",
|
||||
"The Terminal Is the Best UI We Have and It Is Getting Better",
|
||||
"Julia's Notebook",
|
||||
"A tour of modern terminal features (hyperlinks, images, synchronized output) and which tools use them.",
|
||||
1300,
|
||||
),
|
||||
(
|
||||
"Niche Corner",
|
||||
"A Small-Town Bakery's Sourdough Starter Turns One Hundred",
|
||||
"Saveur",
|
||||
"A charming profile of a starter kept alive across four generations and the bread it still makes.",
|
||||
900,
|
||||
),
|
||||
(
|
||||
"Niche Corner",
|
||||
"Inside the Community Keeping 1990s Synthesizers Alive",
|
||||
"Sound on Sound",
|
||||
"Repair collectives, replacement parts and the odd economics of vintage gear.",
|
||||
1700,
|
||||
),
|
||||
];
|
||||
|
||||
fn story_article(index: usize, story: &(&str, &str, &str, &str, i64)) -> Article {
|
||||
let id = index as i64 + 1;
|
||||
let entry_id = 1000 + id;
|
||||
let (_, title, feed, _, words) = *story;
|
||||
let mut article = fixtures::article(id, entry_id, title);
|
||||
article.feed_title = feed.to_string();
|
||||
article.feed_id = 7 + index as i64;
|
||||
article.word_count = words;
|
||||
for source in &mut article.sources {
|
||||
source.feed_title = feed.to_string();
|
||||
source.feed_id = article.feed_id;
|
||||
}
|
||||
article.content_html = body_html(title, words);
|
||||
if !index.is_multiple_of(3) {
|
||||
article.social.clear();
|
||||
}
|
||||
if index == 4 {
|
||||
article.author = None;
|
||||
}
|
||||
article
|
||||
}
|
||||
|
||||
fn body_html(title: &str, words: i64) -> String {
|
||||
let paragraphs = (words / 120).clamp(4, 40);
|
||||
let mut html = format!("<p>Body of <em>{title}</em>. ");
|
||||
html.push_str("This paragraph exists so the article page has something to lay out: a first sentence with a claim, a second with the evidence, and a third that hedges just enough to be honest.</p>");
|
||||
html.push_str("<h2>Where it gets interesting</h2>");
|
||||
for index in 0..paragraphs {
|
||||
if index == 2 {
|
||||
html.push_str("<img src=\"https://picsum.photos/seed/daily-epub/900/500\" alt=\"A chart of the daily figures\">");
|
||||
}
|
||||
if index == 5 {
|
||||
html.push_str("<blockquote><p>A pull quote that the author would rather you remembered than the rest.</p></blockquote>");
|
||||
}
|
||||
if index == 7 {
|
||||
// One block the source marked as Rust, one it left for detection.
|
||||
html.push_str(concat!(
|
||||
"<pre data-lang=\"rust\"><code>/// Greets whoever is reading.\n",
|
||||
"fn main() {\n",
|
||||
" let reader: &str = \"reader\";\n",
|
||||
" for n in 0..3 {\n",
|
||||
" println!(\"hello, {reader} #{n}\"); // three times\n",
|
||||
" }\n",
|
||||
"}</code></pre>",
|
||||
"<pre><code>import json\n\n",
|
||||
"def load(path: str) -> dict:\n",
|
||||
" \"\"\"Read a JSON config, or an empty one.\"\"\"\n",
|
||||
" try:\n",
|
||||
" with open(path) as f:\n",
|
||||
" return json.load(f)\n",
|
||||
" except FileNotFoundError:\n",
|
||||
" return {} # first run\n",
|
||||
"</code></pre>",
|
||||
));
|
||||
}
|
||||
html.push_str("<p>Paragraph ");
|
||||
html.push_str(&(index + 1).to_string());
|
||||
html.push_str(" keeps the argument moving. Long sentences alternate with short ones. Then a list of considerations that the author says nobody weighs properly, followed by the observation that everyone weighs them and simply disagrees about the weights.</p>");
|
||||
}
|
||||
html.push_str("<ul><li>First consideration</li><li>Second consideration</li><li>A third, longer consideration that wraps onto another line on narrow screens</li></ul>");
|
||||
html
|
||||
}
|
||||
|
||||
fn dev_issue(date: Date, issue_number: i64, generated_at: Timestamp) -> Issue {
|
||||
let mut issue = fixtures::issue();
|
||||
let deep = issue.lineup.picks[0]
|
||||
.llm
|
||||
.clone()
|
||||
.expect("the fixture lead carries a deep assessment");
|
||||
issue.meta.date = date;
|
||||
issue.meta.issue_number = issue_number;
|
||||
issue.meta.generated_at = generated_at;
|
||||
issue.meta.display_date = date.strftime("%A, %B %-d, %Y").to_string();
|
||||
issue.lineup.date = date;
|
||||
let mut section_order = Vec::new();
|
||||
let mut picks = Vec::new();
|
||||
let mut summaries = BTreeMap::new();
|
||||
for (index, story) in STORIES.iter().enumerate() {
|
||||
let (section, _, _, summary, _) = *story;
|
||||
if !section_order.iter().any(|s| s == section) {
|
||||
section_order.push(section.to_string());
|
||||
}
|
||||
let position = picks
|
||||
.iter()
|
||||
.filter(|pick: &&Pick| pick.section == section)
|
||||
.count() as i64;
|
||||
let article = story_article(index, story);
|
||||
summaries.insert(article.id, summary.to_string());
|
||||
// A few picks carry facets and matched interests so the
|
||||
// "what the pipeline understood" line shows up; the rest exercise the
|
||||
// nothing-to-say path.
|
||||
let understood = match index {
|
||||
0 => Some((
|
||||
("software_engineering", "analysis_essay", "deep", "advanced"),
|
||||
vec!["copy-on-write", "ZFS", "filesystem design"],
|
||||
vec!["Filesystems", "Rust", "Systems programming"],
|
||||
)),
|
||||
1 => Some((
|
||||
(
|
||||
"software_engineering",
|
||||
"analysis_essay",
|
||||
"standard",
|
||||
"intermediate",
|
||||
),
|
||||
vec!["feature flags", "technical debt"],
|
||||
vec!["Software craft"],
|
||||
)),
|
||||
3 => Some((
|
||||
(
|
||||
"software_engineering",
|
||||
"first_hand_account",
|
||||
"deep",
|
||||
"advanced",
|
||||
),
|
||||
vec!["Raft", "leader election", "incident review"],
|
||||
vec![],
|
||||
)),
|
||||
4 => Some((
|
||||
("history", "analysis_essay", "deep", "nontechnical"),
|
||||
vec!["map projections", "cartography"],
|
||||
vec!["Maps", "History of science"],
|
||||
)),
|
||||
_ => None,
|
||||
};
|
||||
let (llm, top_interests) = match understood {
|
||||
Some(((topic_group, format, depth, technicality), topics, interests)) => (
|
||||
Some(daily_epub::types::Deep {
|
||||
facets: daily_epub::types::Facets {
|
||||
format: Some(format.into()),
|
||||
depth: Some(depth.into()),
|
||||
topic_group: Some(topic_group.into()),
|
||||
technicality: Some(technicality.into()),
|
||||
specific_topics: Some(topics.into_iter().map(String::from).collect()),
|
||||
..Default::default()
|
||||
},
|
||||
..deep.clone()
|
||||
}),
|
||||
interests.into_iter().map(String::from).collect(),
|
||||
),
|
||||
None => (None, Vec::new()),
|
||||
};
|
||||
picks.push(Pick {
|
||||
discussion: (index == 0)
|
||||
.then(|| fixtures::discussion(article.id, article.best_entry_id)),
|
||||
article,
|
||||
section: section.to_string(),
|
||||
position,
|
||||
is_lead: index == 0,
|
||||
summary: Some(summary.to_string()),
|
||||
llm,
|
||||
top_interests,
|
||||
});
|
||||
}
|
||||
issue.meta.article_count = picks.len() as i64;
|
||||
issue.meta.section_count = section_order.len() as i64;
|
||||
issue.meta.total_words = picks.iter().map(|pick| pick.article.word_count).sum();
|
||||
issue.meta.reading_minutes = daily_epub::types::reading_minutes(issue.meta.total_words);
|
||||
issue.lineup.picks = picks;
|
||||
issue.lineup.section_order = section_order;
|
||||
issue.editorial.summaries = summaries;
|
||||
issue.editorial.front_page_html = "<p>Nine stories today. The filesystem rewrite leads because it is the rare systems piece that shows its benchmarks and its mistakes; the consensus outage walkthrough is the long read to save for the train.</p><p>Two contrarian pieces on tooling sit in the middle, and the back pages are for the weekend: a hundred-year-old sourdough starter and the people who keep old synthesizers humming.</p>".into();
|
||||
if let Some(world) = issue.world_briefing.as_mut() {
|
||||
world.date = date;
|
||||
}
|
||||
issue.behind.selected = issue.meta.article_count;
|
||||
issue
|
||||
}
|
||||
|
||||
async fn seed_issue(
|
||||
db: &Db,
|
||||
issue: &mut Issue,
|
||||
with_snapshot: bool,
|
||||
epub_path: Option<&Path>,
|
||||
) -> anyhow::Result<i64> {
|
||||
for pick in &mut issue.lineup.picks {
|
||||
let article = &pick.article;
|
||||
db.upsert_entry(&Entry {
|
||||
id: article.best_entry_id,
|
||||
feed_id: article.feed_id,
|
||||
feed_title: Some(article.feed_title.clone()),
|
||||
category: article.category.clone(),
|
||||
title: article.title.clone(),
|
||||
url: article.url.clone(),
|
||||
canonical_url: Some(article.canonical_url.clone()),
|
||||
author: article.author.clone(),
|
||||
published_at: article.published_at,
|
||||
comments_url: article.comments_url.clone(),
|
||||
raw_content: article.content_html.clone(),
|
||||
fetched_at: article.first_seen,
|
||||
})
|
||||
.await?;
|
||||
let id = db.upsert_article(article).await?;
|
||||
pick.article.id = id;
|
||||
for social in &mut pick.article.social {
|
||||
social.article_id = id;
|
||||
db.upsert_social(social).await?;
|
||||
}
|
||||
}
|
||||
let snapshot = with_snapshot
|
||||
.then(|| {
|
||||
let mut snapshot = issue.clone();
|
||||
for pick in &mut snapshot.lineup.picks {
|
||||
pick.article.content_html.clear();
|
||||
}
|
||||
serde_json::to_string(&snapshot)
|
||||
})
|
||||
.transpose()?;
|
||||
let started_at = issue
|
||||
.meta
|
||||
.generated_at
|
||||
.checked_sub(jiff::Span::new().minutes(22))
|
||||
.unwrap_or(issue.meta.generated_at);
|
||||
let run_id = db.start_run(issue.meta.date, started_at).await?;
|
||||
let mut report = RunReport::new(issue.meta.date, started_at);
|
||||
report.counts.entries_fetched = issue.colophon.entries_fetched;
|
||||
report.counts.feeds_seen = issue.colophon.feeds_seen;
|
||||
report.counts.articles = issue.behind.considered;
|
||||
report.counts.eligible = issue.behind.eligible;
|
||||
report.counts.triaged = issue.behind.triaged;
|
||||
report.counts.assessed = issue.behind.read_closely;
|
||||
report.counts.shortlisted = issue.behind.shortlisted;
|
||||
report.counts.candidates = issue.colophon.candidates;
|
||||
report.counts.selected = issue.meta.article_count;
|
||||
report.counts.admitted_by = issue.behind.admitted_by.clone();
|
||||
report.counts.rated_with_embeddings = issue.behind.rated_with_embeddings;
|
||||
report.counts.knn_gate = issue.behind.knn_gate;
|
||||
report.counts.feed_gate = issue.behind.feed_gate;
|
||||
report.counts.embedded = 300;
|
||||
report.provider_costs = issue
|
||||
.colophon
|
||||
.provider_costs
|
||||
.iter()
|
||||
.map(|(provider, cost_usd)| {
|
||||
(
|
||||
provider.clone(),
|
||||
ProviderUsage {
|
||||
usage: TokenUsage::default(),
|
||||
cost_usd: *cost_usd,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
report.config_json = serde_json::json!({
|
||||
"models": {
|
||||
"bulk": issue.colophon.models.bulk,
|
||||
"editor": issue.colophon.models.editor,
|
||||
"embedding": issue.behind.embedding_model,
|
||||
},
|
||||
"editorial": {"summary_model": "editor"},
|
||||
"voyage": {"model": issue.behind.embedding_model},
|
||||
});
|
||||
report.finish(issue.meta.generated_at);
|
||||
db.finish_run(run_id, &report).await?;
|
||||
let epub_path = epub_path.map(|path| path.to_string_lossy().into_owned());
|
||||
db.upsert_issue(
|
||||
issue.meta.date,
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
epub_path.as_deref(),
|
||||
None,
|
||||
None,
|
||||
Some(&issue.editorial.front_page_html),
|
||||
Some(&serde_json::to_string(&report)?),
|
||||
snapshot.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
db.replace_issue_articles(issue.meta.date, &issue.lineup.picks)
|
||||
.await?;
|
||||
Ok(run_id)
|
||||
}
|
||||
|
||||
async fn seed_near_misses(db: &Db, run_id: i64) -> anyhow::Result<()> {
|
||||
for (index, title, quality, fit, utility) in [
|
||||
(
|
||||
0,
|
||||
"The Database Migration That Almost Made the Cut",
|
||||
8.4,
|
||||
7.2,
|
||||
81.5,
|
||||
),
|
||||
(
|
||||
1,
|
||||
"An Oral History of the First E-Ink Hackers",
|
||||
7.7,
|
||||
8.1,
|
||||
79.8,
|
||||
),
|
||||
(2, "A Very Good Essay About Municipal Trees", 8.0, 6.8, 76.2),
|
||||
] {
|
||||
let mut article = fixtures::article(0, 9001 + index, title);
|
||||
article.feed_title = "The Near-Miss Review".into();
|
||||
let entry = Entry {
|
||||
id: article.best_entry_id,
|
||||
feed_id: article.feed_id,
|
||||
feed_title: Some(article.feed_title.clone()),
|
||||
category: article.category.clone(),
|
||||
title: article.title.clone(),
|
||||
url: article.url.clone(),
|
||||
canonical_url: Some(article.canonical_url.clone()),
|
||||
author: article.author.clone(),
|
||||
published_at: article.published_at,
|
||||
comments_url: article.comments_url.clone(),
|
||||
raw_content: article.content_html.clone(),
|
||||
fetched_at: article.first_seen,
|
||||
};
|
||||
db.upsert_entry(&entry).await?;
|
||||
let article_id = db.upsert_article(&article).await?;
|
||||
let signals = serde_json::json!({
|
||||
"v": 1,
|
||||
"raw": {"quality": quality, "fit": fit},
|
||||
"norm": {"quality": quality / 10.0, "fit": fit / 10.0},
|
||||
"present": {"quality": true, "fit": true},
|
||||
"weights": {"quality": 0.6, "fit": 0.4},
|
||||
})
|
||||
.to_string();
|
||||
telemetry::write(
|
||||
db,
|
||||
&CandidateRun {
|
||||
run_id,
|
||||
article_id,
|
||||
stage: "shortlisted",
|
||||
excluded_reason: Some("not_selected"),
|
||||
admitted_by: Some("[\"blend\"]"),
|
||||
signals_json: &signals,
|
||||
utility: Some(utility),
|
||||
rank_utility: Some(index + 10),
|
||||
cluster_id: Some(index),
|
||||
cluster_rank: Some(1),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_config(dir: &Path) -> anyhow::Result<PathBuf> {
|
||||
let path = dir.join("config.toml");
|
||||
let dir = dir.canonicalize()?;
|
||||
// The profile lives inside the throwaway directory: saving it from
|
||||
// /dashboard/profile must never rewrite the repository's data/profile.md.
|
||||
let profile = dir.join("profile.md");
|
||||
let toml = format!(
|
||||
"database_path = \"{db}\"\nprofile_path = \"{profile}\"\n\n[publish]\nepub_dir = \"{epubs}\"\nxtc_dir = \"{xtc}\"\n\n[server]\nbind = \"127.0.0.1:3599\"\npublic_url = \"http://127.0.0.1:3599\"\njobs_enabled = false\n",
|
||||
db = dir.join("daily-epub.db").display(),
|
||||
profile = profile.display(),
|
||||
epubs = dir.join("epubs").display(),
|
||||
xtc = dir.join("xtc").display(),
|
||||
);
|
||||
std::fs::write(&path, toml)?;
|
||||
if !profile.exists() {
|
||||
let shipped = Path::new("data/profile.md");
|
||||
let body = std::fs::read_to_string(shipped)
|
||||
.unwrap_or_else(|_| "# Reader profile\n\n## Who he is\n\nA developer.\n".to_string());
|
||||
std::fs::write(&profile, body)?;
|
||||
}
|
||||
std::fs::create_dir_all(dir.join("epubs"))?;
|
||||
std::fs::create_dir_all(dir.join("xtc"))?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let dir = std::env::args()
|
||||
.nth(1)
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("dev"));
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
let db_path = dir.join("daily-epub.db");
|
||||
if db_path.exists() {
|
||||
anyhow::bail!("{} already exists; delete it first", db_path.display());
|
||||
}
|
||||
let config_path = write_config(&dir)?;
|
||||
let db = Db::open_and_migrate(&db_path).await?;
|
||||
|
||||
users::add(&db, "admin", ADMIN_PASSWORD, true).await?;
|
||||
users::add(&db, "reader", READER_PASSWORD, false).await?;
|
||||
|
||||
// Legacy issue: published before `issue_json` existed, so the site must
|
||||
// rebuild the colophon and back matter from the run instead.
|
||||
let mut legacy = dev_issue("2026-09-01".parse()?, 18, "2026-09-01T05:31:00Z".parse()?);
|
||||
let mut epub_config = Config::default();
|
||||
epub_config.publish.epub_dir = dir.join("epubs");
|
||||
let legacy_epub = daily_epub::epub::build_edition_with_images(
|
||||
&legacy,
|
||||
Edition::Standard,
|
||||
&epub_config,
|
||||
&epub_config.publish.epub_dir,
|
||||
&[],
|
||||
)?;
|
||||
let legacy_run = seed_issue(&db, &mut legacy, false, Some(&legacy_epub.path)).await?;
|
||||
seed_near_misses(&db, legacy_run).await?;
|
||||
|
||||
let mut latest = dev_issue("2026-09-02".parse()?, 19, "2026-09-02T05:29:00Z".parse()?);
|
||||
seed_issue(&db, &mut latest, true, None).await?;
|
||||
|
||||
println!("seeded {}", db_path.display());
|
||||
println!(
|
||||
"run: DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) cargo run -- --config {} serve",
|
||||
config_path.display()
|
||||
);
|
||||
println!("open: http://127.0.0.1:3599 (admin / {ADMIN_PASSWORD}, reader / {READER_PASSWORD})");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//! Preview an article the way the Xteink X4 will show it.
|
||||
//!
|
||||
//! Browsers are a poor stand-in for the X4: its pages are rendered ahead of
|
||||
//! time by the XTC converter's CREngine, which has its own defaults (`code` is a
|
||||
//! block, `pre` has no margin) and only four grey levels. This builds both
|
||||
//! EPUB editions around one article, runs the real converter on the X4 one and
|
||||
//! decodes every page to `page-NNN.png`.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --example x4_preview -- <article URL | HTML file> <out dir>
|
||||
//! cargo run --example x4_preview -- --convert <x4.epub> <out dir>
|
||||
//! ```
|
||||
//!
|
||||
//! A URL goes through the real extraction path; an HTML file is used as the
|
||||
//! stored body as-is. `--convert` skips the build, for iterating on a
|
||||
//! stylesheet inside an unzipped EPUB. Needs the converter at
|
||||
//! `/opt/epub-to-xtc-converter` and runs it with `xtc-settings.example.json`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use daily_epub::config::Config;
|
||||
use daily_epub::epub;
|
||||
use daily_epub::extract::Extractor;
|
||||
use daily_epub::types::Edition;
|
||||
|
||||
const CONVERTER: &str = "/opt/epub-to-xtc-converter/cli/index.js";
|
||||
const SETTINGS: &str = "xtc-settings.example.json";
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.as_slice() {
|
||||
[flag, epub, out] if flag == "--convert" => {
|
||||
let pages = convert(Path::new(epub), Path::new(out))?;
|
||||
println!("{pages} pages in {out}");
|
||||
}
|
||||
[source, out] => {
|
||||
let out = PathBuf::from(out);
|
||||
std::fs::create_dir_all(&out)?;
|
||||
let html = if source.starts_with("http") {
|
||||
let extractor = Extractor::new(reqwest::Client::new(), vec![]);
|
||||
let page = extractor.fetch_readable(source).await?;
|
||||
extractor.finish_readable(source, &page).content_html
|
||||
} else {
|
||||
std::fs::read_to_string(source)?
|
||||
};
|
||||
std::fs::write(out.join("stored.html"), &html)?;
|
||||
|
||||
let mut issue = epub::fixtures::issue();
|
||||
issue.lineup.picks.truncate(1);
|
||||
issue.lineup.picks[0].article.content_html = html;
|
||||
let config = Config::default();
|
||||
for edition in [Edition::Standard, Edition::X4] {
|
||||
let artifact =
|
||||
epub::build_edition_with_images(&issue, edition, &config, &out, &[])?;
|
||||
println!("{edition:?}: {}", artifact.path.display());
|
||||
if edition == Edition::X4 {
|
||||
let pages = convert(&artifact.path, &out)?;
|
||||
println!("{pages} X4 pages in {}", out.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!(
|
||||
"usage: x4_preview <URL | HTML file> <out dir> | --convert <epub> <out dir>"
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the converter on `epub` and decode its pages into `out`.
|
||||
fn convert(epub: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
std::fs::create_dir_all(out)?;
|
||||
let xtch = out.join("x4.xtch");
|
||||
let status = Command::new("node")
|
||||
.args([CONVERTER, "convert"])
|
||||
.arg(epub)
|
||||
.arg("-o")
|
||||
.arg(&xtch)
|
||||
.args(["-f", "xtch", "-c", SETTINGS])
|
||||
.stdout(Stdio::null())
|
||||
.status()?;
|
||||
anyhow::ensure!(status.success(), "converter failed: {status}");
|
||||
decode_xtch(&xtch, out)
|
||||
}
|
||||
|
||||
fn u16_at(bytes: &[u8], at: usize) -> usize {
|
||||
u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize
|
||||
}
|
||||
|
||||
fn u64_at(bytes: &[u8], at: usize) -> usize {
|
||||
u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize
|
||||
}
|
||||
|
||||
/// Decode every page of an XTCH container to a PNG.
|
||||
///
|
||||
/// Layout per the converter's `docs/xtc-format-spec.md`: the page index is at
|
||||
/// the offset in header bytes 0x18..0x20, 16 bytes per page (offset, size,
|
||||
/// width, height). Each page is a 22-byte header and two bit planes stored
|
||||
/// column by column, right to left, 8 vertical pixels per byte. The plane
|
||||
/// order follows the encoder (`cli/encoder.js`), not the spec's table: the
|
||||
/// first plane holds the level's low bit, and 0b01 is dark grey.
|
||||
fn decode_xtch(path: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
let bytes = std::fs::read(path)?;
|
||||
let pages = u16_at(&bytes, 0x06);
|
||||
let index = u64_at(&bytes, 0x18);
|
||||
for page in 0..pages {
|
||||
let entry = index + page * 16;
|
||||
let (offset, width, height) = (
|
||||
u64_at(&bytes, entry),
|
||||
u16_at(&bytes, entry + 12),
|
||||
u16_at(&bytes, entry + 14),
|
||||
);
|
||||
let plane = (width * height).div_ceil(8);
|
||||
let column = height.div_ceil(8);
|
||||
let data = &bytes[offset + 22..];
|
||||
let mut image = image::GrayImage::new(width as u32, height as u32);
|
||||
for x in 0..width {
|
||||
for y in 0..height {
|
||||
let byte = (width - 1 - x) * column + y / 8;
|
||||
let bit = 7 - (y % 8);
|
||||
let low = (data[byte] >> bit) & 1;
|
||||
let high = (data[plane + byte] >> bit) & 1;
|
||||
let shade = [255, 85, 170, 0][usize::from(high << 1 | low)];
|
||||
image.put_pixel(x as u32, y as u32, image::Luma([shade]));
|
||||
}
|
||||
}
|
||||
image.save(out.join(format!("page-{page:03}.png")))?;
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
# Operator and developer shortcuts for The Daily EPUB.
|
||||
#
|
||||
# just list recipes
|
||||
# just de <args> run any daily-epub subcommand on the server as the service user
|
||||
#
|
||||
# Server recipes run the binary the way the systemd units do: same user, working
|
||||
# directory and EnvironmentFile (`/etc/daily-epub/env`, where the API keys live),
|
||||
# via a transient `systemd-run` unit. Plain `sudo -u daily-epub` never loads that
|
||||
# file, so `miniflux api key is not configured` from a hand-run command means the
|
||||
# environment, not the config.
|
||||
|
||||
set positional-arguments
|
||||
|
||||
bin := "/usr/local/bin/daily-epub"
|
||||
config := "/etc/daily-epub/config.toml"
|
||||
env_file := "/etc/daily-epub/env"
|
||||
work_dir := "/var/lib/daily-epub"
|
||||
user := "daily-epub"
|
||||
|
||||
# The systemd-run prefix every server recipe uses.
|
||||
run := "sudo systemd-run --quiet --wait --pty --collect --uid=" + user + " --gid=" + user \
|
||||
+ " -p WorkingDirectory=" + work_dir + " -p EnvironmentFile=" + env_file \
|
||||
+ " " + bin + " --config " + config
|
||||
|
||||
default:
|
||||
@just --list --unsorted
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: any subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Run any daily-epub subcommand as the service user with the env file loaded.
|
||||
de *args:
|
||||
{{run}} "$@"
|
||||
|
||||
# Validate the config as `generate` would; one fact per line.
|
||||
config-check:
|
||||
{{run}} config check
|
||||
|
||||
# Run pending migrations (also automatic on every start).
|
||||
db-migrate:
|
||||
{{run}} db migrate
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: issues
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Build and publish an issue by hand (same as the timer, but in the foreground).
|
||||
generate *args:
|
||||
{{run}} generate "$@"
|
||||
|
||||
# Build everything, publish nothing, record no issue.
|
||||
dry-run *args:
|
||||
{{run}} generate --dry-run "$@"
|
||||
|
||||
# Why an article was (or was not) in the paper, from persisted telemetry.
|
||||
explain *args:
|
||||
{{run}} explain "$@"
|
||||
|
||||
# The weekly numbers: issues, ratings, retriever yield, cost, timing.
|
||||
stats *args:
|
||||
{{run}} stats "$@"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: feeds, ratings, profile, features
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Seed /dashboard/feeds from recent aggregator-only articles.
|
||||
feeds-discover days="14" limit="50":
|
||||
{{run}} feeds discover --days {{days}} --limit {{limit}}
|
||||
|
||||
# List current explicit verdicts.
|
||||
ratings-list *args:
|
||||
{{run}} ratings list "$@"
|
||||
|
||||
# Set a verdict: `just ratings-set <article-id> <loved|good|not_for_me> [--note ...]`.
|
||||
ratings-set *args:
|
||||
{{run}} ratings set "$@"
|
||||
|
||||
# Clear a verdict: `just ratings-clear <article-id>`.
|
||||
ratings-clear *args:
|
||||
{{run}} ratings clear "$@"
|
||||
|
||||
# Regenerate the taste profile from ratings history.
|
||||
profile-rebuild:
|
||||
{{run}} profile rebuild
|
||||
|
||||
# Embed rated and published articles, then interests, into the cache.
|
||||
features-backfill *args:
|
||||
{{run}} features backfill "$@"
|
||||
|
||||
# Drop stale embeddings, old telemetry and old assessments per the retention config.
|
||||
features-prune:
|
||||
{{run}} features prune
|
||||
|
||||
# Re-poll social scores for recent entries.
|
||||
backfill-social days="7":
|
||||
{{run}} backfill-social --days {{days}}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: users and jobs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# List dashboard users and open-session counts.
|
||||
users-list:
|
||||
{{run}} users list
|
||||
|
||||
# Add a user: `just users-add NAME [--admin]` (prompts for the password).
|
||||
users-add *args:
|
||||
{{run}} users add "$@"
|
||||
|
||||
# Change a password and revoke sessions: `just users-passwd NAME`.
|
||||
users-passwd *args:
|
||||
{{run}} users passwd "$@"
|
||||
|
||||
# Any other `users` subcommand: `just users role NAME admin`, `just users disable NAME`, ...
|
||||
users *args:
|
||||
{{run}} users "$@"
|
||||
|
||||
# Run one catalogue job in-process (generate, dry-run, profile-rebuild, features-backfill, backfill-social, features-prune, import-ratings, interests-categorize).
|
||||
job name:
|
||||
{{run}} job run {{name}}
|
||||
|
||||
# Start a catalogue job as its systemd unit, the way the dashboard does.
|
||||
job-unit name:
|
||||
sudo systemctl start daily-epub-job@{{name}}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server: services and logs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Status of the web server, the generate timer and its last run.
|
||||
status:
|
||||
systemctl status --no-pager daily-epub.service daily-epub-generate.timer daily-epub-generate.service || true
|
||||
systemctl list-timers --no-pager daily-epub-generate.timer
|
||||
|
||||
# Restart the web server (after installing a new binary or unit).
|
||||
restart:
|
||||
sudo systemctl restart daily-epub.service
|
||||
|
||||
# Follow the web server log.
|
||||
logs:
|
||||
journalctl -u daily-epub.service -f
|
||||
|
||||
# Last N lines of the most recent generate run.
|
||||
logs-generate lines="120":
|
||||
journalctl -u daily-epub-generate.service -n {{lines}} --no-pager
|
||||
|
||||
# Follow one job unit's log: `just logs-job features-prune`.
|
||||
logs-job name:
|
||||
journalctl -u daily-epub-job@{{name}} -f
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deploy (run from a checkout on the server)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Release build, install the binary and restart the web server.
|
||||
deploy: build-release
|
||||
sudo install -m0755 target/release/daily-epub {{bin}}
|
||||
sudo systemctl restart daily-epub.service
|
||||
systemctl is-active daily-epub.service
|
||||
|
||||
# Install the systemd units and the polkit rule, then reload systemd.
|
||||
install-units:
|
||||
sudo install -m0644 systemd/daily-epub.service systemd/daily-epub-generate.service \
|
||||
systemd/daily-epub-generate.timer systemd/daily-epub-job@.service /etc/systemd/system/
|
||||
sudo install -m0644 systemd/50-daily-epub.rules /etc/polkit-1/rules.d/
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
# Edit the environment file (API keys and other secrets).
|
||||
edit-env:
|
||||
sudo -e {{env_file}}
|
||||
|
||||
# Edit the config file as the service user would see it.
|
||||
edit-config:
|
||||
sudo -e {{config}}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Development (local checkout)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
build:
|
||||
cargo build
|
||||
|
||||
build-release:
|
||||
cargo build --release
|
||||
|
||||
# Everything CI expects: format check, clippy, tests, and the committed CSS is current.
|
||||
check: fmt-check clippy test css-check
|
||||
|
||||
test *args:
|
||||
cargo test "$@"
|
||||
|
||||
clippy:
|
||||
cargo clippy --all-targets
|
||||
|
||||
fmt:
|
||||
cargo fmt
|
||||
|
||||
fmt-check:
|
||||
cargo fmt --check
|
||||
|
||||
# Rebuild src/web/static/app.css from the Tailwind source (commit the result).
|
||||
css:
|
||||
npm run css
|
||||
|
||||
# Fail if the committed app.css is out of date.
|
||||
css-check:
|
||||
npm run css:check
|
||||
|
||||
# Rebuild app.css on every template change.
|
||||
css-watch:
|
||||
npm run css:watch
|
||||
|
||||
# Seed a throwaway dev database under ./dev (admin/adminpassword123, reader/readerpassword123).
|
||||
dev-seed:
|
||||
cargo run --example seed_dev_db -- ./dev
|
||||
|
||||
# Serve the dev database on http://127.0.0.1:3599.
|
||||
dev-serve:
|
||||
DAILY_EPUB_SERVER__HMAC_SECRET=$(head -c 48 /dev/urandom | base64) \
|
||||
cargo run -- --config ./dev/config.toml serve
|
||||
|
||||
# Run any subcommand against the local dev config: `just dev config check`.
|
||||
dev *args:
|
||||
cargo run -- --config ./dev/config.toml "$@"
|
||||
|
||||
# Replay extraction + images over published EPUBs: `just image-audit ~/books/*.epub`.
|
||||
image-audit *args:
|
||||
cargo run --release --example image_audit -- --cache /tmp/pagecache "$@"
|
||||
@@ -0,0 +1,96 @@
|
||||
-- Personalized curation v2 schema (plan 2026-09-02 §7).
|
||||
-- `scores` intentionally remains until migration 0003 (step 4).
|
||||
|
||||
CREATE TABLE rating_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
issue_date TEXT,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('explicit', 'implicit')),
|
||||
source TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
value REAL NOT NULL,
|
||||
note TEXT,
|
||||
event_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_rating_events_article ON rating_events(article_id, event_at);
|
||||
CREATE INDEX idx_rating_events_at ON rating_events(event_at);
|
||||
|
||||
-- Filled in by step 3.
|
||||
CREATE TABLE article_embeddings (
|
||||
article_id INTEGER PRIMARY KEY REFERENCES articles(id) ON DELETE CASCADE,
|
||||
model TEXT NOT NULL,
|
||||
dimension INTEGER NOT NULL,
|
||||
input_hash TEXT NOT NULL,
|
||||
embedding BLOB NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Filled in by step 3.
|
||||
CREATE TABLE interest_embeddings (
|
||||
interest TEXT PRIMARY KEY,
|
||||
model TEXT NOT NULL,
|
||||
dimension INTEGER NOT NULL,
|
||||
embedding BLOB NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Triage rows are filled in by step 4; deep rows by step 5.
|
||||
CREATE TABLE article_assessments (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL CHECK (stage IN ('triage', 'deep')),
|
||||
model TEXT NOT NULL,
|
||||
prompt_version INTEGER NOT NULL,
|
||||
profile_version INTEGER,
|
||||
score REAL,
|
||||
fit REAL,
|
||||
kind TEXT,
|
||||
facets_json TEXT,
|
||||
rationale TEXT,
|
||||
category TEXT,
|
||||
paywalled_guess INTEGER NOT NULL DEFAULT 0,
|
||||
assessed_at TEXT NOT NULL,
|
||||
PRIMARY KEY (article_id, stage)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_article_assessments_at ON article_assessments(assessed_at);
|
||||
|
||||
-- Filled in by step 3.
|
||||
CREATE TABLE candidate_runs (
|
||||
run_id INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL,
|
||||
excluded_reason TEXT,
|
||||
admitted_by TEXT,
|
||||
signals_json TEXT NOT NULL,
|
||||
utility REAL,
|
||||
rank_utility INTEGER,
|
||||
cluster_id INTEGER,
|
||||
cluster_rank INTEGER,
|
||||
editor_why TEXT,
|
||||
PRIMARY KEY (run_id, article_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_candidate_runs_article ON candidate_runs(article_id);
|
||||
CREATE INDEX idx_candidate_runs_run_stage ON candidate_runs(run_id, stage);
|
||||
|
||||
-- Filled in by later telemetry/provider steps.
|
||||
ALTER TABLE runs ADD COLUMN config_json TEXT;
|
||||
ALTER TABLE runs ADD COLUMN provider_costs_json TEXT;
|
||||
-- Filled in by step 2.
|
||||
ALTER TABLE issue_articles ADD COLUMN why TEXT;
|
||||
|
||||
INSERT INTO rating_events
|
||||
(article_id, issue_date, kind, source, label, value, note, event_at)
|
||||
SELECT article_id,
|
||||
issue_date,
|
||||
'explicit',
|
||||
'migration',
|
||||
CASE vote WHEN 1 THEN 'loved' ELSE 'not_for_me' END,
|
||||
CASE vote WHEN 1 THEN 1.0 ELSE -1.0 END,
|
||||
NULL,
|
||||
rated_at
|
||||
FROM ratings;
|
||||
|
||||
DROP TABLE ratings;
|
||||
DROP TABLE feed_priors;
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE scores;
|
||||
@@ -0,0 +1,58 @@
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE COLLATE NOCASE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('user', 'admin')),
|
||||
disabled INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
last_login_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
expiry INTEGER NOT NULL,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_sessions_user ON sessions(user_id);
|
||||
CREATE INDEX idx_sessions_expiry ON sessions(expiry);
|
||||
|
||||
ALTER TABLE rating_events ADD COLUMN user_id INTEGER REFERENCES users(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE TABLE config_changes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
key TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_config_changes_at ON config_changes(changed_at);
|
||||
|
||||
CREATE TABLE profile_versions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
content TEXT NOT NULL,
|
||||
saved_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
saved_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
unit TEXT NOT NULL,
|
||||
requested_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
requested_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('requested', 'running', 'ok', 'failed')),
|
||||
message TEXT,
|
||||
run_id INTEGER REFERENCES runs(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX idx_jobs_requested_at ON jobs(requested_at);
|
||||
|
||||
ALTER TABLE runs ADD COLUMN report_json TEXT;
|
||||
ALTER TABLE issues ADD COLUMN issue_json TEXT;
|
||||
|
||||
CREATE INDEX idx_candidate_runs_article_run ON candidate_runs(article_id, run_id DESC);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE issues ADD COLUMN bookorbit_book_id INTEGER;
|
||||
ALTER TABLE issues ADD COLUMN bookorbit_file_id INTEGER;
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE rating_imports (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT NOT NULL,
|
||||
label TEXT NOT NULL CHECK (label IN ('loved', 'good', 'not_for_me')),
|
||||
note TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('pending', 'ok', 'failed')),
|
||||
message TEXT,
|
||||
article_id INTEGER REFERENCES articles(id) ON DELETE SET NULL,
|
||||
requested_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
requested_at TEXT NOT NULL,
|
||||
finished_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_rating_imports_status_id ON rating_imports(status, id);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE account_requests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL COLLATE NOCASE,
|
||||
reason TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('open', 'done')) DEFAULT 'open',
|
||||
requested_at TEXT NOT NULL,
|
||||
handled_at TEXT,
|
||||
handled_by INTEGER REFERENCES users(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_account_requests_open_email
|
||||
ON account_requests(email) WHERE status = 'open';
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE users ADD COLUMN must_change_password INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE account_requests ADD COLUMN username TEXT;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Feed discovery: proposed Miniflux subscriptions found behind aggregator-only
|
||||
-- articles (feed discovery plan §4 step 1).
|
||||
|
||||
CREATE TABLE feed_candidates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
host TEXT NOT NULL,
|
||||
title TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('candidate', 'added', 'dismissed')),
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
miniflux_feed_id INTEGER,
|
||||
decided_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_feed_candidates_status_host ON feed_candidates(status, host);
|
||||
|
||||
CREATE TABLE feed_candidate_articles (
|
||||
candidate_id INTEGER NOT NULL REFERENCES feed_candidates(id) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (candidate_id, article_id)
|
||||
);
|
||||
|
||||
CREATE TABLE feed_discovery_hosts (
|
||||
host TEXT PRIMARY KEY,
|
||||
checked_at TEXT NOT NULL,
|
||||
candidates INTEGER NOT NULL
|
||||
);
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Persist page-extracted authors independently of the best Miniflux entry.
|
||||
|
||||
ALTER TABLE articles ADD COLUMN author TEXT;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Persist page-extracted publication names independently of the best Miniflux entry.
|
||||
|
||||
ALTER TABLE articles ADD COLUMN publication TEXT;
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE interests (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
category TEXT, -- NULL until categorized
|
||||
created_at TEXT NOT NULL,
|
||||
categorized_at TEXT
|
||||
);
|
||||
CREATE INDEX idx_interests_category ON interests(category);
|
||||
|
||||
CREATE TABLE article_interests (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE,
|
||||
cos REAL NOT NULL,
|
||||
z REAL NOT NULL,
|
||||
run_id INTEGER, -- NULL for backfilled rows
|
||||
PRIMARY KEY (article_id, interest_id)
|
||||
);
|
||||
CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Related articles under each web issue article: the nearest embedded articles
|
||||
-- across the whole archive, computed when the issue is published (and by
|
||||
-- `features related` for older issues). Stored rather than computed per page
|
||||
-- view because finding them is a scan of every cached embedding.
|
||||
CREATE TABLE issue_related (
|
||||
issue_date TEXT NOT NULL REFERENCES issues(date) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL, -- 0 = closest
|
||||
related_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
cosine REAL NOT NULL,
|
||||
PRIMARY KEY (issue_date, article_id, position)
|
||||
);
|
||||
Generated
+313
@@ -0,0 +1,313 @@
|
||||
{
|
||||
"name": "the-daily-epub-v2b",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"devDependencies": {
|
||||
"@tailwindcss/cli": "4.3.3",
|
||||
"tailwindcss": "4.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/gen-mapping": {
|
||||
"version": "0.3.13",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/sourcemap-codec": "^1.5.0",
|
||||
"@jridgewell/trace-mapping": "^0.3.24"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/remapping": {
|
||||
"version": "2.3.5",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/gen-mapping": "^0.3.5",
|
||||
"@jridgewell/trace-mapping": "^0.3.24"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/resolve-uri": {
|
||||
"version": "3.1.2",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@jridgewell/sourcemap-codec": {
|
||||
"version": "1.5.5",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@jridgewell/trace-mapping": {
|
||||
"version": "0.3.31",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/resolve-uri": "^3.1.0",
|
||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@parcel/watcher": {
|
||||
"version": "2.5.1",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^1.0.3",
|
||||
"is-glob": "^4.0.3",
|
||||
"micromatch": "^4.0.5",
|
||||
"node-addon-api": "^7.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@parcel/watcher-linux-x64-glibc": "2.5.1",
|
||||
"@parcel/watcher-linux-x64-musl": "2.5.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@parcel/watcher-linux-x64-glibc": {
|
||||
"version": "2.5.1",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@parcel/watcher-linux-x64-musl": {
|
||||
"version": "2.5.1",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@parcel/watcher/node_modules/detect-libc": {
|
||||
"version": "1.0.3",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"detect-libc": "bin/detect-libc.js"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/cli": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/cli/-/cli-4.3.3.tgz",
|
||||
"integrity": "sha512-ZvS/n1ZHOBKcVlhkt8l5NNr1EDXk1NboYO5CYDOs6NUmvT9z6bzkwsosaJftY57T/3gWNzWMJzIXLodZC8ssdw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@parcel/watcher": "2.5.1",
|
||||
"@tailwindcss/node": "4.3.3",
|
||||
"@tailwindcss/oxide": "4.3.3",
|
||||
"enhanced-resolve": "^5.24.1",
|
||||
"mri": "^1.2.0",
|
||||
"picocolors": "^1.1.1",
|
||||
"tailwindcss": "4.3.3"
|
||||
},
|
||||
"bin": {
|
||||
"tailwindcss": "dist/index.mjs"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/node": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz",
|
||||
"integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/remapping": "^2.3.5",
|
||||
"enhanced-resolve": "^5.24.1",
|
||||
"jiti": "^2.7.0",
|
||||
"lightningcss": "1.32.0",
|
||||
"magic-string": "^0.30.21",
|
||||
"source-map-js": "^1.2.1",
|
||||
"tailwindcss": "4.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz",
|
||||
"integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==",
|
||||
"dev": true,
|
||||
"optionalDependencies": {
|
||||
"@tailwindcss/oxide-linux-x64-gnu": "4.3.3",
|
||||
"@tailwindcss/oxide-linux-x64-musl": "4.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-linux-x64-gnu": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz",
|
||||
"integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-linux-x64-musl": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz",
|
||||
"integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/braces": {
|
||||
"version": "3.0.3",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"fill-range": "^7.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-libc": {
|
||||
"version": "2.1.2",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/enhanced-resolve": {
|
||||
"version": "5.24.2",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"graceful-fs": "^4.2.4",
|
||||
"tapable": "^2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/fill-range": {
|
||||
"version": "7.1.1",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"to-regex-range": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/graceful-fs": {
|
||||
"version": "4.2.11",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/is-extglob": {
|
||||
"version": "2.1.1",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/is-glob": {
|
||||
"version": "4.0.3",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"is-extglob": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/is-number": {
|
||||
"version": "7.0.0",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/jiti": {
|
||||
"version": "2.7.0",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"jiti": "lib/jiti-cli.mjs"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss": {
|
||||
"version": "1.32.0",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.3"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"lightningcss-linux-x64-gnu": "1.32.0",
|
||||
"lightningcss-linux-x64-musl": "1.32.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lightningcss-linux-x64-gnu": {
|
||||
"version": "1.32.0",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/lightningcss-linux-x64-musl": {
|
||||
"version": "1.32.0",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "0.30.21",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/micromatch": {
|
||||
"version": "4.0.8",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"braces": "^3.0.3",
|
||||
"picomatch": "^2.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/mri": {
|
||||
"version": "1.2.0",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "7.1.1",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/tailwindcss": {
|
||||
"version": "4.3.3",
|
||||
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz",
|
||||
"integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/tapable": {
|
||||
"version": "2.3.3",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/to-regex-range": {
|
||||
"version": "5.0.1",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"is-number": "^7.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"css": "tailwindcss -i src/web/tailwind.css -o src/web/static/app.css --minify",
|
||||
"css:watch": "tailwindcss -i src/web/tailwind.css -o src/web/static/app.css --watch",
|
||||
"css:check": "tmp=$(mktemp) && trap 'rm -f \"$tmp\"' EXIT && tailwindcss -i src/web/tailwind.css -o \"$tmp\" --minify && diff -u src/web/static/app.css \"$tmp\""
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/cli": "4.3.3",
|
||||
"tailwindcss": "4.3.3"
|
||||
}
|
||||
}
|
||||
+53
-33
@@ -5,14 +5,14 @@
|
||||
//! both sides. It lives here and nowhere else:
|
||||
//!
|
||||
//! ```text
|
||||
//! message = "{issue_date}/{article_id}/{up|down}"
|
||||
//! message = "{issue_date}/{article_id}/{loved|good|down}"
|
||||
//! token = hex(hmac_sha256(secret, message))[..16]
|
||||
//! link = {public_url}/r/{issue_date}/{article_id}/{vote}?t={token}
|
||||
//! ```
|
||||
//!
|
||||
//! Pinned test vector, asserted from three places (here, `epub::build`,
|
||||
//! `tests/m7_server.rs`): `secret = "test-secret"`, date `2026-08-15`,
|
||||
//! article `42`, `up` → `3b314cf7e6d8f50f`.
|
||||
//! article `42`, `loved` (with legacy `up` verification).
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use jiff::civil::Date;
|
||||
@@ -23,17 +23,26 @@ use crate::types::{ArticleId, Vote};
|
||||
/// Characters of the hex HMAC kept in rating links (§3.9).
|
||||
pub const TOKEN_LEN: usize = 16;
|
||||
|
||||
/// The exact signed string: `{issue_date}/{article_id}/{up|down}` (§3.9).
|
||||
/// The exact signed string: `{issue_date}/{article_id}/{loved|good|down}` (§3.9).
|
||||
pub fn rating_message(issue_date: Date, article_id: ArticleId, vote: Vote) -> String {
|
||||
format!("{issue_date}/{article_id}/{}", vote.as_str())
|
||||
}
|
||||
|
||||
/// `hex(hmac_sha256(secret, "{issue_date}/{article_id}/{vote}"))[..16]` (§3.9).
|
||||
pub fn rating_token(secret: &str, issue_date: Date, article_id: ArticleId, vote: Vote) -> String {
|
||||
rating_token_for_segment(secret, issue_date, article_id, vote.as_str())
|
||||
}
|
||||
|
||||
fn rating_token_for_segment(
|
||||
secret: &str,
|
||||
issue_date: Date,
|
||||
article_id: ArticleId,
|
||||
segment: &str,
|
||||
) -> String {
|
||||
// `Hmac` derives a fixed-size key from any input length, so this never fails.
|
||||
let mut mac = <Hmac<Sha256> as KeyInit>::new_from_slice(secret.as_bytes())
|
||||
.expect("HMAC accepts keys of any length");
|
||||
mac.update(rating_message(issue_date, article_id, vote).as_bytes());
|
||||
mac.update(format!("{issue_date}/{article_id}/{segment}").as_bytes());
|
||||
let digest = hex::encode(mac.finalize().into_bytes());
|
||||
digest[..TOKEN_LEN].to_string()
|
||||
}
|
||||
@@ -46,8 +55,14 @@ pub fn verify_token(
|
||||
vote: Vote,
|
||||
token: &str,
|
||||
) -> bool {
|
||||
constant_time_eq(
|
||||
rating_token(secret, issue_date, article_id, vote).as_bytes(),
|
||||
let current = rating_token(secret, issue_date, article_id, vote);
|
||||
if constant_time_eq(current.as_bytes(), token.as_bytes()) {
|
||||
return true;
|
||||
}
|
||||
// Already-published `up` links were signed over the literal legacy segment.
|
||||
vote == Vote::Loved
|
||||
&& constant_time_eq(
|
||||
rating_token_for_segment(secret, issue_date, article_id, "up").as_bytes(),
|
||||
token.as_bytes(),
|
||||
)
|
||||
}
|
||||
@@ -67,7 +82,7 @@ pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
|
||||
}
|
||||
|
||||
/// Full rating URL embedded in an article footer:
|
||||
/// `{public_url}/r/{date}/{article_id}/{up|down}?t={token}` (§3.9).
|
||||
/// `{public_url}/r/{date}/{article_id}/{loved|good|down}?t={token}` (§3.9).
|
||||
pub fn rating_url(
|
||||
public_url: &str,
|
||||
secret: &str,
|
||||
@@ -92,50 +107,55 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_matches_the_shared_vector() {
|
||||
assert_eq!(rating_message(date(), 42, Vote::Up), "2026-08-15/42/up");
|
||||
assert_eq!(
|
||||
rating_token("test-secret", date(), 42, Vote::Up),
|
||||
"3b314cf7e6d8f50f"
|
||||
);
|
||||
assert_eq!(rating_token("test-secret", date(), 42, Vote::Up).len(), 16);
|
||||
fn all_three_tokens_verify_and_are_distinct() {
|
||||
let votes = [Vote::Loved, Vote::Good, Vote::NotForMe];
|
||||
let tokens: Vec<String> = votes
|
||||
.iter()
|
||||
.map(|vote| rating_token("test-secret", date(), 42, *vote))
|
||||
.collect();
|
||||
assert_eq!(tokens.len(), 3);
|
||||
assert!(tokens.iter().all(|token| token.len() == TOKEN_LEN));
|
||||
assert_ne!(tokens[0], tokens[1]);
|
||||
assert_ne!(tokens[1], tokens[2]);
|
||||
for (vote, token) in votes.into_iter().zip(tokens) {
|
||||
assert!(verify_token("test-secret", date(), 42, vote, &token));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_are_per_article_and_per_vote() {
|
||||
let up = rating_token("s", date(), 42, Vote::Up);
|
||||
assert_ne!(up, rating_token("s", date(), 42, Vote::Down));
|
||||
assert_ne!(up, rating_token("s", date(), 43, Vote::Up));
|
||||
assert_ne!(up, rating_token("other", date(), 42, Vote::Up));
|
||||
let tomorrow: Date = "2026-08-16".parse().unwrap();
|
||||
assert_ne!(up, rating_token("s", tomorrow, 42, Vote::Up));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_is_exact() {
|
||||
fn legacy_up_token_still_verifies_as_loved() {
|
||||
let legacy = rating_token_for_segment("test-secret", date(), 42, "up");
|
||||
assert_eq!(legacy, "3b314cf7e6d8f50f");
|
||||
assert!(verify_token(
|
||||
"s",
|
||||
"test-secret",
|
||||
date(),
|
||||
42,
|
||||
Vote::Up,
|
||||
&rating_token("s", date(), 42, Vote::Up)
|
||||
Vote::Loved,
|
||||
&legacy
|
||||
));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, "deadbeefdeadbeef"));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, ""));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, "short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_rejects_tampering() {
|
||||
let token = rating_token("s", date(), 42, Vote::Loved);
|
||||
assert!(!verify_token("s", date(), 42, Vote::Good, &token));
|
||||
assert!(!verify_token("s", date(), 43, Vote::Loved, &token));
|
||||
assert!(!verify_token("other", date(), 42, Vote::Loved, &token));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Loved, "short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_shape_matches_the_spec() {
|
||||
let token = rating_token("test-secret", date(), 42, Vote::Good);
|
||||
assert_eq!(
|
||||
rating_url(
|
||||
"https://daily.hallada.net/",
|
||||
"test-secret",
|
||||
date(),
|
||||
42,
|
||||
Vote::Up
|
||||
Vote::Good
|
||||
),
|
||||
"https://daily.hallada.net/r/2026-08-15/42/up?t=3b314cf7e6d8f50f"
|
||||
format!("https://daily.hallada.net/r/2026-08-15/42/good?t={token}")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
//! BookOrbit OPDS catalog lookup support.
|
||||
//!
|
||||
//! BookOrbit is the companion library and browser-based EPUB reader used by
|
||||
//! this service. Its OPDS catalog is preferable to its JSON API here because
|
||||
//! OPDS uses static HTTP Basic credentials and therefore needs no JWT/refresh
|
||||
//! token lifecycle. Searches use `/api/v1/opds/catalog?q=<date>`, acquisition
|
||||
//! links expose `/api/v1/opds/<book_id>/download?fileId=<file_id>`, and browser
|
||||
//! links use `/read/<book_id>/<file_id>`. See
|
||||
//! `docs/plans/2026-09-05-bookorbit-read-link.md` for the integration design.
|
||||
|
||||
use jiff::civil::Date;
|
||||
use reqwest::header::ACCEPT;
|
||||
|
||||
const ACQUISITION_REL: &str = "http://opds-spec.org/acquisition";
|
||||
const DOWNLOAD_PREFIX: &str = "/api/v1/opds/";
|
||||
|
||||
/// The BookOrbit book and file identifiers required by its reader route.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct BookorbitIds {
|
||||
/// BookOrbit's identifier for the issue's book record.
|
||||
pub book_id: i64,
|
||||
/// BookOrbit's identifier for the EPUB file attached to the book.
|
||||
pub file_id: i64,
|
||||
}
|
||||
|
||||
/// A failure while querying or parsing BookOrbit's OPDS catalog.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BookorbitError {
|
||||
/// The OPDS endpoint could not be reached.
|
||||
#[error("BookOrbit unreachable: {0}")]
|
||||
Unreachable(#[source] reqwest::Error),
|
||||
/// BookOrbit rejected the configured OPDS Basic credentials.
|
||||
#[error("BookOrbit rejected the OPDS credentials")]
|
||||
Unauthorized,
|
||||
/// BookOrbit returned an unexpected non-success status.
|
||||
#[error("BookOrbit returned HTTP {0}")]
|
||||
Status(reqwest::StatusCode),
|
||||
/// BookOrbit returned a body that could not be read as the expected feed.
|
||||
#[error("BookOrbit returned an unreadable OPDS feed: {0}")]
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
/// Search BookOrbit's OPDS catalog for the Standard edition of the issue dated `date`.
|
||||
///
|
||||
/// `api_url` must have no trailing slash. `Ok(None)` means BookOrbit has not
|
||||
/// indexed the issue yet.
|
||||
pub async fn find_issue(
|
||||
client: &reqwest::Client,
|
||||
api_url: &str,
|
||||
opds_user: &str,
|
||||
opds_pass: &str,
|
||||
issue_title: &str,
|
||||
date: Date,
|
||||
) -> Result<Option<BookorbitIds>, BookorbitError> {
|
||||
let response = client
|
||||
.get(format!("{api_url}/api/v1/opds/catalog"))
|
||||
.query(&[("q", date.to_string())])
|
||||
.basic_auth(opds_user, Some(opds_pass))
|
||||
.header(ACCEPT, "application/atom+xml")
|
||||
.send()
|
||||
.await
|
||||
.map_err(BookorbitError::Unreachable)?;
|
||||
|
||||
let status = response.status();
|
||||
if matches!(
|
||||
status,
|
||||
reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN
|
||||
) {
|
||||
return Err(BookorbitError::Unauthorized);
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(BookorbitError::Status(status));
|
||||
}
|
||||
|
||||
let feed_xml = response.text().await.map_err(|error| {
|
||||
if error.is_timeout() || error.is_connect() {
|
||||
BookorbitError::Unreachable(error)
|
||||
} else {
|
||||
BookorbitError::Malformed(error.to_string())
|
||||
}
|
||||
})?;
|
||||
let ids = select_issue_entry(&feed_xml, issue_title, date)?;
|
||||
if let Some(ids) = ids {
|
||||
tracing::info!(
|
||||
book_id = ids.book_id,
|
||||
file_id = ids.file_id,
|
||||
"resolved BookOrbit issue"
|
||||
);
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// Select the Standard issue from an Atom feed without performing network I/O.
|
||||
pub fn select_issue_entry(
|
||||
feed_xml: &str,
|
||||
issue_title: &str,
|
||||
date: Date,
|
||||
) -> Result<Option<BookorbitIds>, BookorbitError> {
|
||||
if find_open_tag(feed_xml, "feed", 0).is_none() {
|
||||
return Err(BookorbitError::Malformed(
|
||||
"response does not contain an Atom <feed> element".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let fallback_title = format!("The Daily EPUB - {date}");
|
||||
let mut fallback_entry = None;
|
||||
|
||||
for entry in entry_bodies(feed_xml) {
|
||||
let Some(title) = element_text(entry, "title") else {
|
||||
continue;
|
||||
};
|
||||
let title = xml_unescape(title);
|
||||
let title = title.trim();
|
||||
if title.ends_with("(X4)") {
|
||||
continue;
|
||||
}
|
||||
if title == issue_title {
|
||||
return acquisition_ids(entry);
|
||||
}
|
||||
if title == fallback_title && fallback_entry.is_none() {
|
||||
fallback_entry = Some(entry);
|
||||
}
|
||||
}
|
||||
|
||||
fallback_entry.map_or(Ok(None), acquisition_ids)
|
||||
}
|
||||
|
||||
/// Build the public BookOrbit reader URL for `ids`.
|
||||
///
|
||||
/// `public_url` must have no trailing slash.
|
||||
pub fn reader_url(public_url: &str, ids: BookorbitIds) -> String {
|
||||
format!("{public_url}/read/{}/{}", ids.book_id, ids.file_id)
|
||||
}
|
||||
|
||||
fn entry_bodies(feed_xml: &str) -> Vec<&str> {
|
||||
let mut entries = Vec::new();
|
||||
let mut cursor = 0;
|
||||
|
||||
while let Some(start) = find_open_tag(feed_xml, "entry", cursor) {
|
||||
let Some(open_end_offset) = feed_xml[start..].find('>') else {
|
||||
break;
|
||||
};
|
||||
let body_start = start + open_end_offset + 1;
|
||||
let Some(close_offset) = feed_xml[body_start..].find("</entry>") else {
|
||||
break;
|
||||
};
|
||||
let body_end = body_start + close_offset;
|
||||
entries.push(&feed_xml[body_start..body_end]);
|
||||
cursor = body_end + "</entry>".len();
|
||||
}
|
||||
|
||||
entries
|
||||
}
|
||||
|
||||
fn element_text<'a>(xml: &'a str, name: &str) -> Option<&'a str> {
|
||||
let start = find_open_tag(xml, name, 0)?;
|
||||
let open_end = start + xml[start..].find('>')?;
|
||||
let text_start = open_end + 1;
|
||||
let close = format!("</{name}>");
|
||||
let text_end = text_start + xml[text_start..].find(&close)?;
|
||||
Some(&xml[text_start..text_end])
|
||||
}
|
||||
|
||||
fn find_open_tag(xml: &str, name: &str, mut cursor: usize) -> Option<usize> {
|
||||
let needle = format!("<{name}");
|
||||
while let Some(offset) = xml[cursor..].find(&needle) {
|
||||
let start = cursor + offset;
|
||||
let after_name = start + needle.len();
|
||||
if xml
|
||||
.as_bytes()
|
||||
.get(after_name)
|
||||
.is_some_and(|byte| byte.is_ascii_whitespace() || matches!(byte, b'>' | b'/'))
|
||||
{
|
||||
return Some(start);
|
||||
}
|
||||
cursor = after_name;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn acquisition_ids(entry: &str) -> Result<Option<BookorbitIds>, BookorbitError> {
|
||||
let mut cursor = 0;
|
||||
let mut saw_acquisition = false;
|
||||
|
||||
while let Some(start) = find_open_tag(entry, "link", cursor) {
|
||||
let Some(end_offset) = entry[start..].find('>') else {
|
||||
break;
|
||||
};
|
||||
let end = start + end_offset;
|
||||
let attributes = &entry[start + "<link".len()..end];
|
||||
if attribute_value(attributes, "rel") == Some(ACQUISITION_REL) {
|
||||
saw_acquisition = true;
|
||||
if let Some(href) = attribute_value(attributes, "href") {
|
||||
let href = xml_unescape(href);
|
||||
if let Some(ids) = parse_download_href(&href) {
|
||||
return Ok(Some(ids));
|
||||
}
|
||||
}
|
||||
}
|
||||
cursor = end + 1;
|
||||
}
|
||||
|
||||
if saw_acquisition {
|
||||
Err(BookorbitError::Malformed(
|
||||
"qualifying entry has an invalid acquisition href".to_string(),
|
||||
))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
fn attribute_value<'a>(attributes: &'a str, wanted: &str) -> Option<&'a str> {
|
||||
let bytes = attributes.as_bytes();
|
||||
let mut cursor = 0;
|
||||
|
||||
while cursor < bytes.len() {
|
||||
while cursor < bytes.len() && (bytes[cursor].is_ascii_whitespace() || bytes[cursor] == b'/')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
let name_start = cursor;
|
||||
while cursor < bytes.len()
|
||||
&& !bytes[cursor].is_ascii_whitespace()
|
||||
&& !matches!(bytes[cursor], b'=' | b'/' | b'>')
|
||||
{
|
||||
cursor += 1;
|
||||
}
|
||||
if name_start == cursor {
|
||||
cursor += 1;
|
||||
continue;
|
||||
}
|
||||
let name = &attributes[name_start..cursor];
|
||||
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() {
|
||||
cursor += 1;
|
||||
}
|
||||
if bytes.get(cursor) != Some(&b'=') {
|
||||
continue;
|
||||
}
|
||||
cursor += 1;
|
||||
while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() {
|
||||
cursor += 1;
|
||||
}
|
||||
|
||||
let quote = *bytes.get(cursor)?;
|
||||
if !matches!(quote, b'\'' | b'"') {
|
||||
return None;
|
||||
}
|
||||
cursor += 1;
|
||||
let value_start = cursor;
|
||||
while cursor < bytes.len() && bytes[cursor] != quote {
|
||||
cursor += 1;
|
||||
}
|
||||
if cursor == bytes.len() {
|
||||
return None;
|
||||
}
|
||||
let value = &attributes[value_start..cursor];
|
||||
cursor += 1;
|
||||
if name == wanted {
|
||||
return Some(value);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn parse_download_href(href: &str) -> Option<BookorbitIds> {
|
||||
let path = href.strip_prefix(DOWNLOAD_PREFIX)?;
|
||||
let (book_id, query) = path.split_once("/download?fileId=")?;
|
||||
if book_id.is_empty() || !book_id.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let file_id = match query.split_once('&') {
|
||||
Some((file_id, extra_params))
|
||||
if !extra_params.is_empty() && !extra_params.split('&').any(str::is_empty) =>
|
||||
{
|
||||
file_id
|
||||
}
|
||||
Some(_) => return None,
|
||||
None => query,
|
||||
};
|
||||
if file_id.is_empty() || !file_id.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(BookorbitIds {
|
||||
book_id: book_id.parse().ok()?,
|
||||
file_id: file_id.parse().ok()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn xml_unescape(text: &str) -> String {
|
||||
let mut output = String::with_capacity(text.len());
|
||||
let mut cursor = 0;
|
||||
|
||||
while let Some(offset) = text[cursor..].find('&') {
|
||||
let ampersand = cursor + offset;
|
||||
output.push_str(&text[cursor..ampersand]);
|
||||
let entity_start = ampersand + 1;
|
||||
let Some(end_offset) = text[entity_start..].find(';') else {
|
||||
output.push_str(&text[ampersand..]);
|
||||
return output;
|
||||
};
|
||||
let entity_end = entity_start + end_offset;
|
||||
let entity = &text[entity_start..entity_end];
|
||||
if let Some(character) = decode_entity(entity) {
|
||||
output.push(character);
|
||||
} else {
|
||||
output.push_str(&text[ampersand..=entity_end]);
|
||||
}
|
||||
cursor = entity_end + 1;
|
||||
}
|
||||
|
||||
output.push_str(&text[cursor..]);
|
||||
output
|
||||
}
|
||||
|
||||
fn decode_entity(entity: &str) -> Option<char> {
|
||||
match entity {
|
||||
"amp" => Some('&'),
|
||||
"lt" => Some('<'),
|
||||
"gt" => Some('>'),
|
||||
"quot" => Some('"'),
|
||||
"apos" => Some('\''),
|
||||
_ => entity
|
||||
.strip_prefix("#x")
|
||||
.or_else(|| entity.strip_prefix("#X"))
|
||||
.and_then(|digits| u32::from_str_radix(digits, 16).ok())
|
||||
.or_else(|| {
|
||||
entity
|
||||
.strip_prefix('#')
|
||||
.and_then(|digits| digits.parse().ok())
|
||||
})
|
||||
.and_then(char::from_u32),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const REAL_SHAPE_FEED: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<id>urn:bookorbit:catalog</id>
|
||||
<title>BookOrbit Catalog</title>
|
||||
<entry data-source="watch-folder">
|
||||
<title>The Daily EPUB — 2026-09-05 (X4)</title>
|
||||
<id>urn:bookorbit:book:410</id>
|
||||
<link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/410/download?fileId=901" type="application/epub+zip" title="EPUB"/>
|
||||
</entry>
|
||||
<entry data-source="watch-folder">
|
||||
<title>The Daily EPUB — 2026-09-05</title>
|
||||
<id>urn:bookorbit:book:411</id>
|
||||
<link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/411/download?fileId=902" type="application/epub+zip" title="EPUB"/>
|
||||
</entry>
|
||||
</feed>"#;
|
||||
|
||||
fn date() -> Date {
|
||||
"2026-09-05".parse().expect("date")
|
||||
}
|
||||
|
||||
fn feed(entries: &str) -> String {
|
||||
format!(r#"<feed xmlns="http://www.w3.org/2005/Atom">{entries}</feed>"#)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_title_match_returns_the_right_ids() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>Another book</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/1/download?fileId=2"/></entry>
|
||||
<entry><title>The Daily EPUB — 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/31/download?fileId=47"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 31,
|
||||
file_id: 47
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x4_entry_listed_first_is_skipped_for_standard() {
|
||||
assert_eq!(
|
||||
select_issue_entry(REAL_SHAPE_FEED, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 411,
|
||||
file_id: 902
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hyphen_form_fallback_works_without_em_dash_title() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>The Daily EPUB - 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/52/download?fileId=81"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 52,
|
||||
file_id: 81
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_title_outranks_an_earlier_fallback() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>The Daily EPUB - 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/1/download?fileId=2"/></entry>
|
||||
<entry><title>The Daily EPUB — 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/3/download?fileId=4"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 3,
|
||||
file_id: 4
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_matching_entry_returns_none() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>Unrelated</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/7/download?fileId=8"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
select_issue_entry(&feed(""), "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_acquisition_href_is_an_error_for_a_match() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>The Daily EPUB — 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/nope/download?fileId=8"/></entry>"#,
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()),
|
||||
Err(BookorbitError::Malformed(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn title_entities_are_unescaped_before_comparison() {
|
||||
let xml = feed(
|
||||
r#"<entry><title>Books & News — 2026-09-05</title><link rel="http://opds-spec.org/acquisition" href="/api/v1/opds/12/download?fileId=13"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "Books & News — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 12,
|
||||
file_id: 13
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attributed_entry_and_reordered_link_attributes_parse() {
|
||||
let xml = feed(
|
||||
r#"<entry data-index="1"><title type="text">The Daily EPUB — 2026-09-05</title><link title="EPUB" href="/api/v1/opds/63/download?fileId=64&download=true" type="application/epub+zip" rel="http://opds-spec.org/acquisition"/></entry>"#,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
select_issue_entry(&xml, "The Daily EPUB — 2026-09-05", date()).unwrap(),
|
||||
Some(BookorbitIds {
|
||||
book_id: 63,
|
||||
file_id: 64
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reader_url_formats_the_reader_route() {
|
||||
assert_eq!(
|
||||
reader_url(
|
||||
"https://bookorbit.example",
|
||||
BookorbitIds {
|
||||
book_id: 14,
|
||||
file_id: 29
|
||||
}
|
||||
),
|
||||
"https://bookorbit.example/read/14/29"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_without_feed_is_malformed() {
|
||||
assert!(matches!(
|
||||
select_issue_entry(
|
||||
"<html><body>not an Atom feed</body></html>",
|
||||
"The Daily EPUB — 2026-09-05",
|
||||
date()
|
||||
),
|
||||
Err(BookorbitError::Malformed(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
+1724
-54
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,542 @@
|
||||
//! Hygiene and union admission into the deep set (plan §8.1, §11).
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::Row as _;
|
||||
|
||||
use super::{prefilter, telemetry};
|
||||
use crate::config::{CurationConfig, RankingConfig};
|
||||
use crate::db::{Db, fmt_ts};
|
||||
use crate::types::{Article, ArticleId, Candidate};
|
||||
|
||||
/// Run hygiene before embeddings and write thin telemetry rows for exclusions.
|
||||
pub async fn hygiene(
|
||||
db: &Db,
|
||||
run_id: i64,
|
||||
articles: Vec<Article>,
|
||||
date: Date,
|
||||
config: &CurationConfig,
|
||||
now: Timestamp,
|
||||
) -> anyhow::Result<Vec<Candidate>> {
|
||||
let published = db
|
||||
.previously_published_ids_before(date)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
let since = now - jiff::Span::new().hours(config.recent_rejection_days.max(0) * 24);
|
||||
let rows = sqlx::query(
|
||||
"SELECT DISTINCT article_id FROM article_assessments
|
||||
WHERE stage IN ('triage', 'deep') AND score IS NOT NULL AND score < ?
|
||||
AND assessed_at >= ?",
|
||||
)
|
||||
.bind(config.recent_rejection_floor)
|
||||
.bind(fmt_ts(since))
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
let rejected = rows
|
||||
.iter()
|
||||
.map(|row| row.get::<i64, _>("article_id"))
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
let mut eligible = Vec::new();
|
||||
for article in articles {
|
||||
let auto_include = prefilter::is_auto_include(&article, config);
|
||||
let reason = if auto_include {
|
||||
None
|
||||
} else if prefilter::is_blocked(&article, config) {
|
||||
Some("blocked")
|
||||
} else if published.contains(&article.id) {
|
||||
Some("published_before")
|
||||
} else if rejected.contains(&article.id) {
|
||||
Some("recently_rejected")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(reason) = reason {
|
||||
telemetry::thin_excluded(db, run_id, article.id, reason).await?;
|
||||
} else {
|
||||
eligible.push(Candidate::new(article, auto_include));
|
||||
}
|
||||
}
|
||||
Ok(eligible)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AdmissionSummary {
|
||||
pub admitted: usize,
|
||||
pub admitted_by: BTreeMap<String, usize>,
|
||||
pub exploration_admitted: usize,
|
||||
}
|
||||
|
||||
pub fn admit(
|
||||
candidates: &mut [Candidate],
|
||||
date: Date,
|
||||
ranking: &RankingConfig,
|
||||
) -> AdmissionSummary {
|
||||
for candidate in candidates.iter_mut() {
|
||||
candidate.admitted_by.clear();
|
||||
candidate.exploration = false;
|
||||
if candidate.excluded_reason.as_deref() != Some("not_admitted") {
|
||||
candidate.excluded_reason = None;
|
||||
}
|
||||
}
|
||||
let mut admitted = HashSet::new();
|
||||
for (index, candidate) in candidates.iter_mut().enumerate() {
|
||||
if candidate.excluded_reason.is_none() && candidate.auto_include {
|
||||
candidate.admitted_by.push("auto_include".into());
|
||||
admitted.insert(index);
|
||||
}
|
||||
}
|
||||
|
||||
let capacity = |admitted: &HashSet<usize>| ranking.deep_keep.saturating_sub(admitted.len());
|
||||
|
||||
let triage = ranked(candidates, |candidate| {
|
||||
candidate
|
||||
.assessment
|
||||
.triage
|
||||
.as_ref()
|
||||
.filter(|triage| triage.interest >= 5.0)
|
||||
.map(|triage| triage.interest)
|
||||
});
|
||||
let quota = ranking.quotas.triage.min(capacity(&admitted));
|
||||
take_retriever(
|
||||
candidates,
|
||||
&mut admitted,
|
||||
&triage,
|
||||
ranking.quotas.triage,
|
||||
quota,
|
||||
"triage",
|
||||
);
|
||||
|
||||
let interest = ranked(candidates, |candidate| {
|
||||
semantic_floor(candidate, ranking)
|
||||
.then_some(candidate.signals.interest)
|
||||
.flatten()
|
||||
});
|
||||
if !interest.is_empty() {
|
||||
let quota = ranking.quotas.interest.min(capacity(&admitted));
|
||||
take_retriever(
|
||||
candidates,
|
||||
&mut admitted,
|
||||
&interest,
|
||||
ranking.quotas.interest,
|
||||
quota,
|
||||
"interest",
|
||||
);
|
||||
}
|
||||
|
||||
let knn = ranked(candidates, |candidate| {
|
||||
semantic_floor(candidate, ranking)
|
||||
.then_some(candidate.signals.knn.filter(|score| *score > 0.0))
|
||||
.flatten()
|
||||
});
|
||||
if !knn.is_empty() {
|
||||
let quota = ranking.quotas.knn.min(capacity(&admitted));
|
||||
take_retriever(
|
||||
candidates,
|
||||
&mut admitted,
|
||||
&knn,
|
||||
ranking.quotas.knn,
|
||||
quota,
|
||||
"knn",
|
||||
);
|
||||
}
|
||||
|
||||
let mut by_blend = ranked(candidates, |candidate| candidate.signals.preliminary);
|
||||
let band_end = ((ranking.deep_keep as f64) * 2.5).ceil() as usize;
|
||||
let band_start = ranking.deep_keep.min(by_blend.len());
|
||||
by_blend.truncate(band_end.min(by_blend.len()));
|
||||
let mut exploration = by_blend
|
||||
.into_iter()
|
||||
.skip(band_start)
|
||||
.filter(|index| {
|
||||
let candidate = &candidates[*index];
|
||||
candidate.article.word_count >= 300
|
||||
&& !prefilter::looks_like_roundup(&candidate.article.title)
|
||||
&& candidate
|
||||
.assessment
|
||||
.triage
|
||||
.as_ref()
|
||||
.is_some_and(|triage| triage.interest >= 4.0)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
exploration.sort_by_key(|index| exploration_key(date, candidates[*index].article.id));
|
||||
let quota = ranking.exploration_slots.min(capacity(&admitted));
|
||||
take_retriever(
|
||||
candidates,
|
||||
&mut admitted,
|
||||
&exploration,
|
||||
ranking.exploration_slots,
|
||||
quota,
|
||||
"exploration",
|
||||
);
|
||||
for index in &admitted {
|
||||
if candidates[*index]
|
||||
.admitted_by
|
||||
.first()
|
||||
.is_some_and(|name| name == "exploration")
|
||||
{
|
||||
candidates[*index].exploration = true;
|
||||
}
|
||||
}
|
||||
|
||||
let blend = ranked(candidates, |candidate| candidate.signals.preliminary);
|
||||
let quota = capacity(&admitted);
|
||||
take_retriever(candidates, &mut admitted, &blend, quota, quota, "blend");
|
||||
|
||||
for (index, candidate) in candidates.iter_mut().enumerate() {
|
||||
if admitted.contains(&index) {
|
||||
candidate.stage = "admitted".into();
|
||||
candidate.excluded_reason = None;
|
||||
} else if candidate.excluded_reason.is_none() {
|
||||
candidate.stage = if candidate.assessment.triage.is_some() {
|
||||
"triaged".into()
|
||||
} else {
|
||||
"eligible".into()
|
||||
};
|
||||
candidate.excluded_reason = Some("not_admitted".into());
|
||||
}
|
||||
}
|
||||
let mut summary = AdmissionSummary {
|
||||
admitted: admitted.len(),
|
||||
exploration_admitted: admitted
|
||||
.iter()
|
||||
.filter(|index| candidates[**index].exploration)
|
||||
.count(),
|
||||
..AdmissionSummary::default()
|
||||
};
|
||||
for index in admitted {
|
||||
if let Some(first) = candidates[index].admitted_by.first() {
|
||||
*summary.admitted_by.entry(first.clone()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
summary
|
||||
}
|
||||
|
||||
fn semantic_floor(candidate: &Candidate, ranking: &RankingConfig) -> bool {
|
||||
candidate.article.word_count >= ranking.semantic_min_words
|
||||
&& !prefilter::looks_like_roundup(&candidate.article.title)
|
||||
&& candidate
|
||||
.assessment
|
||||
.triage
|
||||
.as_ref()
|
||||
.is_none_or(|triage| triage.interest >= 3.0)
|
||||
}
|
||||
|
||||
fn ranked(candidates: &[Candidate], signal: impl Fn(&Candidate) -> Option<f64>) -> Vec<usize> {
|
||||
let mut values = candidates
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, candidate)| candidate.excluded_reason.is_none())
|
||||
.filter_map(|(index, candidate)| signal(candidate).map(|value| (index, value)))
|
||||
.collect::<Vec<_>>();
|
||||
values.sort_by(|(left_index, left), (right_index, right)| {
|
||||
right.total_cmp(left).then_with(|| {
|
||||
candidates[*left_index]
|
||||
.article
|
||||
.id
|
||||
.cmp(&candidates[*right_index].article.id)
|
||||
})
|
||||
});
|
||||
values.into_iter().map(|(index, _)| index).collect()
|
||||
}
|
||||
|
||||
fn take_retriever(
|
||||
candidates: &mut [Candidate],
|
||||
admitted: &mut HashSet<usize>,
|
||||
ranked: &[usize],
|
||||
would_take: usize,
|
||||
admit_quota: usize,
|
||||
name: &str,
|
||||
) {
|
||||
// Record overlap among this retriever's own top-N.
|
||||
for index in ranked.iter().take(would_take) {
|
||||
if admitted.contains(index)
|
||||
&& !candidates[*index]
|
||||
.admitted_by
|
||||
.iter()
|
||||
.any(|value| value == name)
|
||||
{
|
||||
candidates[*index].admitted_by.push(name.into());
|
||||
}
|
||||
}
|
||||
let mut taken = 0;
|
||||
for index in ranked {
|
||||
if admitted.contains(index) {
|
||||
continue;
|
||||
}
|
||||
if taken >= admit_quota {
|
||||
break;
|
||||
}
|
||||
candidates[*index].admitted_by.push(name.into());
|
||||
admitted.insert(*index);
|
||||
taken += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn exploration_key(date: Date, article_id: ArticleId) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(date.to_string().as_bytes());
|
||||
hasher.update(article_id.to_string().as_bytes());
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{CurationConfig, RankingConfig, RankingQuotas};
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::types::Triage;
|
||||
|
||||
fn candidate(
|
||||
id: i64,
|
||||
words: i64,
|
||||
interest: Option<f64>,
|
||||
knn: Option<f64>,
|
||||
triage: Option<f64>,
|
||||
) -> Candidate {
|
||||
let mut candidate = Candidate::new(article(id, &format!("article {id}"), words), false);
|
||||
candidate.signals.interest = interest;
|
||||
candidate.signals.knn = knn;
|
||||
candidate.signals.preliminary = Some(id as f64);
|
||||
candidate.assessment.triage = triage.map(|interest| Triage {
|
||||
interest,
|
||||
kind: "essay".into(),
|
||||
why: "specific".into(),
|
||||
model: "mock".into(),
|
||||
prompt_version: 1,
|
||||
assessed_at: "2026-09-02T05:30:00Z".parse().expect("timestamp"),
|
||||
});
|
||||
candidate
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_retrievers_reject_stubs_and_honor_quotas() {
|
||||
let config = RankingConfig {
|
||||
deep_keep: 4,
|
||||
exploration_slots: 0,
|
||||
quotas: RankingQuotas {
|
||||
triage: 0,
|
||||
interest: 2,
|
||||
knn: 1,
|
||||
},
|
||||
..RankingConfig::default()
|
||||
};
|
||||
let mut candidates = vec![
|
||||
candidate(1, 60, Some(100.0), Some(100.0), None),
|
||||
candidate(2, 600, Some(9.0), None, None),
|
||||
candidate(3, 600, Some(8.0), None, None),
|
||||
candidate(4, 600, None, Some(0.8), None),
|
||||
candidate(5, 600, None, None, None),
|
||||
];
|
||||
let summary = admit(
|
||||
&mut candidates,
|
||||
"2026-09-02".parse().expect("date"),
|
||||
&config,
|
||||
);
|
||||
assert_eq!(summary.admitted_by.get("interest"), Some(&2));
|
||||
assert_eq!(summary.admitted_by.get("knn"), Some(&1));
|
||||
assert!(
|
||||
!candidates[0]
|
||||
.admitted_by
|
||||
.iter()
|
||||
.any(|by| by == "interest" || by == "knn")
|
||||
);
|
||||
assert_eq!(summary.admitted, 4);
|
||||
assert_eq!(summary.admitted_by.get("blend"), Some(&1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strong_interest_weak_heuristic_reaches_deep_set_and_auto_always_wins() {
|
||||
let config = RankingConfig {
|
||||
deep_keep: 2,
|
||||
exploration_slots: 0,
|
||||
quotas: RankingQuotas {
|
||||
triage: 0,
|
||||
interest: 1,
|
||||
knn: 0,
|
||||
},
|
||||
..RankingConfig::default()
|
||||
};
|
||||
let mut candidates = vec![
|
||||
candidate(1, 400, Some(9.0), None, None),
|
||||
candidate(2, 100, None, None, None),
|
||||
candidate(3, 4000, None, None, None),
|
||||
];
|
||||
candidates[0].signals.heuristic = Some(0.0);
|
||||
candidates[1].auto_include = true;
|
||||
let summary = admit(
|
||||
&mut candidates,
|
||||
"2026-09-02".parse().expect("date"),
|
||||
&config,
|
||||
);
|
||||
assert_eq!(summary.admitted, 2);
|
||||
assert_eq!(
|
||||
candidates[0].admitted_by.first().map(String::as_str),
|
||||
Some("interest")
|
||||
);
|
||||
assert_eq!(
|
||||
candidates[1].admitted_by.first().map(String::as_str),
|
||||
Some("auto_include")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inactive_retrievers_release_slots_to_blend() {
|
||||
let config = RankingConfig {
|
||||
deep_keep: 3,
|
||||
exploration_slots: 0,
|
||||
..RankingConfig::default()
|
||||
};
|
||||
let mut candidates = (1..=5)
|
||||
.map(|id| candidate(id, 500, None, None, None))
|
||||
.collect::<Vec<_>>();
|
||||
let summary = admit(
|
||||
&mut candidates,
|
||||
"2026-09-02".parse().expect("date"),
|
||||
&config,
|
||||
);
|
||||
assert_eq!(summary.admitted_by.get("blend"), Some(&3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exploration_is_stable_for_a_date_and_rotates() {
|
||||
let config = RankingConfig {
|
||||
deep_keep: 4,
|
||||
exploration_slots: 2,
|
||||
quotas: RankingQuotas {
|
||||
triage: 0,
|
||||
interest: 0,
|
||||
knn: 0,
|
||||
},
|
||||
..RankingConfig::default()
|
||||
};
|
||||
let base = (1..=12)
|
||||
.map(|id| candidate(id, 500, None, None, Some(5.0)))
|
||||
.collect::<Vec<_>>();
|
||||
let mut first = base.clone();
|
||||
admit(&mut first, "2026-09-02".parse().expect("date"), &config);
|
||||
let ids = |items: &[Candidate]| {
|
||||
items
|
||||
.iter()
|
||||
.filter(|candidate| candidate.exploration)
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let expected = ids(&first);
|
||||
let mut again = base.clone();
|
||||
admit(&mut again, "2026-09-02".parse().expect("date"), &config);
|
||||
assert_eq!(ids(&again), expected);
|
||||
let mut next = base;
|
||||
admit(&mut next, "2026-09-03".parse().expect("date"), &config);
|
||||
assert_ne!(ids(&next), expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recent_low_triage_is_excluded_but_auto_include_is_spared() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = Db::open_and_migrate(&dir.path().join("hygiene.db"))
|
||||
.await
|
||||
.expect("db");
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||
(1, 'https://example.com/1', 'Rejected', '2026-09-02T00:00:00Z'),
|
||||
(2, 'https://example.com/2', 'Auto', '2026-09-02T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("articles");
|
||||
sqlx::query(
|
||||
"INSERT INTO article_assessments
|
||||
(article_id, stage, model, prompt_version, score, assessed_at) VALUES
|
||||
(1, 'triage', 'model', 1, 2.0, '2026-09-02T04:00:00Z'),
|
||||
(2, 'triage', 'model', 1, 1.0, '2026-09-02T04:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("assessments");
|
||||
let run_id = db
|
||||
.start_run(
|
||||
"2026-09-02".parse().expect("date"),
|
||||
"2026-09-02T05:30:00Z".parse().expect("timestamp"),
|
||||
)
|
||||
.await
|
||||
.expect("run");
|
||||
let config = CurationConfig {
|
||||
always_include_feeds: vec!["99".into()],
|
||||
..CurationConfig::default()
|
||||
};
|
||||
let normal = article(1, "Rejected", 500);
|
||||
let mut auto = article(2, "Auto", 500);
|
||||
auto.feed_id = 99;
|
||||
let eligible = hygiene(
|
||||
&db,
|
||||
run_id,
|
||||
vec![normal, auto],
|
||||
"2026-09-02".parse().expect("date"),
|
||||
&config,
|
||||
"2026-09-02T05:30:00Z".parse().expect("timestamp"),
|
||||
)
|
||||
.await
|
||||
.expect("hygiene");
|
||||
assert_eq!(eligible.len(), 1);
|
||||
assert_eq!(eligible[0].article.id, 2);
|
||||
assert!(eligible[0].auto_include);
|
||||
let reason: String = sqlx::query_scalar(
|
||||
"SELECT excluded_reason FROM candidate_runs WHERE run_id = ? AND article_id = 1",
|
||||
)
|
||||
.bind(run_id)
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.expect("thin row");
|
||||
assert_eq!(reason, "recently_rejected");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_rejected_rows_do_not_mark_an_article_recently_rejected() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = Db::open_and_migrate(&dir.path().join("hygiene.db"))
|
||||
.await
|
||||
.expect("db");
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||
(1, 'https://example.com/1', 'Refused', '2026-09-02T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("article");
|
||||
sqlx::query(
|
||||
"INSERT INTO article_assessments
|
||||
(article_id, stage, model, prompt_version, score, fit, kind, rationale, assessed_at)
|
||||
VALUES (1, 'triage', 'model', 1, NULL, NULL, 'provider_rejected',
|
||||
'deepseek: Content Exists Risk', '2026-09-02T04:00:00Z'),
|
||||
(1, 'deep', 'model', 1, NULL, NULL, 'provider_rejected',
|
||||
'deepseek: Content Exists Risk', '2026-09-02T04:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("rejection rows");
|
||||
let run_id = db
|
||||
.start_run(
|
||||
"2026-09-02".parse().expect("date"),
|
||||
"2026-09-02T05:30:00Z".parse().expect("timestamp"),
|
||||
)
|
||||
.await
|
||||
.expect("run");
|
||||
let eligible = hygiene(
|
||||
&db,
|
||||
run_id,
|
||||
vec![article(1, "Refused", 500)],
|
||||
"2026-09-02".parse().expect("date"),
|
||||
&CurationConfig::default(),
|
||||
"2026-09-02T05:30:00Z".parse().expect("timestamp"),
|
||||
)
|
||||
.await
|
||||
.expect("hygiene");
|
||||
assert_eq!(eligible.len(), 1, "a NULL score is not a low score");
|
||||
assert_eq!(eligible[0].article.id, 1);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,839 @@
|
||||
//! Bisecting batch runner shared by triage and deep assessment.
|
||||
//!
|
||||
//! DeepSeek's input content filter rejects a whole request when any one
|
||||
//! article in the batch trips it (`400 Content Exists Risk`) and does not say
|
||||
//! which. Sending the batch once and giving up cost every other article its
|
||||
//! assessment and retried them all the next day. Instead, a batch that fails
|
||||
//! with a non-transient error is split in half and both halves are sent
|
||||
//! again, down to single articles. A single article that still fails is
|
||||
//! "rejected": it is retried once on the editor client when that runs on a
|
||||
//! different provider, and otherwise reported so the caller can persist a
|
||||
//! `provider_rejected` row and stop asking (§10, §12.1, §17).
|
||||
//!
|
||||
//! Transient errors are retried inside [`LlmClient::complete`] and, once
|
||||
//! exhausted, leave the batch unassessed as before; a budget trip stops the
|
||||
//! stage. Concurrency is the caller's `max_concurrent_requests` across the
|
||||
//! original batches; the bisection inside one batch runs sequentially.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use futures::{StreamExt, stream};
|
||||
|
||||
use super::llm::{LlmClient, LlmError};
|
||||
use crate::types::{ArticleId, Candidate};
|
||||
|
||||
/// Longest rejection message kept in `article_assessments.rationale`.
|
||||
pub const REJECTION_MESSAGE_CHARS: usize = 200;
|
||||
|
||||
/// A parsed per-article item the runner can attribute to a candidate.
|
||||
pub trait Assessed {
|
||||
fn article_id(&self) -> ArticleId;
|
||||
}
|
||||
|
||||
/// One stage's prompt builder, parser and clients.
|
||||
pub struct BatchRunner<'a, T> {
|
||||
/// The bulk client every batch goes to first.
|
||||
pub llm: &'a LlmClient,
|
||||
/// The editor client, tried once per rejected article when it runs on a
|
||||
/// different provider than `llm` and its meter is not tripped.
|
||||
pub fallback: Option<&'a LlmClient>,
|
||||
pub temperature: f32,
|
||||
pub build_prompt: &'a (dyn Fn(&[&Candidate]) -> String + Sync),
|
||||
pub parse: &'a (dyn Fn(&str) -> Vec<T> + Sync),
|
||||
}
|
||||
|
||||
impl<T> BatchRunner<'_, T> {
|
||||
/// The editor client when it is a real alternative to the bulk one.
|
||||
pub fn usable_fallback(&self) -> Option<&LlmClient> {
|
||||
self.fallback
|
||||
.filter(|fallback| fallback.provider() != self.llm.provider())
|
||||
.filter(|fallback| fallback.meter.check_budget().is_ok())
|
||||
}
|
||||
|
||||
/// The name of the provider rejected articles are retried on, if any.
|
||||
pub fn fallback_provider(&self) -> Option<String> {
|
||||
self.fallback
|
||||
.filter(|fallback| fallback.provider() != self.llm.provider())
|
||||
.map(|fallback| fallback.provider().to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// An item together with the model that produced it (bulk or editor).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Scored<T> {
|
||||
pub item: T,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
/// A single article both the bulk provider and the fallback refused.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Rejection {
|
||||
pub id: ArticleId,
|
||||
/// The bulk provider's name.
|
||||
pub provider: String,
|
||||
/// The bulk provider's message, at most [`REJECTION_MESSAGE_CHARS`] long.
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl Rejection {
|
||||
/// `<provider>: <message>`, the `rationale` of a `provider_rejected` row.
|
||||
pub fn rationale(&self) -> String {
|
||||
format!("{}: {}", self.provider, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
/// What a set of batches produced.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct BatchOutcome<T> {
|
||||
pub items: Vec<Scored<T>>,
|
||||
/// Articles nobody would assess; the caller persists these.
|
||||
pub rejections: Vec<Rejection>,
|
||||
/// Single articles the bulk provider rejected, recovered or not.
|
||||
pub rejected: usize,
|
||||
/// Of `rejected`, those the fallback client assessed.
|
||||
pub recovered: usize,
|
||||
/// Requests made on the bulk client.
|
||||
pub requests: usize,
|
||||
/// True when the bulk budget tripped and work was left undone.
|
||||
pub budget_stopped: bool,
|
||||
}
|
||||
|
||||
impl<T> Default for BatchOutcome<T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
items: Vec::new(),
|
||||
rejections: Vec::new(),
|
||||
rejected: 0,
|
||||
recovered: 0,
|
||||
requests: 0,
|
||||
budget_stopped: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> BatchOutcome<T> {
|
||||
fn absorb(&mut self, other: Self) {
|
||||
self.items.extend(other.items);
|
||||
self.rejections.extend(other.rejections);
|
||||
self.rejected += other.rejected;
|
||||
self.recovered += other.recovered;
|
||||
self.requests += other.requests;
|
||||
self.budget_stopped |= other.budget_stopped;
|
||||
}
|
||||
}
|
||||
|
||||
/// The per-stage totals behind the `triage:` / `assess:` log line and the
|
||||
/// run report's `*_reused` / `*_rejected` counts.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct StageSummary {
|
||||
pub stage: &'static str,
|
||||
/// Articles the stage was responsible for this run.
|
||||
pub pool: usize,
|
||||
/// Served from cached rows within `assessment_reuse_days`.
|
||||
pub reused: usize,
|
||||
/// Skipped because a `provider_rejected` row is still fresh.
|
||||
pub known_rejected: usize,
|
||||
/// Sent to the bulk provider.
|
||||
pub requested: usize,
|
||||
/// Original batches before any bisection.
|
||||
pub batches: usize,
|
||||
/// Assessments written this run (bulk and editor).
|
||||
pub applied: usize,
|
||||
/// Single articles the bulk provider rejected this run.
|
||||
pub rejected: usize,
|
||||
/// Of `rejected`, those the editor client assessed instead.
|
||||
pub recovered: usize,
|
||||
pub fallback_provider: Option<String>,
|
||||
}
|
||||
|
||||
impl StageSummary {
|
||||
/// Articles carrying this stage's assessment after the run.
|
||||
pub fn assessed(&self) -> usize {
|
||||
self.reused + self.applied
|
||||
}
|
||||
|
||||
/// Articles without an assessment because a provider rejected them:
|
||||
/// this run's unrecovered rejections plus the cached ones.
|
||||
pub fn rejected_total(&self) -> usize {
|
||||
self.known_rejected + self.rejected.saturating_sub(self.recovered)
|
||||
}
|
||||
|
||||
/// `triage: 398 in pool · 210 reused · 188 requested in 8 batches · 3 rejected (2 recovered on gemini)`
|
||||
pub fn info_line(&self) -> String {
|
||||
let mut line = format!(
|
||||
"{}: {} in pool · {} reused · {} requested in {} batches · {} rejected",
|
||||
self.stage, self.pool, self.reused, self.requested, self.batches, self.rejected
|
||||
);
|
||||
if self.recovered > 0 {
|
||||
line.push_str(&format!(
|
||||
" ({} recovered on {})",
|
||||
self.recovered,
|
||||
self.fallback_provider.as_deref().unwrap_or("editor")
|
||||
));
|
||||
}
|
||||
if self.known_rejected > 0 {
|
||||
line.push_str(&format!(" · {} known rejected", self.known_rejected));
|
||||
}
|
||||
line
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors the provider will keep returning for the same input.
|
||||
fn is_rejection(error: &LlmError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
LlmError::Api { .. } | LlmError::Refusal { .. } | LlmError::EmptyResponse { .. }
|
||||
)
|
||||
}
|
||||
|
||||
/// The provider's own words, without the `<provider> request failed:` prefix.
|
||||
fn rejection_message(error: &LlmError) -> String {
|
||||
match error {
|
||||
LlmError::Api { message, .. } => message.clone(),
|
||||
LlmError::Refusal { .. } => "returned a refusal".into(),
|
||||
LlmError::EmptyResponse { .. } => "returned an empty completion".into(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_chars(text: &str, max_chars: usize) -> String {
|
||||
let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if collapsed.chars().count() <= max_chars {
|
||||
return collapsed;
|
||||
}
|
||||
let mut cut = collapsed
|
||||
.chars()
|
||||
.take(max_chars.saturating_sub(1))
|
||||
.collect::<String>();
|
||||
cut.push('…');
|
||||
cut
|
||||
}
|
||||
|
||||
enum Failure {
|
||||
Rejected(String),
|
||||
NoItems,
|
||||
}
|
||||
|
||||
struct Work<'c> {
|
||||
batch: Vec<&'c Candidate>,
|
||||
/// A batch that parses to zero items is split once; its halves are not.
|
||||
zero_split: bool,
|
||||
}
|
||||
|
||||
/// Run every batch on the bulk client, `max_concurrent` in flight.
|
||||
pub async fn run_batches<T: Assessed>(
|
||||
runner: &BatchRunner<'_, T>,
|
||||
batches: Vec<Vec<&Candidate>>,
|
||||
max_concurrent: usize,
|
||||
) -> BatchOutcome<T> {
|
||||
let outcomes = stream::iter(batches)
|
||||
.map(|batch| run_one(runner, batch))
|
||||
.buffer_unordered(max_concurrent.max(1))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let mut merged = BatchOutcome::default();
|
||||
for outcome in outcomes {
|
||||
merged.absorb(outcome);
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
/// One original batch, bisected as far as it needs to be.
|
||||
async fn run_one<T: Assessed>(
|
||||
runner: &BatchRunner<'_, T>,
|
||||
batch: Vec<&Candidate>,
|
||||
) -> BatchOutcome<T> {
|
||||
let original = batch.len();
|
||||
let provider = runner.llm.provider();
|
||||
let mut out = BatchOutcome::default();
|
||||
let mut warned = false;
|
||||
// Explicit LIFO stack instead of recursion: the left half is pushed last
|
||||
// so it runs first, which keeps the request order predictable.
|
||||
let mut stack = vec![Work {
|
||||
batch,
|
||||
zero_split: true,
|
||||
}];
|
||||
while let Some(work) = stack.pop() {
|
||||
if let Err(error) = runner.llm.meter.check_budget() {
|
||||
tracing::warn!(%error, "bulk budget tripped; skipping the rest of the batch");
|
||||
out.budget_stopped = true;
|
||||
return out;
|
||||
}
|
||||
let size = work.batch.len();
|
||||
let prompt = (runner.build_prompt)(&work.batch);
|
||||
let allowed = work
|
||||
.batch
|
||||
.iter()
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<HashSet<_>>();
|
||||
out.requests += 1;
|
||||
let failure = match runner.llm.complete(&prompt, runner.temperature, true).await {
|
||||
Ok(raw) => {
|
||||
let items = (runner.parse)(&raw)
|
||||
.into_iter()
|
||||
.filter(|item| allowed.contains(&item.article_id()))
|
||||
.collect::<Vec<_>>();
|
||||
if items.is_empty() {
|
||||
Failure::NoItems
|
||||
} else {
|
||||
out.items.extend(items.into_iter().map(|item| Scored {
|
||||
item,
|
||||
model: runner.llm.model.clone(),
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(LlmError::BudgetExceeded { spent, limit }) => {
|
||||
tracing::warn!(
|
||||
spent,
|
||||
limit,
|
||||
"bulk budget tripped; skipping the rest of the batch"
|
||||
);
|
||||
out.budget_stopped = true;
|
||||
return out;
|
||||
}
|
||||
Err(error) if is_rejection(&error) => Failure::Rejected(rejection_message(&error)),
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, size, "batch failed; its articles remain unassessed");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if size == 1 {
|
||||
let candidate = work.batch[0];
|
||||
let message = match failure {
|
||||
Failure::Rejected(message) => message,
|
||||
Failure::NoItems => "returned no assessment for the article".into(),
|
||||
};
|
||||
resolve_single(runner, candidate, &prompt, message, &mut out).await;
|
||||
continue;
|
||||
}
|
||||
match failure {
|
||||
Failure::Rejected(message) => {
|
||||
if !warned {
|
||||
tracing::warn!(
|
||||
provider,
|
||||
message = %truncate_chars(&message, REJECTION_MESSAGE_CHARS),
|
||||
"batch of {original} rejected by {provider}; bisecting"
|
||||
);
|
||||
warned = true;
|
||||
}
|
||||
push_halves(&mut stack, work.batch, work.zero_split);
|
||||
}
|
||||
Failure::NoItems if work.zero_split => {
|
||||
tracing::warn!(provider, size, "batch parsed to zero items; bisecting once");
|
||||
push_halves(&mut stack, work.batch, false);
|
||||
}
|
||||
Failure::NoItems => {
|
||||
tracing::warn!(
|
||||
provider,
|
||||
size,
|
||||
"half batch parsed to zero items again; its articles remain unassessed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn push_halves<'c>(stack: &mut Vec<Work<'c>>, batch: Vec<&'c Candidate>, zero_split: bool) {
|
||||
let mut left = batch;
|
||||
let right = left.split_off(left.len() / 2);
|
||||
stack.push(Work {
|
||||
batch: right,
|
||||
zero_split,
|
||||
});
|
||||
stack.push(Work {
|
||||
batch: left,
|
||||
zero_split,
|
||||
});
|
||||
}
|
||||
|
||||
/// A single article the bulk provider would not assess: try the editor
|
||||
/// client once, else report it as rejected.
|
||||
async fn resolve_single<T: Assessed>(
|
||||
runner: &BatchRunner<'_, T>,
|
||||
candidate: &Candidate,
|
||||
prompt: &str,
|
||||
message: String,
|
||||
out: &mut BatchOutcome<T>,
|
||||
) {
|
||||
let id = candidate.article.id;
|
||||
let provider = runner.llm.provider();
|
||||
out.rejected += 1;
|
||||
if let Some(fallback) = runner.usable_fallback() {
|
||||
let fallback_provider = fallback.provider();
|
||||
match fallback.complete(prompt, runner.temperature, true).await {
|
||||
Ok(raw) => {
|
||||
if let Some(item) = (runner.parse)(&raw)
|
||||
.into_iter()
|
||||
.find(|item| item.article_id() == id)
|
||||
{
|
||||
tracing::info!(
|
||||
article_id = id,
|
||||
provider = fallback_provider,
|
||||
"article rejected by {provider}; assessed on {fallback_provider} instead"
|
||||
);
|
||||
out.items.push(Scored {
|
||||
item,
|
||||
model: fallback.model.clone(),
|
||||
});
|
||||
out.recovered += 1;
|
||||
return;
|
||||
}
|
||||
tracing::warn!(
|
||||
article_id = id,
|
||||
provider = fallback_provider,
|
||||
"fallback returned no assessment for the rejected article"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
article_id = id,
|
||||
provider = fallback_provider,
|
||||
%error,
|
||||
"fallback failed for the rejected article"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let message = truncate_chars(&message, REJECTION_MESSAGE_CHARS);
|
||||
tracing::warn!(
|
||||
article_id = id,
|
||||
provider,
|
||||
%message,
|
||||
"article rejected by the provider; recorded so it is not retried"
|
||||
);
|
||||
out.rejections.push(Rejection {
|
||||
id,
|
||||
provider: provider.to_string(),
|
||||
message,
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::config::ProviderConfig;
|
||||
use crate::curate::llm::{
|
||||
ChatBackend, ChatCompletion, ChatRequest, MockBackend, PriceTable, UsageMeter,
|
||||
};
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::curate::triage::{TriageItem, build_batch_prompt, parse_triage_response};
|
||||
use crate::http::RetryPolicy;
|
||||
use crate::types::TokenUsage;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
/// A backend that answers for every id it finds in the prompt unless the
|
||||
/// prompt mentions a forbidden title, in which case it rejects the whole
|
||||
/// request the way DeepSeek's content filter does.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct FilterBackend {
|
||||
forbidden: Vec<String>,
|
||||
/// Prompts answered with `{"articles": []}` (by request index).
|
||||
empty_on: Vec<usize>,
|
||||
answer: fn(&[i64]) -> String,
|
||||
seen: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl FilterBackend {
|
||||
/// Answers in the triage shape.
|
||||
pub(crate) fn new(forbidden: &[&str], empty_on: &[usize]) -> Arc<Self> {
|
||||
Self::with_answer(forbidden, empty_on, answer_for)
|
||||
}
|
||||
|
||||
/// Answers in the deep-assessment shape.
|
||||
pub(crate) fn deep(forbidden: &[&str]) -> Arc<Self> {
|
||||
Self::with_answer(forbidden, &[], deep_answer_for)
|
||||
}
|
||||
|
||||
fn with_answer(
|
||||
forbidden: &[&str],
|
||||
empty_on: &[usize],
|
||||
answer: fn(&[i64]) -> String,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
forbidden: forbidden.iter().map(|s| s.to_string()).collect(),
|
||||
empty_on: empty_on.to_vec(),
|
||||
answer,
|
||||
seen: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn calls(&self) -> usize {
|
||||
self.seen.lock().map(|seen| seen.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The prompt of the first request that carried only `id`.
|
||||
pub(crate) fn prompts_for_single(&self, id: i64) -> String {
|
||||
self.seen
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|seen| {
|
||||
seen.iter()
|
||||
.find(|prompt| ids_in(prompt) == vec![id])
|
||||
.cloned()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The ids in each request, in order.
|
||||
pub(crate) fn requests(&self) -> Vec<Vec<i64>> {
|
||||
self.seen
|
||||
.lock()
|
||||
.map(|seen| seen.iter().map(|prompt| ids_in(prompt)).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
fn ids_in(prompt: &str) -> Vec<i64> {
|
||||
prompt
|
||||
.lines()
|
||||
.filter_map(|line| line.strip_prefix("--- id: "))
|
||||
.filter_map(|id| id.trim().parse().ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A triage answer for `ids`.
|
||||
pub(crate) fn answer_for(ids: &[i64]) -> String {
|
||||
let items = ids
|
||||
.iter()
|
||||
.map(|id| format!(r#"{{"id":{id},"interest":6,"kind":"essay","why":"fine"}}"#))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(r#"{{"articles":[{items}]}}"#)
|
||||
}
|
||||
|
||||
/// A deep-assessment answer for `ids`.
|
||||
pub(crate) fn deep_answer_for(ids: &[i64]) -> String {
|
||||
let items = ids
|
||||
.iter()
|
||||
.map(|id| {
|
||||
format!(
|
||||
r#"{{"id":{id},"quality":7,"fit":6,"category":"Top Stories","rationale":"fine","facets":{{"format":"analysis_essay"}}}}"#
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(r#"{{"articles":[{items}]}}"#)
|
||||
}
|
||||
|
||||
impl ChatBackend for FilterBackend {
|
||||
fn complete<'a>(
|
||||
&'a self,
|
||||
req: ChatRequest,
|
||||
) -> Pin<Box<dyn Future<Output = Result<ChatCompletion, LlmError>> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
let index = self.calls();
|
||||
if let Ok(mut seen) = self.seen.lock() {
|
||||
seen.push(req.user.clone());
|
||||
}
|
||||
if self.forbidden.iter().any(|title| req.user.contains(title)) {
|
||||
return Err(LlmError::api(
|
||||
"deepseek",
|
||||
"400 Bad Request: {\"error\":{\"message\":\"Content Exists Risk\"}}",
|
||||
));
|
||||
}
|
||||
let content = if self.empty_on.contains(&index) {
|
||||
r#"{"articles":[]}"#.to_string()
|
||||
} else {
|
||||
(self.answer)(&ids_in(&req.user))
|
||||
};
|
||||
Ok(ChatCompletion {
|
||||
content,
|
||||
usage: TokenUsage::default(),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn client(provider: &str, model: &str, backend: Arc<dyn ChatBackend>) -> LlmClient {
|
||||
LlmClient::with_backend_options(
|
||||
provider,
|
||||
model,
|
||||
"SYSTEM".into(),
|
||||
None,
|
||||
UsageMeter::with_prices(PriceTable::from(&ProviderConfig::deepseek()), 10.0),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
fn candidates(n: i64) -> Vec<Candidate> {
|
||||
(1..=n)
|
||||
.map(|id| Candidate::new(article(id, &format!("Title number {id}"), 800), false))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn run(
|
||||
llm: &LlmClient,
|
||||
fallback: Option<&LlmClient>,
|
||||
candidates: &[Candidate],
|
||||
batch_size: usize,
|
||||
) -> BatchOutcome<TriageItem> {
|
||||
let runner = BatchRunner {
|
||||
llm,
|
||||
fallback,
|
||||
temperature: 0.3,
|
||||
build_prompt: &build_batch_prompt,
|
||||
parse: &parse_triage_response,
|
||||
};
|
||||
let all = candidates.iter().collect::<Vec<_>>();
|
||||
let batches = all.chunks(batch_size).map(<[&Candidate]>::to_vec).collect();
|
||||
run_batches(&runner, batches, 4).await
|
||||
}
|
||||
|
||||
fn sorted_ids(outcome: &BatchOutcome<TriageItem>) -> Vec<i64> {
|
||||
let mut ids = outcome
|
||||
.items
|
||||
.iter()
|
||||
.map(|scored| scored.item.id)
|
||||
.collect::<Vec<_>>();
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn one_bad_article_in_four_costs_five_requests_and_one_rejection() {
|
||||
let backend = FilterBackend::new(&["Title number 3"], &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let candidates = candidates(4);
|
||||
let outcome = run(&llm, None, &candidates, 4).await;
|
||||
|
||||
// [1,2,3,4] is rejected → split. [1,2] succeeds. [3,4] is rejected →
|
||||
// split. [3] alone is rejected → recorded. [4] succeeds. 1 + 2 + 2 = 5.
|
||||
assert_eq!(
|
||||
backend.requests(),
|
||||
vec![vec![1, 2, 3, 4], vec![1, 2], vec![3, 4], vec![3], vec![4]]
|
||||
);
|
||||
assert_eq!(outcome.requests, 5);
|
||||
assert_eq!(sorted_ids(&outcome), vec![1, 2, 4]);
|
||||
assert!(outcome.items.iter().all(|s| s.model == "deepseek-v4-flash"));
|
||||
assert_eq!(outcome.rejected, 1);
|
||||
assert_eq!(outcome.recovered, 0);
|
||||
assert_eq!(outcome.rejections.len(), 1);
|
||||
let rejection = &outcome.rejections[0];
|
||||
assert_eq!(rejection.id, 3);
|
||||
assert_eq!(rejection.provider, "deepseek");
|
||||
assert!(rejection.message.contains("Content Exists Risk"));
|
||||
assert!(
|
||||
rejection
|
||||
.rationale()
|
||||
.starts_with("deepseek: 400 Bad Request"),
|
||||
"{}",
|
||||
rejection.rationale()
|
||||
);
|
||||
assert!(!outcome.budget_stopped);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_article_rejected_gives_every_article_a_rejection_in_2n_minus_1_requests() {
|
||||
let titles = (1..=8)
|
||||
.map(|id| format!("Title number {id}"))
|
||||
.collect::<Vec<_>>();
|
||||
let backend =
|
||||
FilterBackend::new(&titles.iter().map(String::as_str).collect::<Vec<_>>(), &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let candidates = candidates(8);
|
||||
let outcome = run(&llm, None, &candidates, 8).await;
|
||||
assert_eq!(backend.calls(), 2 * 8 - 1);
|
||||
assert_eq!(outcome.requests, 15);
|
||||
assert!(outcome.items.is_empty());
|
||||
let mut rejected = outcome
|
||||
.rejections
|
||||
.iter()
|
||||
.map(|rejection| rejection.id)
|
||||
.collect::<Vec<_>>();
|
||||
rejected.sort_unstable();
|
||||
assert_eq!(rejected, (1..=8).collect::<Vec<_>>());
|
||||
assert_eq!(outcome.rejected, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_rejected_article_is_recovered_on_an_editor_of_another_provider() {
|
||||
let backend = FilterBackend::new(&["Title number 2"], &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let editor_backend = Arc::new(MockBackend::new());
|
||||
editor_backend.push(answer_for(&[2]), TokenUsage::default());
|
||||
let editor = client("gemini", "gemini-3.8-flash", editor_backend.clone());
|
||||
let candidates = candidates(2);
|
||||
let outcome = run(&llm, Some(&editor), &candidates, 2).await;
|
||||
assert_eq!(backend.requests(), vec![vec![1, 2], vec![1], vec![2]]);
|
||||
assert_eq!(editor_backend.calls(), 1);
|
||||
let prompt = editor_backend.prompts()[0].clone();
|
||||
assert_eq!(prompt.system.as_str(), "SYSTEM", "same system prompt");
|
||||
assert!(prompt.user.contains("--- id: 2\n"), "same article prompt");
|
||||
assert_eq!(sorted_ids(&outcome), vec![1, 2]);
|
||||
let recovered = outcome
|
||||
.items
|
||||
.iter()
|
||||
.find(|scored| scored.item.id == 2)
|
||||
.expect("recovered");
|
||||
assert_eq!(recovered.model, "gemini-3.8-flash");
|
||||
assert!(outcome.rejections.is_empty(), "no rejection row");
|
||||
assert_eq!((outcome.rejected, outcome.recovered), (1, 1));
|
||||
|
||||
// The same provider name is no alternative: no fallback attempt.
|
||||
let backend = FilterBackend::new(&["Title number 2"], &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let same_backend = Arc::new(MockBackend::new());
|
||||
same_backend.push(answer_for(&[2]), TokenUsage::default());
|
||||
let same = client("deepseek", "deepseek-v4-flash", same_backend.clone());
|
||||
let outcome = run(&llm, Some(&same), &candidates, 2).await;
|
||||
assert_eq!(same_backend.calls(), 0);
|
||||
assert_eq!(outcome.rejections.len(), 1);
|
||||
assert_eq!(outcome.recovered, 0);
|
||||
|
||||
// A tripped editor meter is skipped too, and a failing editor still
|
||||
// yields a rejection with the bulk provider's message.
|
||||
let backend = FilterBackend::new(&["Title number 2"], &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let tripped_backend = Arc::new(MockBackend::new());
|
||||
let tripped = client("gemini", "gemini-3.8-flash", tripped_backend.clone());
|
||||
tripped.meter.preload_cost(100.0);
|
||||
let outcome = run(&llm, Some(&tripped), &candidates, 2).await;
|
||||
assert_eq!(tripped_backend.calls(), 0);
|
||||
assert_eq!(outcome.rejections.len(), 1);
|
||||
|
||||
let backend = FilterBackend::new(&["Title number 2"], &[]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let failing_backend = Arc::new(MockBackend::new());
|
||||
failing_backend.push_llm_error(LlmError::refusal("gemini"));
|
||||
let failing = client("gemini", "gemini-3.8-flash", failing_backend.clone());
|
||||
let outcome = run(&llm, Some(&failing), &candidates, 2).await;
|
||||
assert_eq!(failing_backend.calls(), 1);
|
||||
assert_eq!(outcome.rejections.len(), 1);
|
||||
assert_eq!(outcome.rejections[0].provider, "deepseek");
|
||||
assert!(
|
||||
outcome.rejections[0]
|
||||
.message
|
||||
.contains("Content Exists Risk")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_items_bisect_once_and_a_single_zero_is_a_rejection() {
|
||||
// Batch of 4 parses to zero items → its two halves are sent; both fine.
|
||||
let backend = FilterBackend::new(&[], &[0]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let candidates = candidates(4);
|
||||
let outcome = run(&llm, None, &candidates, 4).await;
|
||||
assert_eq!(
|
||||
backend.requests(),
|
||||
vec![vec![1, 2, 3, 4], vec![1, 2], vec![3, 4]]
|
||||
);
|
||||
assert_eq!(sorted_ids(&outcome), vec![1, 2, 3, 4]);
|
||||
assert!(outcome.rejections.is_empty());
|
||||
|
||||
// A half that parses to zero items again is not split further: its
|
||||
// articles simply stay unassessed for this run.
|
||||
let backend = FilterBackend::new(&[], &[0, 1]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let outcome = run(&llm, None, &candidates, 4).await;
|
||||
assert_eq!(backend.calls(), 3);
|
||||
assert_eq!(sorted_ids(&outcome), vec![3, 4]);
|
||||
assert!(outcome.rejections.is_empty());
|
||||
|
||||
// A single article that parses to zero items is a rejection.
|
||||
let backend = FilterBackend::new(&[], &[0]);
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
let outcome = run(&llm, None, &candidates[..1], 1).await;
|
||||
assert_eq!(backend.calls(), 1);
|
||||
assert!(outcome.items.is_empty());
|
||||
assert_eq!(outcome.rejections.len(), 1);
|
||||
assert_eq!(
|
||||
outcome.rejections[0].rationale(),
|
||||
"deepseek: returned no assessment for the article"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_and_budget_failures_are_not_bisected() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
for _ in 0..3 {
|
||||
backend.push_llm_error(LlmError::Transient {
|
||||
provider: "deepseek".into(),
|
||||
message: "503".into(),
|
||||
});
|
||||
}
|
||||
let llm =
|
||||
client("deepseek", "deepseek-v4-flash", backend.clone()).with_retry(RetryPolicy {
|
||||
max_attempts: 3,
|
||||
base_delay: Duration::from_millis(1),
|
||||
max_delay: Duration::from_millis(2),
|
||||
});
|
||||
let candidates = candidates(4);
|
||||
// Transient errors are retried inside the client and, once exhausted,
|
||||
// leave the batch unassessed without bisecting it.
|
||||
let outcome = run(&llm, None, &candidates, 4).await;
|
||||
assert_eq!(backend.calls(), 3, "three attempts, one batch");
|
||||
assert!(outcome.items.is_empty());
|
||||
assert!(
|
||||
outcome.rejections.is_empty(),
|
||||
"transient is not a rejection"
|
||||
);
|
||||
assert_eq!(outcome.rejected, 0);
|
||||
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
let llm = client("deepseek", "deepseek-v4-flash", backend.clone());
|
||||
llm.meter.preload_cost(100.0);
|
||||
let outcome = run(&llm, None, &candidates, 2).await;
|
||||
assert_eq!(backend.calls(), 0);
|
||||
assert!(outcome.budget_stopped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summary_line_matches_the_documented_shape() {
|
||||
let summary = StageSummary {
|
||||
stage: "triage",
|
||||
pool: 398,
|
||||
reused: 210,
|
||||
known_rejected: 0,
|
||||
requested: 188,
|
||||
batches: 8,
|
||||
applied: 187,
|
||||
rejected: 3,
|
||||
recovered: 2,
|
||||
fallback_provider: Some("gemini".into()),
|
||||
};
|
||||
assert_eq!(
|
||||
summary.info_line(),
|
||||
"triage: 398 in pool · 210 reused · 188 requested in 8 batches · 3 rejected (2 recovered on gemini)"
|
||||
);
|
||||
assert_eq!(summary.assessed(), 397);
|
||||
assert_eq!(summary.rejected_total(), 1);
|
||||
let quiet = StageSummary {
|
||||
stage: "assess",
|
||||
pool: 120,
|
||||
reused: 20,
|
||||
known_rejected: 2,
|
||||
requested: 98,
|
||||
batches: 13,
|
||||
applied: 98,
|
||||
..StageSummary::default()
|
||||
};
|
||||
assert_eq!(
|
||||
quiet.info_line(),
|
||||
"assess: 120 in pool · 20 reused · 98 requested in 13 batches · 0 rejected · 2 known rejected"
|
||||
);
|
||||
assert_eq!(quiet.rejected_total(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejection_messages_are_short_and_single_line() {
|
||||
let long = "x".repeat(500);
|
||||
let cut = truncate_chars(&long, REJECTION_MESSAGE_CHARS);
|
||||
assert_eq!(cut.chars().count(), REJECTION_MESSAGE_CHARS);
|
||||
assert!(cut.ends_with('…'));
|
||||
assert_eq!(truncate_chars("a\n b", 10), "a b");
|
||||
assert_eq!(
|
||||
rejection_message(&LlmError::refusal("anthropic")),
|
||||
"returned a refusal"
|
||||
);
|
||||
assert_eq!(
|
||||
rejection_message(&LlmError::api("deepseek", "400: nope")),
|
||||
"400: nope"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
+506
-319
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+1169
-214
File diff suppressed because it is too large.
Load diff
+106
-107
@@ -1,154 +1,153 @@
|
||||
//! Curation pipeline: pre-filter → LLM scoring → selection → editorial (spec §3.5, §3.6).
|
||||
//! Personalized curation: signals → triage → admission → assessment → editor.
|
||||
//!
|
||||
//! ```text
|
||||
//! ~400 articles ─prefilter─▶ ~120 candidates ─stage A─▶ scored ─stage B─▶ lineup ─stage C─▶ editorial
|
||||
//! ~400 eligible ─triage─▶ admission (120) ─assess/rank─▶ shortlist (60) ─editor─▶ editorial
|
||||
//! ```
|
||||
//!
|
||||
//! [`Curator`] is the thin orchestration layer the `generate` pipeline calls; the
|
||||
//! interesting logic lives in the stage modules. Every stage is safe to run with
|
||||
//! `llm == None` (`--skip-llm`): the prefilter order stands in for selection and
|
||||
//! feed excerpts stand in for summaries (notes §6).
|
||||
//! no provider at all (`--skip-llm`): the cheap-signal blend stands in for selection
|
||||
//! and feed excerpts stand in for summaries (notes §6). Scoring runs on the bulk
|
||||
//! client; selection and editorial on the editor with per-call bulk fallback.
|
||||
|
||||
pub mod admit;
|
||||
pub mod assess;
|
||||
pub mod batch;
|
||||
pub mod editor;
|
||||
pub mod editorial;
|
||||
pub mod embedding;
|
||||
pub mod llm;
|
||||
pub mod prefilter;
|
||||
pub mod profile;
|
||||
pub mod score;
|
||||
pub mod select;
|
||||
pub mod rank;
|
||||
pub mod signals;
|
||||
pub mod telemetry;
|
||||
pub mod triage;
|
||||
|
||||
use jiff::civil::Date;
|
||||
use jiff::{Timestamp, civil::Date};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::Db;
|
||||
use crate::types::{Article, Editorial, Lineup, ScoredArticle};
|
||||
use crate::types::{Candidate, Editorial, Lineup};
|
||||
|
||||
/// Runs the three curation stages against one day's articles (§3.5, §3.6).
|
||||
/// Runs the LLM curation stages against one day's articles (§12, §13, §14).
|
||||
pub struct Curator {
|
||||
pub config: Config,
|
||||
pub db: Db,
|
||||
pub llm: Option<llm::LlmClient>,
|
||||
pub llms: llm::Llms,
|
||||
}
|
||||
|
||||
impl Curator {
|
||||
/// `llm == None` corresponds to `--skip-llm`: prefilter order is used for
|
||||
/// selection and feed excerpts stand in for summaries (notes §6).
|
||||
pub fn new(config: Config, db: Db, llm: Option<llm::LlmClient>) -> Self {
|
||||
Self { config, db, llm }
|
||||
/// An empty [`llm::Llms`] corresponds to `--skip-llm`: cheap-signal order is
|
||||
/// used for selection and feed excerpts stand in for summaries (notes §6).
|
||||
/// With only `bulk`, every editor call runs on the bulk provider (§4.2).
|
||||
pub fn new(config: Config, db: Db, llms: llm::Llms) -> Self {
|
||||
Self { config, db, llms }
|
||||
}
|
||||
|
||||
/// Heuristic pre-filter: 300–500 articles → `prefilter_keep` (§3.5).
|
||||
/// Deep assessment of the admitted set on the bulk client (§12.1), reusing
|
||||
/// cached `article_assessments` rows within `assessment_reuse_days`.
|
||||
///
|
||||
/// Also persists each candidate's `prefilter_score` for the day so that a
|
||||
/// re-run of the same date is idempotent (notes §12).
|
||||
pub async fn prefilter(
|
||||
/// A no-op under `--skip-llm`: like triage, nothing is read or written and
|
||||
/// utility falls back to the present signals (§12.3, §17). When the bulk
|
||||
/// provider is down or its budget trips, cached rows are still reused and
|
||||
/// the failed batches simply stay unassessed. A batch the provider's
|
||||
/// content filter rejects is bisected, and a rejected single article is
|
||||
/// retried on the editor when that is another provider (`curate::batch`).
|
||||
pub async fn assess(
|
||||
&self,
|
||||
articles: Vec<Article>,
|
||||
date: Date,
|
||||
) -> anyhow::Result<Vec<ScoredArticle>> {
|
||||
let span = tracing::info_span!("prefilter", articles = articles.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let ctx = prefilter::PrefilterContext::load(&self.db, date).await?;
|
||||
let candidates = prefilter::run(articles, &ctx, &self.config);
|
||||
for candidate in &candidates {
|
||||
if candidate.article.id == 0 {
|
||||
continue; // not persisted yet (dry run over synthetic articles)
|
||||
}
|
||||
if let Err(e) = self
|
||||
.db
|
||||
.upsert_score(
|
||||
candidate.article.id,
|
||||
date,
|
||||
Some(candidate.prefilter_score),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(article_id = candidate.article.id, error = %e,
|
||||
"could not persist the prefilter score");
|
||||
}
|
||||
}
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
/// Stage A: batched LLM scoring of the surviving candidates (§3.6).
|
||||
///
|
||||
/// A no-op under `--skip-llm`. Scores are persisted per `(article, date)`.
|
||||
pub async fn score(&self, candidates: &mut [ScoredArticle], date: Date) -> anyhow::Result<()> {
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
tracing::info!("--skip-llm: stage A scoring skipped");
|
||||
return Ok(());
|
||||
candidates: &mut [Candidate],
|
||||
rescore: bool,
|
||||
profile_version: Option<i64>,
|
||||
assessed_at: Timestamp,
|
||||
) -> anyhow::Result<batch::StageSummary> {
|
||||
let Some(bulk) = self.llms.bulk.as_ref() else {
|
||||
tracing::info!("--skip-llm: deep assessment skipped");
|
||||
return Ok(batch::StageSummary::default());
|
||||
};
|
||||
let span = tracing::info_span!("llm_score", candidates = candidates.len());
|
||||
let admitted = candidates
|
||||
.iter()
|
||||
.filter(|candidate| candidate.stage == "admitted")
|
||||
.count();
|
||||
let span = tracing::info_span!("llm_assess", candidates = admitted);
|
||||
let _guard = span.enter();
|
||||
|
||||
let scored = score::score_all(
|
||||
llm,
|
||||
assess::run(
|
||||
&self.db,
|
||||
Some(bulk),
|
||||
self.llms.editor.as_ref(),
|
||||
&bulk.model,
|
||||
candidates,
|
||||
self.config.deepseek.score_batch_size,
|
||||
self.config.llm.deep_batch_size,
|
||||
bulk.max_concurrent_requests,
|
||||
self.config.curation.ranking.assessment_reuse_days,
|
||||
rescore,
|
||||
profile_version,
|
||||
assessed_at,
|
||||
self.config.llm.score_temperature,
|
||||
&self.config.curation.sections,
|
||||
self.config.deepseek.score_temperature,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(scored, total = candidates.len(), "stage A complete");
|
||||
|
||||
for candidate in candidates.iter() {
|
||||
if candidate.article.id == 0 {
|
||||
continue;
|
||||
.await
|
||||
}
|
||||
if let Some(llm_score) = candidate.llm.as_ref()
|
||||
&& let Err(e) = self
|
||||
.db
|
||||
.upsert_score(candidate.article.id, date, None, Some(llm_score))
|
||||
|
||||
/// The editor: one call that assembles the issue from the shortlist (§13).
|
||||
pub async fn select(&self, candidates: Vec<Candidate>, date: Date) -> anyhow::Result<Lineup> {
|
||||
let sections = &self.config.curation.sections;
|
||||
let soft_target = self.config.target_article_count;
|
||||
let hard_max = self.config.curation.max_article_count;
|
||||
let span = tracing::info_span!("llm_editor", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
match editor::select(
|
||||
&self.llms,
|
||||
candidates.clone(),
|
||||
sections,
|
||||
soft_target,
|
||||
hard_max,
|
||||
date,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(article_id = candidate.article.id, error = %e,
|
||||
"could not persist the llm score");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stage B: single-call lineup selection into sections (§3.6).
|
||||
pub async fn select(
|
||||
&self,
|
||||
candidates: Vec<ScoredArticle>,
|
||||
date: Date,
|
||||
) -> anyhow::Result<Lineup> {
|
||||
let sections = &self.config.curation.sections;
|
||||
let target = self.config.target_article_count;
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
tracing::info!("--skip-llm: selecting by prefilter order");
|
||||
return Ok(select::select_without_llm(
|
||||
candidates, sections, target, date,
|
||||
));
|
||||
};
|
||||
let span = tracing::info_span!("llm_select", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
match select::select(llm, candidates.clone(), sections, target, date).await {
|
||||
Ok(lineup) => Ok(lineup),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e,
|
||||
"stage B selection failed; falling back to prefilter order");
|
||||
Ok(select::select_without_llm(
|
||||
candidates, sections, target, date,
|
||||
Err(error) => {
|
||||
tracing::error!(%error, "editor and bulk fallback failed; selecting heuristically");
|
||||
Ok(editor::select_without_llm(
|
||||
candidates,
|
||||
sections,
|
||||
soft_target,
|
||||
hard_max,
|
||||
date,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage C: per-article summaries, section intros and the front page (§3.6).
|
||||
/// Stage C: per-article summaries and the Brief (§14).
|
||||
///
|
||||
/// Never fails the run: a budget trip or an API error degrades to excerpts.
|
||||
pub async fn editorial(&self, lineup: &Lineup) -> anyhow::Result<Editorial> {
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
Ok(self.editorial_timed(lineup).await?.0)
|
||||
}
|
||||
|
||||
/// [`Self::editorial`] plus the `summaries` / `brief` stage timings (§15.4).
|
||||
pub async fn editorial_timed(
|
||||
&self,
|
||||
lineup: &Lineup,
|
||||
) -> anyhow::Result<(Editorial, editorial::EditorialTimings)> {
|
||||
if self.llms.editor_or_bulk().is_none() {
|
||||
tracing::info!("--skip-llm: using feed excerpts as summaries");
|
||||
return Ok(editorial::fallback_editorial(lineup));
|
||||
};
|
||||
return Ok((
|
||||
editorial::fallback_editorial(lineup),
|
||||
editorial::EditorialTimings::default(),
|
||||
));
|
||||
}
|
||||
let span = tracing::info_span!("llm_editorial", picks = lineup.picks.len());
|
||||
let _guard = span.enter();
|
||||
Ok(editorial::run(llm, lineup, self.config.deepseek.editorial_temperature).await)
|
||||
Ok(editorial::run_timed(
|
||||
&self.llms,
|
||||
lineup,
|
||||
&self.config.editorial,
|
||||
self.config.llm.editorial_temperature,
|
||||
)
|
||||
.await)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,7 +156,7 @@ impl Curator {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
||||
/// prose. Only used to size prompt budgets (§3.6 stage C).
|
||||
/// prose. Only used to size prompt budgets.
|
||||
pub fn approx_tokens(text: &str) -> usize {
|
||||
text.len().div_ceil(4)
|
||||
}
|
||||
@@ -224,7 +223,7 @@ pub fn truncate_words(text: &str, max_words: usize) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C).
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary.
|
||||
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.len() <= max_chars {
|
||||
|
||||
+52
-512
@@ -1,29 +1,12 @@
|
||||
//! Heuristic pre-filter: 300–500 articles → ~120 candidates (spec §3.5).
|
||||
//! Hygiene matchers and the text-only heuristic used by personalized ranking.
|
||||
//!
|
||||
//! Pure Rust and free: this is what keeps LLM cost flat as feed volume grows.
|
||||
//!
|
||||
//! The 0–100 score is a sum of bounded components so that no single signal can
|
||||
//! dominate, and every component is monotonic in its input:
|
||||
//!
|
||||
//! | component | range | source |
|
||||
//! |---|---|---|
|
||||
//! | long-form word count | 0 … +35 | §3.5 "0 pts <300 words, max at ~2500+" |
|
||||
//! | social proof | 0 … +25 | §3.4 composite, log-scaled again |
|
||||
//! | came via Scour | +8 | §3.5 (already matched a stated interest) |
|
||||
//! | came via HN frontpage | +8 | §3.5 |
|
||||
//! | carried by several feeds | 0 … +8 | §3.2 (multi-source *is* social proof) |
|
||||
//! | feed prior | −12 … +12 | §3.9 beta-smoothed upvote rate, neutral at 0.5 |
|
||||
//! | excerpt only | −20 | §3.5 (penalized, never banned — §7) |
|
||||
//! | roundup/release-notes title | −15 | §3.5 |
|
||||
//! | blocked domain | excluded | §3.5 |
|
||||
//! This module no longer gates the candidate pool. Admission lives in
|
||||
//! `curate::admit`; these helpers remain here because hygiene and cheap signals
|
||||
//! share them (plan §8.1, §9, §18).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use crate::config::CurationConfig;
|
||||
use crate::types::{Article, FeedId};
|
||||
|
||||
use crate::config::{Config, CurationConfig};
|
||||
use crate::types::{Article, ArticleId, FeedId, FeedPrior, ScoredArticle, SourceKind};
|
||||
|
||||
/// Title patterns that mark low-effort posts: link roundups, release notes,
|
||||
/// sponsor posts (§3.5).
|
||||
pub const PENALTY_TITLE_PATTERNS: &[&str] = &[
|
||||
"link roundup",
|
||||
"links for",
|
||||
@@ -49,96 +32,12 @@ pub const PENALTY_TITLE_PATTERNS: &[&str] = &[
|
||||
"digest #",
|
||||
];
|
||||
|
||||
/// Word count at which the long-form bonus saturates (§3.5).
|
||||
pub const LONGFORM_SATURATION_WORDS: i64 = 2500;
|
||||
/// Below this word count the long-form bonus is zero (§3.5).
|
||||
pub const LONGFORM_FLOOR_WORDS: i64 = 300;
|
||||
/// Articles the LLM scored below this within the last week are not re-scored (§3.5).
|
||||
pub const STALE_LOW_SCORE: f64 = 3.0;
|
||||
/// Lookback for the "don't re-score churn" rule (§3.5).
|
||||
pub const STALE_LOOKBACK_DAYS: i64 = 7;
|
||||
|
||||
/// Maximum contribution of each scoring component (§3.5).
|
||||
pub const MAX_LONGFORM_POINTS: f64 = 35.0;
|
||||
pub const MAX_SOCIAL_POINTS: f64 = 25.0;
|
||||
pub const SCOUR_BONUS: f64 = 8.0;
|
||||
pub const HN_FRONTPAGE_BONUS: f64 = 8.0;
|
||||
pub const MAX_MULTI_SOURCE_POINTS: f64 = 8.0;
|
||||
pub const MAX_FEED_PRIOR_POINTS: f64 = 12.0;
|
||||
pub const EXCERPT_ONLY_PENALTY: f64 = 20.0;
|
||||
pub const ROUNDUP_TITLE_PENALTY: f64 = 15.0;
|
||||
|
||||
/// `composite_social_score` value that earns the full social bonus. Empirically
|
||||
/// ~6.0 is a 1,000-point HN story with 500 comments (§3.4 formula).
|
||||
const SOCIAL_SATURATION: f64 = 6.0;
|
||||
|
||||
/// Everything the pre-filter needs beyond the articles themselves (§3.5, §3.9).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PrefilterContext {
|
||||
/// Per-feed Bayesian upvote rate from ratings history (§3.9).
|
||||
pub feed_priors: HashMap<FeedId, FeedPrior>,
|
||||
/// Article ids already published in a previous issue (§3.5).
|
||||
pub already_published: Vec<ArticleId>,
|
||||
/// Article ids the LLM scored < [`STALE_LOW_SCORE`] recently (§3.5).
|
||||
pub recently_rejected: Vec<ArticleId>,
|
||||
}
|
||||
|
||||
impl PrefilterContext {
|
||||
/// Load the history/priors context from SQLite (§3.5 dedup-vs-history, §3.9).
|
||||
///
|
||||
/// `today` anchors the [`STALE_LOOKBACK_DAYS`] window.
|
||||
pub async fn load(
|
||||
db: &crate::db::Db,
|
||||
today: jiff::civil::Date,
|
||||
) -> Result<Self, crate::db::DbError> {
|
||||
let since = today
|
||||
.checked_sub(jiff::Span::new().days(STALE_LOOKBACK_DAYS))
|
||||
.unwrap_or(today);
|
||||
let feed_priors = db
|
||||
.feed_priors()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|p| (p.feed_id, p))
|
||||
.collect();
|
||||
let already_published = db.previously_published_ids().await?;
|
||||
let recently_rejected = db.recently_low_scored_ids(STALE_LOW_SCORE, since).await?;
|
||||
tracing::debug!(
|
||||
priors = ?feed_priors_len(&feed_priors),
|
||||
published = already_published.len(),
|
||||
rejected = recently_rejected.len(),
|
||||
"loaded prefilter context"
|
||||
);
|
||||
Ok(Self {
|
||||
feed_priors,
|
||||
already_published,
|
||||
recently_rejected,
|
||||
})
|
||||
}
|
||||
|
||||
fn prior_for(&self, article: &Article) -> f64 {
|
||||
// The cluster's feeds are all candidates; take the most favourable one,
|
||||
// since a story carried by a well-rated feed is a better bet.
|
||||
let mut best = self.feed_priors.get(&article.feed_id).map(FeedPrior::rate);
|
||||
for source in &article.sources {
|
||||
if let Some(p) = self.feed_priors.get(&source.feed_id) {
|
||||
let rate = p.rate();
|
||||
best = Some(best.map_or(rate, |b: f64| b.max(rate)));
|
||||
}
|
||||
}
|
||||
best.unwrap_or(0.5)
|
||||
}
|
||||
}
|
||||
|
||||
fn feed_priors_len(m: &HashMap<FeedId, FeedPrior>) -> usize {
|
||||
m.len()
|
||||
}
|
||||
|
||||
/// True when the article's feed is in `curation.always_include_feeds` (§3.5).
|
||||
///
|
||||
/// Entries are matched either as a Miniflux feed id (any feed in the cluster) or
|
||||
/// as a case-insensitive substring of the article/site URL.
|
||||
///
|
||||
/// Auto-includes are still LLM-scored (for section + summary) but can't be dropped.
|
||||
pub fn is_auto_include(article: &Article, cfg: &CurationConfig) -> bool {
|
||||
if cfg.always_include_feeds.is_empty() {
|
||||
return false;
|
||||
@@ -151,38 +50,30 @@ pub fn is_auto_include(article: &Article, cfg: &CurationConfig) -> bool {
|
||||
return false;
|
||||
}
|
||||
if let Ok(id) = needle.parse::<FeedId>()
|
||||
&& (article.feed_id == id || article.sources.iter().any(|s| s.feed_id == id))
|
||||
&& (article.feed_id == id || article.sources.iter().any(|source| source.feed_id == id))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let needle = needle.to_lowercase();
|
||||
// Bare host or full site URL: compare against both URLs we hold.
|
||||
let needle = needle
|
||||
.to_lowercase()
|
||||
.trim_start_matches("https://")
|
||||
.trim_start_matches("http://")
|
||||
.trim_end_matches('/');
|
||||
!needle.is_empty() && (url.contains(needle) || canonical.contains(needle))
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
!needle.is_empty() && (url.contains(&needle) || canonical.contains(&needle))
|
||||
})
|
||||
}
|
||||
|
||||
/// True when the article's host matches `curation.blocked_domains` (§3.5).
|
||||
pub fn is_blocked(article: &Article, cfg: &CurationConfig) -> bool {
|
||||
if cfg.blocked_domains.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let host = host_of(&article.canonical_url)
|
||||
.or_else(|| host_of(&article.url))
|
||||
.unwrap_or_default();
|
||||
if host.is_empty() {
|
||||
return false;
|
||||
}
|
||||
cfg.blocked_domains.iter().any(|raw| {
|
||||
let blocked = raw.trim().trim_start_matches('.').to_lowercase();
|
||||
!blocked.is_empty() && (host == blocked || host.ends_with(&format!(".{blocked}")))
|
||||
})
|
||||
}
|
||||
|
||||
/// Lowercased host of a URL, `www.` stripped.
|
||||
fn host_of(url: &str) -> Option<String> {
|
||||
let rest = url
|
||||
.split_once("://")
|
||||
@@ -190,17 +81,12 @@ fn host_of(url: &str) -> Option<String> {
|
||||
.unwrap_or(url)
|
||||
.split(['/', '?', '#'])
|
||||
.next()?;
|
||||
let host = rest.rsplit_once('@').map(|(_, h)| h).unwrap_or(rest);
|
||||
let host = host.split_once(':').map(|(h, _)| h).unwrap_or(host);
|
||||
let host = rest.rsplit_once('@').map(|(_, host)| host).unwrap_or(rest);
|
||||
let host = host.split_once(':').map(|(host, _)| host).unwrap_or(host);
|
||||
let host = host.trim().to_lowercase();
|
||||
if host.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(host.trim_start_matches("www.").to_string())
|
||||
}
|
||||
(!host.is_empty()).then(|| host.trim_start_matches("www.").to_string())
|
||||
}
|
||||
|
||||
/// True when the title reads like a link roundup / release note / sponsor post (§3.5).
|
||||
pub fn looks_like_roundup(title: &str) -> bool {
|
||||
let lower = title.to_lowercase();
|
||||
PENALTY_TITLE_PATTERNS
|
||||
@@ -208,139 +94,38 @@ pub fn looks_like_roundup(title: &str) -> bool {
|
||||
.any(|pattern| lower.contains(pattern))
|
||||
}
|
||||
|
||||
/// Long-form bonus: zero below [`LONGFORM_FLOOR_WORDS`], saturating at
|
||||
/// [`LONGFORM_SATURATION_WORDS`], with a concave curve so that the jump from a
|
||||
/// 400-word note to a 1,200-word piece matters more than 2,000 → 2,500 (§3.5).
|
||||
pub fn longform_points(word_count: i64) -> f64 {
|
||||
let span = (LONGFORM_SATURATION_WORDS - LONGFORM_FLOOR_WORDS) as f64;
|
||||
let over = (word_count - LONGFORM_FLOOR_WORDS).max(0) as f64;
|
||||
MAX_LONGFORM_POINTS * (over / span).min(1.0).powf(0.65)
|
||||
}
|
||||
|
||||
/// Social proof, log-scaled a second time so that a viral story cannot swamp the
|
||||
/// long-form preference (§3.4, §3.5).
|
||||
pub fn social_points(social_score: f64) -> f64 {
|
||||
if social_score <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
MAX_SOCIAL_POINTS * (social_score / SOCIAL_SATURATION).min(1.0).sqrt()
|
||||
pub fn text_heuristic(article: &Article) -> f64 {
|
||||
longform_points(article.word_count)
|
||||
- excerpt_only_penalty(article)
|
||||
- roundup_penalty(&article.title)
|
||||
}
|
||||
|
||||
/// Score one article 0–100 from word count, social proof, source signals, feed
|
||||
/// prior, and the excerpt/roundup/blocklist penalties (§3.5).
|
||||
pub fn score_article(article: &Article, ctx: &PrefilterContext, cfg: &Config) -> f64 {
|
||||
if is_blocked(article, &cfg.curation) {
|
||||
return 0.0;
|
||||
}
|
||||
let mut score = longform_points(article.word_count);
|
||||
score += social_points(article.social_score());
|
||||
|
||||
if article.came_via(SourceKind::Scour) {
|
||||
score += SCOUR_BONUS;
|
||||
}
|
||||
if article.came_via(SourceKind::HnFrontpage) {
|
||||
score += HN_FRONTPAGE_BONUS;
|
||||
}
|
||||
|
||||
let extra_feeds = article.sources.len().saturating_sub(1) as f64;
|
||||
score += (extra_feeds * 4.0).min(MAX_MULTI_SOURCE_POINTS);
|
||||
|
||||
// Beta-smoothed upvote rate, neutral (0.5) contributing nothing (§3.9).
|
||||
score += (ctx.prior_for(article) - 0.5) * 2.0 * MAX_FEED_PRIOR_POINTS;
|
||||
|
||||
pub fn excerpt_only_penalty(article: &Article) -> f64 {
|
||||
if article.excerpt_only {
|
||||
score -= EXCERPT_ONLY_PENALTY;
|
||||
}
|
||||
if looks_like_roundup(&article.title) {
|
||||
score -= ROUNDUP_TITLE_PENALTY;
|
||||
}
|
||||
|
||||
score.clamp(0.0, 100.0)
|
||||
}
|
||||
|
||||
/// Apply [`score_article`] to everything, drop history duplicates, then keep the
|
||||
/// top `prefilter_keep` plus every auto-include (§3.5).
|
||||
pub fn run(articles: Vec<Article>, ctx: &PrefilterContext, cfg: &Config) -> Vec<ScoredArticle> {
|
||||
let published: HashSet<ArticleId> = ctx.already_published.iter().copied().collect();
|
||||
let rejected: HashSet<ArticleId> = ctx.recently_rejected.iter().copied().collect();
|
||||
|
||||
let total = articles.len();
|
||||
let (mut dropped_history, mut dropped_blocked) = (0usize, 0usize);
|
||||
let mut scored: Vec<ScoredArticle> = Vec::with_capacity(total);
|
||||
|
||||
for article in articles {
|
||||
let auto_include = is_auto_include(&article, &cfg.curation);
|
||||
|
||||
// Never print the same story twice, not even from an always-include feed.
|
||||
if published.contains(&article.id) {
|
||||
dropped_history += 1;
|
||||
continue;
|
||||
}
|
||||
// "Don't re-score churn" (§3.5) — but an always-include feed still gets in.
|
||||
if !auto_include && rejected.contains(&article.id) {
|
||||
dropped_history += 1;
|
||||
continue;
|
||||
}
|
||||
if !auto_include && is_blocked(&article, &cfg.curation) {
|
||||
dropped_blocked += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let prefilter_score = score_article(&article, ctx, cfg);
|
||||
let social_score = article.social_score();
|
||||
let feed_prior = ctx.prior_for(&article);
|
||||
scored.push(ScoredArticle {
|
||||
article,
|
||||
prefilter_score,
|
||||
social_score,
|
||||
feed_prior,
|
||||
llm: None,
|
||||
auto_include,
|
||||
});
|
||||
}
|
||||
|
||||
// Descending by score; ties broken by word count then id so the order is
|
||||
// deterministic across runs (notes §12).
|
||||
sort_by_prefilter(&mut scored);
|
||||
|
||||
let keep = cfg.prefilter_keep.max(cfg.target_article_count);
|
||||
let kept: Vec<ScoredArticle> = if scored.len() <= keep {
|
||||
scored
|
||||
EXCERPT_ONLY_PENALTY
|
||||
} else {
|
||||
let (head, tail) = scored.split_at(keep);
|
||||
let mut kept = head.to_vec();
|
||||
// Auto-includes below the cut are pulled back in — they can't be dropped.
|
||||
kept.extend(tail.iter().filter(|s| s.auto_include).cloned());
|
||||
sort_by_prefilter(&mut kept);
|
||||
kept
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
input = total,
|
||||
kept = kept.len(),
|
||||
auto_includes = kept.iter().filter(|s| s.auto_include).count(),
|
||||
dropped_history,
|
||||
dropped_blocked,
|
||||
"pre-filter complete"
|
||||
);
|
||||
kept
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic ranking: score desc, then longer, then lowest id (notes §12).
|
||||
pub fn sort_by_prefilter(scored: &mut [ScoredArticle]) {
|
||||
scored.sort_by(|a, b| {
|
||||
b.prefilter_score
|
||||
.partial_cmp(&a.prefilter_score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| b.article.word_count.cmp(&a.article.word_count))
|
||||
.then_with(|| a.article.id.cmp(&b.article.id))
|
||||
});
|
||||
pub fn roundup_penalty(title: &str) -> f64 {
|
||||
if looks_like_roundup(title) {
|
||||
ROUNDUP_TITLE_PENALTY
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::types::{ExtractMethod, SocialRef, SocialSource, SourceRef};
|
||||
use crate::types::{ArticleId, ExtractMethod, SocialRef, SocialSource, SourceKind, SourceRef};
|
||||
use jiff::Timestamp;
|
||||
|
||||
pub(crate) fn ts() -> Timestamp {
|
||||
@@ -349,7 +134,6 @@ pub(crate) mod tests {
|
||||
.expect("static timestamp parses")
|
||||
}
|
||||
|
||||
/// A plain 800-word article from feed 7 with no social proof.
|
||||
pub(crate) fn article(id: ArticleId, title: &str, word_count: i64) -> Article {
|
||||
Article {
|
||||
id,
|
||||
@@ -370,6 +154,7 @@ pub(crate) mod tests {
|
||||
first_seen: ts(),
|
||||
url: format!("https://example.com/{id}"),
|
||||
author: Some("A. Writer".into()),
|
||||
publication: None,
|
||||
feed_id: 7,
|
||||
feed_title: "Some Blog".into(),
|
||||
category: Some("Tech".into()),
|
||||
@@ -381,285 +166,40 @@ pub(crate) mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_social(mut a: Article, points: i64, comments: i64) -> Article {
|
||||
a.social = vec![SocialRef {
|
||||
article_id: a.id,
|
||||
pub(crate) fn with_social(mut article: Article, points: i64, comments: i64) -> Article {
|
||||
article.social = vec![SocialRef {
|
||||
article_id: article.id,
|
||||
source: SocialSource::Hn,
|
||||
item_id: Some("1".into()),
|
||||
score: points,
|
||||
num_comments: comments,
|
||||
item_url: Some("https://news.ycombinator.com/item?id=1".into()),
|
||||
item_url: None,
|
||||
fetched_at: ts(),
|
||||
}];
|
||||
a
|
||||
}
|
||||
|
||||
pub(crate) fn via(mut a: Article, kind: SourceKind, feed_id: FeedId) -> Article {
|
||||
a.sources.push(SourceRef {
|
||||
entry_id: a.best_entry_id,
|
||||
feed_id,
|
||||
feed_title: format!("{kind:?} feed"),
|
||||
category: None,
|
||||
kind,
|
||||
});
|
||||
a
|
||||
}
|
||||
|
||||
fn cfg() -> Config {
|
||||
Config {
|
||||
prefilter_keep: 3,
|
||||
target_article_count: 2,
|
||||
..Config::default()
|
||||
}
|
||||
article
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longform_curve_is_monotonic_and_bounded() {
|
||||
assert_eq!(longform_points(0), 0.0);
|
||||
assert_eq!(longform_points(LONGFORM_FLOOR_WORDS), 0.0);
|
||||
let mut prev = -1.0;
|
||||
for wc in [0, 100, 299, 300, 500, 900, 1500, 2200, 2500, 9000] {
|
||||
let pts = longform_points(wc);
|
||||
assert!(pts >= prev, "not monotonic at {wc}");
|
||||
assert!(pts <= MAX_LONGFORM_POINTS);
|
||||
prev = pts;
|
||||
}
|
||||
assert!((longform_points(2500) - MAX_LONGFORM_POINTS).abs() < 1e-9);
|
||||
assert!((longform_points(50_000) - MAX_LONGFORM_POINTS).abs() < 1e-9);
|
||||
fn text_heuristic_has_only_text_terms() {
|
||||
let quiet = article(1, "An essay", 1200);
|
||||
let loud = with_social(quiet.clone(), 500, 200);
|
||||
assert_eq!(text_heuristic(&quiet), text_heuristic(&loud));
|
||||
assert!(text_heuristic(&article(2, "This Week in Rust", 1200)) < text_heuristic(&quiet));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn social_curve_is_monotonic_and_bounded() {
|
||||
let mut prev = -1.0;
|
||||
for s in [0.0, 0.5, 1.0, 2.0, 4.0, 6.0, 20.0] {
|
||||
let pts = social_points(s);
|
||||
assert!(pts >= prev);
|
||||
assert!(pts <= MAX_SOCIAL_POINTS);
|
||||
prev = pts;
|
||||
}
|
||||
assert_eq!(social_points(0.0), 0.0);
|
||||
assert!((social_points(6.0) - MAX_SOCIAL_POINTS).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_rises_with_length_and_social_proof() {
|
||||
let (ctx, cfg) = (PrefilterContext::default(), cfg());
|
||||
let short = score_article(&article(1, "A thought", 200), &ctx, &cfg);
|
||||
let medium = score_article(&article(2, "An essay", 1200), &ctx, &cfg);
|
||||
let long = score_article(&article(3, "A treatise", 3000), &ctx, &cfg);
|
||||
assert!(short < medium, "{short} !< {medium}");
|
||||
assert!(medium < long, "{medium} !< {long}");
|
||||
|
||||
let quiet = score_article(&article(4, "An essay", 1200), &ctx, &cfg);
|
||||
let loud = score_article(
|
||||
&with_social(article(5, "An essay", 1200), 400, 250),
|
||||
&ctx,
|
||||
&cfg,
|
||||
);
|
||||
assert!(loud > quiet);
|
||||
assert!(loud <= 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_bonuses_and_penalties_apply() {
|
||||
let (ctx, cfg) = (PrefilterContext::default(), cfg());
|
||||
// Long enough that the penalties do not run into the 0 floor.
|
||||
let plain = score_article(&article(1, "Deep dive", 3000), &ctx, &cfg);
|
||||
assert!(plain > EXCERPT_ONLY_PENALTY);
|
||||
|
||||
let scoured = score_article(
|
||||
&via(article(2, "Deep dive", 3000), SourceKind::Scour, 42),
|
||||
&ctx,
|
||||
&cfg,
|
||||
);
|
||||
// Scour bonus + one extra feed in the cluster.
|
||||
assert!(scoured > plain + SCOUR_BONUS - 0.001);
|
||||
|
||||
let mut excerpt = article(3, "Deep dive", 3000);
|
||||
excerpt.excerpt_only = true;
|
||||
assert!(
|
||||
(score_article(&excerpt, &ctx, &cfg) - (plain - EXCERPT_ONLY_PENALTY)).abs() < 1e-9
|
||||
);
|
||||
|
||||
let roundup = article(4, "This Week in Rust #612", 3000);
|
||||
assert!(looks_like_roundup(&roundup.title));
|
||||
assert!(
|
||||
(score_article(&roundup, &ctx, &cfg) - (plain - ROUNDUP_TITLE_PENALTY)).abs() < 1e-9
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_prior_moves_the_score_both_ways() {
|
||||
let cfg = cfg();
|
||||
let mut liked = PrefilterContext::default();
|
||||
liked.feed_priors.insert(
|
||||
7,
|
||||
FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 18,
|
||||
downvotes: 0,
|
||||
included: 18,
|
||||
},
|
||||
);
|
||||
let mut disliked = PrefilterContext::default();
|
||||
disliked.feed_priors.insert(
|
||||
7,
|
||||
FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 0,
|
||||
downvotes: 18,
|
||||
included: 18,
|
||||
},
|
||||
);
|
||||
let a = article(1, "Deep dive", 1200);
|
||||
let neutral = score_article(&a, &PrefilterContext::default(), &cfg);
|
||||
assert!(score_article(&a, &liked, &cfg) > neutral);
|
||||
assert!(score_article(&a, &disliked, &cfg) < neutral);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_domains_and_auto_includes_match_urls_and_ids() {
|
||||
let mut cfg = cfg();
|
||||
cfg.curation.blocked_domains = vec!["spam.example".into()];
|
||||
cfg.curation.always_include_feeds = vec!["99".into(), "tyler.blog".into()];
|
||||
|
||||
let mut blocked = article(1, "Buy now", 1200);
|
||||
blocked.canonical_url = "https://news.spam.example/post".into();
|
||||
blocked.url.clone_from(&blocked.canonical_url);
|
||||
assert!(is_blocked(&blocked, &cfg.curation));
|
||||
assert_eq!(
|
||||
score_article(&blocked, &PrefilterContext::default(), &cfg),
|
||||
0.0
|
||||
);
|
||||
|
||||
let mut by_url = article(2, "A rare post", 900);
|
||||
by_url.url = "https://tyler.blog/2026/rare".into();
|
||||
assert!(is_auto_include(&by_url, &cfg.curation));
|
||||
|
||||
let mut by_id = article(3, "Another rare post", 900);
|
||||
by_id.feed_id = 99;
|
||||
assert!(is_auto_include(&by_id, &cfg.curation));
|
||||
|
||||
assert!(!is_auto_include(&article(4, "Normal", 900), &cfg.curation));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_top_n_plus_auto_includes_and_drops_history() {
|
||||
let mut cfg = cfg();
|
||||
cfg.prefilter_keep = 2;
|
||||
cfg.curation.always_include_feeds = vec!["99".into()];
|
||||
|
||||
let mut auto = article(5, "A short personal note", 120);
|
||||
fn blocked_and_auto_include_match() {
|
||||
let cfg = CurationConfig {
|
||||
blocked_domains: vec!["spam.example".into()],
|
||||
always_include_feeds: vec!["99".into(), "tyler.blog".into()],
|
||||
..CurationConfig::default()
|
||||
};
|
||||
let mut blocked = article(1, "spam", 100);
|
||||
blocked.url = "https://news.spam.example/a".into();
|
||||
blocked.canonical_url.clone_from(&blocked.url);
|
||||
assert!(is_blocked(&blocked, &cfg));
|
||||
let mut auto = article(2, "post", 100);
|
||||
auto.feed_id = 99;
|
||||
|
||||
let articles = vec![
|
||||
article(1, "Long treatise", 4000),
|
||||
article(2, "Medium essay", 1500),
|
||||
article(3, "Shorter piece", 700),
|
||||
article(4, "Already printed", 5000),
|
||||
auto,
|
||||
article(6, "Rejected yesterday", 3000),
|
||||
];
|
||||
let ctx = PrefilterContext {
|
||||
already_published: vec![4],
|
||||
recently_rejected: vec![6],
|
||||
..PrefilterContext::default()
|
||||
};
|
||||
|
||||
let kept = run(articles, &ctx, &cfg);
|
||||
let ids: Vec<ArticleId> = kept.iter().map(|s| s.article.id).collect();
|
||||
assert!(!ids.contains(&4), "previously published must be dropped");
|
||||
assert!(!ids.contains(&6), "recently rejected must be dropped");
|
||||
assert!(ids.contains(&5), "auto-include survives below the cut");
|
||||
assert!(ids.contains(&1) && ids.contains(&2));
|
||||
assert!(!ids.contains(&3), "cut at prefilter_keep");
|
||||
assert_eq!(kept.len(), 3); // 2 kept + 1 auto-include
|
||||
|
||||
// Sorted by score, descending.
|
||||
for pair in kept.windows(2) {
|
||||
assert!(pair[0].prefilter_score >= pair[1].prefilter_score);
|
||||
}
|
||||
assert!(
|
||||
kept.iter()
|
||||
.find(|s| s.article.id == 5)
|
||||
.is_some_and(|s| s.auto_include)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_include_survives_the_recently_rejected_list_but_not_republication() {
|
||||
let mut cfg = cfg();
|
||||
cfg.curation.always_include_feeds = vec!["99".into()];
|
||||
let mut a = article(1, "Personal note", 200);
|
||||
a.feed_id = 99;
|
||||
let mut b = article(2, "Personal note two", 200);
|
||||
b.feed_id = 99;
|
||||
|
||||
let ctx = PrefilterContext {
|
||||
recently_rejected: vec![1],
|
||||
already_published: vec![2],
|
||||
..PrefilterContext::default()
|
||||
};
|
||||
let kept = run(vec![a, b], &ctx, &cfg);
|
||||
let ids: Vec<ArticleId> = kept.iter().map(|s| s.article.id).collect();
|
||||
assert_eq!(ids, vec![1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_loads_history_from_sqlite() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = crate::db::Db::open_and_migrate(&dir.path().join("t.db"))
|
||||
.await
|
||||
.expect("db");
|
||||
let date: jiff::civil::Date = "2026-08-15".parse().expect("date");
|
||||
|
||||
db.upsert_feed_prior(&FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 4,
|
||||
downvotes: 1,
|
||||
included: 5,
|
||||
})
|
||||
.await
|
||||
.expect("prior");
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||
(42, 'https://example.com/42', 'Printed', '2026-08-14T00:00:00Z'),
|
||||
(43, 'https://example.com/43', 'Rejected', '2026-08-14T00:00:00Z'),
|
||||
(44, 'https://example.com/44', 'Ancient', '2020-01-01T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("articles");
|
||||
db.upsert_issue(
|
||||
"2026-08-14".parse().expect("date"),
|
||||
1,
|
||||
ts(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("issue");
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position, is_lead)
|
||||
VALUES ('2026-08-14', 42, 'Top Stories', 1, 0)",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("issue article");
|
||||
sqlx::query(
|
||||
"INSERT INTO scores (article_id, run_date, llm_score) VALUES (43, '2026-08-14', 1.5),
|
||||
(44, '2020-01-01', 1.0)",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("scores");
|
||||
|
||||
let ctx = PrefilterContext::load(&db, date).await.expect("context");
|
||||
assert_eq!(ctx.already_published, vec![42]);
|
||||
assert_eq!(ctx.recently_rejected, vec![43], "old rejects age out");
|
||||
assert!((ctx.feed_priors[&7].rate() - 5.0 / 7.0).abs() < 1e-12);
|
||||
assert!(is_auto_include(&auto, &cfg));
|
||||
}
|
||||
}
|
||||
+456
-594
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,610 @@
|
||||
//! Utility normalization, leader clustering and diversified shortlisting (§12.2–§12.5).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::config::{RankingConfig, UtilityWeights};
|
||||
use crate::curate::embedding::dot;
|
||||
use crate::curate::signals;
|
||||
use crate::types::{ArticleId, Candidate};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct RankSummary {
|
||||
pub shortlisted: usize,
|
||||
pub clusters: usize,
|
||||
}
|
||||
|
||||
/// Re-normalize cheap signals over the deep set and calculate utility on 0–100,
|
||||
/// scaled by the slop-author factor (§9.3).
|
||||
pub fn calculate_utility(candidates: &mut [Candidate], configured: &UtilityWeights) {
|
||||
let indices = (0..candidates.len()).collect::<Vec<_>>();
|
||||
calculate_utility_for(candidates, &indices, configured);
|
||||
}
|
||||
|
||||
fn calculate_utility_for(
|
||||
candidates: &mut [Candidate],
|
||||
indices: &[usize],
|
||||
configured: &UtilityWeights,
|
||||
) {
|
||||
let mut normalized = indices
|
||||
.iter()
|
||||
.map(|index| candidates[*index].signals.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for signals in &mut normalized {
|
||||
signals.norm.clear();
|
||||
signals.weights.clear();
|
||||
}
|
||||
let mut signal_refs = normalized.iter_mut().collect::<Vec<_>>();
|
||||
signals::normalize(&mut signal_refs);
|
||||
for (index, signals) in indices.iter().zip(normalized) {
|
||||
candidates[*index].signals.norm = signals.norm;
|
||||
candidates[*index].signals.weights.clear();
|
||||
}
|
||||
|
||||
for index in indices {
|
||||
let candidate = &mut candidates[*index];
|
||||
if let Some(triage) = &candidate.assessment.triage {
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("triage".into(), (triage.interest / 10.0).clamp(0.0, 1.0));
|
||||
}
|
||||
if let Some(deep) = &candidate.assessment.deep {
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("quality".into(), (deep.quality / 10.0).clamp(0.0, 1.0));
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("fit".into(), (deep.fit / 10.0).clamp(0.0, 1.0));
|
||||
}
|
||||
let weighted = [
|
||||
("quality", configured.quality, 1.0),
|
||||
("fit", configured.fit, 1.0),
|
||||
("knn", configured.knn, candidate.signals.knn_gate),
|
||||
(
|
||||
"affinity",
|
||||
configured.affinity,
|
||||
candidate.signals.affinity_gate,
|
||||
),
|
||||
("interest", configured.interest, 1.0),
|
||||
("feed", configured.feed, candidate.signals.feed_gate),
|
||||
("triage", configured.triage, 1.0),
|
||||
("social", configured.social, 1.0),
|
||||
("heuristic", configured.heuristic, 1.0),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(name, weight, gate)| {
|
||||
let value = candidate.signals.norm.get(name).copied()?;
|
||||
let effective = weight * gate;
|
||||
(effective > 0.0).then_some((name, effective, value))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let total = weighted.iter().map(|(_, weight, _)| weight).sum::<f64>();
|
||||
if total <= 0.0 {
|
||||
candidate.utility = None;
|
||||
continue;
|
||||
}
|
||||
candidate.signals.weights = weighted
|
||||
.iter()
|
||||
.map(|(name, weight, _)| ((*name).to_string(), weight / total))
|
||||
.collect();
|
||||
candidate.utility = Some(
|
||||
weighted
|
||||
.iter()
|
||||
.map(|(_, weight, value)| weight / total * value)
|
||||
.sum::<f64>()
|
||||
* 100.0
|
||||
* candidate.signals.slop_factor(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn ranked_indices(candidates: &[Candidate], indices: &[usize]) -> Vec<usize> {
|
||||
let mut sorted = indices.to_vec();
|
||||
sorted.sort_by(|left, right| {
|
||||
candidates[*right]
|
||||
.utility
|
||||
.unwrap_or(f64::NEG_INFINITY)
|
||||
.total_cmp(&candidates[*left].utility.unwrap_or(f64::NEG_INFINITY))
|
||||
.then_with(|| {
|
||||
candidates[*left]
|
||||
.article
|
||||
.id
|
||||
.cmp(&candidates[*right].article.id)
|
||||
})
|
||||
});
|
||||
sorted
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Cluster {
|
||||
id: i64,
|
||||
leader: usize,
|
||||
}
|
||||
|
||||
/// Rank the admitted deep set and leave only the diversified shortlist at
|
||||
/// `stage = shortlisted`. All deep-set articles receive ranks and clusters.
|
||||
pub fn shortlist(
|
||||
candidates: &mut [Candidate],
|
||||
embeddings: &HashMap<ArticleId, Vec<f32>>,
|
||||
ranking: &RankingConfig,
|
||||
) -> RankSummary {
|
||||
let deep = candidates
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, candidate)| matches!(candidate.stage.as_str(), "admitted" | "assessed"))
|
||||
.map(|(index, _)| index)
|
||||
.collect::<Vec<_>>();
|
||||
calculate_utility_for(candidates, &deep, &ranking.weights.utility);
|
||||
let sorted = ranked_indices(candidates, &deep);
|
||||
for (rank, index) in sorted.iter().enumerate() {
|
||||
candidates[*index].rank_utility = Some(rank as i64 + 1);
|
||||
candidates[*index].cluster = None;
|
||||
candidates[*index].cluster_rank = None;
|
||||
}
|
||||
|
||||
let mut clusters = Vec::<Cluster>::new();
|
||||
let mut members: HashMap<i64, Vec<usize>> = HashMap::new();
|
||||
for index in &sorted {
|
||||
let assigned = embeddings
|
||||
.get(&candidates[*index].article.id)
|
||||
.and_then(|vector| {
|
||||
clusters.iter().find_map(|cluster| {
|
||||
let leader_id = candidates[cluster.leader].article.id;
|
||||
let leader = embeddings.get(&leader_id)?;
|
||||
dot(vector, leader)
|
||||
.ok()
|
||||
.filter(|cosine| *cosine >= ranking.diversity.cluster_threshold)
|
||||
.map(|_| cluster.id)
|
||||
})
|
||||
});
|
||||
let cluster_id = assigned.unwrap_or_else(|| {
|
||||
let id = clusters.len() as i64 + 1;
|
||||
clusters.push(Cluster { id, leader: *index });
|
||||
id
|
||||
});
|
||||
let cluster_members = members.entry(cluster_id).or_default();
|
||||
cluster_members.push(*index);
|
||||
candidates[*index].cluster = Some(cluster_id);
|
||||
candidates[*index].cluster_rank = Some(cluster_members.len() as i64);
|
||||
}
|
||||
|
||||
let protected = sorted
|
||||
.iter()
|
||||
.take(ranking.diversity.utility_protected)
|
||||
.copied()
|
||||
.collect::<HashSet<_>>();
|
||||
let mut admitted = HashSet::new();
|
||||
let mut admitted_per_cluster = HashMap::<i64, usize>::new();
|
||||
let admit = |index: usize, admitted: &mut HashSet<usize>, counts: &mut HashMap<i64, usize>| {
|
||||
if admitted.insert(index)
|
||||
&& let Some(cluster) = candidates[index].cluster
|
||||
{
|
||||
*counts.entry(cluster).or_default() += 1;
|
||||
}
|
||||
};
|
||||
|
||||
for index in &sorted {
|
||||
if protected.contains(index) || candidates[*index].auto_include {
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
}
|
||||
for index in sorted
|
||||
.iter()
|
||||
.filter(|index| candidates[**index].exploration)
|
||||
.take(3)
|
||||
{
|
||||
if admitted.len() >= ranking.shortlist_keep {
|
||||
break;
|
||||
}
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
|
||||
let target = ranking.shortlist_keep.max(admitted.len());
|
||||
let mut suppressed_at_base_cap = HashSet::new();
|
||||
admit_under_cap(
|
||||
candidates,
|
||||
&sorted,
|
||||
target,
|
||||
ranking.diversity.per_cluster_cap,
|
||||
&mut admitted,
|
||||
&mut admitted_per_cluster,
|
||||
Some(&mut suppressed_at_base_cap),
|
||||
);
|
||||
if admitted.len() < target {
|
||||
admit_under_cap(
|
||||
candidates,
|
||||
&sorted,
|
||||
target,
|
||||
3,
|
||||
&mut admitted,
|
||||
&mut admitted_per_cluster,
|
||||
None,
|
||||
);
|
||||
}
|
||||
if admitted.len() < target {
|
||||
for index in &sorted {
|
||||
if admitted.len() >= target {
|
||||
break;
|
||||
}
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
}
|
||||
|
||||
for index in deep {
|
||||
if admitted.contains(&index) {
|
||||
candidates[index].stage = "shortlisted".into();
|
||||
candidates[index].excluded_reason = None;
|
||||
} else {
|
||||
// The stage stays where the article stopped (`admitted` when the
|
||||
// deep assessment never happened, else `assessed`).
|
||||
candidates[index].excluded_reason = Some(
|
||||
if suppressed_at_base_cap.contains(&index) {
|
||||
"cluster_suppressed"
|
||||
} else {
|
||||
"shortlist_cap"
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
RankSummary {
|
||||
shortlisted: admitted.len(),
|
||||
clusters: clusters.len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn admit_under_cap(
|
||||
candidates: &[Candidate],
|
||||
sorted: &[usize],
|
||||
target: usize,
|
||||
cap: usize,
|
||||
admitted: &mut HashSet<usize>,
|
||||
admitted_per_cluster: &mut HashMap<i64, usize>,
|
||||
mut suppressed: Option<&mut HashSet<usize>>,
|
||||
) {
|
||||
for index in sorted {
|
||||
if admitted.len() >= target || admitted.contains(index) {
|
||||
continue;
|
||||
}
|
||||
let Some(cluster) = candidates[*index].cluster else {
|
||||
continue;
|
||||
};
|
||||
if admitted_per_cluster.get(&cluster).copied().unwrap_or(0) >= cap {
|
||||
if let Some(suppressed) = suppressed.as_deref_mut() {
|
||||
suppressed.insert(*index);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
admitted.insert(*index);
|
||||
*admitted_per_cluster.entry(cluster).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DiversityConfig;
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::curate::signals::Signals;
|
||||
use crate::types::{Deep, Facets};
|
||||
|
||||
fn candidate(id: i64, utility_hint: f64) -> Candidate {
|
||||
let mut candidate = Candidate::new(article(id, &format!("article {id}"), 800), false);
|
||||
candidate.stage = "assessed".into();
|
||||
candidate.signals = Signals {
|
||||
heuristic: Some(utility_hint),
|
||||
..Signals::default()
|
||||
};
|
||||
candidate.assessment.deep = Some(Deep {
|
||||
quality: utility_hint,
|
||||
fit: utility_hint,
|
||||
category: Some("Top Stories".into()),
|
||||
rationale: "specific".into(),
|
||||
paywalled_guess: false,
|
||||
facets: Facets::default(),
|
||||
model: "mock".into(),
|
||||
prompt_version: 1,
|
||||
assessed_at: "2026-09-02T00:00:00Z".parse().expect("timestamp"),
|
||||
});
|
||||
candidate
|
||||
}
|
||||
|
||||
fn vector(angle: f32) -> Vec<f32> {
|
||||
vec![angle.cos(), angle.sin()]
|
||||
}
|
||||
|
||||
fn ranking(keep: usize, protected: usize, cap: usize, threshold: f64) -> RankingConfig {
|
||||
RankingConfig {
|
||||
shortlist_keep: keep,
|
||||
diversity: DiversityConfig {
|
||||
cluster_threshold: threshold,
|
||||
per_cluster_cap: cap,
|
||||
utility_protected: protected,
|
||||
},
|
||||
..RankingConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_renormalizes_present_signals_and_gates_learned_ones() {
|
||||
let mut a = candidate(1, 8.0);
|
||||
let mut b = candidate(2, 4.0);
|
||||
a.signals.interest = Some(1.0);
|
||||
b.signals.interest = None;
|
||||
a.signals.knn = Some(0.9);
|
||||
a.signals.knn_gate = 0.5;
|
||||
let mut values = vec![a, b];
|
||||
calculate_utility(&mut values, &UtilityWeights::default());
|
||||
let [a, b] = values.as_slice() else {
|
||||
panic!("two values")
|
||||
};
|
||||
for candidate in [&a, &b] {
|
||||
assert!((candidate.signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
assert!(candidate.utility.is_some());
|
||||
}
|
||||
assert!(!b.signals.weights.contains_key("interest"));
|
||||
assert!(
|
||||
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.10 * 0.5) / 0.40).abs()
|
||||
< 1e-9
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slop_authors_keep_their_weights_but_lose_most_of_their_utility() {
|
||||
let mut reported = candidate(1, 8.0);
|
||||
let mut other = candidate(2, 8.0);
|
||||
reported.signals.slop_author = true;
|
||||
reported.signals.slop_penalty = 0.75;
|
||||
other.signals.slop_author = false;
|
||||
other.signals.slop_penalty = 0.75;
|
||||
let mut values = vec![reported, other];
|
||||
calculate_utility(&mut values, &UtilityWeights::default());
|
||||
let [reported, other] = values.as_slice() else {
|
||||
panic!("two values")
|
||||
};
|
||||
assert_eq!(reported.signals.weights, other.signals.weights);
|
||||
assert!((reported.utility.unwrap() - other.utility.unwrap() * 0.25).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_cluster_and_the_third_is_suppressed() {
|
||||
let ranking = ranking(3, 0, 2, 0.85);
|
||||
let mut candidates = vec![
|
||||
candidate(1, 9.0),
|
||||
candidate(2, 8.0),
|
||||
candidate(3, 7.0),
|
||||
candidate(4, 6.0),
|
||||
];
|
||||
let embeddings = HashMap::from([
|
||||
(1, vector(0.0)),
|
||||
(2, vector(0.1)),
|
||||
(3, vector(0.2)),
|
||||
(4, vector(2.0)),
|
||||
]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.shortlisted, 3);
|
||||
assert_eq!(candidates[0].cluster, candidates[1].cluster);
|
||||
assert_eq!(candidates[1].cluster, candidates[2].cluster);
|
||||
assert_eq!(
|
||||
candidates[2].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_items_survive_and_count_toward_the_cap() {
|
||||
let ranking = ranking(3, 2, 1, 0.85);
|
||||
let mut candidates = vec![
|
||||
candidate(1, 9.0),
|
||||
candidate(2, 8.0),
|
||||
candidate(3, 7.0),
|
||||
candidate(4, 6.0),
|
||||
];
|
||||
let embeddings = HashMap::from([
|
||||
(1, vector(0.0)),
|
||||
(2, vector(0.05)),
|
||||
(3, vector(0.1)),
|
||||
(4, vector(2.0)),
|
||||
]);
|
||||
shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(candidates[0].stage, "shortlisted");
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
assert_ne!(candidates[2].stage, "shortlisted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_case_uses_leaders_not_transitive_components() {
|
||||
let ranking = ranking(3, 0, 2, 0.80);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
// A at 0°, B at 72°, C at 36°: A~C and B~C, but A not~B.
|
||||
let embeddings =
|
||||
HashMap::from([(1, vector(0.0)), (2, vector(1.2566)), (3, vector(0.6283))]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.clusters, 2);
|
||||
assert_eq!(candidates[0].cluster, candidates[2].cluster);
|
||||
assert_ne!(candidates[0].cluster, candidates[1].cluster);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_embeddings_are_singletons_and_never_suppressed() {
|
||||
let ranking = ranking(3, 0, 1, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert!(
|
||||
candidates
|
||||
.iter()
|
||||
.all(|candidate| candidate.stage == "shortlisted")
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.filter_map(|candidate| candidate.cluster)
|
||||
.collect::<HashSet<_>>()
|
||||
.len(),
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_relax_to_three_then_uncapped_when_the_shortlist_is_short() {
|
||||
// Six near-duplicates, keep 5: cap 2 admits two, cap 3 admits a third,
|
||||
// and the uncapped pass fills the remaining two slots in utility order.
|
||||
let ranking = ranking(5, 0, 2, 0.85);
|
||||
let mut candidates = (1..=6)
|
||||
.map(|id| candidate(id, 10.0 - id as f64))
|
||||
.collect::<Vec<_>>();
|
||||
let embeddings = (1..=6)
|
||||
.map(|id| (id, vector(0.01 * id as f32)))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.clusters, 1);
|
||||
assert_eq!(summary.shortlisted, 5);
|
||||
let shortlisted = candidates
|
||||
.iter()
|
||||
.filter(|candidate| candidate.stage == "shortlisted")
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(shortlisted, vec![1, 2, 3, 4, 5], "filled in utility order");
|
||||
assert_eq!(candidates[5].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[5].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.map(|c| c.rank_utility)
|
||||
.collect::<Vec<_>>(),
|
||||
(1..=6).map(Some).collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.map(|c| c.cluster_rank)
|
||||
.collect::<Vec<_>>(),
|
||||
(1..=6).map(Some).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shortlist_cap_is_the_reason_beyond_the_keep() {
|
||||
let ranking = ranking(2, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(candidates[0].stage, "shortlisted");
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
assert_eq!(candidates[2].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[2].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exploration_picks_get_up_to_three_reserved_slots() {
|
||||
// Keep 4 with the four best by utility being ordinary articles: three
|
||||
// exploration picks are still reserved seats, the fourth is not.
|
||||
let ranking = ranking(4, 0, 2, 0.85);
|
||||
let mut candidates = (1..=8)
|
||||
.map(|id| candidate(id, 10.0 - id as f64))
|
||||
.collect::<Vec<_>>();
|
||||
for candidate in candidates.iter_mut().skip(4) {
|
||||
candidate.exploration = true;
|
||||
}
|
||||
let summary = shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(summary.shortlisted, 4);
|
||||
let shortlisted = candidates
|
||||
.iter()
|
||||
.filter(|candidate| candidate.stage == "shortlisted")
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(shortlisted, vec![1, 5, 6, 7]);
|
||||
assert_eq!(
|
||||
candidates[7].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_includes_are_admitted_regardless_and_count_toward_their_cluster() {
|
||||
let ranking = ranking(2, 0, 1, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 1.0)];
|
||||
candidates[2].auto_include = true;
|
||||
let embeddings = HashMap::from([(1, vector(0.0)), (2, vector(2.0)), (3, vector(0.05))]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.shortlisted, 2);
|
||||
assert_eq!(candidates[2].stage, "shortlisted", "auto-include survives");
|
||||
// The auto-include filled its cluster's single seat, so the stronger
|
||||
// near-duplicate is suppressed and the unrelated article gets the slot.
|
||||
assert_eq!(candidates[0].cluster, candidates[2].cluster);
|
||||
assert_eq!(candidates[0].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[0].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unassessed_articles_rank_on_present_signals_and_keep_their_stage() {
|
||||
// DeepSeek down: no quality/fit anywhere, utility comes from what is present.
|
||||
let ranking = ranking(2, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 3.0), candidate(2, 6.0), candidate(3, 9.0)];
|
||||
for candidate in &mut candidates {
|
||||
candidate.assessment.deep = None;
|
||||
candidate.stage = "admitted".into();
|
||||
}
|
||||
candidates[0].assessment.triage = Some(crate::types::Triage {
|
||||
interest: 9.0,
|
||||
kind: "essay".into(),
|
||||
why: "promising".into(),
|
||||
model: "mock".into(),
|
||||
prompt_version: 1,
|
||||
assessed_at: "2026-09-02T00:00:00Z".parse().expect("timestamp"),
|
||||
});
|
||||
let summary = shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(summary.shortlisted, 2);
|
||||
for candidate in &candidates {
|
||||
assert!(candidate.utility.is_some(), "scored on present signals");
|
||||
assert!(!candidate.signals.weights.contains_key("quality"));
|
||||
assert!(!candidate.signals.weights.contains_key("fit"));
|
||||
assert!((candidate.signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
}
|
||||
// Triage (0.05) outweighs heuristic (0.02): the triaged article with
|
||||
// the weakest heuristic overtakes the middle one.
|
||||
assert_eq!(candidates[2].rank_utility, Some(1));
|
||||
assert_eq!(candidates[0].rank_utility, Some(2));
|
||||
assert_eq!(candidates[1].rank_utility, Some(3));
|
||||
assert_eq!(
|
||||
candidates[1].stage, "admitted",
|
||||
"never assessed, so not `assessed`"
|
||||
);
|
||||
assert_eq!(
|
||||
candidates[1].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentiles_are_taken_over_the_deep_set_only() {
|
||||
// The eligible-but-not-admitted article has the strongest heuristic;
|
||||
// it must not shift the deep set's percentiles or receive a utility.
|
||||
let ranking = ranking(10, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 5.0), candidate(2, 5.0), candidate(3, 5.0)];
|
||||
candidates[2].stage = "triaged".into();
|
||||
candidates[2].excluded_reason = Some("not_admitted".into());
|
||||
candidates[2].signals.heuristic = Some(99.0);
|
||||
candidates[1].signals.heuristic = Some(5.0);
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(
|
||||
candidates[0].signals.norm["heuristic"], 0.5,
|
||||
"ties share a percentile"
|
||||
);
|
||||
assert_eq!(candidates[1].signals.norm["heuristic"], 0.5);
|
||||
assert!(candidates[2].utility.is_none());
|
||||
assert!(candidates[2].rank_utility.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,593 +0,0 @@
|
||||
//! Stage A — batched LLM scoring (spec §3.6).
|
||||
//!
|
||||
//! Batches of `deepseek.score_batch_size` articles per request. Per article we
|
||||
//! send title, source feed, author, word count, social stats, sources list and a
|
||||
//! ~200-word excerpt; the model returns one JSON object per article.
|
||||
//!
|
||||
//! Parsing is deliberately forgiving: one malformed item must not cost us the
|
||||
//! other eleven, and a failed batch must not fail the run.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::llm::{LlmClient, LlmError, strip_code_fence};
|
||||
use super::{prompt_text, truncate_words};
|
||||
use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind};
|
||||
|
||||
/// Words of article text sent per candidate in stage A (§3.6).
|
||||
pub const EXCERPT_WORDS: usize = 200;
|
||||
|
||||
/// One element of the stage-A JSON response (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreItem {
|
||||
pub id: ArticleId,
|
||||
/// 0–10.
|
||||
pub score: f64,
|
||||
pub category: String,
|
||||
/// ≤ 20 words.
|
||||
#[serde(default)]
|
||||
pub rationale: String,
|
||||
#[serde(default)]
|
||||
pub is_paywalled_guess: bool,
|
||||
}
|
||||
|
||||
impl From<ScoreItem> for LlmScore {
|
||||
fn from(i: ScoreItem) -> Self {
|
||||
LlmScore {
|
||||
score: i.score,
|
||||
category: i.category,
|
||||
rationale: i.rationale,
|
||||
is_paywalled_guess: i.is_paywalled_guess,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Envelope the model is asked to return (`{"articles": [...]}`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreResponse {
|
||||
#[serde(default)]
|
||||
pub articles: Vec<ScoreItem>,
|
||||
}
|
||||
|
||||
/// The invariant instruction block for stage A. Everything article-specific goes
|
||||
/// in the per-batch tail so this prefix stays cacheable (§3.6).
|
||||
pub const SCORE_INSTRUCTIONS: &str = "\
|
||||
TASK: score a batch of candidate articles for today's issue of The Daily EPUB.
|
||||
|
||||
Judge each article against the reader profile in your system prompt — not against \
|
||||
a general audience, and not against what is objectively newsworthy.
|
||||
|
||||
Return one object per input article with these fields:
|
||||
\"id\" integer, copied exactly from the input
|
||||
\"score\" number 0-10, the rubric below
|
||||
\"category\" one short label from the palette below
|
||||
\"rationale\" at most 20 words, concrete, no hedging, no restating the title
|
||||
\"is_paywalled_guess\" true when the text looks truncated, teaser-like or paywalled
|
||||
|
||||
SCORING RUBRIC — calibrate hard; a normal day averages about 4, and a 9 should \
|
||||
appear a couple of times a week, not a couple of times a day:
|
||||
9-10 Exceptional. Original reporting, a deep technical dive, or an essay he \
|
||||
will still be thinking about next week. Evident effort and a real point of view.
|
||||
7-8 Strong. A well-made long-form piece squarely in his interests, or an \
|
||||
outstanding piece outside them.
|
||||
5-6 Worth a slot on a thin day. Solid, useful, a little thin or a little \
|
||||
familiar.
|
||||
3-4 Marginal. Competent news-of-the-day, short posts, incremental updates, \
|
||||
good writing about an over-covered story.
|
||||
1-2 Weak. Announcements, changelogs and release notes, link roundups, \
|
||||
listicles, rewrites of a story available at the source, thin AI-industry churn.
|
||||
0 Unusable. Press releases, sponsored content, engagement bait, spam, \
|
||||
pure crypto promotion, or an entry with no readable body.
|
||||
|
||||
CALIBRATION NOTES
|
||||
- Length alone is not quality; padding scores worse than a tight short piece. But \
|
||||
between two equally good pieces, prefer the one with more substance.
|
||||
- Social proof is evidence, not a verdict: hundreds of HN points mean a critical \
|
||||
audience read it; a quiet post from a good blog can still outrank it.
|
||||
- \"came via scour\" means the story already matched one of his standing \
|
||||
interests. \"came via hn_frontpage\" means it cleared HN's front page.
|
||||
- Boston/New England local stories and ultra-niche community news get a genuine \
|
||||
lift — this paper wants them.
|
||||
- Wire-service world/US news should score low here: the World Briefing section \
|
||||
covers that separately.
|
||||
- Excerpt-only or paywalled text is a real cost to the reader; score it lower \
|
||||
unless the piece is clearly excellent.
|
||||
|
||||
Return JSON exactly in this shape, with one entry per input article and nothing \
|
||||
else:
|
||||
{\"articles\": [{\"id\": 123, \"score\": 7.5, \"category\": \"Tech & Engineering\", \
|
||||
\"rationale\": \"first-hand account of migrating 40TB off Postgres\", \
|
||||
\"is_paywalled_guess\": false}]}";
|
||||
|
||||
/// Render the user prompt for one batch (§3.6).
|
||||
pub fn build_batch_prompt(batch: &[ScoredArticle], sections: &[String]) -> String {
|
||||
let mut prompt = String::with_capacity(4096 + batch.len() * 1500);
|
||||
prompt.push_str(SCORE_INSTRUCTIONS);
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nCATEGORY PALETTE (use one of these exact strings): {}\n\nARTICLES ({} in this batch)\n",
|
||||
sections.join(" | "),
|
||||
batch.len()
|
||||
);
|
||||
for candidate in batch {
|
||||
prompt.push('\n');
|
||||
prompt.push_str(&render_candidate(candidate));
|
||||
}
|
||||
prompt
|
||||
}
|
||||
|
||||
/// One article's block in the stage-A prompt (§3.6).
|
||||
fn render_candidate(candidate: &ScoredArticle) -> String {
|
||||
let a = &candidate.article;
|
||||
let mut block = String::with_capacity(1500);
|
||||
let _ = writeln!(block, "--- id: {}", a.id);
|
||||
let _ = writeln!(block, "title: {}", a.title.trim());
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"feed: {}{}",
|
||||
if a.feed_title.is_empty() {
|
||||
"unknown"
|
||||
} else {
|
||||
a.feed_title.trim()
|
||||
},
|
||||
a.category
|
||||
.as_deref()
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(|c| format!(" (category: {c})"))
|
||||
.unwrap_or_default()
|
||||
);
|
||||
if let Some(author) = a.author.as_deref().filter(|s| !s.trim().is_empty()) {
|
||||
let _ = writeln!(block, "author: {}", author.trim());
|
||||
}
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"length: {} words (~{} min read){}",
|
||||
a.word_count,
|
||||
a.reading_minutes(),
|
||||
if a.excerpt_only {
|
||||
" [EXCERPT ONLY — full text unavailable]"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
let _ = writeln!(block, "social: {}", social_line(candidate));
|
||||
let _ = writeln!(block, "came via: {}", sources_line(candidate));
|
||||
let excerpt = truncate_words(&prompt_text(&a.content_html), EXCERPT_WORDS);
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"excerpt: {}",
|
||||
if excerpt.is_empty() {
|
||||
"(no body text extracted)"
|
||||
} else {
|
||||
&excerpt
|
||||
}
|
||||
);
|
||||
block
|
||||
}
|
||||
|
||||
fn social_line(candidate: &ScoredArticle) -> String {
|
||||
if candidate.article.social.is_empty() {
|
||||
return "none found".into();
|
||||
}
|
||||
let mut parts: Vec<String> = candidate
|
||||
.article
|
||||
.social
|
||||
.iter()
|
||||
.map(|s| {
|
||||
format!(
|
||||
"{} {} points / {} comments",
|
||||
s.source.display_name(),
|
||||
s.score,
|
||||
s.num_comments
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
parts.push(format!("composite {:.2}", candidate.social_score));
|
||||
parts.join("; ")
|
||||
}
|
||||
|
||||
fn sources_line(candidate: &ScoredArticle) -> String {
|
||||
let mut kinds: Vec<&str> = candidate
|
||||
.article
|
||||
.sources
|
||||
.iter()
|
||||
.map(|s| match s.kind {
|
||||
SourceKind::Scour => "scour",
|
||||
SourceKind::HnFrontpage => "hn_frontpage",
|
||||
SourceKind::Lobsters => "lobsters",
|
||||
SourceKind::Reddit => "reddit",
|
||||
SourceKind::Feed => "feed",
|
||||
})
|
||||
.collect();
|
||||
kinds.sort_unstable();
|
||||
kinds.dedup();
|
||||
if candidate.auto_include {
|
||||
kinds.push("always-include feed (cannot be dropped)");
|
||||
}
|
||||
if kinds.is_empty() {
|
||||
"feed".into()
|
||||
} else {
|
||||
kinds.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response parsing (§3.6: tolerate anything the model does to us)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Keys the model might wrap the array in, in preference order.
|
||||
const ARRAY_KEYS: &[&str] = &["articles", "scores", "results", "items", "data"];
|
||||
|
||||
/// Parse a stage-A response leniently: missing optional fields default, scores
|
||||
/// are clamped to 0–10, and malformed items are skipped with a warning (§3.6).
|
||||
pub fn parse_score_response(raw: &str) -> Vec<ScoreItem> {
|
||||
let cleaned = strip_code_fence(raw);
|
||||
let value: Value = match serde_json::from_str(cleaned) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "stage A response was not JSON at all");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let array = match &value {
|
||||
Value::Array(items) => Some(items),
|
||||
Value::Object(map) => ARRAY_KEYS
|
||||
.iter()
|
||||
.find_map(|k| map.get(*k).and_then(Value::as_array))
|
||||
// Some models return {"1234": {...}} or a single bare object.
|
||||
.or_else(|| map.values().find_map(Value::as_array)),
|
||||
_ => None,
|
||||
};
|
||||
let Some(array) = array else {
|
||||
tracing::warn!("stage A response contained no array of scores");
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut out = Vec::with_capacity(array.len());
|
||||
let mut skipped = 0usize;
|
||||
for item in array {
|
||||
match parse_item(item) {
|
||||
Some(parsed) => out.push(parsed),
|
||||
None => {
|
||||
skipped += 1;
|
||||
tracing::warn!(item = %truncate_debug(item), "skipping malformed stage A item");
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, kept = out.len(), "stage A items were dropped");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn parse_item(item: &Value) -> Option<ScoreItem> {
|
||||
let obj = item.as_object()?;
|
||||
let id = obj.get("id").and_then(as_i64_lenient)?;
|
||||
let score = obj
|
||||
.get("score")
|
||||
.and_then(as_f64_lenient)
|
||||
.or_else(|| obj.get("rating").and_then(as_f64_lenient))?;
|
||||
Some(ScoreItem {
|
||||
id,
|
||||
score: score.clamp(0.0, 10.0),
|
||||
category: obj
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
rationale: obj
|
||||
.get("rationale")
|
||||
.or_else(|| obj.get("reason"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
is_paywalled_guess: obj
|
||||
.get("is_paywalled_guess")
|
||||
.or_else(|| obj.get("paywalled"))
|
||||
.and_then(as_bool_lenient)
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn as_i64_lenient(v: &Value) -> Option<i64> {
|
||||
v.as_i64()
|
||||
.or_else(|| v.as_f64().map(|f| f as i64))
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
}
|
||||
|
||||
fn as_f64_lenient(v: &Value) -> Option<f64> {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
.filter(|f| f.is_finite())
|
||||
}
|
||||
|
||||
fn as_bool_lenient(v: &Value) -> Option<bool> {
|
||||
v.as_bool().or_else(|| match v.as_str()?.trim() {
|
||||
"true" | "yes" => Some(true),
|
||||
"false" | "no" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn truncate_debug(v: &Value) -> String {
|
||||
v.to_string().chars().take(160).collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Score every candidate, filling in [`ScoredArticle::llm`] (§3.6).
|
||||
///
|
||||
/// Batches that fail are logged and left unscored rather than aborting the run.
|
||||
/// Returns how many candidates came back with a score.
|
||||
pub async fn score_all(
|
||||
llm: &LlmClient,
|
||||
candidates: &mut [ScoredArticle],
|
||||
batch_size: usize,
|
||||
sections: &[String],
|
||||
temperature: f32,
|
||||
) -> Result<usize, LlmError> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let batch_size = batch_size.max(1);
|
||||
let batches = candidates.len().div_ceil(batch_size);
|
||||
let mut scores: HashMap<ArticleId, LlmScore> = HashMap::with_capacity(candidates.len());
|
||||
|
||||
for (n, batch) in candidates.chunks(batch_size).enumerate() {
|
||||
if let Err(e) = llm.meter.check_budget() {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
unscored = candidates.len() - scores.len(),
|
||||
"COST CEILING HIT during stage A scoring — remaining batches skipped; \
|
||||
the lineup will fall back to heuristic ranking for them"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let prompt = build_batch_prompt(batch, sections);
|
||||
tracing::debug!(
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
articles = batch.len(),
|
||||
approx_tokens = super::approx_tokens(&prompt),
|
||||
"stage A request"
|
||||
);
|
||||
match llm.complete(&prompt, temperature, true).await {
|
||||
Ok(raw) => {
|
||||
let items = parse_score_response(&raw);
|
||||
if items.is_empty() {
|
||||
tracing::warn!(
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
"stage A batch returned no scores"
|
||||
);
|
||||
}
|
||||
for item in items {
|
||||
scores.insert(item.id, item.into());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(batch = n + 1, of = batches, error = %e,
|
||||
"stage A batch failed; its articles stay unscored");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut applied = 0usize;
|
||||
for candidate in candidates.iter_mut() {
|
||||
if let Some(score) = scores.remove(&candidate.article.id) {
|
||||
candidate.llm = Some(score);
|
||||
applied += 1;
|
||||
}
|
||||
}
|
||||
if !scores.is_empty() {
|
||||
tracing::warn!(
|
||||
unknown_ids = scores.len(),
|
||||
"stage A returned scores for ids that were not in the batch"
|
||||
);
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DeepseekConfig;
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::curate::prefilter::tests::{article, via, with_social};
|
||||
use crate::types::TokenUsage;
|
||||
use std::sync::Arc;
|
||||
|
||||
const BATCH_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch.json"
|
||||
));
|
||||
const MESSY_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch_messy.json"
|
||||
));
|
||||
|
||||
fn sections() -> Vec<String> {
|
||||
crate::config::CurationConfig::default().sections
|
||||
}
|
||||
|
||||
fn candidate(id: i64, title: &str, words: i64) -> ScoredArticle {
|
||||
ScoredArticle {
|
||||
article: article(id, title, words),
|
||||
prefilter_score: 50.0,
|
||||
social_score: 0.0,
|
||||
feed_prior: 0.5,
|
||||
llm: None,
|
||||
auto_include: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_prompt_carries_every_documented_signal() {
|
||||
let mut c = candidate(12, "Migrating 40TB off Postgres", 3200);
|
||||
c.article = via(
|
||||
with_social(c.article, 342, 210),
|
||||
SourceKind::HnFrontpage,
|
||||
9001,
|
||||
);
|
||||
c.social_score = c.article.social_score();
|
||||
c.auto_include = true;
|
||||
let prompt = build_batch_prompt(&[c], §ions());
|
||||
|
||||
assert!(prompt.starts_with(SCORE_INSTRUCTIONS));
|
||||
assert!(prompt.contains("--- id: 12"));
|
||||
assert!(prompt.contains("title: Migrating 40TB off Postgres"));
|
||||
assert!(prompt.contains("feed: Some Blog (category: Tech)"));
|
||||
assert!(prompt.contains("author: A. Writer"));
|
||||
assert!(prompt.contains("length: 3200 words"));
|
||||
assert!(prompt.contains("HN 342 points / 210 comments"));
|
||||
assert!(prompt.contains("hn_frontpage"));
|
||||
assert!(prompt.contains("always-include feed"));
|
||||
assert!(prompt.contains("excerpt: word word"));
|
||||
assert!(prompt.contains("Tech & Engineering"));
|
||||
// The excerpt is capped.
|
||||
let excerpt_line = prompt
|
||||
.lines()
|
||||
.find(|l| l.starts_with("excerpt:"))
|
||||
.expect("excerpt line");
|
||||
assert!(excerpt_line.split_whitespace().count() <= EXCERPT_WORDS + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_realistic_deepseek_batch() {
|
||||
let items = parse_score_response(BATCH_FIXTURE);
|
||||
assert_eq!(items.len(), 4);
|
||||
assert_eq!(items[0].id, 101);
|
||||
assert!((items[0].score - 8.5).abs() < 1e-9);
|
||||
assert_eq!(items[0].category, "Tech & Engineering");
|
||||
assert!(items[0].rationale.split_whitespace().count() <= 20);
|
||||
assert!(!items[0].is_paywalled_guess);
|
||||
assert!(items[3].is_paywalled_guess);
|
||||
let score: LlmScore = items[0].clone().into();
|
||||
assert_eq!(score.category, "Tech & Engineering");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_survives_everything_a_model_might_do() {
|
||||
let items = parse_score_response(MESSY_FIXTURE);
|
||||
let ids: Vec<ArticleId> = items.iter().map(|i| i.id).collect();
|
||||
// 201 fine; 202 string score clamped; 203 missing rationale/category;
|
||||
// 204 out-of-range clamped; the two malformed entries are dropped.
|
||||
assert_eq!(ids, vec![201, 202, 203, 204]);
|
||||
assert!((items[1].score - 6.0).abs() < 1e-9);
|
||||
assert_eq!(items[2].rationale, "");
|
||||
assert_eq!(items[2].category, "");
|
||||
assert!(
|
||||
(items[3].score - 10.0).abs() < 1e-9,
|
||||
"clamped to the 0-10 range"
|
||||
);
|
||||
assert!(items.iter().all(|i| (0.0..=10.0).contains(&i.score)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_tolerates_fences_arrays_and_junk() {
|
||||
assert_eq!(
|
||||
parse_score_response("```json\n{\"articles\":[{\"id\":1,\"score\":5}]}\n```").len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(parse_score_response("[{\"id\": 2, \"score\": 3}]").len(), 1);
|
||||
assert_eq!(
|
||||
parse_score_response("{\"results\":[{\"id\":3,\"score\":\"4.5\"}]}")[0].score,
|
||||
4.5
|
||||
);
|
||||
assert!(parse_score_response("I'm sorry, I can't do that").is_empty());
|
||||
assert!(parse_score_response("{\"articles\": {}}").is_empty());
|
||||
}
|
||||
|
||||
fn client(backend: Arc<MockBackend>, limit_usd: f64) -> LlmClient {
|
||||
LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), limit_usd),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scores_are_applied_batch_by_batch() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":8,"category":"Tech & Engineering","rationale":"good"},
|
||||
{"id":2,"score":2,"category":"Niche Corner","rationale":"thin"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":3,"score":6.5,"category":"Culture & Essays","rationale":"solid"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
let mut candidates = vec![
|
||||
candidate(1, "One", 1000),
|
||||
candidate(2, "Two", 1000),
|
||||
candidate(3, "Three", 1000),
|
||||
];
|
||||
let scored = score_all(&llm, &mut candidates, 2, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 3);
|
||||
assert_eq!(backend.calls(), 2, "batched by score_batch_size");
|
||||
assert_eq!(candidates[0].llm.as_ref().map(|l| l.score), Some(8.0));
|
||||
assert_eq!(candidates[2].llm.as_ref().map(|l| l.score), Some(6.5));
|
||||
// combined_score now reflects the LLM verdict.
|
||||
assert!(candidates[0].combined_score() > candidates[1].combined_score());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_batch_does_not_sink_the_run() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push_error("500 upstream exploded");
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":7,"category":"Top Stories","rationale":"ok"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring must not abort");
|
||||
assert_eq!(scored, 1);
|
||||
assert!(candidates[0].llm.is_none());
|
||||
assert!(candidates[1].llm.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoring_stops_when_the_budget_is_gone() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
// First batch alone blows a $0.05 ceiling ($0.14 per 1M input tokens).
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 0,
|
||||
output_tokens: 0,
|
||||
},
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 0.05);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 1, "only the first batch ran");
|
||||
assert_eq!(backend.calls(), 1);
|
||||
assert!(llm.meter.budget_exceeded());
|
||||
}
|
||||
}
|
||||
@@ -1,1000 +0,0 @@
|
||||
//! Stage B — lineup selection (spec §3.6).
|
||||
//!
|
||||
//! One call: send the top ~40 candidates by [`ScoredArticle::combined_score`] with
|
||||
//! their rationales; the model returns the final 15–25 picks, each with a section
|
||||
//! from the configured palette, an ordering, and exactly one `lead_story`.
|
||||
//!
|
||||
//! The model's answer is treated as a proposal, never as gospel: sections are
|
||||
//! validated against the palette, the lead is forced to be unique, auto-include
|
||||
//! feeds are re-inserted if they were dropped, and the size is clamped to
|
||||
//! `target_article_count ± 5`.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use jiff::civil::Date;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::llm::{LlmClient, LlmError, strip_code_fence};
|
||||
use super::{prompt_text, truncate_words};
|
||||
use crate::types::{ArticleId, Lineup, Pick, ScoredArticle, SourceKind, WORLD_BRIEFING_SECTION};
|
||||
|
||||
/// How many candidates are offered to stage B (§3.6).
|
||||
pub const SHORTLIST_SIZE: usize = 40;
|
||||
/// How far the final count may drift from `target_article_count` (§3.6: 15–25
|
||||
/// around a default target of 20).
|
||||
pub const TARGET_TOLERANCE: usize = 5;
|
||||
/// Words of lead-in text shown per candidate in the stage-B prompt.
|
||||
const BLURB_WORDS: usize = 45;
|
||||
|
||||
/// One element of the stage-B JSON response (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SelectionItem {
|
||||
pub id: ArticleId,
|
||||
/// Must be one of `curation.sections`.
|
||||
pub section: String,
|
||||
pub position: i64,
|
||||
#[serde(default)]
|
||||
pub lead_story: bool,
|
||||
}
|
||||
|
||||
/// Envelope the model is asked to return.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SelectionResponse {
|
||||
#[serde(default)]
|
||||
pub picks: Vec<SelectionItem>,
|
||||
}
|
||||
|
||||
/// The invariant instruction block for stage B (§3.6).
|
||||
pub const SELECT_INSTRUCTIONS: &str = "\
|
||||
TASK: assemble today's issue of The Daily EPUB from the shortlist below.
|
||||
|
||||
You are choosing what one specific reader — the profile in your system prompt — \
|
||||
will actually read on an e-ink screen over breakfast. Build a paper, not a \
|
||||
ranking: it should have a shape, a range of subjects, and a clear front page.
|
||||
|
||||
RULES
|
||||
1. Pick articles by id from the shortlist only. Never invent an id.
|
||||
2. Give every pick a section from the palette below, spelled exactly as given.
|
||||
3. Number picks within each section from 1 upward, best first.
|
||||
4. Flag exactly one pick as \"lead_story\": true — the day's strongest, most \
|
||||
substantial piece. It must sit in the first section you use.
|
||||
5. Any candidate marked \"always-include\" MUST appear; place it in \"From the \
|
||||
Blogroll\" unless it clearly belongs elsewhere.
|
||||
6. Do not select two articles that tell the same story; keep the better one.
|
||||
|
||||
EDITORIAL JUDGEMENT
|
||||
- Favour depth over coverage: a slim issue of excellent pieces beats a full one \
|
||||
padded with filler. Drop anything you would not defend.
|
||||
- Mix the day up. Several long technical dives in a row is a bad breakfast; \
|
||||
alternate register and subject across sections.
|
||||
- Keep the local and ultra-niche picks — a Boston story and a small-scene story \
|
||||
are worth more here than a third AI-industry item.
|
||||
- Score is evidence, not an instruction: overrule it when the paper reads better \
|
||||
for it, and say so through your placement.
|
||||
- Leave a section out entirely rather than padding it; empty sections are dropped.
|
||||
|
||||
Return JSON exactly in this shape and nothing else:
|
||||
{\"picks\": [{\"id\": 123, \"section\": \"Top Stories\", \"position\": 1, \
|
||||
\"lead_story\": true}]}";
|
||||
|
||||
/// Render the stage-B user prompt (§3.6).
|
||||
pub fn build_prompt(shortlist: &[ScoredArticle], sections: &[String], target: usize) -> String {
|
||||
let (min, max) = size_bounds(target);
|
||||
let mut prompt = String::with_capacity(2048 + shortlist.len() * 400);
|
||||
prompt.push_str(SELECT_INSTRUCTIONS);
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nSECTION PALETTE (exact strings, use only these): {}\n\
|
||||
Reserved and unavailable: \"{WORLD_BRIEFING_SECTION}\" is compiled separately.\n\n\
|
||||
SIZE: choose {target} articles; never fewer than {min} and never more than {max}.\n\n\
|
||||
SHORTLIST ({} candidates, best-ranked first)\n",
|
||||
sections.join(" | "),
|
||||
shortlist.len()
|
||||
);
|
||||
for candidate in shortlist {
|
||||
prompt.push('\n');
|
||||
prompt.push_str(&render_candidate(candidate));
|
||||
}
|
||||
prompt
|
||||
}
|
||||
|
||||
fn render_candidate(candidate: &ScoredArticle) -> String {
|
||||
let a = &candidate.article;
|
||||
let mut block = String::with_capacity(400);
|
||||
let _ = writeln!(block, "--- id: {}", a.id);
|
||||
let _ = writeln!(block, "title: {}", a.title.trim());
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"feed: {} · {} words (~{} min){}",
|
||||
if a.feed_title.is_empty() {
|
||||
"unknown"
|
||||
} else {
|
||||
a.feed_title.trim()
|
||||
},
|
||||
a.word_count,
|
||||
a.reading_minutes(),
|
||||
if a.excerpt_only {
|
||||
" · EXCERPT ONLY"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
match candidate.llm.as_ref() {
|
||||
Some(llm) => {
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"score: {:.1} ({}) — {}",
|
||||
llm.score,
|
||||
if llm.category.is_empty() {
|
||||
"uncategorized"
|
||||
} else {
|
||||
llm.category.as_str()
|
||||
},
|
||||
llm.rationale.trim()
|
||||
);
|
||||
}
|
||||
None => {
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"score: unscored (heuristic rank {:.0}/100)",
|
||||
candidate.prefilter_score
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"signals: social {:.2}; feed prior {:.2}; via {}{}",
|
||||
candidate.social_score,
|
||||
candidate.feed_prior,
|
||||
source_kinds(candidate),
|
||||
if candidate.auto_include {
|
||||
"; ALWAYS-INCLUDE"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
let blurb = truncate_words(&prompt_text(&a.content_html), BLURB_WORDS);
|
||||
if !blurb.is_empty() {
|
||||
let _ = writeln!(block, "opening: {blurb}");
|
||||
}
|
||||
block
|
||||
}
|
||||
|
||||
fn source_kinds(candidate: &ScoredArticle) -> String {
|
||||
let mut kinds: Vec<&str> = candidate
|
||||
.article
|
||||
.sources
|
||||
.iter()
|
||||
.map(|s| match s.kind {
|
||||
SourceKind::Scour => "scour",
|
||||
SourceKind::HnFrontpage => "hn_frontpage",
|
||||
SourceKind::Lobsters => "lobsters",
|
||||
SourceKind::Reddit => "reddit",
|
||||
SourceKind::Feed => "feed",
|
||||
})
|
||||
.collect();
|
||||
kinds.sort_unstable();
|
||||
kinds.dedup();
|
||||
if kinds.is_empty() {
|
||||
"feed".into()
|
||||
} else {
|
||||
kinds.join("+")
|
||||
}
|
||||
}
|
||||
|
||||
/// `target ± TARGET_TOLERANCE`, floored at one article (§3.6).
|
||||
pub fn size_bounds(target: usize) -> (usize, usize) {
|
||||
let target = target.max(1);
|
||||
(
|
||||
target.saturating_sub(TARGET_TOLERANCE).max(1),
|
||||
target + TARGET_TOLERANCE,
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Section validation (§3.6: the model may only use the configured palette)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The section unrecognized labels fall back to.
|
||||
pub fn default_section(sections: &[String]) -> String {
|
||||
sections
|
||||
.iter()
|
||||
.find(|s| s.as_str() == "Top Stories")
|
||||
.or_else(|| sections.first())
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "Top Stories".to_string())
|
||||
}
|
||||
|
||||
/// Map whatever the model said onto the configured palette (§3.6).
|
||||
///
|
||||
/// Exact match → case-insensitive match → best word-overlap match → default.
|
||||
pub fn resolve_section(raw: &str, sections: &[String]) -> String {
|
||||
let candidate = raw.trim();
|
||||
if candidate.is_empty() || candidate.eq_ignore_ascii_case(WORLD_BRIEFING_SECTION) {
|
||||
return default_section(sections);
|
||||
}
|
||||
if let Some(exact) = sections.iter().find(|s| s.as_str() == candidate) {
|
||||
return exact.clone();
|
||||
}
|
||||
if let Some(ci) = sections.iter().find(|s| s.eq_ignore_ascii_case(candidate)) {
|
||||
return ci.clone();
|
||||
}
|
||||
let wanted = words_of(candidate);
|
||||
let best = sections
|
||||
.iter()
|
||||
.map(|s| (s, words_of(s).intersection(&wanted).count()))
|
||||
.filter(|(_, overlap)| *overlap > 0)
|
||||
.max_by_key(|(_, overlap)| *overlap);
|
||||
match best {
|
||||
Some((section, _)) => {
|
||||
tracing::debug!(raw = candidate, mapped = %section, "mapped an off-palette section");
|
||||
section.clone()
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(raw = candidate, "unknown section; using the default");
|
||||
default_section(sections)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn words_of(s: &str) -> HashSet<String> {
|
||||
s.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| w.len() > 2 && !w.eq_ignore_ascii_case("and"))
|
||||
.map(str::to_lowercase)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Section guess from feed metadata, used by `--skip-llm` and by top-ups (§3.6).
|
||||
pub fn heuristic_section(candidate: &ScoredArticle, sections: &[String]) -> String {
|
||||
if candidate.auto_include {
|
||||
return resolve_section("From the Blogroll", sections);
|
||||
}
|
||||
let haystack = format!(
|
||||
"{} {} {}",
|
||||
candidate.article.category.as_deref().unwrap_or_default(),
|
||||
candidate.article.feed_title,
|
||||
candidate.article.title
|
||||
)
|
||||
.to_lowercase();
|
||||
|
||||
const RULES: &[(&str, &[&str])] = &[
|
||||
(
|
||||
"Boston & Local",
|
||||
&[
|
||||
"boston",
|
||||
"massachusetts",
|
||||
"cambridge",
|
||||
"mbta",
|
||||
"new england",
|
||||
],
|
||||
),
|
||||
(
|
||||
"AI & Machine Learning",
|
||||
&[
|
||||
" ai ",
|
||||
"ai:",
|
||||
"llm",
|
||||
"machine learning",
|
||||
"neural",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"gpt",
|
||||
"diffusion",
|
||||
],
|
||||
),
|
||||
(
|
||||
"Science & Space",
|
||||
&[
|
||||
"science",
|
||||
"space",
|
||||
"nasa",
|
||||
"astronom",
|
||||
"physics",
|
||||
"biology",
|
||||
"climate",
|
||||
"aerospace",
|
||||
],
|
||||
),
|
||||
(
|
||||
"Culture & Essays",
|
||||
&[
|
||||
"essay", "culture", "book", "fiction", "poetry", "film", "music", "art", "review",
|
||||
],
|
||||
),
|
||||
(
|
||||
"Niche Corner",
|
||||
&[
|
||||
"hobby",
|
||||
"retro",
|
||||
"keyboard",
|
||||
"board game",
|
||||
"coffee",
|
||||
"e-ink",
|
||||
"eink",
|
||||
],
|
||||
),
|
||||
(
|
||||
"Tech & Engineering",
|
||||
&[
|
||||
"tech",
|
||||
"programming",
|
||||
"engineering",
|
||||
"software",
|
||||
"developer",
|
||||
"rust",
|
||||
"linux",
|
||||
"database",
|
||||
"systems",
|
||||
"web",
|
||||
],
|
||||
),
|
||||
];
|
||||
for (section, needles) in RULES {
|
||||
if needles.iter().any(|n| haystack.contains(n))
|
||||
&& let Some(found) = sections.iter().find(|s| s.as_str() == *section)
|
||||
{
|
||||
return found.clone();
|
||||
}
|
||||
}
|
||||
default_section(sections)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Keys the model might wrap the array in.
|
||||
const ARRAY_KEYS: &[&str] = &["picks", "lineup", "articles", "selection", "items"];
|
||||
|
||||
/// Lenient parse of the stage-B response (§3.6).
|
||||
pub fn parse_selection_response(raw: &str) -> Vec<SelectionItem> {
|
||||
let cleaned = strip_code_fence(raw);
|
||||
let value: serde_json::Value = match serde_json::from_str(cleaned) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "stage B response was not JSON");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let array = match &value {
|
||||
serde_json::Value::Array(items) => Some(items),
|
||||
serde_json::Value::Object(map) => ARRAY_KEYS
|
||||
.iter()
|
||||
.find_map(|k| map.get(*k).and_then(serde_json::Value::as_array))
|
||||
.or_else(|| map.values().find_map(serde_json::Value::as_array)),
|
||||
_ => None,
|
||||
};
|
||||
let Some(array) = array else {
|
||||
tracing::warn!("stage B response contained no array of picks");
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut out = Vec::with_capacity(array.len());
|
||||
for (idx, item) in array.iter().enumerate() {
|
||||
let Some(obj) = item.as_object() else {
|
||||
tracing::warn!("skipping a non-object stage B pick");
|
||||
continue;
|
||||
};
|
||||
let Some(id) = obj.get("id").and_then(|v| {
|
||||
v.as_i64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
}) else {
|
||||
tracing::warn!("skipping a stage B pick without an id");
|
||||
continue;
|
||||
};
|
||||
out.push(SelectionItem {
|
||||
id,
|
||||
section: obj
|
||||
.get("section")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
position: obj
|
||||
.get("position")
|
||||
.and_then(|v| {
|
||||
v.as_i64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
})
|
||||
.unwrap_or(idx as i64 + 1),
|
||||
lead_story: obj
|
||||
.get("lead_story")
|
||||
.or_else(|| obj.get("is_lead"))
|
||||
.and_then(|v| {
|
||||
v.as_bool()
|
||||
.or_else(|| v.as_str().map(|s| s.eq_ignore_ascii_case("true")))
|
||||
})
|
||||
.unwrap_or(false),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Ask the model for the day's lineup, validating that every section is from the
|
||||
/// configured palette and exactly one pick is the lead (§3.6).
|
||||
pub async fn select(
|
||||
llm: &LlmClient,
|
||||
candidates: Vec<ScoredArticle>,
|
||||
sections: &[String],
|
||||
target: usize,
|
||||
date: Date,
|
||||
) -> Result<Lineup, LlmError> {
|
||||
if candidates.is_empty() {
|
||||
tracing::warn!("stage B had no candidates");
|
||||
return Ok(Lineup {
|
||||
date,
|
||||
picks: Vec::new(),
|
||||
section_order: Vec::new(),
|
||||
});
|
||||
}
|
||||
if let Err(e) = llm.meter.check_budget() {
|
||||
tracing::error!(error = %e,
|
||||
"COST CEILING HIT before stage B selection — falling back to heuristic ranking");
|
||||
return Ok(select_without_llm(candidates, sections, target, date));
|
||||
}
|
||||
|
||||
let shortlist = shortlist(&candidates, target);
|
||||
let prompt = build_prompt(&shortlist, sections, target);
|
||||
tracing::debug!(
|
||||
shortlist = shortlist.len(),
|
||||
approx_tokens = super::approx_tokens(&prompt),
|
||||
"stage B request"
|
||||
);
|
||||
|
||||
let raw = llm.complete(&prompt, llm_temperature(), true).await?;
|
||||
let items = parse_selection_response(&raw);
|
||||
if items.is_empty() {
|
||||
tracing::error!("stage B returned no usable picks; falling back to heuristic ranking");
|
||||
return Ok(select_without_llm(candidates, sections, target, date));
|
||||
}
|
||||
|
||||
let by_id: HashMap<ArticleId, &ScoredArticle> =
|
||||
candidates.iter().map(|c| (c.article.id, c)).collect();
|
||||
let mut chosen: Vec<(SelectionItem, ScoredArticle)> = Vec::with_capacity(items.len());
|
||||
let mut seen: HashSet<ArticleId> = HashSet::new();
|
||||
for item in items {
|
||||
if !seen.insert(item.id) {
|
||||
tracing::warn!(id = item.id, "stage B picked the same article twice");
|
||||
continue;
|
||||
}
|
||||
match by_id.get(&item.id) {
|
||||
Some(candidate) => chosen.push((item, (*candidate).clone())),
|
||||
None => tracing::warn!(id = item.id, "stage B invented an id that was not offered"),
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-include feeds can never be dropped (§3.5).
|
||||
for candidate in &candidates {
|
||||
if candidate.auto_include && seen.insert(candidate.article.id) {
|
||||
tracing::info!(
|
||||
id = candidate.article.id,
|
||||
title = %candidate.article.title,
|
||||
"re-inserting an always-include article the model dropped"
|
||||
);
|
||||
chosen.push((
|
||||
SelectionItem {
|
||||
id: candidate.article.id,
|
||||
section: "From the Blogroll".into(),
|
||||
position: i64::MAX,
|
||||
lead_story: false,
|
||||
},
|
||||
candidate.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let lineup = assemble(chosen, &candidates, sections, target, date);
|
||||
tracing::info!(
|
||||
picks = lineup.picks.len(),
|
||||
sections = lineup.section_order.len(),
|
||||
lead = lineup.lead().map(|p| p.article.id),
|
||||
"stage B lineup ready"
|
||||
);
|
||||
Ok(lineup)
|
||||
}
|
||||
|
||||
/// Stage B runs at the scoring temperature: this is a judgement call, not prose.
|
||||
fn llm_temperature() -> f32 {
|
||||
0.4
|
||||
}
|
||||
|
||||
/// Top [`SHORTLIST_SIZE`] candidates by combined score, always including the
|
||||
/// auto-includes (§3.6).
|
||||
fn shortlist(candidates: &[ScoredArticle], target: usize) -> Vec<ScoredArticle> {
|
||||
let mut ranked: Vec<ScoredArticle> = candidates.to_vec();
|
||||
sort_by_combined(&mut ranked);
|
||||
let keep = SHORTLIST_SIZE.max(target * 2);
|
||||
if ranked.len() <= keep {
|
||||
return ranked;
|
||||
}
|
||||
let (head, tail) = ranked.split_at(keep);
|
||||
let mut out = head.to_vec();
|
||||
out.extend(tail.iter().filter(|c| c.auto_include).cloned());
|
||||
out
|
||||
}
|
||||
|
||||
fn sort_by_combined(candidates: &mut [ScoredArticle]) {
|
||||
candidates.sort_by(|a, b| {
|
||||
b.combined_score()
|
||||
.partial_cmp(&a.combined_score())
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| a.article.id.cmp(&b.article.id))
|
||||
});
|
||||
}
|
||||
|
||||
/// Turn validated picks into a [`Lineup`]: clamp the size, force a single lead,
|
||||
/// order the sections and renumber positions (§3.6).
|
||||
fn assemble(
|
||||
mut chosen: Vec<(SelectionItem, ScoredArticle)>,
|
||||
all: &[ScoredArticle],
|
||||
sections: &[String],
|
||||
target: usize,
|
||||
date: Date,
|
||||
) -> Lineup {
|
||||
let (min, max) = size_bounds(target);
|
||||
|
||||
// Too many: drop the weakest non-auto-include picks.
|
||||
if chosen.len() > max {
|
||||
chosen.sort_by(|a, b| {
|
||||
b.1.auto_include.cmp(&a.1.auto_include).then_with(|| {
|
||||
b.1.combined_score()
|
||||
.partial_cmp(&a.1.combined_score())
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
});
|
||||
let dropped = chosen.len() - max;
|
||||
chosen.truncate(max);
|
||||
tracing::info!(dropped, max, "trimmed the lineup to the size ceiling");
|
||||
}
|
||||
|
||||
// Too few: top up from the best unpicked candidates.
|
||||
if chosen.len() < min {
|
||||
let taken: HashSet<ArticleId> = chosen.iter().map(|(i, _)| i.id).collect();
|
||||
let mut rest: Vec<ScoredArticle> = all
|
||||
.iter()
|
||||
.filter(|c| !taken.contains(&c.article.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
sort_by_combined(&mut rest);
|
||||
let wanted = min - chosen.len();
|
||||
let added = rest.len().min(wanted);
|
||||
for candidate in rest.into_iter().take(wanted) {
|
||||
let section = heuristic_section(&candidate, sections);
|
||||
chosen.push((
|
||||
SelectionItem {
|
||||
id: candidate.article.id,
|
||||
section,
|
||||
position: i64::MAX,
|
||||
lead_story: false,
|
||||
},
|
||||
candidate,
|
||||
));
|
||||
}
|
||||
tracing::info!(added, min, "topped the lineup up to the size floor");
|
||||
}
|
||||
|
||||
// Normalize sections and pick the section order.
|
||||
for (item, _) in chosen.iter_mut() {
|
||||
item.section = resolve_section(&item.section, sections);
|
||||
}
|
||||
let used: HashSet<&str> = chosen.iter().map(|(i, _)| i.section.as_str()).collect();
|
||||
let mut section_order: Vec<String> = sections
|
||||
.iter()
|
||||
.filter(|s| used.contains(s.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
for (item, _) in &chosen {
|
||||
if !section_order.contains(&item.section) {
|
||||
section_order.push(item.section.clone());
|
||||
}
|
||||
}
|
||||
let section_rank: HashMap<&str, usize> = section_order
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, s)| (s.as_str(), i))
|
||||
.collect();
|
||||
|
||||
// Order: section, then the model's position, then quality, then id.
|
||||
chosen.sort_by(|a, b| {
|
||||
section_rank
|
||||
.get(a.0.section.as_str())
|
||||
.cmp(§ion_rank.get(b.0.section.as_str()))
|
||||
.then_with(|| a.0.position.cmp(&b.0.position))
|
||||
.then_with(|| {
|
||||
b.1.combined_score()
|
||||
.partial_cmp(&a.1.combined_score())
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
.then_with(|| a.1.article.id.cmp(&b.1.article.id))
|
||||
});
|
||||
|
||||
// Exactly one lead, and it must live in the first section (§3.6 rule 4).
|
||||
let lead_id = chosen
|
||||
.iter()
|
||||
.find(|(item, _)| item.lead_story)
|
||||
.filter(|(item, _)| section_rank.get(item.section.as_str()) == Some(&0))
|
||||
.or_else(|| chosen.first())
|
||||
.map(|(item, _)| item.id);
|
||||
|
||||
let mut per_section: BTreeMap<String, i64> = BTreeMap::new();
|
||||
let picks = chosen
|
||||
.into_iter()
|
||||
.map(|(item, candidate)| {
|
||||
let position = per_section
|
||||
.entry(item.section.clone())
|
||||
.and_modify(|n| *n += 1)
|
||||
.or_insert(1);
|
||||
Pick {
|
||||
section: item.section,
|
||||
position: *position,
|
||||
is_lead: Some(item.id) == lead_id,
|
||||
summary: None,
|
||||
llm: candidate.llm.clone(),
|
||||
discussion: None,
|
||||
article: candidate.article,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Lineup {
|
||||
date,
|
||||
picks,
|
||||
section_order,
|
||||
}
|
||||
}
|
||||
|
||||
/// `--skip-llm` fallback: take the top `target` by prefilter score and bucket them
|
||||
/// into sections by feed category (notes §6).
|
||||
pub fn select_without_llm(
|
||||
candidates: Vec<ScoredArticle>,
|
||||
sections: &[String],
|
||||
target: usize,
|
||||
date: Date,
|
||||
) -> Lineup {
|
||||
let mut ranked = candidates.clone();
|
||||
super::prefilter::sort_by_prefilter(&mut ranked);
|
||||
|
||||
let mut chosen: Vec<(SelectionItem, ScoredArticle)> = Vec::new();
|
||||
let mut seen: HashSet<ArticleId> = HashSet::new();
|
||||
for candidate in ranked.into_iter() {
|
||||
let auto = candidate.auto_include;
|
||||
if chosen.len() >= target && !auto {
|
||||
continue;
|
||||
}
|
||||
if !seen.insert(candidate.article.id) {
|
||||
continue;
|
||||
}
|
||||
let section = heuristic_section(&candidate, sections);
|
||||
chosen.push((
|
||||
SelectionItem {
|
||||
id: candidate.article.id,
|
||||
section,
|
||||
position: chosen.len() as i64 + 1,
|
||||
lead_story: false,
|
||||
},
|
||||
candidate,
|
||||
));
|
||||
}
|
||||
let lineup = assemble(chosen, &candidates, sections, target, date);
|
||||
tracing::info!(
|
||||
picks = lineup.picks.len(),
|
||||
sections = lineup.section_order.len(),
|
||||
"skip-llm lineup ready"
|
||||
);
|
||||
lineup
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{CurationConfig, DeepseekConfig};
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::types::{LlmScore, TokenUsage};
|
||||
use std::sync::Arc;
|
||||
|
||||
const LINEUP_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_lineup.json"
|
||||
));
|
||||
|
||||
fn sections() -> Vec<String> {
|
||||
CurationConfig::default().sections
|
||||
}
|
||||
|
||||
fn date() -> Date {
|
||||
"2026-08-15".parse().expect("date")
|
||||
}
|
||||
|
||||
fn candidate(id: i64, title: &str, words: i64, score: f64) -> ScoredArticle {
|
||||
ScoredArticle {
|
||||
article: article(id, title, words),
|
||||
prefilter_score: 40.0 + score,
|
||||
social_score: 1.0,
|
||||
feed_prior: 0.5,
|
||||
llm: Some(LlmScore {
|
||||
score,
|
||||
category: "Tech & Engineering".into(),
|
||||
rationale: "solid".into(),
|
||||
is_paywalled_guess: false,
|
||||
}),
|
||||
auto_include: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn candidates(n: i64) -> Vec<ScoredArticle> {
|
||||
(1..=n)
|
||||
.map(|i| {
|
||||
candidate(
|
||||
i,
|
||||
&format!("Article {i}"),
|
||||
500 + i * 10,
|
||||
(10.0 - i as f64 * 0.1).max(0.0),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn section_resolution_maps_onto_the_palette() {
|
||||
let s = sections();
|
||||
assert_eq!(resolve_section("Top Stories", &s), "Top Stories");
|
||||
assert_eq!(resolve_section(" top stories ", &s), "Top Stories");
|
||||
assert_eq!(
|
||||
resolve_section("Technology & Engineering", &s),
|
||||
"Tech & Engineering"
|
||||
);
|
||||
assert_eq!(resolve_section("Science", &s), "Science & Space");
|
||||
assert_eq!(resolve_section("Sports", &s), "Top Stories");
|
||||
assert_eq!(resolve_section("", &s), "Top Stories");
|
||||
// The reserved section is never allowed through (§3.6).
|
||||
assert_eq!(resolve_section(WORLD_BRIEFING_SECTION, &s), "Top Stories");
|
||||
// A palette without "Top Stories" falls back to its first entry.
|
||||
let tiny = vec!["Niche Corner".to_string()];
|
||||
assert_eq!(resolve_section("Whatever", &tiny), "Niche Corner");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heuristic_sections_follow_feed_metadata() {
|
||||
let s = sections();
|
||||
let mut c = candidate(1, "MBTA slow zones, charted", 900, 6.0);
|
||||
c.article.category = Some("News".into());
|
||||
assert_eq!(heuristic_section(&c, &s), "Boston & Local");
|
||||
|
||||
let mut ai = candidate(2, "A new LLM benchmark", 900, 6.0);
|
||||
ai.article.category = Some("Machine Learning".into());
|
||||
assert_eq!(heuristic_section(&ai, &s), "AI & Machine Learning");
|
||||
|
||||
let mut blog = candidate(3, "Notes from my week", 900, 6.0);
|
||||
blog.auto_include = true;
|
||||
assert_eq!(heuristic_section(&blog, &s), "From the Blogroll");
|
||||
|
||||
let mut plain = candidate(4, "Untitled musing", 900, 6.0);
|
||||
plain.article.category = None;
|
||||
plain.article.feed_title = "A Journal".into();
|
||||
assert_eq!(heuristic_section(&plain, &s), "Top Stories");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_realistic_lineup_response() {
|
||||
let items = parse_selection_response(LINEUP_FIXTURE);
|
||||
assert_eq!(items.len(), 6);
|
||||
assert_eq!(items[0].id, 101);
|
||||
assert!(items[0].lead_story);
|
||||
assert_eq!(items[0].section, "Top Stories");
|
||||
assert_eq!(items.iter().filter(|i| i.lead_story).count(), 1);
|
||||
// Junk entries in the fixture are dropped, not fatal.
|
||||
assert!(items.iter().all(|i| i.id != 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_bounds_follow_the_spec() {
|
||||
assert_eq!(size_bounds(20), (15, 25));
|
||||
assert_eq!(size_bounds(6), (1, 11));
|
||||
assert_eq!(size_bounds(0), (1, 6));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selection_builds_a_valid_lineup() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(LINEUP_FIXTURE, TokenUsage::default());
|
||||
let llm = LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), 2.0),
|
||||
backend.clone(),
|
||||
);
|
||||
|
||||
// ids 101..=112 so the fixture's picks resolve.
|
||||
let pool: Vec<ScoredArticle> = (101..=112)
|
||||
.map(|i| candidate(i, &format!("Article {i}"), 800, 7.0))
|
||||
.collect();
|
||||
let lineup = select(&llm, pool, §ions(), 6, date())
|
||||
.await
|
||||
.expect("selection");
|
||||
|
||||
assert_eq!(lineup.date, date());
|
||||
assert_eq!(lineup.picks.len(), 6);
|
||||
assert_eq!(lineup.picks.iter().filter(|p| p.is_lead).count(), 1);
|
||||
assert_eq!(lineup.lead().map(|p| p.article.id), Some(101));
|
||||
// Every section is from the palette and non-empty.
|
||||
for section in &lineup.section_order {
|
||||
assert!(sections().contains(section), "{section} is off-palette");
|
||||
assert!(!lineup.section_picks(section).is_empty());
|
||||
}
|
||||
// Positions restart at 1 inside each section and ascend.
|
||||
for section in &lineup.section_order {
|
||||
let positions: Vec<i64> = lineup
|
||||
.section_picks(section)
|
||||
.iter()
|
||||
.map(|p| p.position)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
positions,
|
||||
(1..=positions.len() as i64).collect::<Vec<_>>(),
|
||||
"{section} positions"
|
||||
);
|
||||
}
|
||||
// The lead sits in the first section used.
|
||||
assert_eq!(
|
||||
lineup.lead().map(|p| p.section.clone()),
|
||||
lineup.section_order.first().cloned()
|
||||
);
|
||||
// The prompt carried the shortlist and the palette.
|
||||
let prompt = &backend.prompts()[0].user;
|
||||
assert!(prompt.starts_with(SELECT_INSTRUCTIONS));
|
||||
assert!(prompt.contains("--- id: 101"));
|
||||
assert!(prompt.contains("never fewer than 1 and never more than 11"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hallucinated_ids_and_missing_leads_are_repaired() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"picks":[{"id":9999,"section":"Top Stories","position":1,"lead_story":true},
|
||||
{"id":1,"section":"Sportsball","position":2},
|
||||
{"id":2,"section":"Niche Corner","position":1}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), 2.0),
|
||||
backend,
|
||||
);
|
||||
let lineup = select(&llm, candidates(6), §ions(), 2, date())
|
||||
.await
|
||||
.expect("selection");
|
||||
assert!(lineup.picks.iter().all(|p| p.article.id != 9999));
|
||||
assert_eq!(lineup.picks.iter().filter(|p| p.is_lead).count(), 1);
|
||||
for pick in &lineup.picks {
|
||||
assert!(sections().contains(&pick.section));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oversized_and_undersized_answers_are_clamped() {
|
||||
// Undersized: the model returns one pick but the floor is 5.
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"picks":[{"id":1,"section":"Top Stories","position":1,"lead_story":true}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), 2.0),
|
||||
backend,
|
||||
);
|
||||
let lineup = select(&llm, candidates(30), §ions(), 10, date())
|
||||
.await
|
||||
.expect("selection");
|
||||
assert!(lineup.picks.len() >= 5, "{}", lineup.picks.len());
|
||||
|
||||
// Oversized: 30 picks against a target of 6 (ceiling 11).
|
||||
let picks: Vec<String> = (1..=30)
|
||||
.map(|i| format!(r#"{{"id":{i},"section":"Top Stories","position":{i}}}"#))
|
||||
.collect();
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
format!(r#"{{"picks":[{}]}}"#, picks.join(",")),
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), 2.0),
|
||||
backend,
|
||||
);
|
||||
let lineup = select(&llm, candidates(30), §ions(), 6, date())
|
||||
.await
|
||||
.expect("selection");
|
||||
assert_eq!(lineup.picks.len(), 11);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn always_include_articles_are_reinserted() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"picks":[{"id":1,"section":"Top Stories","position":1,"lead_story":true}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), 2.0),
|
||||
backend,
|
||||
);
|
||||
let mut pool = candidates(3);
|
||||
pool[2].auto_include = true;
|
||||
let lineup = select(&llm, pool, §ions(), 1, date())
|
||||
.await
|
||||
.expect("selection");
|
||||
let ids: Vec<ArticleId> = lineup.picks.iter().map(|p| p.article.id).collect();
|
||||
assert!(ids.contains(&3), "auto-include must survive: {ids:?}");
|
||||
assert_eq!(
|
||||
lineup
|
||||
.picks
|
||||
.iter()
|
||||
.find(|p| p.article.id == 3)
|
||||
.map(|p| p.section.as_str()),
|
||||
Some("From the Blogroll")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_tripped_budget_falls_back_without_calling_the_model() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
let meter = UsageMeter::new(&DeepseekConfig::default(), 0.001);
|
||||
meter.record(TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 0,
|
||||
output_tokens: 0,
|
||||
});
|
||||
let llm =
|
||||
LlmClient::with_backend("deepseek-v4-flash", "SYSTEM".into(), meter, backend.clone());
|
||||
let lineup = select(&llm, candidates(20), §ions(), 6, date())
|
||||
.await
|
||||
.expect("fallback");
|
||||
assert_eq!(backend.calls(), 0);
|
||||
assert_eq!(lineup.picks.len(), 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skip_llm_lineup_uses_prefilter_order() {
|
||||
let mut pool = candidates(10);
|
||||
pool.iter_mut().for_each(|c| c.llm = None);
|
||||
pool[7].prefilter_score = 99.0; // id 8 is the strongest heuristically
|
||||
pool[9].auto_include = true; // id 10 is a personal blog
|
||||
pool[9].prefilter_score = 1.0;
|
||||
|
||||
let lineup = select_without_llm(pool, §ions(), 4, date());
|
||||
assert_eq!(lineup.picks.len(), 5, "4 picks + the auto-include");
|
||||
assert_eq!(lineup.lead().map(|p| p.article.id), Some(8));
|
||||
assert_eq!(lineup.picks.iter().filter(|p| p.is_lead).count(), 1);
|
||||
assert!(
|
||||
lineup
|
||||
.picks
|
||||
.iter()
|
||||
.any(|p| p.article.id == 10 && p.section == "From the Blogroll")
|
||||
);
|
||||
for pick in &lineup.picks {
|
||||
assert!(sections().contains(&pick.section));
|
||||
assert!(pick.summary.is_none());
|
||||
}
|
||||
assert!(!lineup.section_order.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_yields_an_empty_lineup() {
|
||||
let lineup = select_without_llm(Vec::new(), §ions(), 20, date());
|
||||
assert!(lineup.picks.is_empty());
|
||||
assert!(lineup.section_order.is_empty());
|
||||
assert!(lineup.lead().is_none());
|
||||
}
|
||||
}
|
||||
Loaded 100 of 221 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user