From d403c51edf74ed1f750beab6a22079e96bbb6208 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Wed, 2 Sep 2026 16:32:18 +0000 Subject: [PATCH] Curation v2 step 7: cleanup, prune paths, implementation notes Dead code and stale v1 comments removed (clippy -W dead_code clean, the three world.rs warnings fixed), the Brief chapter's TOC title renamed from "From the Editor", features prune now also sweeps article_assessments and generate runs the sweep once after publishing, the example config is tested key-for-key against Config::default(), README commands match --help, and docs/plans/2026-08-15-implementation-notes.md records the Anthropic and Voyage facts, the new tables, the budget-day rule and the lock. Implemented by a Claude agent from docs/plans/curation-v2-briefs/step7.md. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe --- README.md | 11 +-- config.example.toml | 2 +- docs/plans/2026-08-15-implementation-notes.md | 82 +++++++++++++++++-- src/config.rs | 76 +++++++++++++++-- src/curate/llm.rs | 3 +- src/curate/mod.rs | 4 +- src/curate/profile/mod.rs | 5 +- src/curate/telemetry.rs | 72 +++++++++++----- src/db.rs | 8 +- src/epub/chapters.rs | 8 +- src/epub/mod.rs | 11 --- src/main.rs | 20 ++++- src/pipeline.rs | 32 +++++++- src/report.rs | 4 +- src/types.rs | 24 +++--- src/world.rs | 6 +- tests/e2e_pipeline.rs | 2 +- tests/m3_curation.rs | 6 +- 18 files changed, 288 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index dfaafc3..9cda068 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,15 @@ sudo install -m0755 target/release/daily-epub /usr/local/bin/ daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] [--skip-embeddings] [--rescore] daily-epub serve # rating endpoints + OPDS catalog + downloads daily-epub profile rebuild # regenerate learned profile adjustments -daily-epub ratings list --days 90 +daily-epub ratings list [--days 90] [--label loved|good|down|cleared] daily-epub ratings set --article 42 --label loved --note "excellent" daily-epub ratings clear --url https://example.com/article daily-epub explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N] daily-epub explain --date YYYY-MM-DD --near-misses [N] daily-epub stats [--days 14] # the evaluation framework, one fact per line daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes] -daily-epub features prune # stale embeddings + old candidate telemetry -daily-epub backfill-social # re-poll social scores for recent articles +daily-epub features prune # stale embeddings, old telemetry and assessments +daily-epub backfill-social [--days 7] # re-poll social scores for recent articles daily-epub db migrate # run migrations (also automatic on every start) ``` @@ -156,8 +156,9 @@ set), then the standing interests, then — only with `--all` — every other article first seen in the window. It prints an estimate and asks before spending more than 5M tokens unless `--yes`; a warm cache makes zero calls. `features prune` drops embeddings of articles neither rated nor published that are older -than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows of -runs older than `curation.ranking.telemetry_retention_days`. +than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows and +`article_assessments` older than `curation.ranking.telemetry_retention_days`. +`generate` runs the same sweep once after publishing, best effort. --- diff --git a/config.example.toml b/config.example.toml index ef8f17b..b423329 100644 --- a/config.example.toml +++ b/config.example.toml @@ -123,7 +123,7 @@ feed_full = 40 semantic_min_words = 300 exploration_slots = 5 embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors -telemetry_retention_days = 180 # `features prune`: candidate_runs rows +telemetry_retention_days = 180 # `features prune`: candidate_runs and article_assessments [curation.ranking.quotas] triage = 60 diff --git a/docs/plans/2026-08-15-implementation-notes.md b/docs/plans/2026-08-15-implementation-notes.md index 6ae4e08..8f4f669 100644 --- a/docs/plans/2026-08-15-implementation-notes.md +++ b/docs/plans/2026-08-15-implementation-notes.md @@ -1,6 +1,8 @@ # Implementation notes (shared brief for all implementation agents) Authoritative spec: `docs/plans/2026-08-15-the-daily-epub.md`. Read it fully before writing code. +For curation (§3.5, §3.6 and §3.9 of that spec) the authority is now +`docs/plans/2026-09-02-personalized-curation-v2.md`; see "Curation v2" below. This file records implementation-time decisions and verified external facts. Follow both. ## Verified external facts (2026-08-15) @@ -68,6 +70,31 @@ This file records implementation-time decisions and verified external facts. Fol walks back up to `MAX_LOOKBACK_DAYS` and the section is datelined with the day it actually covers, not the masthead date. +## Verified external facts (2026-09-02, curation v2) + +- **Anthropic Messages API** (verified 2026-09-02 against the bundled Claude API reference): + `POST https://api.anthropic.com/v1/messages` with headers `x-api-key`, + `anthropic-version: 2023-06-01`, `content-type: application/json` and + `anthropic-beta: server-side-fallback-2026-07-01`. Model id `claude-opus-5`; pricing + **$5.00 / M input, $25.00 / M output**, cache reads 0.1× input ($0.50/M), cache writes + 1.25× ($6.25/M); the minimum cacheable prefix is 512 tokens. **No sampling parameters** + (`temperature`, `top_p`, `top_k` are a 400) and no `thinking` block — adaptive thinking is on + by default and depth is set with `output_config: {"effort": "high"}` (`low | medium | high | + xhigh | max`). The system prompt goes in `system: [{type: "text", text, cache_control: + {type: "ephemeral"}}]`; no assistant prefill, JSON is asked for in the prompt and parsed + tolerantly. `"fallbacks": "default"` (with the beta header) routes a request the safety + classifiers would refuse to a fallback model server-side; a response can still end with + `stop_reason: "refusal"` on HTTP 200, which the code treats as an error that degrades the + call to DeepSeek. Usage fields: `input_tokens` (uncached remainder), + `cache_creation_input_tokens`, `cache_read_input_tokens`, `output_tokens`. Timeout 300 s; + retry 429/5xx/network, never 400. Key only from `DAILY_EPUB_ANTHROPIC__API_KEY`. +- **Voyage AI embeddings** (verified 2026-09-02): `POST https://api.voyageai.com/v1/embeddings` + with `Authorization: Bearer `; body `{input: [...], model: "voyage-4-lite", input_type: + "document" | "query", truncation: true, output_dimension: 512, output_dtype: "float"}`. Up + to 1,000 inputs and 1M tokens per request, 32k tokens per input. Vectors are + unit-normalized, so dot product = cosine. **$0.02 / M tokens** after a 200M-token free + allocation. Key only from `DAILY_EPUB_VOYAGE__API_KEY`. + ## Cross-cutting implementation decisions 1. **sqlx usage**: use *runtime* queries (`sqlx::query(...).bind(...)`) and manual row mapping @@ -80,12 +107,20 @@ This file records implementation-time decisions and verified external facts. Fol Pipeline stages are best-effort where the spec says so (social, XTC, world briefing, images). 4. **HTTP**: one shared `reqwest::Client` (rustls, gzip, no cookies, 10s timeouts, UA `the-daily-epub/1.0 (personal rss digest; contact tyler@hallada.net)`), passed by clone. -5. **LLM**: `async-openai` with custom base URL. All LLM calls go through `curate/llm.rs` - `LlmClient` which tracks token usage into a shared `UsageMeter` (input/cached/output tokens, - cost usd) and enforces `max_daily_usd`. +5. **LLM**: a hand-rolled `reqwest` client, not `async-openai` (the published crate exposes + neither `Client` nor `CreateChatCompletionRequest` at the pinned version). Every LLM call + goes through `curate/llm.rs`: `LlmClient { system_prompt, model, meter, backend, retry }` + over the `ChatBackend` trait, with `DeepseekBackend` (OpenAI-compatible chat completions, + `response_format: json_object`) and `AnthropicBackend` (Messages API, facts above). The + pipeline holds `Llms { bulk, editor }`; `editor_or_bulk()` degrades to DeepSeek when the + Claude client is missing or its meter is tripped. One `UsageMeter` per provider + (DeepSeek, Anthropic, Voyage) with its own price table and `max_daily_usd`. The system + prompt is sent first and byte-identical within a run so both providers' prefix caches hit. 6. **Testing**: unit tests inline per module; integration tests in `tests/` over fixture JSON in - `tests/fixtures/`. Never hit the network in tests. LLM stage mockable via `--skip-llm` - (prefilter order used for selection, feed excerpts as summaries). + `tests/fixtures/`. Never hit the network in tests: `MockBackend` (`ChatBackend`) and the + embedding mock (`EmbeddingBackend`) stand in for all three providers. `--skip-llm` makes + zero LLM calls (admission by cheap signals, `select_without_llm` by utility, excerpt + summaries); `--skip-embeddings` makes zero Voyage calls. 7. **Style**: rustfmt defaults, `cargo clippy` clean-ish, no `unwrap()` outside tests, tracing spans per pipeline stage. 8. **File ownership**: waves of agents work in parallel on disjoint files. Do not edit files @@ -99,3 +134,40 @@ This file records implementation-time decisions and verified external facts. Fol `style-x4.css`). Askama 0.12+ configured via `askama.toml` if needed. 12. **Determinism**: chapter ids `art-{entry_id}`, stable filenames, issue regeneration for the same date replaces prior rows/files (idempotent upsert everywhere). + +## Curation v2 (2026-09-02) + +The personalized ranker is specified in `docs/plans/2026-09-02-personalized-curation-v2.md` +(§0 settled decisions, §3 target pipeline, §19 configuration, §21 the seven landed steps); +`docs/plans/2026-09-02-curation-v2-progress.md` records per-step deviations. Facts an +implementer needs that are easy to get wrong: + +- **Tables** (`migrations/0002_curation_v2.sql`, `0003_drop_scores.sql`; never edit + `0001_init.sql`): `rating_events` (append-only; the current verdict is the latest + `explicit` event), `article_embeddings` and `interest_embeddings` (f32 little-endian BLOBs, + `input_hash` = sha256 of the embedded text), `article_assessments` (`stage IN ('triage', + 'deep')`, reused while `model` and `prompt_version` match and `assessed_at` is within + `assessment_reuse_days`; `--rescore` ignores the cache), `candidate_runs` (one row per + considered article per run, upserted with every column set on each stage transition), + `runs.config_json` / `runs.provider_costs_json`, `issue_articles.why`. `ratings`, + `feed_priors` and `scores` are dropped; `kv` keeps `ingest_watermark`, `taste_profile`, + `taste_profile_learned`, `profile_version`. +- **Feedback**: `Vote` is `loved | good | down` (`NotForMe`); the HMAC message stays + `{issue_date}/{article_id}/{vote}`. `Vote::parse("up")` → `Loved` and `auth::verify_token` + still accepts tokens signed over the literal `up` segment because published issues carry + those links. Keep both. +- **Budget day**: each provider's `UsageMeter` is preloaded with the spend of earlier runs on + the **UTC date of the run's `started_at`**, summed from `runs.provider_costs_json` + (`db::spend_for_date` by nominal issue date is gone). A tripped meter skips that provider's + remaining calls; the paper always publishes. +- **Lock**: `src/lock.rs` takes `libc::flock(LOCK_EX | LOCK_NB)` on `.lock` + for `generate`, `profile rebuild`, `features backfill` and `backfill-social`; a second + writer exits with " is already running". `serve`, `explain`, `stats`, `ratings`, + `features prune` and `db migrate` never take it. +- **Retention**: `telemetry::prune` removes `article_embeddings` of unrated, unpublished + articles older than `embedding_retention_days` (120) and `candidate_runs` rows plus + `article_assessments` older than `telemetry_retention_days` (180). `features prune` runs it + on demand; `generate` runs it once after publishing, best effort. +- **Keys**: `DAILY_EPUB_ANTHROPIC__API_KEY` and `DAILY_EPUB_VOYAGE__API_KEY` map onto + `AnthropicConfig.api_key` / `VoyageConfig.api_key` through figment; the fields exist only + for that mapping and are never documented in TOML, logged, or stored. diff --git a/src/config.rs b/src/config.rs index 8efab9b..344f709 100644 --- a/src/config.rs +++ b/src/config.rs @@ -47,7 +47,8 @@ pub struct Config { pub timezone: String, /// Ingest window size in hours (§3.1). pub lookback_hours: u32, - /// How many articles the lineup should contain (§3.6 stage B). + /// Soft target for the lineup size (§13); `curation.max_article_count` + /// is the ceiling and there is no minimum. pub target_article_count: usize, /// Days of published EPUBs kept in `publish.epub_dir` (§3.11). pub retention_days: u32, @@ -56,7 +57,8 @@ pub struct Config { /// Counted, not dated, because an XTCH issue is ~80–100 MB of pre-rendered /// page bitmaps: the constraint is disk, not age. pub xtc_retention_count: u32, - /// Hard cost ceiling per run (§3.6 guardrail). + /// DeepSeek spend ceiling per UTC day (§5); `[anthropic]` and `[voyage]` + /// carry their own. pub max_daily_usd: f64, /// Include the Wikipedia Current Events section (§3.8). pub world_briefing: bool, @@ -129,7 +131,7 @@ impl Default for MinifluxConfig { } } -/// `[deepseek]` — LLM endpoint, model and pricing (§3.6, notes "verified facts"). +/// `[deepseek]` — bulk LLM endpoint, model and pricing (§4.1, notes "verified facts"). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct DeepseekConfig { @@ -263,7 +265,7 @@ impl Default for VoyageConfig { } } -/// `[curation]` — pre-filter and section palette (§3.5, §3.6). +/// `[curation]` — hygiene, feedback weights, the ranker and the section palette (§19). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct CurationConfig { @@ -278,7 +280,7 @@ pub struct CurationConfig { /// Extra paywalled hosts, merged with [`crate::extract::DEFAULT_PAYWALL_DOMAINS`] /// by the extraction stage's `excerpt_only` heuristic (§3.3). pub paywall_domains: Vec, - /// The only section names the LLM may use (§3.6 stage B). + /// The only section names the editor may use (§13). pub sections: Vec, pub feedback: FeedbackConfig, pub ranking: RankingConfig, @@ -313,9 +315,7 @@ impl Default for CurationConfig { } /// `[curation.ranking]` — every weight, quota, gate and threshold of the -/// personalized ranker (plan §19). Steps 4–5 consume most of these; step 3 -/// uses the learned-signal gates, the preliminary weights and the retention -/// windows. +/// personalized ranker (plan §19). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct RankingConfig { @@ -925,6 +925,66 @@ mod tests { assert_eq!(c.editorial.summary_input_tokens, 3000); } + /// `config.example.toml` documents the plan's numbers (§19), which are also + /// `Config::default()`: every documented key in these sections must exist + /// on the struct with the default value, and every struct field (except + /// the env-only `api_key`) must be documented in the file. + #[test] + fn shipped_example_config_matches_the_defaults_key_for_key() { + let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("config.example.toml"); + let documented: serde_json::Value = Figment::from(Toml::file(&example)) + .extract() + .expect("config.example.toml must parse as a table"); + let defaults = serde_json::to_value(Config::default()).expect("defaults serialize"); + + fn compare(path: &str, documented: &serde_json::Value, default: &serde_json::Value) { + let (Some(documented), Some(default)) = (documented.as_object(), default.as_object()) + else { + // TOML `60` and the f64 default `60.0` are the same setting, and + // an f32 field (`editorial_temperature`) widens inexactly. + match (documented.as_f64(), default.as_f64()) { + (Some(doc), Some(def)) => assert!( + (doc - def).abs() <= 1e-6 * def.abs().max(1.0), + "{path}: documented {doc} vs default {def}" + ), + _ => assert_eq!(documented, default, "{path}"), + } + return; + }; + for (key, value) in default { + if key == "api_key" { + assert!( + !documented.contains_key(key), + "{path}.{key} must stay out of the TOML (env only)" + ); + continue; + } + let doc = documented + .get(key) + .unwrap_or_else(|| panic!("{path}.{key} is missing from config.example.toml")); + compare(&format!("{path}.{key}"), doc, value); + } + for key in documented.keys() { + assert!( + default.contains_key(key), + "{path}.{key} is documented but not a config field" + ); + } + } + + for (key, default) in defaults.as_object().expect("config is a table") { + let section = match key.as_str() { + "curation" | "anthropic" | "voyage" | "editorial" | "deepseek" => key, + "target_article_count" | "max_daily_usd" | "profile_path" | "interests_opml" => key, + _ => continue, + }; + let documented = documented + .get(section) + .unwrap_or_else(|| panic!("{section} is missing from config.example.toml")); + compare(section, documented, default); + } + } + #[test] fn provider_validation_rejects_nonsense() { let mut c = Config::default(); diff --git a/src/curate/llm.rs b/src/curate/llm.rs index 90e6265..b81dc4f 100644 --- a/src/curate/llm.rs +++ b/src/curate/llm.rs @@ -122,7 +122,8 @@ pub struct UsageMeter { } impl UsageMeter { - /// Compatibility constructor for the existing DeepSeek call sites. + /// A meter priced from the `[deepseek]` table; the other providers build + /// theirs with [`UsageMeter::with_prices`]. pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self { Self::with_prices(PriceTable::deepseek(cfg), limit_usd) } diff --git a/src/curate/mod.rs b/src/curate/mod.rs index 992749e..c1677da 100644 --- a/src/curate/mod.rs +++ b/src/curate/mod.rs @@ -148,7 +148,7 @@ impl Curator { // --------------------------------------------------------------------------- /// Crude token estimate: DeepSeek averages ~4 characters per token for English -/// prose. Only used to size prompt budgets (§3.6 stage C). +/// prose. Only used to size prompt budgets. pub fn approx_tokens(text: &str) -> usize { text.len().div_ceil(4) } @@ -215,7 +215,7 @@ pub fn truncate_words(text: &str, max_words: usize) -> String { out } -/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C). +/// Truncate to roughly `max_tokens` tokens on a word boundary. pub fn truncate_tokens(text: &str, max_tokens: usize) -> String { let max_chars = max_tokens.saturating_mul(4); if text.len() <= max_chars { diff --git a/src/curate/profile/mod.rs b/src/curate/profile/mod.rs index 47f4125..7e26577 100644 --- a/src/curate/profile/mod.rs +++ b/src/curate/profile/mod.rs @@ -16,7 +16,10 @@ use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE}; use crate::types::{Facets, RatedArticle, TasteProfile}; pub const REBUILD_INTERVAL_DAYS: i64 = 7; -pub const RATINGS_LOOKBACK_DAYS: i64 = 36_500; +/// The verdict block and the weekly rebuild are bounded by count +/// (`verdicts_in_prompt`, [`MAX_RATINGS_IN_REBUILD`]), not by age (§8.3, §8.4), +/// so their `current_ratings` lookback is effectively unbounded. +const RATINGS_LOOKBACK_DAYS: i64 = 36_500; pub const KV_LEARNED_ADJUSTMENTS: &str = "taste_profile_learned"; const MAX_RATINGS_IN_REBUILD: usize = 200; diff --git a/src/curate/telemetry.rs b/src/curate/telemetry.rs index da339ee..e32ec13 100644 --- a/src/curate/telemetry.rs +++ b/src/curate/telemetry.rs @@ -18,18 +18,7 @@ use crate::db::{Db, fmt_ts}; use crate::report::RunReport; use crate::types::{ArticleId, Candidate, NearMiss}; -/// The stage vocabulary of §7.4, in pipeline order. -pub const STAGES: [&str; 7] = [ - "excluded", - "eligible", - "triaged", - "admitted", - "assessed", - "shortlisted", - "selected", -]; - -/// Signal names rendered by `explain`, including the LLM ones steps 4–5 add. +/// Signal names rendered by `explain`, in the order of §7.5. const RENDERED_SIGNALS: [&str; 8] = [ "interest", "knn", @@ -850,15 +839,29 @@ pub async fn stats(db: &Db, days: i64, now: Timestamp) -> anyhow::Result // `features prune` (§7.1, §7.4) // --------------------------------------------------------------------------- +/// Rows removed by one [`prune`] pass. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Pruned { + /// `article_embeddings` of unrated, unpublished articles past + /// `embedding_retention_days`. + pub embeddings: u64, + /// `candidate_runs` rows of runs past `telemetry_retention_days`. + pub telemetry: u64, + /// `article_assessments` assessed more than `telemetry_retention_days` ago. + pub assessments: u64, +} + /// Delete `article_embeddings` for articles neither rated nor published that -/// are older than `embedding_retention_days`, and `candidate_runs` rows whose -/// run started more than `telemetry_retention_days` ago. Returns the counts. +/// are older than `embedding_retention_days`, `candidate_runs` rows whose run +/// started more than `telemetry_retention_days` ago, and `article_assessments` +/// older than the same window (§7.1, §7.4). Runs from `features prune` and +/// once per `generate` after publishing. pub async fn prune( db: &Db, embedding_retention_days: i64, telemetry_retention_days: i64, now: Timestamp, -) -> Result<(u64, u64), sqlx::Error> { +) -> Result { let cutoff = |days: i64| { now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24))) .unwrap_or(Timestamp::UNIX_EPOCH) @@ -886,7 +889,17 @@ pub async fn prune( .execute(db.pool()) .await? .rows_affected(); - Ok((embeddings, telemetry)) + + let assessments = sqlx::query("DELETE FROM article_assessments WHERE assessed_at < ?") + .bind(fmt_ts(cutoff(telemetry_retention_days))) + .execute(db.pool()) + .await? + .rows_affected(); + Ok(Pruned { + embeddings, + telemetry, + assessments, + }) } #[cfg(test)] @@ -1523,13 +1536,26 @@ mod tests { let new_run = db.start_run(date(), now).await.unwrap(); thin_excluded(&db, old_run, 1, "blocked").await.unwrap(); thin_excluded(&db, new_run, 1, "blocked").await.unwrap(); + for (id, assessed_at) in [(1, old.clone()), (2, fmt_ts(now))] { + sqlx::query( + "INSERT INTO article_assessments + (article_id, stage, model, prompt_version, score, assessed_at) + VALUES (?, 'triage', 'deepseek-v4-flash', 1, 7.0, ?)", + ) + .bind(id) + .bind(&assessed_at) + .execute(db.pool()) + .await + .unwrap(); + } - let (embeddings, telemetry) = prune(&db, 120, 180, now).await.unwrap(); + let pruned = prune(&db, 120, 180, now).await.unwrap(); assert_eq!( - embeddings, 1, + pruned.embeddings, 1, "only the old, unrated, unpublished article 3" ); - assert_eq!(telemetry, 1, "only the old run's rows"); + assert_eq!(pruned.telemetry, 1, "only the old run's rows"); + assert_eq!(pruned.assessments, 1, "only the 200-day-old assessment"); let remaining: Vec = sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id") .fetch_all(db.pool()) @@ -1541,5 +1567,13 @@ mod tests { .await .unwrap(); assert_eq!(runs, vec![new_run]); + let assessed: Vec = sqlx::query_scalar("SELECT article_id FROM article_assessments") + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(assessed, vec![2]); + + // A second pass finds nothing left to remove. + assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default()); } } diff --git a/src/db.rs b/src/db.rs index 2e29a70..ea3ee82 100644 --- a/src/db.rs +++ b/src/db.rs @@ -25,9 +25,9 @@ pub static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations"); /// `kv` key holding the ingest watermark (§3.1). pub const KV_WATERMARK: &str = "ingest_watermark"; -/// `kv` key holding the current taste profile document (§3.6). +/// `kv` key holding the current system-prompt profile document (§8.4). pub const KV_TASTE_PROFILE: &str = "taste_profile"; -/// `kv` key holding the taste profile version/build time (§3.6). +/// `kv` key holding the profile version/build time (§8.2). pub const KV_PROFILE_VERSION: &str = "profile_version"; #[derive(Debug, thiserror::Error)] @@ -107,7 +107,7 @@ impl Db { } // ----------------------------------------------------------------- - // kv (§3.1 watermark, §3.6 taste profile) + // kv (§3.1 watermark, §8 profile) // ----------------------------------------------------------------- pub async fn kv_get(&self, key: &str) -> Result> { @@ -635,7 +635,7 @@ impl Db { } // ----------------------------------------------------------------- - // runs (§3.6 cost guardrail, §3.13) + // runs (§3.13, plan §7.6) // ----------------------------------------------------------------- /// Insert a `running` row at the top of `generate`; returns `runs.id`. diff --git a/src/epub/chapters.rs b/src/epub/chapters.rs index c41d65c..9eebf6e 100644 --- a/src/epub/chapters.rs +++ b/src/epub/chapters.rs @@ -237,7 +237,7 @@ fn published_display(pick: &Pick) -> Option { .map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string()) } -/// "From the Editor" front page plus the issue stats line (§3.10). +/// The Brief (§14.2) under the masthead, plus the issue stats line (§3.10). pub fn render_front_page(issue: &Issue) -> Result { let body = issue.editorial.front_page_html.trim(); let body_html = if body.is_empty() { @@ -249,7 +249,7 @@ pub fn render_front_page(issue: &Issue) -> Result { to_xhtml(&ammonia::clean(body)) }; let tpl = FrontPage { - title: "From the Editor".into(), + title: "The Brief".into(), display_date: issue.meta.display_date.clone(), issue_number: issue.meta.issue_number, stats_line: issue.meta.stats_line(), @@ -258,7 +258,7 @@ pub fn render_front_page(issue: &Issue) -> Result { Ok(Chapter { id: "front".into(), href: "front.xhtml".into(), - title: "From the Editor".into(), + title: "The Brief".into(), xhtml: tpl.render()?, toc_level: 1, }) @@ -726,7 +726,7 @@ mod tests { let issue = issue(); let chapter = render_front_page(&issue).unwrap(); assert_eq!(chapter.href, "front.xhtml"); - assert!(chapter.xhtml.contains("From the Editor")); + assert!(chapter.xhtml.contains("The Brief")); assert!(chapter.xhtml.contains("2 articles")); assert!(chapter.xhtml.contains("both worth your coffee")); assert!(chapter.xhtml.contains("No. 42")); diff --git a/src/epub/mod.rs b/src/epub/mod.rs index e13a913..87e0dee 100644 --- a/src/epub/mod.rs +++ b/src/epub/mod.rs @@ -16,17 +16,6 @@ use crate::config::Config; use crate::images; use crate::types::{Artifact, Edition, ImageAsset, Issue}; -/// Chapter order inside an issue (§3.10). -pub const CHAPTER_ORDER: &[&str] = &[ - "cover", - "from-the-editor", - "in-this-issue", - "sections", - "world-briefing", - "behind-the-paper", - "colophon", -]; - #[derive(Debug, thiserror::Error)] pub enum EpubError { #[error("epub build failed: {0}")] diff --git a/src/main.rs b/src/main.rs index 91289f3..81bff95 100644 --- a/src/main.rs +++ b/src/main.rs @@ -150,28 +150,36 @@ impl RatingSetLabel { #[derive(Debug, clap::Args)] struct RatingsListArgs { + /// How many days of verdicts to list. #[arg(long, default_value_t = 90)] days: i64, + /// Only verdicts with this label. #[arg(long, value_enum)] label: Option, } #[derive(Debug, clap::Args)] struct RatingsSetArgs { + /// Article id, as printed by `ratings list` or `explain`. #[arg(long, required_unless_present = "url", conflicts_with = "url")] article: Option, + /// Article URL; canonicalized before lookup. #[arg(long, required_unless_present = "article", conflicts_with = "article")] url: Option, + /// The verdict to record. #[arg(long, value_enum)] label: RatingSetLabel, + /// Free-text note shown to the weekly profile rebuild. #[arg(long)] note: Option, } #[derive(Debug, clap::Args)] struct RatingsClearArgs { + /// Article id, as printed by `ratings list` or `explain`. #[arg(long, required_unless_present = "url", conflicts_with = "url")] article: Option, + /// Article URL; canonicalized before lookup. #[arg(long, required_unless_present = "article", conflicts_with = "article")] url: Option, } @@ -213,7 +221,7 @@ struct StatsArgs { enum FeaturesCommand { /// Embed rated and published articles, then interests, into the cache. Backfill(BackfillArgs), - /// Drop stale embeddings and old candidate telemetry per the retention config. + /// Drop stale embeddings, old candidate telemetry and old assessments per the retention config. Prune, } @@ -668,7 +676,7 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res } FeaturesCommand::Prune => { let ranking = &config.curation.ranking; - let (embeddings, rows) = telemetry::prune( + let pruned = telemetry::prune( db, ranking.embedding_retention_days, ranking.telemetry_retention_days, @@ -676,8 +684,12 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res ) .await?; println!( - "pruned {embeddings} embeddings older than {} days and {rows} candidate rows older than {} days", - ranking.embedding_retention_days, ranking.telemetry_retention_days + "pruned {} embeddings older than {} days, {} candidate rows and {} assessments older than {} days", + pruned.embeddings, + ranking.embedding_retention_days, + pruned.telemetry, + pruned.assessments, + ranking.telemetry_retention_days ); } } diff --git a/src/pipeline.rs b/src/pipeline.rs index 4a60ccc..80a4dd7 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -254,6 +254,9 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul }; db.finish_run(run_id, &report).await?; + if stages.published.is_some() { + prune_retention(config, db).await; + } // The issue row is written before the report is costed, so stamp the finished // report onto it now (the paths are preserved by `COALESCE`, §3.13). if !opts.dry_run @@ -282,6 +285,28 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul }) } +/// The retention sweep of `features prune` (§7.1, §7.4), run once per +/// published issue. Best effort: a failure is logged and never touches the run. +async fn prune_retention(config: &Config, db: &Db) { + let ranking = &config.curation.ranking; + match telemetry::prune( + db, + ranking.embedding_retention_days, + ranking.telemetry_retention_days, + Timestamp::now(), + ) + .await + { + Ok(pruned) => tracing::info!( + embeddings = pruned.embeddings, + candidate_rows = pruned.telemetry, + assessments = pruned.assessments, + "retention prune complete" + ), + Err(error) => tracing::warn!(%error, "retention prune failed; continuing"), + } +} + /// What [`run_stages`] hands back; [`generate`] pairs it with the costed report. #[derive(Debug)] struct StageOutput { @@ -1044,10 +1069,11 @@ async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<( Ok(()) } -/// Build the DeepSeek client, running the weekly profile rebuild when it is due. +/// Build the bulk (DeepSeek) and editor (Claude) clients, running the weekly +/// profile rebuild when it is due. /// -/// Returns `None` for `--skip-llm` and for every configuration/API problem: the -/// caller then curates heuristically instead of failing the run (§3.6). +/// Each client is `None` for `--skip-llm` and for every configuration/API +/// problem: the pipeline then degrades per §17 instead of failing the run. async fn build_llms( ctx: &StageContext<'_>, bulk_meter: &UsageMeter, diff --git a/src/report.rs b/src/report.rs index 76aaf00..17b5482 100644 --- a/src/report.rs +++ b/src/report.rs @@ -22,7 +22,7 @@ pub enum RunStatus { /// Everything completed. Ok, /// The issue was produced but a best-effort stage failed (social, XTC, - /// world briefing, images) or the cost guardrail tripped (§3.6). + /// world briefing, images) or a provider budget tripped (§5). Degraded, /// No issue was produced. Failed, @@ -98,7 +98,7 @@ pub struct StageCounts { pub clusters: i64, /// Admitted deep-set count retained for the colophon and runs table. pub candidates: i64, - /// Articles in the final lineup (§3.6 stage B). + /// Articles in the final lineup (§13). pub selected: i64, /// Discussion chapters rendered (§3.7). pub discussions: i64, diff --git a/src/types.rs b/src/types.rs index fa22e81..eda3336 100644 --- a/src/types.rs +++ b/src/types.rs @@ -246,7 +246,7 @@ pub fn composite_social_score(refs: &[SocialRef]) -> f64 { } // --------------------------------------------------------------------------- -// Curation (§3.5, §3.6) +// Curation (plan §10–§13) // --------------------------------------------------------------------------- /// DeepSeek's close read of one article (§12.1). @@ -322,7 +322,7 @@ impl Candidate { } } -/// One selected article with its section placement (§3.6 stage B). +/// One selected article with its section placement (§13). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Pick { pub article: Article, @@ -340,13 +340,14 @@ pub struct Pick { pub discussion: Option, } -/// The day's final lineup: 15–25 picks grouped into sections (§3.6 stage B). +/// The day's final lineup grouped into sections (§13): no minimum size, +/// `curation.max_article_count` (or `--max-articles`) as the ceiling. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Lineup { pub date: Date, /// Sorted by (section order, position). pub picks: Vec, - /// Section names in issue order; empty sections are omitted (§3.6). + /// Section names in issue order; empty sections are omitted. pub section_order: Vec, } @@ -367,16 +368,17 @@ impl Lineup { } } -/// Stage-C editorial output (§3.6). +/// Editorial output: the Brief and the per-article summaries (§14). #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct Editorial { - /// "From the Editor", 250–400 words, already sanitized XHTML. + /// The Brief (§14.2), 120–200 words, already sanitized XHTML. pub front_page_html: String, - /// Article id → 2–3 sentence newspaper abstract. + /// Article id → 2–3 sentence newspaper abstract (§14.1). pub summaries: BTreeMap, } -/// The taste profile that forms the DeepSeek system prompt (§3.6, `kv`). +/// The reader profile that forms the system prompt shared by every LLM call +/// (§8.4); the current text lives in `kv`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TasteProfile { /// Full ~600-word prompt document. @@ -743,10 +745,10 @@ pub struct RatedArticle { } // --------------------------------------------------------------------------- -// LLM accounting (§3.6 cost guardrail) +// LLM accounting (§5 per-provider budgets) // --------------------------------------------------------------------------- -/// Token counters accumulated across every DeepSeek call in a run (§3.6). +/// Token counters accumulated across one provider's calls in a run (§5). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct TokenUsage { /// Cache-miss input tokens (billed at the full input rate). @@ -766,7 +768,7 @@ impl TokenUsage { self.output_tokens += other.output_tokens; } - /// USD cost given the per-1M-token prices from `[deepseek]` config (§3.6). + /// USD cost given a provider's per-1M-token prices (§4.1–§4.2). pub fn cost_usd( &self, price_input: f64, diff --git a/src/world.rs b/src/world.rs index fa87175..42d9b88 100644 --- a/src/world.rs +++ b/src/world.rs @@ -160,11 +160,11 @@ fn dropped(element: &scraper::node::Element) -> bool { fn node_text(node: NodeRef<'_>, skip_lists: bool, out: &mut String) { match node.value() { Node::Text(text) => { - out.push_str(&text); + out.push_str(text); out.push(' '); } Node::Element(element) => { - if dropped(&element) || (skip_lists && matches!(element.name(), "ul" | "ol")) { + if dropped(element) || (skip_lists && matches!(element.name(), "ul" | "ol")) { return; } for child in node.children() { @@ -208,7 +208,7 @@ fn links_without_child_lists(element: ElementRef<'_>, base: &Url) -> Vec let Node::Element(element) = node.value() else { return; }; - if dropped(&element) || matches!(element.name(), "ul" | "ol") { + if dropped(element) || matches!(element.name(), "ul" | "ol") { return; } if element.name() == "a" diff --git a/tests/e2e_pipeline.rs b/tests/e2e_pipeline.rs index 523c89a..0e184a7 100644 --- a/tests/e2e_pipeline.rs +++ b/tests/e2e_pipeline.rs @@ -596,7 +596,7 @@ async fn llm_pipeline_runs_against_a_mock_backend() { "every candidate came back assessed" ); - let lineup = curator.select(candidates, date()).await.expect("stage B"); + let lineup = curator.select(candidates, date()).await.expect("editor"); assert_eq!(lineup.picks.len(), 5); assert_eq!( lineup.lead().map(|p| p.article.id), diff --git a/tests/m3_curation.rs b/tests/m3_curation.rs index ed225d0..9a3602c 100644 --- a/tests/m3_curation.rs +++ b/tests/m3_curation.rs @@ -109,10 +109,10 @@ fn deep_messy_fixture_is_salvaged_not_rejected() { assert!(parse_deep_response("", §ions).is_empty()); } -/// Stage B responses must carry `{id, section, position, lead_story}` with -/// exactly one lead, and use only palette section names (§3.6). +/// Editor responses must carry `{id, section, position, lead_story}` with +/// exactly one lead, and use only palette section names (plan §13). #[test] -fn stage_b_fixture_parses_into_a_lineup() { +fn editor_fixture_parses_into_a_lineup() { // The palette from `CurationConfig::default()` (§3.14). let sections = daily_epub::config::CurationConfig::default().sections;