Curation v2 step 7: cleanup, prune paths, implementation notes
Dead code and stale v1 comments removed (clippy -W dead_code clean, the three world.rs warnings fixed), the Brief chapter's TOC title renamed from "From the Editor", features prune now also sweeps article_assessments and generate runs the sweep once after publishing, the example config is tested key-for-key against Config::default(), README commands match --help, and docs/plans/2026-08-15-implementation-notes.md records the Anthropic and Voyage facts, the new tables, the budget-day rule and the lock. Implemented by a Claude agent from docs/plans/curation-v2-briefs/step7.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
This commit is contained in:
@@ -109,15 +109,15 @@ sudo install -m0755 target/release/daily-epub /usr/local/bin/
|
|||||||
daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] [--skip-embeddings] [--rescore]
|
daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] [--skip-embeddings] [--rescore]
|
||||||
daily-epub serve # rating endpoints + OPDS catalog + downloads
|
daily-epub serve # rating endpoints + OPDS catalog + downloads
|
||||||
daily-epub profile rebuild # regenerate learned profile adjustments
|
daily-epub profile rebuild # regenerate learned profile adjustments
|
||||||
daily-epub ratings list --days 90
|
daily-epub ratings list [--days 90] [--label loved|good|down|cleared]
|
||||||
daily-epub ratings set --article 42 --label loved --note "excellent"
|
daily-epub ratings set --article 42 --label loved --note "excellent"
|
||||||
daily-epub ratings clear --url https://example.com/article
|
daily-epub ratings clear --url https://example.com/article
|
||||||
daily-epub explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N]
|
daily-epub explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N]
|
||||||
daily-epub explain --date YYYY-MM-DD --near-misses [N]
|
daily-epub explain --date YYYY-MM-DD --near-misses [N]
|
||||||
daily-epub stats [--days 14] # the evaluation framework, one fact per line
|
daily-epub stats [--days 14] # the evaluation framework, one fact per line
|
||||||
daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes]
|
daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes]
|
||||||
daily-epub features prune # stale embeddings + old candidate telemetry
|
daily-epub features prune # stale embeddings, old telemetry and assessments
|
||||||
daily-epub backfill-social # re-poll social scores for recent articles
|
daily-epub backfill-social [--days 7] # re-poll social scores for recent articles
|
||||||
daily-epub db migrate # run migrations (also automatic on every start)
|
daily-epub db migrate # run migrations (also automatic on every start)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -156,8 +156,9 @@ set), then the standing interests, then — only with `--all` — every other
|
|||||||
article first seen in the window. It prints an estimate and asks before spending
|
article first seen in the window. It prints an estimate and asks before spending
|
||||||
more than 5M tokens unless `--yes`; a warm cache makes zero calls. `features
|
more than 5M tokens unless `--yes`; a warm cache makes zero calls. `features
|
||||||
prune` drops embeddings of articles neither rated nor published that are older
|
prune` drops embeddings of articles neither rated nor published that are older
|
||||||
than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows of
|
than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows and
|
||||||
runs older than `curation.ranking.telemetry_retention_days`.
|
`article_assessments` older than `curation.ranking.telemetry_retention_days`.
|
||||||
|
`generate` runs the same sweep once after publishing, best effort.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -123,7 +123,7 @@ feed_full = 40
|
|||||||
semantic_min_words = 300
|
semantic_min_words = 300
|
||||||
exploration_slots = 5
|
exploration_slots = 5
|
||||||
embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors
|
embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors
|
||||||
telemetry_retention_days = 180 # `features prune`: candidate_runs rows
|
telemetry_retention_days = 180 # `features prune`: candidate_runs and article_assessments
|
||||||
|
|
||||||
[curation.ranking.quotas]
|
[curation.ranking.quotas]
|
||||||
triage = 60
|
triage = 60
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
# Implementation notes (shared brief for all implementation agents)
|
# Implementation notes (shared brief for all implementation agents)
|
||||||
|
|
||||||
Authoritative spec: `docs/plans/2026-08-15-the-daily-epub.md`. Read it fully before writing code.
|
Authoritative spec: `docs/plans/2026-08-15-the-daily-epub.md`. Read it fully before writing code.
|
||||||
|
For curation (§3.5, §3.6 and §3.9 of that spec) the authority is now
|
||||||
|
`docs/plans/2026-09-02-personalized-curation-v2.md`; see "Curation v2" below.
|
||||||
This file records implementation-time decisions and verified external facts. Follow both.
|
This file records implementation-time decisions and verified external facts. Follow both.
|
||||||
|
|
||||||
## Verified external facts (2026-08-15)
|
## Verified external facts (2026-08-15)
|
||||||
@@ -68,6 +70,31 @@ This file records implementation-time decisions and verified external facts. Fol
|
|||||||
walks back up to `MAX_LOOKBACK_DAYS` and the section is datelined with the day it actually
|
walks back up to `MAX_LOOKBACK_DAYS` and the section is datelined with the day it actually
|
||||||
covers, not the masthead date.
|
covers, not the masthead date.
|
||||||
|
|
||||||
|
## Verified external facts (2026-09-02, curation v2)
|
||||||
|
|
||||||
|
- **Anthropic Messages API** (verified 2026-09-02 against the bundled Claude API reference):
|
||||||
|
`POST https://api.anthropic.com/v1/messages` with headers `x-api-key`,
|
||||||
|
`anthropic-version: 2023-06-01`, `content-type: application/json` and
|
||||||
|
`anthropic-beta: server-side-fallback-2026-07-01`. Model id `claude-opus-5`; pricing
|
||||||
|
**$5.00 / M input, $25.00 / M output**, cache reads 0.1× input ($0.50/M), cache writes
|
||||||
|
1.25× ($6.25/M); the minimum cacheable prefix is 512 tokens. **No sampling parameters**
|
||||||
|
(`temperature`, `top_p`, `top_k` are a 400) and no `thinking` block — adaptive thinking is on
|
||||||
|
by default and depth is set with `output_config: {"effort": "high"}` (`low | medium | high |
|
||||||
|
xhigh | max`). The system prompt goes in `system: [{type: "text", text, cache_control:
|
||||||
|
{type: "ephemeral"}}]`; no assistant prefill, JSON is asked for in the prompt and parsed
|
||||||
|
tolerantly. `"fallbacks": "default"` (with the beta header) routes a request the safety
|
||||||
|
classifiers would refuse to a fallback model server-side; a response can still end with
|
||||||
|
`stop_reason: "refusal"` on HTTP 200, which the code treats as an error that degrades the
|
||||||
|
call to DeepSeek. Usage fields: `input_tokens` (uncached remainder),
|
||||||
|
`cache_creation_input_tokens`, `cache_read_input_tokens`, `output_tokens`. Timeout 300 s;
|
||||||
|
retry 429/5xx/network, never 400. Key only from `DAILY_EPUB_ANTHROPIC__API_KEY`.
|
||||||
|
- **Voyage AI embeddings** (verified 2026-09-02): `POST https://api.voyageai.com/v1/embeddings`
|
||||||
|
with `Authorization: Bearer <key>`; body `{input: [...], model: "voyage-4-lite", input_type:
|
||||||
|
"document" | "query", truncation: true, output_dimension: 512, output_dtype: "float"}`. Up
|
||||||
|
to 1,000 inputs and 1M tokens per request, 32k tokens per input. Vectors are
|
||||||
|
unit-normalized, so dot product = cosine. **$0.02 / M tokens** after a 200M-token free
|
||||||
|
allocation. Key only from `DAILY_EPUB_VOYAGE__API_KEY`.
|
||||||
|
|
||||||
## Cross-cutting implementation decisions
|
## Cross-cutting implementation decisions
|
||||||
|
|
||||||
1. **sqlx usage**: use *runtime* queries (`sqlx::query(...).bind(...)`) and manual row mapping
|
1. **sqlx usage**: use *runtime* queries (`sqlx::query(...).bind(...)`) and manual row mapping
|
||||||
@@ -80,12 +107,20 @@ This file records implementation-time decisions and verified external facts. Fol
|
|||||||
Pipeline stages are best-effort where the spec says so (social, XTC, world briefing, images).
|
Pipeline stages are best-effort where the spec says so (social, XTC, world briefing, images).
|
||||||
4. **HTTP**: one shared `reqwest::Client` (rustls, gzip, no cookies, 10s timeouts, UA
|
4. **HTTP**: one shared `reqwest::Client` (rustls, gzip, no cookies, 10s timeouts, UA
|
||||||
`the-daily-epub/1.0 (personal rss digest; contact tyler@hallada.net)`), passed by clone.
|
`the-daily-epub/1.0 (personal rss digest; contact tyler@hallada.net)`), passed by clone.
|
||||||
5. **LLM**: `async-openai` with custom base URL. All LLM calls go through `curate/llm.rs`
|
5. **LLM**: a hand-rolled `reqwest` client, not `async-openai` (the published crate exposes
|
||||||
`LlmClient` which tracks token usage into a shared `UsageMeter` (input/cached/output tokens,
|
neither `Client` nor `CreateChatCompletionRequest` at the pinned version). Every LLM call
|
||||||
cost usd) and enforces `max_daily_usd`.
|
goes through `curate/llm.rs`: `LlmClient { system_prompt, model, meter, backend, retry }`
|
||||||
|
over the `ChatBackend` trait, with `DeepseekBackend` (OpenAI-compatible chat completions,
|
||||||
|
`response_format: json_object`) and `AnthropicBackend` (Messages API, facts above). The
|
||||||
|
pipeline holds `Llms { bulk, editor }`; `editor_or_bulk()` degrades to DeepSeek when the
|
||||||
|
Claude client is missing or its meter is tripped. One `UsageMeter` per provider
|
||||||
|
(DeepSeek, Anthropic, Voyage) with its own price table and `max_daily_usd`. The system
|
||||||
|
prompt is sent first and byte-identical within a run so both providers' prefix caches hit.
|
||||||
6. **Testing**: unit tests inline per module; integration tests in `tests/` over fixture JSON in
|
6. **Testing**: unit tests inline per module; integration tests in `tests/` over fixture JSON in
|
||||||
`tests/fixtures/`. Never hit the network in tests. LLM stage mockable via `--skip-llm`
|
`tests/fixtures/`. Never hit the network in tests: `MockBackend` (`ChatBackend`) and the
|
||||||
(prefilter order used for selection, feed excerpts as summaries).
|
embedding mock (`EmbeddingBackend`) stand in for all three providers. `--skip-llm` makes
|
||||||
|
zero LLM calls (admission by cheap signals, `select_without_llm` by utility, excerpt
|
||||||
|
summaries); `--skip-embeddings` makes zero Voyage calls.
|
||||||
7. **Style**: rustfmt defaults, `cargo clippy` clean-ish, no `unwrap()` outside tests, tracing
|
7. **Style**: rustfmt defaults, `cargo clippy` clean-ish, no `unwrap()` outside tests, tracing
|
||||||
spans per pipeline stage.
|
spans per pipeline stage.
|
||||||
8. **File ownership**: waves of agents work in parallel on disjoint files. Do not edit files
|
8. **File ownership**: waves of agents work in parallel on disjoint files. Do not edit files
|
||||||
@@ -99,3 +134,40 @@ This file records implementation-time decisions and verified external facts. Fol
|
|||||||
`style-x4.css`). Askama 0.12+ configured via `askama.toml` if needed.
|
`style-x4.css`). Askama 0.12+ configured via `askama.toml` if needed.
|
||||||
12. **Determinism**: chapter ids `art-{entry_id}`, stable filenames, issue regeneration for the
|
12. **Determinism**: chapter ids `art-{entry_id}`, stable filenames, issue regeneration for the
|
||||||
same date replaces prior rows/files (idempotent upsert everywhere).
|
same date replaces prior rows/files (idempotent upsert everywhere).
|
||||||
|
|
||||||
|
## Curation v2 (2026-09-02)
|
||||||
|
|
||||||
|
The personalized ranker is specified in `docs/plans/2026-09-02-personalized-curation-v2.md`
|
||||||
|
(§0 settled decisions, §3 target pipeline, §19 configuration, §21 the seven landed steps);
|
||||||
|
`docs/plans/2026-09-02-curation-v2-progress.md` records per-step deviations. Facts an
|
||||||
|
implementer needs that are easy to get wrong:
|
||||||
|
|
||||||
|
- **Tables** (`migrations/0002_curation_v2.sql`, `0003_drop_scores.sql`; never edit
|
||||||
|
`0001_init.sql`): `rating_events` (append-only; the current verdict is the latest
|
||||||
|
`explicit` event), `article_embeddings` and `interest_embeddings` (f32 little-endian BLOBs,
|
||||||
|
`input_hash` = sha256 of the embedded text), `article_assessments` (`stage IN ('triage',
|
||||||
|
'deep')`, reused while `model` and `prompt_version` match and `assessed_at` is within
|
||||||
|
`assessment_reuse_days`; `--rescore` ignores the cache), `candidate_runs` (one row per
|
||||||
|
considered article per run, upserted with every column set on each stage transition),
|
||||||
|
`runs.config_json` / `runs.provider_costs_json`, `issue_articles.why`. `ratings`,
|
||||||
|
`feed_priors` and `scores` are dropped; `kv` keeps `ingest_watermark`, `taste_profile`,
|
||||||
|
`taste_profile_learned`, `profile_version`.
|
||||||
|
- **Feedback**: `Vote` is `loved | good | down` (`NotForMe`); the HMAC message stays
|
||||||
|
`{issue_date}/{article_id}/{vote}`. `Vote::parse("up")` → `Loved` and `auth::verify_token`
|
||||||
|
still accepts tokens signed over the literal `up` segment because published issues carry
|
||||||
|
those links. Keep both.
|
||||||
|
- **Budget day**: each provider's `UsageMeter` is preloaded with the spend of earlier runs on
|
||||||
|
the **UTC date of the run's `started_at`**, summed from `runs.provider_costs_json`
|
||||||
|
(`db::spend_for_date` by nominal issue date is gone). A tripped meter skips that provider's
|
||||||
|
remaining calls; the paper always publishes.
|
||||||
|
- **Lock**: `src/lock.rs` takes `libc::flock(LOCK_EX | LOCK_NB)` on `<database_path>.lock`
|
||||||
|
for `generate`, `profile rebuild`, `features backfill` and `backfill-social`; a second
|
||||||
|
writer exits with "<command> is already running". `serve`, `explain`, `stats`, `ratings`,
|
||||||
|
`features prune` and `db migrate` never take it.
|
||||||
|
- **Retention**: `telemetry::prune` removes `article_embeddings` of unrated, unpublished
|
||||||
|
articles older than `embedding_retention_days` (120) and `candidate_runs` rows plus
|
||||||
|
`article_assessments` older than `telemetry_retention_days` (180). `features prune` runs it
|
||||||
|
on demand; `generate` runs it once after publishing, best effort.
|
||||||
|
- **Keys**: `DAILY_EPUB_ANTHROPIC__API_KEY` and `DAILY_EPUB_VOYAGE__API_KEY` map onto
|
||||||
|
`AnthropicConfig.api_key` / `VoyageConfig.api_key` through figment; the fields exist only
|
||||||
|
for that mapping and are never documented in TOML, logged, or stored.
|
||||||
|
|||||||
+68
-8
@@ -47,7 +47,8 @@ pub struct Config {
|
|||||||
pub timezone: String,
|
pub timezone: String,
|
||||||
/// Ingest window size in hours (§3.1).
|
/// Ingest window size in hours (§3.1).
|
||||||
pub lookback_hours: u32,
|
pub lookback_hours: u32,
|
||||||
/// How many articles the lineup should contain (§3.6 stage B).
|
/// Soft target for the lineup size (§13); `curation.max_article_count`
|
||||||
|
/// is the ceiling and there is no minimum.
|
||||||
pub target_article_count: usize,
|
pub target_article_count: usize,
|
||||||
/// Days of published EPUBs kept in `publish.epub_dir` (§3.11).
|
/// Days of published EPUBs kept in `publish.epub_dir` (§3.11).
|
||||||
pub retention_days: u32,
|
pub retention_days: u32,
|
||||||
@@ -56,7 +57,8 @@ pub struct Config {
|
|||||||
/// Counted, not dated, because an XTCH issue is ~80–100 MB of pre-rendered
|
/// Counted, not dated, because an XTCH issue is ~80–100 MB of pre-rendered
|
||||||
/// page bitmaps: the constraint is disk, not age.
|
/// page bitmaps: the constraint is disk, not age.
|
||||||
pub xtc_retention_count: u32,
|
pub xtc_retention_count: u32,
|
||||||
/// Hard cost ceiling per run (§3.6 guardrail).
|
/// DeepSeek spend ceiling per UTC day (§5); `[anthropic]` and `[voyage]`
|
||||||
|
/// carry their own.
|
||||||
pub max_daily_usd: f64,
|
pub max_daily_usd: f64,
|
||||||
/// Include the Wikipedia Current Events section (§3.8).
|
/// Include the Wikipedia Current Events section (§3.8).
|
||||||
pub world_briefing: bool,
|
pub world_briefing: bool,
|
||||||
@@ -129,7 +131,7 @@ impl Default for MinifluxConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `[deepseek]` — LLM endpoint, model and pricing (§3.6, notes "verified facts").
|
/// `[deepseek]` — bulk LLM endpoint, model and pricing (§4.1, notes "verified facts").
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields, default)]
|
#[serde(deny_unknown_fields, default)]
|
||||||
pub struct DeepseekConfig {
|
pub struct DeepseekConfig {
|
||||||
@@ -263,7 +265,7 @@ impl Default for VoyageConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `[curation]` — pre-filter and section palette (§3.5, §3.6).
|
/// `[curation]` — hygiene, feedback weights, the ranker and the section palette (§19).
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields, default)]
|
#[serde(deny_unknown_fields, default)]
|
||||||
pub struct CurationConfig {
|
pub struct CurationConfig {
|
||||||
@@ -278,7 +280,7 @@ pub struct CurationConfig {
|
|||||||
/// Extra paywalled hosts, merged with [`crate::extract::DEFAULT_PAYWALL_DOMAINS`]
|
/// Extra paywalled hosts, merged with [`crate::extract::DEFAULT_PAYWALL_DOMAINS`]
|
||||||
/// by the extraction stage's `excerpt_only` heuristic (§3.3).
|
/// by the extraction stage's `excerpt_only` heuristic (§3.3).
|
||||||
pub paywall_domains: Vec<String>,
|
pub paywall_domains: Vec<String>,
|
||||||
/// The only section names the LLM may use (§3.6 stage B).
|
/// The only section names the editor may use (§13).
|
||||||
pub sections: Vec<String>,
|
pub sections: Vec<String>,
|
||||||
pub feedback: FeedbackConfig,
|
pub feedback: FeedbackConfig,
|
||||||
pub ranking: RankingConfig,
|
pub ranking: RankingConfig,
|
||||||
@@ -313,9 +315,7 @@ impl Default for CurationConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// `[curation.ranking]` — every weight, quota, gate and threshold of the
|
/// `[curation.ranking]` — every weight, quota, gate and threshold of the
|
||||||
/// personalized ranker (plan §19). Steps 4–5 consume most of these; step 3
|
/// personalized ranker (plan §19).
|
||||||
/// uses the learned-signal gates, the preliminary weights and the retention
|
|
||||||
/// windows.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields, default)]
|
#[serde(deny_unknown_fields, default)]
|
||||||
pub struct RankingConfig {
|
pub struct RankingConfig {
|
||||||
@@ -925,6 +925,66 @@ mod tests {
|
|||||||
assert_eq!(c.editorial.summary_input_tokens, 3000);
|
assert_eq!(c.editorial.summary_input_tokens, 3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `config.example.toml` documents the plan's numbers (§19), which are also
|
||||||
|
/// `Config::default()`: every documented key in these sections must exist
|
||||||
|
/// on the struct with the default value, and every struct field (except
|
||||||
|
/// the env-only `api_key`) must be documented in the file.
|
||||||
|
#[test]
|
||||||
|
fn shipped_example_config_matches_the_defaults_key_for_key() {
|
||||||
|
let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("config.example.toml");
|
||||||
|
let documented: serde_json::Value = Figment::from(Toml::file(&example))
|
||||||
|
.extract()
|
||||||
|
.expect("config.example.toml must parse as a table");
|
||||||
|
let defaults = serde_json::to_value(Config::default()).expect("defaults serialize");
|
||||||
|
|
||||||
|
fn compare(path: &str, documented: &serde_json::Value, default: &serde_json::Value) {
|
||||||
|
let (Some(documented), Some(default)) = (documented.as_object(), default.as_object())
|
||||||
|
else {
|
||||||
|
// TOML `60` and the f64 default `60.0` are the same setting, and
|
||||||
|
// an f32 field (`editorial_temperature`) widens inexactly.
|
||||||
|
match (documented.as_f64(), default.as_f64()) {
|
||||||
|
(Some(doc), Some(def)) => assert!(
|
||||||
|
(doc - def).abs() <= 1e-6 * def.abs().max(1.0),
|
||||||
|
"{path}: documented {doc} vs default {def}"
|
||||||
|
),
|
||||||
|
_ => assert_eq!(documented, default, "{path}"),
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for (key, value) in default {
|
||||||
|
if key == "api_key" {
|
||||||
|
assert!(
|
||||||
|
!documented.contains_key(key),
|
||||||
|
"{path}.{key} must stay out of the TOML (env only)"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let doc = documented
|
||||||
|
.get(key)
|
||||||
|
.unwrap_or_else(|| panic!("{path}.{key} is missing from config.example.toml"));
|
||||||
|
compare(&format!("{path}.{key}"), doc, value);
|
||||||
|
}
|
||||||
|
for key in documented.keys() {
|
||||||
|
assert!(
|
||||||
|
default.contains_key(key),
|
||||||
|
"{path}.{key} is documented but not a config field"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (key, default) in defaults.as_object().expect("config is a table") {
|
||||||
|
let section = match key.as_str() {
|
||||||
|
"curation" | "anthropic" | "voyage" | "editorial" | "deepseek" => key,
|
||||||
|
"target_article_count" | "max_daily_usd" | "profile_path" | "interests_opml" => key,
|
||||||
|
_ => continue,
|
||||||
|
};
|
||||||
|
let documented = documented
|
||||||
|
.get(section)
|
||||||
|
.unwrap_or_else(|| panic!("{section} is missing from config.example.toml"));
|
||||||
|
compare(section, documented, default);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn provider_validation_rejects_nonsense() {
|
fn provider_validation_rejects_nonsense() {
|
||||||
let mut c = Config::default();
|
let mut c = Config::default();
|
||||||
|
|||||||
+2
-1
@@ -122,7 +122,8 @@ pub struct UsageMeter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl UsageMeter {
|
impl UsageMeter {
|
||||||
/// Compatibility constructor for the existing DeepSeek call sites.
|
/// A meter priced from the `[deepseek]` table; the other providers build
|
||||||
|
/// theirs with [`UsageMeter::with_prices`].
|
||||||
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
||||||
Self::with_prices(PriceTable::deepseek(cfg), limit_usd)
|
Self::with_prices(PriceTable::deepseek(cfg), limit_usd)
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -148,7 +148,7 @@ impl Curator {
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
||||||
/// prose. Only used to size prompt budgets (§3.6 stage C).
|
/// prose. Only used to size prompt budgets.
|
||||||
pub fn approx_tokens(text: &str) -> usize {
|
pub fn approx_tokens(text: &str) -> usize {
|
||||||
text.len().div_ceil(4)
|
text.len().div_ceil(4)
|
||||||
}
|
}
|
||||||
@@ -215,7 +215,7 @@ pub fn truncate_words(text: &str, max_words: usize) -> String {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C).
|
/// Truncate to roughly `max_tokens` tokens on a word boundary.
|
||||||
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
||||||
let max_chars = max_tokens.saturating_mul(4);
|
let max_chars = max_tokens.saturating_mul(4);
|
||||||
if text.len() <= max_chars {
|
if text.len() <= max_chars {
|
||||||
|
|||||||
@@ -16,7 +16,10 @@ use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
|||||||
use crate::types::{Facets, RatedArticle, TasteProfile};
|
use crate::types::{Facets, RatedArticle, TasteProfile};
|
||||||
|
|
||||||
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
||||||
pub const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
/// The verdict block and the weekly rebuild are bounded by count
|
||||||
|
/// (`verdicts_in_prompt`, [`MAX_RATINGS_IN_REBUILD`]), not by age (§8.3, §8.4),
|
||||||
|
/// so their `current_ratings` lookback is effectively unbounded.
|
||||||
|
const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
||||||
pub const KV_LEARNED_ADJUSTMENTS: &str = "taste_profile_learned";
|
pub const KV_LEARNED_ADJUSTMENTS: &str = "taste_profile_learned";
|
||||||
const MAX_RATINGS_IN_REBUILD: usize = 200;
|
const MAX_RATINGS_IN_REBUILD: usize = 200;
|
||||||
|
|
||||||
|
|||||||
+53
-19
@@ -18,18 +18,7 @@ use crate::db::{Db, fmt_ts};
|
|||||||
use crate::report::RunReport;
|
use crate::report::RunReport;
|
||||||
use crate::types::{ArticleId, Candidate, NearMiss};
|
use crate::types::{ArticleId, Candidate, NearMiss};
|
||||||
|
|
||||||
/// The stage vocabulary of §7.4, in pipeline order.
|
/// Signal names rendered by `explain`, in the order of §7.5.
|
||||||
pub const STAGES: [&str; 7] = [
|
|
||||||
"excluded",
|
|
||||||
"eligible",
|
|
||||||
"triaged",
|
|
||||||
"admitted",
|
|
||||||
"assessed",
|
|
||||||
"shortlisted",
|
|
||||||
"selected",
|
|
||||||
];
|
|
||||||
|
|
||||||
/// Signal names rendered by `explain`, including the LLM ones steps 4–5 add.
|
|
||||||
const RENDERED_SIGNALS: [&str; 8] = [
|
const RENDERED_SIGNALS: [&str; 8] = [
|
||||||
"interest",
|
"interest",
|
||||||
"knn",
|
"knn",
|
||||||
@@ -850,15 +839,29 @@ pub async fn stats(db: &Db, days: i64, now: Timestamp) -> anyhow::Result<String>
|
|||||||
// `features prune` (§7.1, §7.4)
|
// `features prune` (§7.1, §7.4)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Rows removed by one [`prune`] pass.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
|
pub struct Pruned {
|
||||||
|
/// `article_embeddings` of unrated, unpublished articles past
|
||||||
|
/// `embedding_retention_days`.
|
||||||
|
pub embeddings: u64,
|
||||||
|
/// `candidate_runs` rows of runs past `telemetry_retention_days`.
|
||||||
|
pub telemetry: u64,
|
||||||
|
/// `article_assessments` assessed more than `telemetry_retention_days` ago.
|
||||||
|
pub assessments: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Delete `article_embeddings` for articles neither rated nor published that
|
/// Delete `article_embeddings` for articles neither rated nor published that
|
||||||
/// are older than `embedding_retention_days`, and `candidate_runs` rows whose
|
/// are older than `embedding_retention_days`, `candidate_runs` rows whose run
|
||||||
/// run started more than `telemetry_retention_days` ago. Returns the counts.
|
/// started more than `telemetry_retention_days` ago, and `article_assessments`
|
||||||
|
/// older than the same window (§7.1, §7.4). Runs from `features prune` and
|
||||||
|
/// once per `generate` after publishing.
|
||||||
pub async fn prune(
|
pub async fn prune(
|
||||||
db: &Db,
|
db: &Db,
|
||||||
embedding_retention_days: i64,
|
embedding_retention_days: i64,
|
||||||
telemetry_retention_days: i64,
|
telemetry_retention_days: i64,
|
||||||
now: Timestamp,
|
now: Timestamp,
|
||||||
) -> Result<(u64, u64), sqlx::Error> {
|
) -> Result<Pruned, sqlx::Error> {
|
||||||
let cutoff = |days: i64| {
|
let cutoff = |days: i64| {
|
||||||
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
||||||
.unwrap_or(Timestamp::UNIX_EPOCH)
|
.unwrap_or(Timestamp::UNIX_EPOCH)
|
||||||
@@ -886,7 +889,17 @@ pub async fn prune(
|
|||||||
.execute(db.pool())
|
.execute(db.pool())
|
||||||
.await?
|
.await?
|
||||||
.rows_affected();
|
.rows_affected();
|
||||||
Ok((embeddings, telemetry))
|
|
||||||
|
let assessments = sqlx::query("DELETE FROM article_assessments WHERE assessed_at < ?")
|
||||||
|
.bind(fmt_ts(cutoff(telemetry_retention_days)))
|
||||||
|
.execute(db.pool())
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
Ok(Pruned {
|
||||||
|
embeddings,
|
||||||
|
telemetry,
|
||||||
|
assessments,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -1523,13 +1536,26 @@ mod tests {
|
|||||||
let new_run = db.start_run(date(), now).await.unwrap();
|
let new_run = db.start_run(date(), now).await.unwrap();
|
||||||
thin_excluded(&db, old_run, 1, "blocked").await.unwrap();
|
thin_excluded(&db, old_run, 1, "blocked").await.unwrap();
|
||||||
thin_excluded(&db, new_run, 1, "blocked").await.unwrap();
|
thin_excluded(&db, new_run, 1, "blocked").await.unwrap();
|
||||||
|
for (id, assessed_at) in [(1, old.clone()), (2, fmt_ts(now))] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO article_assessments
|
||||||
|
(article_id, stage, model, prompt_version, score, assessed_at)
|
||||||
|
VALUES (?, 'triage', 'deepseek-v4-flash', 1, 7.0, ?)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(&assessed_at)
|
||||||
|
.execute(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
let (embeddings, telemetry) = prune(&db, 120, 180, now).await.unwrap();
|
let pruned = prune(&db, 120, 180, now).await.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
embeddings, 1,
|
pruned.embeddings, 1,
|
||||||
"only the old, unrated, unpublished article 3"
|
"only the old, unrated, unpublished article 3"
|
||||||
);
|
);
|
||||||
assert_eq!(telemetry, 1, "only the old run's rows");
|
assert_eq!(pruned.telemetry, 1, "only the old run's rows");
|
||||||
|
assert_eq!(pruned.assessments, 1, "only the 200-day-old assessment");
|
||||||
let remaining: Vec<i64> =
|
let remaining: Vec<i64> =
|
||||||
sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id")
|
sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id")
|
||||||
.fetch_all(db.pool())
|
.fetch_all(db.pool())
|
||||||
@@ -1541,5 +1567,13 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(runs, vec![new_run]);
|
assert_eq!(runs, vec![new_run]);
|
||||||
|
let assessed: Vec<i64> = sqlx::query_scalar("SELECT article_id FROM article_assessments")
|
||||||
|
.fetch_all(db.pool())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(assessed, vec![2]);
|
||||||
|
|
||||||
|
// A second pass finds nothing left to remove.
|
||||||
|
assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ pub static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations");
|
|||||||
|
|
||||||
/// `kv` key holding the ingest watermark (§3.1).
|
/// `kv` key holding the ingest watermark (§3.1).
|
||||||
pub const KV_WATERMARK: &str = "ingest_watermark";
|
pub const KV_WATERMARK: &str = "ingest_watermark";
|
||||||
/// `kv` key holding the current taste profile document (§3.6).
|
/// `kv` key holding the current system-prompt profile document (§8.4).
|
||||||
pub const KV_TASTE_PROFILE: &str = "taste_profile";
|
pub const KV_TASTE_PROFILE: &str = "taste_profile";
|
||||||
/// `kv` key holding the taste profile version/build time (§3.6).
|
/// `kv` key holding the profile version/build time (§8.2).
|
||||||
pub const KV_PROFILE_VERSION: &str = "profile_version";
|
pub const KV_PROFILE_VERSION: &str = "profile_version";
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
@@ -107,7 +107,7 @@ impl Db {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------
|
// -----------------------------------------------------------------
|
||||||
// kv (§3.1 watermark, §3.6 taste profile)
|
// kv (§3.1 watermark, §8 profile)
|
||||||
// -----------------------------------------------------------------
|
// -----------------------------------------------------------------
|
||||||
|
|
||||||
pub async fn kv_get(&self, key: &str) -> Result<Option<String>> {
|
pub async fn kv_get(&self, key: &str) -> Result<Option<String>> {
|
||||||
@@ -635,7 +635,7 @@ impl Db {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------
|
// -----------------------------------------------------------------
|
||||||
// runs (§3.6 cost guardrail, §3.13)
|
// runs (§3.13, plan §7.6)
|
||||||
// -----------------------------------------------------------------
|
// -----------------------------------------------------------------
|
||||||
|
|
||||||
/// Insert a `running` row at the top of `generate`; returns `runs.id`.
|
/// Insert a `running` row at the top of `generate`; returns `runs.id`.
|
||||||
|
|||||||
@@ -237,7 +237,7 @@ fn published_display(pick: &Pick) -> Option<String> {
|
|||||||
.map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string())
|
.map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// "From the Editor" front page plus the issue stats line (§3.10).
|
/// The Brief (§14.2) under the masthead, plus the issue stats line (§3.10).
|
||||||
pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||||
let body = issue.editorial.front_page_html.trim();
|
let body = issue.editorial.front_page_html.trim();
|
||||||
let body_html = if body.is_empty() {
|
let body_html = if body.is_empty() {
|
||||||
@@ -249,7 +249,7 @@ pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
|||||||
to_xhtml(&ammonia::clean(body))
|
to_xhtml(&ammonia::clean(body))
|
||||||
};
|
};
|
||||||
let tpl = FrontPage {
|
let tpl = FrontPage {
|
||||||
title: "From the Editor".into(),
|
title: "The Brief".into(),
|
||||||
display_date: issue.meta.display_date.clone(),
|
display_date: issue.meta.display_date.clone(),
|
||||||
issue_number: issue.meta.issue_number,
|
issue_number: issue.meta.issue_number,
|
||||||
stats_line: issue.meta.stats_line(),
|
stats_line: issue.meta.stats_line(),
|
||||||
@@ -258,7 +258,7 @@ pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
|||||||
Ok(Chapter {
|
Ok(Chapter {
|
||||||
id: "front".into(),
|
id: "front".into(),
|
||||||
href: "front.xhtml".into(),
|
href: "front.xhtml".into(),
|
||||||
title: "From the Editor".into(),
|
title: "The Brief".into(),
|
||||||
xhtml: tpl.render()?,
|
xhtml: tpl.render()?,
|
||||||
toc_level: 1,
|
toc_level: 1,
|
||||||
})
|
})
|
||||||
@@ -726,7 +726,7 @@ mod tests {
|
|||||||
let issue = issue();
|
let issue = issue();
|
||||||
let chapter = render_front_page(&issue).unwrap();
|
let chapter = render_front_page(&issue).unwrap();
|
||||||
assert_eq!(chapter.href, "front.xhtml");
|
assert_eq!(chapter.href, "front.xhtml");
|
||||||
assert!(chapter.xhtml.contains("From the Editor"));
|
assert!(chapter.xhtml.contains("The Brief"));
|
||||||
assert!(chapter.xhtml.contains("2 articles"));
|
assert!(chapter.xhtml.contains("2 articles"));
|
||||||
assert!(chapter.xhtml.contains("both worth your coffee"));
|
assert!(chapter.xhtml.contains("both worth your coffee"));
|
||||||
assert!(chapter.xhtml.contains("No. 42"));
|
assert!(chapter.xhtml.contains("No. 42"));
|
||||||
|
|||||||
@@ -16,17 +16,6 @@ use crate::config::Config;
|
|||||||
use crate::images;
|
use crate::images;
|
||||||
use crate::types::{Artifact, Edition, ImageAsset, Issue};
|
use crate::types::{Artifact, Edition, ImageAsset, Issue};
|
||||||
|
|
||||||
/// Chapter order inside an issue (§3.10).
|
|
||||||
pub const CHAPTER_ORDER: &[&str] = &[
|
|
||||||
"cover",
|
|
||||||
"from-the-editor",
|
|
||||||
"in-this-issue",
|
|
||||||
"sections",
|
|
||||||
"world-briefing",
|
|
||||||
"behind-the-paper",
|
|
||||||
"colophon",
|
|
||||||
];
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum EpubError {
|
pub enum EpubError {
|
||||||
#[error("epub build failed: {0}")]
|
#[error("epub build failed: {0}")]
|
||||||
|
|||||||
+16
-4
@@ -150,28 +150,36 @@ impl RatingSetLabel {
|
|||||||
|
|
||||||
#[derive(Debug, clap::Args)]
|
#[derive(Debug, clap::Args)]
|
||||||
struct RatingsListArgs {
|
struct RatingsListArgs {
|
||||||
|
/// How many days of verdicts to list.
|
||||||
#[arg(long, default_value_t = 90)]
|
#[arg(long, default_value_t = 90)]
|
||||||
days: i64,
|
days: i64,
|
||||||
|
/// Only verdicts with this label.
|
||||||
#[arg(long, value_enum)]
|
#[arg(long, value_enum)]
|
||||||
label: Option<RatingListLabel>,
|
label: Option<RatingListLabel>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, clap::Args)]
|
#[derive(Debug, clap::Args)]
|
||||||
struct RatingsSetArgs {
|
struct RatingsSetArgs {
|
||||||
|
/// Article id, as printed by `ratings list` or `explain`.
|
||||||
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
||||||
article: Option<ArticleId>,
|
article: Option<ArticleId>,
|
||||||
|
/// Article URL; canonicalized before lookup.
|
||||||
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
||||||
url: Option<String>,
|
url: Option<String>,
|
||||||
|
/// The verdict to record.
|
||||||
#[arg(long, value_enum)]
|
#[arg(long, value_enum)]
|
||||||
label: RatingSetLabel,
|
label: RatingSetLabel,
|
||||||
|
/// Free-text note shown to the weekly profile rebuild.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
note: Option<String>,
|
note: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, clap::Args)]
|
#[derive(Debug, clap::Args)]
|
||||||
struct RatingsClearArgs {
|
struct RatingsClearArgs {
|
||||||
|
/// Article id, as printed by `ratings list` or `explain`.
|
||||||
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
||||||
article: Option<ArticleId>,
|
article: Option<ArticleId>,
|
||||||
|
/// Article URL; canonicalized before lookup.
|
||||||
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
||||||
url: Option<String>,
|
url: Option<String>,
|
||||||
}
|
}
|
||||||
@@ -213,7 +221,7 @@ struct StatsArgs {
|
|||||||
enum FeaturesCommand {
|
enum FeaturesCommand {
|
||||||
/// Embed rated and published articles, then interests, into the cache.
|
/// Embed rated and published articles, then interests, into the cache.
|
||||||
Backfill(BackfillArgs),
|
Backfill(BackfillArgs),
|
||||||
/// Drop stale embeddings and old candidate telemetry per the retention config.
|
/// Drop stale embeddings, old candidate telemetry and old assessments per the retention config.
|
||||||
Prune,
|
Prune,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -668,7 +676,7 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
|||||||
}
|
}
|
||||||
FeaturesCommand::Prune => {
|
FeaturesCommand::Prune => {
|
||||||
let ranking = &config.curation.ranking;
|
let ranking = &config.curation.ranking;
|
||||||
let (embeddings, rows) = telemetry::prune(
|
let pruned = telemetry::prune(
|
||||||
db,
|
db,
|
||||||
ranking.embedding_retention_days,
|
ranking.embedding_retention_days,
|
||||||
ranking.telemetry_retention_days,
|
ranking.telemetry_retention_days,
|
||||||
@@ -676,8 +684,12 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
println!(
|
println!(
|
||||||
"pruned {embeddings} embeddings older than {} days and {rows} candidate rows older than {} days",
|
"pruned {} embeddings older than {} days, {} candidate rows and {} assessments older than {} days",
|
||||||
ranking.embedding_retention_days, ranking.telemetry_retention_days
|
pruned.embeddings,
|
||||||
|
ranking.embedding_retention_days,
|
||||||
|
pruned.telemetry,
|
||||||
|
pruned.assessments,
|
||||||
|
ranking.telemetry_retention_days
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+29
-3
@@ -254,6 +254,9 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
|||||||
};
|
};
|
||||||
|
|
||||||
db.finish_run(run_id, &report).await?;
|
db.finish_run(run_id, &report).await?;
|
||||||
|
if stages.published.is_some() {
|
||||||
|
prune_retention(config, db).await;
|
||||||
|
}
|
||||||
// The issue row is written before the report is costed, so stamp the finished
|
// The issue row is written before the report is costed, so stamp the finished
|
||||||
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
||||||
if !opts.dry_run
|
if !opts.dry_run
|
||||||
@@ -282,6 +285,28 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The retention sweep of `features prune` (§7.1, §7.4), run once per
|
||||||
|
/// published issue. Best effort: a failure is logged and never touches the run.
|
||||||
|
async fn prune_retention(config: &Config, db: &Db) {
|
||||||
|
let ranking = &config.curation.ranking;
|
||||||
|
match telemetry::prune(
|
||||||
|
db,
|
||||||
|
ranking.embedding_retention_days,
|
||||||
|
ranking.telemetry_retention_days,
|
||||||
|
Timestamp::now(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(pruned) => tracing::info!(
|
||||||
|
embeddings = pruned.embeddings,
|
||||||
|
candidate_rows = pruned.telemetry,
|
||||||
|
assessments = pruned.assessments,
|
||||||
|
"retention prune complete"
|
||||||
|
),
|
||||||
|
Err(error) => tracing::warn!(%error, "retention prune failed; continuing"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct StageOutput {
|
struct StageOutput {
|
||||||
@@ -1044,10 +1069,11 @@ async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the DeepSeek client, running the weekly profile rebuild when it is due.
|
/// Build the bulk (DeepSeek) and editor (Claude) clients, running the weekly
|
||||||
|
/// profile rebuild when it is due.
|
||||||
///
|
///
|
||||||
/// Returns `None` for `--skip-llm` and for every configuration/API problem: the
|
/// Each client is `None` for `--skip-llm` and for every configuration/API
|
||||||
/// caller then curates heuristically instead of failing the run (§3.6).
|
/// problem: the pipeline then degrades per §17 instead of failing the run.
|
||||||
async fn build_llms(
|
async fn build_llms(
|
||||||
ctx: &StageContext<'_>,
|
ctx: &StageContext<'_>,
|
||||||
bulk_meter: &UsageMeter,
|
bulk_meter: &UsageMeter,
|
||||||
|
|||||||
+2
-2
@@ -22,7 +22,7 @@ pub enum RunStatus {
|
|||||||
/// Everything completed.
|
/// Everything completed.
|
||||||
Ok,
|
Ok,
|
||||||
/// The issue was produced but a best-effort stage failed (social, XTC,
|
/// The issue was produced but a best-effort stage failed (social, XTC,
|
||||||
/// world briefing, images) or the cost guardrail tripped (§3.6).
|
/// world briefing, images) or a provider budget tripped (§5).
|
||||||
Degraded,
|
Degraded,
|
||||||
/// No issue was produced.
|
/// No issue was produced.
|
||||||
Failed,
|
Failed,
|
||||||
@@ -98,7 +98,7 @@ pub struct StageCounts {
|
|||||||
pub clusters: i64,
|
pub clusters: i64,
|
||||||
/// Admitted deep-set count retained for the colophon and runs table.
|
/// Admitted deep-set count retained for the colophon and runs table.
|
||||||
pub candidates: i64,
|
pub candidates: i64,
|
||||||
/// Articles in the final lineup (§3.6 stage B).
|
/// Articles in the final lineup (§13).
|
||||||
pub selected: i64,
|
pub selected: i64,
|
||||||
/// Discussion chapters rendered (§3.7).
|
/// Discussion chapters rendered (§3.7).
|
||||||
pub discussions: i64,
|
pub discussions: i64,
|
||||||
|
|||||||
+13
-11
@@ -246,7 +246,7 @@ pub fn composite_social_score(refs: &[SocialRef]) -> f64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Curation (§3.5, §3.6)
|
// Curation (plan §10–§13)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// DeepSeek's close read of one article (§12.1).
|
/// DeepSeek's close read of one article (§12.1).
|
||||||
@@ -322,7 +322,7 @@ impl Candidate {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One selected article with its section placement (§3.6 stage B).
|
/// One selected article with its section placement (§13).
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct Pick {
|
pub struct Pick {
|
||||||
pub article: Article,
|
pub article: Article,
|
||||||
@@ -340,13 +340,14 @@ pub struct Pick {
|
|||||||
pub discussion: Option<Discussion>,
|
pub discussion: Option<Discussion>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The day's final lineup: 15–25 picks grouped into sections (§3.6 stage B).
|
/// The day's final lineup grouped into sections (§13): no minimum size,
|
||||||
|
/// `curation.max_article_count` (or `--max-articles`) as the ceiling.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct Lineup {
|
pub struct Lineup {
|
||||||
pub date: Date,
|
pub date: Date,
|
||||||
/// Sorted by (section order, position).
|
/// Sorted by (section order, position).
|
||||||
pub picks: Vec<Pick>,
|
pub picks: Vec<Pick>,
|
||||||
/// Section names in issue order; empty sections are omitted (§3.6).
|
/// Section names in issue order; empty sections are omitted.
|
||||||
pub section_order: Vec<String>,
|
pub section_order: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -367,16 +368,17 @@ impl Lineup {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stage-C editorial output (§3.6).
|
/// Editorial output: the Brief and the per-article summaries (§14).
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct Editorial {
|
pub struct Editorial {
|
||||||
/// "From the Editor", 250–400 words, already sanitized XHTML.
|
/// The Brief (§14.2), 120–200 words, already sanitized XHTML.
|
||||||
pub front_page_html: String,
|
pub front_page_html: String,
|
||||||
/// Article id → 2–3 sentence newspaper abstract.
|
/// Article id → 2–3 sentence newspaper abstract (§14.1).
|
||||||
pub summaries: BTreeMap<ArticleId, String>,
|
pub summaries: BTreeMap<ArticleId, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The taste profile that forms the DeepSeek system prompt (§3.6, `kv`).
|
/// The reader profile that forms the system prompt shared by every LLM call
|
||||||
|
/// (§8.4); the current text lives in `kv`.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct TasteProfile {
|
pub struct TasteProfile {
|
||||||
/// Full ~600-word prompt document.
|
/// Full ~600-word prompt document.
|
||||||
@@ -743,10 +745,10 @@ pub struct RatedArticle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// LLM accounting (§3.6 cost guardrail)
|
// LLM accounting (§5 per-provider budgets)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// Token counters accumulated across every DeepSeek call in a run (§3.6).
|
/// Token counters accumulated across one provider's calls in a run (§5).
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct TokenUsage {
|
pub struct TokenUsage {
|
||||||
/// Cache-miss input tokens (billed at the full input rate).
|
/// Cache-miss input tokens (billed at the full input rate).
|
||||||
@@ -766,7 +768,7 @@ impl TokenUsage {
|
|||||||
self.output_tokens += other.output_tokens;
|
self.output_tokens += other.output_tokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// USD cost given the per-1M-token prices from `[deepseek]` config (§3.6).
|
/// USD cost given a provider's per-1M-token prices (§4.1–§4.2).
|
||||||
pub fn cost_usd(
|
pub fn cost_usd(
|
||||||
&self,
|
&self,
|
||||||
price_input: f64,
|
price_input: f64,
|
||||||
|
|||||||
+3
-3
@@ -160,11 +160,11 @@ fn dropped(element: &scraper::node::Element) -> bool {
|
|||||||
fn node_text(node: NodeRef<'_>, skip_lists: bool, out: &mut String) {
|
fn node_text(node: NodeRef<'_>, skip_lists: bool, out: &mut String) {
|
||||||
match node.value() {
|
match node.value() {
|
||||||
Node::Text(text) => {
|
Node::Text(text) => {
|
||||||
out.push_str(&text);
|
out.push_str(text);
|
||||||
out.push(' ');
|
out.push(' ');
|
||||||
}
|
}
|
||||||
Node::Element(element) => {
|
Node::Element(element) => {
|
||||||
if dropped(&element) || (skip_lists && matches!(element.name(), "ul" | "ol")) {
|
if dropped(element) || (skip_lists && matches!(element.name(), "ul" | "ol")) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for child in node.children() {
|
for child in node.children() {
|
||||||
@@ -208,7 +208,7 @@ fn links_without_child_lists(element: ElementRef<'_>, base: &Url) -> Vec<String>
|
|||||||
let Node::Element(element) = node.value() else {
|
let Node::Element(element) = node.value() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if dropped(&element) || matches!(element.name(), "ul" | "ol") {
|
if dropped(element) || matches!(element.name(), "ul" | "ol") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if element.name() == "a"
|
if element.name() == "a"
|
||||||
|
|||||||
@@ -596,7 +596,7 @@ async fn llm_pipeline_runs_against_a_mock_backend() {
|
|||||||
"every candidate came back assessed"
|
"every candidate came back assessed"
|
||||||
);
|
);
|
||||||
|
|
||||||
let lineup = curator.select(candidates, date()).await.expect("stage B");
|
let lineup = curator.select(candidates, date()).await.expect("editor");
|
||||||
assert_eq!(lineup.picks.len(), 5);
|
assert_eq!(lineup.picks.len(), 5);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
lineup.lead().map(|p| p.article.id),
|
lineup.lead().map(|p| p.article.id),
|
||||||
|
|||||||
@@ -109,10 +109,10 @@ fn deep_messy_fixture_is_salvaged_not_rejected() {
|
|||||||
assert!(parse_deep_response("", §ions).is_empty());
|
assert!(parse_deep_response("", §ions).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stage B responses must carry `{id, section, position, lead_story}` with
|
/// Editor responses must carry `{id, section, position, lead_story}` with
|
||||||
/// exactly one lead, and use only palette section names (§3.6).
|
/// exactly one lead, and use only palette section names (plan §13).
|
||||||
#[test]
|
#[test]
|
||||||
fn stage_b_fixture_parses_into_a_lineup() {
|
fn editor_fixture_parses_into_a_lineup() {
|
||||||
// The palette from `CurationConfig::default()` (§3.14).
|
// The palette from `CurationConfig::default()` (§3.14).
|
||||||
let sections = daily_epub::config::CurationConfig::default().sections;
|
let sections = daily_epub::config::CurationConfig::default().sections;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user