//! Personalized curation: signals → triage → admission → assessment → editor. //! //! ```text //! ~400 eligible ─triage─▶ admission (120) ─assess/rank─▶ shortlist (60) ─editor─▶ editorial //! ``` //! //! [`Curator`] is the thin orchestration layer the `generate` pipeline calls; the //! interesting logic lives in the stage modules. Every stage is safe to run with //! no provider at all (`--skip-llm`): the cheap-signal blend stands in for selection //! and feed excerpts stand in for summaries (notes §6). Scoring runs on the bulk //! client; selection and editorial on the editor with per-call bulk fallback. pub mod admit; pub mod assess; pub mod batch; pub mod editor; pub mod editorial; pub mod embedding; pub mod llm; pub mod prefilter; pub mod profile; pub mod rank; pub mod signals; pub mod telemetry; pub mod triage; use jiff::{Timestamp, civil::Date}; use crate::config::Config; use crate::db::Db; use crate::types::{Candidate, Editorial, Lineup}; /// Runs the LLM curation stages against one day's articles (§12, §13, §14). pub struct Curator { pub config: Config, pub db: Db, pub llms: llm::Llms, } impl Curator { /// An empty [`llm::Llms`] corresponds to `--skip-llm`: cheap-signal order is /// used for selection and feed excerpts stand in for summaries (notes §6). /// With only `bulk`, every editor call runs on the bulk provider (§4.2). pub fn new(config: Config, db: Db, llms: llm::Llms) -> Self { Self { config, db, llms } } /// Deep assessment of the admitted set on the bulk client (§12.1), reusing /// cached `article_assessments` rows within `assessment_reuse_days`. /// /// A no-op under `--skip-llm`: like triage, nothing is read or written and /// utility falls back to the present signals (§12.3, §17). When the bulk /// provider is down or its budget trips, cached rows are still reused and /// the failed batches simply stay unassessed. A batch the provider's /// content filter rejects is bisected, and a rejected single article is /// retried on the editor when that is another provider (`curate::batch`). pub async fn assess( &self, candidates: &mut [Candidate], rescore: bool, profile_version: Option, assessed_at: Timestamp, ) -> anyhow::Result { let Some(bulk) = self.llms.bulk.as_ref() else { tracing::info!("--skip-llm: deep assessment skipped"); return Ok(batch::StageSummary::default()); }; let admitted = candidates .iter() .filter(|candidate| candidate.stage == "admitted") .count(); let span = tracing::info_span!("llm_assess", candidates = admitted); let _guard = span.enter(); assess::run( &self.db, Some(bulk), self.llms.editor.as_ref(), &bulk.model, candidates, self.config.llm.deep_batch_size, bulk.max_concurrent_requests, self.config.curation.ranking.assessment_reuse_days, rescore, profile_version, assessed_at, self.config.llm.score_temperature, &self.config.curation.sections, ) .await } /// The editor: one call that assembles the issue from the shortlist (§13). pub async fn select(&self, candidates: Vec, date: Date) -> anyhow::Result { let sections = &self.config.curation.sections; let soft_target = self.config.target_article_count; let hard_max = self.config.curation.max_article_count; let span = tracing::info_span!("llm_editor", candidates = candidates.len()); let _guard = span.enter(); match editor::select( &self.llms, candidates.clone(), sections, soft_target, hard_max, date, ) .await { Ok(lineup) => Ok(lineup), Err(error) => { tracing::error!(%error, "editor and bulk fallback failed; selecting heuristically"); Ok(editor::select_without_llm( candidates, sections, soft_target, hard_max, date, )) } } } /// Stage C: per-article summaries and the Brief (§14). /// /// Never fails the run: a budget trip or an API error degrades to excerpts. pub async fn editorial(&self, lineup: &Lineup) -> anyhow::Result { Ok(self.editorial_timed(lineup).await?.0) } /// [`Self::editorial`] plus the `summaries` / `brief` stage timings (§15.4). pub async fn editorial_timed( &self, lineup: &Lineup, ) -> anyhow::Result<(Editorial, editorial::EditorialTimings)> { if self.llms.editor_or_bulk().is_none() { tracing::info!("--skip-llm: using feed excerpts as summaries"); return Ok(( editorial::fallback_editorial(lineup), editorial::EditorialTimings::default(), )); } let span = tracing::info_span!("llm_editorial", picks = lineup.picks.len()); let _guard = span.enter(); Ok(editorial::run_timed( &self.llms, lineup, &self.config.editorial, self.config.llm.editorial_temperature, ) .await) } } // --------------------------------------------------------------------------- // Small text helpers shared by the prompt builders // --------------------------------------------------------------------------- /// Crude token estimate: DeepSeek averages ~4 characters per token for English /// prose. Only used to size prompt budgets. pub fn approx_tokens(text: &str) -> usize { text.len().div_ceil(4) } /// Strip markup and collapse whitespace, so article bodies can go into prompts /// as plain text (cheaper and less confusing for the model than raw HTML). /// /// Deliberately not [`crate::html::html_to_text`]: this one collapses runs of /// whitespace and never builds a DOM, because it runs over every candidate /// body on every run and only has to be good enough to size a prompt. pub fn prompt_text(html: &str) -> String { /// Does `tail` open the named element, i.e. ` bool { let bytes = tail.as_bytes(); bytes.len() > name.len() && bytes[1..=name.len()].eq_ignore_ascii_case(name.as_bytes()) } let mut out = String::with_capacity(html.len()); let mut rest = html; while let Some(ch) = rest.chars().next() { if ch != '<' { out.push(ch); rest = &rest[ch.len_utf8()..]; continue; } // Drop \

Second
line

"; assert_eq!(prompt_text(html), "Title First & best. Second line"); assert_eq!(prompt_text(""), ""); assert_eq!(prompt_text("no markup at all"), "no markup at all"); // Unicode survives byte-wise walking. assert_eq!(prompt_text("

café — naïve

"), "café — naïve"); } #[test] fn word_and_token_truncation() { assert_eq!(truncate_words("one two three", 5), "one two three"); assert_eq!(truncate_words("one two three", 2), "one two…"); let long = "word ".repeat(1000); // 10 tokens ≈ 40 characters, cut back to a word boundary, plus the ellipsis. let cut = truncate_tokens(&long, 10); assert!(cut.len() <= 43, "{}", cut.len()); assert!(cut.split_whitespace().count() <= 10); assert!(cut.ends_with('…')); assert_eq!(truncate_tokens("short", 10), "short"); assert!(approx_tokens("abcd") <= 1); } #[test] fn escaping_and_paragraphs() { assert_eq!(escape_html("a&'\""), "a<b>&'""); assert_eq!( text_to_paragraphs("One\nline.\n\nTwo ."), "

One line.

\n

Two <b>.

" ); assert_eq!(text_to_paragraphs(" "), ""); } }