Curation v2 step 7: cleanup, prune paths, implementation notes
Dead code and stale v1 comments removed (clippy -W dead_code clean, the three world.rs warnings fixed), the Brief chapter's TOC title renamed from "From the Editor", features prune now also sweeps article_assessments and generate runs the sweep once after publishing, the example config is tested key-for-key against Config::default(), README commands match --help, and docs/plans/2026-08-15-implementation-notes.md records the Anthropic and Voyage facts, the new tables, the budget-day rule and the lock. Implemented by a Claude agent from docs/plans/curation-v2-briefs/step7.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
This commit is contained in:
Claude Fable 5.1
+68
-8
@@ -47,7 +47,8 @@ pub struct Config {
|
||||
pub timezone: String,
|
||||
/// Ingest window size in hours (§3.1).
|
||||
pub lookback_hours: u32,
|
||||
/// How many articles the lineup should contain (§3.6 stage B).
|
||||
/// Soft target for the lineup size (§13); `curation.max_article_count`
|
||||
/// is the ceiling and there is no minimum.
|
||||
pub target_article_count: usize,
|
||||
/// Days of published EPUBs kept in `publish.epub_dir` (§3.11).
|
||||
pub retention_days: u32,
|
||||
@@ -56,7 +57,8 @@ pub struct Config {
|
||||
/// Counted, not dated, because an XTCH issue is ~80–100 MB of pre-rendered
|
||||
/// page bitmaps: the constraint is disk, not age.
|
||||
pub xtc_retention_count: u32,
|
||||
/// Hard cost ceiling per run (§3.6 guardrail).
|
||||
/// DeepSeek spend ceiling per UTC day (§5); `[anthropic]` and `[voyage]`
|
||||
/// carry their own.
|
||||
pub max_daily_usd: f64,
|
||||
/// Include the Wikipedia Current Events section (§3.8).
|
||||
pub world_briefing: bool,
|
||||
@@ -129,7 +131,7 @@ impl Default for MinifluxConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// `[deepseek]` — LLM endpoint, model and pricing (§3.6, notes "verified facts").
|
||||
/// `[deepseek]` — bulk LLM endpoint, model and pricing (§4.1, notes "verified facts").
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct DeepseekConfig {
|
||||
@@ -263,7 +265,7 @@ impl Default for VoyageConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// `[curation]` — pre-filter and section palette (§3.5, §3.6).
|
||||
/// `[curation]` — hygiene, feedback weights, the ranker and the section palette (§19).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct CurationConfig {
|
||||
@@ -278,7 +280,7 @@ pub struct CurationConfig {
|
||||
/// Extra paywalled hosts, merged with [`crate::extract::DEFAULT_PAYWALL_DOMAINS`]
|
||||
/// by the extraction stage's `excerpt_only` heuristic (§3.3).
|
||||
pub paywall_domains: Vec<String>,
|
||||
/// The only section names the LLM may use (§3.6 stage B).
|
||||
/// The only section names the editor may use (§13).
|
||||
pub sections: Vec<String>,
|
||||
pub feedback: FeedbackConfig,
|
||||
pub ranking: RankingConfig,
|
||||
@@ -313,9 +315,7 @@ impl Default for CurationConfig {
|
||||
}
|
||||
|
||||
/// `[curation.ranking]` — every weight, quota, gate and threshold of the
|
||||
/// personalized ranker (plan §19). Steps 4–5 consume most of these; step 3
|
||||
/// uses the learned-signal gates, the preliminary weights and the retention
|
||||
/// windows.
|
||||
/// personalized ranker (plan §19).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct RankingConfig {
|
||||
@@ -925,6 +925,66 @@ mod tests {
|
||||
assert_eq!(c.editorial.summary_input_tokens, 3000);
|
||||
}
|
||||
|
||||
/// `config.example.toml` documents the plan's numbers (§19), which are also
|
||||
/// `Config::default()`: every documented key in these sections must exist
|
||||
/// on the struct with the default value, and every struct field (except
|
||||
/// the env-only `api_key`) must be documented in the file.
|
||||
#[test]
|
||||
fn shipped_example_config_matches_the_defaults_key_for_key() {
|
||||
let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("config.example.toml");
|
||||
let documented: serde_json::Value = Figment::from(Toml::file(&example))
|
||||
.extract()
|
||||
.expect("config.example.toml must parse as a table");
|
||||
let defaults = serde_json::to_value(Config::default()).expect("defaults serialize");
|
||||
|
||||
fn compare(path: &str, documented: &serde_json::Value, default: &serde_json::Value) {
|
||||
let (Some(documented), Some(default)) = (documented.as_object(), default.as_object())
|
||||
else {
|
||||
// TOML `60` and the f64 default `60.0` are the same setting, and
|
||||
// an f32 field (`editorial_temperature`) widens inexactly.
|
||||
match (documented.as_f64(), default.as_f64()) {
|
||||
(Some(doc), Some(def)) => assert!(
|
||||
(doc - def).abs() <= 1e-6 * def.abs().max(1.0),
|
||||
"{path}: documented {doc} vs default {def}"
|
||||
),
|
||||
_ => assert_eq!(documented, default, "{path}"),
|
||||
}
|
||||
return;
|
||||
};
|
||||
for (key, value) in default {
|
||||
if key == "api_key" {
|
||||
assert!(
|
||||
!documented.contains_key(key),
|
||||
"{path}.{key} must stay out of the TOML (env only)"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let doc = documented
|
||||
.get(key)
|
||||
.unwrap_or_else(|| panic!("{path}.{key} is missing from config.example.toml"));
|
||||
compare(&format!("{path}.{key}"), doc, value);
|
||||
}
|
||||
for key in documented.keys() {
|
||||
assert!(
|
||||
default.contains_key(key),
|
||||
"{path}.{key} is documented but not a config field"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (key, default) in defaults.as_object().expect("config is a table") {
|
||||
let section = match key.as_str() {
|
||||
"curation" | "anthropic" | "voyage" | "editorial" | "deepseek" => key,
|
||||
"target_article_count" | "max_daily_usd" | "profile_path" | "interests_opml" => key,
|
||||
_ => continue,
|
||||
};
|
||||
let documented = documented
|
||||
.get(section)
|
||||
.unwrap_or_else(|| panic!("{section} is missing from config.example.toml"));
|
||||
compare(section, documented, default);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_validation_rejects_nonsense() {
|
||||
let mut c = Config::default();
|
||||
|
||||
+2
-1
@@ -122,7 +122,8 @@ pub struct UsageMeter {
|
||||
}
|
||||
|
||||
impl UsageMeter {
|
||||
/// Compatibility constructor for the existing DeepSeek call sites.
|
||||
/// A meter priced from the `[deepseek]` table; the other providers build
|
||||
/// theirs with [`UsageMeter::with_prices`].
|
||||
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
||||
Self::with_prices(PriceTable::deepseek(cfg), limit_usd)
|
||||
}
|
||||
|
||||
+2
-2
@@ -148,7 +148,7 @@ impl Curator {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
||||
/// prose. Only used to size prompt budgets (§3.6 stage C).
|
||||
/// prose. Only used to size prompt budgets.
|
||||
pub fn approx_tokens(text: &str) -> usize {
|
||||
text.len().div_ceil(4)
|
||||
}
|
||||
@@ -215,7 +215,7 @@ pub fn truncate_words(text: &str, max_words: usize) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C).
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary.
|
||||
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.len() <= max_chars {
|
||||
|
||||
@@ -16,7 +16,10 @@ use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
||||
use crate::types::{Facets, RatedArticle, TasteProfile};
|
||||
|
||||
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
||||
pub const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
||||
/// The verdict block and the weekly rebuild are bounded by count
|
||||
/// (`verdicts_in_prompt`, [`MAX_RATINGS_IN_REBUILD`]), not by age (§8.3, §8.4),
|
||||
/// so their `current_ratings` lookback is effectively unbounded.
|
||||
const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
||||
pub const KV_LEARNED_ADJUSTMENTS: &str = "taste_profile_learned";
|
||||
const MAX_RATINGS_IN_REBUILD: usize = 200;
|
||||
|
||||
|
||||
+53
-19
@@ -18,18 +18,7 @@ use crate::db::{Db, fmt_ts};
|
||||
use crate::report::RunReport;
|
||||
use crate::types::{ArticleId, Candidate, NearMiss};
|
||||
|
||||
/// The stage vocabulary of §7.4, in pipeline order.
|
||||
pub const STAGES: [&str; 7] = [
|
||||
"excluded",
|
||||
"eligible",
|
||||
"triaged",
|
||||
"admitted",
|
||||
"assessed",
|
||||
"shortlisted",
|
||||
"selected",
|
||||
];
|
||||
|
||||
/// Signal names rendered by `explain`, including the LLM ones steps 4–5 add.
|
||||
/// Signal names rendered by `explain`, in the order of §7.5.
|
||||
const RENDERED_SIGNALS: [&str; 8] = [
|
||||
"interest",
|
||||
"knn",
|
||||
@@ -850,15 +839,29 @@ pub async fn stats(db: &Db, days: i64, now: Timestamp) -> anyhow::Result<String>
|
||||
// `features prune` (§7.1, §7.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Rows removed by one [`prune`] pass.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Pruned {
|
||||
/// `article_embeddings` of unrated, unpublished articles past
|
||||
/// `embedding_retention_days`.
|
||||
pub embeddings: u64,
|
||||
/// `candidate_runs` rows of runs past `telemetry_retention_days`.
|
||||
pub telemetry: u64,
|
||||
/// `article_assessments` assessed more than `telemetry_retention_days` ago.
|
||||
pub assessments: u64,
|
||||
}
|
||||
|
||||
/// Delete `article_embeddings` for articles neither rated nor published that
|
||||
/// are older than `embedding_retention_days`, and `candidate_runs` rows whose
|
||||
/// run started more than `telemetry_retention_days` ago. Returns the counts.
|
||||
/// are older than `embedding_retention_days`, `candidate_runs` rows whose run
|
||||
/// started more than `telemetry_retention_days` ago, and `article_assessments`
|
||||
/// older than the same window (§7.1, §7.4). Runs from `features prune` and
|
||||
/// once per `generate` after publishing.
|
||||
pub async fn prune(
|
||||
db: &Db,
|
||||
embedding_retention_days: i64,
|
||||
telemetry_retention_days: i64,
|
||||
now: Timestamp,
|
||||
) -> Result<(u64, u64), sqlx::Error> {
|
||||
) -> Result<Pruned, sqlx::Error> {
|
||||
let cutoff = |days: i64| {
|
||||
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
||||
.unwrap_or(Timestamp::UNIX_EPOCH)
|
||||
@@ -886,7 +889,17 @@ pub async fn prune(
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok((embeddings, telemetry))
|
||||
|
||||
let assessments = sqlx::query("DELETE FROM article_assessments WHERE assessed_at < ?")
|
||||
.bind(fmt_ts(cutoff(telemetry_retention_days)))
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok(Pruned {
|
||||
embeddings,
|
||||
telemetry,
|
||||
assessments,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1523,13 +1536,26 @@ mod tests {
|
||||
let new_run = db.start_run(date(), now).await.unwrap();
|
||||
thin_excluded(&db, old_run, 1, "blocked").await.unwrap();
|
||||
thin_excluded(&db, new_run, 1, "blocked").await.unwrap();
|
||||
for (id, assessed_at) in [(1, old.clone()), (2, fmt_ts(now))] {
|
||||
sqlx::query(
|
||||
"INSERT INTO article_assessments
|
||||
(article_id, stage, model, prompt_version, score, assessed_at)
|
||||
VALUES (?, 'triage', 'deepseek-v4-flash', 1, 7.0, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&assessed_at)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let (embeddings, telemetry) = prune(&db, 120, 180, now).await.unwrap();
|
||||
let pruned = prune(&db, 120, 180, now).await.unwrap();
|
||||
assert_eq!(
|
||||
embeddings, 1,
|
||||
pruned.embeddings, 1,
|
||||
"only the old, unrated, unpublished article 3"
|
||||
);
|
||||
assert_eq!(telemetry, 1, "only the old run's rows");
|
||||
assert_eq!(pruned.telemetry, 1, "only the old run's rows");
|
||||
assert_eq!(pruned.assessments, 1, "only the 200-day-old assessment");
|
||||
let remaining: Vec<i64> =
|
||||
sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id")
|
||||
.fetch_all(db.pool())
|
||||
@@ -1541,5 +1567,13 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(runs, vec![new_run]);
|
||||
let assessed: Vec<i64> = sqlx::query_scalar("SELECT article_id FROM article_assessments")
|
||||
.fetch_all(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(assessed, vec![2]);
|
||||
|
||||
// A second pass finds nothing left to remove.
|
||||
assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,9 +25,9 @@ pub static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations");
|
||||
|
||||
/// `kv` key holding the ingest watermark (§3.1).
|
||||
pub const KV_WATERMARK: &str = "ingest_watermark";
|
||||
/// `kv` key holding the current taste profile document (§3.6).
|
||||
/// `kv` key holding the current system-prompt profile document (§8.4).
|
||||
pub const KV_TASTE_PROFILE: &str = "taste_profile";
|
||||
/// `kv` key holding the taste profile version/build time (§3.6).
|
||||
/// `kv` key holding the profile version/build time (§8.2).
|
||||
pub const KV_PROFILE_VERSION: &str = "profile_version";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -107,7 +107,7 @@ impl Db {
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// kv (§3.1 watermark, §3.6 taste profile)
|
||||
// kv (§3.1 watermark, §8 profile)
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
pub async fn kv_get(&self, key: &str) -> Result<Option<String>> {
|
||||
@@ -635,7 +635,7 @@ impl Db {
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// runs (§3.6 cost guardrail, §3.13)
|
||||
// runs (§3.13, plan §7.6)
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
/// Insert a `running` row at the top of `generate`; returns `runs.id`.
|
||||
|
||||
@@ -237,7 +237,7 @@ fn published_display(pick: &Pick) -> Option<String> {
|
||||
.map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string())
|
||||
}
|
||||
|
||||
/// "From the Editor" front page plus the issue stats line (§3.10).
|
||||
/// The Brief (§14.2) under the masthead, plus the issue stats line (§3.10).
|
||||
pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||
let body = issue.editorial.front_page_html.trim();
|
||||
let body_html = if body.is_empty() {
|
||||
@@ -249,7 +249,7 @@ pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||
to_xhtml(&ammonia::clean(body))
|
||||
};
|
||||
let tpl = FrontPage {
|
||||
title: "From the Editor".into(),
|
||||
title: "The Brief".into(),
|
||||
display_date: issue.meta.display_date.clone(),
|
||||
issue_number: issue.meta.issue_number,
|
||||
stats_line: issue.meta.stats_line(),
|
||||
@@ -258,7 +258,7 @@ pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
|
||||
Ok(Chapter {
|
||||
id: "front".into(),
|
||||
href: "front.xhtml".into(),
|
||||
title: "From the Editor".into(),
|
||||
title: "The Brief".into(),
|
||||
xhtml: tpl.render()?,
|
||||
toc_level: 1,
|
||||
})
|
||||
@@ -726,7 +726,7 @@ mod tests {
|
||||
let issue = issue();
|
||||
let chapter = render_front_page(&issue).unwrap();
|
||||
assert_eq!(chapter.href, "front.xhtml");
|
||||
assert!(chapter.xhtml.contains("From the Editor"));
|
||||
assert!(chapter.xhtml.contains("The Brief"));
|
||||
assert!(chapter.xhtml.contains("2 articles"));
|
||||
assert!(chapter.xhtml.contains("both worth your coffee"));
|
||||
assert!(chapter.xhtml.contains("No. 42"));
|
||||
|
||||
@@ -16,17 +16,6 @@ use crate::config::Config;
|
||||
use crate::images;
|
||||
use crate::types::{Artifact, Edition, ImageAsset, Issue};
|
||||
|
||||
/// Chapter order inside an issue (§3.10).
|
||||
pub const CHAPTER_ORDER: &[&str] = &[
|
||||
"cover",
|
||||
"from-the-editor",
|
||||
"in-this-issue",
|
||||
"sections",
|
||||
"world-briefing",
|
||||
"behind-the-paper",
|
||||
"colophon",
|
||||
];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum EpubError {
|
||||
#[error("epub build failed: {0}")]
|
||||
|
||||
+16
-4
@@ -150,28 +150,36 @@ impl RatingSetLabel {
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RatingsListArgs {
|
||||
/// How many days of verdicts to list.
|
||||
#[arg(long, default_value_t = 90)]
|
||||
days: i64,
|
||||
/// Only verdicts with this label.
|
||||
#[arg(long, value_enum)]
|
||||
label: Option<RatingListLabel>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RatingsSetArgs {
|
||||
/// Article id, as printed by `ratings list` or `explain`.
|
||||
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
||||
article: Option<ArticleId>,
|
||||
/// Article URL; canonicalized before lookup.
|
||||
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
||||
url: Option<String>,
|
||||
/// The verdict to record.
|
||||
#[arg(long, value_enum)]
|
||||
label: RatingSetLabel,
|
||||
/// Free-text note shown to the weekly profile rebuild.
|
||||
#[arg(long)]
|
||||
note: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RatingsClearArgs {
|
||||
/// Article id, as printed by `ratings list` or `explain`.
|
||||
#[arg(long, required_unless_present = "url", conflicts_with = "url")]
|
||||
article: Option<ArticleId>,
|
||||
/// Article URL; canonicalized before lookup.
|
||||
#[arg(long, required_unless_present = "article", conflicts_with = "article")]
|
||||
url: Option<String>,
|
||||
}
|
||||
@@ -213,7 +221,7 @@ struct StatsArgs {
|
||||
enum FeaturesCommand {
|
||||
/// Embed rated and published articles, then interests, into the cache.
|
||||
Backfill(BackfillArgs),
|
||||
/// Drop stale embeddings and old candidate telemetry per the retention config.
|
||||
/// Drop stale embeddings, old candidate telemetry and old assessments per the retention config.
|
||||
Prune,
|
||||
}
|
||||
|
||||
@@ -668,7 +676,7 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
||||
}
|
||||
FeaturesCommand::Prune => {
|
||||
let ranking = &config.curation.ranking;
|
||||
let (embeddings, rows) = telemetry::prune(
|
||||
let pruned = telemetry::prune(
|
||||
db,
|
||||
ranking.embedding_retention_days,
|
||||
ranking.telemetry_retention_days,
|
||||
@@ -676,8 +684,12 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
||||
)
|
||||
.await?;
|
||||
println!(
|
||||
"pruned {embeddings} embeddings older than {} days and {rows} candidate rows older than {} days",
|
||||
ranking.embedding_retention_days, ranking.telemetry_retention_days
|
||||
"pruned {} embeddings older than {} days, {} candidate rows and {} assessments older than {} days",
|
||||
pruned.embeddings,
|
||||
ranking.embedding_retention_days,
|
||||
pruned.telemetry,
|
||||
pruned.assessments,
|
||||
ranking.telemetry_retention_days
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+29
-3
@@ -254,6 +254,9 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
||||
};
|
||||
|
||||
db.finish_run(run_id, &report).await?;
|
||||
if stages.published.is_some() {
|
||||
prune_retention(config, db).await;
|
||||
}
|
||||
// The issue row is written before the report is costed, so stamp the finished
|
||||
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
||||
if !opts.dry_run
|
||||
@@ -282,6 +285,28 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
||||
})
|
||||
}
|
||||
|
||||
/// The retention sweep of `features prune` (§7.1, §7.4), run once per
|
||||
/// published issue. Best effort: a failure is logged and never touches the run.
|
||||
async fn prune_retention(config: &Config, db: &Db) {
|
||||
let ranking = &config.curation.ranking;
|
||||
match telemetry::prune(
|
||||
db,
|
||||
ranking.embedding_retention_days,
|
||||
ranking.telemetry_retention_days,
|
||||
Timestamp::now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(pruned) => tracing::info!(
|
||||
embeddings = pruned.embeddings,
|
||||
candidate_rows = pruned.telemetry,
|
||||
assessments = pruned.assessments,
|
||||
"retention prune complete"
|
||||
),
|
||||
Err(error) => tracing::warn!(%error, "retention prune failed; continuing"),
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
||||
#[derive(Debug)]
|
||||
struct StageOutput {
|
||||
@@ -1044,10 +1069,11 @@ async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the DeepSeek client, running the weekly profile rebuild when it is due.
|
||||
/// Build the bulk (DeepSeek) and editor (Claude) clients, running the weekly
|
||||
/// profile rebuild when it is due.
|
||||
///
|
||||
/// Returns `None` for `--skip-llm` and for every configuration/API problem: the
|
||||
/// caller then curates heuristically instead of failing the run (§3.6).
|
||||
/// Each client is `None` for `--skip-llm` and for every configuration/API
|
||||
/// problem: the pipeline then degrades per §17 instead of failing the run.
|
||||
async fn build_llms(
|
||||
ctx: &StageContext<'_>,
|
||||
bulk_meter: &UsageMeter,
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ pub enum RunStatus {
|
||||
/// Everything completed.
|
||||
Ok,
|
||||
/// The issue was produced but a best-effort stage failed (social, XTC,
|
||||
/// world briefing, images) or the cost guardrail tripped (§3.6).
|
||||
/// world briefing, images) or a provider budget tripped (§5).
|
||||
Degraded,
|
||||
/// No issue was produced.
|
||||
Failed,
|
||||
@@ -98,7 +98,7 @@ pub struct StageCounts {
|
||||
pub clusters: i64,
|
||||
/// Admitted deep-set count retained for the colophon and runs table.
|
||||
pub candidates: i64,
|
||||
/// Articles in the final lineup (§3.6 stage B).
|
||||
/// Articles in the final lineup (§13).
|
||||
pub selected: i64,
|
||||
/// Discussion chapters rendered (§3.7).
|
||||
pub discussions: i64,
|
||||
|
||||
+13
-11
@@ -246,7 +246,7 @@ pub fn composite_social_score(refs: &[SocialRef]) -> f64 {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Curation (§3.5, §3.6)
|
||||
// Curation (plan §10–§13)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// DeepSeek's close read of one article (§12.1).
|
||||
@@ -322,7 +322,7 @@ impl Candidate {
|
||||
}
|
||||
}
|
||||
|
||||
/// One selected article with its section placement (§3.6 stage B).
|
||||
/// One selected article with its section placement (§13).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Pick {
|
||||
pub article: Article,
|
||||
@@ -340,13 +340,14 @@ pub struct Pick {
|
||||
pub discussion: Option<Discussion>,
|
||||
}
|
||||
|
||||
/// The day's final lineup: 15–25 picks grouped into sections (§3.6 stage B).
|
||||
/// The day's final lineup grouped into sections (§13): no minimum size,
|
||||
/// `curation.max_article_count` (or `--max-articles`) as the ceiling.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Lineup {
|
||||
pub date: Date,
|
||||
/// Sorted by (section order, position).
|
||||
pub picks: Vec<Pick>,
|
||||
/// Section names in issue order; empty sections are omitted (§3.6).
|
||||
/// Section names in issue order; empty sections are omitted.
|
||||
pub section_order: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -367,16 +368,17 @@ impl Lineup {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage-C editorial output (§3.6).
|
||||
/// Editorial output: the Brief and the per-article summaries (§14).
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Editorial {
|
||||
/// "From the Editor", 250–400 words, already sanitized XHTML.
|
||||
/// The Brief (§14.2), 120–200 words, already sanitized XHTML.
|
||||
pub front_page_html: String,
|
||||
/// Article id → 2–3 sentence newspaper abstract.
|
||||
/// Article id → 2–3 sentence newspaper abstract (§14.1).
|
||||
pub summaries: BTreeMap<ArticleId, String>,
|
||||
}
|
||||
|
||||
/// The taste profile that forms the DeepSeek system prompt (§3.6, `kv`).
|
||||
/// The reader profile that forms the system prompt shared by every LLM call
|
||||
/// (§8.4); the current text lives in `kv`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TasteProfile {
|
||||
/// Full ~600-word prompt document.
|
||||
@@ -743,10 +745,10 @@ pub struct RatedArticle {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM accounting (§3.6 cost guardrail)
|
||||
// LLM accounting (§5 per-provider budgets)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Token counters accumulated across every DeepSeek call in a run (§3.6).
|
||||
/// Token counters accumulated across one provider's calls in a run (§5).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TokenUsage {
|
||||
/// Cache-miss input tokens (billed at the full input rate).
|
||||
@@ -766,7 +768,7 @@ impl TokenUsage {
|
||||
self.output_tokens += other.output_tokens;
|
||||
}
|
||||
|
||||
/// USD cost given the per-1M-token prices from `[deepseek]` config (§3.6).
|
||||
/// USD cost given a provider's per-1M-token prices (§4.1–§4.2).
|
||||
pub fn cost_usd(
|
||||
&self,
|
||||
price_input: f64,
|
||||
|
||||
+3
-3
@@ -160,11 +160,11 @@ fn dropped(element: &scraper::node::Element) -> bool {
|
||||
fn node_text(node: NodeRef<'_>, skip_lists: bool, out: &mut String) {
|
||||
match node.value() {
|
||||
Node::Text(text) => {
|
||||
out.push_str(&text);
|
||||
out.push_str(text);
|
||||
out.push(' ');
|
||||
}
|
||||
Node::Element(element) => {
|
||||
if dropped(&element) || (skip_lists && matches!(element.name(), "ul" | "ol")) {
|
||||
if dropped(element) || (skip_lists && matches!(element.name(), "ul" | "ol")) {
|
||||
return;
|
||||
}
|
||||
for child in node.children() {
|
||||
@@ -208,7 +208,7 @@ fn links_without_child_lists(element: ElementRef<'_>, base: &Url) -> Vec<String>
|
||||
let Node::Element(element) = node.value() else {
|
||||
return;
|
||||
};
|
||||
if dropped(&element) || matches!(element.name(), "ul" | "ol") {
|
||||
if dropped(element) || matches!(element.name(), "ul" | "ol") {
|
||||
return;
|
||||
}
|
||||
if element.name() == "a"
|
||||
|
||||
Reference in New Issue
Block a user