thallada/the-daily-epub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Curation v2 step 5: deep assessment, utility, diversified shortlist

Browse Source 
assess.rs replaces score.rs (DEEP_INSTRUCTIONS, representative sample
with [BEGINNING]/[MIDDLE]/[END], facets, cached deep rows with --rescore
bypass), rank.rs adds the utility blend over present signals with gate
ramps and the leader-clustered shortlist (cap 2 → 3 → uncapped, protected
top-N, exploration reserve), editor.rs replaces select.rs with the §13
rendering and utility-ordered fallbacks. ScoredArticle is gone; Candidate
is the only flow type. deep_batch_size replaces score_batch_size.

Started by Codex (cut off by its usage limit mid-verification) and
finished by a Claude agent from docs/plans/curation-v2-briefs/step5.md;
reviewed against plan §12–§13.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
This commit is contained in:
thallada
2026-09-02 16:04:43 +00:00
co-authored by Claude Fable 5.1
parent 10f4afd091
commit 05a74a0dcf
22 changed files with 2710 additions and 1298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e_pipeline.rs
+28 -17
View File
@@ -18,7 +18,7 @@
//! no article in the fixtures carries an image, so the EPUB builder's image
//! downloader has nothing to fetch.
use std::collections::BTreeMap;
use std::collections::{BTreeMap, HashMap};
use std::path::Path;
use jiff::Timestamp;
@@ -26,7 +26,7 @@ use jiff::civil::Date;
use daily_epub::config::{Config, PublishConfig, ServerConfig, XtcConfig};
use daily_epub::curate::llm::{LlmClient, Llms, MockBackend, UsageMeter};
use daily_epub::curate::{Curator, admit, editorial};
use daily_epub::curate::{Curator, admit, editorial, rank};
use daily_epub::db::Db;
use daily_epub::extract::Extractor;
use daily_epub::types::{
@@ -412,7 +412,6 @@ async fn skip_llm_pipeline_produces_a_published_issue() {
let candidates = personalized
.into_iter()
.filter(|candidate| candidate.stage == "admitted")
.map(Candidate::into_legacy_scored)
.collect::<Vec<_>>();
assert_eq!(candidates.len(), 5, "nothing is dropped at this volume");
// The excerpt-only story is penalized (§3.5).
@@ -421,10 +420,10 @@ async fn skip_llm_pipeline_produces_a_published_issue() {
.find(|c| c.article.excerpt_only)
.expect("the allocator teaser survived");
assert!(
allocator.prefilter_score
allocator.signals.heuristic.unwrap_or_default()
< candidates
.iter()
.map(|candidate| candidate.prefilter_score)
.filter_map(|candidate| candidate.signals.heuristic)
.fold(f64::NEG_INFINITY, f64::max)
);
@@ -516,13 +515,12 @@ async fn llm_pipeline_runs_against_a_mock_backend() {
let candidates = personalized
.into_iter()
.filter(|candidate| candidate.stage == "admitted")
.map(Candidate::into_legacy_scored)
.collect::<Vec<_>>();
let ids: Vec<i64> = candidates.iter().map(|c| c.article.id).collect();
assert_eq!(ids.len(), 5);
// --- Script DeepSeek: one stage-A batch, one stage-B call, five stage-C
// summaries and one front page (§3.6). ---
// --- Script DeepSeek: one deep-assessment batch, one editor call, five
// summaries and one brief. ---
let backend = std::sync::Arc::new(MockBackend::new());
let usage = daily_epub::types::TokenUsage {
input_tokens: 1000,
@@ -535,8 +533,8 @@ async fn llm_pipeline_runs_against_a_mock_backend() {
.enumerate()
.map(|(i, id)| {
format!(
r#"{{"id": {id}, "score": {}, "category": "Tech & Engineering",
"rationale": "solid systems writeup", "is_paywalled_guess": false}}"#,
r#"{{"id": {id}, "quality": {}, "fit": 7, "category": "Tech & Engineering",
"rationale": "solid systems writeup", "paywalled_guess": false, "facets": {{"format":"analysis_essay"}}}}"#,
9 - i
)
})
@@ -590,12 +588,12 @@ async fn llm_pipeline_runs_against_a_mock_backend() {
let mut candidates = candidates;
curator
.score(&mut candidates, date())
.assess(&mut candidates, true, None, jiff::Timestamp::now())
.await
.expect("stage A");
.expect("deep assessment");
assert!(
candidates.iter().all(|c| c.llm.is_some()),
"every candidate came back scored"
candidates.iter().all(|c| c.assessment.deep.is_some()),
"every candidate came back assessed"
);
let lineup = curator.select(candidates, date()).await.expect("stage B");
@@ -680,7 +678,6 @@ async fn failing_deepseek_still_publishes_with_heuristic_fallbacks() {
let mut candidates = personalized
.into_iter()
.filter(|candidate| candidate.stage == "admitted")
.map(Candidate::into_legacy_scored)
.collect::<Vec<_>>();
let backend = std::sync::Arc::new(MockBackend::new());
@@ -699,10 +696,24 @@ async fn failing_deepseek_still_publishes_with_heuristic_fallbacks() {
},
);
curator
.score(&mut candidates, date())
.assess(&mut candidates, true, None, jiff::Timestamp::now())
.await
.expect("failed batches degrade, not abort");
assert!(candidates.iter().all(|candidate| candidate.llm.is_none()));
assert!(
candidates
.iter()
.all(|candidate| candidate.assessment.deep.is_none())
);
// Utility over the present signals, clustered as singletons without
// embeddings: every admitted article is shortlisted with a cluster id.
let ranked = rank::shortlist(&mut candidates, &HashMap::new(), &cfg.curation.ranking);
assert_eq!(ranked.shortlisted, candidates.len());
assert_eq!(ranked.clusters, candidates.len());
for candidate in &candidates {
assert_eq!(candidate.stage, "shortlisted");
assert!(candidate.utility.is_some() && candidate.cluster.is_some());
assert!(candidate.rank_utility.is_some());
}
let mut lineup = curator
.select(candidates, date())
.await
tests/fixtures/deepseek_deep_batch.json
Vendored
+76
View File
@@ -0,0 +1,76 @@
{
"articles": [
{
"id": 101,
"quality": 8.5,
"fit": 7.0,
"category": "Tech & Engineering",
"rationale": "First-hand 40TB Postgres migration with failure timeline, numbers and a rollback plan",
"paywalled_guess": false,
"facets": {
"format": "first_hand_account",
"depth": "deep",
"evidence": "first_hand",
"commerciality": "none",
"topic_group": "software_engineering",
"technicality": "advanced",
"locality": "not_applicable",
"specific_topics": ["Postgres migration", "storage failover", "rollback planning"]
}
},
{
"id": 102,
"quality": 3.0,
"fit": 2.5,
"category": "AI & Machine Learning",
"rationale": "Model release announcement, benchmark table from the vendor, no independent evaluation",
"paywalled_guess": false,
"facets": {
"format": "announcement_roundup",
"depth": "brief",
"evidence": "speculative",
"commerciality": "promotional",
"topic_group": "ai_ml",
"technicality": "light",
"locality": "international",
"specific_topics": ["model launch"]
}
},
{
"id": 103,
"quality": 6.5,
"fit": 8.0,
"category": "Boston & Local",
"rationale": "MBTA slow-zone data pulled from the tracker and charted by line with original analysis",
"paywalled_guess": false,
"facets": {
"format": "analysis_essay",
"depth": "standard",
"evidence": "data_or_experiment",
"commerciality": "none",
"topic_group": "boston_new_england",
"technicality": "intermediate",
"locality": "boston_new_england",
"specific_topics": ["MBTA slow zones", "transit data"]
}
},
{
"id": 104,
"quality": 5.0,
"fit": 6.0,
"category": "Culture & Essays",
"rationale": "Promising essay on typesetting history; the body reads cut off after the second section",
"paywalled_guess": true,
"facets": {
"format": "analysis_essay",
"depth": "standard",
"evidence": "synthesis",
"commerciality": "none",
"topic_group": "books_writing",
"technicality": "nontechnical",
"locality": "not_applicable",
"specific_topics": ["typesetting", "print history"]
}
}
]
}
tests/fixtures/deepseek_deep_batch_messy.json
Vendored
+67
View File
@@ -0,0 +1,67 @@
{
"articles": [
{
"id": 201,
"quality": 7.0,
"fit": 6.5,
"category": "Science & Space",
"rationale": "careful write-up of an amateur radio occultation measurement",
"paywalled_guess": false,
"facets": {
"format": "first_hand_account",
"depth": "standard",
"evidence": "data_or_experiment",
"commerciality": "none",
"topic_group": "science_space",
"technicality": "intermediate",
"locality": "us",
"specific_topics": ["radio occultation"]
}
},
{
"id": "202",
"quality": "6",
"fit": "5.5",
"category": "Niche Corner",
"rationale": "mailing-list argument about tape drives, oddly gripping",
"paywalled_guess": "false",
"facets": {
"format": "discussion_thread",
"depth": "standard",
"evidence": "anecdote",
"commerciality": "none",
"topic_group": "retro_computing",
"technicality": "intermediate",
"locality": "not_applicable",
"specific_topics": ["LTO tape", "backups", "archival", "vendors", "pricing"]
}
},
{
"id": 203,
"quality": 4,
"fit": 4
},
{
"id": 204,
"quality": 12.5,
"fit": -1,
"category": "Sports",
"rationale": "model ignored the rubric ceiling and invented a section here",
"paywalled_guess": false,
"facets": "not an object"
},
{
"id": 205,
"quality": 8.0,
"category": "Top Stories",
"rationale": "fit is missing, so this item is unusable"
},
{
"quality": 9.0,
"fit": 9.0,
"category": "Top Stories",
"rationale": "no id at all, unusable"
},
"a bare string where an object belongs"
]
}
tests/fixtures/deepseek_score_batch.json
Vendored
-32
View File
@@ -1,32 +0,0 @@
{
"articles": [
{
"id": 101,
"score": 8.5,
"category": "Tech & Engineering",
"rationale": "first-hand 40TB Postgres migration with numbers, failures and rollback plan",
"is_paywalled_guess": false
},
{
"id": 102,
"score": 3.0,
"category": "AI & Machine Learning",
"rationale": "model release announcement, no independent evaluation",
"is_paywalled_guess": false
},
{
"id": 103,
"score": 6.5,
"category": "Boston & Local",
"rationale": "MBTA slow-zone data analysis with original charts",
"is_paywalled_guess": false
},
{
"id": 104,
"score": 5.0,
"category": "Culture & Essays",
"rationale": "promising essay on typesetting, body appears truncated",
"is_paywalled_guess": true
}
]
}
tests/fixtures/deepseek_score_batch_messy.json
Vendored
-35
View File
@@ -1,35 +0,0 @@
{
"articles": [
{
"id": 201,
"score": 7.0,
"category": "Science & Space",
"rationale": "careful write-up of an amateur radio occultation measurement",
"is_paywalled_guess": false
},
{
"id": "202",
"score": "6",
"category": "Niche Corner",
"rationale": "mailing-list argument about tape drives, oddly gripping",
"is_paywalled_guess": "false"
},
{
"id": 203,
"score": 4
},
{
"id": 204,
"score": 12.5,
"category": "Top Stories",
"rationale": "model ignored the rubric ceiling here",
"is_paywalled_guess": false
},
{
"score": 9.0,
"category": "Top Stories",
"rationale": "no id at all, unusable"
},
"a bare string where an object belongs"
]
}
tests/m3_curation.rs
+54 -38
View File
@@ -3,8 +3,8 @@
//! The stage logic is unit-tested inside `src/curate/*`. What this file guards is
//! the contract *between* the curation stages and everything around them:
//!
//! * the recorded DeepSeek fixtures still parse through the real
//! `score.rs` / `select.rs` / `editorial.rs` parsers into the structures the
//! * recorded DeepSeek fixtures still parse through the real
//! `assess.rs` / `editor.rs` / `editorial.rs` parsers into the structures the
//! pipeline consumes, and the lenient parsers still cope with the messy one;
//! * the shipped Scour OPML still yields the ~220 interests the taste profile is
//! assembled from (§3.6a);
@@ -17,11 +17,10 @@ use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::process::Command;
use daily_epub::curate::assess::parse_deep_response;
use daily_epub::curate::editor::parse_selection_response;
use daily_epub::curate::editorial::BriefResponse;
use daily_epub::curate::profile;
use daily_epub::curate::score::parse_score_response;
use daily_epub::curate::select::parse_selection_response;
use daily_epub::types::LlmScore;
fn repo(rel: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join(rel)
@@ -32,65 +31,82 @@ fn fixture(name: &str) -> String {
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
}
/// Stage A responses must parse into `{id, score, category, rationale,
/// is_paywalled_guess}` per article (§3.6).
/// Deep responses must parse into `{id, quality, fit, category, rationale,
/// paywalled_guess, facets}` per article (§12.1).
#[test]
fn stage_a_fixture_parses_into_scores() {
let items = parse_score_response(&fixture("deepseek_score_batch.json"));
fn deep_fixture_parses_into_assessments() {
let sections = daily_epub::config::CurationConfig::default().sections;
let items = parse_deep_response(&fixture("deepseek_deep_batch.json"), &sections);
assert!(items.len() >= 4, "fixture should cover a realistic batch");
for item in &items {
assert!(item.id > 0, "every item carries a positive article id");
assert!((0.0..=10.0).contains(&item.quality));
assert!((0.0..=10.0).contains(&item.fit));
assert!(item.category.is_some(), "every category is on the palette");
assert!(
(0.0..=10.0).contains(&item.score),
"score {} out of range",
item.score
);
assert!(!item.category.is_empty());
assert!(
item.rationale.split_whitespace().count() <= 20,
"rationale must stay under 20 words: {:?}",
item.rationale.split_whitespace().count() <= 25,
"rationale must stay under 25 words: {:?}",
item.rationale
);
assert!(item.facets.format.is_some() && item.facets.topic_group.is_some());
}
assert!(
items.iter().any(|i| i.is_paywalled_guess),
items.iter().any(|item| item.paywalled_guess),
"the fixture should exercise the paywall flag"
);
// Quality and fit are judged separately: the fixture has an article whose
// fit exceeds its quality and one the other way round.
assert!(items.iter().any(|item| item.fit > item.quality));
assert!(items.iter().any(|item| item.quality > item.fit));
// The batch spans the rubric rather than clustering at one score.
let scores: Vec<f64> = items.iter().map(|i| i.score).collect();
let spread = scores.iter().cloned().fold(f64::MIN, f64::max)
- scores.iter().cloned().fold(f64::MAX, f64::min);
let qualities: Vec<f64> = items.iter().map(|i| i.quality).collect();
let spread = qualities.iter().cloned().fold(f64::MIN, f64::max)
- qualities.iter().cloned().fold(f64::MAX, f64::min);
assert!(spread >= 3.0, "fixture scores are too uniform to be useful");
// Every item converts into the shared curation type.
let converted: Vec<LlmScore> = items.into_iter().map(LlmScore::from).collect();
assert!(converted.iter().all(|s| (0.0..=10.0).contains(&s.score)));
}
/// The messy fixture must stay messy: it is what proves the parser is lenient
/// (string ids, string scores, out-of-range scores, junk entries).
/// (string ids and scores, out-of-range scores, unknown facet tokens, junk).
#[test]
fn stage_a_messy_fixture_is_salvaged_not_rejected() {
let raw = fixture("deepseek_score_batch_messy.json");
// The hard cases are still present in the recording…
fn deep_messy_fixture_is_salvaged_not_rejected() {
let sections = daily_epub::config::CurationConfig::default().sections;
let raw = fixture("deepseek_deep_batch_messy.json");
assert!(raw.contains("\"id\": \""), "needs a string id");
assert!(raw.contains("\"score\": \""), "needs a string score");
assert!(raw.contains("\"quality\": \""), "needs a string score");
assert!(
raw.contains("discussion_thread"),
"needs an unknown facet token"
);
// …and the real parser copes with all of them.
let items = parse_score_response(&raw);
let items = parse_deep_response(&raw, &sections);
assert!(!items.is_empty(), "the parser salvaged nothing");
assert!(
items.iter().all(|i| (0.0..=10.0).contains(&i.score)),
"out-of-range scores must be clamped: {:?}",
items.iter().map(|i| i.score).collect::<Vec<_>>()
items
.iter()
.all(|i| (0.0..=10.0).contains(&i.quality) && (0.0..=10.0).contains(&i.fit)),
"out-of-range scores must be clamped"
);
assert!(items.iter().all(|i| i.id > 0), "id-less items are skipped");
let messy = items
.iter()
.find(|i| i.id == 202)
.expect("string-valued item");
assert!(
messy.facets.format.is_none(),
"unknown facet tokens become None"
);
assert_eq!(messy.facets.depth.as_deref(), Some("standard"));
let off_palette = items.iter().find(|i| i.id == 204).expect("clamped item");
assert!(
off_palette.category.is_none(),
"invented sections become None"
);
// A response that is not JSON at all degrades to "no scores", never a panic.
assert!(parse_score_response("I'm sorry, I can't do that.").is_empty());
assert!(parse_score_response("").is_empty());
// A response that is not JSON at all degrades to "no assessments", never a panic.
assert!(parse_deep_response("I'm sorry, I can't do that.", &sections).is_empty());
assert!(parse_deep_response("", &sections).is_empty());
}
/// Stage B responses must carry `{id, section, position, lead_story}` with