Curation v2 step 5: deep assessment, utility, diversified shortlist
assess.rs replaces score.rs (DEEP_INSTRUCTIONS, representative sample with [BEGINNING]/[MIDDLE]/[END], facets, cached deep rows with --rescore bypass), rank.rs adds the utility blend over present signals with gate ramps and the leader-clustered shortlist (cap 2 → 3 → uncapped, protected top-N, exploration reserve), editor.rs replaces select.rs with the §13 rendering and utility-ordered fallbacks. ScoredArticle is gone; Candidate is the only flow type. deep_batch_size replaces score_batch_size. Started by Codex (cut off by its usage limit mid-verification) and finished by a Claude agent from docs/plans/curation-v2-briefs/step5.md; reviewed against plan §12–§13. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
This commit is contained in:
Claude Fable 5.1
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+13
-6
@@ -195,22 +195,28 @@ pub fn build_brief_prompt(lineup: &Lineup, summaries: &BTreeMap<ArticleId, Strin
|
||||
for section in &lineup.section_order {
|
||||
let _ = writeln!(prompt, "\n## {section}");
|
||||
for pick in lineup.section_picks(section) {
|
||||
let score = pick
|
||||
let quality = pick
|
||||
.llm
|
||||
.as_ref()
|
||||
.map(|score| format!("{:.1}", score.score))
|
||||
.unwrap_or_else(|| "unscored".into());
|
||||
.map(|assessment| format!("{:.1}", assessment.quality))
|
||||
.unwrap_or_else(|| "unassessed".into());
|
||||
let fit = pick
|
||||
.llm
|
||||
.as_ref()
|
||||
.map(|assessment| format!("{:.1}", assessment.fit))
|
||||
.unwrap_or_else(|| "unassessed".into());
|
||||
let summary = summaries
|
||||
.get(&pick.article.id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| excerpt_summary(pick));
|
||||
let _ = writeln!(
|
||||
prompt,
|
||||
"- {}\n feed: {}\n why: {}\n score: {}\n summary: {}",
|
||||
"- {}\n feed: {}\n why: {}\n quality: {}\n fit: {}\n summary: {}",
|
||||
pick.article.title.trim(),
|
||||
pick.article.feed_title.trim(),
|
||||
pick.why.as_deref().unwrap_or("not supplied"),
|
||||
score,
|
||||
quality,
|
||||
fit,
|
||||
summary
|
||||
);
|
||||
}
|
||||
@@ -462,7 +468,8 @@ mod tests {
|
||||
assert!(prompt.contains("- Migrating 40TB off Postgres"));
|
||||
assert!(prompt.contains("why: the Migrating 40TB off Postgres piece you'd argue with"));
|
||||
assert!(prompt.contains("summary: A migration story with numbers."));
|
||||
assert!(prompt.contains("score: unscored"));
|
||||
assert!(prompt.contains("quality: unassessed"));
|
||||
assert!(prompt.contains("fit: unassessed"));
|
||||
assert!(prompt.contains("2026-08-15"));
|
||||
assert!(
|
||||
!prompt.contains("section_intros"),
|
||||
|
||||
+39
-30
@@ -1,7 +1,7 @@
|
||||
//! Personalized curation: signals → triage → admission → assessment → editor.
|
||||
//!
|
||||
//! ```text
|
||||
//! ~400 eligible ─triage─▶ union admission (120) ─stage A─▶ editor ─▶ editorial
|
||||
//! ~400 eligible ─triage─▶ admission (120) ─assess/rank─▶ shortlist (60) ─editor─▶ editorial
|
||||
//! ```
|
||||
//!
|
||||
//! [`Curator`] is the thin orchestration layer the `generate` pipeline calls; the
|
||||
@@ -11,24 +11,25 @@
|
||||
//! client; selection and editorial on the editor with per-call bulk fallback.
|
||||
|
||||
pub mod admit;
|
||||
pub mod assess;
|
||||
pub mod editor;
|
||||
pub mod editorial;
|
||||
pub mod embedding;
|
||||
pub mod llm;
|
||||
pub mod prefilter;
|
||||
pub mod profile;
|
||||
pub mod score;
|
||||
pub mod select;
|
||||
pub mod rank;
|
||||
pub mod signals;
|
||||
pub mod telemetry;
|
||||
pub mod triage;
|
||||
|
||||
use jiff::civil::Date;
|
||||
use jiff::{Timestamp, civil::Date};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::Db;
|
||||
use crate::types::{Editorial, Lineup, ScoredArticle};
|
||||
use crate::types::{Candidate, Editorial, Lineup};
|
||||
|
||||
/// Runs the three curation stages against one day's articles (§3.5, §3.6).
|
||||
/// Runs the LLM curation stages against one day's articles (§12, §13, §14).
|
||||
pub struct Curator {
|
||||
pub config: Config,
|
||||
pub db: Db,
|
||||
@@ -43,43 +44,51 @@ impl Curator {
|
||||
Self { config, db, llms }
|
||||
}
|
||||
|
||||
/// Stage A: batched LLM scoring of the surviving candidates (§3.6).
|
||||
/// Deep assessment of the admitted set on the bulk client (§12.1), reusing
|
||||
/// cached `article_assessments` rows within `assessment_reuse_days`.
|
||||
///
|
||||
/// A no-op under `--skip-llm`. Scores are persisted per `(article, date)`.
|
||||
pub async fn score(&self, candidates: &mut [ScoredArticle], _date: Date) -> anyhow::Result<()> {
|
||||
let Some(llm) = self.llms.bulk.as_ref() else {
|
||||
tracing::info!("--skip-llm: stage A scoring skipped");
|
||||
return Ok(());
|
||||
/// A no-op under `--skip-llm`: like triage, nothing is read or written and
|
||||
/// utility falls back to the present signals (§12.3, §17). When the bulk
|
||||
/// provider is down or its budget trips, cached rows are still reused and
|
||||
/// the failed batches simply stay unassessed.
|
||||
pub async fn assess(
|
||||
&self,
|
||||
candidates: &mut [Candidate],
|
||||
rescore: bool,
|
||||
profile_version: Option<i64>,
|
||||
assessed_at: Timestamp,
|
||||
) -> anyhow::Result<usize> {
|
||||
let Some(bulk) = self.llms.bulk.as_ref() else {
|
||||
tracing::info!("--skip-llm: deep assessment skipped");
|
||||
return Ok(0);
|
||||
};
|
||||
let span = tracing::info_span!("llm_score", candidates = candidates.len());
|
||||
let span = tracing::info_span!("llm_assess", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let scored = score::score_all(
|
||||
llm,
|
||||
assess::run(
|
||||
&self.db,
|
||||
Some(bulk),
|
||||
&self.config.deepseek.model,
|
||||
candidates,
|
||||
self.config.deepseek.score_batch_size,
|
||||
self.config.deepseek.deep_batch_size,
|
||||
self.config.deepseek.max_concurrent_requests,
|
||||
&self.config.curation.sections,
|
||||
self.config.curation.ranking.assessment_reuse_days,
|
||||
rescore,
|
||||
profile_version,
|
||||
assessed_at,
|
||||
self.config.deepseek.score_temperature,
|
||||
&self.config.curation.sections,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(scored, total = candidates.len(), "stage A complete");
|
||||
|
||||
Ok(())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stage B: single-call lineup selection into sections (§3.6).
|
||||
pub async fn select(
|
||||
&self,
|
||||
candidates: Vec<ScoredArticle>,
|
||||
date: Date,
|
||||
) -> anyhow::Result<Lineup> {
|
||||
/// The editor: one call that assembles the issue from the shortlist (§13).
|
||||
pub async fn select(&self, candidates: Vec<Candidate>, date: Date) -> anyhow::Result<Lineup> {
|
||||
let sections = &self.config.curation.sections;
|
||||
let soft_target = self.config.target_article_count;
|
||||
let hard_max = self.config.curation.max_article_count;
|
||||
let span = tracing::info_span!("llm_editor", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
match select::select(
|
||||
match editor::select(
|
||||
&self.llms,
|
||||
candidates.clone(),
|
||||
sections,
|
||||
@@ -92,7 +101,7 @@ impl Curator {
|
||||
Ok(lineup) => Ok(lineup),
|
||||
Err(error) => {
|
||||
tracing::error!(%error, "editor and bulk fallback failed; selecting heuristically");
|
||||
Ok(select::select_without_llm(
|
||||
Ok(editor::select_without_llm(
|
||||
candidates,
|
||||
sections,
|
||||
soft_target,
|
||||
|
||||
+1
-14
@@ -125,9 +125,7 @@ pub fn roundup_penalty(title: &str) -> f64 {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::types::{
|
||||
ArticleId, ExtractMethod, FeedId, SocialRef, SocialSource, SourceKind, SourceRef,
|
||||
};
|
||||
use crate::types::{ArticleId, ExtractMethod, SocialRef, SocialSource, SourceKind, SourceRef};
|
||||
use jiff::Timestamp;
|
||||
|
||||
pub(crate) fn ts() -> Timestamp {
|
||||
@@ -180,17 +178,6 @@ pub(crate) mod tests {
|
||||
article
|
||||
}
|
||||
|
||||
pub(crate) fn via(mut article: Article, kind: SourceKind, feed_id: FeedId) -> Article {
|
||||
article.sources.push(SourceRef {
|
||||
entry_id: article.best_entry_id,
|
||||
feed_id,
|
||||
feed_title: format!("{kind:?} feed"),
|
||||
category: None,
|
||||
kind,
|
||||
});
|
||||
article
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_heuristic_has_only_text_terms() {
|
||||
let quiet = article(1, "An essay", 1200);
|
||||
|
||||
@@ -0,0 +1,586 @@
|
||||
//! Utility normalization, leader clustering and diversified shortlisting (§12.2–§12.5).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::config::{RankingConfig, UtilityWeights};
|
||||
use crate::curate::embedding::dot;
|
||||
use crate::curate::signals;
|
||||
use crate::types::{ArticleId, Candidate};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct RankSummary {
|
||||
pub shortlisted: usize,
|
||||
pub clusters: usize,
|
||||
}
|
||||
|
||||
/// Re-normalize cheap signals over the deep set and calculate utility on 0–100.
|
||||
pub fn calculate_utility(candidates: &mut [Candidate], configured: &UtilityWeights) {
|
||||
let indices = (0..candidates.len()).collect::<Vec<_>>();
|
||||
calculate_utility_for(candidates, &indices, configured);
|
||||
}
|
||||
|
||||
fn calculate_utility_for(
|
||||
candidates: &mut [Candidate],
|
||||
indices: &[usize],
|
||||
configured: &UtilityWeights,
|
||||
) {
|
||||
let mut normalized = indices
|
||||
.iter()
|
||||
.map(|index| candidates[*index].signals.clone())
|
||||
.collect::<Vec<_>>();
|
||||
for signals in &mut normalized {
|
||||
signals.norm.clear();
|
||||
signals.weights.clear();
|
||||
}
|
||||
let mut signal_refs = normalized.iter_mut().collect::<Vec<_>>();
|
||||
signals::normalize(&mut signal_refs);
|
||||
for (index, signals) in indices.iter().zip(normalized) {
|
||||
candidates[*index].signals.norm = signals.norm;
|
||||
candidates[*index].signals.weights.clear();
|
||||
}
|
||||
|
||||
for index in indices {
|
||||
let candidate = &mut candidates[*index];
|
||||
if let Some(triage) = &candidate.assessment.triage {
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("triage".into(), (triage.interest / 10.0).clamp(0.0, 1.0));
|
||||
}
|
||||
if let Some(deep) = &candidate.assessment.deep {
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("quality".into(), (deep.quality / 10.0).clamp(0.0, 1.0));
|
||||
candidate
|
||||
.signals
|
||||
.norm
|
||||
.insert("fit".into(), (deep.fit / 10.0).clamp(0.0, 1.0));
|
||||
}
|
||||
let weighted = [
|
||||
("quality", configured.quality, 1.0),
|
||||
("fit", configured.fit, 1.0),
|
||||
("knn", configured.knn, candidate.signals.knn_gate),
|
||||
("interest", configured.interest, 1.0),
|
||||
("feed", configured.feed, candidate.signals.feed_gate),
|
||||
("triage", configured.triage, 1.0),
|
||||
("social", configured.social, 1.0),
|
||||
("heuristic", configured.heuristic, 1.0),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(name, weight, gate)| {
|
||||
let value = candidate.signals.norm.get(name).copied()?;
|
||||
let effective = weight * gate;
|
||||
(effective > 0.0).then_some((name, effective, value))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let total = weighted.iter().map(|(_, weight, _)| weight).sum::<f64>();
|
||||
if total <= 0.0 {
|
||||
candidate.utility = None;
|
||||
continue;
|
||||
}
|
||||
candidate.signals.weights = weighted
|
||||
.iter()
|
||||
.map(|(name, weight, _)| ((*name).to_string(), weight / total))
|
||||
.collect();
|
||||
candidate.utility = Some(
|
||||
weighted
|
||||
.iter()
|
||||
.map(|(_, weight, value)| weight / total * value)
|
||||
.sum::<f64>()
|
||||
* 100.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn ranked_indices(candidates: &[Candidate], indices: &[usize]) -> Vec<usize> {
|
||||
let mut sorted = indices.to_vec();
|
||||
sorted.sort_by(|left, right| {
|
||||
candidates[*right]
|
||||
.utility
|
||||
.unwrap_or(f64::NEG_INFINITY)
|
||||
.total_cmp(&candidates[*left].utility.unwrap_or(f64::NEG_INFINITY))
|
||||
.then_with(|| {
|
||||
candidates[*left]
|
||||
.article
|
||||
.id
|
||||
.cmp(&candidates[*right].article.id)
|
||||
})
|
||||
});
|
||||
sorted
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Cluster {
|
||||
id: i64,
|
||||
leader: usize,
|
||||
}
|
||||
|
||||
/// Rank the admitted deep set and leave only the diversified shortlist at
|
||||
/// `stage = shortlisted`. All deep-set articles receive ranks and clusters.
|
||||
pub fn shortlist(
|
||||
candidates: &mut [Candidate],
|
||||
embeddings: &HashMap<ArticleId, Vec<f32>>,
|
||||
ranking: &RankingConfig,
|
||||
) -> RankSummary {
|
||||
let deep = candidates
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, candidate)| matches!(candidate.stage.as_str(), "admitted" | "assessed"))
|
||||
.map(|(index, _)| index)
|
||||
.collect::<Vec<_>>();
|
||||
calculate_utility_for(candidates, &deep, &ranking.weights.utility);
|
||||
let sorted = ranked_indices(candidates, &deep);
|
||||
for (rank, index) in sorted.iter().enumerate() {
|
||||
candidates[*index].rank_utility = Some(rank as i64 + 1);
|
||||
candidates[*index].cluster = None;
|
||||
candidates[*index].cluster_rank = None;
|
||||
}
|
||||
|
||||
let mut clusters = Vec::<Cluster>::new();
|
||||
let mut members: HashMap<i64, Vec<usize>> = HashMap::new();
|
||||
for index in &sorted {
|
||||
let assigned = embeddings
|
||||
.get(&candidates[*index].article.id)
|
||||
.and_then(|vector| {
|
||||
clusters.iter().find_map(|cluster| {
|
||||
let leader_id = candidates[cluster.leader].article.id;
|
||||
let leader = embeddings.get(&leader_id)?;
|
||||
dot(vector, leader)
|
||||
.ok()
|
||||
.filter(|cosine| *cosine >= ranking.diversity.cluster_threshold)
|
||||
.map(|_| cluster.id)
|
||||
})
|
||||
});
|
||||
let cluster_id = assigned.unwrap_or_else(|| {
|
||||
let id = clusters.len() as i64 + 1;
|
||||
clusters.push(Cluster { id, leader: *index });
|
||||
id
|
||||
});
|
||||
let cluster_members = members.entry(cluster_id).or_default();
|
||||
cluster_members.push(*index);
|
||||
candidates[*index].cluster = Some(cluster_id);
|
||||
candidates[*index].cluster_rank = Some(cluster_members.len() as i64);
|
||||
}
|
||||
|
||||
let protected = sorted
|
||||
.iter()
|
||||
.take(ranking.diversity.utility_protected)
|
||||
.copied()
|
||||
.collect::<HashSet<_>>();
|
||||
let mut admitted = HashSet::new();
|
||||
let mut admitted_per_cluster = HashMap::<i64, usize>::new();
|
||||
let admit = |index: usize, admitted: &mut HashSet<usize>, counts: &mut HashMap<i64, usize>| {
|
||||
if admitted.insert(index)
|
||||
&& let Some(cluster) = candidates[index].cluster
|
||||
{
|
||||
*counts.entry(cluster).or_default() += 1;
|
||||
}
|
||||
};
|
||||
|
||||
for index in &sorted {
|
||||
if protected.contains(index) || candidates[*index].auto_include {
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
}
|
||||
for index in sorted
|
||||
.iter()
|
||||
.filter(|index| candidates[**index].exploration)
|
||||
.take(3)
|
||||
{
|
||||
if admitted.len() >= ranking.shortlist_keep {
|
||||
break;
|
||||
}
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
|
||||
let target = ranking.shortlist_keep.max(admitted.len());
|
||||
let mut suppressed_at_base_cap = HashSet::new();
|
||||
admit_under_cap(
|
||||
candidates,
|
||||
&sorted,
|
||||
target,
|
||||
ranking.diversity.per_cluster_cap,
|
||||
&mut admitted,
|
||||
&mut admitted_per_cluster,
|
||||
Some(&mut suppressed_at_base_cap),
|
||||
);
|
||||
if admitted.len() < target {
|
||||
admit_under_cap(
|
||||
candidates,
|
||||
&sorted,
|
||||
target,
|
||||
3,
|
||||
&mut admitted,
|
||||
&mut admitted_per_cluster,
|
||||
None,
|
||||
);
|
||||
}
|
||||
if admitted.len() < target {
|
||||
for index in &sorted {
|
||||
if admitted.len() >= target {
|
||||
break;
|
||||
}
|
||||
admit(*index, &mut admitted, &mut admitted_per_cluster);
|
||||
}
|
||||
}
|
||||
|
||||
for index in deep {
|
||||
if admitted.contains(&index) {
|
||||
candidates[index].stage = "shortlisted".into();
|
||||
candidates[index].excluded_reason = None;
|
||||
} else {
|
||||
// The stage stays where the article stopped (`admitted` when the
|
||||
// deep assessment never happened, else `assessed`).
|
||||
candidates[index].excluded_reason = Some(
|
||||
if suppressed_at_base_cap.contains(&index) {
|
||||
"cluster_suppressed"
|
||||
} else {
|
||||
"shortlist_cap"
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
RankSummary {
|
||||
shortlisted: admitted.len(),
|
||||
clusters: clusters.len(),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn admit_under_cap(
|
||||
candidates: &[Candidate],
|
||||
sorted: &[usize],
|
||||
target: usize,
|
||||
cap: usize,
|
||||
admitted: &mut HashSet<usize>,
|
||||
admitted_per_cluster: &mut HashMap<i64, usize>,
|
||||
mut suppressed: Option<&mut HashSet<usize>>,
|
||||
) {
|
||||
for index in sorted {
|
||||
if admitted.len() >= target || admitted.contains(index) {
|
||||
continue;
|
||||
}
|
||||
let Some(cluster) = candidates[*index].cluster else {
|
||||
continue;
|
||||
};
|
||||
if admitted_per_cluster.get(&cluster).copied().unwrap_or(0) >= cap {
|
||||
if let Some(suppressed) = suppressed.as_deref_mut() {
|
||||
suppressed.insert(*index);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
admitted.insert(*index);
|
||||
*admitted_per_cluster.entry(cluster).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DiversityConfig;
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::curate::signals::Signals;
|
||||
use crate::types::{Deep, Facets};
|
||||
|
||||
fn candidate(id: i64, utility_hint: f64) -> Candidate {
|
||||
let mut candidate = Candidate::new(article(id, &format!("article {id}"), 800), false);
|
||||
candidate.stage = "assessed".into();
|
||||
candidate.signals = Signals {
|
||||
heuristic: Some(utility_hint),
|
||||
..Signals::default()
|
||||
};
|
||||
candidate.assessment.deep = Some(Deep {
|
||||
quality: utility_hint,
|
||||
fit: utility_hint,
|
||||
category: Some("Top Stories".into()),
|
||||
rationale: "specific".into(),
|
||||
paywalled_guess: false,
|
||||
facets: Facets::default(),
|
||||
model: "mock".into(),
|
||||
prompt_version: 1,
|
||||
assessed_at: "2026-09-02T00:00:00Z".parse().expect("timestamp"),
|
||||
});
|
||||
candidate
|
||||
}
|
||||
|
||||
fn vector(angle: f32) -> Vec<f32> {
|
||||
vec![angle.cos(), angle.sin()]
|
||||
}
|
||||
|
||||
fn ranking(keep: usize, protected: usize, cap: usize, threshold: f64) -> RankingConfig {
|
||||
RankingConfig {
|
||||
shortlist_keep: keep,
|
||||
diversity: DiversityConfig {
|
||||
cluster_threshold: threshold,
|
||||
per_cluster_cap: cap,
|
||||
utility_protected: protected,
|
||||
},
|
||||
..RankingConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_renormalizes_present_signals_and_gates_learned_ones() {
|
||||
let mut a = candidate(1, 8.0);
|
||||
let mut b = candidate(2, 4.0);
|
||||
a.signals.interest = Some(1.0);
|
||||
b.signals.interest = None;
|
||||
a.signals.knn = Some(0.9);
|
||||
a.signals.knn_gate = 0.5;
|
||||
let mut values = vec![a, b];
|
||||
calculate_utility(&mut values, &UtilityWeights::default());
|
||||
let [a, b] = values.as_slice() else {
|
||||
panic!("two values")
|
||||
};
|
||||
for candidate in [&a, &b] {
|
||||
assert!((candidate.signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
assert!(candidate.utility.is_some());
|
||||
}
|
||||
assert!(!b.signals.weights.contains_key("interest"));
|
||||
assert!(
|
||||
(a.signals.weights["knn"] / a.signals.weights["quality"] - (0.15 * 0.5) / 0.40).abs()
|
||||
< 1e-9
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicates_cluster_and_the_third_is_suppressed() {
|
||||
let ranking = ranking(3, 0, 2, 0.85);
|
||||
let mut candidates = vec![
|
||||
candidate(1, 9.0),
|
||||
candidate(2, 8.0),
|
||||
candidate(3, 7.0),
|
||||
candidate(4, 6.0),
|
||||
];
|
||||
let embeddings = HashMap::from([
|
||||
(1, vector(0.0)),
|
||||
(2, vector(0.1)),
|
||||
(3, vector(0.2)),
|
||||
(4, vector(2.0)),
|
||||
]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.shortlisted, 3);
|
||||
assert_eq!(candidates[0].cluster, candidates[1].cluster);
|
||||
assert_eq!(candidates[1].cluster, candidates[2].cluster);
|
||||
assert_eq!(
|
||||
candidates[2].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protected_items_survive_and_count_toward_the_cap() {
|
||||
let ranking = ranking(3, 2, 1, 0.85);
|
||||
let mut candidates = vec![
|
||||
candidate(1, 9.0),
|
||||
candidate(2, 8.0),
|
||||
candidate(3, 7.0),
|
||||
candidate(4, 6.0),
|
||||
];
|
||||
let embeddings = HashMap::from([
|
||||
(1, vector(0.0)),
|
||||
(2, vector(0.05)),
|
||||
(3, vector(0.1)),
|
||||
(4, vector(2.0)),
|
||||
]);
|
||||
shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(candidates[0].stage, "shortlisted");
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
assert_ne!(candidates[2].stage, "shortlisted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_case_uses_leaders_not_transitive_components() {
|
||||
let ranking = ranking(3, 0, 2, 0.80);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
// A at 0°, B at 72°, C at 36°: A~C and B~C, but A not~B.
|
||||
let embeddings =
|
||||
HashMap::from([(1, vector(0.0)), (2, vector(1.2566)), (3, vector(0.6283))]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.clusters, 2);
|
||||
assert_eq!(candidates[0].cluster, candidates[2].cluster);
|
||||
assert_ne!(candidates[0].cluster, candidates[1].cluster);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_embeddings_are_singletons_and_never_suppressed() {
|
||||
let ranking = ranking(3, 0, 1, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert!(
|
||||
candidates
|
||||
.iter()
|
||||
.all(|candidate| candidate.stage == "shortlisted")
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.filter_map(|candidate| candidate.cluster)
|
||||
.collect::<HashSet<_>>()
|
||||
.len(),
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_relax_to_three_then_uncapped_when_the_shortlist_is_short() {
|
||||
// Six near-duplicates, keep 5: cap 2 admits two, cap 3 admits a third,
|
||||
// and the uncapped pass fills the remaining two slots in utility order.
|
||||
let ranking = ranking(5, 0, 2, 0.85);
|
||||
let mut candidates = (1..=6)
|
||||
.map(|id| candidate(id, 10.0 - id as f64))
|
||||
.collect::<Vec<_>>();
|
||||
let embeddings = (1..=6)
|
||||
.map(|id| (id, vector(0.01 * id as f32)))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.clusters, 1);
|
||||
assert_eq!(summary.shortlisted, 5);
|
||||
let shortlisted = candidates
|
||||
.iter()
|
||||
.filter(|candidate| candidate.stage == "shortlisted")
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(shortlisted, vec![1, 2, 3, 4, 5], "filled in utility order");
|
||||
assert_eq!(candidates[5].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[5].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.map(|c| c.rank_utility)
|
||||
.collect::<Vec<_>>(),
|
||||
(1..=6).map(Some).collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(
|
||||
candidates
|
||||
.iter()
|
||||
.map(|c| c.cluster_rank)
|
||||
.collect::<Vec<_>>(),
|
||||
(1..=6).map(Some).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shortlist_cap_is_the_reason_beyond_the_keep() {
|
||||
let ranking = ranking(2, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 7.0)];
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(candidates[0].stage, "shortlisted");
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
assert_eq!(candidates[2].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[2].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exploration_picks_get_up_to_three_reserved_slots() {
|
||||
// Keep 4 with the four best by utility being ordinary articles: three
|
||||
// exploration picks are still reserved seats, the fourth is not.
|
||||
let ranking = ranking(4, 0, 2, 0.85);
|
||||
let mut candidates = (1..=8)
|
||||
.map(|id| candidate(id, 10.0 - id as f64))
|
||||
.collect::<Vec<_>>();
|
||||
for candidate in candidates.iter_mut().skip(4) {
|
||||
candidate.exploration = true;
|
||||
}
|
||||
let summary = shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(summary.shortlisted, 4);
|
||||
let shortlisted = candidates
|
||||
.iter()
|
||||
.filter(|candidate| candidate.stage == "shortlisted")
|
||||
.map(|candidate| candidate.article.id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(shortlisted, vec![1, 5, 6, 7]);
|
||||
assert_eq!(
|
||||
candidates[7].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_includes_are_admitted_regardless_and_count_toward_their_cluster() {
|
||||
let ranking = ranking(2, 0, 1, 0.85);
|
||||
let mut candidates = vec![candidate(1, 9.0), candidate(2, 8.0), candidate(3, 1.0)];
|
||||
candidates[2].auto_include = true;
|
||||
let embeddings = HashMap::from([(1, vector(0.0)), (2, vector(2.0)), (3, vector(0.05))]);
|
||||
let summary = shortlist(&mut candidates, &embeddings, &ranking);
|
||||
assert_eq!(summary.shortlisted, 2);
|
||||
assert_eq!(candidates[2].stage, "shortlisted", "auto-include survives");
|
||||
// The auto-include filled its cluster's single seat, so the stronger
|
||||
// near-duplicate is suppressed and the unrelated article gets the slot.
|
||||
assert_eq!(candidates[0].cluster, candidates[2].cluster);
|
||||
assert_eq!(candidates[0].stage, "assessed");
|
||||
assert_eq!(
|
||||
candidates[0].excluded_reason.as_deref(),
|
||||
Some("cluster_suppressed")
|
||||
);
|
||||
assert_eq!(candidates[1].stage, "shortlisted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unassessed_articles_rank_on_present_signals_and_keep_their_stage() {
|
||||
// DeepSeek down: no quality/fit anywhere, utility comes from what is present.
|
||||
let ranking = ranking(2, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 3.0), candidate(2, 6.0), candidate(3, 9.0)];
|
||||
for candidate in &mut candidates {
|
||||
candidate.assessment.deep = None;
|
||||
candidate.stage = "admitted".into();
|
||||
}
|
||||
candidates[0].assessment.triage = Some(crate::types::Triage {
|
||||
interest: 9.0,
|
||||
kind: "essay".into(),
|
||||
why: "promising".into(),
|
||||
model: "mock".into(),
|
||||
prompt_version: 1,
|
||||
assessed_at: "2026-09-02T00:00:00Z".parse().expect("timestamp"),
|
||||
});
|
||||
let summary = shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(summary.shortlisted, 2);
|
||||
for candidate in &candidates {
|
||||
assert!(candidate.utility.is_some(), "scored on present signals");
|
||||
assert!(!candidate.signals.weights.contains_key("quality"));
|
||||
assert!(!candidate.signals.weights.contains_key("fit"));
|
||||
assert!((candidate.signals.weights.values().sum::<f64>() - 1.0).abs() < 1e-9);
|
||||
}
|
||||
// Triage (0.05) outweighs heuristic (0.02): the triaged article with
|
||||
// the weakest heuristic overtakes the middle one.
|
||||
assert_eq!(candidates[2].rank_utility, Some(1));
|
||||
assert_eq!(candidates[0].rank_utility, Some(2));
|
||||
assert_eq!(candidates[1].rank_utility, Some(3));
|
||||
assert_eq!(
|
||||
candidates[1].stage, "admitted",
|
||||
"never assessed, so not `assessed`"
|
||||
);
|
||||
assert_eq!(
|
||||
candidates[1].excluded_reason.as_deref(),
|
||||
Some("shortlist_cap")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentiles_are_taken_over_the_deep_set_only() {
|
||||
// The eligible-but-not-admitted article has the strongest heuristic;
|
||||
// it must not shift the deep set's percentiles or receive a utility.
|
||||
let ranking = ranking(10, 0, 2, 0.85);
|
||||
let mut candidates = vec![candidate(1, 5.0), candidate(2, 5.0), candidate(3, 5.0)];
|
||||
candidates[2].stage = "triaged".into();
|
||||
candidates[2].excluded_reason = Some("not_admitted".into());
|
||||
candidates[2].signals.heuristic = Some(99.0);
|
||||
candidates[1].signals.heuristic = Some(5.0);
|
||||
shortlist(&mut candidates, &HashMap::new(), &ranking);
|
||||
assert_eq!(
|
||||
candidates[0].signals.norm["heuristic"], 0.5,
|
||||
"ties share a percentile"
|
||||
);
|
||||
assert_eq!(candidates[1].signals.norm["heuristic"], 0.5);
|
||||
assert!(candidates[2].utility.is_none());
|
||||
assert!(candidates[2].rank_utility.is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,584 +0,0 @@
|
||||
//! Stage A — batched LLM scoring (spec §3.6).
|
||||
//!
|
||||
//! Batches of `deepseek.score_batch_size` articles per request. Per article we
|
||||
//! send title, source feed, author, word count, social stats, sources list and a
|
||||
//! ~200-word excerpt; the model returns one JSON object per article.
|
||||
//!
|
||||
//! Parsing is deliberately forgiving: one malformed item must not cost us the
|
||||
//! other eleven, and a failed batch must not fail the run.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use futures::{StreamExt, stream};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::llm::{LlmClient, LlmError, strip_code_fence};
|
||||
use super::{prompt_text, truncate_words};
|
||||
use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind};
|
||||
|
||||
/// Words of article text sent per candidate in stage A (§3.6).
|
||||
pub const EXCERPT_WORDS: usize = 200;
|
||||
|
||||
/// One element of the stage-A JSON response (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreItem {
|
||||
pub id: ArticleId,
|
||||
/// 0–10.
|
||||
pub score: f64,
|
||||
pub category: String,
|
||||
/// ≤ 20 words.
|
||||
#[serde(default)]
|
||||
pub rationale: String,
|
||||
#[serde(default)]
|
||||
pub is_paywalled_guess: bool,
|
||||
}
|
||||
|
||||
impl From<ScoreItem> for LlmScore {
|
||||
fn from(i: ScoreItem) -> Self {
|
||||
LlmScore {
|
||||
score: i.score,
|
||||
category: i.category,
|
||||
rationale: i.rationale,
|
||||
is_paywalled_guess: i.is_paywalled_guess,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Envelope the model is asked to return (`{"articles": [...]}`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreResponse {
|
||||
#[serde(default)]
|
||||
pub articles: Vec<ScoreItem>,
|
||||
}
|
||||
|
||||
/// The invariant instruction block for stage A. Everything article-specific goes
|
||||
/// in the per-batch tail so this prefix stays cacheable (§3.6).
|
||||
pub const SCORE_INSTRUCTIONS: &str = "\
|
||||
TASK: score a batch of candidate articles for today's issue of The Daily EPUB.
|
||||
|
||||
Judge each article against the reader profile in your system prompt — not against \
|
||||
a general audience, and not against what is objectively newsworthy.
|
||||
|
||||
Return one object per input article with these fields:
|
||||
\"id\" integer, copied exactly from the input
|
||||
\"score\" number 0-10, the rubric below
|
||||
\"category\" one short label from the palette below
|
||||
\"rationale\" at most 20 words, concrete, no hedging, no restating the title
|
||||
\"is_paywalled_guess\" true when the text looks truncated, teaser-like or paywalled
|
||||
|
||||
SCORING RUBRIC — calibrate hard; a normal day averages about 4, and a 9 should \
|
||||
appear a couple of times a week, not a couple of times a day:
|
||||
9-10 Exceptional. Original reporting, a deep technical dive, or an essay he \
|
||||
will still be thinking about next week. Evident effort and a real point of view.
|
||||
7-8 Strong. A well-made long-form piece squarely in his interests, or an \
|
||||
outstanding piece outside them.
|
||||
5-6 Worth a slot on a thin day. Solid, useful, a little thin or a little \
|
||||
familiar.
|
||||
3-4 Marginal. Competent news-of-the-day, short posts, incremental updates, \
|
||||
good writing about an over-covered story.
|
||||
1-2 Weak. Announcements, changelogs and release notes, link roundups, \
|
||||
listicles, rewrites of a story available at the source, thin AI-industry churn.
|
||||
0 Unusable. Press releases, sponsored content, engagement bait, spam, \
|
||||
pure crypto promotion, or an entry with no readable body.
|
||||
|
||||
CALIBRATION NOTES
|
||||
- Length alone is not quality; padding scores worse than a tight short piece. But \
|
||||
between two equally good pieces, prefer the one with more substance.
|
||||
- Social proof is evidence, not a verdict: hundreds of HN points mean a critical \
|
||||
audience read it; a quiet post from a good blog can still outrank it.
|
||||
- \"came via scour\" means the story already matched one of his standing \
|
||||
interests. \"came via hn_frontpage\" means it cleared HN's front page.
|
||||
- Boston/New England local stories and ultra-niche community news get a genuine \
|
||||
lift — this paper wants them.
|
||||
- Wire-service world/US news should score low here: the World Briefing section \
|
||||
covers that separately.
|
||||
- Excerpt-only or paywalled text is a real cost to the reader; score it lower \
|
||||
unless the piece is clearly excellent.
|
||||
|
||||
Return JSON exactly in this shape, with one entry per input article and nothing \
|
||||
else:
|
||||
{\"articles\": [{\"id\": 123, \"score\": 7.5, \"category\": \"Tech & Engineering\", \
|
||||
\"rationale\": \"first-hand account of migrating 40TB off Postgres\", \
|
||||
\"is_paywalled_guess\": false}]}";
|
||||
|
||||
/// Render the user prompt for one batch (§3.6).
|
||||
pub fn build_batch_prompt(batch: &[ScoredArticle], sections: &[String]) -> String {
|
||||
let mut prompt = String::with_capacity(4096 + batch.len() * 1500);
|
||||
prompt.push_str(SCORE_INSTRUCTIONS);
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nCATEGORY PALETTE (use one of these exact strings): {}\n\nARTICLES ({} in this batch)\n",
|
||||
sections.join(" | "),
|
||||
batch.len()
|
||||
);
|
||||
for candidate in batch {
|
||||
prompt.push('\n');
|
||||
prompt.push_str(&render_candidate(candidate));
|
||||
}
|
||||
prompt
|
||||
}
|
||||
|
||||
/// One article's block in the stage-A prompt (§3.6).
|
||||
fn render_candidate(candidate: &ScoredArticle) -> String {
|
||||
let a = &candidate.article;
|
||||
let mut block = String::with_capacity(1500);
|
||||
let _ = writeln!(block, "--- id: {}", a.id);
|
||||
let _ = writeln!(block, "title: {}", a.title.trim());
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"feed: {}{}",
|
||||
if a.feed_title.is_empty() {
|
||||
"unknown"
|
||||
} else {
|
||||
a.feed_title.trim()
|
||||
},
|
||||
a.category
|
||||
.as_deref()
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(|c| format!(" (category: {c})"))
|
||||
.unwrap_or_default()
|
||||
);
|
||||
if let Some(author) = a.author.as_deref().filter(|s| !s.trim().is_empty()) {
|
||||
let _ = writeln!(block, "author: {}", author.trim());
|
||||
}
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"length: {} words (~{} min read){}",
|
||||
a.word_count,
|
||||
a.reading_minutes(),
|
||||
if a.excerpt_only {
|
||||
" [EXCERPT ONLY — full text unavailable]"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
let _ = writeln!(block, "social: {}", social_line(candidate));
|
||||
let _ = writeln!(block, "came via: {}", sources_line(candidate));
|
||||
let excerpt = truncate_words(&prompt_text(&a.content_html), EXCERPT_WORDS);
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"excerpt: {}",
|
||||
if excerpt.is_empty() {
|
||||
"(no body text extracted)"
|
||||
} else {
|
||||
&excerpt
|
||||
}
|
||||
);
|
||||
block
|
||||
}
|
||||
|
||||
fn social_line(candidate: &ScoredArticle) -> String {
|
||||
if candidate.article.social.is_empty() {
|
||||
return "none found".into();
|
||||
}
|
||||
let mut parts: Vec<String> = candidate
|
||||
.article
|
||||
.social
|
||||
.iter()
|
||||
.map(|s| {
|
||||
format!(
|
||||
"{} {} points / {} comments",
|
||||
s.source.display_name(),
|
||||
s.score,
|
||||
s.num_comments
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
parts.push(format!("composite {:.2}", candidate.social_score));
|
||||
parts.join("; ")
|
||||
}
|
||||
|
||||
fn sources_line(candidate: &ScoredArticle) -> String {
|
||||
let mut kinds: Vec<&str> = candidate
|
||||
.article
|
||||
.sources
|
||||
.iter()
|
||||
.map(|s| match s.kind {
|
||||
SourceKind::Scour => "scour",
|
||||
SourceKind::HnFrontpage => "hn_frontpage",
|
||||
SourceKind::Lobsters => "lobsters",
|
||||
SourceKind::Reddit => "reddit",
|
||||
SourceKind::Feed => "feed",
|
||||
})
|
||||
.collect();
|
||||
kinds.sort_unstable();
|
||||
kinds.dedup();
|
||||
if candidate.auto_include {
|
||||
kinds.push("always-include feed (cannot be dropped)");
|
||||
}
|
||||
if kinds.is_empty() {
|
||||
"feed".into()
|
||||
} else {
|
||||
kinds.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response parsing (§3.6: tolerate anything the model does to us)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Keys the model might wrap the array in, in preference order.
|
||||
const ARRAY_KEYS: &[&str] = &["articles", "scores", "results", "items", "data"];
|
||||
|
||||
/// Parse a stage-A response leniently: missing optional fields default, scores
|
||||
/// are clamped to 0–10, and malformed items are skipped with a warning (§3.6).
|
||||
pub fn parse_score_response(raw: &str) -> Vec<ScoreItem> {
|
||||
let cleaned = strip_code_fence(raw);
|
||||
let value: Value = match serde_json::from_str(cleaned) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "stage A response was not JSON at all");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let array = match &value {
|
||||
Value::Array(items) => Some(items),
|
||||
Value::Object(map) => ARRAY_KEYS
|
||||
.iter()
|
||||
.find_map(|k| map.get(*k).and_then(Value::as_array))
|
||||
// Some models return {"1234": {...}} or a single bare object.
|
||||
.or_else(|| map.values().find_map(Value::as_array)),
|
||||
_ => None,
|
||||
};
|
||||
let Some(array) = array else {
|
||||
tracing::warn!("stage A response contained no array of scores");
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut out = Vec::with_capacity(array.len());
|
||||
let mut skipped = 0usize;
|
||||
for item in array {
|
||||
match parse_item(item) {
|
||||
Some(parsed) => out.push(parsed),
|
||||
None => {
|
||||
skipped += 1;
|
||||
tracing::warn!(item = %truncate_debug(item), "skipping malformed stage A item");
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, kept = out.len(), "stage A items were dropped");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn parse_item(item: &Value) -> Option<ScoreItem> {
|
||||
let obj = item.as_object()?;
|
||||
let id = obj.get("id").and_then(as_i64_lenient)?;
|
||||
let score = obj
|
||||
.get("score")
|
||||
.and_then(as_f64_lenient)
|
||||
.or_else(|| obj.get("rating").and_then(as_f64_lenient))?;
|
||||
Some(ScoreItem {
|
||||
id,
|
||||
score: score.clamp(0.0, 10.0),
|
||||
category: obj
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
rationale: obj
|
||||
.get("rationale")
|
||||
.or_else(|| obj.get("reason"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
is_paywalled_guess: obj
|
||||
.get("is_paywalled_guess")
|
||||
.or_else(|| obj.get("paywalled"))
|
||||
.and_then(as_bool_lenient)
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn as_i64_lenient(v: &Value) -> Option<i64> {
|
||||
v.as_i64()
|
||||
.or_else(|| v.as_f64().map(|f| f as i64))
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
}
|
||||
|
||||
fn as_f64_lenient(v: &Value) -> Option<f64> {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
.filter(|f| f.is_finite())
|
||||
}
|
||||
|
||||
fn as_bool_lenient(v: &Value) -> Option<bool> {
|
||||
v.as_bool().or_else(|| match v.as_str()?.trim() {
|
||||
"true" | "yes" => Some(true),
|
||||
"false" | "no" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn truncate_debug(v: &Value) -> String {
|
||||
v.to_string().chars().take(160).collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Score every candidate, filling in [`ScoredArticle::llm`] (§3.6).
|
||||
///
|
||||
/// Batches that fail are logged and left unscored rather than aborting the run.
|
||||
/// Returns how many candidates came back with a score.
|
||||
pub async fn score_all(
|
||||
llm: &LlmClient,
|
||||
candidates: &mut [ScoredArticle],
|
||||
batch_size: usize,
|
||||
max_concurrent_requests: usize,
|
||||
sections: &[String],
|
||||
temperature: f32,
|
||||
) -> Result<usize, LlmError> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let batch_size = batch_size.max(1);
|
||||
let batches = candidates.len().div_ceil(batch_size);
|
||||
let prompts = candidates
|
||||
.chunks(batch_size)
|
||||
.enumerate()
|
||||
.map(|(index, batch)| (index, batch.len(), build_batch_prompt(batch, sections)))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let results = stream::iter(prompts)
|
||||
.map(|(index, article_count, prompt)| async move {
|
||||
if let Err(error) = llm.meter.check_budget() {
|
||||
tracing::warn!(batch = index + 1, of = batches, %error, "bulk budget tripped; skipping stage A batch");
|
||||
return (index, Vec::new());
|
||||
}
|
||||
tracing::debug!(batch = index + 1, of = batches, articles = article_count, approx_tokens = super::approx_tokens(&prompt), "stage A request");
|
||||
let items = match llm.complete(&prompt, temperature, true).await {
|
||||
Ok(raw) => parse_score_response(&raw),
|
||||
Err(error) => {
|
||||
tracing::warn!(batch = index + 1, of = batches, %error, "stage A batch failed; its articles stay unscored");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
(index, items)
|
||||
})
|
||||
.buffer_unordered(max_concurrent_requests.max(1))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let mut scores: HashMap<ArticleId, LlmScore> = HashMap::with_capacity(candidates.len());
|
||||
for (_, items) in results {
|
||||
for item in items {
|
||||
scores.insert(item.id, item.into());
|
||||
}
|
||||
}
|
||||
let mut applied = 0;
|
||||
for candidate in candidates {
|
||||
if let Some(score) = scores.remove(&candidate.article.id) {
|
||||
candidate.llm = Some(score);
|
||||
applied += 1;
|
||||
}
|
||||
}
|
||||
if !scores.is_empty() {
|
||||
tracing::warn!(unknown_ids = scores.len(), "stage A returned unknown ids");
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DeepseekConfig;
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::curate::prefilter::tests::{article, via, with_social};
|
||||
use crate::types::TokenUsage;
|
||||
use std::sync::Arc;
|
||||
|
||||
const BATCH_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch.json"
|
||||
));
|
||||
const MESSY_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch_messy.json"
|
||||
));
|
||||
|
||||
fn sections() -> Vec<String> {
|
||||
crate::config::CurationConfig::default().sections
|
||||
}
|
||||
|
||||
fn candidate(id: i64, title: &str, words: i64) -> ScoredArticle {
|
||||
ScoredArticle {
|
||||
article: article(id, title, words),
|
||||
prefilter_score: 50.0,
|
||||
social_score: 0.0,
|
||||
llm: None,
|
||||
triage: None,
|
||||
auto_include: false,
|
||||
exploration: false,
|
||||
admitted_by: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_prompt_carries_every_documented_signal() {
|
||||
let mut c = candidate(12, "Migrating 40TB off Postgres", 3200);
|
||||
c.article = via(
|
||||
with_social(c.article, 342, 210),
|
||||
SourceKind::HnFrontpage,
|
||||
9001,
|
||||
);
|
||||
c.social_score = c.article.social_score();
|
||||
c.auto_include = true;
|
||||
let prompt = build_batch_prompt(&[c], §ions());
|
||||
|
||||
assert!(prompt.starts_with(SCORE_INSTRUCTIONS));
|
||||
assert!(prompt.contains("--- id: 12"));
|
||||
assert!(prompt.contains("title: Migrating 40TB off Postgres"));
|
||||
assert!(prompt.contains("feed: Some Blog (category: Tech)"));
|
||||
assert!(prompt.contains("author: A. Writer"));
|
||||
assert!(prompt.contains("length: 3200 words"));
|
||||
assert!(prompt.contains("HN 342 points / 210 comments"));
|
||||
assert!(prompt.contains("hn_frontpage"));
|
||||
assert!(prompt.contains("always-include feed"));
|
||||
assert!(prompt.contains("excerpt: word word"));
|
||||
assert!(prompt.contains("Tech & Engineering"));
|
||||
// The excerpt is capped.
|
||||
let excerpt_line = prompt
|
||||
.lines()
|
||||
.find(|l| l.starts_with("excerpt:"))
|
||||
.expect("excerpt line");
|
||||
assert!(excerpt_line.split_whitespace().count() <= EXCERPT_WORDS + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_realistic_deepseek_batch() {
|
||||
let items = parse_score_response(BATCH_FIXTURE);
|
||||
assert_eq!(items.len(), 4);
|
||||
assert_eq!(items[0].id, 101);
|
||||
assert!((items[0].score - 8.5).abs() < 1e-9);
|
||||
assert_eq!(items[0].category, "Tech & Engineering");
|
||||
assert!(items[0].rationale.split_whitespace().count() <= 20);
|
||||
assert!(!items[0].is_paywalled_guess);
|
||||
assert!(items[3].is_paywalled_guess);
|
||||
let score: LlmScore = items[0].clone().into();
|
||||
assert_eq!(score.category, "Tech & Engineering");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_survives_everything_a_model_might_do() {
|
||||
let items = parse_score_response(MESSY_FIXTURE);
|
||||
let ids: Vec<ArticleId> = items.iter().map(|i| i.id).collect();
|
||||
// 201 fine; 202 string score clamped; 203 missing rationale/category;
|
||||
// 204 out-of-range clamped; the two malformed entries are dropped.
|
||||
assert_eq!(ids, vec![201, 202, 203, 204]);
|
||||
assert!((items[1].score - 6.0).abs() < 1e-9);
|
||||
assert_eq!(items[2].rationale, "");
|
||||
assert_eq!(items[2].category, "");
|
||||
assert!(
|
||||
(items[3].score - 10.0).abs() < 1e-9,
|
||||
"clamped to the 0-10 range"
|
||||
);
|
||||
assert!(items.iter().all(|i| (0.0..=10.0).contains(&i.score)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_tolerates_fences_arrays_and_junk() {
|
||||
assert_eq!(
|
||||
parse_score_response("```json\n{\"articles\":[{\"id\":1,\"score\":5}]}\n```").len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(parse_score_response("[{\"id\": 2, \"score\": 3}]").len(), 1);
|
||||
assert_eq!(
|
||||
parse_score_response("{\"results\":[{\"id\":3,\"score\":\"4.5\"}]}")[0].score,
|
||||
4.5
|
||||
);
|
||||
assert!(parse_score_response("I'm sorry, I can't do that").is_empty());
|
||||
assert!(parse_score_response("{\"articles\": {}}").is_empty());
|
||||
}
|
||||
|
||||
fn client(backend: Arc<MockBackend>, limit_usd: f64) -> LlmClient {
|
||||
LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), limit_usd),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scores_are_applied_batch_by_batch() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":8,"category":"Tech & Engineering","rationale":"good"},
|
||||
{"id":2,"score":2,"category":"Niche Corner","rationale":"thin"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":3,"score":6.5,"category":"Culture & Essays","rationale":"solid"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
let mut candidates = vec![
|
||||
candidate(1, "One", 1000),
|
||||
candidate(2, "Two", 1000),
|
||||
candidate(3, "Three", 1000),
|
||||
];
|
||||
let scored = score_all(&llm, &mut candidates, 2, 4, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 3);
|
||||
assert_eq!(backend.calls(), 2, "batched by score_batch_size");
|
||||
assert_eq!(candidates[0].llm.as_ref().map(|l| l.score), Some(8.0));
|
||||
assert_eq!(candidates[2].llm.as_ref().map(|l| l.score), Some(6.5));
|
||||
// combined_score now reflects the LLM verdict.
|
||||
assert!(candidates[0].combined_score() > candidates[1].combined_score());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_batch_does_not_sink_the_run() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push_error("500 upstream exploded");
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":7,"category":"Top Stories","rationale":"ok"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, 4, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring must not abort");
|
||||
assert_eq!(scored, 1);
|
||||
assert!(candidates[0].llm.is_none());
|
||||
assert!(candidates[1].llm.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoring_stops_when_the_budget_is_gone() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
// First batch alone blows a $0.05 ceiling ($0.14 per 1M input tokens).
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
output_tokens: 0,
|
||||
},
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 0.05);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, 4, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 1, "only the first batch ran");
|
||||
assert_eq!(backend.calls(), 1);
|
||||
assert!(llm.meter.budget_exceeded());
|
||||
}
|
||||
}
|
||||
+79
-33
@@ -205,6 +205,18 @@ pub fn serialize_candidate(candidate: &Candidate) -> String {
|
||||
.insert("triage".into(), (triage.interest / 10.0).clamp(0.0, 1.0));
|
||||
value.present.insert("triage".into(), true);
|
||||
}
|
||||
if let Some(deep) = candidate.assessment.deep.as_ref() {
|
||||
value.raw.insert("quality".into(), deep.quality);
|
||||
value.raw.insert("fit".into(), deep.fit);
|
||||
value
|
||||
.norm
|
||||
.insert("quality".into(), (deep.quality / 10.0).clamp(0.0, 1.0));
|
||||
value
|
||||
.norm
|
||||
.insert("fit".into(), (deep.fit / 10.0).clamp(0.0, 1.0));
|
||||
value.present.insert("quality".into(), true);
|
||||
value.present.insert("fit".into(), true);
|
||||
}
|
||||
serde_json::to_string(&value).unwrap_or_else(|_| "{}".into())
|
||||
}
|
||||
|
||||
@@ -251,7 +263,7 @@ impl ExplainRow {
|
||||
serde_json::from_str(&self.signals_json).ok()
|
||||
}
|
||||
|
||||
/// Utility when step 5 has written it, else the preliminary blend.
|
||||
/// Utility once the deep set has been ranked, else the preliminary blend.
|
||||
pub fn score(&self) -> Option<f64> {
|
||||
self.utility
|
||||
.or_else(|| self.signals().and_then(|signals| signals.blend()))
|
||||
@@ -305,7 +317,8 @@ pub async fn explain_row(
|
||||
Ok(row.as_ref().map(ExplainRow::from_row))
|
||||
}
|
||||
|
||||
/// The top `limit` rows by utility-or-blend that were not selected (§15.2).
|
||||
/// The top `limit` rows that were not selected, by utility, falling back to
|
||||
/// the preliminary blend for rows the ranker never reached (§15.2).
|
||||
pub async fn near_misses(
|
||||
db: &Db,
|
||||
run_id: i64,
|
||||
@@ -413,24 +426,34 @@ pub async fn render_explain(db: &Db, row: &ExplainRow) -> Result<String, sqlx::E
|
||||
if !assessments.is_empty() {
|
||||
let _ = writeln!(out, "assessments:");
|
||||
for assessment in assessments {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" {} · {} · score {} · fit {} · kind {} · category {} · paywalled={} · {}",
|
||||
assessment.get::<String, _>("stage"),
|
||||
assessment.get::<String, _>("model"),
|
||||
fmt_opt(assessment.get::<Option<f64>, _>("score")),
|
||||
fmt_opt(assessment.get::<Option<f64>, _>("fit")),
|
||||
assessment
|
||||
.get::<Option<String>, _>("kind")
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
assessment
|
||||
.get::<Option<String>, _>("category")
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
assessment.get::<i64, _>("paywalled_guess") != 0,
|
||||
assessment
|
||||
.get::<Option<String>, _>("rationale")
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
let stage = assessment.get::<String, _>("stage");
|
||||
let model = assessment.get::<String, _>("model");
|
||||
let score = fmt_opt(assessment.get::<Option<f64>, _>("score"));
|
||||
let rationale = assessment
|
||||
.get::<Option<String>, _>("rationale")
|
||||
.unwrap_or_default();
|
||||
if stage == "deep" {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" deep · {model} · quality {score} · fit {} · format {} · category {} · paywalled={} · {rationale}",
|
||||
fmt_opt(assessment.get::<Option<f64>, _>("fit")),
|
||||
assessment
|
||||
.get::<Option<String>, _>("kind")
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
assessment
|
||||
.get::<Option<String>, _>("category")
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
assessment.get::<i64, _>("paywalled_guess") != 0,
|
||||
);
|
||||
} else {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" triage · {model} · interest {score} · kind {} · {rationale}",
|
||||
assessment
|
||||
.get::<Option<String>, _>("kind")
|
||||
.unwrap_or_else(|| "—".into()),
|
||||
);
|
||||
}
|
||||
if let Some(facets) = assessment.get::<Option<String>, _>("facets_json") {
|
||||
let _ = writeln!(out, " facets: {facets}");
|
||||
}
|
||||
@@ -876,16 +899,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn near_misses_rank_by_blend_and_skip_selected_and_excluded() {
|
||||
let (_dir, db) = db_with_articles(&[1, 2, 3, 4, 5]).await;
|
||||
async fn near_misses_rank_by_utility_then_blend_and_skip_selected_and_excluded() {
|
||||
let (_dir, db) = db_with_articles(&[1, 2, 3, 4, 5, 6]).await;
|
||||
let run_id = db.start_run(date(), Timestamp::now()).await.unwrap();
|
||||
// Utility decides wherever the ranker wrote one; the preliminary blend
|
||||
// only stands in for rows the deep set never reached. Article 4's blend
|
||||
// would put it first, but its utility is the lowest; article 3 never
|
||||
// got a utility and ranks on its blend.
|
||||
let rows = [
|
||||
(1, "selected", None, 0.9),
|
||||
(2, "shortlisted", Some("not_selected"), 0.7),
|
||||
(3, "eligible", Some("not_admitted"), 0.95),
|
||||
(4, "shortlisted", Some("not_selected"), 0.1),
|
||||
(1, "selected", None, 0.9, Some(90.0)),
|
||||
(2, "shortlisted", Some("not_selected"), 0.1, Some(70.0)),
|
||||
(3, "eligible", Some("not_admitted"), 0.95, None),
|
||||
(4, "shortlisted", Some("not_selected"), 0.99, Some(10.0)),
|
||||
(6, "assessed", Some("cluster_suppressed"), 0.5, Some(40.0)),
|
||||
];
|
||||
for (id, stage, reason, norm) in rows {
|
||||
for (id, stage, reason, norm, utility) in rows {
|
||||
let json = serialize_signals(&signals(10.0, norm), false);
|
||||
write(
|
||||
&db,
|
||||
@@ -896,7 +924,7 @@ mod tests {
|
||||
excluded_reason: reason,
|
||||
admitted_by: None,
|
||||
signals_json: &json,
|
||||
utility: None,
|
||||
utility,
|
||||
rank_utility: None,
|
||||
cluster_id: None,
|
||||
cluster_rank: None,
|
||||
@@ -912,18 +940,36 @@ mod tests {
|
||||
let misses = near_misses(&db, run_id, 10).await.unwrap();
|
||||
assert_eq!(
|
||||
misses.iter().map(|row| row.article_id).collect::<Vec<_>>(),
|
||||
vec![3, 2, 4]
|
||||
vec![3, 2, 6, 4]
|
||||
);
|
||||
let text = explain_near_misses(&db, date(), None, 2).await.unwrap();
|
||||
assert!(
|
||||
text.contains("top 2 not selected, by preliminary blend"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(text.contains("top 2 not selected, by utility"), "{text}");
|
||||
assert!(
|
||||
text.contains("Article 3 · eligible, not_admitted"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("Article 2 · shortlisted, not_selected"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(!text.contains("Article 4"), "{text}");
|
||||
|
||||
// Without any utility the listing says so and orders by the blend.
|
||||
sqlx::query("UPDATE candidate_runs SET utility = NULL WHERE run_id = ?")
|
||||
.bind(run_id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let misses = near_misses(&db, run_id, 10).await.unwrap();
|
||||
assert_eq!(
|
||||
misses.iter().map(|row| row.article_id).collect::<Vec<_>>(),
|
||||
vec![4, 3, 6, 2]
|
||||
);
|
||||
let text = explain_near_misses(&db, date(), None, 1).await.unwrap();
|
||||
assert!(
|
||||
text.contains("top 1 not selected, by preliminary blend"),
|
||||
"{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -357,11 +357,14 @@ pub async fn run(
|
||||
let rows = sqlx::query(
|
||||
"SELECT article_id, stage, score, kind, rationale, assessed_at
|
||||
FROM article_assessments
|
||||
WHERE model = ? AND prompt_version = ? AND assessed_at >= ?",
|
||||
WHERE model = ? AND assessed_at >= ?
|
||||
AND ((stage = 'triage' AND prompt_version = ?)
|
||||
OR (stage = 'deep' AND prompt_version = ?))",
|
||||
)
|
||||
.bind(&llm.model)
|
||||
.bind(TRIAGE_PROMPT_VERSION)
|
||||
.bind(fmt_ts(since))
|
||||
.bind(TRIAGE_PROMPT_VERSION)
|
||||
.bind(super::assess::DEEP_PROMPT_VERSION)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
let pool_ids = pool;
|
||||
|
||||
Reference in New Issue
Block a user