Curation v2 step 7: cleanup, prune paths, implementation notes
Dead code and stale v1 comments removed (clippy -W dead_code clean, the three world.rs warnings fixed), the Brief chapter's TOC title renamed from "From the Editor", features prune now also sweeps article_assessments and generate runs the sweep once after publishing, the example config is tested key-for-key against Config::default(), README commands match --help, and docs/plans/2026-08-15-implementation-notes.md records the Anthropic and Voyage facts, the new tables, the budget-day rule and the lock. Implemented by a Claude agent from docs/plans/curation-v2-briefs/step7.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
This commit is contained in:
+2
-1
@@ -122,7 +122,8 @@ pub struct UsageMeter {
|
||||
}
|
||||
|
||||
impl UsageMeter {
|
||||
/// Compatibility constructor for the existing DeepSeek call sites.
|
||||
/// A meter priced from the `[deepseek]` table; the other providers build
|
||||
/// theirs with [`UsageMeter::with_prices`].
|
||||
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
||||
Self::with_prices(PriceTable::deepseek(cfg), limit_usd)
|
||||
}
|
||||
|
||||
+2
-2
@@ -148,7 +148,7 @@ impl Curator {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
||||
/// prose. Only used to size prompt budgets (§3.6 stage C).
|
||||
/// prose. Only used to size prompt budgets.
|
||||
pub fn approx_tokens(text: &str) -> usize {
|
||||
text.len().div_ceil(4)
|
||||
}
|
||||
@@ -215,7 +215,7 @@ pub fn truncate_words(text: &str, max_words: usize) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C).
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary.
|
||||
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.len() <= max_chars {
|
||||
|
||||
@@ -16,7 +16,10 @@ use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE};
|
||||
use crate::types::{Facets, RatedArticle, TasteProfile};
|
||||
|
||||
pub const REBUILD_INTERVAL_DAYS: i64 = 7;
|
||||
pub const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
||||
/// The verdict block and the weekly rebuild are bounded by count
|
||||
/// (`verdicts_in_prompt`, [`MAX_RATINGS_IN_REBUILD`]), not by age (§8.3, §8.4),
|
||||
/// so their `current_ratings` lookback is effectively unbounded.
|
||||
const RATINGS_LOOKBACK_DAYS: i64 = 36_500;
|
||||
pub const KV_LEARNED_ADJUSTMENTS: &str = "taste_profile_learned";
|
||||
const MAX_RATINGS_IN_REBUILD: usize = 200;
|
||||
|
||||
|
||||
+53
-19
@@ -18,18 +18,7 @@ use crate::db::{Db, fmt_ts};
|
||||
use crate::report::RunReport;
|
||||
use crate::types::{ArticleId, Candidate, NearMiss};
|
||||
|
||||
/// The stage vocabulary of §7.4, in pipeline order.
|
||||
pub const STAGES: [&str; 7] = [
|
||||
"excluded",
|
||||
"eligible",
|
||||
"triaged",
|
||||
"admitted",
|
||||
"assessed",
|
||||
"shortlisted",
|
||||
"selected",
|
||||
];
|
||||
|
||||
/// Signal names rendered by `explain`, including the LLM ones steps 4–5 add.
|
||||
/// Signal names rendered by `explain`, in the order of §7.5.
|
||||
const RENDERED_SIGNALS: [&str; 8] = [
|
||||
"interest",
|
||||
"knn",
|
||||
@@ -850,15 +839,29 @@ pub async fn stats(db: &Db, days: i64, now: Timestamp) -> anyhow::Result<String>
|
||||
// `features prune` (§7.1, §7.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Rows removed by one [`prune`] pass.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Pruned {
|
||||
/// `article_embeddings` of unrated, unpublished articles past
|
||||
/// `embedding_retention_days`.
|
||||
pub embeddings: u64,
|
||||
/// `candidate_runs` rows of runs past `telemetry_retention_days`.
|
||||
pub telemetry: u64,
|
||||
/// `article_assessments` assessed more than `telemetry_retention_days` ago.
|
||||
pub assessments: u64,
|
||||
}
|
||||
|
||||
/// Delete `article_embeddings` for articles neither rated nor published that
|
||||
/// are older than `embedding_retention_days`, and `candidate_runs` rows whose
|
||||
/// run started more than `telemetry_retention_days` ago. Returns the counts.
|
||||
/// are older than `embedding_retention_days`, `candidate_runs` rows whose run
|
||||
/// started more than `telemetry_retention_days` ago, and `article_assessments`
|
||||
/// older than the same window (§7.1, §7.4). Runs from `features prune` and
|
||||
/// once per `generate` after publishing.
|
||||
pub async fn prune(
|
||||
db: &Db,
|
||||
embedding_retention_days: i64,
|
||||
telemetry_retention_days: i64,
|
||||
now: Timestamp,
|
||||
) -> Result<(u64, u64), sqlx::Error> {
|
||||
) -> Result<Pruned, sqlx::Error> {
|
||||
let cutoff = |days: i64| {
|
||||
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
||||
.unwrap_or(Timestamp::UNIX_EPOCH)
|
||||
@@ -886,7 +889,17 @@ pub async fn prune(
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok((embeddings, telemetry))
|
||||
|
||||
let assessments = sqlx::query("DELETE FROM article_assessments WHERE assessed_at < ?")
|
||||
.bind(fmt_ts(cutoff(telemetry_retention_days)))
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok(Pruned {
|
||||
embeddings,
|
||||
telemetry,
|
||||
assessments,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1523,13 +1536,26 @@ mod tests {
|
||||
let new_run = db.start_run(date(), now).await.unwrap();
|
||||
thin_excluded(&db, old_run, 1, "blocked").await.unwrap();
|
||||
thin_excluded(&db, new_run, 1, "blocked").await.unwrap();
|
||||
for (id, assessed_at) in [(1, old.clone()), (2, fmt_ts(now))] {
|
||||
sqlx::query(
|
||||
"INSERT INTO article_assessments
|
||||
(article_id, stage, model, prompt_version, score, assessed_at)
|
||||
VALUES (?, 'triage', 'deepseek-v4-flash', 1, 7.0, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&assessed_at)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let (embeddings, telemetry) = prune(&db, 120, 180, now).await.unwrap();
|
||||
let pruned = prune(&db, 120, 180, now).await.unwrap();
|
||||
assert_eq!(
|
||||
embeddings, 1,
|
||||
pruned.embeddings, 1,
|
||||
"only the old, unrated, unpublished article 3"
|
||||
);
|
||||
assert_eq!(telemetry, 1, "only the old run's rows");
|
||||
assert_eq!(pruned.telemetry, 1, "only the old run's rows");
|
||||
assert_eq!(pruned.assessments, 1, "only the 200-day-old assessment");
|
||||
let remaining: Vec<i64> =
|
||||
sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id")
|
||||
.fetch_all(db.pool())
|
||||
@@ -1541,5 +1567,13 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(runs, vec![new_run]);
|
||||
let assessed: Vec<i64> = sqlx::query_scalar("SELECT article_id FROM article_assessments")
|
||||
.fetch_all(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(assessed, vec![2]);
|
||||
|
||||
// A second pass finds nothing left to remove.
|
||||
assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user