From f0c0927ab8ff2ae6416d6faa0059ffd09920222c Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Sun, 13 Sep 2026 05:00:56 +0000 Subject: [PATCH 1/8] Add the interests table and module (first-class interests, step 1) Standing interests get a table of their own plus article_interests, the per-run top-3 matches, and a pure rates() that derives each interest's Beta-smoothed weight from current ratings the way feed affinity does. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc --- migrations/0013_interests.sql | 18 ++ src/db.rs | 6 + src/interests.rs | 534 ++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 4 files changed, 559 insertions(+) create mode 100644 migrations/0013_interests.sql create mode 100644 src/interests.rs diff --git a/migrations/0013_interests.sql b/migrations/0013_interests.sql new file mode 100644 index 0000000..e8867b4 --- /dev/null +++ b/migrations/0013_interests.sql @@ -0,0 +1,18 @@ +CREATE TABLE interests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL COLLATE NOCASE UNIQUE, + category TEXT, -- NULL until categorized + created_at TEXT NOT NULL, + categorized_at TEXT +); +CREATE INDEX idx_interests_category ON interests(category); + +CREATE TABLE article_interests ( + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE, + cos REAL NOT NULL, + z REAL NOT NULL, + run_id INTEGER, -- NULL for backfilled rows + PRIMARY KEY (article_id, interest_id) +); +CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC); diff --git a/src/db.rs b/src/db.rs index 5dae60f..9db8b1d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1689,6 +1689,10 @@ mod tests { .execute(&pool) .await .unwrap(); + sqlx::raw_sql(include_str!("../migrations/0013_interests.sql")) + .execute(&pool) + .await + .unwrap(); let rows = sqlx::query( "SELECT article_id, issue_date, kind, source, label, value, event_at, user_id @@ -1724,6 +1728,8 @@ mod tests { "interest_embeddings", "article_assessments", "candidate_runs", + "interests", + "article_interests", "users", "sessions", "config_changes", diff --git a/src/interests.rs b/src/interests.rs new file mode 100644 index 0000000..918ed6f --- /dev/null +++ b/src/interests.rs @@ -0,0 +1,534 @@ +//! Standing-interest storage and rating-derived weights. +//! +//! Interest queries stay here so the central database layer remains focused on +//! the pipeline's shared records. + +use std::collections::HashMap; + +use anyhow::{Result, bail}; +use jiff::Timestamp; +use sqlx::Row as _; + +use crate::curate::signals::TopInterest; +use crate::db::{Db, fmt_ts}; +use crate::types::ArticleId; + +/// One standing interest and its optional prompt category. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Interest { + pub id: i64, + pub name: String, + pub category: Option, + pub created_at: String, + pub categorized_at: Option, +} + +/// Result of adding a name whose uniqueness is case-insensitive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AddOutcome { + Added(i64), + Duplicate, +} + +/// One stored article-to-interest match. +#[derive(Debug, Clone, PartialEq)] +pub struct MatchRow { + pub article_id: ArticleId, + pub interest_id: i64, + pub name: String, + pub cos: f64, + pub z: f64, +} + +/// Rating credit accumulated for one interest. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct Rate { + pub up: f64, + pub down: f64, + pub n: usize, +} + +impl Rate { + /// Beta smoothing keeps an unrated interest neutral. + pub fn weight(&self) -> f64 { + (self.up + 1.0) / (self.up + self.down + 2.0) + } +} + +/// Interest rates plus the number of ratings that could affect them. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Rates { + pub by_interest: HashMap, + pub attributable: usize, +} + +const INTEREST_COLUMNS: &str = "id, name, category, created_at, categorized_at"; + +fn interest_from(row: &sqlx::sqlite::SqliteRow) -> Interest { + Interest { + id: row.get("id"), + name: row.get("name"), + category: row.get("category"), + created_at: row.get("created_at"), + categorized_at: row.get("categorized_at"), + } +} + +/// All interests, ordered case-insensitively by name. +pub async fn list(db: &Db) -> Result> { + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT {INTEREST_COLUMNS} FROM interests ORDER BY name COLLATE NOCASE, name" + ))) + .fetch_all(db.pool()) + .await?; + Ok(rows.iter().map(interest_from).collect()) +} + +/// Add one trimmed, non-empty name of at most 80 characters. +pub async fn add( + db: &Db, + name: &str, + category: Option<&str>, + now: Timestamp, +) -> Result { + let name = name.trim(); + let len = name.chars().count(); + if !(1..=80).contains(&len) { + bail!("interest name must be 1–80 characters"); + } + + let result = sqlx::query( + "INSERT OR IGNORE INTO interests (name, category, created_at, categorized_at) + VALUES (?, ?, ?, ?)", + ) + .bind(name) + .bind(category) + .bind(fmt_ts(now)) + .bind(category.map(|_| fmt_ts(now))) + .execute(db.pool()) + .await?; + if result.rows_affected() == 0 { + Ok(AddOutcome::Duplicate) + } else { + Ok(AddOutcome::Added(result.last_insert_rowid())) + } +} + +/// Set or clear an interest category and its categorization timestamp together. +pub async fn set_category(db: &Db, id: i64, category: Option<&str>, now: Timestamp) -> Result<()> { + sqlx::query("UPDATE interests SET category = ?, categorized_at = ? WHERE id = ?") + .bind(category) + .bind(category.map(|_| fmt_ts(now))) + .bind(id) + .execute(db.pool()) + .await?; + Ok(()) +} + +/// Delete an interest, its match rows, and its name-keyed cached embedding. +pub async fn delete(db: &Db, id: i64) -> Result<()> { + let mut tx = db.pool().begin().await?; + let name: Option = sqlx::query_scalar("SELECT name FROM interests WHERE id = ?") + .bind(id) + .fetch_optional(&mut *tx) + .await?; + sqlx::query("DELETE FROM interests WHERE id = ?") + .bind(id) + .execute(&mut *tx) + .await?; + if let Some(name) = name { + sqlx::query("DELETE FROM interest_embeddings WHERE interest = ?") + .bind(name) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) +} + +/// All names in the stable order used for embedding requests. +pub async fn names(db: &Db) -> Result> { + Ok( + sqlx::query_scalar("SELECT name FROM interests ORDER BY name COLLATE NOCASE, name") + .fetch_all(db.pool()) + .await?, + ) +} + +/// Names grouped for the prompt, with uncategorized interests last. +pub async fn grouped(db: &Db) -> Result)>> { + let mut by_category: HashMap> = HashMap::new(); + let mut other = Vec::new(); + for interest in list(db).await? { + if let Some(category) = interest.category { + by_category.entry(category).or_default().push(interest.name); + } else { + other.push(interest.name); + } + } + + let mut groups: Vec<_> = by_category.into_iter().collect(); + groups.sort_by(|left, right| { + left.0 + .to_lowercase() + .cmp(&right.0.to_lowercase()) + .then_with(|| left.0.cmp(&right.0)) + }); + for (_, members) in &mut groups { + members.sort_by(|left, right| { + left.to_lowercase() + .cmp(&right.to_lowercase()) + .then_with(|| left.cmp(right)) + }); + } + if !other.is_empty() { + groups.push(("Other standing interests".to_string(), other)); + } + Ok(groups) +} + +/// Interests awaiting the categorizer, ordered case-insensitively by name. +pub async fn uncategorized(db: &Db) -> Result> { + let rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT {INTEREST_COLUMNS} FROM interests WHERE category IS NULL + ORDER BY name COLLATE NOCASE, name" + ))) + .fetch_all(db.pool()) + .await?; + Ok(rows.iter().map(interest_from).collect()) +} + +/// Upsert the current run's recorded top-interest matches in one transaction. +pub async fn replace_matches( + db: &Db, + run_id: Option, + matches: &[(ArticleId, Vec)], + ids: &HashMap, +) -> Result<()> { + let mut tx = db.pool().begin().await?; + for (article_id, top_interests) in matches { + for top in top_interests { + let Some(interest_id) = ids.get(&top.name) else { + continue; + }; + sqlx::query( + "INSERT INTO article_interests (article_id, interest_id, cos, z, run_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(article_id, interest_id) DO UPDATE SET + cos = excluded.cos, z = excluded.z, run_id = excluded.run_id", + ) + .bind(article_id) + .bind(interest_id) + .bind(top.cos) + .bind(top.z) + .bind(run_id) + .execute(&mut *tx) + .await?; + } + } + tx.commit().await?; + Ok(()) +} + +/// Stored matches for the requested articles. +pub async fn matches_for_articles(db: &Db, article_ids: &[ArticleId]) -> Result> { + let mut matches = Vec::new(); + for chunk in article_ids.chunks(500) { + let placeholders = vec!["?"; chunk.len()].join(", "); + let mut query = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT ai.article_id, ai.interest_id, i.name, ai.cos, ai.z + FROM article_interests ai + JOIN interests i ON i.id = ai.interest_id + WHERE ai.article_id IN ({placeholders})" + ))); + for article_id in chunk { + query = query.bind(article_id); + } + for row in query.fetch_all(db.pool()).await? { + matches.push(MatchRow { + article_id: row.get("article_id"), + interest_id: row.get("interest_id"), + name: row.get("name"), + cos: row.get("cos"), + z: row.get("z"), + }); + } + } + matches.sort_by(|left, right| { + left.article_id + .cmp(&right.article_id) + .then_with(|| right.z.total_cmp(&left.z)) + .then_with(|| left.interest_id.cmp(&right.interest_id)) + }); + Ok(matches) +} + +/// Number of stored article matches for each interest. +pub async fn match_counts(db: &Db) -> Result> { + let rows = sqlx::query( + "SELECT interest_id, COUNT(*) AS matches FROM article_interests GROUP BY interest_id", + ) + .fetch_all(db.pool()) + .await?; + Ok(rows + .iter() + .map(|row| (row.get("interest_id"), row.get("matches"))) + .collect()) +} + +/// Derive smoothed interest rates from current ratings and their match rows. +pub fn rates(ratings: &[(ArticleId, f64, f64)], rows: &[(ArticleId, i64, f64)]) -> Rates { + let mut rows_by_article: HashMap> = HashMap::new(); + for &(article_id, interest_id, z) in rows { + rows_by_article + .entry(article_id) + .or_default() + .push((interest_id, z)); + } + + let mut result = Rates::default(); + for &(article_id, value, decay) in ratings { + let mut attributed = false; + if let Some(matches) = rows_by_article.get(&article_id) { + for &(interest_id, z) in matches { + let strength = (z / 3.0).clamp(0.0, 1.0); + if strength <= 0.0 { + continue; + } + attributed = true; + let credit = value * decay * strength; + let rate = result.by_interest.entry(interest_id).or_default(); + rate.up += credit.max(0.0); + rate.down += (-credit).max(0.0); + rate.n += 1; + } + } + if attributed { + result.attributable += 1; + } + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ts(value: &str) -> Timestamp { + value.parse().unwrap() + } + + async fn test_db() -> (tempfile::TempDir, Db) { + let dir = tempfile::tempdir().unwrap(); + let db = Db::open_and_migrate(&dir.path().join("db.sqlite")) + .await + .unwrap(); + (dir, db) + } + + async fn seed_article(db: &Db, id: ArticleId) { + sqlx::query( + "INSERT INTO articles (id, canonical_url, title, first_seen) VALUES (?, ?, ?, ?)", + ) + .bind(id) + .bind(format!("https://example.com/{id}")) + .bind(format!("Article {id}")) + .bind("2026-09-12T00:00:00Z") + .execute(db.pool()) + .await + .unwrap(); + } + + #[tokio::test] + async fn add_trims_names_and_uniqueness_is_case_insensitive() { + let (_dir, db) = test_db().await; + let now = ts("2026-09-12T12:00:00Z"); + let AddOutcome::Added(id) = add(&db, " Rust ", Some("Software"), now).await.unwrap() + else { + panic!("first insert should succeed"); + }; + assert_eq!( + add(&db, "rust", None, now).await.unwrap(), + AddOutcome::Duplicate + ); + assert!(add(&db, " ", None, now).await.is_err()); + assert!(add(&db, &"x".repeat(81), None, now).await.is_err()); + + let interests = list(&db).await.unwrap(); + assert_eq!(interests.len(), 1); + assert_eq!(interests[0].id, id); + assert_eq!(interests[0].name, "Rust"); + assert_eq!(interests[0].category.as_deref(), Some("Software")); + assert_eq!( + interests[0].categorized_at.as_deref(), + Some("2026-09-12T12:00:00Z") + ); + } + + #[test] + fn rates_apply_value_decay_strength_and_negative_credit() { + let ratings = [ + (1, 1.0, 1.0), + (2, 0.35, 1.0), + (3, 1.0, 0.5), + (4, -1.0, 0.5), + (5, -1.0, 1.0), + ]; + let rows = [ + (1, 10, 3.0), + (2, 10, 1.5), + (3, 11, 3.0), + (4, 10, 0.9), + (5, 12, 0.0), + ]; + let rates = rates(&ratings, &rows); + + assert_eq!(rates.attributable, 4); + let ten = rates.by_interest[&10]; + assert!((ten.up - 1.175).abs() < 1e-12); + assert!((ten.down - 0.15).abs() < 1e-12); + assert_eq!(ten.n, 3); + assert!((ten.weight() - 2.175 / 3.325).abs() < 1e-12); + assert_eq!( + rates.by_interest[&11], + Rate { + up: 0.5, + down: 0.0, + n: 1 + } + ); + assert!(!rates.by_interest.contains_key(&12)); + assert_eq!(Rate::default().weight(), 0.5); + } + + #[tokio::test] + async fn delete_cascades_matches_and_removes_the_embedding() { + let (_dir, db) = test_db().await; + seed_article(&db, 1).await; + let now = ts("2026-09-12T12:00:00Z"); + let AddOutcome::Added(id) = add(&db, "Rust", None, now).await.unwrap() else { + unreachable!(); + }; + sqlx::query( + "INSERT INTO interest_embeddings + (interest, model, dimension, embedding, created_at) VALUES (?, ?, ?, ?, ?)", + ) + .bind("Rust") + .bind("test") + .bind(1_i64) + .bind(vec![0_u8; 4]) + .bind("2026-09-12T12:00:00Z") + .execute(db.pool()) + .await + .unwrap(); + + let ids = HashMap::from([("Rust".to_string(), id)]); + replace_matches( + &db, + Some(7), + &[( + 1, + vec![ + TopInterest { + name: "Rust".into(), + cos: 0.7, + z: 1.2, + }, + TopInterest { + name: "Unknown".into(), + cos: 0.9, + z: 2.0, + }, + ], + )], + &ids, + ) + .await + .unwrap(); + replace_matches( + &db, + None, + &[( + 1, + vec![TopInterest { + name: "Rust".into(), + cos: 0.8, + z: 1.5, + }], + )], + &ids, + ) + .await + .unwrap(); + + assert_eq!(match_counts(&db).await.unwrap(), HashMap::from([(id, 1)])); + assert_eq!( + matches_for_articles(&db, &[1]).await.unwrap(), + [MatchRow { + article_id: 1, + interest_id: id, + name: "Rust".into(), + cos: 0.8, + z: 1.5 + }] + ); + let run_id: Option = sqlx::query_scalar( + "SELECT run_id FROM article_interests WHERE article_id = 1 AND interest_id = ?", + ) + .bind(id) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(run_id, None); + + delete(&db, id).await.unwrap(); + let match_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_interests") + .fetch_one(db.pool()) + .await + .unwrap(); + let embedding_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM interest_embeddings") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(match_rows, 0); + assert_eq!(embedding_rows, 0); + } + + #[tokio::test] + async fn grouped_sorts_categories_and_puts_uncategorized_last() { + let (_dir, db) = test_db().await; + let now = ts("2026-09-12T12:00:00Z"); + add(&db, "zebra", Some("Animals"), now).await.unwrap(); + add(&db, "Alpaca", Some("Animals"), now).await.unwrap(); + let AddOutcome::Added(id) = add(&db, "rust", None, now).await.unwrap() else { + unreachable!(); + }; + add(&db, "Baking", Some("cooking"), now).await.unwrap(); + + assert_eq!( + grouped(&db).await.unwrap(), + [ + ("Animals".into(), vec!["Alpaca".into(), "zebra".into()]), + ("cooking".into(), vec!["Baking".into()]), + ("Other standing interests".into(), vec!["rust".into()]), + ] + ); + assert_eq!(uncategorized(&db).await.unwrap()[0].id, id); + + set_category(&db, id, Some("Software"), now).await.unwrap(); + assert!(uncategorized(&db).await.unwrap().is_empty()); + set_category(&db, id, None, now).await.unwrap(); + let rust = uncategorized(&db).await.unwrap().pop().unwrap(); + assert_eq!(rust.categorized_at, None); + assert_eq!( + names(&db).await.unwrap(), + ["Alpaca", "Baking", "rust", "zebra"] + ); + } + + #[tokio::test] + async fn empty_article_lookup_is_a_no_op() { + let (_dir, db) = test_db().await; + assert!(matches_for_articles(&db, &[]).await.unwrap().is_empty()); + } +} diff --git a/src/lib.rs b/src/lib.rs index c7fb7d9..dc01ca6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub mod html; pub mod http; pub mod images; pub mod imports; +pub mod interests; pub mod jobs; pub mod lock; pub mod mail; From 2d857e3e10e6986599a7a53d578e8974438b7143 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Sun, 13 Sep 2026 05:20:05 +0000 Subject: [PATCH 2/8] Cut weak interest matches and add the rating-driven affinity signal (step 2) An interest matches an article only when it is in the top three by z and z >= 1.0, so the Matches line, the stored rows and the weights agree. The new bounded affinity signal blends each matched interest's rating-derived weight, gated on attributable ratings like feed affinity. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc --- README.md | 23 ++- config.example.toml | 12 +- src/config.rs | 31 +++- src/curate/rank.rs | 7 +- src/curate/signals.rs | 314 ++++++++++++++++++++++++++++++++-- src/curate/telemetry.rs | 23 ++- src/web/dashboard/mod.rs | 3 +- src/web/dashboard/settings.rs | 4 + 8 files changed, 384 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 315c7ae..139bd43 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,12 @@ prints what resolved. | `curation.recent_rejection_days` | `7` | Churn window for recent low triage/deep assessments. | | `curation.recent_rejection_floor` | `3.0` | Scores below this floor are excluded during the churn window (except auto-includes). | | `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. | +| `curation.ranking.affinity_floor` / `affinity_full` | `15` / `40` | Interest-attributable ratings where affinity starts and reaches full weight. | +| `curation.ranking.weights.preliminary.affinity` | `0.10` | Rating-derived interest affinity in the preliminary blend. | +| `curation.ranking.weights.preliminary.interest` | `0.30` | Interest similarity in the preliminary blend. | +| `curation.ranking.weights.preliminary.social` | `0.05` | Social signal in the preliminary blend. | +| `curation.ranking.weights.utility.affinity` | `0.05` | Rating-derived interest affinity in the utility score. | +| `curation.ranking.weights.utility.knn` | `0.10` | Rated-neighbour preference in the utility score. | | `editorial.summary_model` | `editor` | Which `[llm]` role writes the per-article summaries: `editor` (with per-article bulk fallback) or `bulk`. | | `editorial.summary_input_tokens` | `3000` | Article text offered to the summary prompt. | | `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. | @@ -448,7 +454,9 @@ prints what resolved. gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to `knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from `feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the -signal is absent. Ratings decay with `rating_half_life_days` (60) over +signal is absent. `affinity` (rating-derived interest affinity) likewise ramps +from `affinity_floor` (15) to `affinity_full` (40) interest-attributable ratings. +Ratings decay with `rating_half_life_days` (60) over `rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and `negative_coefficient` (0.75) shape the signal. `slop_author_penalty` (0.75) is the fraction of the blend and utility removed from every candidate whose @@ -457,13 +465,13 @@ limit. `triage_max` (800), `deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3), `semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]` (`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]` -(`quality` 0.40 · `fit` 0.20 · `knn` 0.15 · `interest` 0.10 · `feed` 0.05 · +(`quality` 0.40 · `fit` 0.20 · `knn` 0.10 · `affinity` 0.05 · `interest` 0.10 · `feed` 0.05 · `triage` 0.05 · `social` 0.03 · `heuristic` 0.02, over the signals present for each article of the deep set) and `[curation.ranking.diversity]` (`cluster_threshold` 0.85, `per_cluster_cap` 2, `utility_protected` 10) drive the LLM triage, deep assessment, utility ranking and diversification stages. -`[curation.ranking.weights.preliminary]` (`interest` 0.35 · `knn` 0.25 · -`heuristic` 0.20 · `feed` 0.10 · `social` 0.10) blends the cheap signals; weights +`[curation.ranking.weights.preliminary]` (`interest` 0.30 · `knn` 0.25 · +`affinity` 0.10 · `heuristic` 0.20 · `feed` 0.10 · `social` 0.05) blends the cheap signals; weights are renormalized over the signals present for each article, so they need not sum to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥ @@ -992,11 +1000,12 @@ From spec §7, plus what implementation turned up: ceiling; a protocol that is neither means another impl. Voyage AI embeddings sit behind the analogous `EmbeddingBackend` trait in `curate/embedding.rs`. - **Triage and union admission replace the heuristic gate.** Every eligible - article gets interest, rated-neighbour, feed-affinity, social and heuristic - signals, then DeepSeek reads its opening (up to `triage_max`). The deep set is + article gets interest, rated-neighbour, feed-affinity, interest-affinity, + social and heuristic signals, then DeepSeek reads its opening (up to + `triage_max`). The deep set is the union of triage, interest, neighbour, exploration, blend and auto-include retrievers. `explain` shows the assessment and `admitted_by`. Learned signals - stay absent until their gates open (8 and 15 ratings respectively). + stay absent until their gates open (8, 15, and 15 ratings respectively). - **Deep assessment and diversity are live.** DeepSeek reads a representative beginning/middle/end sample, separates editorial quality from reader fit, and records descriptive facets. Utility is normalized over the deep set; embedding diff --git a/config.example.toml b/config.example.toml index 99153b5..dc3c7c0 100644 --- a/config.example.toml +++ b/config.example.toml @@ -138,7 +138,7 @@ slop_value = -1.0 # AI slop: a full negative; the author pena verdicts_in_prompt = 60 # Every weight, quota, gate and threshold of the personalized ranker. The -# learned signals (`knn`, `feed`) contribute nothing until their gates open: +# learned signals (`knn`, `feed`, `affinity`) contribute nothing until their gates open: # the weight ramps linearly from `*_floor` to `*_full` rated articles. [curation.ranking] triage_max = 800 # eligible articles the triage LLM reads @@ -153,6 +153,8 @@ knn_floor = 8 knn_full = 25 feed_floor = 15 feed_full = 40 +affinity_floor = 15 +affinity_full = 40 slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict semantic_min_words = 300 exploration_slots = 5 @@ -166,16 +168,18 @@ knn = 20 # Weights need not sum to 1; they are renormalized over the present signals. [curation.ranking.weights.preliminary] -interest = 0.35 +interest = 0.30 knn = 0.25 +affinity = 0.10 heuristic = 0.20 feed = 0.10 -social = 0.10 +social = 0.05 [curation.ranking.weights.utility] quality = 0.40 fit = 0.20 -knn = 0.15 +knn = 0.10 +affinity = 0.05 interest = 0.10 feed = 0.05 triage = 0.05 diff --git a/src/config.rs b/src/config.rs index 8e32a1e..d5bceb1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -488,6 +488,8 @@ pub struct RankingConfig { pub knn_full: usize, pub feed_floor: usize, pub feed_full: usize, + pub affinity_floor: usize, + pub affinity_full: usize, /// Fraction of the preliminary blend and the utility removed from any /// candidate whose author has a current *AI slop* verdict (§9.3). `1.0` /// zeroes such candidates; `0.0` disables the penalty. @@ -516,6 +518,8 @@ impl Default for RankingConfig { knn_full: 25, feed_floor: 15, feed_full: 40, + affinity_floor: 15, + affinity_full: 40, slop_author_penalty: 0.75, semantic_min_words: 300, exploration_slots: 5, @@ -558,6 +562,7 @@ pub struct RankingWeights { pub struct PreliminaryWeights { pub interest: f64, pub knn: f64, + pub affinity: f64, pub heuristic: f64, pub feed: f64, pub social: f64, @@ -566,11 +571,12 @@ pub struct PreliminaryWeights { impl Default for PreliminaryWeights { fn default() -> Self { Self { - interest: 0.35, + interest: 0.30, knn: 0.25, + affinity: 0.10, heuristic: 0.20, feed: 0.10, - social: 0.10, + social: 0.05, } } } @@ -581,6 +587,7 @@ pub struct UtilityWeights { pub quality: f64, pub fit: f64, pub knn: f64, + pub affinity: f64, pub interest: f64, pub feed: f64, pub triage: f64, @@ -593,7 +600,8 @@ impl Default for UtilityWeights { Self { quality: 0.40, fit: 0.20, - knn: 0.15, + knn: 0.10, + affinity: 0.05, interest: 0.10, feed: 0.05, triage: 0.05, @@ -1204,7 +1212,10 @@ impl Config { .into(), )); } - if ranking.knn_full <= ranking.knn_floor || ranking.feed_full <= ranking.feed_floor { + if ranking.knn_full <= ranking.knn_floor + || ranking.feed_full <= ranking.feed_floor + || ranking.affinity_full <= ranking.affinity_floor + { return Err(ConfigError::Invalid( "curation.ranking *_full must be > *_floor >= 0".into(), )); @@ -1229,12 +1240,14 @@ impl Config { let weights = [ preliminary.interest, preliminary.knn, + preliminary.affinity, preliminary.heuristic, preliminary.feed, preliminary.social, utility.quality, utility.fit, utility.knn, + utility.affinity, utility.interest, utility.feed, utility.triage, @@ -2084,10 +2097,15 @@ mod tests { ); assert_eq!((ranking.knn_floor, ranking.knn_full), (8, 25)); assert_eq!((ranking.feed_floor, ranking.feed_full), (15, 40)); + assert_eq!((ranking.affinity_floor, ranking.affinity_full), (15, 40)); assert_eq!(ranking.rating_half_life_days, 60.0); assert_eq!(ranking.negative_coefficient, 0.75); - assert_eq!(ranking.weights.preliminary.interest, 0.35); + assert_eq!(ranking.weights.preliminary.interest, 0.30); + assert_eq!(ranking.weights.preliminary.affinity, 0.10); + assert_eq!(ranking.weights.preliminary.social, 0.05); assert_eq!(ranking.weights.utility.quality, 0.40); + assert_eq!(ranking.weights.utility.knn, 0.10); + assert_eq!(ranking.weights.utility.affinity, 0.05); assert_eq!(ranking.diversity.per_cluster_cap, 2); assert_eq!(ranking.embedding_retention_days, 120); assert_eq!(ranking.telemetry_retention_days, 180); @@ -2106,6 +2124,9 @@ mod tests { bad.curation.ranking.knn_full = bad.curation.ranking.knn_floor; assert!(bad.validate().is_err(), "*_full must exceed *_floor"); let mut bad = Config::default(); + bad.curation.ranking.affinity_full = bad.curation.ranking.affinity_floor; + assert!(bad.validate().is_err(), "*_full must exceed *_floor"); + let mut bad = Config::default(); bad.curation.ranking.shortlist_keep = bad.curation.ranking.deep_keep + 1; assert!(bad.validate().is_err(), "deep_keep >= shortlist_keep"); let mut bad = Config::default(); diff --git a/src/curate/rank.rs b/src/curate/rank.rs index 85452b2..e8e7c6d 100644 --- a/src/curate/rank.rs +++ b/src/curate/rank.rs @@ -62,6 +62,11 @@ fn calculate_utility_for( ("quality", configured.quality, 1.0), ("fit", configured.fit, 1.0), ("knn", configured.knn, candidate.signals.knn_gate), + ( + "affinity", + configured.affinity, + candidate.signals.affinity_gate, + ), ("interest", configured.interest, 1.0), ("feed", configured.feed, candidate.signals.feed_gate), ("triage", configured.triage, 1.0), @@ -342,7 +347,7 @@ mod tests { } assert!(!b.signals.weights.contains_key("interest")); assert!( - (a.signals.weights["knn"] / a.signals.weights["quality"] - (0.15 * 0.5) / 0.40).abs() + (a.signals.weights["knn"] / a.signals.weights["quality"] - (0.10 * 0.5) / 0.40).abs() < 1e-9 ); } diff --git a/src/curate/signals.rs b/src/curate/signals.rs index aefd9cf..4d1c715 100644 --- a/src/curate/signals.rs +++ b/src/curate/signals.rs @@ -14,11 +14,14 @@ use crate::config::{PreliminaryWeights, RankingConfig, VoyageConfig}; use crate::curate::embedding::{dot, load_article_embeddings}; use crate::curate::prefilter; use crate::db::Db; +use crate::interests::{self, Rate}; use crate::types::{Article, ArticleId, FeedId, SourceKind}; /// Below this many embedded eligible articles the z-score is too noisy, so the /// interest signal falls back to the raw top-1 cosine (§9.1). pub const INTEREST_ZSCORE_MIN_ARTICLES: usize = 30; +/// Weak top-three matches are omitted everywhere they are presented or credited. +pub const MATCH_MIN_Z: f64 = 1.0; /// Standard-deviation floor for the per-interest z-score (§9.1). const ZSCORE_STD_FLOOR: f64 = 1e-3; /// How many interests and rated neighbours `signals_json` records (§7.5). @@ -30,7 +33,8 @@ pub const AGGREGATOR_FEED_SHARE: f64 = 0.25; /// The signal names that go through the percentile normalizer, in the order /// they are rendered (§12.2). LLM scores (`triage`, `quality`, `fit`) are /// absolute and arrive in steps 4–5. -pub const PERCENTILE_SIGNALS: [&str; 5] = ["interest", "knn", "feed", "social", "heuristic"]; +pub const PERCENTILE_SIGNALS: [&str; 6] = + ["interest", "knn", "feed", "affinity", "social", "heuristic"]; #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct TopInterest { @@ -56,6 +60,7 @@ pub struct Signals { pub interest_top1_cos: Option, pub knn: Option, pub feed: Option, + pub affinity: Option, pub social: Option, pub heuristic: Option, /// Mid-rank percentiles of the present signals (§12.2). @@ -80,6 +85,8 @@ pub struct Signals { pub knn_gate: f64, #[serde(skip)] pub feed_gate: f64, + #[serde(skip)] + pub affinity_gate: f64, /// `ranking.slop_author_penalty`, applied when `slop_author` is set. #[serde(skip)] pub slop_penalty: f64, @@ -105,6 +112,7 @@ impl Signals { "interest_top1_cos" => self.interest_top1_cos, "knn" => self.knn, "feed" => self.feed, + "affinity" => self.affinity, "social" => self.social, "heuristic" => self.heuristic, _ => None, @@ -130,8 +138,10 @@ impl Signals { pub struct PreferenceSummary { pub rated_with_embeddings: usize, pub attributable_feed_ratings: usize, + pub attributable_interest_ratings: usize, pub knn_gate: f64, pub feed_gate: f64, + pub affinity_gate: f64, } /// One rated article with an embedding: the unit of the preference state (§9.2). @@ -179,12 +189,15 @@ pub struct PreferenceState { pub examples: Vec, feed_rates: HashMap, author_rates: HashMap, + interest_rates: HashMap, /// Normalized keys of authors with a current *AI slop* verdict (§9.3). slop_authors: HashSet, pub slop_author_penalty: f64, pub attributable_feed_ratings: usize, + pub attributable_interest_ratings: usize, pub knn_gate: f64, pub feed_gate: f64, + pub affinity_gate: f64, } impl PreferenceState { @@ -204,12 +217,28 @@ impl PreferenceState { examples, feed_rates, author_rates, + interest_rates: HashMap::new(), slop_authors: HashSet::new(), slop_author_penalty: ranking.slop_author_penalty, attributable_feed_ratings, + attributable_interest_ratings: 0, + affinity_gate: 0.0, } } + /// Register rating-derived interest rates after the embedding examples are built. + pub fn with_interest_rates( + mut self, + rates_by_name: HashMap, + attributable: usize, + ranking: &RankingConfig, + ) -> Self { + self.interest_rates = rates_by_name; + self.attributable_interest_ratings = attributable; + self.affinity_gate = gate(attributable, ranking.affinity_floor, ranking.affinity_full); + self + } + /// Register the authors whose current verdict is *AI slop*; keys are /// normalized like [`normalize_author`] and empty ones are dropped. pub fn with_slop_authors(mut self, authors: I) -> Self @@ -234,8 +263,7 @@ impl PreferenceState { self.slop_authors.len() } - /// Load `db::current_ratings(rating_lookback_days)` joined to - /// `article_embeddings`; ratings without an embedding are skipped (§9.2). + /// Load current ratings; only kNN/feed examples require an embedding. pub async fn load( db: &Db, voyage: &VoyageConfig, @@ -249,7 +277,7 @@ impl PreferenceState { .collect::>(); let embeddings = load_article_embeddings(db, voyage, &ids).await?; let mut examples = Vec::new(); - for rating in ratings { + for rating in &ratings { let Some(embedding) = embeddings.get(&rating.article_id).cloned() else { continue; }; @@ -264,8 +292,8 @@ impl PreferenceState { let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0; examples.push(RatedExample { article_id: rating.article_id, - label: rating.label, - title: rating.title, + label: rating.label.clone(), + title: rating.title.clone(), value: rating.value, decay: decay(age_days, ranking.rating_half_life_days), embedding, @@ -274,16 +302,51 @@ impl PreferenceState { aggregator_only, }); } + let match_rows = interests::matches_for_articles(db, &ids).await?; + let rated = ratings + .iter() + .map(|rating| { + let age_days = + (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0; + ( + rating.article_id, + rating.value, + decay(age_days, ranking.rating_half_life_days), + ) + }) + .collect::>(); + let matched = match_rows + .iter() + .map(|row| (row.article_id, row.interest_id, row.z)) + .collect::>(); + let rates = interests::rates(&rated, &matched); + let names = match_rows + .iter() + .map(|row| (row.interest_id, row.name.as_str())) + .collect::>(); + let rates_by_name = rates + .by_interest + .into_iter() + .filter_map(|(interest_id, rate)| { + names + .get(&interest_id) + .map(|name| ((*name).to_string(), rate)) + }) + .collect(); let slop_authors = db.slop_authors().await?; - Ok(Self::build(examples, ranking).with_slop_authors(slop_authors)) + Ok(Self::build(examples, ranking) + .with_interest_rates(rates_by_name, rates.attributable, ranking) + .with_slop_authors(slop_authors)) } pub fn summary(&self) -> PreferenceSummary { PreferenceSummary { rated_with_embeddings: self.examples.len(), attributable_feed_ratings: self.attributable_feed_ratings, + attributable_interest_ratings: self.attributable_interest_ratings, knn_gate: self.knn_gate, feed_gate: self.feed_gate, + affinity_gate: self.affinity_gate, } } @@ -297,21 +360,54 @@ impl PreferenceState { self.attributable_feed_ratings, ranking.feed_floor ) }; + let affinity_detail = if self.affinity_gate > 0.0 { + format!("(n={})", self.attributable_interest_ratings) + } else { + format!( + "(n={} < {})", + self.attributable_interest_ratings, ranking.affinity_floor + ) + }; tracing::info!( rated_with_embeddings = self.examples.len(), knn_gate = self.knn_gate, feed_gate = self.feed_gate, + affinity_gate = self.affinity_gate, slop_authors = self.slop_authors.len(), - "preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; {} slop authors (penalty {:.2})", + "preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}; affinity gate {:.1} {}; {} slop authors (penalty {:.2})", self.examples.len(), self.knn_gate, self.feed_gate, feed_detail, + self.affinity_gate, + affinity_detail, self.slop_authors.len(), self.slop_author_penalty ); } + /// Match-strength-weighted preference for the article's rated interests. + fn affinity(&self, top: &[TopInterest]) -> Option { + if self.affinity_gate <= 0.0 { + return None; + } + let mut weighted = 0.0; + let mut strength_sum = 0.0; + for interest in top { + let Some(rate) = self + .interest_rates + .get(&interest.name) + .filter(|rate| rate.n > 0) + else { + continue; + }; + let strength = (interest.z / 3.0).clamp(0.0, 1.0); + weighted += strength * (rate.weight() - 0.5); + strength_sum += strength; + } + (strength_sum > 0.0).then_some(weighted / strength_sum) + } + /// Signed rated-neighbour preference and the three nearest rated articles /// (§9.2). Absent when the gate is closed or there are no examples. pub fn knn(&self, candidate: &[f32], ranking: &RankingConfig) -> (Option, Vec) { @@ -552,6 +648,7 @@ pub fn interest_matches( let top_mean = all.iter().map(|item| item.z).sum::() / all.len() as f64; 0.7 * all[0].z + 0.3 * top_mean }; + all.retain(|interest| interest.z >= MATCH_MIN_Z); ( article_id, InterestMatch { @@ -580,10 +677,12 @@ pub fn compute( let mut signals = Signals::baseline(article); signals.knn_gate = preference.knn_gate; signals.feed_gate = preference.feed_gate; + signals.affinity_gate = preference.affinity_gate; if let Some(matched) = interests.get(&article.id) { signals.interest = Some(matched.score); signals.interest_top1_cos = Some(matched.top1_cos); signals.top_interests = matched.top_interests.clone(); + signals.affinity = preference.affinity(&matched.top_interests); } if let Some(embedding) = article_embeddings.get(&article.id) { let (knn, neighbours) = preference.knn(embedding, ranking); @@ -680,6 +779,7 @@ pub fn preliminary_blend(signals: &mut Signals, configured: &PreliminaryWeights) let candidates = [ ("interest", configured.interest, 1.0), ("knn", configured.knn, signals.knn_gate), + ("affinity", configured.affinity, signals.affinity_gate), ("heuristic", configured.heuristic, 1.0), ("feed", configured.feed, signals.feed_gate), ("social", configured.social, 1.0), @@ -817,6 +917,47 @@ mod tests { assert!((matched[&2].top1_cos - 0.707).abs() < 0.01); } + #[test] + fn match_cut_keeps_the_score_from_the_uncut_top_three() { + let mut articles = HashMap::new(); + for id in 1..=30 { + let mut vector = vec![0.0; 30]; + vector[id - 1] = 1.0; + articles.insert(id as ArticleId, vector); + } + let interest_at_z = |target: f64| { + let mean = -target / 29.0; + let spread = ((30.0 - target * target - target * target / 29.0) / 812.0).sqrt(); + let mut vector = vec![mean + spread; 30]; + vector[0] = target; + vector[29] = mean - 28.0 * spread; + unit( + &vector + .into_iter() + .map(|value| value as f32) + .collect::>(), + ) + }; + let interests = HashMap::from([ + ("first".to_string(), interest_at_z(2.0)), + ("second".to_string(), interest_at_z(1.5)), + ("weak third".to_string(), interest_at_z(0.4)), + ]); + + let matched = interest_matches(&articles, &interests); + let first = &matched[&1]; + assert_eq!( + first + .top_interests + .iter() + .map(|interest| interest.name.as_str()) + .collect::>(), + vec!["first", "second"] + ); + let uncut_score = 0.7 * 2.0 + 0.3 * ((2.0 + 1.5 + 0.4) / 3.0); + assert!((first.score - uncut_score).abs() < 1e-5, "{}", first.score); + } + #[test] fn interest_falls_back_to_raw_cosine_under_thirty_articles() { let (articles, interests) = interest_fixture(10); @@ -826,6 +967,11 @@ mod tests { (m.score - m.top1_cos).abs() < 1e-9, "article {id} should use raw top-1" ); + assert!( + m.top_interests + .iter() + .all(|interest| interest.z >= MATCH_MIN_Z) + ); } assert!((matched[&2].score - 0.707).abs() < 0.01); } @@ -930,6 +1076,99 @@ mod tests { assert_eq!(state.knn(&unit(&[1.0, 0.0]), &ranking), (None, Vec::new())); } + #[test] + fn affinity_is_absent_under_the_gate_and_without_rated_interests() { + let top = [TopInterest { + name: "Rust".into(), + z: 3.0, + cos: 0.8, + }]; + let rates = HashMap::from([( + "Rust".to_string(), + Rate { + up: 3.0, + down: 0.0, + n: 1, + }, + )]); + let closed = PreferenceState::build(Vec::new(), &ranking()).with_interest_rates( + rates, + 1, + &ranking(), + ); + assert_eq!(closed.affinity(&top), None); + + let mut open_ranking = ranking(); + open_ranking.affinity_floor = 0; + open_ranking.affinity_full = 1; + let empty = PreferenceState::build(Vec::new(), &open_ranking).with_interest_rates( + HashMap::new(), + 1, + &open_ranking, + ); + assert_eq!(empty.affinity(&top), None); + } + + #[tokio::test] + async fn affinity_load_counts_ratings_without_embeddings() { + let dir = tempfile::tempdir().unwrap(); + let db = Db::open_and_migrate(&dir.path().join("signals.db")) + .await + .unwrap(); + sqlx::query( + "INSERT INTO articles (id, canonical_url, title, first_seen) + VALUES (1, 'https://example.com/1', 'Rated', '2026-09-13T00:00:00Z')", + ) + .execute(db.pool()) + .await + .unwrap(); + let now = Timestamp::now(); + let interests::AddOutcome::Added(interest_id) = + interests::add(&db, "Rust", None, now).await.unwrap() + else { + unreachable!(); + }; + sqlx::query( + "INSERT INTO article_interests (article_id, interest_id, cos, z) + VALUES (1, ?, 0.8, 3.0)", + ) + .bind(interest_id) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO rating_events + (article_id, kind, source, label, value, event_at) + VALUES (1, 'explicit', 'test', 'loved', 1.0, ?)", + ) + .bind(now.to_string()) + .execute(db.pool()) + .await + .unwrap(); + let mut ranking = ranking(); + ranking.affinity_floor = 0; + ranking.affinity_full = 1; + let voyage = VoyageConfig { + enabled: false, + ..VoyageConfig::default() + }; + + let state = PreferenceState::load(&db, &voyage, &ranking, now) + .await + .unwrap(); + assert!(state.examples.is_empty()); + assert_eq!(state.attributable_interest_ratings, 1); + assert_eq!(state.affinity_gate, 1.0); + let affinity = state + .affinity(&[TopInterest { + name: "Rust".into(), + z: 3.0, + cos: 0.8, + }]) + .unwrap(); + assert!((affinity - 1.0 / 6.0).abs() < 1e-9); + } + // --- §9.3 feed affinity --- #[test] @@ -1165,8 +1404,8 @@ mod tests { assert!((signals.weights.values().sum::() - 1.0).abs() < 1e-9); assert!(!signals.weights.contains_key("knn")); assert!(!signals.weights.contains_key("social")); - // 0.35/0.55 × 0.8 + 0.20/0.55 × 0.4 = 0.6545… - assert!((blend - 65.4545).abs() < 0.01, "{blend}"); + // 0.30/0.50 × 0.8 + 0.20/0.50 × 0.4 = 0.64. + assert!((blend - 64.0).abs() < 1e-9, "{blend}"); let mut only_heuristic = Signals { heuristic: Some(2.0), @@ -1192,6 +1431,61 @@ mod tests { assert!((signals.weights["knn"] - 0.125 / 0.325).abs() < 1e-9); } + #[test] + fn liked_interest_outranks_disliked_interest_with_affinity_in_the_blend() { + let mut ranking = ranking(); + ranking.affinity_floor = 0; + ranking.affinity_full = 1; + let rates = HashMap::from([ + ( + "liked".to_string(), + Rate { + up: 3.0, + down: 0.0, + n: 1, + }, + ), + ( + "disliked".to_string(), + Rate { + up: 0.0, + down: 4.0 / 3.0, + n: 1, + }, + ), + ]); + let state = + PreferenceState::build(Vec::new(), &ranking).with_interest_rates(rates, 1, &ranking); + let top = |name: &str| { + vec![TopInterest { + name: name.into(), + z: 3.0, + cos: 0.8, + }] + }; + let mut signals = [ + Signals { + affinity: state.affinity(&top("liked")), + heuristic: Some(1.0), + affinity_gate: state.affinity_gate, + ..Signals::default() + }, + Signals { + affinity: state.affinity(&top("disliked")), + heuristic: Some(1.0), + affinity_gate: state.affinity_gate, + ..Signals::default() + }, + ]; + normalize(&mut signals.iter_mut().collect::>()); + for signal in &mut signals { + preliminary_blend(signal, &ranking.weights.preliminary); + assert!((signal.weights.values().sum::() - 1.0).abs() < 1e-9); + assert!(signal.weights.contains_key("affinity")); + } + assert!(signals[0].preliminary > signals[1].preliminary); + } + #[test] fn compute_scores_every_article_and_leaves_ungated_signals_absent() { let articles = vec![article(1, &[10]), article(2, &[20]), article(3, &[30])]; diff --git a/src/curate/telemetry.rs b/src/curate/telemetry.rs index f1685e3..76512d2 100644 --- a/src/curate/telemetry.rs +++ b/src/curate/telemetry.rs @@ -19,10 +19,11 @@ use crate::report::RunReport; use crate::types::{ArticleId, Candidate, NearMiss}; /// Signal names rendered by `explain`, in the order of §7.5. -const RENDERED_SIGNALS: [&str; 8] = [ +const RENDERED_SIGNALS: [&str; 9] = [ "interest", "knn", "feed", + "affinity", "social", "heuristic", "triage", @@ -159,6 +160,7 @@ pub fn serialize_signals(signals: &Signals, auto_include: bool) -> String { "interest_top1_cos", "knn", "feed", + "affinity", "social", "heuristic", ] { @@ -1157,11 +1159,17 @@ mod tests { Signals { interest: Some(1.2), interest_top1_cos: Some(0.61), + affinity: Some(0.2), heuristic: Some(heuristic), - norm: BTreeMap::from([("heuristic".into(), norm), ("interest".into(), 0.9)]), + norm: BTreeMap::from([ + ("affinity".into(), 0.7), + ("heuristic".into(), norm), + ("interest".into(), 0.9), + ]), weights: BTreeMap::from([ - ("heuristic".into(), 0.2 / 0.55), - ("interest".into(), 0.35 / 0.55), + ("affinity".into(), 0.1 / 0.6), + ("heuristic".into(), 0.2 / 0.6), + ("interest".into(), 0.3 / 0.6), ]), top_interests: vec![TopInterest { name: "Gaussian Splatting".into(), @@ -1186,6 +1194,7 @@ mod tests { assert_eq!(parsed["v"], 1); assert_eq!(parsed["raw"]["heuristic"], 41.0); assert_eq!(parsed["raw"]["interest_top1_cos"], 0.61); + assert_eq!(parsed["raw"]["affinity"], 0.2); assert_eq!(parsed["present"]["heuristic"], true); assert_eq!(parsed["present"]["knn"], false); assert_eq!(parsed["present"]["quality"], false); @@ -1385,7 +1394,11 @@ mod tests { ); let squashed = text.split_whitespace().collect::>().join(" "); assert!( - squashed.contains("heuristic 41.000 · 0.550 · 0.364"), + squashed.contains("heuristic 41.000 · 0.550 · 0.333"), + "{text}" + ); + assert!( + squashed.contains("affinity 0.200 · 0.700 · 0.167"), "{text}" ); assert!(squashed.contains("knn absent"), "{text}"); diff --git a/src/web/dashboard/mod.rs b/src/web/dashboard/mod.rs index 43fcc5c..7330bbc 100644 --- a/src/web/dashboard/mod.rs +++ b/src/web/dashboard/mod.rs @@ -58,10 +58,11 @@ pub fn router() -> Router { // --------------------------------------------------------------------------- /// Signal names in the order of curation plan §7.5. -pub const SIGNAL_NAMES: [&str; 8] = [ +pub const SIGNAL_NAMES: [&str; 9] = [ "interest", "knn", "feed", + "affinity", "social", "heuristic", "triage", diff --git a/src/web/dashboard/settings.rs b/src/web/dashboard/settings.rs index 13f6ca3..158d92f 100644 --- a/src/web/dashboard/settings.rs +++ b/src/web/dashboard/settings.rs @@ -351,6 +351,8 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[ ("curation.ranking.knn_full", "Rated articles at which the knn signal reaches full weight. Must be > knn_floor."), ("curation.ranking.feed_floor", "Attributable ratings before the feed-affinity signal starts to count."), ("curation.ranking.feed_full", "Attributable ratings at which feed affinity reaches full weight. Must be > feed_floor."), + ("curation.ranking.affinity_floor", "Interest-attributable ratings before the affinity signal starts to count."), + ("curation.ranking.affinity_full", "Interest-attributable ratings at which affinity reaches full weight. Must be > affinity_floor."), ("curation.ranking.semantic_min_words", "Bodies shorter than this are not embedded."), ("curation.ranking.exploration_slots", "Shortlist slots reserved for exploration picks."), ("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted."), @@ -360,12 +362,14 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[ ("curation.ranking.quotas.knn", "Deep-set slots filled by rated-neighbour preference."), ("curation.ranking.weights.preliminary.interest", "Interest similarity in the preliminary blend."), ("curation.ranking.weights.preliminary.knn", "Rated-neighbour preference in the preliminary blend."), + ("curation.ranking.weights.preliminary.affinity", "Rating-derived interest affinity in the preliminary blend."), ("curation.ranking.weights.preliminary.heuristic", "Heuristic score in the preliminary blend."), ("curation.ranking.weights.preliminary.feed", "Feed affinity in the preliminary blend."), ("curation.ranking.weights.preliminary.social", "Social signal in the preliminary blend."), ("curation.ranking.weights.utility.quality", "Deep-assessment quality in the utility score."), ("curation.ranking.weights.utility.fit", "Deep-assessment fit in the utility score."), ("curation.ranking.weights.utility.knn", "Rated-neighbour preference in the utility score."), + ("curation.ranking.weights.utility.affinity", "Rating-derived interest affinity in the utility score."), ("curation.ranking.weights.utility.interest", "Interest similarity in the utility score."), ("curation.ranking.weights.utility.feed", "Feed affinity in the utility score."), ("curation.ranking.weights.utility.triage", "Triage score in the utility score."), From a78e44f56eec7ebe2e8571bdb4f1432be91d8ffc Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Sun, 13 Sep 2026 05:20:16 +0000 Subject: [PATCH 3/8] Make the interests table the only source of standing interests (step 3) The OPML file and the profile's ## Interests section become one-time import inputs; the prompt groups by the stored category and the OPML config key is gone. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc --- README.md | 4 +- config.example.toml | 3 +- docs/runbooks/curation-v2-migration.md | 12 +- src/config.rs | 19 ++- src/curate/embedding.rs | 29 ++-- src/curate/profile/mod.rs | 167 +++++++---------------- src/interests.rs | 54 +++++++- src/main.rs | 2 - src/pipeline.rs | 31 ++--- src/web/dashboard/profile.rs | 71 +++++----- src/web/dashboard/settings.rs | 2 - src/web/templates/dashboard/profile.html | 11 +- tests/config_check.rs | 5 +- tests/m3_curation.rs | 17 ++- 14 files changed, 190 insertions(+), 237 deletions(-) diff --git a/README.md b/README.md index 315c7ae..9c852ab 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,7 @@ enabled on every request to an `anthropic`-kind provider. | **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node /cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. | | A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. | -`data/profile.md` is the hand-maintained reader profile; its optional interests are merged -with `data/scour-interests.opml`. Both paths are configurable. +`data/profile.md` is the hand-maintained reader profile; its path is configurable. --- @@ -366,7 +365,6 @@ prints what resolved. | `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. | | `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. | | `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. | -| `interests_opml` | `data/scour-interests.opml` | Scour interests merged with the profile interests. | | `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). | | `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. | | `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. | diff --git a/config.example.toml b/config.example.toml index 99153b5..fa4b822 100644 --- a/config.example.toml +++ b/config.example.toml @@ -24,9 +24,8 @@ database_path = "/var/lib/daily-epub/daily-epub.db" # Default output directory for generated artifacts (overridden by `--out`). out_dir = "/var/lib/daily-epub/out" -# Hand-maintained reader profile and Scour interests merged into the system prompt. +# Hand-maintained reader profile loaded into the system prompt. profile_path = "data/profile.md" -interests_opml = "data/scour-interests.opml" [miniflux] base_url = "http://127.0.0.1:8082" diff --git a/docs/runbooks/curation-v2-migration.md b/docs/runbooks/curation-v2-migration.md index 87e5c18..4fb1d6f 100644 --- a/docs/runbooks/curation-v2-migration.md +++ b/docs/runbooks/curation-v2-migration.md @@ -9,7 +9,7 @@ What changes for the operator, in one paragraph: the binary is replaced; the SQL tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first); `config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a `profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained -`profile.md` is installed next to the OPML; the systemd units are unchanged. +`profile.md` is installed for the hand-maintained reader profile; the systemd units are unchanged. ## 0. Before touching the server @@ -70,12 +70,13 @@ Everything you do not mention keeps its documented default, so the edit is small | Old key | Why | |---|---| +| `interests_opml = …` (top level) | standing interests now live in SQLite; remove this before rollout because unknown keys fail startup | | `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) | | `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` | | the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) | | any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` | -**Add** near the top, next to `interests_opml`: +**Add** near the top: ```toml profile_path = "/var/lib/daily-epub/data/profile.md" @@ -83,8 +84,7 @@ profile_path = "/var/lib/daily-epub/data/profile.md" Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an -explicit path survives running one-off commands from another directory. Point -`interests_opml` at an absolute path too if it is still relative. +explicit path survives running one-off commands from another directory. **Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old `[deepseek]` table if you had changed them; the values shown are the defaults. @@ -165,11 +165,9 @@ Voyage dashboards: the in-app `max_daily_usd` meters are runaway guards, not acc ```sh sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md -# if the OPML is not already there: -sudo install -m0640 -o daily-epub -g daily-epub data/scour-interests.opml /var/lib/daily-epub/data/ ``` -If the file is missing the run does not fail; it logs a warning and uses the OPML interests only, +If the file is missing the run does not fail; it logs a warning and uses empty profile prose, which is a much worse prompt. `config check` in the next step tells you whether it was found. ## 6. Check the config as the service user diff --git a/src/config.rs b/src/config.rs index 8e32a1e..40e8c15 100644 --- a/src/config.rs +++ b/src/config.rs @@ -37,12 +37,9 @@ impl From for ConfigError { /// Legacy/alternate env var for the rating-link HMAC key (spec §1). pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET"; -/// Root configuration document (§3.14). -/// -/// Unknown *top-level* keys are ignored on purpose: the prefix `DAILY_EPUB_` is -/// shared with plain operator env vars such as [`ENV_SECRET_ALIAS`]. +/// Root configuration document; unknown keys fail so retired settings stay visible. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(default)] +#[serde(deny_unknown_fields, default)] pub struct Config { /// IANA tz used for day boundaries and `--date` (§3.14, notes §2). pub timezone: String, @@ -65,8 +62,6 @@ pub struct Config { pub database_path: PathBuf, /// Default artifact output directory (overridden by `generate --out`). pub out_dir: PathBuf, - /// Scour interests OPML used to seed the taste profile (§3.6). - pub interests_opml: PathBuf, /// Hand-maintained reader profile loaded for every curation run (§8.2). pub profile_path: PathBuf, @@ -99,7 +94,6 @@ impl Default for Config { world_briefing: true, database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"), out_dir: PathBuf::from("/var/lib/daily-epub/out"), - interests_opml: PathBuf::from("data/scour-interests.opml"), profile_path: PathBuf::from("data/profile.md"), miniflux: MinifluxConfig::default(), llm: LlmConfig::default(), @@ -934,7 +928,11 @@ impl Config { fig = fig.merge(Toml::file(p)); } } - Ok(fig.merge(Env::prefixed(ENV_PREFIX).split(ENV_SPLIT))) + Ok(fig.merge( + Env::prefixed(ENV_PREFIX) + .ignore(&["secret"]) + .split(ENV_SPLIT), + )) } /// The file `load` reads: the explicit `--config` path, else `./config.toml` @@ -1059,7 +1057,6 @@ impl Config { }); lines.push(file_line("database_path", &self.database_path)); lines.push(file_line("profile_path", &self.profile_path)); - lines.push(file_line("interests_opml", &self.interests_opml)); for (role, name) in self.llm.roles() { match self.providers.get(name) { Some(provider) => lines.push(provider_line(&format!("llm.{role}"), name, provider)), @@ -2018,7 +2015,7 @@ mod tests { for (key, default) in defaults.as_object().expect("config is a table") { let section = match key.as_str() { "curation" | "llm" | "providers" | "voyage" | "editorial" => key, - "target_article_count" | "profile_path" | "interests_opml" => key, + "target_article_count" | "profile_path" => key, _ => continue, }; let documented = documented diff --git a/src/curate/embedding.rs b/src/curate/embedding.rs index 405a85f..4539f69 100644 --- a/src/curate/embedding.rs +++ b/src/curate/embedding.rs @@ -19,9 +19,10 @@ use sha2::{Digest as _, Sha256}; use sqlx::Row as _; use crate::config::{Config, VoyageConfig}; -use crate::curate::{approx_tokens, profile, prompt_text}; +use crate::curate::{approx_tokens, prompt_text}; use crate::db::{Db, fmt_ts}; use crate::http::RetryPolicy; +use crate::interests; use crate::types::{Article, ArticleId}; /// The only place the Voyage key comes from (§4.3). @@ -898,14 +899,7 @@ pub async fn plan_backfill( } } - let interests = - match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { - Ok(interests) => interests, - Err(error) => { - tracing::warn!(%error, "could not load standing interests; skipping them"); - Vec::new() - } - }; + let interest_names = interests::names(db).await?; let mut plan = BackfillPlan::default(); let mut keep = |articles: Vec
, misses: Vec<(ArticleId, i64)>| -> Vec
{ @@ -922,8 +916,8 @@ pub async fn plan_backfill( let other_misses = service.uncached_articles(&others).await?; plan.others = keep(others, other_misses); - let interest_misses = service.uncached_interests(&interests).await?; - plan.cached += interests.len() - interest_misses.len(); + let interest_misses = service.uncached_interests(&interest_names).await?; + plan.cached += interest_names.len() - interest_misses.len(); plan.estimated_tokens += interest_misses .iter() .map(|interest| approx_tokens(interest) as i64) @@ -1391,7 +1385,7 @@ mod tests { #[tokio::test] async fn backfill_prioritizes_the_learned_set_and_is_idempotent() { - let (dir, db) = db_with_articles(&[1, 2, 3]).await; + let (_dir, db) = db_with_articles(&[1, 2, 3]).await; // Article 1 is rated, article 2 is published, article 3 is neither. sqlx::query( "INSERT INTO rating_events (article_id, kind, source, label, value, event_at) @@ -1418,16 +1412,11 @@ mod tests { let config = Config { voyage: small_config(), - interests_opml: dir.path().join("interests.opml"), - profile_path: dir.path().join("profile.md"), ..Config::default() }; - std::fs::write( - &config.interests_opml, - "", - ) - .unwrap(); - std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); + interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now()) + .await + .unwrap(); let backend = Arc::new(MockBackend::auto(4)); let svc = service(db.clone(), config.voyage.clone(), backend.clone()); diff --git a/src/curate/profile/mod.rs b/src/curate/profile/mod.rs index 3a05bf5..054bec2 100644 --- a/src/curate/profile/mod.rs +++ b/src/curate/profile/mod.rs @@ -3,7 +3,6 @@ //! Every run rebuilds one byte-stable prompt from the hand-maintained profile, //! standing interests, stored weekly adjustments, and current explicit verdicts. -use std::collections::BTreeSet; use std::fmt::Write as _; use std::path::Path; @@ -13,6 +12,7 @@ use serde::{Deserialize, Serialize}; use super::llm::LlmClient; use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE}; +use crate::interests; use crate::types::{Facets, RatedArticle, TasteProfile}; pub const REBUILD_INTERVAL_DAYS: i64 = 7; @@ -29,50 +29,9 @@ pub const NO_LEARNED_ADJUSTMENTS: &str = "No reader ratings have been collected const EDITOR_IN_CHIEF_FRAMING: &str = "You are the editor-in-chief of *The Daily EPUB*, a personal morning newspaper assembled every day for exactly one reader. Everything you are asked to do — score, select, place, summarize, introduce — serves his taste, not a general audience's. When a judgement call is close, re-read this profile and decide the way he would."; // --------------------------------------------------------------------------- -// Interest and profile-file parsing +// Profile-file parsing // --------------------------------------------------------------------------- -pub fn parse_interests(opml_path: &Path) -> anyhow::Result> { - let raw = std::fs::read_to_string(opml_path) - .with_context(|| format!("reading the interests OPML at {}", opml_path.display()))?; - let interests = parse_interests_str(&raw); - if interests.is_empty() { - anyhow::bail!( - "no interests found in {}", - opml_path.display() - ); - } - tracing::debug!(count = interests.len(), "parsed scour interests"); - Ok(interests) -} - -pub fn parse_interests_str(raw: &str) -> Vec { - let mut seen = BTreeSet::new(); - let mut out = Vec::new(); - for chunk in raw.split("text=\"").skip(1) { - let Some((value, _)) = chunk.split_once('"') else { - continue; - }; - let name = xml_unescape(value).trim().to_string(); - if !name.is_empty() && seen.insert(name.to_lowercase()) { - out.push(name); - } - } - out -} - -fn xml_unescape(s: &str) -> String { - if !s.contains('&') { - return s.to_string(); - } - s.replace("<", "<") - .replace(">", ">") - .replace(""", "\"") - .replace("'", "'") - .replace("'", "'") - .replace("&", "&") -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProfileFile { /// Original Markdown with every `## Interests` section removed. @@ -116,7 +75,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result { match std::fs::read_to_string(path) { Ok(raw) => Ok(parse_profile_str(&raw)), Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - tracing::warn!(path = %path.display(), "profile file is missing; using OPML interests only"); + tracing::warn!(path = %path.display(), "profile file is missing; using empty profile prose"); Ok(ProfileFile { body: String::new(), interests: Vec::new(), @@ -128,30 +87,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result { } } -/// Load the exact standing-interest union used in the system prompt. -pub fn load_standing_interests( - opml_path: &Path, - profile_path: &Path, -) -> anyhow::Result> { - let opml = parse_interests(opml_path)?; - let profile = load_profile(profile_path)?; - Ok(union_interests(opml, profile.interests)) -} - -fn union_interests(opml: Vec, profile: Vec) -> Vec { - let mut seen = BTreeSet::new(); - let mut out = Vec::new(); - for interest in opml.into_iter().chain(profile) { - let interest = interest.trim(); - if !interest.is_empty() && seen.insert(interest.to_lowercase()) { - out.push(interest.to_string()); - } - } - out -} - pub mod themes; -pub use themes::group_into_themes; // --------------------------------------------------------------------------- // Prompt assembly @@ -174,7 +110,7 @@ fn one_line(text: &str) -> String { /// Assemble sections in the exact cache-friendly order required by §8.4. pub fn build( profile_body: &str, - interests: &[String], + grouped: &[(String, Vec)], learned_adjustments: &str, ratings: &[RatedArticle], verdict_limit: usize, @@ -193,8 +129,8 @@ pub fn build( doc.push_str("## Standing interests\n\n"); doc.push_str("These are his subscribed interest topics, grouped. They raise the floor for a match, but never cap the paper: an outstanding article on none of these still belongs.\n\n"); - for (theme, members) in group_into_themes(interests) { - let _ = writeln!(doc, "- **{}**: {}", theme, members.join(", ")); + for (category, members) in grouped { + let _ = writeln!(doc, "- **{}**: {}", category, members.join(", ")); } doc.push_str("\n## Learned adjustments (rebuilt weekly from ratings)\n\n"); @@ -272,26 +208,27 @@ async fn store_version(db: &Db, version: i64, built_at: Timestamp) -> anyhow::Re async fn prompt_inputs( db: &Db, - opml_path: &Path, profile_path: &Path, -) -> anyhow::Result<(ProfileFile, Vec, Vec, String)> { - let opml = parse_interests(opml_path)?; +) -> anyhow::Result<( + ProfileFile, + Vec<(String, Vec)>, + Vec, + String, +)> { let profile = load_profile(profile_path)?; - let interests = union_interests(opml, profile.interests.clone()); + let grouped = interests::grouped(db).await?; let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?; let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); - Ok((profile, interests, ratings, learned)) + Ok((profile, grouped, ratings, learned)) } /// Rebuild the complete system prompt from its live inputs on every run. pub async fn load_or_build( db: &Db, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result { - let (profile_file, interests, ratings, learned) = - prompt_inputs(db, opml_path, profile_path).await?; + let (profile_file, grouped, ratings, learned) = prompt_inputs(db, profile_path).await?; let (version, built_at) = match stored_version(db).await? { Some(stored) => stored, None => { @@ -303,7 +240,7 @@ pub async fn load_or_build( let profile = TasteProfile { text: build( &profile_file.body, - &interests, + &grouped, &learned, &ratings, verdict_limit, @@ -315,7 +252,10 @@ pub async fn load_or_build( db.kv_set(KV_TASTE_PROFILE, &profile.text).await?; tracing::debug!( version, - interests = interests.len(), + interests = grouped + .iter() + .map(|(_, members)| members.len()) + .sum::(), verdicts = ratings.len().min(verdict_limit), chars = profile.text.len(), "rebuilt the taste profile prompt" @@ -334,7 +274,6 @@ pub async fn is_stale(db: &Db) -> anyhow::Result { pub async fn weekly_rebuild_if_due( db: &Db, llm: &LlmClient, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result> { @@ -346,9 +285,7 @@ pub async fn weekly_rebuild_if_due( return Ok(None); } tracing::info!("taste profile is over a week old; rebuilding learned adjustments"); - Ok(Some( - rebuild(db, llm, opml_path, profile_path, verdict_limit).await?, - )) + Ok(Some(rebuild(db, llm, profile_path, verdict_limit).await?)) } // --------------------------------------------------------------------------- @@ -429,15 +366,12 @@ pub fn build_rebuild_prompt(ratings: &[RatedArticle]) -> String { pub async fn rebuild( db: &Db, llm: &LlmClient, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result { - // Read the prompt inputs first: a rebuild that dies on a missing OPML must - // stay due and must not have spent a model call getting there. - let opml = parse_interests(opml_path)?; + // Read the prompt inputs before spending a model call. let profile_file = load_profile(profile_path)?; - let interests = union_interests(opml, profile_file.interests.clone()); + let grouped = interests::grouped(db).await?; let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?; let previous = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); let learned = if ratings.is_empty() { @@ -473,7 +407,7 @@ pub async fn rebuild( let profile = TasteProfile { text: build( &profile_file.body, - &interests, + &grouped, &learned, &ratings, verdict_limit, @@ -495,18 +429,15 @@ pub async fn rebuild( mod tests { use super::*; - const OPML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/scour-interests.opml"); const PROFILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/profile.md"); #[test] - fn profile_interests_are_removed_and_union_case_insensitively() { + fn profile_interests_are_removed_for_the_importer() { let parsed = parse_profile_str( "# P\n\n## Interests\n- Rust\nBoston Tech\n- rust\n\n## Notes\nKeep this.\n", ); assert_eq!(parsed.body, "# P\n\n## Notes\nKeep this.\n"); assert_eq!(parsed.interests, ["Rust", "Boston Tech", "rust"]); - let union = union_interests(vec!["rust".into(), "E-Ink".into()], parsed.interests); - assert_eq!(union, ["rust", "E-Ink", "Boston Tech"]); } #[test] @@ -526,7 +457,7 @@ mod tests { }; let prompt = build( "# Reader profile\n\nProfile prose.", - &["Rust".into()], + &[("Software".into(), vec!["Rust".into(), "SQLite".into()])], "- Adjust.", &[rating], 60, @@ -539,17 +470,16 @@ mod tests { assert!( framing < profile && profile < interests && interests < learned && learned < verdicts ); + assert!(prompt.contains("- **Software**: Rust, SQLite")); assert!(prompt.contains("NOT FOR ME | A title | A feed | A summary with whitespace.")); } #[test] - fn shipped_profile_and_opml_parse() { + fn shipped_profile_parses() { let profile = load_profile(Path::new(PROFILE_PATH)).unwrap(); assert!(profile.body.contains("## Who he is")); assert!(!profile.body.contains("## Interests")); assert!(profile.interests.is_empty()); - let interests = parse_interests(Path::new(OPML_PATH)).unwrap(); - assert!(interests.iter().any(|interest| interest == "Rust")); } #[test] @@ -594,19 +524,21 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write( &profile_path, "# Reader profile\n\nOriginal prose.\n\n## Interests\n- Custom Topic\n", ) .unwrap(); - let first = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let first = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(first.version, 1); assert!(first.text.contains("Original prose.")); - assert!(first.text.contains("Custom Topic")); + assert!(first.text.contains("- **Software**: Rust")); + assert!(!first.text.contains("Custom Topic")); assert!(!first.text.contains("## Interests")); std::fs::write( @@ -614,11 +546,11 @@ mod tests { "# Reader profile\n\nChanged prose.\n\n## Interests\n- Another Topic\n", ) .unwrap(); - let second = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let second = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(second.version, first.version); assert_eq!(second.built_at, first.built_at); assert!(second.text.contains("Changed prose.")); - assert!(second.text.contains("Another Topic")); + assert!(!second.text.contains("Another Topic")); assert!(!second.text.contains("Original prose.")); let missing = load_profile(&dir.path().join("missing.md")).unwrap(); @@ -637,11 +569,12 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap(); - let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let initial = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(initial.version, 1); sqlx::query( @@ -677,7 +610,7 @@ mod tests { UsageMeter::for_provider(&ProviderConfig::deepseek()), backend.clone(), ); - let rebuilt = rebuild(&db, &llm, &opml, &profile_path, 60).await.unwrap(); + let rebuilt = rebuild(&db, &llm, &profile_path, 60).await.unwrap(); assert_eq!(rebuilt.version, 2); assert!(rebuilt.text.contains("Rank first-hand reports higher.")); assert!( @@ -700,16 +633,16 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap(); - let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let initial = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(initial.version, 1); - // The OPML goes missing the way a relative path does under a service - // whose working directory is not the checkout. - std::fs::remove_file(&opml).unwrap(); + std::fs::remove_file(&profile_path).unwrap(); + std::fs::create_dir(&profile_path).unwrap(); let backend = Arc::new(MockBackend::new()); let llm = LlmClient::with_backend( @@ -718,10 +651,10 @@ mod tests { UsageMeter::for_provider(&ProviderConfig::deepseek()), backend.clone(), ); - let error = rebuild(&db, &llm, &opml, &profile_path, 60) + let error = rebuild(&db, &llm, &profile_path, 60) .await - .expect_err("a missing OPML fails the rebuild"); - assert!(format!("{error:#}").contains("reading the interests OPML")); + .expect_err("an unreadable profile fails the rebuild"); + assert!(format!("{error:#}").contains("reading the reader profile")); // Still version 1, so the profile stays stale and the rebuild is retried. assert_eq!(stored_version(&db).await.unwrap().unwrap().0, 1); diff --git a/src/interests.rs b/src/interests.rs index 918ed6f..aae23c2 100644 --- a/src/interests.rs +++ b/src/interests.rs @@ -3,7 +3,7 @@ //! Interest queries stay here so the central database layer remains focused on //! the pipeline's shared records. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use anyhow::{Result, bail}; use jiff::Timestamp; @@ -64,6 +64,35 @@ pub struct Rates { const INTEREST_COLUMNS: &str = "id, name, category, created_at, categorized_at"; +/// Parse an OPML export for the one-time interests importer. +pub fn parse_opml(raw: &str) -> Vec { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for chunk in raw.split("text=\"").skip(1) { + let Some((value, _)) = chunk.split_once('"') else { + continue; + }; + let name = xml_unescape(value).trim().to_string(); + if !name.is_empty() && seen.insert(name.to_lowercase()) { + out.push(name); + } + } + out +} + +fn xml_unescape(value: &str) -> String { + if !value.contains('&') { + return value.to_string(); + } + value + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("&", "&") +} + fn interest_from(row: &sqlx::sqlite::SqliteRow) -> Interest { Interest { id: row.get("id"), @@ -326,6 +355,29 @@ mod tests { (dir, db) } + #[test] + fn opml_parser_unescapes_trims_and_deduplicates_names() { + let interests = parse_opml( + r#" + + + + + + + "#, + ); + assert_eq!( + interests, + [ + "Rust", + "E-Ink & RSS", + "Quotes \"and\" apostrophes 'x' 'y'", + "Markup ", + ] + ); + } + async fn seed_article(db: &Db, id: ArticleId) { sqlx::query( "INSERT INTO articles (id, canonical_url, title, first_seen) VALUES (?, ?, ?, ?)", diff --git a/src/main.rs b/src/main.rs index bc04c74..eb9b2a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -724,7 +724,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result { use curate::llm::{Llms, provider_meters}; let profile = curate::profile::load_or_build( db, - &config.interests_opml, &config.profile_path, config.curation.feedback.verdicts_in_prompt, ) @@ -749,7 +748,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result { let rebuilt = curate::profile::rebuild( db, llm, - &config.interests_opml, &config.profile_path, config.curation.feedback.verdicts_in_prompt, ) diff --git a/src/pipeline.rs b/src/pipeline.rs index f5b3b8e..c13ece6 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -46,7 +46,7 @@ use crate::types::{ Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta, Lineup, Models, TokenUsage, reading_minutes, }; -use crate::{comments, dedupe, discovery, epub, http, miniflux, publish, social, world}; +use crate::{comments, dedupe, discovery, epub, http, interests, miniflux, publish, social, world}; /// One `generate` invocation's inputs — the CLI flags, already parsed (§2). #[derive(Debug, Clone, Default)] @@ -943,15 +943,14 @@ async fn prepare_features( } }; report.counts.embedded = article_embeddings.len() as i64; - let interests = - match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { - Ok(interests) => interests, - Err(error) => { - tracing::warn!(%error, "could not load standing interests for embeddings"); - Vec::new() - } - }; - let interest_embeddings = match service.interests(&interests).await { + let interest_names = match interests::names(db).await { + Ok(interests) => interests, + Err(error) => { + tracing::warn!(%error, "could not load standing interests for embeddings"); + Vec::new() + } + }; + let interest_embeddings = match service.interests(&interest_names).await { Ok(embeddings) => embeddings, Err(error) => { report.warn(format!("interest embedding stage degraded: {error}")); @@ -1138,7 +1137,6 @@ async fn build_llms( ) -> Llms { let profile = match profile::load_or_build( ctx.db, - &ctx.config.interests_opml, &ctx.config.profile_path, ctx.config.curation.feedback.verdicts_in_prompt, ) @@ -1168,7 +1166,6 @@ async fn build_llms( match profile::weekly_rebuild_if_due( ctx.db, rebuild_client, - &ctx.config.interests_opml, &ctx.config.profile_path, ctx.config.curation.feedback.verdicts_in_prompt, ) @@ -1577,12 +1574,9 @@ mod tests { config.curation.blocked_domains = vec!["blocked.example".into()]; config.voyage.output_dimension = 4; config.target_article_count = 1; - config.interests_opml = dir.path().join("interests.opml"); - std::fs::write( - &config.interests_opml, - "", - ) - .unwrap(); + interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now()) + .await + .unwrap(); config.profile_path = dir.path().join("profile.md"); std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); @@ -1708,6 +1702,7 @@ mod tests { assert!(report.voyage_tokens > 0); // One batch for the two articles, one for the interest. assert_eq!(backend.calls(), 2); + assert_eq!(backend.requests()[1].input, ["Writerdeck"]); let signals = &features .iter() .find(|candidate| candidate.article.id == a) diff --git a/src/web/dashboard/profile.rs b/src/web/dashboard/profile.rs index 2e49475..f3eb33f 100644 --- a/src/web/dashboard/profile.rs +++ b/src/web/dashboard/profile.rs @@ -1,8 +1,7 @@ //! Dashboard: the profile page (`/dashboard/profile`, web plan §11). //! -//! Edits `profile.md` with version history, shows what the loader parses out -//! of it, the standing OPML interests by theme, the stored system prompt and -//! the weekly learned adjustments, and offers the `profile-rebuild` job. +//! Edits `profile.md` with version history and shows the stored interests, +//! system prompt, and weekly learned adjustments. use std::path::Path; @@ -18,6 +17,7 @@ use sqlx::Row; use crate::curate::profile::{self, KV_LEARNED_ADJUSTMENTS, ProfileFile, REBUILD_INTERVAL_DAYS}; use crate::db::{Db, DbError, KV_TASTE_PROFILE}; +use crate::interests; use crate::server::AppState; use crate::web::session::{AuthSession, Viewer}; use crate::web::{Flash, Html, Page, WebError, format_time, take_flash}; @@ -104,8 +104,7 @@ fn read_profile(path: &Path) -> anyhow::Result> { } } -/// The live preview of what the loader extracts (§11): the passthrough body -/// and the `## Interests` lines. +/// The live preview of the prose that reaches the prompt. pub fn preview(content: &str) -> ProfileFile { profile::parse_profile_str(content) } @@ -191,7 +190,7 @@ async fn versions(db: &Db, config: &crate::config::Config) -> Result, versions: Vec, - opml_path: String, - opml_count: usize, - opml_error: String, - themes: Vec, + interest_count: usize, + category_count: usize, + categories: Vec, prompt: String, prompt_chars: usize, prompt_version: String, @@ -254,20 +251,17 @@ async fn show( let content = stored.unwrap_or_default(); let parsed = preview(&content); - let (opml_count, opml_error, themes) = match profile::parse_interests(&config.interests_opml) { - Ok(interests) => { - let themes = profile::group_into_themes(&interests) - .into_iter() - .map(|(name, members)| ThemeView { - name, - count: members.len(), - members: members.join(", "), - }) - .collect(); - (interests.len(), String::new(), themes) - } - Err(error) => (0, format!("{error:#}"), Vec::new()), - }; + let grouped = interests::grouped(db).await.map_err(WebError::Internal)?; + let interest_count = grouped.iter().map(|(_, members)| members.len()).sum(); + let category_count = grouped.len(); + let categories = grouped + .into_iter() + .map(|(name, members)| CategoryView { + name, + count: members.len(), + members: members.join(", "), + }) + .collect(); let prompt = db.kv_get(KV_TASTE_PROFILE).await?.unwrap_or_default(); let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); @@ -297,12 +291,10 @@ async fn show( max_bytes: MAX_PROFILE_BYTES, content, preview_body: parsed.body, - preview_interests: parsed.interests, versions: versions(db, &config).await?, - opml_path: config.interests_opml.display().to_string(), - opml_count, - opml_error, - themes, + interest_count, + category_count, + categories, prompt_chars: prompt.len(), prompt_verdicts: count_verdict_lines(&prompt), prompt, @@ -516,15 +508,15 @@ mod tests { .unwrap(); let config = Config { profile_path: dir.path().join("profile.md"), - interests_opml: dir.path().join("interests.opml"), ..Config::default() }; std::fs::write(&config.profile_path, "# Original\n\nProse.\n").unwrap(); - std::fs::write( - &config.interests_opml, - r#""#, - ) - .unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); + interests::add(&db, "Boston", Some("Places"), Timestamp::now()) + .await + .unwrap(); db.kv_set( KV_TASTE_PROFILE, "system prompt text\n\n## Recent verdicts\n\nLOVED | x\n", @@ -617,7 +609,7 @@ mod tests { } #[tokio::test] - async fn profile_page_shows_editor_preview_interests_prompt_and_rebuild_form() { + async fn profile_page_shows_editor_standing_interests_prompt_and_rebuild_form() { let (_dir, _state, app, cookie) = setup().await; let response = get(&app, Some(&cookie)).await; assert_eq!(response.status(), StatusCode::OK); @@ -625,7 +617,8 @@ mod tests { assert!(body.contains("# Original")); assert!(body.contains("Prose.")); assert!(body.contains("Rust, Boston") || body.contains("Rust") && body.contains("Boston")); - assert!(body.contains("2 interests")); + assert!(body.contains("2 standing interests in 2 categories")); + assert!(body.contains("Interests page")); assert!(body.contains("system prompt text")); assert!(body.contains("Rank depth higher.")); assert!(body.contains("never built")); @@ -666,7 +659,7 @@ mod tests { let page = text(get(&app, Some(&cookie)).await).await; assert!(page.contains("Saved; the next run rebuilds the system prompt.")); - assert!(page.contains("Writerdeck")); + assert!(page.contains("section is ignored")); assert!(page.contains("# Original")); assert!(page.contains(">tyler<")); diff --git a/src/web/dashboard/settings.rs b/src/web/dashboard/settings.rs index 13f6ca3..40e1dbe 100644 --- a/src/web/dashboard/settings.rs +++ b/src/web/dashboard/settings.rs @@ -246,7 +246,6 @@ const PATH_KEYS: &[&str] = &[ "database_path", "out_dir", "profile_path", - "interests_opml", "publish.epub_dir", "publish.xtc_dir", "xtc.settings", @@ -295,7 +294,6 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[ ("database_path", "SQLite file; parent directories are created on demand."), ("out_dir", "Where generate writes artifacts before publishing (overridden by --out)."), ("profile_path", "Hand-maintained reader profile, loaded every run."), - ("interests_opml", "Scour interests OPML merged with the profile interests."), ("miniflux.base_url", "Miniflux root (no /v1)."), ("miniflux.public_url", "Browser-facing Miniflux web UI URL for dashboard links. Defaults to miniflux.base_url."), ("miniflux.api_key", "X-Auth-Token for Miniflux. Required; environment only."), diff --git a/src/web/templates/dashboard/profile.html b/src/web/templates/dashboard/profile.html index a2d7535..d57660c 100644 --- a/src/web/templates/dashboard/profile.html +++ b/src/web/templates/dashboard/profile.html @@ -1,13 +1,13 @@ {% extends "layout.html" %}{% block content %}

Profile

-

The standing taste file the curator reads before every run: what you like, what the OPML declares, and what the editor model has learned from your verdicts.

+

The standing taste file the curator reads before every run, alongside stored interests and what the editor model has learned from your verdicts.

RatingsFeedback settings

profile.md

-

{{ path }}{% if !exists %} — missing; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any ## Interests section is parsed one interest per line; everything else goes into the system prompt verbatim.

+

{{ path }}{% if !exists %} — missing; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any ## Interests section is ignored; everything else goes into the system prompt verbatim.

The next run rebuilds the system prompt from the saved file.
@@ -19,8 +19,6 @@

A live read of the text on the left, exactly as curate::profile splits it.

Passthrough sections

{% if preview_body.trim().is_empty() %}

Nothing passes through — the file is empty or only has an Interests section.

{% else %}
{{ preview_body }}
{% endif %} -

Extracted ## Interests lines

-{% if preview_interests.is_empty() %}

None. The prompt uses the OPML interests alone.

{% else %}
    {% for interest in preview_interests %}
  • {{ interest }}
    • {% endfor %}
{% endif %}
@@ -40,9 +38,8 @@ {% endif %}

Standing interests

-

{{ opml_path }} · {{ opml_count }} interests, grouped the way the system prompt lists them. The union of these and the ## Interests lines above is what the prompt uses; edit the OPML file to change them.

-{% if !opml_error.is_empty() %}

{{ opml_error }}

{% endif %} -{% if !themes.is_empty() %}
{% for theme in themes %}
{{ theme.name }} ({{ theme.count }})
{{ theme.members }}
{% endfor %}
{% endif %} +

{{ interest_count }} standing interests in {{ category_count }} categories — manage them on the Interests page.

+{% if !categories.is_empty() %}
{% for category in categories %}
{{ category.name }} ({{ category.count }})
{{ category.members }}
{% endfor %}
{% endif %}

Learned adjustments

Rebuilt weekly from ratings by the editor model (prompt version {{ prompt_version }}, built {{ prompt_built_at }}, {{ learned_age }}). {% if rebuild_due %}A rebuild is due — the next run performs it, or start it now.{% else %}The next scheduled rebuild is at least {{ rebuild_interval_days }} days after the last one; the next run performs it when due.{% endif %}

diff --git a/tests/config_check.rs b/tests/config_check.rs index 57a4b3e..172ae0f 100644 --- a/tests/config_check.rs +++ b/tests/config_check.rs @@ -35,7 +35,6 @@ fn config_check_prints_the_facts_and_exits_zero_without_keys() { "config: ", "database_path: /var/lib/daily-epub/daily-epub.db", "profile_path: data/profile.md", - "interests_opml: data/scour-interests.opml", "llm.bulk: deepseek · openai · deepseek-v4-flash", "key MISSING (set DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY)", "llm.editor: anthropic · anthropic · claude-opus-5 · effort high · max_daily_usd $3.00", @@ -66,4 +65,8 @@ fn config_check_exits_non_zero_on_an_invalid_config() { let (code, _, stderr) = run("[deepseek]\nmodel = \"x\"\n"); assert_ne!(code, 0); assert!(stderr.contains("[providers.deepseek]"), "{stderr}"); + + let (code, _, stderr) = run("interests_opml = \"data/scour-interests.opml\"\n"); + assert_ne!(code, 0); + assert!(stderr.contains("interests_opml"), "{stderr}"); } diff --git a/tests/m3_curation.rs b/tests/m3_curation.rs index 96212a3..f18c909 100644 --- a/tests/m3_curation.rs +++ b/tests/m3_curation.rs @@ -21,6 +21,7 @@ use daily_epub::curate::assess::parse_deep_response; use daily_epub::curate::editor::parse_selection_response; use daily_epub::curate::editorial::BriefResponse; use daily_epub::curate::profile; +use daily_epub::interests; fn repo(rel: &str) -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join(rel) @@ -183,13 +184,14 @@ fn stage_c_fixture_parses_into_the_brief() { assert!(value.get("section_intros").is_none()); } -/// The taste profile is seeded from this file; a broken export would silently -/// gut the system prompt (§3.6a). +/// The importer relies on this file; a broken export would silently lose +/// standing interests (§3.6a). #[test] fn scour_opml_still_yields_the_interest_list() { - let interests = profile::parse_interests(&repo("data/scour-interests.opml")) - .expect("the shipped OPML must parse"); - let unique: BTreeSet = interests.iter().map(|n| n.to_lowercase()).collect(); + let raw = std::fs::read_to_string(repo("data/scour-interests.opml")) + .expect("the shipped OPML must be readable"); + let names = interests::parse_opml(&raw); + let unique: BTreeSet = names.iter().map(|n| n.to_lowercase()).collect(); assert!( unique.len() > 180, @@ -206,7 +208,7 @@ fn scour_opml_still_yields_the_interest_list() { assert!(unique.contains(expected), "{expected} disappeared"); } assert!( - !interests.iter().any(|n| n.contains("token=")), + !names.iter().any(|n| n.contains("token=")), "interest names must not leak the Scour token" ); @@ -217,9 +219,10 @@ fn scour_opml_still_yields_the_interest_list() { "/data/profile.md" ))) .expect("profile file"); + let grouped = vec![("Imported interests".to_string(), names)]; let document = profile::build( &profile_file.body, - &interests, + &grouped, profile::NO_LEARNED_ADJUSTMENTS, &[], 60, From 22cfb34788b523aa220367af28359a3dc17f6016 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Sun, 13 Sep 2026 05:22:25 +0000 Subject: [PATCH 4/8] Add the first-class interests plan Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc --- .../plans/2026-09-12-first-class-interests.md | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 docs/plans/2026-09-12-first-class-interests.md diff --git a/docs/plans/2026-09-12-first-class-interests.md b/docs/plans/2026-09-12-first-class-interests.md new file mode 100644 index 0000000..78414f4 --- /dev/null +++ b/docs/plans/2026-09-12-first-class-interests.md @@ -0,0 +1,247 @@ +# First-class interests: a table, a page, categories, links, and a rating-driven weight + +**Date:** 2026-09-12 +**Repository:** `thallada/the-daily-epub` +**Status:** implementation plan, ready to execute +**Builds on:** `docs/plans/2026-09-02-personalized-curation-v2.md` (§8 profile and interests, §9 cheap signals, §12 blend and utility), `docs/plans/2026-09-03-web-dashboard.md` (page conventions), `docs/plans/2026-09-07-feed-discovery.md` (the most recent page + job + CLI addition; copy its shapes) + +Written for a fresh implementation agent. Facts about this repo were checked against `main` (`e82f7b0`) on 2026-09-12. Nothing here depends on an external service beyond the LLM providers and Voyage that the pipeline already uses. + +--- + +## 1. Goal + +Today the reader's ~230 standing interests live in a Scour OPML export (`data/scour-interests.opml`) plus an optional `## Interests` section of `profile.md`. They are parsed on every run, embedded, matched against each day's articles (`signals::interest_matches`), grouped for the system prompt by a hand-written keyword table (`profile/themes.rs`), and — since the last commit — shown to the reader as the **Matches:** line under every article header. They cannot be added from the dashboard, ratings never touch them, and nothing links from an interest to the articles that matched it. + +This plan makes interests a first-class record: + +1. **A database table is the only source of interests.** The OPML and the profile's `## Interests` section become one-time import inputs. +2. **An Interests dashboard page** lists them with category, weight, and match counts; adds new ones; filters by category; changes a category by hand; deletes. +3. **Categories** are kept (the grouped view the operator likes) and assigned by an LLM in one daily batch, only when uncategorized interests exist, with a manual **Categorize now** button. +4. **Every interest name in the web UI is a link** to the Articles page filtered (and sorted) to the articles that matched it, with a cut-off so unrelated articles never appear. The Articles page gains an interest filter and an interests column. +5. **Ratings nudge interests.** Each explicit rating credits the interests the article matched, scaled by how strongly it matched; the resulting per-interest weight is shown on the Interests page (default sort: highest first) and feeds curation as one bounded signal, so diversity is preserved. +6. **Backfill** is two CLI commands the operator runs once. + +## 2. Verified facts + +### Interests today + +- `profile::parse_interests(opml)` reads `` names; `parse_profile_str` strips `## Interests` from `profile.md` and returns its lines; `union_interests` dedupes case-insensitively (`src/curate/profile/mod.rs:36–150`). `load_standing_interests(opml, profile)` is called from `pipeline::prepare_features` (`src/pipeline.rs:946`), `embedding::plan_backfill` (`src/curate/embedding.rs:901`), and `profile::load_or_build`/`rebuild` (through `prompt_inputs`). `config.interests_opml` is referenced in 20 places in `src/` (mostly tests that write a temp OPML), `config.example.toml:29`, README (lines 103 and 369), and `docs/runbooks/curation-v2-migration.md`. +- The system prompt's "Standing interests" section groups names with `profile::group_into_themes` (`src/curate/profile/themes.rs`): twelve keyword themes plus "Other standing interests", deterministic, sorted. The profile dashboard page renders the same grouping (`src/web/dashboard/profile.rs:249`, template `dashboard/profile.html:42`). +- `interest_embeddings` (migration 0002) is keyed by the interest **name**; `EmbeddingService::interests(&[String])` returns cached vectors and fetches misses (`src/curate/embedding.rs:743`). A new name is embedded automatically the next time it is passed. +- `signals::interest_matches(article_embeddings, interest_embeddings)` (`src/curate/signals.rs:492`) computes cosine per interest × article, z-scores each interest across the day's embedded articles (std floored at 1e-3; raw top-1 cosine fallback under 30 articles), keeps the top `RECORDED_TOP = 3` per article as `TopInterest { name, z, cos }`, and scores `0.7·z₁ + 0.3·mean(top-3 z)`. `signals.top_interests` is serialized into `candidate_runs.signals_json` (`telemetry::serialize_signals`), copied into `Pick.top_interests` as bare names by the editor (`src/curate/editor.rs:598`), and rendered as the Matches line by `chapters::understanding` → `_understanding.html` (web) and `chapter.xhtml` / `in_this_issue.xhtml` (EPUB). Prompts list interests with `z ≥ 1.5` as "matches" (`triage.rs:135`, `assess.rs:192`, `editor.rs:159`). +- Interest names also appear on: the dashboard article detail (`dashboard/article.html:61`, "Top interests" table), `_signals_table.html:5` (runs and article history), and the Feeds page "Why" badges (`dashboard/feeds.html:14`, from `discovery::why`). +- Daily volume: ~365 entries → a few hundred articles per run; 512-dim `voyage-4-lite` vectors; `embedding_retention_days = 120`, so the embedding cache holds on the order of 30–50 k vectors (≈2 KB each). Scanning the cache per web request is not an option; a per-run cosine pass is milliseconds (§9.1 of the curation plan). + +### Ratings and the learned signals + +- `rating_events` is append-only; `Db::current_ratings(lookback_days) -> Vec` returns the latest explicit verdict per article with `value` (`loved` 1.0, `good` 0.35, `not_for_me` −1.0, `slop` per `feedback.slop_value`) and `event_at` (`src/db.rs:713`). Ratings are written by `web::rate::post` (dashboard and issue pages), `rate::record_explicit` (CLI), and `imports::run`. +- `signals::PreferenceState::load` (`src/curate/signals.rs:238`) builds the run's learned state from current ratings: decayed weights `value × 0.5^(age/half_life)`, kNN over rated embeddings, and **feed/author affinity** as Beta-smoothed rates `(up+1)/(up+down+2)` with a gate `gate(n, feed_floor=15, feed_full=40)`. Every cheap signal is `Option`, percentile-normalized over the eligible set (`normalize`, `PERCENTILE_SIGNALS`), blended with renormalized weights (`preliminary_blend`; `rank::calculate_utility_for` for the deep set). Absent is never zero. The slop-author factor scales both blends. Diversity is enforced downstream by `rank::shortlist` (cluster threshold 0.85, `per_cluster_cap = 2`, `utility_protected = 10`) and by the editor prompt. +- Signal names are enumerated in: `signals::PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend`'s candidate list, `rank::calculate_utility_for`'s weighted list, `telemetry::serialize_signals` (raw list) and `RENDERED_SIGNALS`, `dashboard::SIGNAL_NAMES`, `config::{PreliminaryWeights, UtilityWeights}`, and `settings::SETTINGS_HELP`. A new signal touches all of them. +- The Ratings dashboard page computes each verdict's feed credit on the fly with a pure function (`ratings::contribution`) rather than storing it. Follow that precedent. + +### Dashboard, jobs, CLI, config + +- One submodule per page group under `src/web/dashboard/`, each with `routes()`, merged in `dashboard::router()`; admin gating is applied by the caller. Nav tabs are hard-coded in `src/web/templates/layout.html` (keyed on `page.active_nav`); overview tiles in `dashboard/overview.html`. POST → flash → redirect via `jobs::set_flash`. `Page::is_admin()` is available in every template. Pager partial `dashboard/_pager.html`; client-side row filter `data-table-filter`. +- The Articles list (`src/web/dashboard/articles.rs`) builds `ARTICLE_INNER` (articles ⨝ best entry ⨝ latest `candidate_runs` row ⨝ assessments, with correlated subqueries for rating and publication) and wraps it in `SELECT * FROM (…) x WHERE 1=1 {clauses} ORDER BY {sort}`. Filters are allow-listed; sorts come from `ARTICLE_SORTS`; `Pager::new(pagination, path, &filters.params())` round-trips them. +- Jobs are a fixed catalogue (`jobs::Job`, `CATALOGUE`, `parse`, `name`, `description`, `takes_lock`, `dangerous`) run as `daily-epub-job@.service` and dispatched in `main::run_job`. The profile page starts `profile-rebuild` with a plain form posting to `/dashboard/jobs/profile-rebuild`. The only daily entry point is the `daily-epub-generate.timer` (05:30 America/New_York). +- `pipeline::build_llms` (`src/pipeline.rs:1133`) builds the taste-profile prompt, the clients, and runs the weekly learned-adjustments rebuild, then rebuilds the clients with the new prompt. An LLM step that must precede the prompt goes here. +- Config sections are `#[serde(deny_unknown_fields, default)]`; an unknown top-level key makes `Config` fail to load. The settings page has a hard-coded `GROUP_ORDER`, `PATH_KEYS`, and `SETTINGS_HELP`, with a test over the section list. Migrations: `sqlx::migrate!("./migrations")`; latest is `0012_article_publication.sql`, so the new file is `0013_interests.sql`. `db.rs` has a migration test asserting a table list (`src/db.rs:1720`). + +## 3. Options considered + +### 3.1 Where "articles matching interest X" comes from + +| Option | How | Verdict | +|---|---|---| +| A. Compute on request | Load the interest vector and every cached article embedding, dot, sort. | **Rejected.** 30–50 k × 2 KB per request; the dashboard is 1–7 ms today and should stay there. | +| B. Query `candidate_runs.signals_json` with `json_each` | The top-3 names are already persisted per run. | **Rejected.** A JSON scan over every telemetry row (hundreds of thousands, pruned at 180 days) per request, no index, and rows vanish with telemetry retention. | +| C. **A junction table written by the signals stage** | `article_interests(article_id, interest_id, cos, z)`: the same top-3 the run already computes, one row each, indexed by interest. The filter is an indexed join. | **Chosen.** ~1 k rows/day, zero extra computation, and the backfill is one pass over cached embeddings. | + +### 3.2 How an interest weight is computed and stored + +| Option | How | Verdict | +|---|---|---| +| A. Stored counters updated on every rating event | Add `up/down` columns to `interests`; the rating handlers, CLI, and importer bump them; write a backfill migration script. | **Rejected.** Four write paths to keep in sync, decay cannot be stored (it is a function of *now*), and it duplicates the ratings history that already exists. | +| B. **Derived on the fly from current ratings × matches** | One pure function over `current_ratings` and their `article_interests` rows, exactly like feed affinity. The run computes it in `PreferenceState::load`; the Interests page computes it on render. | **Chosen.** No new write path, always current, decay and lookback for free, backfill is "make sure rated articles have match rows". ≤ a few hundred ratings × 3 rows: microseconds. | +| C. Ask the weekly learned-adjustments rebuild to write per-interest weights | An LLM judges the rating history per interest. | **Rejected.** Non-deterministic, weekly, and the numbers would not be explainable. The existing prose rebuild already sees the ratings. | + +### 3.3 How the weight enters curation + +| Option | How | Verdict | +|---|---|---| +| A. **A new bounded cheap signal, `affinity`** | Per article: match-strength-weighted mean of its matched interests' weights; percentile-normalized; gated on rating count; small configured weight in the preliminary blend and the utility. | **Chosen.** Fits the existing design (absent ≠ zero, renormalized weights, `explain` shows it), and its influence is capped at its weight share, so one runaway interest cannot dominate. Diversity machinery downstream is untouched. | +| B. Multiply each interest's z by its weight before the top-3 is taken | Changes which interests appear as matches. | **Rejected.** Entangles "what does this article match" with "what does he like", and the Matches line would drift with ratings. | +| C. Annotate the prompt's Standing interests with ↑/↓ | Cheap and the LLM would use it. | **Deferred** (§9). Worth adding once the weights have a few weeks of ratings behind them; it is a five-line change on top of this plan. | +| D. A new deep-set admission retriever by affinity | Like the `interest` and `knn` quotas. | **Rejected.** More slots for the same signal; the blend fill already admits high-affinity articles. | + +### 3.4 Categories + +| Option | Verdict | +|---|---| +| A. **`interests.category TEXT NULL`; the category set is the distinct values** | **Chosen.** No FK, no second page, renaming is an `UPDATE`. | +| B. A separate `interest_categories` table with FK | Rejected: a table with one meaningful column. | +| C. Keep the keyword table in `themes.rs` | Rejected as the source of truth (the user wants DB-tracked interests and LLM categorization), but **kept for the one-time import** so the current grouping survives unchanged. | + +### 3.5 When the categorizer runs + +| Option | Verdict | +|---|---| +| A. **Inside `generate`, before the prompt is built, only when uncategorized interests exist; plus a catalogue job for the button** | **Chosen.** The morning timer is the only daily trigger that exists; a run without new interests spends nothing. | +| B. Its own systemd timer | Rejected: another unit to install for a call that takes seconds. | +| C. Synchronously in the Add handler | Rejected: an LLM call in a request path, and the user asked for a daily batch. | + +## 4. Design decisions (settled) + +| Topic | Decision | +|---|---| +| Match rule (one definition everywhere) | An interest **matches** an article when it is among the article's top three interests by z **and** `z ≥ MATCH_MIN_Z = 1.0`. `interest_matches` applies this when it truncates, so `signals.top_interests`, the Matches line, the stored rows, the Articles filter, and the interest weights all agree. The score formula is unchanged (computed before the cut). The prompts keep their stricter `z ≥ 1.5` for "matches interests". | +| Stored rows | `article_interests(article_id, interest_id, cos, z, run_id)`, upserted per eligible article per run (the same article can be eligible on consecutive days; the latest run wins; cosine is stable, z is that day's). Rows are never pruned (≈40 bytes each, ~1 k/day). | +| Link target and filter key | `/dashboard/articles?interest=`: names are unique (case-insensitive), human-readable, and `Pick.top_interests` already carries names, so no id has to travel through `issue_json`. The handler resolves the name to an id; an unknown name yields an empty list, never an error. When `interest` is set and `sort` is absent, the sort defaults to `match` (cosine descending). | +| Who sees links | The dashboard is admin-only, so the Matches line links only for admin viewers (`page.is_admin()`); readers and anonymous visitors see plain text as today. The EPUB never links. | +| Interest weight | Beta-smoothed rate over decayed, strength-scaled credits (§5.2): `(up + 1) / (up + down + 2)`, in (0, 1), 0.5 = no information. Shown with two decimals plus `up`, `down`, and the number of rated matches. | +| Match strength | `s = clamp(z / 3, 0, 1)`: a z of 3 credits the full rating, a bare match (z = 1) a third. | +| Curation signal | `affinity` (§5.3): signed, centred on zero, gated on the number of ratings that credited at least one interest (`affinity_floor = 15`, `affinity_full = 40`, same shape as the feed gate). Default weights: preliminary 0.10 (taken from `interest` 0.35→0.30 and `social` 0.10→0.05), utility 0.05 (taken from `knn` 0.15→0.10). | +| Diversity | Guaranteed by construction: the signal's share of the blend is its configured weight (≤ 10 % / 5 %), percentile normalization caps a favourite interest's articles at percentile 1.0 of that one signal, Beta smoothing means a single *loved* moves a weight from 0.50 to at most 0.67, ratings decay with the 60-day half-life, and `rank::shortlist`'s per-cluster cap and the editor's diversity instructions are unchanged. No new cap is needed. | +| Source of truth | The `interests` table. `interests_opml` is **removed** from config; `profile.md`'s `## Interests` section is still stripped by the parser but no longer read (the importer consumes it once; the profile page says so). | +| Prompt grouping | "Standing interests" groups by `interests.category` (sorted by category name, members sorted case-insensitively); `NULL` renders under the existing label "Other standing interests". `themes.rs` is used only by the importer. | +| Categorizer | One JSON call on the bulk provider over every uncategorized interest, given the existing category list; may create a category only when none fits. Unassigned names stay `NULL` and are retried the next day. Also the catalogue job `interests-categorize` behind the **Categorize now** button. | +| Interests page actions | Add (name + optional category), change category (per-row select + Save), Delete (confirm; cascades match rows, deletes the cached embedding). No rename (delete + add). | +| Backfill | `daily-epub interests import` (OPML + profile section → rows, categorized by the keyword table) and `daily-epub interests backfill` (match rows for every cached embedding). Weights need no backfill: they are derived. | +| Config | No new section. `[curation.ranking]` gains `affinity_floor`, `affinity_full`; the two weight tables gain `affinity`. | + +## 5. The numbers + +### 5.1 Match rows + +In `prepare_features` (`src/pipeline.rs`), after `signals::compute_all` and before the candidates are handed on: for every candidate with non-empty `signals.top_interests`, upsert `(article_id, interest_id, cos, z, run_id)`. One transaction, `INSERT … ON CONFLICT(article_id, interest_id) DO UPDATE SET cos, z, run_id`. Names map to ids through the `interests` rows loaded at the top of the stage (the same rows whose names go to `service.interests`). Best effort: a failure is a report warning, never a failed run. + +### 5.2 Interest weight (pure, `interests::rates`) + +Inputs: `current_ratings(rating_lookback_days)` and the `article_interests` rows of those articles. + +```text +for each current rating r on article a (value v_r, decay d_r = 0.5^(age_days / half_life_days)) + for each match row (a, i, z): + s = clamp(z / 3, 0, 1) + credit = v_r × d_r × s + up_i += max(credit, 0) + down_i += max(−credit, 0) + n_i += 1 +weight_i = (up_i + 1) / (up_i + down_i + 2) # 0.5 when n_i = 0 +``` + +`cleared` verdicts are already excluded by `current_ratings`; `slop` carries `feedback.slop_value` like everywhere else. A rating is *attributable to interests* when it credits at least one interest; that count drives the gate. + +### 5.3 The `affinity` signal (pure, in `PreferenceState`) + +```text +matched = the article's top_interests with n_i > 0 +affinity = Σ s_i × (weight_i − 0.5) / Σ s_i absent when matched is empty or the gate is 0 +gate = gate(attributable_interest_ratings, affinity_floor, affinity_full) +``` + +The raw value lives in [−0.5, 0.5]; it is percentile-normalized with the other cheap signals, weighted by `weights.preliminary.affinity × gate` in the blend and `weights.utility.affinity × gate` in the utility. `explain` and `_signals_table.html` show it like any other signal. The once-per-run preference log line gains `affinity gate … (n=…)`. + +## 6. Data model — `migrations/0013_interests.sql` + +```sql +CREATE TABLE interests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL COLLATE NOCASE UNIQUE, + category TEXT, -- NULL until categorized + created_at TEXT NOT NULL, + categorized_at TEXT +); +CREATE INDEX idx_interests_category ON interests(category); + +CREATE TABLE article_interests ( + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + interest_id INTEGER NOT NULL REFERENCES interests(id) ON DELETE CASCADE, + cos REAL NOT NULL, + z REAL NOT NULL, + run_id INTEGER, -- NULL for backfilled rows + PRIMARY KEY (article_id, interest_id) +); +CREATE INDEX idx_article_interests_interest ON article_interests(interest_id, cos DESC); +``` + +`interest_embeddings` stays keyed by name (no migration): a deleted interest also deletes its embedding row; a renamed one is a new interest. + +## 7. Implementation steps + +Sizes are rough line counts including tests. Steps 1–3 must be in order; 4–8 can proceed in parallel after 3; 9 last. + +1. **Migration + `src/interests.rs`** (~350). The table above; the db-list test in `db.rs` gains both tables. Module (following `discovery.rs`: data access, pure functions, and the LLM step in one file): + - `Interest { id, name, category, created_at, categorized_at }`; `list(db) -> Vec` (ordered by name), `add(db, name, category, now) -> Result` (trimmed, 1–80 chars, unique case-insensitive), `set_category(db, id, Option, now)`, `delete(db, id)` (also `DELETE FROM interest_embeddings WHERE interest = name`), `names(db) -> Vec`, `grouped(db) -> Vec<(String, Vec)>` (for the prompt), `uncategorized(db)`. + - `replace_matches(db, run_id, &[(ArticleId, &[TopInterest])], names→ids)`; `matches_for_articles(db, ids) -> Vec`; `match_counts(db) -> HashMap`. + - `pub fn rates(ratings: &[(ArticleId, value, decay)], rows: &[(ArticleId, interest_id, z)]) -> (HashMap, attributable)` per §5.2, and `Rate::weight()`. + - Tests: uniqueness is case-insensitive; `rates` matches hand-checked numbers (a loved z=3 → up 1.0; a good z=1.5 → up 0.175; a not-for-me z=0.9 → nothing, below the cut); delete cascades; `grouped` puts `NULL` last under "Other standing interests". + +2. **Signals** (~250). In `signals.rs`: `MATCH_MIN_Z = 1.0` applied inside `interest_matches` after truncation; `Signals.affinity: Option`; `"affinity"` in `PERCENTILE_SIGNALS`, `Signals::raw`, `preliminary_blend` (gate `affinity_gate`); `PreferenceState` gains `interest_rates: HashMap` (keyed by name — `TopInterest` carries names), `affinity_gate`, `attributable_interest_ratings`, `fn affinity(&self, top: &[TopInterest]) -> Option`, loaded in `load` from `interests::matches_for_articles(rated ids)` and exposed on `PreferenceSummary` and in `log`. `rank::calculate_utility_for` adds `("affinity", configured.affinity, candidate.signals.affinity_gate)`. `telemetry`: `serialize_signals` raw list and `RENDERED_SIGNALS`; `render_explain` prints it with the others. `dashboard::SIGNAL_NAMES` and `_signals_table.html` pick it up automatically once the name is in the list. Config: `RankingConfig { affinity_floor: 15, affinity_full: 40 }`, `PreliminaryWeights { affinity: 0.10, interest: 0.30, social: 0.05 }`, `UtilityWeights { affinity: 0.05, knn: 0.10 }`, `SETTINGS_HELP` lines, `config.example.toml`, README table. + Tests: the top-3 cut drops a z=0.4 third interest; affinity is absent under the gate and with no rated interests; a candidate matching a 0.8-weight interest outranks an otherwise identical one matching a 0.3-weight interest in the blend; blend weights still renormalize to 1. + +3. **Replace the OPML/profile plumbing** (~300, mostly deletions and test edits). `profile::load_standing_interests`, `parse_interests`, `union_interests`, and `prompt_inputs`' OPML argument go; `build`, `load_or_build`, `rebuild`, and `weekly_rebuild_if_due` take `grouped: Vec<(String, Vec)>` from `interests::grouped(db)` instead of `interests: &[String]` + `group_into_themes`. `pipeline::prepare_features` and `embedding::plan_backfill` take names from `interests::names(db)`. `themes.rs` stays but is only referenced by the importer (step 8); `pub use themes::group_into_themes` is removed from the profile module. Remove `Config.interests_opml` (struct, `Default`, `config check` line, the `deny_unknown_fields` implication is a rollout note in §10), `settings::PATH_KEYS`/`SETTINGS_HELP`, `config.example.toml`, README (lines 103, 369), and the runbook mention. Every test that writes a temp OPML instead inserts rows with `interests::add`. The profile page loses the OPML card and the "Extracted `## Interests` lines" preview; in their place one line: "N standing interests in M categories — manage them on the Interests page", and the editor note says the `## Interests` section is ignored. + +4. **Match rows in the run** (~80). `prepare_features` writes them per §5.1 (timing folded into the existing `signals` timing; count `interest_matches` on the report counts is optional — skip unless free). + +5. **Articles page** (~200). `ArticlesQuery.interest: Option` → `ArticleFilters.interest: Option<(i64, String)>` resolved by name (case-insensitive) in `from_query`'s caller (it needs the db; resolve in `list` before building filters, or make `from_query` async — pick the former). When set: `ARTICLE_INNER` gains `JOIN article_interests ai ON ai.article_id = a.id AND ai.interest_id = ?` (inside the inner query so `idx_article_interests_interest` drives it; the join is a `{interest_join}` placeholder that is empty otherwise) and exposes `ai.cos AS match_cos`; `ARTICLE_SORTS` gains `("match", "x.match_cos DESC, x.id DESC")`, which is the default when `interest` is set and `sort` is absent, and is ignored (falls back to `first_seen`) when it is not. Filter UI: a ` + @@ -18,11 +19,12 @@ {% include "dashboard/_pager.html" %} {% if articles.len() > 1 %}{% endif %}
- +{% for article in articles %} + @@ -32,6 +34,6 @@ -{% endfor %}{% if articles.is_empty() %}{% endif %}
first seentitlefeedwordslast stagereasonutilitytriagequalityfitratingpublished
first seentitlefeedinterestswordslast stagereasonutilitytriagequalityfitratingpublished
{{ article.first_seen }} {{ article.title }} {% if let Some(feed_id) = article.feed_id %}{{ article.feed }}{% else %}{{ article.feed }}{% endif %}{% for interest in article.interests %}{{ interest.name }}{% if !loop.last %} {% endif %}{% endfor %} {{ article.words }} {% if let Some(stage) = article.stage %}{{ stage }}{% if let Some(run_id) = article.run_id %} {% if let Some(date) = article.run_date %}{{ date }}{% else %}run {{ run_id }}{% endif %}{% endif %}{% else %}never considered{% endif %} {% if let Some(reason) = article.reason %}{{ reason }}{% endif %}{{ article.fit }} {% if let Some(rating) = article.rating %}{{ rating }}{% endif %} {% if let Some(date) = article.published %}{{ date }}{% endif %}
No articles match this filter.
+{% endfor %}{% if articles.is_empty() %}No articles match this filter.{% endif %} {% include "dashboard/_pager.html" %}
{% endblock %} diff --git a/src/web/templates/dashboard/feeds.html b/src/web/templates/dashboard/feeds.html index 905ac01..4255ab1 100644 --- a/src/web/templates/dashboard/feeds.html +++ b/src/web/templates/dashboard/feeds.html @@ -11,7 +11,7 @@ {% for row in rows %} {{ row.score }} {{ row.label }}{{ row.host }} -{% for interest in row.interests %}{{ interest }} {% endfor %}{% for article in row.articles %}
{{ article.title }}
{% endfor %} +{% for interest in row.interests %}{{ interest.name }} {% endfor %}{% for article in row.articles %}
{{ article.title }}
{% endfor %} {{ row.article_count }} {{ row.seen }} {% if status == "candidate" %}
{% else %}{{ row.decided_at }}{% if let Some(href) = row.miniflux_href %} · in Miniflux{% endif %}{% endif %} diff --git a/src/web/templates/dashboard/interests.html b/src/web/templates/dashboard/interests.html new file mode 100644 index 0000000..806cd86 --- /dev/null +++ b/src/web/templates/dashboard/interests.html @@ -0,0 +1,33 @@ +{% extends "layout.html" %}{% block content %}
+
+

Interests

+

{{ total }} standing interests. Weights use the last {{ lookback_days }} days of ratings with a {{ half_life_days }}-day half-life · affinity gate {{ affinity_gate }} · {{ attributable }} of {{ affinity_full }} attributable ratings.

+
{{ uncategorized }} uncategorized
{% if !jobs_enabled %} Jobs are disabled on this server (server.jobs_enabled = false); run daily-epub job run interests-categorize instead.{% endif %}
+ +

Add an interest

+ + +
+
+ +
+ + +
Reset
+
+{% if rows.len() > 1 %}{% endif %} +
+ +{% for row in rows %} + + + + + + + + + +{% endfor %}{% if rows.is_empty() %}{% endif %} +
interestcategoryweightupdownrated matchesmatched articlesaddedactions
{{ row.name }}{% if let Some(category) = row.category %}{{ category }}{% else %}uncategorized{% endif %}{{ row.weight }}{{ row.up }}{{ row.down }}{{ row.rated_matches }}{{ row.matched_articles }}{{ row.added }}
No interests match this filter.
+
{% endblock %} diff --git a/src/web/templates/dashboard/overview.html b/src/web/templates/dashboard/overview.html index e1e3e1b..0488cbc 100644 --- a/src/web/templates/dashboard/overview.html +++ b/src/web/templates/dashboard/overview.html @@ -11,6 +11,7 @@
Unrated picks{{ unrated.len() }}from the last three issues
Active jobs{{ active_jobs.len() }}all jobs
Feed candidates{{ feed_candidates }}review feeds
+
Interests{{ interests_total }}{{ uncategorized_interests }} uncategorized
Access requests{{ access_requests }}review requests
diff --git a/src/web/templates/layout.html b/src/web/templates/layout.html index 5d629ca..83abf07 100644 --- a/src/web/templates/layout.html +++ b/src/web/templates/layout.html @@ -47,6 +47,7 @@ Runs Articles Ratings + Interests Feeds Profile Stats From 7063b835db1574d5972a57b1cb90704d73d4c941 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Sun, 13 Sep 2026 05:55:06 +0000 Subject: [PATCH 8/8] interests backfill: use cached vectors when the Voyage key is unset Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01K9PrjtUS16PAQve8D4bHgc --- src/main.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index c27d6cf..8cbf2d0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -638,12 +638,19 @@ async fn cmd_interests_backfill(config: &Config, db: &Db) -> Result { .into_iter() .map(|interest| (interest.name, interest.id)) .collect::>(); - let service = if config.voyage.enabled { - embedding::EmbeddingService::real(db.clone(), config.voyage.clone()) - .context("building the Voyage client")? - } else { - embedding::EmbeddingService::cached_only(db.clone(), config.voyage.clone()) + // Like the pipeline: a missing key means cached vectors only, not a failure. + let service = match config.voyage.enabled { + true => match embedding::EmbeddingService::real(db.clone(), config.voyage.clone()) { + Ok(service) => Some(service), + Err(embedding::EmbeddingError::MissingApiKey) => None, + Err(error) => return Err(error).context("building the Voyage client"), + }, + false => None, }; + let cached_only = service.is_none(); + let service = service.unwrap_or_else(|| { + embedding::EmbeddingService::cached_only(db.clone(), config.voyage.clone()) + }); let interest_embeddings = service.interests(&names).await?; let mut matches = curate::signals::interest_matches(&article_embeddings, &interest_embeddings) .into_iter() @@ -653,12 +660,12 @@ async fn cmd_interests_backfill(config: &Config, db: &Db) -> Result { .collect::>(); matches.sort_by_key(|(article_id, _)| *article_id); let inserted = interests::insert_matches_if_absent(db, &matches, &ids).await?; - if config.voyage.enabled { - Ok(format!("wrote {inserted} interest match rows")) - } else { + if cached_only { Ok(format!( - "voyage disabled; used cached interest vectors and wrote {inserted} interest match rows" + "no Voyage client; used cached interest vectors and wrote {inserted} interest match rows" )) + } else { + Ok(format!("wrote {inserted} interest match rows")) } } @@ -1827,7 +1834,7 @@ mod tests { let message = cmd_interests_backfill(&config, &db).await.unwrap(); assert_eq!( message, - "voyage disabled; used cached interest vectors and wrote 1 interest match rows" + "no Voyage client; used cached interest vectors and wrote 1 interest match rows" ); let rows = sqlx::query( "SELECT article_id, interest_id, run_id, cos, z