From ea3b1413736214a7bc05a7166f302dda9a7845d0 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Wed, 2 Sep 2026 04:08:45 +0000 Subject: [PATCH] Curation v2 step 3: Voyage embeddings, cheap signals, candidate telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - embedding.rs: EmbeddingBackend + VoyageBackend, batched bounded-concurrency client with its own UsageMeter, f32 BLOB codec, article/interest embedding cache keyed by model, dimension and sha256 of the embedded text. - signals.rs: z-scored interest match, decayed rated-neighbour preference with the knn gate, feed affinity with the feed gate, social, text heuristic without social terms, mid-rank percentile normalizer, preliminary blend. - telemetry.rs: candidate_runs writer with §7.5 signals_json, explain and near-misses renderers, prune. - [voyage] and the full [curation.ranking] config with validation. - CLI: explain, features backfill|prune, generate --skip-embeddings. - Pipeline: hygiene rows, embed and signals stages before the old prefilter; same-date regeneration no longer excludes its own picks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe --- README.md | 73 +- config.example.toml | 63 ++ src/config.rs | 318 ++++++++ src/curate/embedding.rs | 1446 +++++++++++++++++++++++++++++++++++++ src/curate/mod.rs | 3 + src/curate/prefilter.rs | 25 +- src/curate/profile/mod.rs | 10 + src/curate/signals.rs | 981 +++++++++++++++++++++++++ src/curate/telemetry.rs | 962 ++++++++++++++++++++++++ src/db.rs | 39 + src/main.rs | 312 +++++++- src/pipeline.rs | 656 ++++++++++++++++- src/report.rs | 17 +- 13 files changed, 4888 insertions(+), 17 deletions(-) create mode 100644 src/curate/embedding.rs create mode 100644 src/curate/signals.rs create mode 100644 src/curate/telemetry.rs diff --git a/README.md b/README.md index c35b816..337bec0 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ selects, feed excerpts stand in for summaries) instead of losing the day's issue | Rust (2024 edition toolchain) | building | `cargo build --release` | | **Miniflux** with an API key | the only content source | Settings → API Keys. The client is read-only and never mutates read state. | | **DeepSeek API key** | curation + editorial | . Optional: `--skip-llm` runs the whole pipeline without it. | +| **Voyage AI API key** | article and interest embeddings behind the learned ranking signals | . Optional: without it (or with `--skip-embeddings`) the run uses cached vectors only and the learned signals are absent, never a penalty. | | A 32+ byte random secret | signs the article rating links | `openssl rand -hex 32` | | **BookOrbit** library + watched folder | *optional* — a richer library UI on top of the same folder | Delivery does not need it: `daily-epub serve` has its own OPDS catalog over `publish.epub_dir`. If you do run it, create a dedicated "The Daily EPUB" library, enable *Watch folders*, and point `publish.epub_dir` at it. | | **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node /cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. | @@ -70,12 +71,16 @@ sudo install -m0755 target/release/daily-epub /usr/local/bin/ ### Commands ``` -daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] +daily-epub generate [--date YYYY-MM-DD] [--dry-run] [--out DIR] [--max-articles N] [--skip-llm] [--skip-embeddings] daily-epub serve # rating endpoints + OPDS catalog + downloads daily-epub profile rebuild # regenerate learned profile adjustments daily-epub ratings list --days 90 daily-epub ratings set --article 42 --label loved --note "excellent" daily-epub ratings clear --url https://example.com/article +daily-epub explain --date YYYY-MM-DD (--article ID | --url URL) [--run-id N] +daily-epub explain --date YYYY-MM-DD --near-misses [N] +daily-epub features backfill [--days 30] [--rated-only] [--all] [--yes] +daily-epub features prune # stale embeddings + old candidate telemetry daily-epub backfill-social # re-poll social scores for recent articles daily-epub db migrate # run migrations (also automatic on every start) ``` @@ -85,6 +90,25 @@ articles, curates and **builds both EPUBs into `--out`**, but it does not copy t BookOrbit, does not run the retention sweep, does not write the `issues` row and does not advance the ingest watermark. It prints the lineup and the cost report. +`--skip-embeddings` reads the embedding cache but makes zero Voyage calls. + +`explain` answers "why was this (not) in the paper" from the `candidate_runs` +row the run persisted for every considered article: the stage it reached and the +reason it stopped, every raw and normalized signal with its presence and +effective weight, the top interests, the nearest rated neighbours, any cached +LLM assessments, and the editor's reason for a pick. `--url` canonicalizes the +address; an article that is not in the database at all is reported as never +ingested (a feed problem, not a ranking one). `--near-misses` lists the highest +ranked articles that were not selected. + +`features backfill` embeds the rated and published articles first (the learned +set), then the standing interests, then — only with `--all` — every other +article first seen in the window. It prints an estimate and asks before spending +more than 5M tokens unless `--yes`; a warm cache makes zero calls. `features +prune` drops embeddings of articles neither rated nor published that are older +than `curation.ranking.embedding_retention_days`, and `candidate_runs` rows of +runs older than `curation.ranking.telemetry_retention_days`. + --- ## Configuration @@ -131,6 +155,16 @@ Secrets belong in the environment file, never in the TOML. | `deepseek.price_input_per_mtok` | `0.14` | USD per 1M cache-miss input tokens (cost guardrail arithmetic). | | `deepseek.price_cached_input_per_mtok` | `0.0028` | USD per 1M prefix-cache-hit input tokens. | | `deepseek.price_output_per_mtok` | `0.28` | USD per 1M output tokens. | +| `voyage.enabled` | `true` | Embed articles and interests with Voyage AI. `false` ⇒ cached vectors only. | +| `voyage.base_url` | `https://api.voyageai.com/v1` | `POST {base_url}/embeddings`. | +| `voyage.model` | `voyage-4-lite` | Embedding model; changing it invalidates the cache. | +| `voyage.api_key` | — | **`DAILY_EPUB_VOYAGE__API_KEY`**. Absent ⇒ cached vectors only. | +| `voyage.output_dimension` | `512` | One of 256, 512, 1024, 2048. | +| `voyage.batch_size` | `32` | Texts per request. | +| `voyage.max_concurrent_requests` | `4` | Requests in flight. | +| `voyage.max_input_chars` | `60000` | Per-article cut, on a char boundary. | +| `voyage.max_daily_usd` | `0.50` | Runaway guard at $0.02/M tokens. | +| `curation.ranking.*` | see below | Every weight, quota, gate and threshold of the personalized ranker. | | `curation.always_include_feeds` | `[]` | Miniflux feed ids or URL substrings that can never be dropped. | | `curation.blocked_domains` | `[]` | Hosts excluded outright. | | `curation.paywall_domains` | `[]` | Extra paywalled hosts, merged with the built-in list (nytimes, wsj, ft, economist, …). | @@ -139,6 +173,27 @@ Secrets belong in the environment file, never in the TOML. | `curation.feedback.good_value` | `0.35` | Weight for a Good verdict. | | `curation.feedback.not_for_me_value` | `-1.0` | Weight for a Not for me verdict. | | `curation.feedback.verdicts_in_prompt` | `60` | Recent explicit verdicts included in the system prompt. | + +`[curation.ranking]` holds the ranker's tunables. The learned signals are +gated: `knn` (rated-neighbour preference) ramps from `knn_floor` (8) to +`knn_full` (25) rated articles with embeddings, `feed` (feed affinity) from +`feed_floor` (15) to `feed_full` (40) attributable ratings; below the floor the +signal is absent. Ratings decay with `rating_half_life_days` (60) over +`rating_lookback_days` (180); `neighbour_k` (5) neighbours per side and +`negative_coefficient` (0.75) shape the signal. `triage_max` (800), +`deep_keep` (120), `shortlist_keep` (60), `assessment_reuse_days` (3), +`semantic_min_words` (300), `exploration_slots` (5), `[curation.ranking.quotas]` +(`triage` 60 · `interest` 20 · `knn` 20), `[curation.ranking.weights.utility]` +and `[curation.ranking.diversity]` (`cluster_threshold` 0.85, `per_cluster_cap` +2, `utility_protected` 10) are validated now and drive the LLM triage, deep +assessment and diversification stages as they land. +`[curation.ranking.weights.preliminary]` (`interest` 0.35 · `knn` 0.25 · +`heuristic` 0.20 · `feed` 0.10 · `social` 0.10) blends the cheap signals; weights +are renormalized over the signals present for each article, so they need not sum +to 1. `embedding_retention_days` (120) and `telemetry_retention_days` (180) are +what `features prune` enforces. Validation: weights non-negative; `deep_keep ≥ +shortlist_keep ≥ target_article_count`; `*_full > *_floor`; `0 ≤ +cluster_threshold ≤ 1`; `per_cluster_cap ≥ 1`; batch sizes ≥ 1. | `publish.epub_dir` | `/srv/bookorbit/libraries/daily-epub` | Both EPUB editions land here by atomic copy, and this is the directory the OPDS feed lists. The editions are distinguished by a `(X4)` tag in **both** the filename and `dc:title` — libraries and OPDS clients list books by title, so the filename alone would make them look identical. Point a BookOrbit watched folder at it if you want its UI too. **Renamed from `bookorbit_dir`**; the old key is a hard config error. | | `publish.xtc_dir` | `/var/lib/daily-epub/xtc` | XTC artifacts. **Not** listed in the OPDS feed — CrossPoint cannot acquire them — but downloadable at `/files/xtc/` for sideloading. | | `xtc.enabled` | `true` | Set `false` to skip the converter entirely. | @@ -174,6 +229,7 @@ sudo -e /etc/daily-epub/config.toml # set publish dirs, xtc args, pub sudo tee /etc/daily-epub/env >/dev/null < world.rs the world briefing @@ -485,8 +542,14 @@ From spec §7, plus what implementation turned up: stack, so `curate/llm.rs` speaks the OpenAI-compatible wire protocol over the shared `reqwest` client instead, behind a `ChatBackend` trait. The dependency was removed. -- **Only DeepSeek is wired.** Another provider means another `ChatBackend` impl. -- **No embedding-based personal ranker yet** (spec §3.9 future work); the schema - is ready for it once ~200 ratings exist. +- **Only DeepSeek is wired for chat.** Another provider means another + `ChatBackend` impl. Voyage AI embeddings sit behind the analogous + `EmbeddingBackend` trait in `curate/embedding.rs`. +- **The learned signals are computed but do not yet gate selection.** Every + eligible article gets interest, rated-neighbour, feed-affinity, social and + heuristic signals persisted to `candidate_runs.signals_json` (read them with + `explain`), while the heuristic pre-filter still decides what the LLM sees. + The rated-neighbour and feed signals stay absent until their gates open + (8 and 15 ratings respectively). - **One reader, one issue per day.** There is no multi-user support and no weekly/retrospective edition (spec §6). diff --git a/config.example.toml b/config.example.toml index 74eeb33..64b8fe1 100644 --- a/config.example.toml +++ b/config.example.toml @@ -43,6 +43,20 @@ price_input_per_mtok = 0.14 price_cached_input_per_mtok = 0.0028 price_output_per_mtok = 0.28 +# Voyage AI embeddings behind the interest and rated-neighbour signals. Set +# `enabled = false` (or leave the key unset) and the paper still builds: the +# learned signals are simply absent, never a penalty. +[voyage] +enabled = true +base_url = "https://api.voyageai.com/v1" +model = "voyage-4-lite" +# api_key via DAILY_EPUB_VOYAGE__API_KEY env +output_dimension = 512 # 256 | 512 | 1024 | 2048 +batch_size = 32 +max_concurrent_requests = 4 +max_input_chars = 60000 # per article, cut on a char boundary +max_daily_usd = 0.50 # runaway guard ($0.02 / M tokens) + [curation] always_include_feeds = [] # miniflux feed ids or site urls blocked_domains = [] @@ -66,6 +80,55 @@ good_value = 0.35 not_for_me_value = -1.0 verdicts_in_prompt = 60 +# Every weight, quota, gate and threshold of the personalized ranker. The +# learned signals (`knn`, `feed`) contribute nothing until their gates open: +# the weight ramps linearly from `*_floor` to `*_full` rated articles. +[curation.ranking] +triage_max = 800 # eligible articles the triage LLM reads +deep_keep = 120 # deep-assessment set +shortlist_keep = 60 # what the editor sees +assessment_reuse_days = 3 +rating_lookback_days = 180 +rating_half_life_days = 60 +neighbour_k = 5 +negative_coefficient = 0.75 +knn_floor = 8 +knn_full = 25 +feed_floor = 15 +feed_full = 40 +semantic_min_words = 300 +exploration_slots = 5 +embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors +telemetry_retention_days = 180 # `features prune`: candidate_runs rows + +[curation.ranking.quotas] +triage = 60 +interest = 20 +knn = 20 + +# Weights need not sum to 1; they are renormalized over the present signals. +[curation.ranking.weights.preliminary] +interest = 0.35 +knn = 0.25 +heuristic = 0.20 +feed = 0.10 +social = 0.10 + +[curation.ranking.weights.utility] +quality = 0.40 +fit = 0.20 +knn = 0.15 +interest = 0.10 +feed = 0.05 +triage = 0.05 +social = 0.03 +heuristic = 0.02 + +[curation.ranking.diversity] +cluster_threshold = 0.85 +per_cluster_cap = 2 +utility_protected = 10 + [publish] # Where both EPUB editions land, and what the OPDS feed lists. BookOrbit is # optional — it just watches this folder if you run it. diff --git a/src/config.rs b/src/config.rs index 3c42a58..b2a21a8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -74,6 +74,7 @@ pub struct Config { pub miniflux: MinifluxConfig, pub deepseek: DeepseekConfig, + pub voyage: VoyageConfig, pub curation: CurationConfig, pub publish: PublishConfig, pub xtc: XtcConfig, @@ -97,6 +98,7 @@ impl Default for Config { profile_path: PathBuf::from("data/profile.md"), miniflux: MinifluxConfig::default(), deepseek: DeepseekConfig::default(), + voyage: VoyageConfig::default(), curation: CurationConfig::default(), publish: PublishConfig::default(), xtc: XtcConfig::default(), @@ -162,6 +164,38 @@ impl Default for DeepseekConfig { } } +/// `[voyage]` — embedding endpoint and cache shape (§4.3). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct VoyageConfig { + pub enabled: bool, + pub base_url: String, + pub model: String, + /// Supply via `DAILY_EPUB_VOYAGE__API_KEY`; never put it in the TOML. + pub api_key: Option, + pub output_dimension: usize, + pub batch_size: usize, + pub max_concurrent_requests: usize, + pub max_input_chars: usize, + pub max_daily_usd: f64, +} + +impl Default for VoyageConfig { + fn default() -> Self { + Self { + enabled: true, + base_url: "https://api.voyageai.com/v1".into(), + model: "voyage-4-lite".into(), + api_key: None, + output_dimension: 512, + batch_size: 32, + max_concurrent_requests: 4, + max_input_chars: 60_000, + max_daily_usd: 0.50, + } + } +} + /// `[curation]` — pre-filter and section palette (§3.5, §3.6). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] @@ -176,6 +210,7 @@ pub struct CurationConfig { /// The only section names the LLM may use (§3.6 stage B). pub sections: Vec, pub feedback: FeedbackConfig, + pub ranking: RankingConfig, } impl Default for CurationConfig { @@ -198,6 +233,154 @@ impl Default for CurationConfig { .map(|s| s.to_string()) .collect(), feedback: FeedbackConfig::default(), + ranking: RankingConfig::default(), + } + } +} + +/// `[curation.ranking]` — every weight, quota, gate and threshold of the +/// personalized ranker (plan §19). Steps 4–5 consume most of these; step 3 +/// uses the learned-signal gates, the preliminary weights and the retention +/// windows. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RankingConfig { + pub triage_max: usize, + pub deep_keep: usize, + pub shortlist_keep: usize, + pub assessment_reuse_days: i64, + pub rating_lookback_days: i64, + pub rating_half_life_days: f64, + pub neighbour_k: usize, + pub negative_coefficient: f64, + pub knn_floor: usize, + pub knn_full: usize, + pub feed_floor: usize, + pub feed_full: usize, + pub semantic_min_words: i64, + pub exploration_slots: usize, + pub embedding_retention_days: i64, + pub telemetry_retention_days: i64, + pub quotas: RankingQuotas, + pub weights: RankingWeights, + pub diversity: DiversityConfig, +} + +impl Default for RankingConfig { + fn default() -> Self { + Self { + triage_max: 800, + deep_keep: 120, + shortlist_keep: 60, + assessment_reuse_days: 3, + rating_lookback_days: 180, + rating_half_life_days: 60.0, + neighbour_k: 5, + negative_coefficient: 0.75, + knn_floor: 8, + knn_full: 25, + feed_floor: 15, + feed_full: 40, + semantic_min_words: 300, + exploration_slots: 5, + embedding_retention_days: 120, + telemetry_retention_days: 180, + quotas: RankingQuotas::default(), + weights: RankingWeights::default(), + diversity: DiversityConfig::default(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RankingQuotas { + pub triage: usize, + pub interest: usize, + pub knn: usize, +} + +impl Default for RankingQuotas { + fn default() -> Self { + Self { + triage: 60, + interest: 20, + knn: 20, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct RankingWeights { + pub preliminary: PreliminaryWeights, + pub utility: UtilityWeights, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct PreliminaryWeights { + pub interest: f64, + pub knn: f64, + pub heuristic: f64, + pub feed: f64, + pub social: f64, +} + +impl Default for PreliminaryWeights { + fn default() -> Self { + Self { + interest: 0.35, + knn: 0.25, + heuristic: 0.20, + feed: 0.10, + social: 0.10, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct UtilityWeights { + pub quality: f64, + pub fit: f64, + pub knn: f64, + pub interest: f64, + pub feed: f64, + pub triage: f64, + pub social: f64, + pub heuristic: f64, +} + +impl Default for UtilityWeights { + fn default() -> Self { + Self { + quality: 0.40, + fit: 0.20, + knn: 0.15, + interest: 0.10, + feed: 0.05, + triage: 0.05, + social: 0.03, + heuristic: 0.02, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct DiversityConfig { + pub cluster_threshold: f64, + pub per_cluster_cap: usize, + pub utility_protected: usize, +} + +impl Default for DiversityConfig { + fn default() -> Self { + Self { + cluster_threshold: 0.85, + per_cluster_cap: 2, + utility_protected: 10, } } } @@ -376,6 +559,73 @@ impl Config { "prefilter_keep must be >= target_article_count".into(), )); } + let ranking = &self.curation.ranking; + if ranking.deep_keep < ranking.shortlist_keep + || ranking.shortlist_keep < self.target_article_count + { + return Err(ConfigError::Invalid( + "curation.ranking must satisfy deep_keep >= shortlist_keep >= target_article_count" + .into(), + )); + } + if ranking.knn_full <= ranking.knn_floor || ranking.feed_full <= ranking.feed_floor { + return Err(ConfigError::Invalid( + "curation.ranking *_full must be > *_floor >= 0".into(), + )); + } + if !(0.0..=1.0).contains(&ranking.diversity.cluster_threshold) { + return Err(ConfigError::Invalid( + "curation.ranking.diversity.cluster_threshold must be between 0 and 1".into(), + )); + } + if ranking.diversity.per_cluster_cap == 0 { + return Err(ConfigError::Invalid( + "curation.ranking.diversity.per_cluster_cap must be >= 1".into(), + )); + } + let preliminary = &ranking.weights.preliminary; + let utility = &ranking.weights.utility; + let weights = [ + preliminary.interest, + preliminary.knn, + preliminary.heuristic, + preliminary.feed, + preliminary.social, + utility.quality, + utility.fit, + utility.knn, + utility.interest, + utility.feed, + utility.triage, + utility.social, + utility.heuristic, + ]; + if weights + .iter() + .any(|weight| !weight.is_finite() || *weight < 0.0) + { + return Err(ConfigError::Invalid( + "curation.ranking weights must be finite and non-negative".into(), + )); + } + if self.deepseek.score_batch_size == 0 + || self.voyage.batch_size == 0 + || self.voyage.max_concurrent_requests == 0 + { + return Err(ConfigError::Invalid( + "provider batch sizes must be >= 1".into(), + )); + } + if ![256, 512, 1024, 2048].contains(&self.voyage.output_dimension) { + return Err(ConfigError::Invalid( + "voyage.output_dimension must be one of 256, 512, 1024, 2048".into(), + )); + } + if ranking.rating_half_life_days <= 0.0 || !ranking.rating_half_life_days.is_finite() { + return Err(ConfigError::Invalid( + "curation.ranking.rating_half_life_days must be > 0".into(), + )); + } if self.curation.sections.is_empty() { return Err(ConfigError::Invalid( "curation.sections must not be empty".into(), @@ -442,8 +692,12 @@ mod tests { jail.set_env("DAILY_EPUB_MINIFLUX__API_KEY", "secret-token"); jail.set_env("DAILY_EPUB_TARGET_ARTICLE_COUNT", "12"); jail.set_env("DAILY_EPUB_SERVER__HMAC_SECRET", "hunter2"); + jail.set_env("DAILY_EPUB_VOYAGE__API_KEY", "voyage-key"); + jail.set_env("DAILY_EPUB_VOYAGE__ENABLED", "false"); let c = Config::load(None).map_err(|e| figment::Error::from(e.to_string()))?; + assert_eq!(c.voyage.api_key.as_deref(), Some("voyage-key")); + assert!(!c.voyage.enabled); // from file assert_eq!(c.lookback_hours, 30); assert!(!c.world_briefing); @@ -500,6 +754,70 @@ mod tests { assert_eq!(c.deepseek.base_url, "https://api.deepseek.com/v1"); } + #[test] + fn voyage_and_ranking_defaults_and_validation() { + let cfg = Config::default(); + assert!(cfg.voyage.enabled); + assert_eq!(cfg.voyage.base_url, "https://api.voyageai.com/v1"); + assert_eq!(cfg.voyage.model, "voyage-4-lite"); + assert_eq!(cfg.voyage.output_dimension, 512); + assert_eq!(cfg.voyage.batch_size, 32); + assert_eq!(cfg.voyage.max_concurrent_requests, 4); + assert_eq!(cfg.voyage.max_input_chars, 60_000); + assert_eq!(cfg.voyage.max_daily_usd, 0.50); + let ranking = &cfg.curation.ranking; + assert_eq!( + ( + ranking.triage_max, + ranking.deep_keep, + ranking.shortlist_keep + ), + (800, 120, 60) + ); + assert_eq!((ranking.knn_floor, ranking.knn_full), (8, 25)); + assert_eq!((ranking.feed_floor, ranking.feed_full), (15, 40)); + assert_eq!(ranking.rating_half_life_days, 60.0); + assert_eq!(ranking.negative_coefficient, 0.75); + assert_eq!(ranking.weights.preliminary.interest, 0.35); + assert_eq!(ranking.weights.utility.quality, 0.40); + assert_eq!(ranking.diversity.per_cluster_cap, 2); + assert_eq!(ranking.embedding_retention_days, 120); + assert_eq!(ranking.telemetry_retention_days, 180); + cfg.validate().unwrap(); + + let mut bad = Config::default(); + bad.voyage.output_dimension = 300; + assert!(bad.validate().is_err(), "dimension must be a Voyage size"); + let mut bad = Config::default(); + bad.voyage.batch_size = 0; + assert!(bad.validate().is_err()); + let mut bad = Config::default(); + bad.curation.ranking.weights.preliminary.knn = -0.1; + assert!(bad.validate().is_err(), "weights are non-negative"); + let mut bad = Config::default(); + bad.curation.ranking.knn_full = bad.curation.ranking.knn_floor; + assert!(bad.validate().is_err(), "*_full must exceed *_floor"); + let mut bad = Config::default(); + bad.curation.ranking.shortlist_keep = bad.curation.ranking.deep_keep + 1; + assert!(bad.validate().is_err(), "deep_keep >= shortlist_keep"); + let mut bad = Config::default(); + bad.curation.ranking.shortlist_keep = bad.target_article_count - 1; + assert!(bad.validate().is_err(), "shortlist_keep >= target"); + let mut bad = Config::default(); + bad.curation.ranking.diversity.cluster_threshold = 1.5; + assert!(bad.validate().is_err()); + let mut bad = Config::default(); + bad.curation.ranking.diversity.per_cluster_cap = 0; + assert!(bad.validate().is_err()); + + // Unknown keys inside a known section fail loudly. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, "[voyage]\nenabled = true\nnot_a_key = 1\n").unwrap(); + let err = Config::load(Some(&path)).expect_err("unknown voyage key must be rejected"); + assert!(err.to_string().contains("not_a_key"), "{err}"); + } + #[test] fn validation_rejects_nonsense() { assert!( diff --git a/src/curate/embedding.rs b/src/curate/embedding.rs new file mode 100644 index 0000000..710cfa4 --- /dev/null +++ b/src/curate/embedding.rs @@ -0,0 +1,1446 @@ +//! Voyage embeddings, the f32 BLOB codec, and the SQLite cache (plan §4.3, +//! §7.1–7.2, §16 `features backfill`). +//! +//! The network is reached through an [`EmbeddingBackend`] so tests can inject +//! canned vectors ([`MockBackend`]) without touching the wire, mirroring +//! `ChatBackend` in `llm.rs`. Nothing here is fatal to a run: a failed batch +//! leaves its articles without embeddings and the caller carries on (§17). +//! Raw vectors never reach logs or reports. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use futures::{StreamExt as _, stream}; +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use sqlx::Row as _; + +use crate::config::{Config, VoyageConfig}; +use crate::curate::{approx_tokens, profile, prompt_text}; +use crate::db::{Db, fmt_ts}; +use crate::http::RetryPolicy; +use crate::types::{Article, ArticleId}; + +/// The only place the Voyage key comes from (§4.3). +pub const VOYAGE_API_KEY_ENV: &str = "DAILY_EPUB_VOYAGE__API_KEY"; +/// USD per million tokens, `voyage-4-lite` (§4.3, verified 2026-08-17). +pub const VOYAGE_PRICE_PER_MTOK: f64 = 0.02; +/// `features backfill` asks before spending more than this without `--yes` (§16). +pub const BACKFILL_CONFIRM_TOKENS: i64 = 5_000_000; +const EMBEDDING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +#[derive(Debug, thiserror::Error)] +pub enum EmbeddingError { + #[error("voyage api key is not configured (set DAILY_EPUB_VOYAGE__API_KEY)")] + MissingApiKey, + #[error("voyage request failed: {0}")] + Api(String), + /// A 5xx/429/network failure: worth retrying. + #[error("voyage request failed (transient): {0}")] + Transient(String), + #[error("voyage response index {index} is invalid for {len} inputs")] + InvalidIndex { index: usize, len: usize }, + #[error("voyage response contained duplicate index {0}")] + DuplicateIndex(usize), + #[error("voyage response returned {actual} vectors for {expected} inputs")] + ResponseLength { expected: usize, actual: usize }, + #[error("embedding dimension mismatch: expected {expected}, got {actual}")] + Dimension { expected: usize, actual: usize }, + #[error("embedding contains a non-finite value")] + NonFinite, + #[error("embedding blob length {actual} does not match dimension {dimension}")] + BlobLength { dimension: usize, actual: usize }, + #[error("voyage daily cost ceiling of ${limit:.2} reached")] + BudgetExceeded { limit: f64 }, + #[error(transparent)] + Db(#[from] crate::db::DbError), + #[error(transparent)] + Sqlx(#[from] sqlx::Error), +} + +impl EmbeddingError { + fn is_transient(&self) -> bool { + matches!(self, Self::Transient(_)) + } +} + +/// Voyage's `input_type`: documents for articles, queries for interests (§7.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum InputType { + Document, + Query, +} + +#[derive(Debug, Clone)] +pub struct EmbeddingRequest { + pub input: Vec, + pub model: String, + pub input_type: InputType, + pub output_dimension: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct IndexedEmbedding { + pub index: usize, + pub embedding: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct EmbeddingCompletion { + /// Ordered by `index`, one per input. + pub data: Vec, + pub total_tokens: i64, +} + +/// Network seam matching `ChatBackend`; tests inject canned completions. +pub trait EmbeddingBackend: std::fmt::Debug + Send + Sync { + fn embed<'a>( + &'a self, + request: EmbeddingRequest, + ) -> BoxFuture<'a, Result>; +} + +/// `POST {base_url}/embeddings` with the bearer key from +/// `DAILY_EPUB_VOYAGE__API_KEY` (§4.3). The key is never logged. +#[derive(Debug, Clone)] +pub struct VoyageBackend { + http: reqwest::Client, + endpoint: String, + api_key: String, +} + +impl VoyageBackend { + pub fn new(config: &VoyageConfig) -> Result { + // The config field is how figment carries the env var; the direct + // read covers callers that built the config by hand. + let api_key = config + .api_key + .clone() + .or_else(|| std::env::var(VOYAGE_API_KEY_ENV).ok()) + .filter(|value| !value.trim().is_empty()) + .ok_or(EmbeddingError::MissingApiKey)?; + let http = crate::http::build_client(EMBEDDING_TIMEOUT) + .map_err(|error| EmbeddingError::Api(format!("building HTTP client: {error}")))?; + Ok(Self { + http, + endpoint: format!("{}/embeddings", config.base_url.trim_end_matches('/')), + api_key, + }) + } +} + +#[derive(Debug, Serialize)] +struct ApiRequest<'a> { + input: &'a [String], + model: &'a str, + input_type: InputType, + truncation: bool, + output_dimension: usize, + output_dtype: &'static str, +} + +#[derive(Debug, Deserialize)] +struct ApiResponse { + #[serde(default)] + data: Vec, + #[serde(default)] + usage: ApiUsage, +} + +#[derive(Debug, Deserialize)] +struct ApiEmbedding { + index: usize, + embedding: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct ApiUsage { + #[serde(default)] + total_tokens: i64, +} + +impl EmbeddingBackend for VoyageBackend { + fn embed<'a>( + &'a self, + request: EmbeddingRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let response = self + .http + .post(&self.endpoint) + .bearer_auth(&self.api_key) + .json(&ApiRequest { + input: &request.input, + model: &request.model, + input_type: request.input_type, + truncation: true, + output_dimension: request.output_dimension, + output_dtype: "float", + }) + .send() + .await + .map_err(|error| { + if crate::http::is_retryable(&error) { + EmbeddingError::Transient(error.to_string()) + } else { + EmbeddingError::Api(error.to_string()) + } + })?; + let status = response.status(); + if !status.is_success() { + let detail = response.text().await.unwrap_or_default(); + let message = format!("{status}: {}", detail.chars().take(500).collect::()); + return Err(if status.is_server_error() || status.as_u16() == 429 { + EmbeddingError::Transient(message) + } else { + EmbeddingError::Api(message) + }); + } + let parsed: ApiResponse = response + .json() + .await + .map_err(|error| EmbeddingError::Api(format!("decoding response: {error}")))?; + map_response(parsed, request.input.len(), request.output_dimension) + }) + } +} + +/// Order the response by `index` and reject short, long, duplicate or +/// malformed vectors (§4.3). +fn map_response( + response: ApiResponse, + expected: usize, + dimension: usize, +) -> Result { + let actual = response.data.len(); + if actual != expected { + return Err(EmbeddingError::ResponseLength { expected, actual }); + } + let mut ordered: Vec> = vec![None; expected]; + for item in response.data { + if item.index >= expected { + return Err(EmbeddingError::InvalidIndex { + index: item.index, + len: expected, + }); + } + validate_vector(&item.embedding, dimension)?; + let index = item.index; + if ordered[index] + .replace(IndexedEmbedding { + index, + embedding: item.embedding, + }) + .is_some() + { + return Err(EmbeddingError::DuplicateIndex(index)); + } + } + Ok(EmbeddingCompletion { + data: ordered.into_iter().flatten().collect(), + total_tokens: response.usage.total_tokens.max(0), + }) +} + +/// Voyage token meter with the `max_daily_usd` runaway guard (§5). +#[derive(Debug, Clone)] +pub struct UsageMeter { + tokens: Arc>, + max_daily_usd: f64, +} + +impl UsageMeter { + pub fn new(max_daily_usd: f64) -> Self { + Self { + tokens: Arc::new(Mutex::new(0)), + max_daily_usd, + } + } + + pub fn total_tokens(&self) -> i64 { + match self.tokens.lock() { + Ok(tokens) => *tokens, + Err(poisoned) => *poisoned.into_inner(), + } + } + + pub fn cost_usd(&self) -> f64 { + cost_for_tokens(self.total_tokens()) + } + + fn check(&self) -> Result<(), EmbeddingError> { + if self.max_daily_usd > 0.0 && self.cost_usd() >= self.max_daily_usd { + Err(EmbeddingError::BudgetExceeded { + limit: self.max_daily_usd, + }) + } else { + Ok(()) + } + } + + fn record(&self, tokens: i64) { + match self.tokens.lock() { + Ok(mut total) => *total += tokens.max(0), + Err(poisoned) => *poisoned.into_inner() += tokens.max(0), + } + } +} + +pub fn cost_for_tokens(tokens: i64) -> f64 { + tokens as f64 * VOYAGE_PRICE_PER_MTOK / 1_000_000.0 +} + +/// Batching, bounded concurrency, retries and metering over a backend (§4.3). +#[derive(Debug, Clone)] +pub struct EmbeddingClient { + config: VoyageConfig, + backend: Arc, + retry: RetryPolicy, + pub meter: UsageMeter, +} + +impl EmbeddingClient { + pub fn new(config: &VoyageConfig) -> Result { + Ok(Self::with_backend( + config.clone(), + Arc::new(VoyageBackend::new(config)?), + )) + } + + pub fn with_backend(config: VoyageConfig, backend: Arc) -> Self { + Self { + meter: UsageMeter::new(config.max_daily_usd), + config, + backend, + retry: RetryPolicy::default(), + } + } + + /// Embed every text in `batch_size` chunks, at most `max_concurrent_requests` + /// in flight. A failed batch yields `None` for its texts and is logged. + pub async fn embed_many( + &self, + texts: &[String], + input_type: InputType, + ) -> Vec>> { + if texts.is_empty() { + return Vec::new(); + } + let batch_size = self.config.batch_size.max(1); + let batches = texts + .chunks(batch_size) + .enumerate() + .map(|(batch_index, chunk)| (batch_index * batch_size, chunk.to_vec())); + let client = self.clone(); + let mut completed = stream::iter(batches.map(move |(offset, input)| { + let client = client.clone(); + async move { + let result = client.embed_batch(input, input_type).await; + (offset, result) + } + })) + .buffer_unordered(self.config.max_concurrent_requests.max(1)); + + let mut output = vec![None; texts.len()]; + while let Some((offset, result)) = completed.next().await { + match result { + Ok(vectors) => { + for (index, vector) in vectors.into_iter().enumerate() { + if let Some(slot) = output.get_mut(offset + index) { + *slot = Some(vector); + } + } + } + Err(error) => { + tracing::warn!(%error, offset, "voyage batch failed; leaving its embeddings absent") + } + } + } + output + } + + async fn embed_batch( + &self, + input: Vec, + input_type: InputType, + ) -> Result>, EmbeddingError> { + self.meter.check()?; + let request = EmbeddingRequest { + input, + model: self.config.model.clone(), + input_type, + output_dimension: self.config.output_dimension, + }; + let completion = self + .retry + .run("voyage embeddings", EmbeddingError::is_transient, || { + self.backend.embed(request.clone()) + }) + .await?; + self.meter.record(completion.total_tokens); + Ok(completion + .data + .into_iter() + .map(|item| item.embedding) + .collect()) + } +} + +// --------------------------------------------------------------------------- +// Codec and arithmetic (§7.1) +// --------------------------------------------------------------------------- + +/// f32 little-endian, `dimension * 4` bytes; non-finite values are rejected. +pub fn encode_blob(vector: &[f32]) -> Result, EmbeddingError> { + if vector.iter().any(|value| !value.is_finite()) { + return Err(EmbeddingError::NonFinite); + } + let mut bytes = Vec::with_capacity(vector.len() * 4); + for value in vector { + bytes.extend_from_slice(&value.to_le_bytes()); + } + Ok(bytes) +} + +pub fn decode_blob(bytes: &[u8], dimension: usize) -> Result, EmbeddingError> { + if bytes.len() != dimension.saturating_mul(4) { + return Err(EmbeddingError::BlobLength { + dimension, + actual: bytes.len(), + }); + } + let mut vector = Vec::with_capacity(dimension); + for chunk in bytes.chunks_exact(4) { + let value = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + if !value.is_finite() { + return Err(EmbeddingError::NonFinite); + } + vector.push(value); + } + Ok(vector) +} + +/// Dot product (= cosine, Voyage vectors are unit-normalized) with a dimension check. +pub fn dot(left: &[f32], right: &[f32]) -> Result { + if left.len() != right.len() { + return Err(EmbeddingError::Dimension { + expected: left.len(), + actual: right.len(), + }); + } + Ok(left + .iter() + .zip(right) + .map(|(a, b)| f64::from(*a) * f64::from(*b)) + .sum()) +} + +fn validate_vector(vector: &[f32], dimension: usize) -> Result<(), EmbeddingError> { + if vector.len() != dimension { + return Err(EmbeddingError::Dimension { + expected: dimension, + actual: vector.len(), + }); + } + encode_blob(vector).map(|_| ()) +} + +// --------------------------------------------------------------------------- +// Embedded text (§7.1) +// --------------------------------------------------------------------------- + +/// `"Title: {title}\n\n{plain body}"`, whitespace collapsed, cut at +/// `max_chars` on a char boundary. Deliberately no feed name, author or scores. +pub fn article_input(article: &Article, max_chars: usize) -> String { + let body = prompt_text(&article.content_html); + let title = article + .title + .split_whitespace() + .collect::>() + .join(" "); + truncate_chars(&format!("Title: {title}\n\n{body}"), max_chars) +} + +fn truncate_chars(text: &str, max_chars: usize) -> String { + match text.char_indices().nth(max_chars) { + Some((byte, _)) => text[..byte].to_string(), + None => text.to_string(), + } +} + +/// `sha256` of the embedded text, the cache key alongside model and dimension. +pub fn input_hash(text: &str) -> String { + hex::encode(Sha256::digest(text.as_bytes())) +} + +// --------------------------------------------------------------------------- +// Cache orchestration (§7.1, §7.2) +// --------------------------------------------------------------------------- + +/// A cache miss waiting for the network. +#[derive(Debug, Clone)] +struct ArticleMiss { + article_id: ArticleId, + text: String, + hash: String, +} + +/// The `article_embeddings` / `interest_embeddings` cache in front of a client. +/// +/// Without a client (`--skip-embeddings`, Voyage disabled, no key) it answers +/// from the cache only and never touches the network. +#[derive(Debug, Clone)] +pub struct EmbeddingService { + db: Db, + config: VoyageConfig, + client: Option, +} + +impl EmbeddingService { + pub fn cached_only(db: Db, config: VoyageConfig) -> Self { + Self { + db, + config, + client: None, + } + } + + pub fn real(db: Db, config: VoyageConfig) -> Result { + let client = EmbeddingClient::new(&config)?; + Ok(Self::with_client(db, config, client)) + } + + pub fn with_client(db: Db, config: VoyageConfig, client: EmbeddingClient) -> Self { + Self { + db, + config, + client: Some(client), + } + } + + pub fn config(&self) -> &VoyageConfig { + &self.config + } + + /// `None` when the service is cache-only. + pub fn meter(&self) -> Option<&UsageMeter> { + self.client.as_ref().map(|client| &client.meter) + } + + pub fn has_client(&self) -> bool { + self.client.is_some() + } + + /// Split the articles into cached vectors and misses (no network). + async fn lookup_articles( + &self, + articles: &[Article], + ) -> Result<(HashMap>, Vec), EmbeddingError> { + let mut found = HashMap::new(); + let mut misses = Vec::new(); + for article in articles { + let text = article_input(article, self.config.max_input_chars); + let hash = input_hash(&text); + let row = sqlx::query( + "SELECT embedding FROM article_embeddings + WHERE article_id = ? AND model = ? AND dimension = ? AND input_hash = ?", + ) + .bind(article.id) + .bind(&self.config.model) + .bind(self.config.output_dimension as i64) + .bind(&hash) + .fetch_optional(self.db.pool()) + .await?; + let cached = row.and_then(|row| { + decode_blob( + &row.get::, _>("embedding"), + self.config.output_dimension, + ) + .map_err(|error| { + tracing::warn!(article_id = article.id, %error, "ignoring a malformed cached embedding") + }) + .ok() + }); + match cached { + Some(vector) => { + found.insert(article.id, vector); + } + None => misses.push(ArticleMiss { + article_id: article.id, + text, + hash, + }), + } + } + Ok((found, misses)) + } + + /// Articles with no usable cached vector, with their estimated token cost. + pub async fn uncached_articles( + &self, + articles: &[Article], + ) -> Result, EmbeddingError> { + let (_, misses) = self.lookup_articles(articles).await?; + Ok(misses + .into_iter() + .map(|miss| (miss.article_id, approx_tokens(&miss.text) as i64)) + .collect()) + } + + /// Cached-or-fetched vectors for every article that has one (§7.1). + pub async fn articles( + &self, + articles: &[Article], + ) -> Result>, EmbeddingError> { + let (mut found, misses) = self.lookup_articles(articles).await?; + let Some(client) = self.client.as_ref() else { + return Ok(found); + }; + if misses.is_empty() { + return Ok(found); + } + let texts = misses + .iter() + .map(|miss| miss.text.clone()) + .collect::>(); + let vectors = client.embed_many(&texts, InputType::Document).await; + let now = fmt_ts(Timestamp::now()); + for (miss, vector) in misses.into_iter().zip(vectors) { + let Some(vector) = vector else { continue }; + let blob = encode_blob(&vector)?; + sqlx::query( + "INSERT INTO article_embeddings + (article_id, model, dimension, input_hash, embedding, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(article_id) DO UPDATE SET + model = excluded.model, dimension = excluded.dimension, + input_hash = excluded.input_hash, embedding = excluded.embedding, + created_at = excluded.created_at", + ) + .bind(miss.article_id) + .bind(&self.config.model) + .bind(self.config.output_dimension as i64) + .bind(&miss.hash) + .bind(blob) + .bind(&now) + .execute(self.db.pool()) + .await?; + found.insert(miss.article_id, vector); + } + Ok(found) + } + + async fn lookup_interests( + &self, + interests: &[String], + ) -> Result<(HashMap>, Vec), EmbeddingError> { + let mut found = HashMap::new(); + let mut misses = Vec::new(); + for interest in interests { + let row = sqlx::query( + "SELECT embedding FROM interest_embeddings + WHERE interest = ? AND model = ? AND dimension = ?", + ) + .bind(interest) + .bind(&self.config.model) + .bind(self.config.output_dimension as i64) + .fetch_optional(self.db.pool()) + .await?; + let cached = row.and_then(|row| { + decode_blob( + &row.get::, _>("embedding"), + self.config.output_dimension, + ) + .map_err(|error| { + tracing::warn!(interest, %error, "ignoring a malformed cached interest embedding") + }) + .ok() + }); + match cached { + Some(vector) => { + found.insert(interest.clone(), vector); + } + None => misses.push(interest.clone()), + } + } + Ok((found, misses)) + } + + pub async fn uncached_interests( + &self, + interests: &[String], + ) -> Result, EmbeddingError> { + Ok(self.lookup_interests(interests).await?.1) + } + + /// Cached-or-fetched query vectors for the bare interest strings (§7.2). + pub async fn interests( + &self, + interests: &[String], + ) -> Result>, EmbeddingError> { + let (mut found, misses) = self.lookup_interests(interests).await?; + let Some(client) = self.client.as_ref() else { + return Ok(found); + }; + if misses.is_empty() { + return Ok(found); + } + let vectors = client.embed_many(&misses, InputType::Query).await; + let now = fmt_ts(Timestamp::now()); + for (interest, vector) in misses.into_iter().zip(vectors) { + let Some(vector) = vector else { continue }; + let blob = encode_blob(&vector)?; + sqlx::query( + "INSERT INTO interest_embeddings + (interest, model, dimension, embedding, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(interest) DO UPDATE SET + model = excluded.model, dimension = excluded.dimension, + embedding = excluded.embedding, created_at = excluded.created_at", + ) + .bind(&interest) + .bind(&self.config.model) + .bind(self.config.output_dimension as i64) + .bind(blob) + .bind(&now) + .execute(self.db.pool()) + .await?; + found.insert(interest, vector); + } + Ok(found) + } +} + +/// Cached vectors for the given ids under the configured model and dimension, +/// whatever text they were computed from (the rated set, §9.2). +pub async fn load_article_embeddings( + db: &Db, + config: &VoyageConfig, + article_ids: &[ArticleId], +) -> Result>, EmbeddingError> { + let mut output = HashMap::new(); + for article_id in article_ids { + let row = sqlx::query( + "SELECT embedding FROM article_embeddings + WHERE article_id = ? AND model = ? AND dimension = ?", + ) + .bind(article_id) + .bind(&config.model) + .bind(config.output_dimension as i64) + .fetch_optional(db.pool()) + .await?; + if let Some(row) = row { + match decode_blob(&row.get::, _>("embedding"), config.output_dimension) { + Ok(vector) => { + output.insert(*article_id, vector); + } + Err(error) => tracing::warn!(article_id, %error, "ignoring a malformed embedding"), + } + } + } + Ok(output) +} + +// --------------------------------------------------------------------------- +// `features backfill` (§16) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default)] +pub struct BackfillOptions { + /// Window for published and (under `--all`) other articles, in days. + pub days: i64, + /// Only the rated set. + pub rated_only: bool, + /// Also every other article first seen inside the window. + pub all: bool, +} + +/// What a backfill would embed: cache misses only, in priority order. +#[derive(Debug, Default)] +pub struct BackfillPlan { + /// Rated and published articles (the learned set), rated first. + pub learned: Vec
, + pub interests: Vec, + /// Other recent articles; only under `--all`. + pub others: Vec
, + pub estimated_tokens: i64, + /// Articles and interests that were already cached and will be skipped. + pub cached: usize, +} + +impl BackfillPlan { + pub fn is_empty(&self) -> bool { + self.learned.is_empty() && self.interests.is_empty() && self.others.is_empty() + } + + pub fn article_count(&self) -> usize { + self.learned.len() + self.others.len() + } + + pub fn estimated_cost_usd(&self) -> f64 { + cost_for_tokens(self.estimated_tokens) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct BackfillOutcome { + pub articles_embedded: usize, + pub interests_embedded: usize, + pub tokens: i64, + pub cost_usd: f64, +} + +/// Decide what `features backfill` would embed without calling Voyage. +pub async fn plan_backfill( + db: &Db, + config: &Config, + service: &EmbeddingService, + opts: &BackfillOptions, +) -> anyhow::Result { + let since = Timestamp::now() + .checked_sub(jiff::Span::new().hours(opts.days.max(0).saturating_mul(24))) + .unwrap_or(Timestamp::UNIX_EPOCH); + + let mut ids = Vec::new(); + for rating in db + .current_ratings(config.curation.ranking.rating_lookback_days) + .await? + { + ids.push(rating.article_id); + } + if !opts.rated_only { + ids.extend(db.published_article_ids_since(since).await?); + } + let mut seen = std::collections::HashSet::new(); + let mut learned = Vec::new(); + for id in ids { + if seen.insert(id) + && let Some(article) = db.get_article(id).await? + { + learned.push(article); + } + } + + let mut others = Vec::new(); + if opts.all && !opts.rated_only { + for id in db.article_ids_since(since).await? { + if seen.insert(id) + && let Some(article) = db.get_article(id).await? + { + others.push(article); + } + } + } + + let interests = + match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { + Ok(interests) => interests, + Err(error) => { + tracing::warn!(%error, "could not load standing interests; skipping them"); + Vec::new() + } + }; + + let mut plan = BackfillPlan::default(); + let mut keep = |articles: Vec
, misses: Vec<(ArticleId, i64)>| -> Vec
{ + let wanted: HashMap = misses.into_iter().collect(); + plan.cached += articles.len() - wanted.len(); + plan.estimated_tokens += wanted.values().sum::(); + articles + .into_iter() + .filter(|article| wanted.contains_key(&article.id)) + .collect() + }; + let learned_misses = service.uncached_articles(&learned).await?; + plan.learned = keep(learned, learned_misses); + let other_misses = service.uncached_articles(&others).await?; + plan.others = keep(others, other_misses); + + let interest_misses = service.uncached_interests(&interests).await?; + plan.cached += interests.len() - interest_misses.len(); + plan.estimated_tokens += interest_misses + .iter() + .map(|interest| approx_tokens(interest) as i64) + .sum::(); + plan.interests = interest_misses; + Ok(plan) +} + +/// Embed the plan in priority order: learned set, interests, then the rest. +pub async fn run_backfill( + service: &EmbeddingService, + plan: &BackfillPlan, +) -> anyhow::Result { + let mut outcome = BackfillOutcome::default(); + outcome.articles_embedded += service.articles(&plan.learned).await?.len(); + outcome.interests_embedded += service.interests(&plan.interests).await?.len(); + outcome.articles_embedded += service.articles(&plan.others).await?.len(); + if let Some(meter) = service.meter() { + outcome.tokens = meter.total_tokens(); + outcome.cost_usd = meter.cost_usd(); + } + Ok(outcome) +} + +// --------------------------------------------------------------------------- +// Test backend +// --------------------------------------------------------------------------- + +/// Canned-vector backend for tests: pops scripted replies in order. +#[cfg(test)] +#[derive(Debug, Default)] +pub struct MockBackend { + scripted: Mutex>>, + /// Every request the code under test sent, in order. + pub seen: Mutex>, + /// When set, every request is answered with this many-dimensional unit + /// vectors derived from the input text (deterministic, no scripting). + auto_dimension: Mutex>, +} + +#[cfg(test)] +impl MockBackend { + pub fn new() -> Self { + Self::default() + } + + /// Answer every request with deterministic vectors of this dimension. + pub fn auto(dimension: usize) -> Self { + Self { + auto_dimension: Mutex::new(Some(dimension)), + ..Self::default() + } + } + + pub fn push(&self, completion: EmbeddingCompletion) { + self.scripted + .lock() + .expect("mock mutex") + .push_back(Ok(completion)); + } + + pub fn push_error(&self, error: &str) { + self.scripted + .lock() + .expect("mock mutex") + .push_back(Err(error.to_string())); + } + + pub fn calls(&self) -> usize { + self.seen.lock().expect("mock mutex").len() + } + + pub fn requests(&self) -> Vec { + self.seen.lock().expect("mock mutex").clone() + } + + /// A unit vector that depends only on the text, for cache tests. + pub fn vector_for(text: &str, dimension: usize) -> Vec { + let digest = Sha256::digest(text.as_bytes()); + let mut vector = (0..dimension) + .map(|i| f32::from(digest[i % digest.len()]) / 255.0 - 0.5) + .collect::>(); + let norm = vector.iter().map(|v| v * v).sum::().sqrt().max(1e-6); + vector.iter_mut().for_each(|v| *v /= norm); + vector + } +} + +#[cfg(test)] +impl EmbeddingBackend for MockBackend { + fn embed<'a>( + &'a self, + request: EmbeddingRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let auto = *self.auto_dimension.lock().expect("mock mutex"); + self.seen.lock().expect("mock mutex").push(request.clone()); + if let Some(dimension) = auto { + return Ok(EmbeddingCompletion { + data: request + .input + .iter() + .enumerate() + .map(|(index, text)| IndexedEmbedding { + index, + embedding: Self::vector_for(text, dimension), + }) + .collect(), + total_tokens: request + .input + .iter() + .map(|text| approx_tokens(text) as i64) + .sum(), + }); + } + match self.scripted.lock().expect("mock mutex").pop_front() { + Some(Ok(completion)) => Ok(completion), + Some(Err(error)) => Err(EmbeddingError::Api(error)), + None => Err(EmbeddingError::Api("mock exhausted".into())), + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ExtractMethod, SourceKind, SourceRef}; + + fn vector(index: usize, values: &[f32]) -> IndexedEmbedding { + IndexedEmbedding { + index, + embedding: values.to_vec(), + } + } + + fn small_config() -> VoyageConfig { + VoyageConfig { + output_dimension: 4, + batch_size: 2, + max_concurrent_requests: 2, + ..VoyageConfig::default() + } + } + + fn article(id: ArticleId, title: &str, body: &str) -> Article { + Article { + id, + canonical_url: format!("https://example.com/{id}"), + title: title.into(), + best_entry_id: id, + content_html: body.into(), + word_count: 2, + excerpt_only: false, + image_count: 0, + sources: vec![SourceRef { + entry_id: id, + feed_id: 9, + feed_title: "Secret Feed".into(), + category: None, + kind: SourceKind::Feed, + }], + first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), + url: format!("https://example.com/{id}"), + author: Some("Secret Author".into()), + feed_id: 9, + feed_title: "Secret Feed".into(), + category: None, + published_at: None, + comments_url: None, + image_urls: vec![], + social: vec![], + extract_method: ExtractMethod::Miniflux, + } + } + + async fn db_with_articles(ids: &[ArticleId]) -> (tempfile::TempDir, Db) { + let dir = tempfile::tempdir().unwrap(); + let db = Db::open_and_migrate(&dir.path().join("embed.db")) + .await + .unwrap(); + for id in ids { + sqlx::query( + "INSERT INTO articles (id, canonical_url, title, first_seen) + VALUES (?, ?, 'Article', '2026-08-15T00:00:00Z')", + ) + .bind(id) + .bind(format!("https://example.com/{id}")) + .execute(db.pool()) + .await + .unwrap(); + } + (dir, db) + } + + fn service(db: Db, config: VoyageConfig, backend: Arc) -> EmbeddingService { + let client = EmbeddingClient::with_backend(config.clone(), backend); + EmbeddingService::with_client(db, config, client) + } + + #[test] + fn blob_round_trip_and_validation() { + let values = vec![0.25, -1.5, 3.0]; + assert_eq!( + decode_blob(&encode_blob(&values).unwrap(), 3).unwrap(), + values + ); + assert!(matches!( + decode_blob(&[0; 4], 2), + Err(EmbeddingError::BlobLength { .. }) + )); + assert!(matches!( + encode_blob(&[f32::NAN]), + Err(EmbeddingError::NonFinite) + )); + let mut bytes = encode_blob(&[1.0, 2.0]).unwrap(); + bytes[4..].copy_from_slice(&f32::INFINITY.to_le_bytes()); + assert!(matches!( + decode_blob(&bytes, 2), + Err(EmbeddingError::NonFinite) + )); + } + + #[test] + fn dot_checks_dimensions() { + assert!((dot(&[1.0, 0.0], &[1.0, 0.0]).unwrap() - 1.0).abs() < 1e-9); + assert!(matches!( + dot(&[1.0], &[1.0, 0.0]), + Err(EmbeddingError::Dimension { .. }) + )); + } + + #[test] + fn response_is_mapped_by_index_and_checked() { + let response = ApiResponse { + data: vec![ + ApiEmbedding { + index: 1, + embedding: vec![0.0, 1.0], + }, + ApiEmbedding { + index: 0, + embedding: vec![1.0, 0.0], + }, + ], + usage: ApiUsage { total_tokens: 12 }, + }; + let mapped = map_response(response, 2, 2).unwrap(); + assert_eq!(mapped.data[0].embedding, [1.0, 0.0]); + assert_eq!(mapped.data[1].embedding, [0.0, 1.0]); + assert_eq!(mapped.total_tokens, 12); + + let short = ApiResponse { + data: vec![ApiEmbedding { + index: 0, + embedding: vec![1.0], + }], + usage: ApiUsage::default(), + }; + assert!(matches!( + map_response(short, 2, 1), + Err(EmbeddingError::ResponseLength { .. }) + )); + let wrong_dimension = ApiResponse { + data: vec![ApiEmbedding { + index: 0, + embedding: vec![1.0, 2.0, 3.0], + }], + usage: ApiUsage::default(), + }; + assert!(matches!( + map_response(wrong_dimension, 1, 2), + Err(EmbeddingError::Dimension { .. }) + )); + let duplicate = ApiResponse { + data: vec![ + ApiEmbedding { + index: 0, + embedding: vec![1.0], + }, + ApiEmbedding { + index: 0, + embedding: vec![1.0], + }, + ], + usage: ApiUsage::default(), + }; + assert!(matches!( + map_response(duplicate, 2, 1), + Err(EmbeddingError::DuplicateIndex(0)) + )); + } + + #[tokio::test] + async fn one_failed_batch_does_not_abort_another() { + let config = VoyageConfig { + batch_size: 1, + max_concurrent_requests: 1, + output_dimension: 2, + ..VoyageConfig::default() + }; + let backend = Arc::new(MockBackend::new()); + backend.push_error("failed"); + backend.push(EmbeddingCompletion { + data: vec![vector(0, &[1.0, 0.0])], + total_tokens: 3, + }); + let client = EmbeddingClient::with_backend(config, backend.clone()); + let result = client + .embed_many(&["a".into(), "b".into()], InputType::Document) + .await; + assert!(result[0].is_none()); + assert_eq!(result[1].as_deref(), Some([1.0, 0.0].as_slice())); + assert_eq!(backend.calls(), 2); + assert_eq!(client.meter.total_tokens(), 3); + } + + #[tokio::test] + async fn the_budget_guard_stops_further_batches() { + let config = VoyageConfig { + batch_size: 1, + max_concurrent_requests: 1, + output_dimension: 1, + max_daily_usd: 0.000_000_02, // one token + ..VoyageConfig::default() + }; + let backend = Arc::new(MockBackend::new()); + backend.push(EmbeddingCompletion { + data: vec![vector(0, &[1.0])], + total_tokens: 1, + }); + backend.push(EmbeddingCompletion { + data: vec![vector(0, &[1.0])], + total_tokens: 1, + }); + let client = EmbeddingClient::with_backend(config, backend.clone()); + let result = client + .embed_many(&["a".into(), "b".into()], InputType::Document) + .await; + assert!(result[0].is_some()); + assert!(result[1].is_none()); + assert_eq!(backend.calls(), 1); + } + + #[test] + fn article_text_excludes_feed_author_and_collapses_markup() { + let text = article_input(&article(1, "A title", "

Hello world

"), 60_000); + assert_eq!(text, "Title: A title\n\nHello world"); + assert!(!text.contains("Secret Feed") && !text.contains("Secret Author")); + // Cut on a char boundary. + let cut = article_input(&article(1, "T", "héllo wörld"), 12); + assert_eq!(cut.chars().count(), 12); + assert!(cut.starts_with("Title: T\n\nh")); + } + + #[tokio::test] + async fn cache_hits_on_same_hash_and_misses_on_changed_text_model_or_dimension() { + let (_dir, db) = db_with_articles(&[1]).await; + let backend = Arc::new(MockBackend::auto(4)); + let config = small_config(); + let svc = service(db.clone(), config.clone(), backend.clone()); + let a = article(1, "Title", "

body

"); + + let first = svc.articles(std::slice::from_ref(&a)).await.unwrap(); + assert_eq!(backend.calls(), 1); + assert_eq!(first[&1].len(), 4); + let request = &backend.requests()[0]; + assert_eq!(request.input_type, InputType::Document); + assert_eq!(request.input[0], "Title: Title\n\nbody"); + assert_eq!(request.output_dimension, 4); + + // Same text → cache hit, no call. + let again = svc.articles(std::slice::from_ref(&a)).await.unwrap(); + assert_eq!(backend.calls(), 1); + assert_eq!(again[&1], first[&1]); + assert!( + svc.uncached_articles(std::slice::from_ref(&a)) + .await + .unwrap() + .is_empty() + ); + + // Changed text → new hash → miss, row overwritten. + let edited = article(1, "Title", "

new body

"); + let after_edit = svc.articles(std::slice::from_ref(&edited)).await.unwrap(); + assert_eq!(backend.calls(), 2); + assert_ne!(after_edit[&1], first[&1]); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_embeddings") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(rows, 1, "one row per article, overwritten"); + + // Changed model → miss. + let other_model = EmbeddingService::with_client( + db.clone(), + VoyageConfig { + model: "voyage-other".into(), + ..config.clone() + }, + EmbeddingClient::with_backend( + VoyageConfig { + model: "voyage-other".into(), + ..config.clone() + }, + backend.clone(), + ), + ); + other_model + .articles(std::slice::from_ref(&edited)) + .await + .unwrap(); + assert_eq!(backend.calls(), 3); + + // Changed dimension → miss (the mock answers in the requested dimension). + let backend8 = Arc::new(MockBackend::auto(8)); + let dim8 = VoyageConfig { + output_dimension: 8, + ..config.clone() + }; + let other_dimension = service(db.clone(), dim8, backend8.clone()); + let vectors = other_dimension + .articles(std::slice::from_ref(&edited)) + .await + .unwrap(); + assert_eq!(backend8.calls(), 1); + assert_eq!(vectors[&1].len(), 8); + + // Cache-only: no client, so a miss stays a miss and nothing is called. + let cache_only = EmbeddingService::cached_only(db.clone(), small_config()); + let fresh = article(1, "Title", "

yet another body

"); + assert!( + cache_only + .articles(std::slice::from_ref(&fresh)) + .await + .unwrap() + .is_empty() + ); + assert!(cache_only.meter().is_none()); + } + + #[tokio::test] + async fn interests_are_embedded_as_bare_queries_and_cached() { + let (_dir, db) = db_with_articles(&[]).await; + let backend = Arc::new(MockBackend::auto(4)); + let svc = service(db.clone(), small_config(), backend.clone()); + let interests = vec!["Writerdeck".to_string(), "Gaussian Splatting".to_string()]; + let first = svc.interests(&interests).await.unwrap(); + assert_eq!(first.len(), 2); + assert_eq!(backend.calls(), 1); + let request = &backend.requests()[0]; + assert_eq!(request.input_type, InputType::Query); + assert_eq!(request.input, interests); + svc.interests(&interests).await.unwrap(); + assert_eq!(backend.calls(), 1, "warm cache makes no call"); + assert_eq!( + load_article_embeddings(&db, &small_config(), &[1]) + .await + .unwrap() + .len(), + 0 + ); + } + + #[tokio::test] + async fn backfill_prioritizes_the_learned_set_and_is_idempotent() { + let (dir, db) = db_with_articles(&[1, 2, 3]).await; + // Article 1 is rated, article 2 is published, article 3 is neither. + sqlx::query( + "INSERT INTO rating_events (article_id, kind, source, label, value, event_at) + VALUES (1, 'explicit', 'cli', 'loved', 1.0, ?)", + ) + .bind(fmt_ts(Timestamp::now())) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO issues (date, issue_number, generated_at) VALUES ('2026-08-15', 1, '2026-08-15T12:00:00Z'); + INSERT INTO issue_articles (issue_date, article_id, section) VALUES ('2026-08-15', 2, 'Top Stories');", + ) + .execute(db.pool()) + .await + .unwrap(); + // Article rows carry no body in this fixture; refresh them with one so + // that `get_article` yields embeddable text. + sqlx::query("UPDATE articles SET content_html = '

some body text

', first_seen = ?") + .bind(fmt_ts(Timestamp::now())) + .execute(db.pool()) + .await + .unwrap(); + + let config = Config { + voyage: small_config(), + interests_opml: dir.path().join("interests.opml"), + profile_path: dir.path().join("profile.md"), + ..Config::default() + }; + std::fs::write( + &config.interests_opml, + "", + ) + .unwrap(); + std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); + + let backend = Arc::new(MockBackend::auto(4)); + let svc = service(db.clone(), config.voyage.clone(), backend.clone()); + let opts = BackfillOptions { + days: 30, + rated_only: false, + all: false, + }; + let plan = plan_backfill(&db, &config, &svc, &opts).await.unwrap(); + assert_eq!( + plan.learned.iter().map(|a| a.id).collect::>(), + vec![1, 2], + "rated first, then published; article 3 needs --all" + ); + assert_eq!(plan.interests, vec!["Writerdeck".to_string()]); + assert!(plan.others.is_empty()); + assert!(plan.estimated_tokens > 0); + + let outcome = run_backfill(&svc, &plan).await.unwrap(); + assert_eq!(outcome.articles_embedded, 2); + assert_eq!(outcome.interests_embedded, 1); + let calls = backend.calls(); + assert!(calls >= 2); + + // Warm cache ⇒ empty plan and zero calls. + let plan = plan_backfill(&db, &config, &svc, &opts).await.unwrap(); + assert!(plan.is_empty()); + assert_eq!(plan.cached, 3); + run_backfill(&svc, &plan).await.unwrap(); + assert_eq!(backend.calls(), calls); + + // --all picks up the third article; --rated-only limits to the rated set. + let all = plan_backfill( + &db, + &config, + &svc, + &BackfillOptions { + all: true, + ..opts.clone() + }, + ) + .await + .unwrap(); + assert_eq!(all.others.iter().map(|a| a.id).collect::>(), vec![3]); + let (_dir2, fresh_db) = db_with_articles(&[1, 2]).await; + sqlx::query( + "INSERT INTO rating_events (article_id, kind, source, label, value, event_at) + VALUES (1, 'explicit', 'cli', 'loved', 1.0, ?);", + ) + .bind(fmt_ts(Timestamp::now())) + .execute(fresh_db.pool()) + .await + .unwrap(); + let fresh_svc = service( + fresh_db.clone(), + config.voyage.clone(), + Arc::new(MockBackend::auto(4)), + ); + let rated_only = plan_backfill( + &fresh_db, + &config, + &fresh_svc, + &BackfillOptions { + rated_only: true, + all: true, + days: 30, + }, + ) + .await + .unwrap(); + assert_eq!( + rated_only.learned.iter().map(|a| a.id).collect::>(), + vec![1] + ); + assert!(rated_only.others.is_empty()); + } +} diff --git a/src/curate/mod.rs b/src/curate/mod.rs index 93c4130..d5dff65 100644 --- a/src/curate/mod.rs +++ b/src/curate/mod.rs @@ -10,11 +10,14 @@ //! feed excerpts stand in for summaries (notes §6). pub mod editorial; +pub mod embedding; pub mod llm; pub mod prefilter; pub mod profile; pub mod score; pub mod select; +pub mod signals; +pub mod telemetry; use jiff::civil::Date; diff --git a/src/curate/prefilter.rs b/src/curate/prefilter.rs index 07ad7b0..55d5e87 100644 --- a/src/curate/prefilter.rs +++ b/src/curate/prefilter.rs @@ -90,7 +90,7 @@ impl PrefilterContext { let since = today .checked_sub(jiff::Span::new().days(STALE_LOOKBACK_DAYS)) .unwrap_or(today); - let already_published = db.previously_published_ids().await?; + let already_published = db.previously_published_ids_before(today).await?; let recently_rejected = db.recently_low_scored_ids(STALE_LOW_SCORE, since).await?; tracing::debug!( published = already_published.len(), @@ -197,6 +197,29 @@ pub fn social_points(social_score: f64) -> f64 { MAX_SOCIAL_POINTS * (social_score / SOCIAL_SATURATION).min(1.0).sqrt() } +/// Text-only heuristic used by personalized ranking (§9.3). +pub fn text_heuristic(article: &Article) -> f64 { + longform_points(article.word_count) + - excerpt_only_penalty(article) + - roundup_penalty(&article.title) +} + +pub fn excerpt_only_penalty(article: &Article) -> f64 { + if article.excerpt_only { + EXCERPT_ONLY_PENALTY + } else { + 0.0 + } +} + +pub fn roundup_penalty(title: &str) -> f64 { + if looks_like_roundup(title) { + ROUNDUP_TITLE_PENALTY + } else { + 0.0 + } +} + /// Score one article 0–100 from word count, social proof, source signals, /// and the excerpt/roundup/blocklist penalties (§3.5). pub fn score_article(article: &Article, _ctx: &PrefilterContext, cfg: &Config) -> f64 { diff --git a/src/curate/profile/mod.rs b/src/curate/profile/mod.rs index 544ea82..52d1fe6 100644 --- a/src/curate/profile/mod.rs +++ b/src/curate/profile/mod.rs @@ -125,6 +125,16 @@ pub fn load_profile(path: &Path) -> anyhow::Result { } } +/// Load the exact standing-interest union used in the system prompt. +pub fn load_standing_interests( + opml_path: &Path, + profile_path: &Path, +) -> anyhow::Result> { + let opml = parse_interests(opml_path)?; + let profile = load_profile(profile_path)?; + Ok(union_interests(opml, profile.interests)) +} + fn union_interests(opml: Vec, profile: Vec) -> Vec { let mut seen = BTreeSet::new(); let mut out = Vec::new(); diff --git a/src/curate/signals.rs b/src/curate/signals.rs new file mode 100644 index 0000000..4af1625 --- /dev/null +++ b/src/curate/signals.rs @@ -0,0 +1,981 @@ +//! Cheap per-article ranking signals, the mid-rank percentile normalizer and the +//! preliminary blend (plan §9, §12.2, §12.4). +//! +//! Every signal is an `Option`: `None` means *absent*, which is never a +//! numeric zero. Absent signals are left out of the percentile computation and +//! of the blend, whose remaining weights are renormalized (§12.2, §12.4). + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; + +use crate::config::{PreliminaryWeights, RankingConfig, VoyageConfig}; +use crate::curate::embedding::{dot, load_article_embeddings}; +use crate::curate::prefilter; +use crate::db::Db; +use crate::types::{Article, ArticleId, FeedId, SourceKind}; + +/// Below this many embedded eligible articles the z-score is too noisy, so the +/// interest signal falls back to the raw top-1 cosine (§9.1). +pub const INTEREST_ZSCORE_MIN_ARTICLES: usize = 30; +/// Standard-deviation floor for the per-interest z-score (§9.1). +const ZSCORE_STD_FLOOR: f64 = 1e-3; +/// How many interests and rated neighbours `signals_json` records (§7.5). +const RECORDED_TOP: usize = 3; + +/// The signal names that go through the percentile normalizer, in the order +/// they are rendered (§12.2). LLM scores (`triage`, `quality`, `fit`) are +/// absolute and arrive in steps 4–5. +pub const PERCENTILE_SIGNALS: [&str; 5] = ["interest", "knn", "feed", "social", "heuristic"]; + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct TopInterest { + pub name: String, + pub z: f64, + pub cos: f64, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Neighbour { + pub article_id: ArticleId, + pub label: String, + pub cos: f64, + pub title: String, +} + +/// Every cheap signal for one article, plus what the normalizer and the blend +/// derived from them (§9, §12.2, §12.4). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Signals { + pub interest: Option, + /// Raw top-1 cosine behind `interest`, recorded for `explain` (§7.5). + pub interest_top1_cos: Option, + pub knn: Option, + pub feed: Option, + pub social: Option, + pub heuristic: Option, + /// Mid-rank percentiles of the present signals (§12.2). + #[serde(default)] + pub norm: BTreeMap, + /// Effective preliminary weights after gating and renormalization (§12.4). + #[serde(default)] + pub weights: BTreeMap, + #[serde(default)] + pub top_interests: Vec, + #[serde(default)] + pub neighbours: Vec, + #[serde(default)] + pub notes: Vec, + /// Preliminary blend on a 0–100 scale; `None` when nothing is present. + pub preliminary: Option, + /// Gate ramps applied to the learned signals' weights (§9.2, §9.3). + #[serde(skip)] + pub knn_gate: f64, + #[serde(skip)] + pub feed_gate: f64, +} + +impl Signals { + /// The raw value of a named signal, `None` when absent or unknown. + pub fn raw(&self, name: &str) -> Option { + match name { + "interest" => self.interest, + "interest_top1_cos" => self.interest_top1_cos, + "knn" => self.knn, + "feed" => self.feed, + "social" => self.social, + "heuristic" => self.heuristic, + _ => None, + } + } + + pub fn present(&self, name: &str) -> bool { + self.raw(name).is_some() + } + + /// The signals every eligible article gets without embeddings or ratings. + pub fn baseline(article: &Article) -> Self { + Self { + social: (!article.social.is_empty()).then(|| article.social_score()), + heuristic: Some(prefilter::text_heuristic(article)), + ..Self::default() + } + } +} + +/// What the run log and the report say about the learned signals (§9.2, §15.4). +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct PreferenceSummary { + pub rated_with_embeddings: usize, + pub attributable_feed_ratings: usize, + pub knn_gate: f64, + pub feed_gate: f64, +} + +/// One rated article with an embedding: the unit of the preference state (§9.2). +#[derive(Debug, Clone, PartialEq)] +pub struct RatedExample { + pub article_id: ArticleId, + pub label: String, + pub title: String, + /// The vote's value (`loved` 1.0, `good` 0.35, `not_for_me` −1.0). + pub value: f64, + /// `0.5 ^ (age_days / half_life_days)` at the time of the run. + pub decay: f64, + pub embedding: Vec, + /// Distinct direct feeds that carried the rated article (§9.3). + pub feeds: Vec, +} + +impl RatedExample { + /// `weight_i = value_i × decay_i` (§9.2). + pub fn weight(&self) -> f64 { + self.value * self.decay + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +struct FeedRate { + up: f64, + down: f64, +} + +impl FeedRate { + /// Beta-smoothed rate `(up + 1) / (up + down + 2)` (§9.3). + fn rate(self) -> f64 { + (self.up + 1.0) / (self.up + self.down + 2.0) + } +} + +/// Rated-neighbour and feed-affinity state, built once per run (§9.2, §9.3). +#[derive(Debug, Clone, Default)] +pub struct PreferenceState { + pub examples: Vec, + feed_rates: HashMap, + pub attributable_feed_ratings: usize, + pub knn_gate: f64, + pub feed_gate: f64, +} + +impl PreferenceState { + /// Build the state from already-loaded examples (pure; tests use this). + pub fn build(examples: Vec, ranking: &RankingConfig) -> Self { + let (feed_rates, attributable_feed_ratings) = feed_rates(&examples); + Self { + knn_gate: gate(examples.len(), ranking.knn_floor, ranking.knn_full), + feed_gate: gate( + attributable_feed_ratings, + ranking.feed_floor, + ranking.feed_full, + ), + examples, + feed_rates, + attributable_feed_ratings, + } + } + + /// Load `db::current_ratings(rating_lookback_days)` joined to + /// `article_embeddings`; ratings without an embedding are skipped (§9.2). + pub async fn load( + db: &Db, + voyage: &VoyageConfig, + ranking: &RankingConfig, + now: Timestamp, + ) -> anyhow::Result { + let ratings = db.current_ratings(ranking.rating_lookback_days).await?; + let ids = ratings + .iter() + .map(|rating| rating.article_id) + .collect::>(); + let embeddings = load_article_embeddings(db, voyage, &ids).await?; + let mut examples = Vec::new(); + for rating in ratings { + let Some(embedding) = embeddings.get(&rating.article_id).cloned() else { + continue; + }; + let feeds = db + .get_article(rating.article_id) + .await? + .as_ref() + .map(direct_feeds) + .unwrap_or_default(); + let age_days = (now.as_second() - rating.event_at.as_second()).max(0) as f64 / 86_400.0; + examples.push(RatedExample { + article_id: rating.article_id, + label: rating.label, + title: rating.title, + value: rating.value, + decay: decay(age_days, ranking.rating_half_life_days), + embedding, + feeds, + }); + } + Ok(Self::build(examples, ranking)) + } + + pub fn summary(&self) -> PreferenceSummary { + PreferenceSummary { + rated_with_embeddings: self.examples.len(), + attributable_feed_ratings: self.attributable_feed_ratings, + knn_gate: self.knn_gate, + feed_gate: self.feed_gate, + } + } + + /// The once-per-run log line of §9.2. + pub fn log(&self, ranking: &RankingConfig) { + let feed_detail = if self.feed_gate > 0.0 { + format!("(n={})", self.attributable_feed_ratings) + } else { + format!( + "(n={} < {})", + self.attributable_feed_ratings, ranking.feed_floor + ) + }; + tracing::info!( + rated_with_embeddings = self.examples.len(), + knn_gate = self.knn_gate, + feed_gate = self.feed_gate, + "preference: {} rated articles with embeddings → knn gate {:.2}; feed gate {:.1} {}", + self.examples.len(), + self.knn_gate, + self.feed_gate, + feed_detail + ); + } + + /// Signed rated-neighbour preference and the three nearest rated articles + /// (§9.2). Absent when the gate is closed or there are no examples. + pub fn knn(&self, candidate: &[f32], ranking: &RankingConfig) -> (Option, Vec) { + if self.knn_gate <= 0.0 || self.examples.is_empty() { + return (None, Vec::new()); + } + let mut scored = self + .examples + .iter() + .filter_map(|example| { + dot(candidate, &example.embedding) + .ok() + .map(|s| (s, example)) + }) + .collect::>(); + scored.sort_by(|left, right| right.0.total_cmp(&left.0)); + + let side = |positive: bool| -> Option { + let chosen = scored + .iter() + .filter(|(_, example)| (example.weight() > 0.0) == positive) + .take(ranking.neighbour_k.max(1)) + .collect::>(); + let denominator = chosen + .iter() + .map(|(_, example)| example.weight().abs()) + .sum::(); + (denominator > 0.0).then(|| { + chosen + .iter() + .map(|(similarity, example)| example.weight().abs() * similarity) + .sum::() + / denominator + }) + }; + let positive = side(true); + let negative = side(false); + let knn = (positive.is_some() || negative.is_some()).then(|| { + positive.unwrap_or(0.0) - ranking.negative_coefficient * negative.unwrap_or(0.0) + }); + let neighbours = scored + .iter() + .take(RECORDED_TOP) + .map(|(cos, example)| Neighbour { + article_id: example.article_id, + label: example.label.clone(), + cos: *cos, + title: example.title.clone(), + }) + .collect(); + (knn, neighbours) + } + + /// Mean Beta-smoothed rate over the article's rated direct feeds (§9.3). + pub fn feed(&self, article: &Article) -> Option { + if self.feed_gate <= 0.0 { + return None; + } + let rates = direct_feeds(article) + .into_iter() + .filter_map(|feed| self.feed_rates.get(&feed)) + .map(|rate| rate.rate()) + .collect::>(); + (!rates.is_empty()).then(|| rates.iter().sum::() / rates.len() as f64) + } + + /// Per-feed `(up, down)` credit, exposed for tests of §9.3. + pub fn feed_credit(&self, feed: FeedId) -> Option<(f64, f64)> { + self.feed_rates.get(&feed).map(|rate| (rate.up, rate.down)) + } +} + +/// `0.5 ^ (age_days / half_life_days)` (§9.2). +pub fn decay(age_days: f64, half_life_days: f64) -> f64 { + if half_life_days <= 0.0 { + return 1.0; + } + 0.5f64.powf(age_days.max(0.0) / half_life_days) +} + +/// `clamp((n − floor) / (full − floor), 0, 1)` (§9.2). +pub fn gate(n: usize, floor: usize, full: usize) -> f64 { + if n <= floor { + 0.0 + } else if n >= full || full <= floor { + 1.0 + } else { + (n - floor) as f64 / (full - floor) as f64 + } +} + +/// Distinct `SourceKind::Feed` feeds that carried the article; the best entry's +/// feed when there are none (§9.3). +pub fn direct_feeds(article: &Article) -> Vec { + let mut feeds = article + .sources + .iter() + .filter(|source| source.kind == SourceKind::Feed) + .map(|source| source.feed_id) + .collect::>() + .into_iter() + .collect::>(); + if feeds.is_empty() && article.feed_id != 0 { + feeds.push(article.feed_id); + } + feeds.sort_unstable(); + feeds +} + +fn feed_rates(examples: &[RatedExample]) -> (HashMap, usize) { + let mut rates: HashMap = HashMap::new(); + let mut attributable = 0; + for example in examples { + if example.feeds.is_empty() { + continue; + } + attributable += 1; + let credit = example.weight() / example.feeds.len() as f64; + for feed in &example.feeds { + let rate = rates.entry(*feed).or_default(); + rate.up += credit.max(0.0); + rate.down += (-credit).max(0.0); + } + } + (rates, attributable) +} + +/// The interest match of §9.1 for one article. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct InterestMatch { + pub score: f64, + pub top1_cos: f64, + pub top_interests: Vec, +} + +/// Z-scored standing-interest match for every embedded article (§9.1). +/// +/// Below [`INTEREST_ZSCORE_MIN_ARTICLES`] embedded articles the score is the raw +/// top-1 cosine instead, and that is logged. +pub fn interest_matches( + articles: &HashMap>, + interests: &HashMap>, +) -> HashMap { + if articles.is_empty() || interests.is_empty() { + return HashMap::new(); + } + let fallback = articles.len() < INTEREST_ZSCORE_MIN_ARTICLES; + if fallback { + tracing::info!( + embedded = articles.len(), + "fewer than {INTEREST_ZSCORE_MIN_ARTICLES} embedded articles; interest uses the raw top-1 cosine" + ); + } + + let mut matches: HashMap> = HashMap::new(); + for (name, interest) in interests { + let similarities = articles + .iter() + .filter_map(|(article_id, article)| { + dot(interest, article).ok().map(|cos| (*article_id, cos)) + }) + .collect::>(); + if similarities.is_empty() { + continue; + } + let n = similarities.len() as f64; + let mean = similarities.iter().map(|(_, cos)| cos).sum::() / n; + let variance = similarities + .iter() + .map(|(_, cos)| (cos - mean).powi(2)) + .sum::() + / n; + let std = variance.sqrt().max(ZSCORE_STD_FLOOR); + for (article_id, cos) in similarities { + matches.entry(article_id).or_default().push(TopInterest { + name: name.clone(), + z: (cos - mean) / std, + cos, + }); + } + } + + matches + .into_iter() + .map(|(article_id, mut all)| { + all.sort_by(|left, right| { + right + .z + .total_cmp(&left.z) + .then_with(|| left.name.cmp(&right.name)) + }); + let top1_cos = all + .iter() + .map(|item| item.cos) + .fold(f64::NEG_INFINITY, f64::max); + all.truncate(RECORDED_TOP); + let score = if fallback { + top1_cos + } else { + let top_mean = all.iter().map(|item| item.z).sum::() / all.len() as f64; + 0.7 * all[0].z + 0.3 * top_mean + }; + ( + article_id, + InterestMatch { + score, + top1_cos, + top_interests: all, + }, + ) + }) + .collect() +} + +/// Every cheap signal for the eligible set, normalized and blended (§9, §12.2, +/// §12.4). Pure: the preference state is already loaded. +pub fn compute( + articles: &[Article], + article_embeddings: &HashMap>, + interest_embeddings: &HashMap>, + preference: &PreferenceState, + ranking: &RankingConfig, +) -> HashMap { + let interests = interest_matches(article_embeddings, interest_embeddings); + let mut all = articles + .iter() + .map(|article| { + let mut signals = Signals::baseline(article); + signals.knn_gate = preference.knn_gate; + signals.feed_gate = preference.feed_gate; + if let Some(matched) = interests.get(&article.id) { + signals.interest = Some(matched.score); + signals.interest_top1_cos = Some(matched.top1_cos); + signals.top_interests = matched.top_interests.clone(); + } + if let Some(embedding) = article_embeddings.get(&article.id) { + let (knn, neighbours) = preference.knn(embedding, ranking); + signals.knn = knn; + signals.neighbours = neighbours; + } + signals.feed = preference.feed(article); + if preference.knn_gate > 0.0 { + signals.notes.push(format!( + "knn gate {:.2} (n={} rated with embeddings)", + preference.knn_gate, + preference.examples.len() + )); + } + signals + }) + .collect::>(); + + normalize(&mut all.iter_mut().collect::>()); + for signals in &mut all { + preliminary_blend(signals, &ranking.weights.preliminary); + } + articles.iter().map(|article| article.id).zip(all).collect() +} + +/// [`compute`] with the preference state loaded from the database. +pub async fn compute_all( + db: &Db, + articles: &[Article], + article_embeddings: &HashMap>, + interest_embeddings: &HashMap>, + voyage: &VoyageConfig, + ranking: &RankingConfig, + now: Timestamp, +) -> anyhow::Result<(HashMap, PreferenceSummary)> { + let preference = PreferenceState::load(db, voyage, ranking, now).await?; + preference.log(ranking); + Ok(( + compute( + articles, + article_embeddings, + interest_embeddings, + &preference, + ranking, + ), + preference.summary(), + )) +} + +/// Mid-rank percentiles over the present values of each signal (§12.2). +/// +/// `p(x) = (count_below + (count_equal + 1) / 2) / n_present`; fewer than two +/// present values or all-equal values give 0.5. Article id never breaks ties. +pub fn normalize(signals: &mut [&mut Signals]) { + for name in PERCENTILE_SIGNALS { + let mut values = signals + .iter() + .filter_map(|signal| signal.raw(name)) + .collect::>(); + if values.is_empty() { + continue; + } + values.sort_by(f64::total_cmp); + let n = values.len() as f64; + let constant = values.len() < 2 || values.first() == values.last(); + for signal in signals.iter_mut() { + let Some(value) = signal.raw(name) else { + continue; + }; + let percentile = if constant { + 0.5 + } else { + let below = values.partition_point(|other| *other < value); + let equal = values.partition_point(|other| *other <= value) - below; + (below as f64 + (equal as f64 + 1.0) / 2.0) / n + }; + signal.norm.insert(name.to_string(), percentile); + } + } +} + +/// The preliminary blend of §12.4 on a 0–100 scale: present-and-active +/// signals only, learned weights multiplied by their gate, renormalized to 1. +pub fn preliminary_blend(signals: &mut Signals, configured: &PreliminaryWeights) -> Option { + let candidates = [ + ("interest", configured.interest, 1.0), + ("knn", configured.knn, signals.knn_gate), + ("heuristic", configured.heuristic, 1.0), + ("feed", configured.feed, signals.feed_gate), + ("social", configured.social, 1.0), + ]; + let active = candidates + .into_iter() + .filter_map(|(name, weight, gate)| { + let norm = *signals.norm.get(name)?; + let effective = weight * gate; + (effective > 0.0).then_some((name, effective, norm)) + }) + .collect::>(); + let total = active.iter().map(|(_, weight, _)| weight).sum::(); + if total <= 0.0 { + signals.weights.clear(); + signals.preliminary = None; + return None; + } + signals.weights = active + .iter() + .map(|(name, weight, _)| ((*name).to_string(), weight / total)) + .collect(); + let blend = active + .iter() + .map(|(_, weight, norm)| weight / total * norm) + .sum::() + * 100.0; + signals.preliminary = Some(blend); + Some(blend) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{ExtractMethod, SourceRef}; + + fn ranking() -> RankingConfig { + RankingConfig::default() + } + + fn unit(values: &[f32]) -> Vec { + let norm = values.iter().map(|v| v * v).sum::().sqrt(); + values.iter().map(|v| v / norm).collect() + } + + fn example(id: ArticleId, label: &str, value: f64, embedding: &[f32]) -> RatedExample { + RatedExample { + article_id: id, + label: label.into(), + title: format!("rated {id}"), + value, + decay: 1.0, + embedding: unit(embedding), + feeds: vec![id], + } + } + + fn article(id: ArticleId, feeds: &[FeedId]) -> Article { + Article { + id, + canonical_url: format!("https://example.com/{id}"), + title: format!("Article {id}"), + best_entry_id: id, + content_html: String::new(), + word_count: 1000, + excerpt_only: false, + image_count: 0, + sources: feeds + .iter() + .map(|feed| SourceRef { + entry_id: id, + feed_id: *feed, + feed_title: format!("feed {feed}"), + category: None, + kind: SourceKind::Feed, + }) + .collect(), + first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), + url: format!("https://example.com/{id}"), + author: None, + feed_id: feeds.first().copied().unwrap_or(0), + feed_title: String::new(), + category: None, + published_at: None, + comments_url: None, + image_urls: vec![], + social: vec![], + extract_method: ExtractMethod::Miniflux, + } + } + + fn with_heuristic(value: Option) -> Signals { + Signals { + heuristic: value, + ..Signals::default() + } + } + + // --- §9.1 interest z-scores --- + + fn interest_fixture(n: usize) -> (HashMap>, HashMap>) { + // Article 1 sits on axis x; the rest sit near axis y with a tiny spread. + let mut articles = HashMap::new(); + articles.insert(1, unit(&[1.0, 0.0, 0.0])); + for id in 2..=n as ArticleId { + articles.insert(id, unit(&[0.0, 1.0, 0.001 * id as f32])); + } + // "Broad" is about equally close to everything; "Specific" matches only article 1. + let mut interests = HashMap::new(); + interests.insert("Broad".to_string(), unit(&[1.0, 1.0, 0.0])); + interests.insert("Specific".to_string(), unit(&[1.0, 0.0, 0.0])); + (articles, interests) + } + + #[test] + fn specific_interest_with_one_strong_match_beats_a_broad_one() { + let (articles, interests) = interest_fixture(40); + let matched = interest_matches(&articles, &interests); + let strong = &matched[&1]; + assert_eq!(strong.top_interests[0].name, "Specific"); + assert!( + strong.top_interests[0].z > 3.0, + "z = {}", + strong.top_interests[0].z + ); + let others = (2..=40) + .map(|id| matched[&id].score) + .fold(f64::NEG_INFINITY, f64::max); + assert!(strong.score > others + 2.0, "{} vs {others}", strong.score); + // Raw cosine would have called Broad a near-tie everywhere (≈0.707). + assert!((matched[&2].top1_cos - 0.707).abs() < 0.01); + } + + #[test] + fn interest_falls_back_to_raw_cosine_under_thirty_articles() { + let (articles, interests) = interest_fixture(10); + let matched = interest_matches(&articles, &interests); + for (id, m) in &matched { + assert!( + (m.score - m.top1_cos).abs() < 1e-9, + "article {id} should use raw top-1" + ); + } + assert!((matched[&2].score - 0.707).abs() < 0.01); + } + + // --- §9.2 preference --- + + #[test] + fn one_loved_article_gives_a_positive_knn_to_a_near_neighbour() { + let mut ranking = ranking(); + ranking.knn_floor = 0; + ranking.knn_full = 1; + let state = PreferenceState::build(vec![example(1, "loved", 1.0, &[1.0, 0.0])], &ranking); + let (knn, neighbours) = state.knn(&unit(&[0.9, 0.1]), &ranking); + assert!(knn.unwrap() > 0.9); + assert_eq!(neighbours.len(), 1); + assert_eq!(neighbours[0].label, "loved"); + let (far, _) = state.knn(&unit(&[0.0, 1.0]), &ranking); + assert!(far.unwrap().abs() < 1e-6); + } + + #[test] + fn two_unrelated_loved_clusters_both_score_high() { + let mut ranking = ranking(); + ranking.knn_floor = 0; + ranking.knn_full = 1; + ranking.neighbour_k = 2; + let state = PreferenceState::build( + vec![ + example(1, "loved", 1.0, &[1.0, 0.0, 0.0]), + example(2, "loved", 1.0, &[0.98, 0.02, 0.0]), + example(3, "loved", 1.0, &[0.0, 1.0, 0.0]), + example(4, "loved", 1.0, &[0.0, 0.98, 0.02]), + ], + &ranking, + ); + let (near_a, _) = state.knn(&unit(&[1.0, 0.0, 0.0]), &ranking); + let (near_b, _) = state.knn(&unit(&[0.0, 1.0, 0.0]), &ranking); + assert!(near_a.unwrap() > 0.95, "{near_a:?}"); + assert!(near_b.unwrap() > 0.95, "{near_b:?}"); + // A centroid would have put both at ~0.7. + } + + #[test] + fn good_carries_a_third_of_loved() { + let loved = example(1, "loved", 1.0, &[1.0, 0.0]); + let good = example(2, "good", 0.35, &[1.0, 0.0]); + assert!((good.weight() / loved.weight() - 0.35).abs() < 1e-9); + + // A mixed neighbourhood: the far example pulls the mean down by 0.35× as + // much weight when it is merely "good" as when it is "loved". + let mut ranking = ranking(); + ranking.knn_floor = 0; + ranking.knn_full = 1; + let near = example(1, "loved", 1.0, &[1.0, 0.0]); + let candidate = unit(&[1.0, 0.0]); + let both_loved = PreferenceState::build( + vec![near.clone(), example(2, "loved", 1.0, &[0.0, 1.0])], + &ranking, + ); + let one_good = + PreferenceState::build(vec![near, example(2, "good", 0.35, &[0.0, 1.0])], &ranking); + let pull_loved = 1.0 - both_loved.knn(&candidate, &ranking).0.unwrap(); + let pull_good = 1.0 - one_good.knn(&candidate, &ranking).0.unwrap(); + assert!(pull_good < pull_loved); + // Weighted means: 0.5 vs 1/1.35 → pulls 0.5 vs 0.35/1.35. + assert!((pull_good / pull_loved - 0.35 / 1.35 / 0.5).abs() < 1e-9); + } + + #[test] + fn negatives_subtract_with_the_negative_coefficient() { + let mut ranking = ranking(); + ranking.knn_floor = 0; + ranking.knn_full = 1; + let state = + PreferenceState::build(vec![example(1, "not_for_me", -1.0, &[1.0, 0.0])], &ranking); + let (knn, _) = state.knn(&unit(&[1.0, 0.0]), &ranking); + assert!((knn.unwrap() + ranking.negative_coefficient).abs() < 1e-9); + } + + #[test] + fn decay_halves_at_the_half_life() { + assert!((decay(60.0, 60.0) - 0.5).abs() < 1e-12); + assert!((decay(0.0, 60.0) - 1.0).abs() < 1e-12); + assert!((decay(120.0, 60.0) - 0.25).abs() < 1e-12); + } + + #[test] + fn gate_is_zero_below_floor_one_at_full_and_linear_between() { + assert_eq!(gate(0, 8, 25), 0.0); + assert_eq!(gate(8, 8, 25), 0.0); + assert_eq!(gate(25, 8, 25), 1.0); + assert_eq!(gate(100, 8, 25), 1.0); + assert!((gate(16, 8, 24) - 0.5).abs() < 1e-9); + assert!((gate(9, 8, 25) - 1.0 / 17.0).abs() < 1e-9); + } + + #[test] + fn knn_is_absent_when_the_gate_is_closed() { + let ranking = ranking(); // knn_floor 8 + let state = PreferenceState::build(vec![example(1, "loved", 1.0, &[1.0, 0.0])], &ranking); + assert_eq!(state.knn_gate, 0.0); + assert_eq!(state.knn(&unit(&[1.0, 0.0]), &ranking), (None, Vec::new())); + } + + // --- §9.3 feed affinity --- + + #[test] + fn feed_credit_sums_to_one_across_direct_feeds() { + let mut rated = example(1, "loved", 1.0, &[1.0, 0.0]); + rated.feeds = vec![10, 20, 30]; + let state = PreferenceState::build(vec![rated], &ranking()); + let total: f64 = [10, 20, 30] + .iter() + .map(|feed| state.feed_credit(*feed).unwrap().0) + .sum(); + assert!((total - 1.0).abs() < 1e-9); + assert!((state.feed_credit(10).unwrap().0 - 1.0 / 3.0).abs() < 1e-9); + } + + #[test] + fn feed_affinity_is_the_mean_over_rated_feeds() { + let mut ranking = ranking(); + ranking.feed_floor = 0; + ranking.feed_full = 1; + let mut loved = example(1, "loved", 1.0, &[1.0, 0.0]); + loved.feeds = vec![10]; + let mut down = example(2, "not_for_me", -1.0, &[1.0, 0.0]); + down.feeds = vec![20]; + let state = PreferenceState::build(vec![loved, down], &ranking); + // feed 10: (1+1)/(1+0+2) = 2/3; feed 20: (0+1)/(0+1+2) = 1/3; unrated 99 ignored. + let both = state.feed(&article(7, &[10, 20, 99])).unwrap(); + assert!((both - 0.5).abs() < 1e-9, "{both}"); + let best = state.feed(&article(8, &[10])).unwrap(); + assert!((best - 2.0 / 3.0).abs() < 1e-9); + assert_eq!(state.feed(&article(9, &[99])), None); + } + + #[test] + fn feed_is_absent_when_the_gate_is_closed() { + let ranking = ranking(); // feed_floor 15 + let mut loved = example(1, "loved", 1.0, &[1.0, 0.0]); + loved.feeds = vec![10]; + let state = PreferenceState::build(vec![loved], &ranking); + assert_eq!(state.feed_gate, 0.0); + assert_eq!(state.feed(&article(7, &[10])), None); + } + + #[test] + fn direct_feeds_fall_back_to_the_best_entry_feed() { + let mut a = article(1, &[]); + a.feed_id = 42; + assert_eq!(direct_feeds(&a), vec![42]); + assert_eq!(direct_feeds(&article(2, &[5, 3, 5])), vec![3, 5]); + } + + // --- §12.2 normalization, §12.4 blend --- + + #[test] + fn constant_signal_normalizes_to_half_for_everyone() { + let mut values = [with_heuristic(Some(7.0)), with_heuristic(Some(7.0))]; + let mut refs = values.iter_mut().collect::>(); + normalize(&mut refs); + assert!(values.iter().all(|v| v.norm["heuristic"] == 0.5)); + let mut single = [with_heuristic(Some(3.0))]; + let mut refs = single.iter_mut().collect::>(); + normalize(&mut refs); + assert_eq!(single[0].norm["heuristic"], 0.5); + } + + #[test] + fn ties_get_equal_percentiles_without_an_id_ramp() { + let mut values = (0..400) + .map(|_| with_heuristic(Some(0.0))) + .collect::>(); + let mut refs = values.iter_mut().collect::>(); + normalize(&mut refs); + assert!(values.iter().all(|v| v.norm["heuristic"] == 0.5)); + + let mut mixed = [ + with_heuristic(Some(1.0)), + with_heuristic(Some(2.0)), + with_heuristic(Some(2.0)), + with_heuristic(Some(3.0)), + ]; + let mut refs = mixed.iter_mut().collect::>(); + normalize(&mut refs); + assert_eq!(mixed[0].norm["heuristic"], 0.25); + assert_eq!(mixed[1].norm["heuristic"], 0.625); + assert_eq!(mixed[2].norm["heuristic"], 0.625); + assert_eq!(mixed[3].norm["heuristic"], 1.0); + } + + #[test] + fn absent_values_do_not_shift_present_values() { + let mut values = [ + with_heuristic(Some(1.0)), + with_heuristic(Some(2.0)), + with_heuristic(None), + ]; + let mut refs = values.iter_mut().collect::>(); + normalize(&mut refs); + assert!(!values[2].norm.contains_key("heuristic")); + // n_present = 2: the absent third value does not widen the scale. + assert_eq!(values[0].norm["heuristic"], 0.5); + assert_eq!(values[1].norm["heuristic"], 1.0); + } + + #[test] + fn effective_weights_sum_to_one_and_missing_signals_are_skipped() { + let mut signals = Signals { + interest: Some(1.0), + heuristic: Some(2.0), + norm: BTreeMap::from([("interest".into(), 0.8), ("heuristic".into(), 0.4)]), + ..Signals::default() + }; + let blend = preliminary_blend(&mut signals, &PreliminaryWeights::default()).unwrap(); + assert!((signals.weights.values().sum::() - 1.0).abs() < 1e-9); + assert!(!signals.weights.contains_key("knn")); + assert!(!signals.weights.contains_key("social")); + // 0.35/0.55 × 0.8 + 0.20/0.55 × 0.4 = 0.6545… + assert!((blend - 65.4545).abs() < 0.01, "{blend}"); + + let mut only_heuristic = Signals { + heuristic: Some(2.0), + norm: BTreeMap::from([("heuristic".into(), 0.4)]), + ..Signals::default() + }; + let blend = preliminary_blend(&mut only_heuristic, &PreliminaryWeights::default()); + assert!((blend.unwrap() - 40.0).abs() < 1e-9); + assert_eq!(only_heuristic.weights["heuristic"], 1.0); + } + + #[test] + fn learned_weights_are_multiplied_by_their_gate() { + let mut signals = Signals { + knn: Some(0.5), + heuristic: Some(2.0), + knn_gate: 0.5, + norm: BTreeMap::from([("knn".into(), 1.0), ("heuristic".into(), 0.0)]), + ..Signals::default() + }; + preliminary_blend(&mut signals, &PreliminaryWeights::default()); + // knn 0.25 × 0.5 = 0.125 against heuristic 0.20. + assert!((signals.weights["knn"] - 0.125 / 0.325).abs() < 1e-9); + } + + #[test] + fn compute_scores_every_article_and_leaves_ungated_signals_absent() { + let articles = vec![article(1, &[10]), article(2, &[20]), article(3, &[30])]; + let mut embeddings = HashMap::new(); + embeddings.insert(1, unit(&[1.0, 0.0])); + embeddings.insert(2, unit(&[0.0, 1.0])); + let mut interests = HashMap::new(); + interests.insert("Axis".to_string(), unit(&[1.0, 0.0])); + let state = PreferenceState::build(vec![example(9, "loved", 1.0, &[1.0, 0.0])], &ranking()); + let signals = compute(&articles, &embeddings, &interests, &state, &ranking()); + assert_eq!(signals.len(), 3); + assert!(signals[&1].interest.is_some()); + assert!(signals[&3].interest.is_none(), "no embedding → absent"); + assert!( + signals + .values() + .all(|s| s.knn.is_none() && s.feed.is_none()) + ); + assert!( + signals + .values() + .all(|s| s.heuristic.is_some() && s.preliminary.is_some()) + ); + } +} diff --git a/src/curate/telemetry.rs b/src/curate/telemetry.rs new file mode 100644 index 0000000..1243c4c --- /dev/null +++ b/src/curate/telemetry.rs @@ -0,0 +1,962 @@ +//! Per-run candidate telemetry: the `candidate_runs` writer, `signals_json`, +//! the `explain` command and feature retention (plan §7.4–7.5, §15.2, §16). +//! +//! One row per considered article per run says where it stopped and why. Rows +//! are upserted on every stage transition with every column set (never +//! `COALESCE`), so the last write for a run is the whole truth. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use jiff::Timestamp; +use jiff::civil::Date; +use serde::{Deserialize, Serialize}; +use sqlx::Row as _; + +use crate::curate::signals::{Neighbour, Signals, TopInterest}; +use crate::db::{Db, fmt_ts}; +use crate::types::ArticleId; + +/// The stage vocabulary of §7.4, in pipeline order. +pub const STAGES: [&str; 7] = [ + "excluded", + "eligible", + "triaged", + "admitted", + "assessed", + "shortlisted", + "selected", +]; + +/// Signal names rendered by `explain`, including the LLM ones steps 4–5 add. +const RENDERED_SIGNALS: [&str; 8] = [ + "interest", + "knn", + "feed", + "social", + "heuristic", + "triage", + "quality", + "fit", +]; + +/// One `candidate_runs` row (§7.4). +#[derive(Debug, Clone)] +pub struct CandidateRun<'a> { + pub run_id: i64, + pub article_id: ArticleId, + pub stage: &'a str, + pub excluded_reason: Option<&'a str>, + /// JSON array of retriever names, first = the one that admitted it. + pub admitted_by: Option<&'a str>, + pub signals_json: &'a str, + pub utility: Option, + pub rank_utility: Option, + pub cluster_id: Option, + pub cluster_rank: Option, + pub editor_why: Option<&'a str>, +} + +/// Upsert one row, setting every column (§7.4). +pub async fn write(db: &Db, row: &CandidateRun<'_>) -> Result<(), sqlx::Error> { + sqlx::query( + "INSERT INTO candidate_runs + (run_id, article_id, stage, excluded_reason, admitted_by, signals_json, + utility, rank_utility, cluster_id, cluster_rank, editor_why) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(run_id, article_id) DO UPDATE SET + stage = excluded.stage, + excluded_reason = excluded.excluded_reason, + admitted_by = excluded.admitted_by, + signals_json = excluded.signals_json, + utility = excluded.utility, + rank_utility = excluded.rank_utility, + cluster_id = excluded.cluster_id, + cluster_rank = excluded.cluster_rank, + editor_why = excluded.editor_why", + ) + .bind(row.run_id) + .bind(row.article_id) + .bind(row.stage) + .bind(row.excluded_reason) + .bind(row.admitted_by) + .bind(row.signals_json) + .bind(row.utility) + .bind(row.rank_utility) + .bind(row.cluster_id) + .bind(row.cluster_rank) + .bind(row.editor_why) + .execute(db.pool()) + .await?; + Ok(()) +} + +/// The thin row of a hygiene exclusion: keys, `stage = 'excluded'`, the reason +/// and `signals_json = '{}'` (§8.1). +pub async fn thin_excluded( + db: &Db, + run_id: i64, + article_id: ArticleId, + reason: &str, +) -> Result<(), sqlx::Error> { + write( + db, + &CandidateRun { + run_id, + article_id, + stage: "excluded", + excluded_reason: Some(reason), + admitted_by: None, + signals_json: "{}", + utility: None, + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: None, + }, + ) + .await +} + +/// `signals_json` (§7.5). Missing signals are absent from `raw`/`norm` and +/// `false` in `present`; `weights` are the effective weights. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SignalsJson { + pub v: i64, + #[serde(default)] + pub raw: BTreeMap, + #[serde(default)] + pub norm: BTreeMap, + #[serde(default)] + pub present: BTreeMap, + #[serde(default)] + pub weights: BTreeMap, + #[serde(default)] + pub top_interests: Vec, + #[serde(default)] + pub neighbours: Vec, + #[serde(default)] + pub exploration: bool, + #[serde(default)] + pub auto_include: bool, + #[serde(default)] + pub notes: Vec, +} + +impl SignalsJson { + /// The blend implied by the stored effective weights, 0–100. + pub fn blend(&self) -> Option { + let mut score = 0.0; + let mut any = false; + for (name, weight) in &self.weights { + if let Some(value) = self.norm.get(name) { + score += weight * value; + any = true; + } + } + any.then_some(score * 100.0) + } +} + +/// Serialize the signals of §7.5 for one article. +pub fn serialize_signals(signals: &Signals, auto_include: bool) -> String { + let mut raw = BTreeMap::new(); + for name in [ + "interest", + "interest_top1_cos", + "knn", + "feed", + "social", + "heuristic", + ] { + if let Some(value) = signals.raw(name) { + raw.insert(name.to_string(), value); + } + } + let present = RENDERED_SIGNALS + .into_iter() + .map(|name| (name.to_string(), signals.present(name))) + .collect(); + serde_json::to_string(&SignalsJson { + v: 1, + raw, + norm: signals.norm.clone(), + present, + weights: signals.weights.clone(), + top_interests: signals.top_interests.clone(), + neighbours: signals.neighbours.clone(), + exploration: false, + auto_include, + notes: signals.notes.clone(), + }) + .unwrap_or_else(|_| "{}".into()) +} + +// --------------------------------------------------------------------------- +// `explain` (§15.2) +// --------------------------------------------------------------------------- + +/// A `candidate_runs` row joined to its article title. +#[derive(Debug, Clone)] +pub struct ExplainRow { + pub run_id: i64, + pub article_id: ArticleId, + pub title: String, + pub stage: String, + pub excluded_reason: Option, + pub admitted_by: Option, + pub signals_json: String, + pub utility: Option, + pub rank_utility: Option, + pub cluster_id: Option, + pub cluster_rank: Option, + pub editor_why: Option, +} + +impl ExplainRow { + fn from_row(row: &sqlx::sqlite::SqliteRow) -> Self { + Self { + run_id: row.get("run_id"), + article_id: row.get("article_id"), + title: row.get("title"), + stage: row.get("stage"), + excluded_reason: row.get("excluded_reason"), + admitted_by: row.get("admitted_by"), + signals_json: row.get("signals_json"), + utility: row.get("utility"), + rank_utility: row.get("rank_utility"), + cluster_id: row.get("cluster_id"), + cluster_rank: row.get("cluster_rank"), + editor_why: row.get("editor_why"), + } + } + + pub fn signals(&self) -> Option { + serde_json::from_str(&self.signals_json).ok() + } + + /// Utility when step 5 has written it, else the preliminary blend. + pub fn score(&self) -> Option { + self.utility + .or_else(|| self.signals().and_then(|signals| signals.blend())) + } +} + +/// The run `explain` reads: `--run-id` when given (and of that date), else the +/// latest non-dry run of the date. +pub async fn resolve_run( + db: &Db, + date: Date, + requested: Option, +) -> Result, sqlx::Error> { + let row = match requested { + Some(run_id) => { + sqlx::query("SELECT id FROM runs WHERE id = ? AND date = ?") + .bind(run_id) + .bind(date.to_string()) + .fetch_optional(db.pool()) + .await? + } + None => { + sqlx::query( + "SELECT id FROM runs WHERE date = ? AND status != 'dry_run' + ORDER BY id DESC LIMIT 1", + ) + .bind(date.to_string()) + .fetch_optional(db.pool()) + .await? + } + }; + Ok(row.map(|row| row.get("id"))) +} + +pub async fn explain_row( + db: &Db, + run_id: i64, + article_id: ArticleId, +) -> Result, sqlx::Error> { + let row = sqlx::query( + "SELECT cr.run_id, cr.article_id, COALESCE(a.title, '') AS title, + cr.stage, cr.excluded_reason, cr.admitted_by, cr.signals_json, + cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank, cr.editor_why + FROM candidate_runs cr JOIN articles a ON a.id = cr.article_id + WHERE cr.run_id = ? AND cr.article_id = ?", + ) + .bind(run_id) + .bind(article_id) + .fetch_optional(db.pool()) + .await?; + Ok(row.as_ref().map(ExplainRow::from_row)) +} + +/// The top `limit` rows by utility-or-blend that were not selected (§15.2). +pub async fn near_misses( + db: &Db, + run_id: i64, + limit: usize, +) -> Result, sqlx::Error> { + let rows = sqlx::query( + "SELECT cr.run_id, cr.article_id, COALESCE(a.title, '') AS title, + cr.stage, cr.excluded_reason, cr.admitted_by, cr.signals_json, + cr.utility, cr.rank_utility, cr.cluster_id, cr.cluster_rank, cr.editor_why + FROM candidate_runs cr JOIN articles a ON a.id = cr.article_id + WHERE cr.run_id = ? AND cr.stage != 'selected' AND cr.stage != 'excluded'", + ) + .bind(run_id) + .fetch_all(db.pool()) + .await?; + let mut output = rows.iter().map(ExplainRow::from_row).collect::>(); + output.sort_by(|left, right| { + right + .score() + .unwrap_or(f64::NEG_INFINITY) + .total_cmp(&left.score().unwrap_or(f64::NEG_INFINITY)) + .then_with(|| left.article_id.cmp(&right.article_id)) + }); + output.truncate(limit); + Ok(output) +} + +fn fmt_opt(value: Option) -> String { + value + .map(|v| format!("{v:.3}")) + .unwrap_or_else(|| "—".into()) +} + +/// Render one persisted row the way §15.2 lists it. +pub async fn render_explain(db: &Db, row: &ExplainRow) -> Result { + let mut out = String::new(); + let _ = writeln!(out, "article {}: {}", row.article_id, row.title); + let _ = write!(out, "run {} · stage: {}", row.run_id, row.stage); + if let Some(reason) = &row.excluded_reason { + let _ = write!(out, " · reason: {reason}"); + } + let _ = writeln!(out); + if let Some(signals) = row.signals() { + let _ = writeln!(out, "signals (raw · norm · weight):"); + for name in RENDERED_SIGNALS { + let present = signals.present.get(name).copied().unwrap_or(false); + if present { + let _ = writeln!( + out, + " {name:<10} {:>8} · {:>6} · {:>6}", + fmt_opt(signals.raw.get(name).copied()), + fmt_opt(signals.norm.get(name).copied()), + fmt_opt(signals.weights.get(name).copied()), + ); + } else { + let _ = writeln!(out, " {name:<10} absent"); + } + } + if let Some(blend) = signals.blend() { + let _ = writeln!(out, "preliminary blend: {blend:.1}"); + } + if let Some(cos) = signals.raw.get("interest_top1_cos") { + let _ = writeln!(out, "interest top-1 cosine: {cos:.3}"); + } + if !signals.top_interests.is_empty() { + let _ = writeln!(out, "top interests:"); + for interest in &signals.top_interests { + let _ = writeln!( + out, + " {} · z {:.2} · cos {:.3}", + interest.name, interest.z, interest.cos + ); + } + } + if !signals.neighbours.is_empty() { + let _ = writeln!(out, "nearest rated neighbours:"); + for neighbour in &signals.neighbours { + let _ = writeln!( + out, + " {} · cos {:.3} · article {} · {}", + neighbour.label, neighbour.cos, neighbour.article_id, neighbour.title + ); + } + } + if signals.exploration || signals.auto_include { + let _ = writeln!( + out, + "flags: exploration={} auto_include={}", + signals.exploration, signals.auto_include + ); + } + for note in &signals.notes { + let _ = writeln!(out, "note: {note}"); + } + } + + let assessments = sqlx::query( + "SELECT stage, model, score, fit, kind, facets_json, rationale, category, + paywalled_guess, assessed_at + FROM article_assessments WHERE article_id = ? ORDER BY stage", + ) + .bind(row.article_id) + .fetch_all(db.pool()) + .await?; + if !assessments.is_empty() { + let _ = writeln!(out, "assessments:"); + for assessment in assessments { + let _ = writeln!( + out, + " {} · {} · score {} · fit {} · kind {} · category {} · paywalled={} · {}", + assessment.get::("stage"), + assessment.get::("model"), + fmt_opt(assessment.get::, _>("score")), + fmt_opt(assessment.get::, _>("fit")), + assessment + .get::, _>("kind") + .unwrap_or_else(|| "—".into()), + assessment + .get::, _>("category") + .unwrap_or_else(|| "—".into()), + assessment.get::("paywalled_guess") != 0, + assessment + .get::, _>("rationale") + .unwrap_or_default(), + ); + if let Some(facets) = assessment.get::, _>("facets_json") { + let _ = writeln!(out, " facets: {facets}"); + } + } + } + if row.utility.is_some() || row.rank_utility.is_some() { + let _ = writeln!( + out, + "utility: {} · rank {}", + fmt_opt(row.utility), + row.rank_utility + .map(|r| r.to_string()) + .unwrap_or_else(|| "—".into()) + ); + } + if let Some(cluster) = row.cluster_id { + let _ = writeln!( + out, + "cluster: {cluster} · rank {}", + row.cluster_rank + .map(|r| r.to_string()) + .unwrap_or_else(|| "—".into()) + ); + } + if let Some(admitted_by) = &row.admitted_by { + let _ = writeln!(out, "admitted by: {admitted_by}"); + } + if let Some(why) = &row.editor_why { + let _ = writeln!(out, "editor: {why}"); + } + Ok(out) +} + +/// What `explain` was asked about. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExplainTarget { + Article(ArticleId), + Url(String), +} + +/// `explain --date D (--article ID | --url URL) [--run-id N]` as text (§15.2). +pub async fn explain( + db: &Db, + date: Date, + run_id: Option, + target: &ExplainTarget, +) -> anyhow::Result { + let article_id = match target { + ExplainTarget::Article(id) => *id, + ExplainTarget::Url(url) => { + let canonical = crate::dedupe::canonical_url(url) + .ok_or_else(|| anyhow::anyhow!("invalid article URL {url:?}"))?; + match db.article_id_for_url(&canonical).await? { + Some(id) => id, + None => { + return Ok(format!( + "{canonical} was never ingested: it is not in `articles`, so this is a feed problem, not a ranking problem." + )); + } + } + } + }; + if db.get_article(article_id).await?.is_none() { + return Ok(format!( + "article {article_id} was never ingested: it is not in `articles`." + )); + } + let Some(run_id) = resolve_run(db, date, run_id).await? else { + return Ok(match run_id { + Some(id) => format!("run {id} is not a run for {date}"), + None => format!("no non-dry run recorded for {date}"), + }); + }; + match explain_row(db, run_id, article_id).await? { + Some(row) => Ok(render_explain(db, &row).await?), + None => Ok(format!( + "article {article_id} was not considered by run {run_id} for {date} (outside its ingest window, or telemetry pruned)." + )), + } +} + +/// `explain --date D --near-misses [N]` as text (§15.2). +pub async fn explain_near_misses( + db: &Db, + date: Date, + run_id: Option, + limit: usize, +) -> anyhow::Result { + let Some(run_id) = resolve_run(db, date, run_id).await? else { + return Ok(format!("no non-dry run recorded for {date}")); + }; + let rows = near_misses(db, run_id, limit).await?; + let mut out = String::new(); + let _ = writeln!( + out, + "run {run_id} · {date} · top {} not selected, by {}:", + rows.len(), + if rows.iter().any(|row| row.utility.is_some()) { + "utility" + } else { + "preliminary blend" + } + ); + for (index, row) in rows.iter().enumerate() { + let reason = row + .excluded_reason + .as_deref() + .map(|reason| format!(", {reason}")) + .unwrap_or_default(); + let _ = writeln!( + out, + "{:>3}. {:>6} · {} · {}{} · article {}", + index + 1, + row.score() + .map(|score| format!("{score:.1}")) + .unwrap_or_else(|| "—".into()), + row.title, + row.stage, + reason, + row.article_id + ); + } + Ok(out) +} + +// --------------------------------------------------------------------------- +// `features prune` (§7.1, §7.4) +// --------------------------------------------------------------------------- + +/// Delete `article_embeddings` for articles neither rated nor published that +/// are older than `embedding_retention_days`, and `candidate_runs` rows whose +/// run started more than `telemetry_retention_days` ago. Returns the counts. +pub async fn prune( + db: &Db, + embedding_retention_days: i64, + telemetry_retention_days: i64, + now: Timestamp, +) -> Result<(u64, u64), sqlx::Error> { + let cutoff = |days: i64| { + now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24))) + .unwrap_or(Timestamp::UNIX_EPOCH) + }; + let embeddings = sqlx::query( + "DELETE FROM article_embeddings + WHERE article_id IN ( + SELECT ae.article_id + FROM article_embeddings ae JOIN articles a ON a.id = ae.article_id + WHERE a.first_seen < ? + AND NOT EXISTS (SELECT 1 FROM rating_events re WHERE re.article_id = ae.article_id) + AND NOT EXISTS (SELECT 1 FROM issue_articles ia WHERE ia.article_id = ae.article_id) + )", + ) + .bind(fmt_ts(cutoff(embedding_retention_days))) + .execute(db.pool()) + .await? + .rows_affected(); + + let telemetry = sqlx::query( + "DELETE FROM candidate_runs + WHERE run_id IN (SELECT id FROM runs WHERE started_at < ?)", + ) + .bind(fmt_ts(cutoff(telemetry_retention_days))) + .execute(db.pool()) + .await? + .rows_affected(); + Ok((embeddings, telemetry)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::curate::embedding::encode_blob; + + async fn db_with_articles(ids: &[ArticleId]) -> (tempfile::TempDir, Db) { + let dir = tempfile::tempdir().unwrap(); + let db = Db::open_and_migrate(&dir.path().join("telemetry.db")) + .await + .unwrap(); + for id in ids { + sqlx::query( + "INSERT INTO articles (id, canonical_url, title, first_seen) + VALUES (?, ?, ?, '2026-08-15T00:00:00Z')", + ) + .bind(id) + .bind(format!("https://example.com/{id}")) + .bind(format!("Article {id}")) + .execute(db.pool()) + .await + .unwrap(); + } + (dir, db) + } + + fn date() -> Date { + "2026-09-02".parse().unwrap() + } + + fn signals(heuristic: f64, norm: f64) -> Signals { + Signals { + interest: Some(1.2), + interest_top1_cos: Some(0.61), + heuristic: Some(heuristic), + norm: BTreeMap::from([("heuristic".into(), norm), ("interest".into(), 0.9)]), + weights: BTreeMap::from([ + ("heuristic".into(), 0.2 / 0.55), + ("interest".into(), 0.35 / 0.55), + ]), + top_interests: vec![TopInterest { + name: "Gaussian Splatting".into(), + z: 3.4, + cos: 0.61, + }], + neighbours: vec![Neighbour { + article_id: 812, + label: "loved".into(), + cos: 0.71, + title: "A rated piece".into(), + }], + notes: vec!["knn gate 0.60 (n=14 rated with embeddings)".into()], + ..Signals::default() + } + } + + #[test] + fn signals_json_follows_the_plan_shape() { + let json = serialize_signals(&signals(41.0, 0.55), false); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["v"], 1); + assert_eq!(parsed["raw"]["heuristic"], 41.0); + assert_eq!(parsed["raw"]["interest_top1_cos"], 0.61); + assert_eq!(parsed["present"]["heuristic"], true); + assert_eq!(parsed["present"]["knn"], false); + assert_eq!(parsed["present"]["quality"], false); + assert!(parsed["raw"].get("knn").is_none()); + assert!(parsed["norm"].get("knn").is_none()); + assert_eq!(parsed["exploration"], false); + assert_eq!(parsed["auto_include"], false); + assert_eq!(parsed["top_interests"][0]["name"], "Gaussian Splatting"); + assert_eq!(parsed["neighbours"][0]["article_id"], 812); + assert_eq!( + parsed["notes"][0], + "knn gate 0.60 (n=14 rated with embeddings)" + ); + let typed: SignalsJson = serde_json::from_str(&json).unwrap(); + let weights: f64 = typed.weights.values().sum(); + assert!((weights - 1.0).abs() < 1e-9); + assert!(typed.blend().is_some()); + } + + #[tokio::test] + async fn rows_are_upserted_with_every_column_replaced() { + let (_dir, db) = db_with_articles(&[1]).await; + let run_id = db.start_run(date(), Timestamp::now()).await.unwrap(); + write( + &db, + &CandidateRun { + run_id, + article_id: 1, + stage: "eligible", + excluded_reason: Some("not_admitted"), + admitted_by: None, + signals_json: "{}", + utility: Some(1.0), + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: None, + }, + ) + .await + .unwrap(); + write( + &db, + &CandidateRun { + run_id, + article_id: 1, + stage: "selected", + excluded_reason: None, + admitted_by: Some("[\"prefilter\"]"), + signals_json: "{\"v\":1}", + utility: None, + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: Some("because"), + }, + ) + .await + .unwrap(); + let row = explain_row(&db, run_id, 1).await.unwrap().unwrap(); + assert_eq!(row.stage, "selected"); + assert_eq!(row.excluded_reason, None, "no COALESCE"); + assert_eq!(row.utility, None); + assert_eq!(row.admitted_by.as_deref(), Some("[\"prefilter\"]")); + assert_eq!(row.editor_why.as_deref(), Some("because")); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM candidate_runs") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(count, 1); + } + + #[tokio::test] + async fn explain_renders_persisted_rows_and_reports_never_ingested() { + let (_dir, db) = db_with_articles(&[1, 2]).await; + let run_id = db.start_run(date(), Timestamp::now()).await.unwrap(); + thin_excluded(&db, run_id, 2, "blocked").await.unwrap(); + let json = serialize_signals(&signals(41.0, 0.55), true); + write( + &db, + &CandidateRun { + run_id, + article_id: 1, + stage: "shortlisted", + excluded_reason: Some("not_selected"), + admitted_by: Some("[\"prefilter\"]"), + signals_json: &json, + utility: None, + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: None, + }, + ) + .await + .unwrap(); + // A later dry run must not shadow the real one. + let dry = db.start_run(date(), Timestamp::now()).await.unwrap(); + sqlx::query("UPDATE runs SET status = 'dry_run' WHERE id = ?") + .bind(dry) + .execute(db.pool()) + .await + .unwrap(); + assert_eq!(resolve_run(&db, date(), None).await.unwrap(), Some(run_id)); + assert_eq!( + resolve_run(&db, date(), Some(dry)).await.unwrap(), + Some(dry) + ); + assert_eq!( + resolve_run(&db, "2026-01-01".parse().unwrap(), Some(dry)) + .await + .unwrap(), + None + ); + + let text = explain(&db, date(), None, &ExplainTarget::Article(1)) + .await + .unwrap(); + assert!(text.contains("article 1: Article 1"), "{text}"); + assert!( + text.contains("stage: shortlisted · reason: not_selected"), + "{text}" + ); + let squashed = text.split_whitespace().collect::>().join(" "); + assert!( + squashed.contains("heuristic 41.000 · 0.550 · 0.364"), + "{text}" + ); + assert!(squashed.contains("knn absent"), "{text}"); + assert!(squashed.contains("quality absent"), "{text}"); + assert!( + text.contains("Gaussian Splatting · z 3.40 · cos 0.610"), + "{text}" + ); + assert!( + text.contains("loved · cos 0.710 · article 812 · A rated piece"), + "{text}" + ); + assert!(text.contains("admitted by: [\"prefilter\"]"), "{text}"); + assert!(text.contains("auto_include=true"), "{text}"); + assert!(text.contains("preliminary blend:"), "{text}"); + assert!(text.contains("note: knn gate"), "{text}"); + + let by_url = explain( + &db, + date(), + None, + &ExplainTarget::Url("https://example.com/1?utm_source=x".into()), + ) + .await + .unwrap(); + assert_eq!(by_url, text, "--url canonicalizes and finds the same row"); + + let thin = explain(&db, date(), None, &ExplainTarget::Article(2)) + .await + .unwrap(); + assert!(thin.contains("stage: excluded · reason: blocked"), "{thin}"); + + let missing = explain( + &db, + date(), + None, + &ExplainTarget::Url("https://nowhere.example/post".into()), + ) + .await + .unwrap(); + assert!(missing.contains("never ingested"), "{missing}"); + let missing_id = explain(&db, date(), None, &ExplainTarget::Article(99)) + .await + .unwrap(); + assert!(missing_id.contains("never ingested"), "{missing_id}"); + + let no_run = explain( + &db, + "2026-01-01".parse().unwrap(), + None, + &ExplainTarget::Article(1), + ) + .await + .unwrap(); + assert!(no_run.contains("no non-dry run"), "{no_run}"); + let not_considered = explain(&db, date(), Some(dry), &ExplainTarget::Article(1)) + .await + .unwrap(); + assert!( + not_considered.contains("was not considered by run"), + "{not_considered}" + ); + } + + #[tokio::test] + async fn near_misses_rank_by_blend_and_skip_selected_and_excluded() { + let (_dir, db) = db_with_articles(&[1, 2, 3, 4, 5]).await; + let run_id = db.start_run(date(), Timestamp::now()).await.unwrap(); + let rows = [ + (1, "selected", None, 0.9), + (2, "shortlisted", Some("not_selected"), 0.7), + (3, "eligible", Some("not_admitted"), 0.95), + (4, "shortlisted", Some("not_selected"), 0.1), + ]; + for (id, stage, reason, norm) in rows { + let json = serialize_signals(&signals(10.0, norm), false); + write( + &db, + &CandidateRun { + run_id, + article_id: id, + stage, + excluded_reason: reason, + admitted_by: None, + signals_json: &json, + utility: None, + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: None, + }, + ) + .await + .unwrap(); + } + thin_excluded(&db, run_id, 5, "published_before") + .await + .unwrap(); + let misses = near_misses(&db, run_id, 10).await.unwrap(); + assert_eq!( + misses.iter().map(|row| row.article_id).collect::>(), + vec![3, 2, 4] + ); + let text = explain_near_misses(&db, date(), None, 2).await.unwrap(); + assert!( + text.contains("top 2 not selected, by preliminary blend"), + "{text}" + ); + assert!( + text.contains("Article 3 · eligible, not_admitted"), + "{text}" + ); + assert!(!text.contains("Article 4"), "{text}"); + } + + #[tokio::test] + async fn prune_respects_rated_and_published() { + let (_dir, db) = db_with_articles(&[1, 2, 3, 4]).await; + let now = Timestamp::now(); + let old = fmt_ts(now - jiff::Span::new().hours(200 * 24)); + sqlx::query("UPDATE articles SET first_seen = ? WHERE id IN (1, 2, 3)") + .bind(&old) + .execute(db.pool()) + .await + .unwrap(); + let blob = encode_blob(&[0.5, 0.5]).unwrap(); + for id in 1..=4 { + sqlx::query( + "INSERT INTO article_embeddings + (article_id, model, dimension, input_hash, embedding, created_at) + VALUES (?, 'voyage-4-lite', 2, 'h', ?, ?)", + ) + .bind(id) + .bind(&blob) + .bind(&old) + .execute(db.pool()) + .await + .unwrap(); + } + sqlx::query( + "INSERT INTO rating_events (article_id, kind, source, label, value, event_at) + VALUES (1, 'explicit', 'cli', 'loved', 1.0, ?)", + ) + .bind(&old) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO issues (date, issue_number, generated_at) VALUES ('2026-02-01', 1, ?); + INSERT INTO issue_articles (issue_date, article_id, section) VALUES ('2026-02-01', 2, 'Top Stories');", + ) + .bind(&old) + .execute(db.pool()) + .await + .unwrap(); + + let old_run = db + .start_run("2026-02-01".parse().unwrap(), now) + .await + .unwrap(); + sqlx::query("UPDATE runs SET started_at = ? WHERE id = ?") + .bind(&old) + .bind(old_run) + .execute(db.pool()) + .await + .unwrap(); + let new_run = db.start_run(date(), now).await.unwrap(); + thin_excluded(&db, old_run, 1, "blocked").await.unwrap(); + thin_excluded(&db, new_run, 1, "blocked").await.unwrap(); + + let (embeddings, telemetry) = prune(&db, 120, 180, now).await.unwrap(); + assert_eq!( + embeddings, 1, + "only the old, unrated, unpublished article 3" + ); + assert_eq!(telemetry, 1, "only the old run's rows"); + let remaining: Vec = + sqlx::query_scalar("SELECT article_id FROM article_embeddings ORDER BY article_id") + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(remaining, vec![1, 2, 4]); + let runs: Vec = sqlx::query_scalar("SELECT run_id FROM candidate_runs") + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(runs, vec![new_run]); + } +} diff --git a/src/db.rs b/src/db.rs index 70eb550..8945579 100644 --- a/src/db.rs +++ b/src/db.rs @@ -321,6 +321,45 @@ impl Db { Ok(rows.iter().map(|r| r.get::("article_id")).collect()) } + /// Article ids published before this issue date; same-date regeneration is allowed (§8.1). + pub async fn previously_published_ids_before(&self, date: Date) -> Result> { + let rows = + sqlx::query("SELECT DISTINCT article_id FROM issue_articles WHERE issue_date < ?") + .bind(date.to_string()) + .fetch_all(&self.pool) + .await?; + Ok(rows + .iter() + .map(|row| row.get::("article_id")) + .collect()) + } + + /// Published article ids first seen at or after `since` (`features backfill`). + pub async fn published_article_ids_since(&self, since: Timestamp) -> Result> { + let rows = sqlx::query( + "SELECT DISTINCT ia.article_id FROM issue_articles ia + JOIN articles a ON a.id = ia.article_id + WHERE a.first_seen >= ? + ORDER BY ia.article_id", + ) + .bind(fmt_ts(since)) + .fetch_all(&self.pool) + .await?; + Ok(rows + .iter() + .map(|row| row.get::("article_id")) + .collect()) + } + + /// Every article id first seen at or after `since` (`features backfill --all`). + pub async fn article_ids_since(&self, since: Timestamp) -> Result> { + let rows = sqlx::query("SELECT id FROM articles WHERE first_seen >= ? ORDER BY id") + .bind(fmt_ts(since)) + .fetch_all(&self.pool) + .await?; + Ok(rows.iter().map(|row| row.get::("id")).collect()) + } + /// Articles the LLM scored below `threshold` within the last `days` (§3.5). pub async fn recently_low_scored_ids( &self, diff --git a/src/main.rs b/src/main.rs index cd9cdcf..fd94fa2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ //! Everything of substance lives in the library (`src/lib.rs`); this binary only //! parses flags, loads config, opens the database and dispatches. +use std::io::Write as _; use std::path::PathBuf; use anyhow::{Context, Result}; @@ -10,6 +11,8 @@ use clap::{Parser, Subcommand, ValueEnum}; use tracing_subscriber::EnvFilter; use daily_epub::config::Config; +use daily_epub::curate::embedding::{self, BACKFILL_CONFIRM_TOKENS}; +use daily_epub::curate::telemetry; use daily_epub::db::Db; use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome}; use daily_epub::report::RunReport; @@ -40,6 +43,11 @@ enum Command { /// Inspect and edit explicit article verdicts. #[command(subcommand)] Ratings(RatingsCommand), + /// Why an article was (not) in the paper, from persisted run telemetry. + Explain(ExplainArgs), + /// Embedding cache and telemetry maintenance. + #[command(subcommand)] + Features(FeaturesCommand), /// Re-poll social scores for recent entries. BackfillSocial(BackfillSocialArgs), /// Database maintenance. @@ -64,6 +72,9 @@ struct GenerateArgs { /// Skip every LLM call: prefilter order selects, excerpts stand in for summaries. #[arg(long)] skip_llm: bool, + /// Use cached embeddings only: zero Voyage calls. + #[arg(long)] + skip_embeddings: bool, } impl From<&GenerateArgs> for GenerateOptions { @@ -74,6 +85,7 @@ impl From<&GenerateArgs> for GenerateOptions { out: args.out.clone(), max_articles: args.max_articles, skip_llm: args.skip_llm, + skip_embeddings: args.skip_embeddings, } } } @@ -158,6 +170,55 @@ struct RatingsClearArgs { url: Option, } +/// `explain --date D (--article ID | --url URL) [--run-id N]` or +/// `explain --date D --near-misses [N]` (plan §15.2). +#[derive(Debug, clap::Args)] +struct ExplainArgs { + /// Issue date whose run to read. + #[arg(long, value_name = "YYYY-MM-DD")] + date: String, + /// Article id, as printed by `ratings list` or `explain --near-misses`. + #[arg( + long, + required_unless_present_any = ["url", "near_misses"], + conflicts_with_all = ["url", "near_misses"] + )] + article: Option, + /// Article URL; canonicalized before lookup. + #[arg(long, conflicts_with = "near_misses")] + url: Option, + /// A specific run of that date instead of the latest non-dry one. + #[arg(long, value_name = "N")] + run_id: Option, + /// The top N articles that were considered but not selected (default 10). + #[arg(long, value_name = "N", num_args = 0..=1, default_missing_value = "10")] + near_misses: Option, +} + +#[derive(Debug, Subcommand)] +enum FeaturesCommand { + /// Embed rated and published articles, then interests, into the cache. + Backfill(BackfillArgs), + /// Drop stale embeddings and old candidate telemetry per the retention config. + Prune, +} + +#[derive(Debug, clap::Args)] +struct BackfillArgs { + /// Window for published (and, with --all, other) articles. + #[arg(long, default_value_t = 30)] + days: i64, + /// Only the rated set. + #[arg(long, conflicts_with = "all")] + rated_only: bool, + /// Also every other article first seen inside the window. + #[arg(long)] + all: bool, + /// Skip the confirmation prompt above the token threshold. + #[arg(long)] + yes: bool, +} + #[derive(Debug, clap::Args)] struct BackfillSocialArgs { /// How many days back to re-poll. @@ -177,6 +238,14 @@ async fn main() -> Result<()> { let cli = Cli::parse(); let config = Config::load(cli.config.as_deref()).context("loading configuration")?; tracing::debug!(?config.database_path, "configuration loaded"); + // The root config ignores unknown sections, so say what was resolved (§19). + tracing::info!( + deepseek_model = %config.deepseek.model, + voyage_enabled = config.voyage.enabled, + voyage_model = %config.voyage.model, + voyage_dimension = config.voyage.output_dimension, + "providers resolved" + ); match cli.command { Command::Generate(args) => { @@ -196,6 +265,14 @@ async fn main() -> Result<()> { let db = Db::open_and_migrate(&config.database_path).await?; cmd_ratings(&config, &db, command).await?; } + Command::Explain(args) => { + let db = Db::open_and_migrate(&config.database_path).await?; + cmd_explain(&db, args).await?; + } + Command::Features(command) => { + let db = Db::open_and_migrate(&config.database_path).await?; + cmd_features(&config, &db, command).await?; + } Command::BackfillSocial(args) => { let db = Db::open_and_migrate(&config.database_path).await?; cmd_backfill_social(&db, args.days).await?; @@ -271,11 +348,22 @@ fn print_report(report: &RunReport) { report.counts.entries_dropped, ); println!( - "tokens: {} input · {} cached · {} output = ${:.4}", + "curation: {} eligible · {} embedded · {} rated w/ embeddings → {} candidates → {} scored → {} selected", + report.counts.eligible, + report.counts.embedded, + report.counts.rated_with_embeddings, + report.counts.candidates, + report.counts.llm_scored, + report.counts.selected, + ); + println!( + "tokens: {} input · {} cached · {} output · {} voyage = ${:.4} (voyage ${:.4})", report.usage.input_tokens, report.usage.cached_tokens, report.usage.output_tokens, + report.voyage_tokens, report.cost_usd, + report.voyage_cost_usd, ); for warning in &report.warnings { println!("warning: {warning}"); @@ -442,6 +530,101 @@ async fn cmd_ratings(config: &Config, db: &Db, command: RatingsCommand) -> Resul Ok(()) } +async fn cmd_explain(db: &Db, args: ExplainArgs) -> Result<()> { + let date: jiff::civil::Date = args + .date + .parse() + .with_context(|| format!("invalid --date {:?}, expected YYYY-MM-DD", args.date))?; + let text = if let Some(limit) = args.near_misses { + telemetry::explain_near_misses(db, date, args.run_id, limit).await? + } else { + let target = match (args.article, args.url) { + (Some(id), _) => telemetry::ExplainTarget::Article(id), + (None, Some(url)) => telemetry::ExplainTarget::Url(url), + (None, None) => anyhow::bail!("provide --article, --url or --near-misses"), + }; + telemetry::explain(db, date, args.run_id, &target).await? + }; + print!("{text}"); + Ok(()) +} + +async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Result<()> { + match command { + FeaturesCommand::Backfill(args) => { + if !config.voyage.enabled { + anyhow::bail!("voyage.enabled is false; nothing to backfill"); + } + let service = embedding::EmbeddingService::real(db.clone(), config.voyage.clone()) + .context("building the Voyage client")?; + let opts = embedding::BackfillOptions { + days: args.days, + rated_only: args.rated_only, + all: args.all, + }; + let plan = embedding::plan_backfill(db, config, &service, &opts).await?; + println!( + "backfill: {} articles ({} learned, {} other) + {} interests to embed, {} already cached", + plan.article_count(), + plan.learned.len(), + plan.others.len(), + plan.interests.len(), + plan.cached + ); + if plan.is_empty() { + println!("cache is warm; nothing to do"); + return Ok(()); + } + println!( + "estimate: ~{} tokens ≈ ${:.4} with {} at ${:.2}/M", + plan.estimated_tokens, + plan.estimated_cost_usd(), + config.voyage.model, + embedding::VOYAGE_PRICE_PER_MTOK + ); + if plan.estimated_tokens > BACKFILL_CONFIRM_TOKENS + && !args.yes + && !confirm("continue?")? + { + println!("aborted"); + return Ok(()); + } + let outcome = embedding::run_backfill(&service, &plan).await?; + println!( + "embedded {} articles and {} interests · {} tokens · ${:.4}", + outcome.articles_embedded, + outcome.interests_embedded, + outcome.tokens, + outcome.cost_usd + ); + } + FeaturesCommand::Prune => { + let ranking = &config.curation.ranking; + let (embeddings, rows) = telemetry::prune( + db, + ranking.embedding_retention_days, + ranking.telemetry_retention_days, + jiff::Timestamp::now(), + ) + .await?; + println!( + "pruned {embeddings} embeddings older than {} days and {rows} candidate rows older than {} days", + ranking.embedding_retention_days, ranking.telemetry_retention_days + ); + } + } + Ok(()) +} + +/// A y/N question on stdin; anything but a leading `y` is a no. +fn confirm(question: &str) -> Result { + print!("{question} [y/N] "); + std::io::stdout().flush()?; + let mut answer = String::new(); + std::io::stdin().read_line(&mut answer)?; + Ok(answer.trim().to_lowercase().starts_with('y')) +} + async fn cmd_backfill_social(db: &Db, days: u32) -> Result<()> { let http = http::build_client(http::DEFAULT_TIMEOUT)?; let enricher = social::SocialEnricher::new(http, db.clone()); @@ -473,6 +656,7 @@ mod tests { "--max-articles", "6", "--skip-llm", + "--skip-embeddings", ]) .unwrap(); match cli.command { @@ -482,10 +666,11 @@ mod tests { assert_eq!(a.out, Some(PathBuf::from("./out"))); assert_eq!(a.max_articles, Some(6)); assert!(a.skip_llm); + assert!(a.skip_embeddings); let opts = GenerateOptions::from(&a); assert_eq!(opts.date.as_deref(), Some("2026-08-15")); - assert!(opts.dry_run && opts.skip_llm); + assert!(opts.dry_run && opts.skip_llm && opts.skip_embeddings); assert_eq!(opts.max_articles, Some(6)); } other => panic!("expected generate, got {other:?}"), @@ -547,6 +732,129 @@ mod tests { assert_eq!(cli.config, Some(PathBuf::from("/tmp/x.toml"))); } + #[test] + fn parses_explain_and_features() { + match Cli::try_parse_from([ + "daily-epub", + "explain", + "--date", + "2026-09-02", + "--article", + "42", + "--run-id", + "7", + ]) + .unwrap() + .command + { + Command::Explain(args) => { + assert_eq!(args.date, "2026-09-02"); + assert_eq!(args.article, Some(42)); + assert_eq!(args.run_id, Some(7)); + assert_eq!(args.near_misses, None); + } + other => panic!("expected explain, got {other:?}"), + } + match Cli::try_parse_from([ + "daily-epub", + "explain", + "--date", + "2026-09-02", + "--url", + "https://example.com/post", + ]) + .unwrap() + .command + { + Command::Explain(args) => { + assert_eq!(args.url.as_deref(), Some("https://example.com/post")) + } + other => panic!("expected explain, got {other:?}"), + } + match Cli::try_parse_from([ + "daily-epub", + "explain", + "--date", + "2026-09-02", + "--near-misses", + ]) + .unwrap() + .command + { + Command::Explain(args) => assert_eq!(args.near_misses, Some(10)), + other => panic!("expected explain, got {other:?}"), + } + match Cli::try_parse_from([ + "daily-epub", + "explain", + "--date", + "2026-09-02", + "--near-misses", + "3", + ]) + .unwrap() + .command + { + Command::Explain(args) => assert_eq!(args.near_misses, Some(3)), + other => panic!("expected explain, got {other:?}"), + } + assert!(Cli::try_parse_from(["daily-epub", "explain", "--date", "2026-09-02"]).is_err()); + assert!( + Cli::try_parse_from([ + "daily-epub", + "explain", + "--date", + "2026-09-02", + "--article", + "1", + "--near-misses" + ]) + .is_err() + ); + + match Cli::try_parse_from([ + "daily-epub", + "features", + "backfill", + "--days", + "60", + "--all", + "--yes", + ]) + .unwrap() + .command + { + Command::Features(FeaturesCommand::Backfill(args)) => { + assert_eq!(args.days, 60); + assert!(args.all && args.yes && !args.rated_only); + } + other => panic!("expected features backfill, got {other:?}"), + } + match Cli::try_parse_from(["daily-epub", "features", "backfill"]) + .unwrap() + .command + { + Command::Features(FeaturesCommand::Backfill(args)) => assert_eq!(args.days, 30), + other => panic!("expected features backfill, got {other:?}"), + } + assert!( + Cli::try_parse_from([ + "daily-epub", + "features", + "backfill", + "--rated-only", + "--all" + ]) + .is_err() + ); + assert!(matches!( + Cli::try_parse_from(["daily-epub", "features", "prune"]) + .unwrap() + .command, + Command::Features(FeaturesCommand::Prune) + )); + } + #[tokio::test] async fn cli_set_and_clear_append_cli_events_with_latest_issue_date() { use sqlx::Row as _; diff --git a/src/pipeline.rs b/src/pipeline.rs index 5155cc6..6b6bbde 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -23,7 +23,7 @@ //! issue itself are upserted, `issue_articles` is replaced wholesale, and the //! published filenames are derived from the date. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use anyhow::{Context, Result}; @@ -32,14 +32,14 @@ use jiff::{Timestamp, Zoned}; use crate::config::Config; use crate::curate::llm::{LlmClient, UsageMeter}; -use crate::curate::{Curator, editorial, profile}; +use crate::curate::{Curator, editorial, embedding, prefilter, profile, signals, telemetry}; use crate::db::Db; use crate::extract::Extractor; use crate::miniflux::MinifluxClient; use crate::publish::Published; use crate::report::{RunReport, RunStatus}; use crate::types::{ - Article, Artifact, Colophon, Edition, Issue, IssueMeta, Lineup, reading_minutes, + Article, ArticleId, Artifact, Colophon, Edition, Issue, IssueMeta, Lineup, reading_minutes, }; use crate::{comments, dedupe, epub, http, miniflux, publish, social, world}; @@ -56,6 +56,8 @@ pub struct GenerateOptions { pub max_articles: Option, /// `--skip-llm`: no DeepSeek call at all. pub skip_llm: bool, + /// `--skip-embeddings`: read the cache but make zero Voyage calls. + pub skip_embeddings: bool, } /// What one run produced, for the caller to print (§3.13). @@ -195,6 +197,8 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul lookback_hours = config.lookback_hours, target, skip_llm = opts.skip_llm, + skip_embeddings = opts.skip_embeddings, + voyage_enabled = config.voyage.enabled, out = %out_dir.display(), "starting run" ); @@ -210,11 +214,13 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul let ctx = StageContext { config, db, + run_id, date, target, out_dir, dry_run: opts.dry_run, skip_llm: opts.skip_llm, + skip_embeddings: opts.skip_embeddings, }; let stages = match run_stages(&ctx, window_start, window_end, &mut report).await { Ok(stages) => { @@ -275,11 +281,13 @@ struct StageOutput { struct StageContext<'a> { config: &'a Config, db: &'a Db, + run_id: i64, date: Date, target: usize, out_dir: PathBuf, dry_run: bool, skip_llm: bool, + skip_embeddings: bool, } async fn run_stages( @@ -367,7 +375,11 @@ async fn run_stages( report.counts.social_hits = enricher.enrich_all(&mut articles).await as i64; report.timings.record("social", elapsed_ms(stage)); - // --- Stage 6: heuristic pre-filter (§3.5) --- + // --- Stage 6: hygiene, embeddings, and cheap signals (§8.1, §9) --- + let embeddings = build_embedding_service(ctx, report); + let feature_signals = prepare_features(ctx, &articles, &embeddings, report).await; + + // --- Stage 6b: the old heuristic pre-filter still gates in this step (§21) --- let stage = Timestamp::now(); let meter = UsageMeter::new(&config.deepseek, config.max_daily_usd); // `max_daily_usd` is a ceiling for the *day*, not for one invocation, so a @@ -392,22 +404,79 @@ async fn run_stages( .await .context("running the heuristic pre-filter")?; report.counts.candidates = candidates.len() as i64; + let admitted = candidates + .iter() + .map(|candidate| candidate.article.id) + .collect::>(); + let admitted_set = admitted.iter().copied().collect::>(); + let not_admitted = feature_signals + .keys() + .copied() + .filter(|id| !admitted_set.contains(id)) + .collect::>(); + record_stage( + ctx, + &feature_signals, + ¬_admitted, + "eligible", + Some("not_admitted"), + ) + .await + .context("recording prefilter telemetry")?; + record_stage(ctx, &feature_signals, &admitted, "admitted", None) + .await + .context("recording prefilter telemetry")?; report.timings.record("prefilter", elapsed_ms(stage)); // --- Stage 7: LLM scoring, then selection (§3.6 A + B) --- let stage = Timestamp::now(); - if llm_available && let Err(e) = curator.score(&mut candidates, date).await { - // A dead API or a tripped budget must not cost us the issue: selection - // degrades to prefilter order exactly as `--skip-llm` does. + if let Err(e) = curator.score(&mut candidates, date).await { report.warn(format!("LLM scoring failed; ranking heuristically: {e:#}")); } report.counts.llm_scored = candidates.iter().filter(|c| c.llm.is_some()).count() as i64; + let assessed = candidates + .iter() + .filter(|candidate| candidate.llm.is_some()) + .map(|candidate| candidate.article.id) + .collect::>(); + record_stage(ctx, &feature_signals, &assessed, "assessed", None) + .await + .context("recording assessment telemetry")?; + // Every prefilter survivor goes to the old selector, scored or not. + record_stage(ctx, &feature_signals, &admitted, "shortlisted", None) + .await + .context("recording shortlist telemetry")?; let mut lineup = curator .select(candidates, date) .await .context("selecting the lineup")?; report.counts.selected = lineup.picks.len() as i64; + let selected = lineup + .picks + .iter() + .map(|pick| pick.article.id) + .collect::>(); + let selected_set = selected.iter().copied().collect::>(); + let not_selected = admitted + .iter() + .copied() + .filter(|id| !selected_set.contains(id)) + .collect::>(); + // `Pick::why` arrives with the Claude editor (step 2); `editor_why` stays + // NULL until a pick carries one. + record_stage(ctx, &feature_signals, &selected, "selected", None) + .await + .context("recording selection telemetry")?; + record_stage( + ctx, + &feature_signals, + ¬_selected, + "shortlisted", + Some("not_selected"), + ) + .await + .context("recording selection telemetry")?; if lineup.picks.is_empty() { report.warn("the lineup is empty — check the lookback window and pre-filter"); } @@ -533,6 +602,241 @@ async fn run_stages( }) } +/// The cheap signals and hygiene outcome for one eligible article (§9). +#[derive(Debug, Clone)] +struct FeatureSignals { + signals: signals::Signals, + auto_include: bool, +} + +/// The embedding cache with a Voyage client behind it, or cache-only under +/// `--skip-embeddings`, `voyage.enabled = false` or a missing key (§16, §17). +fn build_embedding_service( + ctx: &StageContext<'_>, + report: &mut RunReport, +) -> embedding::EmbeddingService { + let (db, voyage) = (ctx.db.clone(), ctx.config.voyage.clone()); + if ctx.skip_embeddings { + tracing::info!("--skip-embeddings: using cached vectors only, no Voyage calls"); + return embedding::EmbeddingService::cached_only(db, voyage); + } + if !voyage.enabled { + tracing::info!("voyage disabled: using cached embeddings only"); + return embedding::EmbeddingService::cached_only(db, voyage); + } + match embedding::EmbeddingService::real(db.clone(), voyage.clone()) { + Ok(service) => service, + Err(embedding::EmbeddingError::MissingApiKey) => { + tracing::warn!( + "voyage enabled but {} is unset; using cached embeddings only", + embedding::VOYAGE_API_KEY_ENV + ); + embedding::EmbeddingService::cached_only(db, voyage) + } + Err(error) => { + report.warn(format!( + "Voyage unavailable; using cached embeddings only: {error}" + )); + embedding::EmbeddingService::cached_only(db, voyage) + } + } +} + +/// Hygiene, embeddings and cheap signals for every article (§8.1, §9). +/// +/// Hygiene-excluded articles get thin `candidate_runs` rows; every other +/// article gets an `eligible` row with its `signals_json`. Nothing here can +/// fail the run: embeddings and the learned signals degrade to absent (§17). +async fn prepare_features( + ctx: &StageContext<'_>, + articles: &[Article], + service: &embedding::EmbeddingService, + report: &mut RunReport, +) -> HashMap { + let (config, db) = (ctx.config, ctx.db); + let hygiene = match prefilter::PrefilterContext::load(db, ctx.date).await { + Ok(context) => context, + Err(error) => { + report.warn(format!( + "could not load hygiene history; signals skipped: {error}" + )); + return HashMap::new(); + } + }; + let published = hygiene + .already_published + .iter() + .copied() + .collect::>(); + let rejected = hygiene + .recently_rejected + .iter() + .copied() + .collect::>(); + let mut eligible = Vec::new(); + for article in articles { + let auto_include = prefilter::is_auto_include(article, &config.curation); + let reason = if published.contains(&article.id) { + Some("published_before") + } else if !auto_include && prefilter::is_blocked(article, &config.curation) { + Some("blocked") + } else if !auto_include && rejected.contains(&article.id) { + Some("recently_rejected") + } else { + None + }; + match reason { + Some(reason) => { + if let Err(error) = + telemetry::thin_excluded(db, ctx.run_id, article.id, reason).await + { + report.warn(format!( + "could not record excluded candidate {}: {error}", + article.id + )); + } + } + None => eligible.push(article.clone()), + } + } + report.counts.eligible = eligible.len() as i64; + + // --- embed (§7.1, §7.2) --- + let stage = Timestamp::now(); + let article_embeddings = match service.articles(&eligible).await { + Ok(embeddings) => embeddings, + Err(error) => { + report.warn(format!("article embedding stage degraded: {error}")); + HashMap::new() + } + }; + report.counts.embedded = article_embeddings.len() as i64; + let interests = + match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { + Ok(interests) => interests, + Err(error) => { + tracing::warn!(%error, "could not load standing interests for embeddings"); + Vec::new() + } + }; + let interest_embeddings = match service.interests(&interests).await { + Ok(embeddings) => embeddings, + Err(error) => { + report.warn(format!("interest embedding stage degraded: {error}")); + HashMap::new() + } + }; + if let Some(meter) = service.meter() { + report.voyage_tokens = meter.total_tokens(); + report.voyage_cost_usd = meter.cost_usd(); + } + tracing::info!( + eligible = eligible.len(), + embedded = article_embeddings.len(), + interests = interest_embeddings.len(), + voyage_tokens = report.voyage_tokens, + "embeddings ready" + ); + report.timings.record("embed", elapsed_ms(stage)); + + // --- signals (§9, §12.2, §12.4) --- + let stage = Timestamp::now(); + let ranking = &config.curation.ranking; + let (mut computed, preference) = match signals::compute_all( + db, + &eligible, + &article_embeddings, + &interest_embeddings, + &config.voyage, + ranking, + Timestamp::now(), + ) + .await + { + Ok(result) => result, + Err(error) => { + report.warn(format!("signal computation degraded: {error:#}")); + let state = signals::PreferenceState::default(); + ( + signals::compute( + &eligible, + &article_embeddings, + &interest_embeddings, + &state, + ranking, + ), + state.summary(), + ) + } + }; + report.counts.rated_with_embeddings = preference.rated_with_embeddings as i64; + let mut output = HashMap::new(); + for article in &eligible { + let auto_include = prefilter::is_auto_include(article, &config.curation); + let signals = computed + .remove(&article.id) + .unwrap_or_else(|| signals::Signals::baseline(article)); + output.insert( + article.id, + FeatureSignals { + signals, + auto_include, + }, + ); + } + let eligible_ids = eligible + .iter() + .map(|article| article.id) + .collect::>(); + if let Err(error) = record_stage(ctx, &output, &eligible_ids, "eligible", None).await { + report.warn(format!("could not record eligible candidates: {error}")); + } + report.timings.record("signals", elapsed_ms(stage)); + output +} + +/// Upsert the `candidate_runs` row of every listed article at a new stage +/// (§7.4). Articles without signals (hygiene-excluded) are left alone. +async fn record_stage( + ctx: &StageContext<'_>, + features: &HashMap, + ids: &[ArticleId], + stage: &str, + excluded_reason: Option<&str>, +) -> Result<()> { + let admitted = matches!(stage, "admitted" | "assessed" | "shortlisted" | "selected"); + for id in ids { + let Some(feature) = features.get(id) else { + continue; + }; + let json = telemetry::serialize_signals(&feature.signals, feature.auto_include); + let admitted_by = admitted.then_some(if feature.auto_include { + "[\"auto\"]" + } else { + "[\"prefilter\"]" + }); + telemetry::write( + ctx.db, + &telemetry::CandidateRun { + run_id: ctx.run_id, + article_id: *id, + stage, + excluded_reason, + admitted_by, + signals_json: &json, + utility: None, + rank_utility: None, + cluster_id: None, + cluster_rank: None, + editor_why: None, + }, + ) + .await + .with_context(|| format!("recording candidate {id} at stage {stage}"))?; + } + Ok(()) +} + /// Insert/refresh the `articles` rows and stamp the returned ids back on (§3.13). async fn persist_articles(db: &Db, articles: &mut [Article]) -> Result<()> { for article in articles.iter_mut() { @@ -724,4 +1028,342 @@ mod tests { assert_eq!(lineup.picks[0].summary.as_deref(), Some("An abstract.")); assert!(lineup.picks[1..].iter().all(|p| p.summary.is_none())); } + + use std::sync::Arc; + + use crate::curate::embedding::{EmbeddingClient, EmbeddingService, MockBackend}; + use crate::types::{Entry, ExtractMethod, SourceKind, SourceRef}; + use sqlx::Row as _; + + fn now() -> Timestamp { + "2026-09-02T09:00:00Z".parse().unwrap() + } + + fn run_date() -> Date { + "2026-09-02".parse().unwrap() + } + + fn fixture_article(entry_id: i64, host: &str, words: usize) -> Article { + let url = format!("https://{host}/post-{entry_id}"); + let body = (0..words) + .map(|i| format!("word{i}")) + .collect::>() + .join(" "); + Article { + id: 0, + canonical_url: url.clone(), + title: format!("Post {entry_id}"), + best_entry_id: entry_id, + content_html: format!("

{body}

"), + word_count: words as i64, + excerpt_only: false, + image_count: 0, + sources: vec![SourceRef { + entry_id, + feed_id: 100 + entry_id, + feed_title: format!("Feed {entry_id}"), + category: None, + kind: SourceKind::Feed, + }], + first_seen: now(), + url, + author: None, + feed_id: 100 + entry_id, + feed_title: format!("Feed {entry_id}"), + category: None, + published_at: None, + comments_url: None, + image_urls: vec![], + social: vec![], + extract_method: ExtractMethod::Miniflux, + } + } + + fn entry_for(article: &Article) -> Entry { + Entry { + id: article.best_entry_id, + feed_id: article.feed_id, + feed_title: Some(article.feed_title.clone()), + category: None, + title: article.title.clone(), + url: article.url.clone(), + canonical_url: Some(article.canonical_url.clone()), + author: None, + published_at: None, + comments_url: None, + raw_content: article.content_html.clone(), + fetched_at: now(), + } + } + + struct Harness { + _dir: tempfile::TempDir, + db: Db, + config: Config, + articles: Vec
, + run_id: i64, + } + + /// Four articles: two ordinary, one on a blocked host, one published yesterday. + async fn harness() -> Harness { + let dir = tempfile::tempdir().unwrap(); + let db = Db::open_and_migrate(&dir.path().join("run.db")) + .await + .unwrap(); + let mut config = Config::default(); + config.curation.blocked_domains = vec!["blocked.example".into()]; + config.voyage.output_dimension = 4; + config.target_article_count = 1; + config.interests_opml = dir.path().join("interests.opml"); + std::fs::write( + &config.interests_opml, + "", + ) + .unwrap(); + config.profile_path = dir.path().join("profile.md"); + std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); + + let mut articles = vec![ + fixture_article(1, "a.example", 1200), + fixture_article(2, "b.example", 900), + fixture_article(3, "blocked.example", 1500), + fixture_article(4, "d.example", 1400), + ]; + let entries = articles.iter().map(entry_for).collect::>(); + db.upsert_entries(&entries).await.unwrap(); + persist_articles(&db, &mut articles).await.unwrap(); + sqlx::query( + "INSERT INTO issues (date, issue_number, generated_at) + VALUES ('2026-09-01', 1, '2026-09-01T12:00:00Z')", + ) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO issue_articles (issue_date, article_id, section) + VALUES ('2026-09-01', ?, 'Top Stories')", + ) + .bind(articles[3].id) + .execute(db.pool()) + .await + .unwrap(); + let run_id = db.start_run(run_date(), now()).await.unwrap(); + Harness { + _dir: dir, + db, + config, + articles, + run_id, + } + } + + fn context<'a>(h: &'a Harness, skip_embeddings: bool) -> StageContext<'a> { + StageContext { + config: &h.config, + db: &h.db, + run_id: h.run_id, + date: run_date(), + target: h.config.target_article_count, + out_dir: PathBuf::from("."), + dry_run: true, + skip_llm: true, + skip_embeddings, + } + } + + fn mock_service(h: &Harness, backend: Arc) -> EmbeddingService { + let client = EmbeddingClient::with_backend(h.config.voyage.clone(), backend); + EmbeddingService::with_client(h.db.clone(), h.config.voyage.clone(), client) + } + + async fn stage_rows( + db: &Db, + run_id: i64, + ) -> BTreeMap, Option)> { + sqlx::query( + "SELECT article_id, stage, excluded_reason, admitted_by FROM candidate_runs + WHERE run_id = ? ORDER BY article_id", + ) + .bind(run_id) + .fetch_all(db.pool()) + .await + .unwrap() + .iter() + .map(|row| { + ( + row.get::("article_id"), + ( + row.get::("stage"), + row.get::, _>("excluded_reason"), + row.get::, _>("admitted_by"), + ), + ) + }) + .collect() + } + + #[tokio::test] + async fn mocked_run_writes_a_candidate_runs_row_for_every_considered_article() { + let h = harness().await; + let ctx = context(&h, false); + let backend = Arc::new(MockBackend::auto(4)); + let service = mock_service(&h, backend.clone()); + let mut report = RunReport::new(run_date(), now()); + + let features = prepare_features(&ctx, &h.articles, &service, &mut report).await; + let [a, b, blocked, published] = [ + h.articles[0].id, + h.articles[1].id, + h.articles[2].id, + h.articles[3].id, + ]; + assert_eq!( + features.keys().copied().collect::>(), + BTreeSet::from([a, b]) + ); + assert_eq!(report.counts.eligible, 2); + assert_eq!(report.counts.embedded, 2); + assert_eq!(report.counts.rated_with_embeddings, 0); + assert!(report.timings.0.contains_key("embed") && report.timings.0.contains_key("signals")); + assert!(report.voyage_tokens > 0); + // One batch for the two articles, one for the interest. + assert_eq!(backend.calls(), 2); + let signals = &features[&a].signals; + assert!(signals.heuristic.is_some()); + assert!( + signals.interest.is_some(), + "interest present under the raw fallback" + ); + assert!( + signals.knn.is_none() && signals.feed.is_none(), + "gates closed" + ); + assert!(signals.preliminary.is_some()); + + let rows = stage_rows(&h.db, h.run_id).await; + assert_eq!(rows.len(), 4, "one row per considered article"); + assert_eq!(rows[&blocked].0, "excluded"); + assert_eq!(rows[&blocked].1.as_deref(), Some("blocked")); + assert_eq!(rows[&published].0, "excluded"); + assert_eq!(rows[&published].1.as_deref(), Some("published_before")); + assert_eq!(rows[&a].0, "eligible"); + assert_eq!(rows[&a].1, None); + let thin: String = + sqlx::query_scalar("SELECT signals_json FROM candidate_runs WHERE article_id = ?") + .bind(blocked) + .fetch_one(h.db.pool()) + .await + .unwrap(); + assert_eq!(thin, "{}"); + + // The old prefilter and selector, with the stage transitions of step 3. + let curator = Curator::new(h.config.clone(), h.db.clone(), None); + let candidates = curator + .prefilter(h.articles.clone(), run_date()) + .await + .unwrap(); + let admitted = candidates.iter().map(|c| c.article.id).collect::>(); + assert_eq!( + admitted.iter().copied().collect::>(), + BTreeSet::from([a, b]) + ); + record_stage(&ctx, &features, &admitted, "admitted", None) + .await + .unwrap(); + record_stage(&ctx, &features, &admitted, "shortlisted", None) + .await + .unwrap(); + let lineup = curator.select(candidates, run_date()).await.unwrap(); + let selected = lineup + .picks + .iter() + .map(|p| p.article.id) + .collect::>(); + assert_eq!(selected.len(), 1); + let not_selected = admitted + .iter() + .copied() + .filter(|id| !selected.contains(id)) + .collect::>(); + record_stage(&ctx, &features, &selected, "selected", None) + .await + .unwrap(); + record_stage( + &ctx, + &features, + ¬_selected, + "shortlisted", + Some("not_selected"), + ) + .await + .unwrap(); + + let rows = stage_rows(&h.db, h.run_id).await; + assert_eq!(rows.len(), 4); + let (winner, loser) = (selected[0], not_selected[0]); + assert_eq!( + rows[&winner], + ("selected".into(), None, Some("[\"prefilter\"]".into())) + ); + assert_eq!( + rows[&loser], + ( + "shortlisted".into(), + Some("not_selected".into()), + Some("[\"prefilter\"]".into()) + ) + ); + let text = telemetry::explain( + &h.db, + run_date(), + Some(h.run_id), + &telemetry::ExplainTarget::Article(loser), + ) + .await + .unwrap(); + assert!( + text.contains("stage: shortlisted · reason: not_selected"), + "{text}" + ); + } + + #[tokio::test] + async fn skip_embeddings_makes_zero_voyage_calls_and_uses_the_cache() { + let h = harness().await; + let ctx = context(&h, true); + let mut report = RunReport::new(run_date(), now()); + let service = build_embedding_service(&ctx, &mut report); + assert!(!service.has_client(), "--skip-embeddings is cache-only"); + assert!(service.meter().is_none()); + let features = prepare_features(&ctx, &h.articles, &service, &mut report).await; + assert_eq!(features.len(), 2); + assert_eq!(report.counts.embedded, 0, "nothing cached yet"); + assert!(features.values().all(|f| f.signals.interest.is_none())); + assert!(features.values().all(|f| f.signals.heuristic.is_some())); + assert_eq!(report.voyage_tokens, 0); + } + + #[tokio::test] + async fn a_voyage_failure_degrades_to_absent_signals_and_the_run_continues() { + let h = harness().await; + let ctx = context(&h, false); + let backend = Arc::new(MockBackend::new()); // nothing scripted: every call fails + let service = mock_service(&h, backend.clone()); + let mut report = RunReport::new(run_date(), now()); + let features = prepare_features(&ctx, &h.articles, &service, &mut report).await; + assert!(backend.calls() >= 1); + assert_eq!(features.len(), 2); + assert_eq!(report.counts.eligible, 2); + assert_eq!(report.counts.embedded, 0); + assert!(report.error.is_none()); + for feature in features.values() { + assert!(feature.signals.interest.is_none() && feature.signals.knn.is_none()); + assert!(feature.signals.heuristic.is_some()); + assert!( + feature.signals.preliminary.is_some(), + "scored on what is present" + ); + } + assert_eq!(stage_rows(&h.db, h.run_id).await.len(), 4); + } } diff --git a/src/report.rs b/src/report.rs index ef0a5e6..89af4da 100644 --- a/src/report.rs +++ b/src/report.rs @@ -67,6 +67,12 @@ pub struct StageCounts { pub excerpt_only: i64, /// Social lookups that returned a hit (§3.4). pub social_hits: i64, + /// Articles passing hygiene and eligible for personalized signals. + pub eligible: i64, + /// Eligible articles with a valid embedding. + pub embedded: i64, + /// Current rated articles with a valid embedding. + pub rated_with_embeddings: i64, /// Articles surviving the heuristic pre-filter (§3.5). pub candidates: i64, /// Articles scored by the LLM (§3.6 stage A). @@ -102,6 +108,9 @@ pub struct RunReport { pub status: RunStatus, pub counts: StageCounts, pub usage: TokenUsage, + /// Voyage document/query tokens and cost for this run. + pub voyage_tokens: i64, + pub voyage_cost_usd: f64, pub cost_usd: f64, pub timings: StageTimings, /// Ingest window actually used, RFC3339 (§3.1). @@ -124,6 +133,8 @@ impl RunReport { status: RunStatus::Running, counts: StageCounts::default(), usage: TokenUsage::default(), + voyage_tokens: 0, + voyage_cost_usd: 0.0, cost_usd: 0.0, timings: StageTimings::default(), window_start: None, @@ -155,7 +166,8 @@ impl RunReport { price_output: f64, ) { self.finished_at = Some(finished_at); - self.cost_usd = self.usage.cost_usd(price_input, price_cached, price_output); + self.cost_usd = + self.usage.cost_usd(price_input, price_cached, price_output) + self.voyage_cost_usd; if self.status == RunStatus::Running { self.status = if self.warnings.is_empty() { RunStatus::Ok @@ -182,11 +194,12 @@ impl RunReport { /// Compact human-readable summary printed at the end of `generate`. pub fn summary_line(&self) -> String { format!( - "{} [{}] {} entries → {} articles → {} candidates → {} selected · ${:.4} · {}s", + "{} [{}] {} entries → {} articles → {} eligible → {} candidates → {} selected · ${:.4} · {}s", self.date, self.status, self.counts.entries_fetched, self.counts.articles, + self.counts.eligible, self.counts.candidates, self.counts.selected, self.cost_usd,