diff --git a/README.md b/README.md index 139bd43..fb85d59 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,7 @@ enabled on every request to an `anthropic`-kind provider. | **Node.js 18+** and a clone of [`epub-to-xtc-converter`](https://github.com/bigbag/epub-to-xtc-converter) | XTC/XTCH output for the Xteink X4 | Optional (`xtc.enabled = false` turns it off). Needs `npm install` **inside `cli/`**, and a settings JSON naming a real TTF/OTF — see below. It has **no global npm bin** — it is invoked as `node /cli/index.js convert …`, which is why `xtc.command`/`xtc.args` are fully general. | | A reverse proxy for `daily.hallada.net` → `127.0.0.1:3499` | rating links must be reachable from e-readers on the internet | TLS via your existing setup. | -`data/profile.md` is the hand-maintained reader profile; its optional interests are merged -with `data/scour-interests.opml`. Both paths are configurable. +`data/profile.md` is the hand-maintained reader profile; its path is configurable. --- @@ -366,7 +365,6 @@ prints what resolved. | `database_path` | `/var/lib/daily-epub/daily-epub.db` | SQLite file; parent dirs are created. | | `out_dir` | `/var/lib/daily-epub/out` | Where `generate` writes artifacts before publishing. | | `profile_path` | `data/profile.md` | Hand-maintained reader profile, loaded every run. | -| `interests_opml` | `data/scour-interests.opml` | Scour interests merged with the profile interests. | | `miniflux.base_url` | `http://127.0.0.1:8082` | Miniflux root (no `/v1`). | | `miniflux.api_key` | — | **`DAILY_EPUB_MINIFLUX__API_KEY`**. Required. | | `miniflux.page_limit` | `250` | Entries per page; Miniflux caps this at 250. | diff --git a/config.example.toml b/config.example.toml index dc3c7c0..b9b6a0a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -24,9 +24,8 @@ database_path = "/var/lib/daily-epub/daily-epub.db" # Default output directory for generated artifacts (overridden by `--out`). out_dir = "/var/lib/daily-epub/out" -# Hand-maintained reader profile and Scour interests merged into the system prompt. +# Hand-maintained reader profile loaded into the system prompt. profile_path = "data/profile.md" -interests_opml = "data/scour-interests.opml" [miniflux] base_url = "http://127.0.0.1:8082" diff --git a/docs/runbooks/curation-v2-migration.md b/docs/runbooks/curation-v2-migration.md index 87e5c18..4fb1d6f 100644 --- a/docs/runbooks/curation-v2-migration.md +++ b/docs/runbooks/curation-v2-migration.md @@ -9,7 +9,7 @@ What changes for the operator, in one paragraph: the binary is replaced; the SQL tables and drops `ratings`, `feed_priors` and `scores` (the migration copies your ratings first); `config.toml` loses a few keys and gains the `[llm]` / `[providers.*]` registry plus a `profile_path`; the env file gains two API keys and renames the DeepSeek one; a hand-maintained -`profile.md` is installed next to the OPML; the systemd units are unchanged. +`profile.md` is installed for the hand-maintained reader profile; the systemd units are unchanged. ## 0. Before touching the server @@ -70,12 +70,13 @@ Everything you do not mention keeps its documented default, so the edit is small | Old key | Why | |---|---| +| `interests_opml = …` (top level) | standing interests now live in SQLite; remove this before rollout because unknown keys fail startup | | `prefilter_keep = …` (top level) | replaced by `curation.ranking.deep_keep` (default 120) | | `max_daily_usd = …` (top level) | now per provider: `providers.deepseek.max_daily_usd` | | the whole `[deepseek]` table | becomes `[providers.deepseek]` + `[llm]` (see below) | | any `[anthropic]` table (only if you added one from an interim build) | becomes `[providers.anthropic]` | -**Add** near the top, next to `interests_opml`: +**Add** near the top: ```toml profile_path = "/var/lib/daily-epub/data/profile.md" @@ -83,8 +84,7 @@ profile_path = "/var/lib/daily-epub/data/profile.md" Use an absolute path. The default is `data/profile.md` *relative to the working directory*, which under the unit is `/var/lib/daily-epub`, so the default would resolve to the same place, but an -explicit path survives running one-off commands from another directory. Point -`interests_opml` at an absolute path too if it is still relative. +explicit path survives running one-off commands from another directory. **Add** the LLM registry. Carry over the `base_url`, `model` and `price_*` values from your old `[deepseek]` table if you had changed them; the values shown are the defaults. @@ -165,11 +165,9 @@ Voyage dashboards: the in-app `max_daily_usd` meters are runaway guards, not acc ```sh sudo install -d -m0750 -o daily-epub -g daily-epub /var/lib/daily-epub/data sudo install -m0640 -o daily-epub -g daily-epub data/profile.md /var/lib/daily-epub/data/profile.md -# if the OPML is not already there: -sudo install -m0640 -o daily-epub -g daily-epub data/scour-interests.opml /var/lib/daily-epub/data/ ``` -If the file is missing the run does not fail; it logs a warning and uses the OPML interests only, +If the file is missing the run does not fail; it logs a warning and uses empty profile prose, which is a much worse prompt. `config check` in the next step tells you whether it was found. ## 6. Check the config as the service user diff --git a/src/config.rs b/src/config.rs index d5bceb1..0de65ac 100644 --- a/src/config.rs +++ b/src/config.rs @@ -37,12 +37,9 @@ impl From for ConfigError { /// Legacy/alternate env var for the rating-link HMAC key (spec §1). pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET"; -/// Root configuration document (§3.14). -/// -/// Unknown *top-level* keys are ignored on purpose: the prefix `DAILY_EPUB_` is -/// shared with plain operator env vars such as [`ENV_SECRET_ALIAS`]. +/// Root configuration document; unknown keys fail so retired settings stay visible. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(default)] +#[serde(deny_unknown_fields, default)] pub struct Config { /// IANA tz used for day boundaries and `--date` (§3.14, notes §2). pub timezone: String, @@ -65,8 +62,6 @@ pub struct Config { pub database_path: PathBuf, /// Default artifact output directory (overridden by `generate --out`). pub out_dir: PathBuf, - /// Scour interests OPML used to seed the taste profile (§3.6). - pub interests_opml: PathBuf, /// Hand-maintained reader profile loaded for every curation run (§8.2). pub profile_path: PathBuf, @@ -99,7 +94,6 @@ impl Default for Config { world_briefing: true, database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"), out_dir: PathBuf::from("/var/lib/daily-epub/out"), - interests_opml: PathBuf::from("data/scour-interests.opml"), profile_path: PathBuf::from("data/profile.md"), miniflux: MinifluxConfig::default(), llm: LlmConfig::default(), @@ -942,7 +936,11 @@ impl Config { fig = fig.merge(Toml::file(p)); } } - Ok(fig.merge(Env::prefixed(ENV_PREFIX).split(ENV_SPLIT))) + Ok(fig.merge( + Env::prefixed(ENV_PREFIX) + .ignore(&["secret"]) + .split(ENV_SPLIT), + )) } /// The file `load` reads: the explicit `--config` path, else `./config.toml` @@ -1067,7 +1065,6 @@ impl Config { }); lines.push(file_line("database_path", &self.database_path)); lines.push(file_line("profile_path", &self.profile_path)); - lines.push(file_line("interests_opml", &self.interests_opml)); for (role, name) in self.llm.roles() { match self.providers.get(name) { Some(provider) => lines.push(provider_line(&format!("llm.{role}"), name, provider)), @@ -2031,7 +2028,7 @@ mod tests { for (key, default) in defaults.as_object().expect("config is a table") { let section = match key.as_str() { "curation" | "llm" | "providers" | "voyage" | "editorial" => key, - "target_article_count" | "profile_path" | "interests_opml" => key, + "target_article_count" | "profile_path" => key, _ => continue, }; let documented = documented diff --git a/src/curate/embedding.rs b/src/curate/embedding.rs index 405a85f..4539f69 100644 --- a/src/curate/embedding.rs +++ b/src/curate/embedding.rs @@ -19,9 +19,10 @@ use sha2::{Digest as _, Sha256}; use sqlx::Row as _; use crate::config::{Config, VoyageConfig}; -use crate::curate::{approx_tokens, profile, prompt_text}; +use crate::curate::{approx_tokens, prompt_text}; use crate::db::{Db, fmt_ts}; use crate::http::RetryPolicy; +use crate::interests; use crate::types::{Article, ArticleId}; /// The only place the Voyage key comes from (§4.3). @@ -898,14 +899,7 @@ pub async fn plan_backfill( } } - let interests = - match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { - Ok(interests) => interests, - Err(error) => { - tracing::warn!(%error, "could not load standing interests; skipping them"); - Vec::new() - } - }; + let interest_names = interests::names(db).await?; let mut plan = BackfillPlan::default(); let mut keep = |articles: Vec
, misses: Vec<(ArticleId, i64)>| -> Vec
{ @@ -922,8 +916,8 @@ pub async fn plan_backfill( let other_misses = service.uncached_articles(&others).await?; plan.others = keep(others, other_misses); - let interest_misses = service.uncached_interests(&interests).await?; - plan.cached += interests.len() - interest_misses.len(); + let interest_misses = service.uncached_interests(&interest_names).await?; + plan.cached += interest_names.len() - interest_misses.len(); plan.estimated_tokens += interest_misses .iter() .map(|interest| approx_tokens(interest) as i64) @@ -1391,7 +1385,7 @@ mod tests { #[tokio::test] async fn backfill_prioritizes_the_learned_set_and_is_idempotent() { - let (dir, db) = db_with_articles(&[1, 2, 3]).await; + let (_dir, db) = db_with_articles(&[1, 2, 3]).await; // Article 1 is rated, article 2 is published, article 3 is neither. sqlx::query( "INSERT INTO rating_events (article_id, kind, source, label, value, event_at) @@ -1418,16 +1412,11 @@ mod tests { let config = Config { voyage: small_config(), - interests_opml: dir.path().join("interests.opml"), - profile_path: dir.path().join("profile.md"), ..Config::default() }; - std::fs::write( - &config.interests_opml, - "", - ) - .unwrap(); - std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); + interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now()) + .await + .unwrap(); let backend = Arc::new(MockBackend::auto(4)); let svc = service(db.clone(), config.voyage.clone(), backend.clone()); diff --git a/src/curate/profile/mod.rs b/src/curate/profile/mod.rs index 3a05bf5..054bec2 100644 --- a/src/curate/profile/mod.rs +++ b/src/curate/profile/mod.rs @@ -3,7 +3,6 @@ //! Every run rebuilds one byte-stable prompt from the hand-maintained profile, //! standing interests, stored weekly adjustments, and current explicit verdicts. -use std::collections::BTreeSet; use std::fmt::Write as _; use std::path::Path; @@ -13,6 +12,7 @@ use serde::{Deserialize, Serialize}; use super::llm::LlmClient; use crate::db::{Db, KV_PROFILE_VERSION, KV_TASTE_PROFILE}; +use crate::interests; use crate::types::{Facets, RatedArticle, TasteProfile}; pub const REBUILD_INTERVAL_DAYS: i64 = 7; @@ -29,50 +29,9 @@ pub const NO_LEARNED_ADJUSTMENTS: &str = "No reader ratings have been collected const EDITOR_IN_CHIEF_FRAMING: &str = "You are the editor-in-chief of *The Daily EPUB*, a personal morning newspaper assembled every day for exactly one reader. Everything you are asked to do — score, select, place, summarize, introduce — serves his taste, not a general audience's. When a judgement call is close, re-read this profile and decide the way he would."; // --------------------------------------------------------------------------- -// Interest and profile-file parsing +// Profile-file parsing // --------------------------------------------------------------------------- -pub fn parse_interests(opml_path: &Path) -> anyhow::Result> { - let raw = std::fs::read_to_string(opml_path) - .with_context(|| format!("reading the interests OPML at {}", opml_path.display()))?; - let interests = parse_interests_str(&raw); - if interests.is_empty() { - anyhow::bail!( - "no interests found in {}", - opml_path.display() - ); - } - tracing::debug!(count = interests.len(), "parsed scour interests"); - Ok(interests) -} - -pub fn parse_interests_str(raw: &str) -> Vec { - let mut seen = BTreeSet::new(); - let mut out = Vec::new(); - for chunk in raw.split("text=\"").skip(1) { - let Some((value, _)) = chunk.split_once('"') else { - continue; - }; - let name = xml_unescape(value).trim().to_string(); - if !name.is_empty() && seen.insert(name.to_lowercase()) { - out.push(name); - } - } - out -} - -fn xml_unescape(s: &str) -> String { - if !s.contains('&') { - return s.to_string(); - } - s.replace("<", "<") - .replace(">", ">") - .replace(""", "\"") - .replace("'", "'") - .replace("'", "'") - .replace("&", "&") -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProfileFile { /// Original Markdown with every `## Interests` section removed. @@ -116,7 +75,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result { match std::fs::read_to_string(path) { Ok(raw) => Ok(parse_profile_str(&raw)), Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - tracing::warn!(path = %path.display(), "profile file is missing; using OPML interests only"); + tracing::warn!(path = %path.display(), "profile file is missing; using empty profile prose"); Ok(ProfileFile { body: String::new(), interests: Vec::new(), @@ -128,30 +87,7 @@ pub fn load_profile(path: &Path) -> anyhow::Result { } } -/// Load the exact standing-interest union used in the system prompt. -pub fn load_standing_interests( - opml_path: &Path, - profile_path: &Path, -) -> anyhow::Result> { - let opml = parse_interests(opml_path)?; - let profile = load_profile(profile_path)?; - Ok(union_interests(opml, profile.interests)) -} - -fn union_interests(opml: Vec, profile: Vec) -> Vec { - let mut seen = BTreeSet::new(); - let mut out = Vec::new(); - for interest in opml.into_iter().chain(profile) { - let interest = interest.trim(); - if !interest.is_empty() && seen.insert(interest.to_lowercase()) { - out.push(interest.to_string()); - } - } - out -} - pub mod themes; -pub use themes::group_into_themes; // --------------------------------------------------------------------------- // Prompt assembly @@ -174,7 +110,7 @@ fn one_line(text: &str) -> String { /// Assemble sections in the exact cache-friendly order required by §8.4. pub fn build( profile_body: &str, - interests: &[String], + grouped: &[(String, Vec)], learned_adjustments: &str, ratings: &[RatedArticle], verdict_limit: usize, @@ -193,8 +129,8 @@ pub fn build( doc.push_str("## Standing interests\n\n"); doc.push_str("These are his subscribed interest topics, grouped. They raise the floor for a match, but never cap the paper: an outstanding article on none of these still belongs.\n\n"); - for (theme, members) in group_into_themes(interests) { - let _ = writeln!(doc, "- **{}**: {}", theme, members.join(", ")); + for (category, members) in grouped { + let _ = writeln!(doc, "- **{}**: {}", category, members.join(", ")); } doc.push_str("\n## Learned adjustments (rebuilt weekly from ratings)\n\n"); @@ -272,26 +208,27 @@ async fn store_version(db: &Db, version: i64, built_at: Timestamp) -> anyhow::Re async fn prompt_inputs( db: &Db, - opml_path: &Path, profile_path: &Path, -) -> anyhow::Result<(ProfileFile, Vec, Vec, String)> { - let opml = parse_interests(opml_path)?; +) -> anyhow::Result<( + ProfileFile, + Vec<(String, Vec)>, + Vec, + String, +)> { let profile = load_profile(profile_path)?; - let interests = union_interests(opml, profile.interests.clone()); + let grouped = interests::grouped(db).await?; let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?; let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); - Ok((profile, interests, ratings, learned)) + Ok((profile, grouped, ratings, learned)) } /// Rebuild the complete system prompt from its live inputs on every run. pub async fn load_or_build( db: &Db, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result { - let (profile_file, interests, ratings, learned) = - prompt_inputs(db, opml_path, profile_path).await?; + let (profile_file, grouped, ratings, learned) = prompt_inputs(db, profile_path).await?; let (version, built_at) = match stored_version(db).await? { Some(stored) => stored, None => { @@ -303,7 +240,7 @@ pub async fn load_or_build( let profile = TasteProfile { text: build( &profile_file.body, - &interests, + &grouped, &learned, &ratings, verdict_limit, @@ -315,7 +252,10 @@ pub async fn load_or_build( db.kv_set(KV_TASTE_PROFILE, &profile.text).await?; tracing::debug!( version, - interests = interests.len(), + interests = grouped + .iter() + .map(|(_, members)| members.len()) + .sum::(), verdicts = ratings.len().min(verdict_limit), chars = profile.text.len(), "rebuilt the taste profile prompt" @@ -334,7 +274,6 @@ pub async fn is_stale(db: &Db) -> anyhow::Result { pub async fn weekly_rebuild_if_due( db: &Db, llm: &LlmClient, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result> { @@ -346,9 +285,7 @@ pub async fn weekly_rebuild_if_due( return Ok(None); } tracing::info!("taste profile is over a week old; rebuilding learned adjustments"); - Ok(Some( - rebuild(db, llm, opml_path, profile_path, verdict_limit).await?, - )) + Ok(Some(rebuild(db, llm, profile_path, verdict_limit).await?)) } // --------------------------------------------------------------------------- @@ -429,15 +366,12 @@ pub fn build_rebuild_prompt(ratings: &[RatedArticle]) -> String { pub async fn rebuild( db: &Db, llm: &LlmClient, - opml_path: &Path, profile_path: &Path, verdict_limit: usize, ) -> anyhow::Result { - // Read the prompt inputs first: a rebuild that dies on a missing OPML must - // stay due and must not have spent a model call getting there. - let opml = parse_interests(opml_path)?; + // Read the prompt inputs before spending a model call. let profile_file = load_profile(profile_path)?; - let interests = union_interests(opml, profile_file.interests.clone()); + let grouped = interests::grouped(db).await?; let ratings = db.current_ratings(RATINGS_LOOKBACK_DAYS).await?; let previous = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); let learned = if ratings.is_empty() { @@ -473,7 +407,7 @@ pub async fn rebuild( let profile = TasteProfile { text: build( &profile_file.body, - &interests, + &grouped, &learned, &ratings, verdict_limit, @@ -495,18 +429,15 @@ pub async fn rebuild( mod tests { use super::*; - const OPML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/scour-interests.opml"); const PROFILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data/profile.md"); #[test] - fn profile_interests_are_removed_and_union_case_insensitively() { + fn profile_interests_are_removed_for_the_importer() { let parsed = parse_profile_str( "# P\n\n## Interests\n- Rust\nBoston Tech\n- rust\n\n## Notes\nKeep this.\n", ); assert_eq!(parsed.body, "# P\n\n## Notes\nKeep this.\n"); assert_eq!(parsed.interests, ["Rust", "Boston Tech", "rust"]); - let union = union_interests(vec!["rust".into(), "E-Ink".into()], parsed.interests); - assert_eq!(union, ["rust", "E-Ink", "Boston Tech"]); } #[test] @@ -526,7 +457,7 @@ mod tests { }; let prompt = build( "# Reader profile\n\nProfile prose.", - &["Rust".into()], + &[("Software".into(), vec!["Rust".into(), "SQLite".into()])], "- Adjust.", &[rating], 60, @@ -539,17 +470,16 @@ mod tests { assert!( framing < profile && profile < interests && interests < learned && learned < verdicts ); + assert!(prompt.contains("- **Software**: Rust, SQLite")); assert!(prompt.contains("NOT FOR ME | A title | A feed | A summary with whitespace.")); } #[test] - fn shipped_profile_and_opml_parse() { + fn shipped_profile_parses() { let profile = load_profile(Path::new(PROFILE_PATH)).unwrap(); assert!(profile.body.contains("## Who he is")); assert!(!profile.body.contains("## Interests")); assert!(profile.interests.is_empty()); - let interests = parse_interests(Path::new(OPML_PATH)).unwrap(); - assert!(interests.iter().any(|interest| interest == "Rust")); } #[test] @@ -594,19 +524,21 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write( &profile_path, "# Reader profile\n\nOriginal prose.\n\n## Interests\n- Custom Topic\n", ) .unwrap(); - let first = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let first = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(first.version, 1); assert!(first.text.contains("Original prose.")); - assert!(first.text.contains("Custom Topic")); + assert!(first.text.contains("- **Software**: Rust")); + assert!(!first.text.contains("Custom Topic")); assert!(!first.text.contains("## Interests")); std::fs::write( @@ -614,11 +546,11 @@ mod tests { "# Reader profile\n\nChanged prose.\n\n## Interests\n- Another Topic\n", ) .unwrap(); - let second = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let second = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(second.version, first.version); assert_eq!(second.built_at, first.built_at); assert!(second.text.contains("Changed prose.")); - assert!(second.text.contains("Another Topic")); + assert!(!second.text.contains("Another Topic")); assert!(!second.text.contains("Original prose.")); let missing = load_profile(&dir.path().join("missing.md")).unwrap(); @@ -637,11 +569,12 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap(); - let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let initial = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(initial.version, 1); sqlx::query( @@ -677,7 +610,7 @@ mod tests { UsageMeter::for_provider(&ProviderConfig::deepseek()), backend.clone(), ); - let rebuilt = rebuild(&db, &llm, &opml, &profile_path, 60).await.unwrap(); + let rebuilt = rebuild(&db, &llm, &profile_path, 60).await.unwrap(); assert_eq!(rebuilt.version, 2); assert!(rebuilt.text.contains("Rank first-hand reports higher.")); assert!( @@ -700,16 +633,16 @@ mod tests { let db = Db::open_and_migrate(&dir.path().join("profile.db")) .await .unwrap(); - let opml = dir.path().join("interests.opml"); let profile_path = dir.path().join("profile.md"); - std::fs::write(&opml, r#""#).unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); std::fs::write(&profile_path, "# Reader profile\n\nLikes depth.\n").unwrap(); - let initial = load_or_build(&db, &opml, &profile_path, 60).await.unwrap(); + let initial = load_or_build(&db, &profile_path, 60).await.unwrap(); assert_eq!(initial.version, 1); - // The OPML goes missing the way a relative path does under a service - // whose working directory is not the checkout. - std::fs::remove_file(&opml).unwrap(); + std::fs::remove_file(&profile_path).unwrap(); + std::fs::create_dir(&profile_path).unwrap(); let backend = Arc::new(MockBackend::new()); let llm = LlmClient::with_backend( @@ -718,10 +651,10 @@ mod tests { UsageMeter::for_provider(&ProviderConfig::deepseek()), backend.clone(), ); - let error = rebuild(&db, &llm, &opml, &profile_path, 60) + let error = rebuild(&db, &llm, &profile_path, 60) .await - .expect_err("a missing OPML fails the rebuild"); - assert!(format!("{error:#}").contains("reading the interests OPML")); + .expect_err("an unreadable profile fails the rebuild"); + assert!(format!("{error:#}").contains("reading the reader profile")); // Still version 1, so the profile stays stale and the rebuild is retried. assert_eq!(stored_version(&db).await.unwrap().unwrap().0, 1); diff --git a/src/interests.rs b/src/interests.rs index 918ed6f..aae23c2 100644 --- a/src/interests.rs +++ b/src/interests.rs @@ -3,7 +3,7 @@ //! Interest queries stay here so the central database layer remains focused on //! the pipeline's shared records. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use anyhow::{Result, bail}; use jiff::Timestamp; @@ -64,6 +64,35 @@ pub struct Rates { const INTEREST_COLUMNS: &str = "id, name, category, created_at, categorized_at"; +/// Parse an OPML export for the one-time interests importer. +pub fn parse_opml(raw: &str) -> Vec { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for chunk in raw.split("text=\"").skip(1) { + let Some((value, _)) = chunk.split_once('"') else { + continue; + }; + let name = xml_unescape(value).trim().to_string(); + if !name.is_empty() && seen.insert(name.to_lowercase()) { + out.push(name); + } + } + out +} + +fn xml_unescape(value: &str) -> String { + if !value.contains('&') { + return value.to_string(); + } + value + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("&", "&") +} + fn interest_from(row: &sqlx::sqlite::SqliteRow) -> Interest { Interest { id: row.get("id"), @@ -326,6 +355,29 @@ mod tests { (dir, db) } + #[test] + fn opml_parser_unescapes_trims_and_deduplicates_names() { + let interests = parse_opml( + r#" + + + + + + + "#, + ); + assert_eq!( + interests, + [ + "Rust", + "E-Ink & RSS", + "Quotes \"and\" apostrophes 'x' 'y'", + "Markup ", + ] + ); + } + async fn seed_article(db: &Db, id: ArticleId) { sqlx::query( "INSERT INTO articles (id, canonical_url, title, first_seen) VALUES (?, ?, ?, ?)", diff --git a/src/main.rs b/src/main.rs index bc04c74..eb9b2a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -724,7 +724,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result { use curate::llm::{Llms, provider_meters}; let profile = curate::profile::load_or_build( db, - &config.interests_opml, &config.profile_path, config.curation.feedback.verdicts_in_prompt, ) @@ -749,7 +748,6 @@ async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result { let rebuilt = curate::profile::rebuild( db, llm, - &config.interests_opml, &config.profile_path, config.curation.feedback.verdicts_in_prompt, ) diff --git a/src/pipeline.rs b/src/pipeline.rs index f5b3b8e..c13ece6 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -46,7 +46,7 @@ use crate::types::{ Article, ArticleId, Artifact, BehindThePaper, Candidate, Colophon, Edition, Issue, IssueMeta, Lineup, Models, TokenUsage, reading_minutes, }; -use crate::{comments, dedupe, discovery, epub, http, miniflux, publish, social, world}; +use crate::{comments, dedupe, discovery, epub, http, interests, miniflux, publish, social, world}; /// One `generate` invocation's inputs — the CLI flags, already parsed (§2). #[derive(Debug, Clone, Default)] @@ -943,15 +943,14 @@ async fn prepare_features( } }; report.counts.embedded = article_embeddings.len() as i64; - let interests = - match profile::load_standing_interests(&config.interests_opml, &config.profile_path) { - Ok(interests) => interests, - Err(error) => { - tracing::warn!(%error, "could not load standing interests for embeddings"); - Vec::new() - } - }; - let interest_embeddings = match service.interests(&interests).await { + let interest_names = match interests::names(db).await { + Ok(interests) => interests, + Err(error) => { + tracing::warn!(%error, "could not load standing interests for embeddings"); + Vec::new() + } + }; + let interest_embeddings = match service.interests(&interest_names).await { Ok(embeddings) => embeddings, Err(error) => { report.warn(format!("interest embedding stage degraded: {error}")); @@ -1138,7 +1137,6 @@ async fn build_llms( ) -> Llms { let profile = match profile::load_or_build( ctx.db, - &ctx.config.interests_opml, &ctx.config.profile_path, ctx.config.curation.feedback.verdicts_in_prompt, ) @@ -1168,7 +1166,6 @@ async fn build_llms( match profile::weekly_rebuild_if_due( ctx.db, rebuild_client, - &ctx.config.interests_opml, &ctx.config.profile_path, ctx.config.curation.feedback.verdicts_in_prompt, ) @@ -1577,12 +1574,9 @@ mod tests { config.curation.blocked_domains = vec!["blocked.example".into()]; config.voyage.output_dimension = 4; config.target_article_count = 1; - config.interests_opml = dir.path().join("interests.opml"); - std::fs::write( - &config.interests_opml, - "", - ) - .unwrap(); + interests::add(&db, "Writerdeck", Some("Publishing"), Timestamp::now()) + .await + .unwrap(); config.profile_path = dir.path().join("profile.md"); std::fs::write(&config.profile_path, "# Reader profile\n").unwrap(); @@ -1708,6 +1702,7 @@ mod tests { assert!(report.voyage_tokens > 0); // One batch for the two articles, one for the interest. assert_eq!(backend.calls(), 2); + assert_eq!(backend.requests()[1].input, ["Writerdeck"]); let signals = &features .iter() .find(|candidate| candidate.article.id == a) diff --git a/src/web/dashboard/profile.rs b/src/web/dashboard/profile.rs index 2e49475..f3eb33f 100644 --- a/src/web/dashboard/profile.rs +++ b/src/web/dashboard/profile.rs @@ -1,8 +1,7 @@ //! Dashboard: the profile page (`/dashboard/profile`, web plan §11). //! -//! Edits `profile.md` with version history, shows what the loader parses out -//! of it, the standing OPML interests by theme, the stored system prompt and -//! the weekly learned adjustments, and offers the `profile-rebuild` job. +//! Edits `profile.md` with version history and shows the stored interests, +//! system prompt, and weekly learned adjustments. use std::path::Path; @@ -18,6 +17,7 @@ use sqlx::Row; use crate::curate::profile::{self, KV_LEARNED_ADJUSTMENTS, ProfileFile, REBUILD_INTERVAL_DAYS}; use crate::db::{Db, DbError, KV_TASTE_PROFILE}; +use crate::interests; use crate::server::AppState; use crate::web::session::{AuthSession, Viewer}; use crate::web::{Flash, Html, Page, WebError, format_time, take_flash}; @@ -104,8 +104,7 @@ fn read_profile(path: &Path) -> anyhow::Result> { } } -/// The live preview of what the loader extracts (§11): the passthrough body -/// and the `## Interests` lines. +/// The live preview of the prose that reaches the prompt. pub fn preview(content: &str) -> ProfileFile { profile::parse_profile_str(content) } @@ -191,7 +190,7 @@ async fn versions(db: &Db, config: &crate::config::Config) -> Result, versions: Vec, - opml_path: String, - opml_count: usize, - opml_error: String, - themes: Vec, + interest_count: usize, + category_count: usize, + categories: Vec, prompt: String, prompt_chars: usize, prompt_version: String, @@ -254,20 +251,17 @@ async fn show( let content = stored.unwrap_or_default(); let parsed = preview(&content); - let (opml_count, opml_error, themes) = match profile::parse_interests(&config.interests_opml) { - Ok(interests) => { - let themes = profile::group_into_themes(&interests) - .into_iter() - .map(|(name, members)| ThemeView { - name, - count: members.len(), - members: members.join(", "), - }) - .collect(); - (interests.len(), String::new(), themes) - } - Err(error) => (0, format!("{error:#}"), Vec::new()), - }; + let grouped = interests::grouped(db).await.map_err(WebError::Internal)?; + let interest_count = grouped.iter().map(|(_, members)| members.len()).sum(); + let category_count = grouped.len(); + let categories = grouped + .into_iter() + .map(|(name, members)| CategoryView { + name, + count: members.len(), + members: members.join(", "), + }) + .collect(); let prompt = db.kv_get(KV_TASTE_PROFILE).await?.unwrap_or_default(); let learned = db.kv_get(KV_LEARNED_ADJUSTMENTS).await?.unwrap_or_default(); @@ -297,12 +291,10 @@ async fn show( max_bytes: MAX_PROFILE_BYTES, content, preview_body: parsed.body, - preview_interests: parsed.interests, versions: versions(db, &config).await?, - opml_path: config.interests_opml.display().to_string(), - opml_count, - opml_error, - themes, + interest_count, + category_count, + categories, prompt_chars: prompt.len(), prompt_verdicts: count_verdict_lines(&prompt), prompt, @@ -516,15 +508,15 @@ mod tests { .unwrap(); let config = Config { profile_path: dir.path().join("profile.md"), - interests_opml: dir.path().join("interests.opml"), ..Config::default() }; std::fs::write(&config.profile_path, "# Original\n\nProse.\n").unwrap(); - std::fs::write( - &config.interests_opml, - r#""#, - ) - .unwrap(); + interests::add(&db, "Rust", Some("Software"), Timestamp::now()) + .await + .unwrap(); + interests::add(&db, "Boston", Some("Places"), Timestamp::now()) + .await + .unwrap(); db.kv_set( KV_TASTE_PROFILE, "system prompt text\n\n## Recent verdicts\n\nLOVED | x\n", @@ -617,7 +609,7 @@ mod tests { } #[tokio::test] - async fn profile_page_shows_editor_preview_interests_prompt_and_rebuild_form() { + async fn profile_page_shows_editor_standing_interests_prompt_and_rebuild_form() { let (_dir, _state, app, cookie) = setup().await; let response = get(&app, Some(&cookie)).await; assert_eq!(response.status(), StatusCode::OK); @@ -625,7 +617,8 @@ mod tests { assert!(body.contains("# Original")); assert!(body.contains("Prose.")); assert!(body.contains("Rust, Boston") || body.contains("Rust") && body.contains("Boston")); - assert!(body.contains("2 interests")); + assert!(body.contains("2 standing interests in 2 categories")); + assert!(body.contains("Interests page")); assert!(body.contains("system prompt text")); assert!(body.contains("Rank depth higher.")); assert!(body.contains("never built")); @@ -666,7 +659,7 @@ mod tests { let page = text(get(&app, Some(&cookie)).await).await; assert!(page.contains("Saved; the next run rebuilds the system prompt.")); - assert!(page.contains("Writerdeck")); + assert!(page.contains("section is ignored")); assert!(page.contains("# Original")); assert!(page.contains(">tyler<")); diff --git a/src/web/dashboard/settings.rs b/src/web/dashboard/settings.rs index 158d92f..4ed7015 100644 --- a/src/web/dashboard/settings.rs +++ b/src/web/dashboard/settings.rs @@ -246,7 +246,6 @@ const PATH_KEYS: &[&str] = &[ "database_path", "out_dir", "profile_path", - "interests_opml", "publish.epub_dir", "publish.xtc_dir", "xtc.settings", @@ -295,7 +294,6 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[ ("database_path", "SQLite file; parent directories are created on demand."), ("out_dir", "Where generate writes artifacts before publishing (overridden by --out)."), ("profile_path", "Hand-maintained reader profile, loaded every run."), - ("interests_opml", "Scour interests OPML merged with the profile interests."), ("miniflux.base_url", "Miniflux root (no /v1)."), ("miniflux.public_url", "Browser-facing Miniflux web UI URL for dashboard links. Defaults to miniflux.base_url."), ("miniflux.api_key", "X-Auth-Token for Miniflux. Required; environment only."), diff --git a/src/web/templates/dashboard/profile.html b/src/web/templates/dashboard/profile.html index a2d7535..d57660c 100644 --- a/src/web/templates/dashboard/profile.html +++ b/src/web/templates/dashboard/profile.html @@ -1,13 +1,13 @@ {% extends "layout.html" %}{% block content %}

Profile

-

The standing taste file the curator reads before every run: what you like, what the OPML declares, and what the editor model has learned from your verdicts.

+

The standing taste file the curator reads before every run, alongside stored interests and what the editor model has learned from your verdicts.

RatingsFeedback settings

profile.md

-

{{ path }}{% if !exists %} — missing; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any ## Interests section is parsed one interest per line; everything else goes into the system prompt verbatim.

+

{{ path }}{% if !exists %} — missing; saving creates it{% else %} · {{ bytes }} bytes{% endif %} · limit {{ max_bytes }} bytes. Any ## Interests section is ignored; everything else goes into the system prompt verbatim.

The next run rebuilds the system prompt from the saved file.
@@ -19,8 +19,6 @@

A live read of the text on the left, exactly as curate::profile splits it.

Passthrough sections

{% if preview_body.trim().is_empty() %}

Nothing passes through — the file is empty or only has an Interests section.

{% else %}
{{ preview_body }}
{% endif %} -

Extracted ## Interests lines

-{% if preview_interests.is_empty() %}

None. The prompt uses the OPML interests alone.

{% else %}
    {% for interest in preview_interests %}
  • {{ interest }}
    • {% endfor %}
{% endif %}
@@ -40,9 +38,8 @@ {% endif %}

Standing interests

-

{{ opml_path }} · {{ opml_count }} interests, grouped the way the system prompt lists them. The union of these and the ## Interests lines above is what the prompt uses; edit the OPML file to change them.

-{% if !opml_error.is_empty() %}

{{ opml_error }}

{% endif %} -{% if !themes.is_empty() %}
{% for theme in themes %}
{{ theme.name }} ({{ theme.count }})
{{ theme.members }}
{% endfor %}
{% endif %} +

{{ interest_count }} standing interests in {{ category_count }} categories — manage them on the Interests page.

+{% if !categories.is_empty() %}
{% for category in categories %}
{{ category.name }} ({{ category.count }})
{{ category.members }}
{% endfor %}
{% endif %}

Learned adjustments

Rebuilt weekly from ratings by the editor model (prompt version {{ prompt_version }}, built {{ prompt_built_at }}, {{ learned_age }}). {% if rebuild_due %}A rebuild is due — the next run performs it, or start it now.{% else %}The next scheduled rebuild is at least {{ rebuild_interval_days }} days after the last one; the next run performs it when due.{% endif %}

diff --git a/tests/config_check.rs b/tests/config_check.rs index 57a4b3e..172ae0f 100644 --- a/tests/config_check.rs +++ b/tests/config_check.rs @@ -35,7 +35,6 @@ fn config_check_prints_the_facts_and_exits_zero_without_keys() { "config: ", "database_path: /var/lib/daily-epub/daily-epub.db", "profile_path: data/profile.md", - "interests_opml: data/scour-interests.opml", "llm.bulk: deepseek · openai · deepseek-v4-flash", "key MISSING (set DAILY_EPUB_PROVIDERS__DEEPSEEK__API_KEY)", "llm.editor: anthropic · anthropic · claude-opus-5 · effort high · max_daily_usd $3.00", @@ -66,4 +65,8 @@ fn config_check_exits_non_zero_on_an_invalid_config() { let (code, _, stderr) = run("[deepseek]\nmodel = \"x\"\n"); assert_ne!(code, 0); assert!(stderr.contains("[providers.deepseek]"), "{stderr}"); + + let (code, _, stderr) = run("interests_opml = \"data/scour-interests.opml\"\n"); + assert_ne!(code, 0); + assert!(stderr.contains("interests_opml"), "{stderr}"); } diff --git a/tests/m3_curation.rs b/tests/m3_curation.rs index 96212a3..f18c909 100644 --- a/tests/m3_curation.rs +++ b/tests/m3_curation.rs @@ -21,6 +21,7 @@ use daily_epub::curate::assess::parse_deep_response; use daily_epub::curate::editor::parse_selection_response; use daily_epub::curate::editorial::BriefResponse; use daily_epub::curate::profile; +use daily_epub::interests; fn repo(rel: &str) -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join(rel) @@ -183,13 +184,14 @@ fn stage_c_fixture_parses_into_the_brief() { assert!(value.get("section_intros").is_none()); } -/// The taste profile is seeded from this file; a broken export would silently -/// gut the system prompt (§3.6a). +/// The importer relies on this file; a broken export would silently lose +/// standing interests (§3.6a). #[test] fn scour_opml_still_yields_the_interest_list() { - let interests = profile::parse_interests(&repo("data/scour-interests.opml")) - .expect("the shipped OPML must parse"); - let unique: BTreeSet = interests.iter().map(|n| n.to_lowercase()).collect(); + let raw = std::fs::read_to_string(repo("data/scour-interests.opml")) + .expect("the shipped OPML must be readable"); + let names = interests::parse_opml(&raw); + let unique: BTreeSet = names.iter().map(|n| n.to_lowercase()).collect(); assert!( unique.len() > 180, @@ -206,7 +208,7 @@ fn scour_opml_still_yields_the_interest_list() { assert!(unique.contains(expected), "{expected} disappeared"); } assert!( - !interests.iter().any(|n| n.contains("token=")), + !names.iter().any(|n| n.contains("token=")), "interest names must not leak the Scour token" ); @@ -217,9 +219,10 @@ fn scour_opml_still_yields_the_interest_list() { "/data/profile.md" ))) .expect("profile file"); + let grouped = vec![("Imported interests".to_string(), names)]; let document = profile::build( &profile_file.body, - &interests, + &grouped, profile::NO_LEARNED_ADJUSTMENTS, &[], 60,