Initial commit: The Daily EPUB full implementation
Full implementation of a personalized daily newspaper delivered as an EPUB. Articles are pulled from a local self-hosted Miniflux instance, enriched with comments, summarized and filtered by DeepSeek AI, and then assembled into two EPUB editions: standard and optimized for the Xteink X4 e-ink reader. Both are served by the local self-hosted BookOrbit OPDS server in a separate library. Then the X4 edition is futher converted to XTC format and served over a separate OPDS server hosted by the Rust binary. Runs are tracked in a local SQLite database so runs are idempotent per date. Full documentation of the plan is in docs/plans and setup and install instructions are in the README.md file.
This commit is contained in:
@@ -0,0 +1,622 @@
|
||||
//! The capstone test: the whole pipeline, driven as a library, with no network
|
||||
//! (spec §2, §5).
|
||||
//!
|
||||
//! ```text
|
||||
//! synthetic entries → dedupe → extract (offline) → persist → prefilter
|
||||
//! → select → editorial → issue → both EPUB editions → publish → OPDS + rows
|
||||
//! ```
|
||||
//!
|
||||
//! Two passes over the same machinery:
|
||||
//!
|
||||
//! * [`skip_llm_pipeline_produces_a_published_issue`] takes the `--skip-llm`
|
||||
//! route (prefilter order selects, feed excerpts stand in for summaries);
|
||||
//! * [`llm_pipeline_runs_against_a_mock_backend`] takes the DeepSeek route with
|
||||
//! [`MockBackend`] standing in for the API, so stages A, B and C are all
|
||||
//! exercised — prompts, parsers, budget accounting and all — offline.
|
||||
//!
|
||||
//! `Extractor::offline` guarantees the extraction stage never opens a socket, and
|
||||
//! no article in the fixtures carries an image, so the EPUB builder's image
|
||||
//! downloader has nothing to fetch.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
|
||||
use daily_epub::config::{Config, PublishConfig, ServerConfig, XtcConfig};
|
||||
use daily_epub::curate::llm::{LlmClient, MockBackend, UsageMeter};
|
||||
use daily_epub::curate::{Curator, editorial, prefilter};
|
||||
use daily_epub::db::Db;
|
||||
use daily_epub::extract::Extractor;
|
||||
use daily_epub::types::{
|
||||
Article, Colophon, Edition, Entry, Issue, Lineup, ScoredArticle, SourceKind, Vote,
|
||||
};
|
||||
use daily_epub::{auth, dedupe, epub, miniflux, pipeline, publish};
|
||||
|
||||
const SECRET: &str = "e2e-secret";
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().expect("timestamp")
|
||||
}
|
||||
|
||||
fn date() -> Date {
|
||||
"2026-08-15".parse().expect("date")
|
||||
}
|
||||
|
||||
fn body(words: usize) -> String {
|
||||
format!(
|
||||
"<p>{}</p>",
|
||||
"a sentence about database internals and page layout ".repeat(words / 8)
|
||||
)
|
||||
}
|
||||
|
||||
/// A config whose every writable path points inside `root`.
|
||||
fn test_config(root: &Path) -> Config {
|
||||
Config {
|
||||
database_path: root.join("db").join("daily-epub.db"),
|
||||
out_dir: root.join("out"),
|
||||
target_article_count: 6,
|
||||
prefilter_keep: 20,
|
||||
world_briefing: false,
|
||||
publish: PublishConfig {
|
||||
bookorbit_dir: root.join("bookorbit"),
|
||||
xtc_dir: root.join("xtc"),
|
||||
},
|
||||
xtc: XtcConfig {
|
||||
// Never shell out to node in a test.
|
||||
enabled: false,
|
||||
..XtcConfig::default()
|
||||
},
|
||||
server: ServerConfig {
|
||||
public_url: "https://daily.hallada.net".into(),
|
||||
hmac_secret: Some(SECRET.into()),
|
||||
..ServerConfig::default()
|
||||
},
|
||||
..Config::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// One day of ingest: eight entries covering duplicates, an excerpt-only story,
|
||||
/// and three things that are not articles at all.
|
||||
fn ingested() -> Vec<Entry> {
|
||||
let base = Entry {
|
||||
id: 0,
|
||||
feed_id: 0,
|
||||
feed_title: None,
|
||||
category: Some("Tech".into()),
|
||||
title: String::new(),
|
||||
url: String::new(),
|
||||
canonical_url: None,
|
||||
author: Some("Dana Author".into()),
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
raw_content: String::new(),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
};
|
||||
|
||||
vec![
|
||||
Entry {
|
||||
id: 101,
|
||||
feed_id: 1,
|
||||
feed_title: Some("Hacker News Front Page".into()),
|
||||
title: "A Deep Dive Into B-Trees".into(),
|
||||
url: "https://blog.dev/b-trees?utm_source=hnrss".into(),
|
||||
comments_url: Some("https://news.ycombinator.com/item?id=41234567".into()),
|
||||
raw_content: "<p>Discussion link only.</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
// Same story, richer body: the cluster keeps this one.
|
||||
Entry {
|
||||
id: 102,
|
||||
feed_id: 2,
|
||||
feed_title: Some("Scour: Databases".into()),
|
||||
title: "A Deep Dive Into B-Trees".into(),
|
||||
url: "https://blog.dev/b-trees#intro".into(),
|
||||
raw_content: body(900),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 103,
|
||||
feed_id: 3,
|
||||
feed_title: Some("The Rust Blog".into()),
|
||||
title: "Async Cancellation, Revisited".into(),
|
||||
url: "https://rust.dev/cancellation".into(),
|
||||
raw_content: body(1400),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 104,
|
||||
feed_id: 4,
|
||||
feed_title: Some("Astronomy Notes".into()),
|
||||
title: "What Webb Saw in the Rings of Uranus".into(),
|
||||
url: "https://space.dev/webb-uranus".into(),
|
||||
raw_content: body(700),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 105,
|
||||
feed_id: 5,
|
||||
feed_title: Some("Boston Civic Tech".into()),
|
||||
title: "The MBTA's New Signal Priority Pilot".into(),
|
||||
url: "https://boston.dev/signal-priority".into(),
|
||||
raw_content: body(600),
|
||||
..base.clone()
|
||||
},
|
||||
// Excerpt only: penalized by the pre-filter but still eligible (§3.5).
|
||||
Entry {
|
||||
id: 106,
|
||||
feed_id: 6,
|
||||
feed_title: Some("Lobsters".into()),
|
||||
title: "Notes on Writing a Toy Allocator".into(),
|
||||
url: "https://other.dev/allocator".into(),
|
||||
comments_url: Some("https://lobste.rs/s/abcdef/notes".into()),
|
||||
raw_content: "<p>A teaser paragraph and nothing else.</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
// Non-articles: a video host and an empty title (§3.2).
|
||||
Entry {
|
||||
id: 107,
|
||||
feed_id: 7,
|
||||
feed_title: Some("Video Feed".into()),
|
||||
title: "A conference talk".into(),
|
||||
url: "https://www.youtube.com/watch?v=abc".into(),
|
||||
raw_content: "<p>watch it</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 108,
|
||||
feed_id: 8,
|
||||
feed_title: Some("Broken Feed".into()),
|
||||
title: " ".into(),
|
||||
url: "https://broken.dev/x".into(),
|
||||
raw_content: body(400),
|
||||
..base.clone()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Stages 1–4: ingest → dedupe → extract → persist, exactly as `pipeline.rs`
|
||||
/// orders them (extraction runs *before* the article rows are written).
|
||||
async fn ingest_dedupe_extract_persist(db: &Db) -> Vec<Article> {
|
||||
let entries = ingested();
|
||||
db.upsert_entries(&entries).await.expect("persist entries");
|
||||
|
||||
let feeds = std::collections::HashMap::from([(
|
||||
2,
|
||||
miniflux::FeedMeta {
|
||||
id: 2,
|
||||
title: "Scour: Databases".into(),
|
||||
site_url: "https://scour.ing".into(),
|
||||
feed_url: "https://scour.ing/feed?interest=databases".into(),
|
||||
category: Some("Interests".into()),
|
||||
},
|
||||
)]);
|
||||
let (mut articles, stats) = dedupe::cluster_with_feeds(entries, &miniflux::feed_urls(&feeds));
|
||||
assert_eq!(stats.entries_in, 8);
|
||||
assert_eq!(stats.dropped_non_article, 2, "video host + empty title");
|
||||
assert_eq!(stats.merged, 1, "the B-trees story arrived twice");
|
||||
assert_eq!(articles.len(), 5);
|
||||
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
assert!(
|
||||
!extractor.can_fetch(),
|
||||
"the test must never hit the network"
|
||||
);
|
||||
let extracted = extractor.extract_all(&mut articles).await;
|
||||
assert_eq!(extracted.excerpt_only, 1, "the allocator teaser");
|
||||
|
||||
for article in &mut articles {
|
||||
article.id = db.upsert_article(article).await.expect("persist article");
|
||||
assert!(article.id > 0);
|
||||
}
|
||||
let b_trees = articles
|
||||
.iter()
|
||||
.find(|a| a.canonical_url == "https://blog.dev/b-trees")
|
||||
.expect("the merged cluster");
|
||||
assert!(b_trees.came_via(SourceKind::Scour));
|
||||
assert!(b_trees.came_via(SourceKind::HnFrontpage));
|
||||
assert_eq!(b_trees.best_entry_id, 102, "the richest body won");
|
||||
|
||||
articles
|
||||
}
|
||||
|
||||
/// Stages 11–14: assemble, build both editions, publish, record.
|
||||
async fn assemble_build_publish(
|
||||
db: &Db,
|
||||
cfg: &Config,
|
||||
lineup: Lineup,
|
||||
colophon: Colophon,
|
||||
) -> Issue {
|
||||
let editorial_doc = editorial::fallback_editorial(&lineup);
|
||||
let mut lineup = lineup;
|
||||
pipeline::apply_summaries(&mut lineup, &editorial_doc);
|
||||
assert!(
|
||||
lineup.picks.iter().all(|p| p.summary.is_some()),
|
||||
"every pick carries a summary before the EPUB is built"
|
||||
);
|
||||
|
||||
let issue_number = db.next_issue_number(date()).await.expect("issue number");
|
||||
let issue = pipeline::build_issue(
|
||||
date(),
|
||||
issue_number,
|
||||
ts("2026-08-15T09:30:00Z"),
|
||||
lineup,
|
||||
editorial_doc,
|
||||
None,
|
||||
colophon,
|
||||
);
|
||||
assert_eq!(issue.meta.display_date, "Saturday, August 15, 2026");
|
||||
|
||||
// --- EPUB: both editions (§3.10) ---
|
||||
let (artifacts, images) = epub::build_all(&issue, cfg, &cfg.out_dir)
|
||||
.await
|
||||
.expect("both editions build");
|
||||
assert_eq!(artifacts.len(), 2);
|
||||
assert_eq!(images, 0, "the fixtures carry no images");
|
||||
for artifact in &artifacts {
|
||||
assert!(artifact.path.exists(), "{}", artifact.path.display());
|
||||
assert!(artifact.bytes > 1000);
|
||||
let zip = std::fs::read(&artifact.path).expect("read epub");
|
||||
assert_eq!(&zip[0..4], b"PK\x03\x04", "is a zip");
|
||||
assert_eq!(&zip[38..58], b"application/epub+zip");
|
||||
}
|
||||
assert!(
|
||||
cfg.out_dir
|
||||
.join("The Daily EPUB - 2026-08-15.epub")
|
||||
.exists()
|
||||
);
|
||||
assert!(
|
||||
cfg.out_dir
|
||||
.join("The Daily EPUB - 2026-08-15 (X4).epub")
|
||||
.exists()
|
||||
);
|
||||
|
||||
// Rating links are signed with the configured secret and are what the
|
||||
// running server verifies (§3.9). The chapters are deflated inside the zip,
|
||||
// so assert on the rendered XHTML the builder just zipped.
|
||||
let chapters = epub::build::render_all(
|
||||
&issue,
|
||||
Edition::Standard,
|
||||
&[],
|
||||
&cfg.server.public_url,
|
||||
cfg.server.hmac_secret.as_deref(),
|
||||
)
|
||||
.expect("render chapters");
|
||||
let first = issue.lineup.picks[0].article.id;
|
||||
let expected = auth::rating_url(&cfg.server.public_url, SECRET, date(), first, Vote::Up);
|
||||
let chapter = chapters
|
||||
.iter()
|
||||
.find(|c| c.id == format!("art-{}", issue.lineup.picks[0].article.best_entry_id))
|
||||
.expect("the first article has a chapter");
|
||||
assert!(
|
||||
chapter.xhtml.contains(&expected),
|
||||
"the article footer must carry {expected}\n{}",
|
||||
chapter.xhtml
|
||||
);
|
||||
assert!(
|
||||
daily_epub::server::verify_token(
|
||||
SECRET,
|
||||
date(),
|
||||
first,
|
||||
Vote::Up,
|
||||
&auth::rating_token(SECRET, date(), first, Vote::Up)
|
||||
),
|
||||
"the server must accept the token the EPUB minted"
|
||||
);
|
||||
|
||||
// --- Publish (§3.11) ---
|
||||
let published = publish::publish_issue(db, cfg, &issue, &artifacts, None)
|
||||
.await
|
||||
.expect("publish");
|
||||
assert_eq!(published.epubs.len(), 2);
|
||||
for artifact in &published.epubs {
|
||||
assert!(artifact.path.starts_with(&cfg.publish.bookorbit_dir));
|
||||
assert!(artifact.path.exists(), "{}", artifact.path.display());
|
||||
}
|
||||
assert!(
|
||||
cfg.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-08-15.epub")
|
||||
.exists()
|
||||
);
|
||||
assert!(
|
||||
cfg.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-08-15 (X4).epub")
|
||||
.exists()
|
||||
);
|
||||
assert!(published.xtc.is_none(), "the converter is disabled here");
|
||||
|
||||
// The OPDS feed is regenerated on every publish, even with no XTC files yet.
|
||||
let opds = published.opds.clone().expect("an OPDS feed was written");
|
||||
assert_eq!(opds, cfg.publish.xtc_dir.join("xtc.xml"));
|
||||
let feed = std::fs::read_to_string(&opds).expect("read the OPDS feed");
|
||||
assert!(feed.starts_with("<?xml"), "{feed}");
|
||||
assert!(feed.contains("<feed xmlns=\"http://www.w3.org/2005/Atom\""));
|
||||
assert!(feed.contains(&cfg.server.public_url));
|
||||
|
||||
// --- Record (§3.13) ---
|
||||
let epub_path = published
|
||||
.epubs
|
||||
.iter()
|
||||
.find(|a| a.edition == Edition::Standard)
|
||||
.map(|a| a.path.display().to_string());
|
||||
let x4_path = published
|
||||
.epubs
|
||||
.iter()
|
||||
.find(|a| a.edition == Edition::X4)
|
||||
.map(|a| a.path.display().to_string());
|
||||
db.upsert_issue(
|
||||
date(),
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
epub_path.as_deref(),
|
||||
x4_path.as_deref(),
|
||||
None,
|
||||
Some(&issue.editorial.front_page_html),
|
||||
Some("{\"status\":\"ok\"}"),
|
||||
)
|
||||
.await
|
||||
.expect("record the issue");
|
||||
db.replace_issue_articles(date(), &issue.lineup.picks)
|
||||
.await
|
||||
.expect("record the lineup");
|
||||
|
||||
let reports = db.recent_reports(5).await.expect("recent reports");
|
||||
assert_eq!(reports.len(), 1);
|
||||
assert_eq!(reports[0].0, date());
|
||||
assert_eq!(reports[0].1.as_deref(), Some("{\"status\":\"ok\"}"));
|
||||
|
||||
let published_ids = db
|
||||
.previously_published_ids()
|
||||
.await
|
||||
.expect("issue_articles rows");
|
||||
assert_eq!(published_ids.len(), issue.lineup.picks.len());
|
||||
|
||||
// Tomorrow's issue is No. 2 (days since the first issue, §3.10).
|
||||
let tomorrow: Date = "2026-08-16".parse().unwrap();
|
||||
assert_eq!(db.next_issue_number(tomorrow).await.unwrap(), 2);
|
||||
|
||||
issue
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skip_llm_pipeline_produces_a_published_issue() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
let cfg = test_config(root.path());
|
||||
let db = Db::open_and_migrate(&cfg.database_path)
|
||||
.await
|
||||
.expect("open db");
|
||||
|
||||
let articles = ingest_dedupe_extract_persist(&db).await;
|
||||
|
||||
// --- Stages 6–7 with no LLM at all (notes §6) ---
|
||||
let curator = Curator::new(cfg.clone(), db.clone(), None);
|
||||
let candidates = curator
|
||||
.prefilter(articles, date())
|
||||
.await
|
||||
.expect("prefilter runs");
|
||||
assert_eq!(candidates.len(), 5, "nothing is dropped at this volume");
|
||||
assert!(
|
||||
candidates
|
||||
.windows(2)
|
||||
.all(|w| w[0].prefilter_score >= w[1].prefilter_score),
|
||||
"candidates come back in prefilter order"
|
||||
);
|
||||
// The excerpt-only story is penalized (§3.5).
|
||||
let allocator = candidates
|
||||
.iter()
|
||||
.find(|c| c.article.excerpt_only)
|
||||
.expect("the allocator teaser survived");
|
||||
assert!(allocator.prefilter_score < candidates[0].prefilter_score);
|
||||
|
||||
let lineup = curator.select(candidates, date()).await.expect("select");
|
||||
assert_eq!(lineup.picks.len(), 5, "target 6, only 5 candidates exist");
|
||||
assert!(lineup.lead().is_some(), "a lead story is always chosen");
|
||||
assert!(!lineup.section_order.is_empty());
|
||||
assert!(
|
||||
lineup
|
||||
.picks
|
||||
.iter()
|
||||
.all(|p| lineup.section_order.contains(&p.section)),
|
||||
"every pick sits in a listed section"
|
||||
);
|
||||
|
||||
let colophon = Colophon {
|
||||
model: "none (--skip-llm)".into(),
|
||||
entries_fetched: 8,
|
||||
feeds_seen: 8,
|
||||
candidates: 5,
|
||||
cost_usd: 0.0,
|
||||
generator_version: format!("daily-epub {}", daily_epub::VERSION),
|
||||
};
|
||||
let issue = assemble_build_publish(&db, &cfg, lineup, colophon).await;
|
||||
|
||||
// Front page and summaries came from excerpts, not from a model.
|
||||
assert!(!issue.editorial.front_page_html.is_empty());
|
||||
assert_eq!(issue.editorial.summaries.len(), issue.lineup.picks.len());
|
||||
|
||||
// Re-running the same date replaces rather than duplicates (notes §12).
|
||||
let republished = publish::publish_issue(
|
||||
&db,
|
||||
&cfg,
|
||||
&issue,
|
||||
&[
|
||||
daily_epub::types::Artifact {
|
||||
edition: Edition::Standard,
|
||||
path: cfg.out_dir.join("The Daily EPUB - 2026-08-15.epub"),
|
||||
bytes: 0,
|
||||
},
|
||||
daily_epub::types::Artifact {
|
||||
edition: Edition::X4,
|
||||
path: cfg.out_dir.join("The Daily EPUB - 2026-08-15 (X4).epub"),
|
||||
bytes: 0,
|
||||
},
|
||||
],
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("republish");
|
||||
assert_eq!(republished.epubs.len(), 2);
|
||||
let files: Vec<String> = std::fs::read_dir(&cfg.publish.bookorbit_dir)
|
||||
.expect("read bookorbit dir")
|
||||
.filter_map(|e| e.ok().map(|e| e.file_name().to_string_lossy().into_owned()))
|
||||
.collect();
|
||||
assert_eq!(files.len(), 2, "no duplicate files: {files:?}");
|
||||
|
||||
db.replace_issue_articles(date(), &issue.lineup.picks)
|
||||
.await
|
||||
.expect("replace lineup");
|
||||
assert_eq!(
|
||||
db.previously_published_ids().await.unwrap().len(),
|
||||
issue.lineup.picks.len(),
|
||||
"issue_articles was replaced, not appended"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn llm_pipeline_runs_against_a_mock_backend() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
let cfg = test_config(root.path());
|
||||
let db = Db::open_and_migrate(&cfg.database_path)
|
||||
.await
|
||||
.expect("open db");
|
||||
|
||||
let articles = ingest_dedupe_extract_persist(&db).await;
|
||||
let ctx = prefilter::PrefilterContext::load(&db, date())
|
||||
.await
|
||||
.expect("prefilter context");
|
||||
let candidates: Vec<ScoredArticle> = prefilter::run(articles, &ctx, &cfg);
|
||||
let ids: Vec<i64> = candidates.iter().map(|c| c.article.id).collect();
|
||||
assert_eq!(ids.len(), 5);
|
||||
|
||||
// --- Script DeepSeek: one stage-A batch, one stage-B call, five stage-C
|
||||
// summaries and one front page (§3.6). ---
|
||||
let backend = std::sync::Arc::new(MockBackend::new());
|
||||
let usage = daily_epub::types::TokenUsage {
|
||||
input_tokens: 1000,
|
||||
cached_tokens: 500,
|
||||
output_tokens: 200,
|
||||
};
|
||||
let scores: Vec<String> = ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, id)| {
|
||||
format!(
|
||||
r#"{{"id": {id}, "score": {}, "category": "Tech & Engineering",
|
||||
"rationale": "solid systems writeup", "is_paywalled_guess": false}}"#,
|
||||
9 - i
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
backend.push(format!("{{\"articles\": [{}]}}", scores.join(",")), usage);
|
||||
|
||||
let picks: Vec<String> = ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, id)| {
|
||||
format!(
|
||||
r#"{{"id": {id}, "section": "{}", "position": {}, "lead_story": {}}}"#,
|
||||
if i == 0 {
|
||||
"Top Stories"
|
||||
} else {
|
||||
"Tech & Engineering"
|
||||
},
|
||||
i + 1,
|
||||
i == 0
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
backend.push(format!("{{\"picks\": [{}]}}", picks.join(",")), usage);
|
||||
|
||||
for id in &ids {
|
||||
backend.push(
|
||||
format!(r#"{{"summary": "A newspaper abstract for article {id}."}}"#),
|
||||
usage,
|
||||
);
|
||||
}
|
||||
backend.push(
|
||||
r#"{"from_the_editor": "Today's issue leans on storage internals.\n\nRead on.",
|
||||
"section_intros": {"Top Stories": "The day in one place."}}"#,
|
||||
usage,
|
||||
);
|
||||
|
||||
let meter = UsageMeter::new(&cfg.deepseek, cfg.max_daily_usd);
|
||||
let llm = LlmClient::with_backend(
|
||||
&cfg.deepseek.model,
|
||||
"You are the editor of The Daily EPUB.".into(),
|
||||
meter.clone(),
|
||||
backend.clone(),
|
||||
);
|
||||
let curator = Curator::new(cfg.clone(), db.clone(), Some(llm));
|
||||
|
||||
let mut candidates = candidates;
|
||||
curator
|
||||
.score(&mut candidates, date())
|
||||
.await
|
||||
.expect("stage A");
|
||||
assert!(
|
||||
candidates.iter().all(|c| c.llm.is_some()),
|
||||
"every candidate came back scored"
|
||||
);
|
||||
|
||||
let lineup = curator.select(candidates, date()).await.expect("stage B");
|
||||
assert_eq!(lineup.picks.len(), 5);
|
||||
assert_eq!(
|
||||
lineup.lead().map(|p| p.article.id),
|
||||
Some(ids[0]),
|
||||
"the model's lead choice is honored"
|
||||
);
|
||||
assert_eq!(
|
||||
lineup.section_order.first().map(String::as_str),
|
||||
Some("Top Stories")
|
||||
);
|
||||
|
||||
let editorial_doc = curator.editorial(&lineup).await.expect("stage C");
|
||||
assert_eq!(editorial_doc.summaries.len(), 5);
|
||||
assert!(
|
||||
editorial_doc
|
||||
.summaries
|
||||
.values()
|
||||
.all(|s| s.contains("newspaper abstract")),
|
||||
"the model's summaries were used, not excerpts"
|
||||
);
|
||||
assert!(editorial_doc.front_page_html.contains("storage internals"));
|
||||
assert_eq!(
|
||||
editorial_doc
|
||||
.section_intros
|
||||
.get("Top Stories")
|
||||
.map(String::as_str),
|
||||
Some("The day in one place.")
|
||||
);
|
||||
|
||||
// Every scripted response was consumed, and the meter priced them (§3.6).
|
||||
assert_eq!(backend.calls(), 1 + 1 + 5 + 1);
|
||||
let total = meter.total();
|
||||
assert_eq!(total.input_tokens, 8 * usage.input_tokens);
|
||||
assert!(meter.cost_usd() > 0.0 && !meter.budget_exceeded());
|
||||
// The taste profile leads every request, byte for byte — that is what makes
|
||||
// DeepSeek's prefix cache hit (§3.6).
|
||||
let prompts = backend.prompts();
|
||||
assert!(
|
||||
prompts
|
||||
.iter()
|
||||
.all(|p| p.system.starts_with("You are the editor")),
|
||||
"the system prompt must be identical across requests"
|
||||
);
|
||||
|
||||
// And it all assembles, builds and publishes like the skip-llm route does.
|
||||
let colophon = Colophon {
|
||||
model: cfg.deepseek.model.clone(),
|
||||
entries_fetched: 8,
|
||||
feeds_seen: 8,
|
||||
candidates: 5,
|
||||
cost_usd: meter.cost_usd(),
|
||||
generator_version: format!("daily-epub {}", daily_epub::VERSION),
|
||||
};
|
||||
let mut lineup = lineup;
|
||||
pipeline::apply_summaries(&mut lineup, &editorial_doc);
|
||||
let issue = assemble_build_publish(&db, &cfg, lineup, colophon).await;
|
||||
assert_eq!(issue.colophon.model, cfg.deepseek.model);
|
||||
assert!(issue.colophon.cost_usd > 0.0);
|
||||
}
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"from_the_editor": "Two of today's pieces are, underneath, the same story: what it costs to move data you no longer trust. The lead — a team hauling forty terabytes off Postgres, rollback plans and all — is the version with the invoices attached, and it earns the front page by refusing to tidy up its failures. Read it first, while the coffee is hot; it rewards attention and it is long.\n\nThe local desk supplies the counterpoint. Somebody has finally put the MBTA's slow-zone data into a shape a rider can argue with, and the charts do more persuading than a year of press releases. It is a short read and a satisfying one, and it pairs unreasonably well with the migration story: both are about institutions discovering what they actually have.\n\nThe rest of the issue is quieter than usual. That is not a complaint — a thin Friday is a good excuse to finish the long one properly rather than skimming six. If you only get through the lead today, you will not have missed much else.",
|
||||
"section_intros": {
|
||||
"Top Stories": "The day's most substantial piece: a full account of a forty-terabyte migration, with the failures left in. It is long, technical and unusually honest about what went wrong.",
|
||||
"Boston & Local": "Transit data gets the treatment it deserves. A rider-built analysis of MBTA slow zones, with charts you can check yourself and a methodology section that holds up.",
|
||||
"Niche Corner": "A section the model wrote an intro for even though nothing was placed in it today — a stray thread about tape-drive firmware and the people who still maintain it. The issue drops intros for sections that never ran."
|
||||
}
|
||||
}
|
||||
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"picks": [
|
||||
{ "id": 101, "section": "Top Stories", "position": 1, "lead_story": true },
|
||||
{ "id": 102, "section": "Tech & Engineering", "position": 1, "lead_story": false },
|
||||
{ "id": 103, "section": "Tech & Engineering", "position": 2, "lead_story": false },
|
||||
{ "id": 104, "section": "AI & Machine Learning", "position": 1, "lead_story": false },
|
||||
{ "id": 105, "section": "Boston & Local", "position": 1, "lead_story": false },
|
||||
{ "id": 106, "section": "Culture & Essays", "position": 1 },
|
||||
{ "section": "Niche Corner", "position": 2, "lead_story": false },
|
||||
"the model sometimes trails off like this"
|
||||
]
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"articles": [
|
||||
{
|
||||
"id": 101,
|
||||
"score": 8.5,
|
||||
"category": "Tech & Engineering",
|
||||
"rationale": "first-hand 40TB Postgres migration with numbers, failures and rollback plan",
|
||||
"is_paywalled_guess": false
|
||||
},
|
||||
{
|
||||
"id": 102,
|
||||
"score": 3.0,
|
||||
"category": "AI & Machine Learning",
|
||||
"rationale": "model release announcement, no independent evaluation",
|
||||
"is_paywalled_guess": false
|
||||
},
|
||||
{
|
||||
"id": 103,
|
||||
"score": 6.5,
|
||||
"category": "Boston & Local",
|
||||
"rationale": "MBTA slow-zone data analysis with original charts",
|
||||
"is_paywalled_guess": false
|
||||
},
|
||||
{
|
||||
"id": 104,
|
||||
"score": 5.0,
|
||||
"category": "Culture & Essays",
|
||||
"rationale": "promising essay on typesetting, body appears truncated",
|
||||
"is_paywalled_guess": true
|
||||
}
|
||||
]
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"articles": [
|
||||
{
|
||||
"id": 201,
|
||||
"score": 7.0,
|
||||
"category": "Science & Space",
|
||||
"rationale": "careful write-up of an amateur radio occultation measurement",
|
||||
"is_paywalled_guess": false
|
||||
},
|
||||
{
|
||||
"id": "202",
|
||||
"score": "6",
|
||||
"category": "Niche Corner",
|
||||
"rationale": "mailing-list argument about tape drives, oddly gripping",
|
||||
"is_paywalled_guess": "false"
|
||||
},
|
||||
{
|
||||
"id": 203,
|
||||
"score": 4
|
||||
},
|
||||
{
|
||||
"id": 204,
|
||||
"score": 12.5,
|
||||
"category": "Top Stories",
|
||||
"rationale": "model ignored the rubric ceiling here",
|
||||
"is_paywalled_guess": false
|
||||
},
|
||||
{
|
||||
"score": 9.0,
|
||||
"category": "Top Stories",
|
||||
"rationale": "no id at all, unusable"
|
||||
},
|
||||
"a bare string where an object belongs"
|
||||
]
|
||||
}
|
||||
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"id": 40100000,
|
||||
"created_at": "2026-08-15T09:12:00.000Z",
|
||||
"type": "story",
|
||||
"author": "poster",
|
||||
"title": "A Story About Databases",
|
||||
"url": "https://example.com/databases",
|
||||
"points": 342,
|
||||
"text": null,
|
||||
"children": [
|
||||
{
|
||||
"id": 40100001,
|
||||
"type": "comment",
|
||||
"author": "alice",
|
||||
"text": "The <i>write path</i> is the interesting part here.<p>Especially fsync batching.",
|
||||
"points": 61,
|
||||
"children": [
|
||||
{
|
||||
"id": 40100002,
|
||||
"type": "comment",
|
||||
"author": "bob",
|
||||
"text": "Agreed — and the group commit numbers match my own benchmarks.",
|
||||
"points": 24,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 40100003,
|
||||
"type": "comment",
|
||||
"author": "carol",
|
||||
"text": "Counterpoint: the benchmark hardware is unrealistic.",
|
||||
"points": 12,
|
||||
"children": []
|
||||
},
|
||||
{
|
||||
"id": 40100004,
|
||||
"type": "comment",
|
||||
"author": null,
|
||||
"text": null,
|
||||
"points": null,
|
||||
"children": [
|
||||
{
|
||||
"id": 40100005,
|
||||
"type": "comment",
|
||||
"author": "dana",
|
||||
"text": "Replying to a since-deleted comment, but the point stands.",
|
||||
"points": 3,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"short_id": "abcdef",
|
||||
"title": "A Story About Databases",
|
||||
"url": "https://example.com/databases",
|
||||
"score": 48,
|
||||
"comment_count": 4,
|
||||
"comments": [
|
||||
{
|
||||
"short_id": "c1",
|
||||
"comment": "<p>Nice writeup of the write path.</p>",
|
||||
"score": 20,
|
||||
"indent_level": 1,
|
||||
"parent_comment": null,
|
||||
"commenting_user": "pushcx"
|
||||
},
|
||||
{
|
||||
"short_id": "c2",
|
||||
"comment": "<p>Do you have numbers for NVMe?</p>",
|
||||
"score": 9,
|
||||
"indent_level": 2,
|
||||
"parent_comment": "c1",
|
||||
"commenting_user": { "username": "second" }
|
||||
},
|
||||
{
|
||||
"short_id": "c3",
|
||||
"comment": "<p>Yes, in the appendix.</p>",
|
||||
"score": 4,
|
||||
"indent_level": 3,
|
||||
"parent_comment": "c2",
|
||||
"commenting_user": "third"
|
||||
},
|
||||
{
|
||||
"short_id": "c4",
|
||||
"comment": "<p>Unrelated: the site's typography is lovely.</p>",
|
||||
"score": 3,
|
||||
"indent_level": 1,
|
||||
"parent_comment": null,
|
||||
"commenting_user": "other"
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"id": 41234567,
|
||||
"created_at": "2026-08-14T13:02:11.000Z",
|
||||
"created_at_i": 1786748531,
|
||||
"type": "story",
|
||||
"author": "tylerh",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"url": "https://blog.dev/post",
|
||||
"text": null,
|
||||
"points": 342,
|
||||
"parent_id": null,
|
||||
"story_id": null,
|
||||
"children": [
|
||||
{
|
||||
"id": 41234600,
|
||||
"created_at": "2026-08-14T13:20:00.000Z",
|
||||
"type": "comment",
|
||||
"author": "dbnerd",
|
||||
"title": null,
|
||||
"url": null,
|
||||
"text": "<p>The section on page splits is the clearest I have read. <i>Bookmarked.</i></p>",
|
||||
"points": 88,
|
||||
"parent_id": 41234567,
|
||||
"story_id": 41234567,
|
||||
"children": [
|
||||
{
|
||||
"id": 41234611,
|
||||
"created_at": "2026-08-14T13:41:00.000Z",
|
||||
"type": "comment",
|
||||
"author": "tylerh",
|
||||
"title": null,
|
||||
"url": null,
|
||||
"text": "<p>Thanks — the diagrams took longer than the prose.</p>",
|
||||
"points": 31,
|
||||
"parent_id": 41234600,
|
||||
"story_id": 41234567,
|
||||
"children": [
|
||||
{
|
||||
"id": 41234620,
|
||||
"created_at": "2026-08-14T14:00:00.000Z",
|
||||
"type": "comment",
|
||||
"author": "gridlines",
|
||||
"title": null,
|
||||
"url": null,
|
||||
"text": "<p>What did you draw them with?</p>",
|
||||
"points": 4,
|
||||
"parent_id": 41234611,
|
||||
"story_id": 41234567,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 41234700,
|
||||
"created_at": "2026-08-14T15:05:00.000Z",
|
||||
"type": "comment",
|
||||
"author": "deleted_user",
|
||||
"title": null,
|
||||
"url": null,
|
||||
"text": null,
|
||||
"points": null,
|
||||
"parent_id": 41234567,
|
||||
"story_id": 41234567,
|
||||
"children": []
|
||||
},
|
||||
{
|
||||
"id": 41234800,
|
||||
"created_at": "2026-08-14T16:12:00.000Z",
|
||||
"type": "comment",
|
||||
"author": "skeptic",
|
||||
"title": null,
|
||||
"url": null,
|
||||
"text": "<p>Counterpoint: LSM trees win on write-heavy workloads.</p>",
|
||||
"points": 45,
|
||||
"parent_id": 41234567,
|
||||
"story_id": 41234567,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"hits": [
|
||||
{
|
||||
"created_at": "2026-08-14T13:02:11.000Z",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"url": "https://blog.dev/post",
|
||||
"author": "tylerh",
|
||||
"points": 342,
|
||||
"story_text": null,
|
||||
"comment_text": null,
|
||||
"num_comments": 210,
|
||||
"story_id": null,
|
||||
"story_title": null,
|
||||
"story_url": null,
|
||||
"parent_id": null,
|
||||
"created_at_i": 1786748531,
|
||||
"_tags": ["story", "author_tylerh", "story_41234567"],
|
||||
"objectID": "41234567",
|
||||
"_highlightResult": {
|
||||
"url": {
|
||||
"value": "https://blog.dev/post",
|
||||
"matchLevel": "full",
|
||||
"matchedWords": ["https://blog.dev/post"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"created_at": "2026-08-10T09:15:00.000Z",
|
||||
"title": "A Deep Dive Into B-Trees (2019)",
|
||||
"url": "https://blog.dev/post?utm_source=twitter",
|
||||
"author": "someoneelse",
|
||||
"points": 12,
|
||||
"story_text": null,
|
||||
"comment_text": null,
|
||||
"num_comments": 3,
|
||||
"story_id": null,
|
||||
"created_at_i": 1786396500,
|
||||
"_tags": ["story", "author_someoneelse", "story_41000000"],
|
||||
"objectID": "41000000"
|
||||
}
|
||||
],
|
||||
"nbHits": 2,
|
||||
"page": 0,
|
||||
"nbPages": 1,
|
||||
"hitsPerPage": 20,
|
||||
"exhaustiveNbHits": true,
|
||||
"query": "https://blog.dev/post",
|
||||
"params": "query=https%3A%2F%2Fblog.dev%2Fpost&restrictSearchableAttributes=url",
|
||||
"processingTimeMS": 3
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"hits": [],
|
||||
"nbHits": 0,
|
||||
"page": 0,
|
||||
"nbPages": 0,
|
||||
"hitsPerPage": 20,
|
||||
"exhaustiveNbHits": true,
|
||||
"query": "https://blog.dev/never-submitted",
|
||||
"params": "query=https%3A%2F%2Fblog.dev%2Fnever-submitted&restrictSearchableAttributes=url",
|
||||
"processingTimeMS": 1
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"short_id": "abcdef",
|
||||
"short_id_url": "https://lobste.rs/s/abcdef",
|
||||
"created_at": "2026-08-14T09:11:04.000-05:00",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"url": "https://blog.dev/post",
|
||||
"score": 78,
|
||||
"flags": 1,
|
||||
"comment_count": 4,
|
||||
"description": "",
|
||||
"description_plain": "",
|
||||
"comments_url": "https://lobste.rs/s/abcdef/a_deep_dive_into_b_trees",
|
||||
"submitter_user": "alice",
|
||||
"user_is_author": false,
|
||||
"tags": ["databases", "rust"],
|
||||
"comments": [
|
||||
{
|
||||
"short_id": "c00001",
|
||||
"short_id_url": "https://lobste.rs/c/c00001",
|
||||
"created_at": "2026-08-14T09:40:00.000-05:00",
|
||||
"is_deleted": false,
|
||||
"is_moderated": false,
|
||||
"score": 21,
|
||||
"flags": 0,
|
||||
"parent_comment": null,
|
||||
"comment": "<p>The page-split diagrams are excellent.</p>",
|
||||
"comment_plain": "The page-split diagrams are excellent.",
|
||||
"indent_level": 1,
|
||||
"commenting_user": "bob"
|
||||
},
|
||||
{
|
||||
"short_id": "c00002",
|
||||
"short_id_url": "https://lobste.rs/c/c00002",
|
||||
"created_at": "2026-08-14T10:02:00.000-05:00",
|
||||
"is_deleted": false,
|
||||
"is_moderated": false,
|
||||
"score": 9,
|
||||
"flags": 0,
|
||||
"parent_comment": "c00001",
|
||||
"comment": "<p>Agreed — and the prose is tight too.</p>",
|
||||
"comment_plain": "Agreed - and the prose is tight too.",
|
||||
"indent_level": 2,
|
||||
"commenting_user": { "username": "carol", "karma": 4021 }
|
||||
},
|
||||
{
|
||||
"short_id": "c00003",
|
||||
"short_id_url": "https://lobste.rs/c/c00003",
|
||||
"created_at": "2026-08-14T11:15:00.000-05:00",
|
||||
"is_deleted": false,
|
||||
"is_moderated": false,
|
||||
"score": 33,
|
||||
"flags": 0,
|
||||
"parent_comment": null,
|
||||
"comment": "<p>Prior art: the 1979 Comer survey covers most of this.</p>",
|
||||
"comment_plain": "Prior art: the 1979 Comer survey covers most of this.",
|
||||
"indent_level": 1,
|
||||
"commenting_user": "dave"
|
||||
},
|
||||
{
|
||||
"short_id": "c00004",
|
||||
"short_id_url": "https://lobste.rs/c/c00004",
|
||||
"created_at": "2026-08-14T11:59:00.000-05:00",
|
||||
"is_deleted": true,
|
||||
"is_moderated": false,
|
||||
"score": 0,
|
||||
"flags": 0,
|
||||
"parent_comment": "c00003",
|
||||
"comment": "",
|
||||
"comment_plain": "",
|
||||
"indent_level": 2,
|
||||
"commenting_user": "erin"
|
||||
}
|
||||
]
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
[
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t3",
|
||||
"data": {
|
||||
"subreddit": "programming",
|
||||
"id": "1abcd2",
|
||||
"name": "t3_1abcd2",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"score": 845,
|
||||
"num_comments": 231,
|
||||
"permalink": "/r/programming/comments/1abcd2/a_deep_dive_into_btrees/",
|
||||
"url": "https://blog.dev/post"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"id": "kx1",
|
||||
"name": "t1_kx1",
|
||||
"author": "index_nerd",
|
||||
"score": 412,
|
||||
"body": "Fan-out is the whole ballgame; everything else is bookkeeping.",
|
||||
"depth": 0,
|
||||
"stickied": false,
|
||||
"replies": {
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"id": "kx2",
|
||||
"name": "t1_kx2",
|
||||
"author": "pagecache",
|
||||
"score": 130,
|
||||
"body": "Until your keys are variable length & the math gets ugly.",
|
||||
"depth": 1,
|
||||
"stickied": false,
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "more",
|
||||
"data": {
|
||||
"count": 18,
|
||||
"name": "t1_more1",
|
||||
"id": "more1",
|
||||
"children": ["kx9", "kx10"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"id": "kx3",
|
||||
"name": "t1_kx3",
|
||||
"author": "AutoModerator",
|
||||
"score": 1,
|
||||
"body": "Please keep discussion civil.",
|
||||
"depth": 0,
|
||||
"stickied": true,
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"id": "kx4",
|
||||
"name": "t1_kx4",
|
||||
"author": "[deleted]",
|
||||
"score": null,
|
||||
"body": "[removed]",
|
||||
"depth": 0,
|
||||
"stickied": false,
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "more",
|
||||
"data": {
|
||||
"count": 205,
|
||||
"name": "t1_more2",
|
||||
"id": "more2",
|
||||
"children": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
Vendored
+69
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"after": null,
|
||||
"dist": 3,
|
||||
"modhash": "",
|
||||
"geo_filter": "",
|
||||
"before": null,
|
||||
"children": [
|
||||
{
|
||||
"kind": "t3",
|
||||
"data": {
|
||||
"subreddit": "programming",
|
||||
"id": "1abcd2",
|
||||
"name": "t3_1abcd2",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"score": 845,
|
||||
"ups": 845,
|
||||
"num_comments": 231,
|
||||
"permalink": "/r/programming/comments/1abcd2/a_deep_dive_into_btrees/",
|
||||
"url": "https://blog.dev/post",
|
||||
"author": "u_someone",
|
||||
"created_utc": 1786748531.0,
|
||||
"over_18": false,
|
||||
"stickied": false,
|
||||
"is_self": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t3",
|
||||
"data": {
|
||||
"subreddit": "databases",
|
||||
"id": "1efgh3",
|
||||
"name": "t3_1efgh3",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"score": 96,
|
||||
"ups": 96,
|
||||
"num_comments": 14,
|
||||
"permalink": "/r/databases/comments/1efgh3/a_deep_dive_into_btrees/",
|
||||
"url": "https://blog.dev/post",
|
||||
"author": "dbperson",
|
||||
"created_utc": 1786752000.0,
|
||||
"over_18": false,
|
||||
"stickied": false,
|
||||
"is_self": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t3",
|
||||
"data": {
|
||||
"subreddit": "rust",
|
||||
"id": "1ijkl4",
|
||||
"name": "t3_1ijkl4",
|
||||
"title": "A Deep Dive Into B-Trees",
|
||||
"score": 12,
|
||||
"ups": 12,
|
||||
"num_comments": 2,
|
||||
"permalink": "/r/rust/comments/1ijkl4/a_deep_dive_into_btrees/",
|
||||
"url": "https://blog.dev/post",
|
||||
"author": "ferris",
|
||||
"created_utc": 1786755000.0,
|
||||
"over_18": false,
|
||||
"stickied": false,
|
||||
"is_self": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"after": null,
|
||||
"dist": 0,
|
||||
"modhash": "",
|
||||
"geo_filter": "",
|
||||
"before": null,
|
||||
"children": []
|
||||
}
|
||||
}
|
||||
Vendored
+76
@@ -0,0 +1,76 @@
|
||||
[
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t3",
|
||||
"data": {
|
||||
"title": "A Story About Databases",
|
||||
"permalink": "/r/rust/comments/abc/title/",
|
||||
"score": 512,
|
||||
"num_comments": 87
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"author": "ferris",
|
||||
"score": 55,
|
||||
"body": "The borrow checker earns its keep here.",
|
||||
"body_html": "<div class=\"md\"><p>The borrow checker earns its keep here.</p></div>",
|
||||
"replies": {
|
||||
"kind": "Listing",
|
||||
"data": {
|
||||
"children": [
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"author": "crab",
|
||||
"score": 12,
|
||||
"body_html": "<div class=\"md\"><p>Only if you never fight it.</p></div>",
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "more",
|
||||
"data": { "count": 14, "children": ["x1", "x2"] }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"author": "[deleted]",
|
||||
"score": 1,
|
||||
"body_html": "<div class=\"md\"><p>[removed]</p></div>",
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "t1",
|
||||
"data": {
|
||||
"author": "rustacean",
|
||||
"score": 31,
|
||||
"body_html": "<div class=\"md\"><p>The section on &quot;unsafe&quot; is the best part.</p></div>",
|
||||
"replies": ""
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "more",
|
||||
"data": { "count": 40, "children": ["y1"] }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
<!DOCTYPE html>
|
||||
<html prefix="dc: http://purl.org/dc/terms/" about="http://en.wikipedia.org/wiki/Special:Redirect/revision/1">
|
||||
<head><meta charset="utf-8"/><title>Portal:Current events/2026 August 15</title></head>
|
||||
<body id="mwAA" lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body-content parsoid-body mediawiki mw-parser-output">
|
||||
<section data-mw-section-id="0" id="mwAQ">
|
||||
<div class="current-events-main vevent" id="mwAg">
|
||||
<div class="current-events-heading">
|
||||
<span class="summary"><span class="current-events-title">August 15, 2026</span></span>
|
||||
<span class="mw-editsection"><a href="/w/index.php?title=Portal:Current_events/2026_August_15&action=edit">edit</a></span>
|
||||
</div>
|
||||
<div class="current-events-content description" id="mwAw">
|
||||
<p><b>Armed conflicts and attacks</b></p>
|
||||
<ul>
|
||||
<li><a href="./Border_dispute" title="Border dispute">Border dispute</a> in the region continues for a fourth day, with mediators from the <a href="./United_Nations" title="United Nations">United Nations</a> arriving overnight.<sup class="reference" id="cite_ref-1"><a href="#cite_note-1">[1]</a></sup>
|
||||
<ul>
|
||||
<li>A second-level detail about the mediation timetable.</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>Officials report no casualties.<sup class="reference"><a href="#cite_note-2">[2]</a></sup></li>
|
||||
</ul>
|
||||
<p><b>Disasters and accidents</b></p>
|
||||
<ul>
|
||||
<li>Heavy rain floods the Charles River basin in <a href="./Boston" title="Boston">Boston</a>, <a href="./Massachusetts" title="Massachusetts">Massachusetts</a>, closing several stations.<sup class="reference"><a href="#cite_note-3">[3]</a></sup></li>
|
||||
</ul>
|
||||
<p><b>Science and technology</b></p>
|
||||
<ul>
|
||||
<li>A long-duration flight test concludes successfully.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="navbox" role="navigation">
|
||||
<ul><li>Ongoing events</li><li>Recent deaths</li></ul>
|
||||
</div>
|
||||
</section>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,17 @@
|
||||
<!DOCTYPE html>
|
||||
<html prefix="dc: http://purl.org/dc/terms/ mw: http://mediawiki.org/rdf/" about="//en.wikipedia.org/wiki/Special:Redirect/revision/1369461094"><head prefix="mwr: //en.wikipedia.org/wiki/Special:Redirect/"><meta charset="utf-8"/><link rel="dc:replaces" resource="mwr:revision/0"/><meta property="mw:revisionSHA1" content="bqap22vqgtaw0lrodhv0ng2kbg1lo4i"/><meta property="dc:modified" content="2026-08-15T03:30:02Z"/><meta property="mw:pageNamespace" content="100"/><meta property="mw:pageId" content="83976994"/><meta property="mw:htmlVersion" content="2.8.0"/><meta property="mw:html:version" content="2.8.0"/><base href="//en.wikipedia.org/wiki/"/><link rel="dc:isVersionOf" href="//en.wikipedia.org/wiki/Portal:Current_events/2026_August_16"/><title>Portal:Current events/2026 August 16</title><meta property="mw:jsConfigVars" content='{"wgParsoidHtmlVersion":"2.8.0"}'/><meta property="mw:moduleStyles" content="mediawiki.skinning.content.parsoid"/><link rel="stylesheet" href="/w/load.php?lang=en&modules=mediawiki.skinning.content.parsoid%7Cmediawiki.skinning.interface%7Csite.styles&only=styles&skin=vector"/><meta http-equiv="content-language" content="en"/><meta http-equiv="vary" content="Accept"/><meta http-equiv="x-mediawiki-render-id" content="9a187c46-9859-11f1-ac0e-b9eb5a8a3066"/></head><body class="mw-content-ltr sitedir-ltr ltr mw-body-content mediawiki mw-parser-output parsoid-body" lang="en" dir="ltr" data-mw-parsoid-version="0.24.0.0-alpha19" data-mw-html-version="2.8.0"><section data-mw-section-id="0" id="mwAQ"><span class="mw-empty-elt" about="#mwt1" typeof="mw:Transclusion" data-mw='{"parts":[{"template":{"target":{"wt":"Current events","href":"./Template:Current_events"},"params":{"year":{"wt":"2026"},"month":{"wt":"08"},"day":{"wt":"16"},"top":{"wt":"yes"}},"i":0}},"\n\n<!-- All news items below this line -->\n*\n<!-- All news items above this line -->\n\n",{"template":{"target":{"wt":"Current events","href":"./Template:Current_events"},"params":{"year":{"wt":"2026"},"month":{"wt":"08"},"day":{"wt":"16"},"bottom":{"wt":"yes"}},"i":1}}]}' id="mwAg"><style data-mw-deduplicate="TemplateStyles:r1305593205" typeof="mw:Extension/templatestyles" about="#mwt2" data-mw='{"name":"templatestyles","attrs":{"src":"Current events/styles.css"}}'>.mw-parser-output .current-events-main{margin:0.5em 0;padding:0.3em;background-color:var(--background-color-base,#fff);color:inherit;border:1px #cef2e0 solid}.mw-parser-output .current-events-heading{background-color:#cef2e0;color:inherit;font-weight:bold}@media screen{html.skin-theme-clientpref-night .mw-parser-output .current-events-heading{background-color:#0b281a}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .current-events-heading{background-color:#0b281a}}.mw-parser-output .current-events-title{padding:0.4em}.mw-parser-output .current-events-navbar{list-style:none;margin:0;font-size:small}.mw-parser-output .current-events-navbar li{display:inline-block;padding:0 0.4em}.mw-parser-output .current-events-content{padding:0 0.3em}.mw-parser-output .current-events-content-heading{margin-top:0.3em;font-weight:bold}.mw-parser-output .current-events-more{border-width:2px;font-size:10pt;font-weight:bold;padding:0.3em 0.6em}.mw-parser-output .current-events-nav{margin:auto;text-align:center;line-height:1.2}.mw-parser-output .current-events-nav a{display:inline-block;margin:0.5em;padding:0.5em;background-color:var(--background-color-neutral,#eaecf0)}.mw-parser-output .current-events-nav a>div{font-weight:bold}@media all and (min-width:480px){.mw-parser-output .current-events-heading{align-items:center;display:flex}.mw-parser-output .current-events-title{flex:1}.mw-parser-output .current-events-navbar{flex:0 auto;text-align:right;white-space:nowrap}.mw-parser-output .current-events-nav{max-width:22em}.mw-parser-output .current-events-nav a{width:9em}}</style></span><div class="current-events" about="#mwt1" id="mwAw">
|
||||
<div role="region" aria-label="August 16" id="2026_August_16" class="current-events-main vevent">
|
||||
<div class="current-events-heading plainlinks">
|
||||
<div class="current-events-title" role="heading"><span class="summary">August<span typeof="mw:Entity"> </span>16,<span typeof="mw:Entity"> </span>2026<span style="display: none;"><span typeof="mw:Entity"> </span>(<span class="bday dtstart published updated itvstart">2026-08-16</span>)</span> (Sunday)</span><link rel="mw:PageProp/Category" href="./Category:2026_by_day#2026-08-16"/>
|
||||
</div>
|
||||
<ul class="current-events-navbar editlink noprint"><li><a rel="mw:ExtLink" href="https://en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=edit&editintro=Portal:Current_events/Edit_instructions" class="external text" data-mw-original-href="//en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=edit&editintro=Portal:Current_events/Edit_instructions">edit</a></li><li><a rel="mw:ExtLink" href="https://en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=history" class="external text" data-mw-original-href="//en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=history">history</a></li><li><a rel="mw:ExtLink" href="https://en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=watch" class="external text" data-mw-original-href="//en.wikipedia.org/w/index.php?title=Portal:Current_events/2026_August_16&action=watch">watch</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="current-events-content description">
|
||||
|
||||
<!-- All news items below this line -->
|
||||
<ul></ul>
|
||||
<!-- All news items above this line -->
|
||||
|
||||
</div></div><div class="current-events-nav" role="navigation">
|
||||
<div><a rel="mw:WikiLink" href="./Portal:Current_events/August_2026" title="Portal:Current events/August 2026">Month<div>August 2026</div></a></div><a rel="mw:WikiLink" href="./Portal:Current_events/2026_August_15" title="Portal:Current events/2026 August 15">Previous day<div>August 15</div></a><a rel="mw:WikiLink" href="./Portal:Current_events/2026_August_17?action=edit&redlink=1" title="Portal:Current events/2026 August 17" class="new" typeof="mw:LocalizedAttrs" data-mw-i18n='{"title":{"lang":"x-page","key":"red-link-title","params":["Portal:Current events/2026 August 17"]}}'>Next day<div>August 17</div></a></div></div></section></body></html>
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Milestone 2 integration test: Miniflux entries → dedupe → extraction (§3.2, §3.3).
|
||||
//!
|
||||
//! No network: the extractor is built with [`Extractor::offline`], so the fetch
|
||||
//! leg of the §3.3 priority chain is skipped and the Miniflux/excerpt legs run
|
||||
//! exactly as they do in production (notes §6).
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
||||
use daily_epub::extract::{self, Extractor};
|
||||
use daily_epub::types::{Article, Entry, ExtractMethod, SourceKind};
|
||||
use daily_epub::{dedupe, miniflux};
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().expect("timestamp")
|
||||
}
|
||||
|
||||
fn body(words: usize) -> String {
|
||||
format!("<p>{}</p>", "sqlite pages and btrees ".repeat(words / 4))
|
||||
}
|
||||
|
||||
/// One page of ingest output: the same story from three feeds plus some noise.
|
||||
fn ingested() -> Vec<Entry> {
|
||||
let base = Entry {
|
||||
id: 0,
|
||||
feed_id: 0,
|
||||
feed_title: None,
|
||||
category: Some("Tech".into()),
|
||||
title: String::new(),
|
||||
url: String::new(),
|
||||
canonical_url: None,
|
||||
author: None,
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
raw_content: String::new(),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
};
|
||||
|
||||
vec![
|
||||
// 1. HN frontpage: title + link only, plus the discussion link.
|
||||
Entry {
|
||||
id: 101,
|
||||
feed_id: 1,
|
||||
feed_title: Some("Hacker News Front Page".into()),
|
||||
title: "A Deep Dive Into B-Trees".into(),
|
||||
url: "https://blog.dev/b-trees?utm_source=hnrss".into(),
|
||||
comments_url: Some("https://news.ycombinator.com/item?id=41234567".into()),
|
||||
raw_content: "<p>Comments</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
// 2. Scour: same story, same URL modulo the fragment, richer summary.
|
||||
Entry {
|
||||
id: 102,
|
||||
feed_id: 2,
|
||||
feed_title: Some("Scour: Databases".into()),
|
||||
title: "A Deep Dive Into B-Trees".into(),
|
||||
url: "https://blog.dev/b-trees#intro".into(),
|
||||
raw_content: format!(
|
||||
r#"<p>Intro.</p><figure><img src="/img/split.png" alt="a page split"></figure>{}"#,
|
||||
body(700)
|
||||
),
|
||||
published_at: Some(ts("2026-08-15T03:00:00Z")),
|
||||
..base.clone()
|
||||
},
|
||||
// 3. The blog's own feed: different URL, same story (title pass merges it).
|
||||
Entry {
|
||||
id: 103,
|
||||
feed_id: 3,
|
||||
feed_title: Some("blog.dev".into()),
|
||||
title: "A Deep Dive into B-Trees!".into(),
|
||||
url: "https://blog.dev/2026/08/b-trees".into(),
|
||||
author: Some("Dana Author".into()),
|
||||
raw_content: "<p>Short summary of the post.</p>".into(),
|
||||
published_at: Some(ts("2026-08-15T02:00:00Z")),
|
||||
..base.clone()
|
||||
},
|
||||
// 4. A different story, feed excerpt only.
|
||||
Entry {
|
||||
id: 104,
|
||||
feed_id: 4,
|
||||
feed_title: Some("Lobsters".into()),
|
||||
title: "Notes on Writing a Toy Allocator".into(),
|
||||
url: "https://other.dev/allocator".into(),
|
||||
comments_url: Some("https://lobste.rs/s/abcdef/notes".into()),
|
||||
raw_content: "<p>A teaser paragraph and nothing else.</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
// 5–7. Non-articles: video host, media enclosure, empty title.
|
||||
Entry {
|
||||
id: 105,
|
||||
feed_id: 5,
|
||||
feed_title: Some("Video Feed".into()),
|
||||
title: "A conference talk".into(),
|
||||
url: "https://www.youtube.com/watch?v=abc".into(),
|
||||
raw_content: "<p>watch it</p>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 106,
|
||||
feed_id: 6,
|
||||
feed_title: Some("A Podcast".into()),
|
||||
title: "Episode 42".into(),
|
||||
url: "https://cdn.dev/episodes/42.mp3".into(),
|
||||
raw_content: "<audio src=\"https://cdn.dev/episodes/42.mp3\"></audio>".into(),
|
||||
..base.clone()
|
||||
},
|
||||
Entry {
|
||||
id: 107,
|
||||
feed_id: 7,
|
||||
feed_title: Some("Broken Feed".into()),
|
||||
title: " ".into(),
|
||||
url: "https://broken.dev/x".into(),
|
||||
raw_content: body(500),
|
||||
..base.clone()
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedupe_then_extract_produces_ready_articles() {
|
||||
let (mut articles, stats) = dedupe::cluster(ingested());
|
||||
|
||||
// --- §3.2 ---
|
||||
assert_eq!(stats.entries_in, 7);
|
||||
assert_eq!(stats.dropped_non_article, 3);
|
||||
assert_eq!(stats.clusters, 2);
|
||||
assert_eq!(stats.merged, 1);
|
||||
assert_eq!(articles.len(), 2);
|
||||
|
||||
let deep_dive = &articles[0];
|
||||
assert_eq!(deep_dive.canonical_url, "https://blog.dev/b-trees");
|
||||
assert_eq!(deep_dive.id, 0, "not persisted yet");
|
||||
assert_eq!(deep_dive.best_entry_id, 102, "richest content wins");
|
||||
assert_eq!(deep_dive.sources.len(), 3);
|
||||
assert!(deep_dive.came_via(SourceKind::HnFrontpage));
|
||||
assert!(deep_dive.came_via(SourceKind::Scour));
|
||||
assert!(deep_dive.came_via(SourceKind::Feed));
|
||||
assert_eq!(deep_dive.author.as_deref(), Some("Dana Author"));
|
||||
assert_eq!(deep_dive.first_seen, ts("2026-08-15T02:00:00Z"));
|
||||
assert_eq!(
|
||||
deep_dive.comments_url.as_deref(),
|
||||
Some("https://news.ycombinator.com/item?id=41234567")
|
||||
);
|
||||
assert_eq!(deep_dive.chapter_id(), "art-102");
|
||||
|
||||
// --- §3.3 ---
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
assert!(!extractor.can_fetch(), "tests never hit the network");
|
||||
let extract_stats = extractor.extract_all(&mut articles).await;
|
||||
assert_eq!(extract_stats.from_miniflux, 1);
|
||||
assert_eq!(extract_stats.from_readability, 0);
|
||||
assert_eq!(extract_stats.excerpt_only, 1);
|
||||
|
||||
let deep_dive = &articles[0];
|
||||
assert_eq!(deep_dive.extract_method, ExtractMethod::Miniflux);
|
||||
assert!(deep_dive.word_count >= extract::FULL_TEXT_MIN_WORDS);
|
||||
assert!(!deep_dive.excerpt_only);
|
||||
// Sanitized body, relative image resolved against the entry URL.
|
||||
assert!(
|
||||
deep_dive.content_html.contains("<figure>"),
|
||||
"allowlisted tag"
|
||||
);
|
||||
assert!(
|
||||
deep_dive
|
||||
.content_html
|
||||
.contains(r#"src="https://blog.dev/img/split.png""#)
|
||||
);
|
||||
assert_eq!(deep_dive.image_count, 1);
|
||||
assert_eq!(deep_dive.image_urls, ["https://blog.dev/img/split.png"]);
|
||||
|
||||
let allocator = &articles[1];
|
||||
assert_eq!(allocator.canonical_url, "https://other.dev/allocator");
|
||||
assert_eq!(allocator.extract_method, ExtractMethod::Excerpt);
|
||||
assert!(allocator.excerpt_only, "penalized by the pre-filter (§3.5)");
|
||||
assert!(allocator.content_html.contains(extract::EXCERPT_NOTE));
|
||||
assert!(allocator.came_via(SourceKind::Lobsters));
|
||||
assert_eq!(allocator.image_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rerunning_the_stage_is_idempotent() {
|
||||
let (mut once, _) = dedupe::cluster(ingested());
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
extractor.extract_all(&mut once).await;
|
||||
|
||||
// Feeding the already-extracted articles back in changes nothing: the
|
||||
// sanitized body is still full text, so the Miniflux leg wins again.
|
||||
let before: Vec<Article> = once.clone();
|
||||
extractor.extract_all(&mut once).await;
|
||||
assert_eq!(once, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paywalled_stubs_are_marked_excerpt_only() {
|
||||
let entry = Entry {
|
||||
id: 200,
|
||||
feed_id: 20,
|
||||
feed_title: Some("NYT".into()),
|
||||
category: None,
|
||||
title: "A Paywalled Story".into(),
|
||||
url: "https://www.nytimes.com/2026/08/15/story.html".into(),
|
||||
canonical_url: None,
|
||||
author: None,
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
// Long enough to pass the full-text bar, short enough to smell like a stub.
|
||||
raw_content: body(300),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
};
|
||||
let (mut articles, _) = dedupe::cluster(vec![entry]);
|
||||
Extractor::offline(vec![]).extract_all(&mut articles).await;
|
||||
|
||||
assert_eq!(articles[0].extract_method, ExtractMethod::Miniflux);
|
||||
assert!(articles[0].word_count >= extract::FULL_TEXT_MIN_WORDS);
|
||||
assert!(articles[0].word_count < extract::PAYWALL_MAX_WORDS);
|
||||
assert!(articles[0].excerpt_only, "known paywall host + short body");
|
||||
}
|
||||
|
||||
/// A Scour interest feed whose *title* says nothing about Scour is still
|
||||
/// recognized when the run's `feed_id → url` map is threaded through (§3.2).
|
||||
#[tokio::test]
|
||||
async fn feed_urls_make_scour_detection_exact() {
|
||||
let entry = Entry {
|
||||
id: 300,
|
||||
feed_id: 30,
|
||||
// Scour names its feeds after the interest, not after itself.
|
||||
feed_title: Some("Rust".into()),
|
||||
category: Some("Interests".into()),
|
||||
title: "Async Cancellation, Revisited".into(),
|
||||
url: "https://blog.dev/cancellation".into(),
|
||||
canonical_url: None,
|
||||
author: None,
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
raw_content: body(600),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
};
|
||||
|
||||
// Without the map, the title/category give nothing away.
|
||||
let (plain, _) = dedupe::cluster(vec![entry.clone()]);
|
||||
assert!(!plain[0].came_via(SourceKind::Scour));
|
||||
|
||||
let feeds = std::collections::HashMap::from([(
|
||||
30,
|
||||
daily_epub::miniflux::FeedMeta {
|
||||
id: 30,
|
||||
title: "Rust".into(),
|
||||
site_url: "https://scour.ing".into(),
|
||||
feed_url: "https://scour.ing/feed?interest=rust&token=secret".into(),
|
||||
category: Some("Interests".into()),
|
||||
},
|
||||
)]);
|
||||
let (exact, _) = dedupe::cluster_with_feeds(vec![entry], &miniflux::feed_urls(&feeds));
|
||||
assert!(
|
||||
exact[0].came_via(SourceKind::Scour),
|
||||
"the feed URL is the only reliable Scour tell"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
//! Milestone 3 integration test: the curation contract (§3.5, §3.6).
|
||||
//!
|
||||
//! The stage logic is unit-tested inside `src/curate/*`. What this file guards is
|
||||
//! the contract *between* the curation stages and everything around them:
|
||||
//!
|
||||
//! * the recorded DeepSeek fixtures still parse through the real
|
||||
//! `score.rs` / `select.rs` / `editorial.rs` parsers into the structures the
|
||||
//! pipeline consumes, and the lenient parsers still cope with the messy one;
|
||||
//! * the shipped Scour OPML still yields the ~220 interests the taste profile is
|
||||
//! assembled from (§3.6a);
|
||||
//! * the binary starts, migrates a fresh database and exposes the `--skip-llm`
|
||||
//! path that lets the pipeline run without a DeepSeek key (notes §6).
|
||||
//!
|
||||
//! No network, no API key, no DeepSeek call.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use daily_epub::curate::editorial::FrontPageResponse;
|
||||
use daily_epub::curate::profile;
|
||||
use daily_epub::curate::score::parse_score_response;
|
||||
use daily_epub::curate::select::parse_selection_response;
|
||||
use daily_epub::types::LlmScore;
|
||||
|
||||
fn repo(rel: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join(rel)
|
||||
}
|
||||
|
||||
fn fixture(name: &str) -> String {
|
||||
let path = repo(&format!("tests/fixtures/{name}"));
|
||||
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
|
||||
}
|
||||
|
||||
/// Stage A responses must parse into `{id, score, category, rationale,
|
||||
/// is_paywalled_guess}` per article (§3.6).
|
||||
#[test]
|
||||
fn stage_a_fixture_parses_into_scores() {
|
||||
let items = parse_score_response(&fixture("deepseek_score_batch.json"));
|
||||
assert!(items.len() >= 4, "fixture should cover a realistic batch");
|
||||
|
||||
for item in &items {
|
||||
assert!(item.id > 0, "every item carries a positive article id");
|
||||
assert!(
|
||||
(0.0..=10.0).contains(&item.score),
|
||||
"score {} out of range",
|
||||
item.score
|
||||
);
|
||||
assert!(!item.category.is_empty());
|
||||
assert!(
|
||||
item.rationale.split_whitespace().count() <= 20,
|
||||
"rationale must stay under 20 words: {:?}",
|
||||
item.rationale
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
items.iter().any(|i| i.is_paywalled_guess),
|
||||
"the fixture should exercise the paywall flag"
|
||||
);
|
||||
|
||||
// The batch spans the rubric rather than clustering at one score.
|
||||
let scores: Vec<f64> = items.iter().map(|i| i.score).collect();
|
||||
let spread = scores.iter().cloned().fold(f64::MIN, f64::max)
|
||||
- scores.iter().cloned().fold(f64::MAX, f64::min);
|
||||
assert!(spread >= 3.0, "fixture scores are too uniform to be useful");
|
||||
|
||||
// Every item converts into the shared curation type.
|
||||
let converted: Vec<LlmScore> = items.into_iter().map(LlmScore::from).collect();
|
||||
assert!(converted.iter().all(|s| (0.0..=10.0).contains(&s.score)));
|
||||
}
|
||||
|
||||
/// The messy fixture must stay messy: it is what proves the parser is lenient
|
||||
/// (string ids, string scores, out-of-range scores, junk entries).
|
||||
#[test]
|
||||
fn stage_a_messy_fixture_is_salvaged_not_rejected() {
|
||||
let raw = fixture("deepseek_score_batch_messy.json");
|
||||
// The hard cases are still present in the recording…
|
||||
assert!(raw.contains("\"id\": \""), "needs a string id");
|
||||
assert!(raw.contains("\"score\": \""), "needs a string score");
|
||||
|
||||
// …and the real parser copes with all of them.
|
||||
let items = parse_score_response(&raw);
|
||||
assert!(!items.is_empty(), "the parser salvaged nothing");
|
||||
assert!(
|
||||
items.iter().all(|i| (0.0..=10.0).contains(&i.score)),
|
||||
"out-of-range scores must be clamped: {:?}",
|
||||
items.iter().map(|i| i.score).collect::<Vec<_>>()
|
||||
);
|
||||
assert!(items.iter().all(|i| i.id > 0), "id-less items are skipped");
|
||||
|
||||
// A response that is not JSON at all degrades to "no scores", never a panic.
|
||||
assert!(parse_score_response("I'm sorry, I can't do that.").is_empty());
|
||||
assert!(parse_score_response("").is_empty());
|
||||
}
|
||||
|
||||
/// Stage B responses must carry `{id, section, position, lead_story}` with
|
||||
/// exactly one lead, and use only palette section names (§3.6).
|
||||
#[test]
|
||||
fn stage_b_fixture_parses_into_a_lineup() {
|
||||
// The palette from `CurationConfig::default()` (§3.14).
|
||||
let sections = daily_epub::config::CurationConfig::default().sections;
|
||||
|
||||
let picks = parse_selection_response(&fixture("deepseek_lineup.json"));
|
||||
assert!(picks.len() >= 5);
|
||||
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut leads = 0;
|
||||
for pick in &picks {
|
||||
assert!(ids.insert(pick.id), "the same article was picked twice");
|
||||
assert!(
|
||||
sections.contains(&pick.section),
|
||||
"{:?} is not in the configured palette",
|
||||
pick.section
|
||||
);
|
||||
assert!(pick.position >= 1);
|
||||
if pick.lead_story {
|
||||
leads += 1;
|
||||
assert_eq!(
|
||||
pick.section, "Top Stories",
|
||||
"the lead sits in the first section"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(leads, 1, "exactly one lead story");
|
||||
// The reserved section is never offered to the model (§3.6, §3.8).
|
||||
assert!(
|
||||
!sections
|
||||
.iter()
|
||||
.any(|s| s == daily_epub::types::WORLD_BRIEFING_SECTION)
|
||||
);
|
||||
}
|
||||
|
||||
/// Stage C's front-page response must deserialize into a 250–400 word editor's
|
||||
/// note plus per-section intros (§3.6).
|
||||
#[test]
|
||||
fn stage_c_fixture_parses_into_a_front_page() {
|
||||
let response: FrontPageResponse = serde_json::from_str(&fixture("deepseek_front_page.json"))
|
||||
.expect("the front-page fixture must match FrontPageResponse");
|
||||
|
||||
let words = response.from_the_editor.split_whitespace().count();
|
||||
assert!(
|
||||
(150..=450).contains(&words),
|
||||
"From the Editor is {words} words; the prompt asks for 250-400"
|
||||
);
|
||||
assert!(
|
||||
response.from_the_editor.contains("\n\n"),
|
||||
"the prompt asks for 2-4 blank-line separated paragraphs"
|
||||
);
|
||||
assert!(
|
||||
!response.from_the_editor.contains("- "),
|
||||
"no bullet lists on the front page"
|
||||
);
|
||||
|
||||
assert!(response.section_intros.len() >= 2);
|
||||
for (section, intro) in &response.section_intros {
|
||||
let words = intro.split_whitespace().count();
|
||||
assert!(
|
||||
(10..=90).contains(&words),
|
||||
"intro for {section} is {words} words; the prompt asks for 35-60"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The taste profile is seeded from this file; a broken export would silently
|
||||
/// gut the system prompt (§3.6a).
|
||||
#[test]
|
||||
fn scour_opml_still_yields_the_interest_list() {
|
||||
let interests = profile::parse_interests(&repo("data/scour-interests.opml"))
|
||||
.expect("the shipped OPML must parse");
|
||||
let unique: BTreeSet<String> = interests.iter().map(|n| n.to_lowercase()).collect();
|
||||
|
||||
assert!(
|
||||
unique.len() > 180,
|
||||
"expected ~220 interests, found {}",
|
||||
unique.len()
|
||||
);
|
||||
for expected in [
|
||||
"rust",
|
||||
"boston tech",
|
||||
"e-ink displays",
|
||||
"self-hosted",
|
||||
"sci-fi",
|
||||
] {
|
||||
assert!(unique.contains(expected), "{expected} disappeared");
|
||||
}
|
||||
assert!(
|
||||
!interests.iter().any(|n| n.contains("token=")),
|
||||
"interest names must not leak the Scour token"
|
||||
);
|
||||
|
||||
// The assembled profile is what actually reaches DeepSeek as the system
|
||||
// prompt; it must mention the stated preferences and the interests (§3.6).
|
||||
let document = profile::build(&interests, profile::NO_LEARNED_ADJUSTMENTS);
|
||||
assert!(document.contains("Rust"));
|
||||
assert!(document.len() > 1000, "the profile is suspiciously short");
|
||||
}
|
||||
|
||||
/// The binary must migrate a fresh database and advertise the offline
|
||||
/// `--skip-llm` path (§2, notes §6).
|
||||
#[test]
|
||||
fn binary_migrates_and_offers_the_skip_llm_path() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db_path = dir.path().join("nested").join("daily-epub.db");
|
||||
let bin = env!("CARGO_BIN_EXE_daily-epub");
|
||||
|
||||
let out = Command::new(bin)
|
||||
.args(["db", "migrate"])
|
||||
.env("DAILY_EPUB_DATABASE_PATH", &db_path)
|
||||
.output()
|
||||
.expect("running `daily-epub db migrate`");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"db migrate failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
assert!(db_path.exists(), "the database was not created");
|
||||
|
||||
// Re-running is idempotent.
|
||||
let out = Command::new(bin)
|
||||
.args(["db", "migrate"])
|
||||
.env("DAILY_EPUB_DATABASE_PATH", &db_path)
|
||||
.output()
|
||||
.expect("re-running `daily-epub db migrate`");
|
||||
assert!(out.status.success());
|
||||
|
||||
let out = Command::new(bin)
|
||||
.args(["generate", "--help"])
|
||||
.output()
|
||||
.expect("running `daily-epub generate --help`");
|
||||
let help = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(out.status.success());
|
||||
assert!(
|
||||
help.contains("--skip-llm"),
|
||||
"generate must expose --skip-llm"
|
||||
);
|
||||
assert!(help.contains("--dry-run"));
|
||||
assert!(help.contains("--max-articles"));
|
||||
}
|
||||
|
||||
/// An unreachable Miniflux must fail the run cleanly, with the failure recorded
|
||||
/// and an error chain a human can act on (§5 verification).
|
||||
#[test]
|
||||
fn generate_fails_cleanly_when_miniflux_is_unreachable() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let bin = env!("CARGO_BIN_EXE_daily-epub");
|
||||
|
||||
let out = Command::new(bin)
|
||||
.args([
|
||||
"generate",
|
||||
"--dry-run",
|
||||
"--skip-llm",
|
||||
"--date",
|
||||
"2026-08-15",
|
||||
])
|
||||
.env("DAILY_EPUB_DATABASE_PATH", dir.path().join("daily-epub.db"))
|
||||
.env("DAILY_EPUB_OUT_DIR", dir.path().join("out"))
|
||||
// Port 1 is reserved and never listening.
|
||||
.env("DAILY_EPUB_MINIFLUX__BASE_URL", "http://127.0.0.1:1")
|
||||
.env("DAILY_EPUB_MINIFLUX__API_KEY", "not-a-real-key")
|
||||
.output()
|
||||
.expect("running `daily-epub generate`");
|
||||
|
||||
assert!(!out.status.success(), "the run must not report success");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(
|
||||
stderr.contains("ingesting entries from miniflux"),
|
||||
"the error chain must name the failing stage:\n{stderr}"
|
||||
);
|
||||
assert!(
|
||||
stderr.contains("miniflux") && stderr.contains("127.0.0.1:1"),
|
||||
"the error chain must name the unreachable endpoint:\n{stderr}"
|
||||
);
|
||||
// Nothing was written to the output directory.
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join("out")
|
||||
.join("The Daily EPUB - 2026-08-15.epub")
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! M4 — a complete issue EPUB, built offline (spec §3.10, §4 M4).
|
||||
//!
|
||||
//! Everything here is offline: the synthetic issue's images are never
|
||||
//! downloaded, so the chapters exercise the placeholder path.
|
||||
|
||||
use daily_epub::config::{self, Config};
|
||||
use daily_epub::epub::build::fixtures;
|
||||
use daily_epub::epub::{self, build};
|
||||
use daily_epub::types::{Edition, Issue, Vote};
|
||||
use daily_epub::{comments, world};
|
||||
|
||||
/// Local file headers store entry names verbatim, so a byte search over the
|
||||
/// archive is enough to assert its contents without a zip reader.
|
||||
fn contains_entry(zip: &[u8], name: &str) -> bool {
|
||||
zip.windows(name.len()).any(|w| w == name.as_bytes())
|
||||
}
|
||||
|
||||
/// Read one entry out of the archive, inflating it.
|
||||
fn read_entry(zip: &[u8], name: &str) -> String {
|
||||
use std::io::Read as _;
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip)).expect("zip opens");
|
||||
let mut file = archive.by_name(name).expect("entry exists");
|
||||
let mut out = String::new();
|
||||
file.read_to_string(&mut out).expect("entry is text");
|
||||
out
|
||||
}
|
||||
|
||||
/// Builds one edition into a temporary directory; the directory is cleaned up
|
||||
/// when the returned `TempDir` is dropped.
|
||||
fn build_edition_to_bytes(
|
||||
issue: &Issue,
|
||||
edition: Edition,
|
||||
) -> (tempfile::TempDir, std::path::PathBuf, Vec<u8>) {
|
||||
let cfg = Config {
|
||||
server: config::ServerConfig {
|
||||
public_url: "https://daily.hallada.net".into(),
|
||||
hmac_secret: Some("integration-secret".into()),
|
||||
..config::ServerConfig::default()
|
||||
},
|
||||
..Config::default()
|
||||
};
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let artifact = epub::build_edition_with_images(issue, edition, &cfg, dir.path(), &[])
|
||||
.expect("edition builds");
|
||||
let bytes = std::fs::read(&artifact.path).expect("read epub");
|
||||
assert_eq!(artifact.bytes as usize, bytes.len());
|
||||
(dir, artifact.path, bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_edition_is_a_well_formed_epub3_archive() {
|
||||
let issue = fixtures::issue();
|
||||
let (_dir, path, zip) = build_edition_to_bytes(&issue, Edition::Standard);
|
||||
|
||||
assert_eq!(
|
||||
path.file_name().and_then(|n| n.to_str()),
|
||||
Some("The Daily EPUB - 2026-08-15.epub")
|
||||
);
|
||||
assert_eq!(&zip[0..4], b"PK\x03\x04", "starts with a zip local header");
|
||||
assert_eq!(
|
||||
&zip[30..38],
|
||||
b"mimetype",
|
||||
"`mimetype` must be the first entry"
|
||||
);
|
||||
assert_eq!(&zip[38..58], b"application/epub+zip");
|
||||
|
||||
for entry in [
|
||||
"META-INF/container.xml",
|
||||
"OEBPS/content.opf",
|
||||
"OEBPS/toc.ncx",
|
||||
"OEBPS/nav.xhtml",
|
||||
"OEBPS/stylesheet.css",
|
||||
"OEBPS/cover.png",
|
||||
"OEBPS/cover.xhtml",
|
||||
"OEBPS/front.xhtml",
|
||||
"OEBPS/in-this-issue.xhtml",
|
||||
"OEBPS/sec-top-stories.xhtml",
|
||||
"OEBPS/art-1001.xhtml",
|
||||
"OEBPS/disc-1001.xhtml",
|
||||
"OEBPS/sec-niche-corner.xhtml",
|
||||
"OEBPS/art-1002.xhtml",
|
||||
"OEBPS/world.xhtml",
|
||||
"OEBPS/colophon.xhtml",
|
||||
] {
|
||||
assert!(contains_entry(&zip, entry), "missing {entry}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x4_edition_is_built_alongside_the_standard_one() {
|
||||
let issue = fixtures::issue();
|
||||
let (_dir, path, zip) = build_edition_to_bytes(&issue, Edition::X4);
|
||||
assert_eq!(
|
||||
path.file_name().and_then(|n| n.to_str()),
|
||||
Some("The Daily EPUB - 2026-08-15 (X4).epub")
|
||||
);
|
||||
assert_eq!(&zip[30..38], b"mimetype");
|
||||
assert!(contains_entry(&zip, "OEBPS/art-1001.xhtml"));
|
||||
assert!(contains_entry(&zip, "OEBPS/cover.png"));
|
||||
}
|
||||
|
||||
/// Both editions land in the same BookOrbit library, which lists books by
|
||||
/// `dc:title` — so the edition has to be in the title, not just the filename
|
||||
/// (§3.10). Without this the two are indistinguishable in the library UI and
|
||||
/// over OPDS.
|
||||
#[test]
|
||||
fn the_two_editions_have_distinct_titles_in_the_opf() {
|
||||
let issue = fixtures::issue();
|
||||
let (_d1, _, standard) = build_edition_to_bytes(&issue, Edition::Standard);
|
||||
let (_d2, _, x4) = build_edition_to_bytes(&issue, Edition::X4);
|
||||
|
||||
let standard_opf = read_entry(&standard, "OEBPS/content.opf");
|
||||
let x4_opf = read_entry(&x4, "OEBPS/content.opf");
|
||||
|
||||
assert!(
|
||||
standard_opf.contains("<dc:title>The Daily EPUB \u{2014} 2026-08-15</dc:title>"),
|
||||
"{standard_opf}"
|
||||
);
|
||||
assert!(
|
||||
x4_opf.contains("<dc:title>The Daily EPUB \u{2014} 2026-08-15 (X4)</dc:title>"),
|
||||
"{x4_opf}"
|
||||
);
|
||||
|
||||
// The series metadata still groups them: same collection, same position, so
|
||||
// they sort together rather than as two unrelated books.
|
||||
for opf in [&standard_opf, &x4_opf] {
|
||||
assert!(opf.contains("belongs-to-collection"), "{opf}");
|
||||
assert!(opf.contains("<dc:date>2026-08-15</dc:date>"), "{opf}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapter_ids_hrefs_and_toc_levels_are_stable() {
|
||||
let issue = fixtures::issue();
|
||||
let first =
|
||||
build::render_all(&issue, Edition::Standard, &[], "https://x.test", None).expect("render");
|
||||
let second =
|
||||
build::render_all(&issue, Edition::Standard, &[], "https://x.test", None).expect("render");
|
||||
assert_eq!(first, second, "rendering is deterministic");
|
||||
|
||||
let map: Vec<(String, String, u8)> = first
|
||||
.iter()
|
||||
.map(|c| (c.id.clone(), c.href.clone(), c.toc_level))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
map,
|
||||
vec![
|
||||
("cover".into(), "cover.xhtml".into(), 1),
|
||||
("front".into(), "front.xhtml".into(), 1),
|
||||
("in-this-issue".into(), "in-this-issue.xhtml".into(), 1),
|
||||
("sec-Top Stories".into(), "sec-top-stories.xhtml".into(), 1),
|
||||
("art-1001".into(), "art-1001.xhtml".into(), 2),
|
||||
("disc-1001".into(), "disc-1001.xhtml".into(), 3),
|
||||
(
|
||||
"sec-Niche Corner".into(),
|
||||
"sec-niche-corner.xhtml".into(),
|
||||
1
|
||||
),
|
||||
("art-1002".into(), "art-1002.xhtml".into(), 2),
|
||||
("world".into(), "world.xhtml".into(), 1),
|
||||
("colophon".into(), "colophon.xhtml".into(), 1),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_chapter_is_parseable_xhtml() {
|
||||
let issue = fixtures::issue();
|
||||
let chapters = build::render_all(
|
||||
&issue,
|
||||
Edition::Standard,
|
||||
&[],
|
||||
"https://daily.hallada.net",
|
||||
Some("integration-secret"),
|
||||
)
|
||||
.expect("render");
|
||||
|
||||
for chapter in &chapters {
|
||||
let xhtml = &chapter.xhtml;
|
||||
assert!(
|
||||
xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>"),
|
||||
"{} lacks an XML prologue",
|
||||
chapter.id
|
||||
);
|
||||
assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\""));
|
||||
assert!(xhtml.trim_end().ends_with("</html>"));
|
||||
// Undefined XML entities (html5ever's ` `) would break XML parsers.
|
||||
assert!(!xhtml.contains(" "), "{} has ", chapter.id);
|
||||
for tag in ["html", "head", "body", "div", "p", "a", "blockquote"] {
|
||||
let opens = xhtml.matches(&format!("<{tag}")).count();
|
||||
let closes = xhtml.matches(&format!("</{tag}>")).count();
|
||||
assert_eq!(opens, closes, "unbalanced <{tag}> in {}", chapter.id);
|
||||
}
|
||||
// Void elements are self-closed.
|
||||
for void in ["<br>", "<hr>", "<link ", "<meta charset=\"utf-8\">"] {
|
||||
assert!(!xhtml.contains(void) || xhtml.contains("/>"), "{void}");
|
||||
}
|
||||
// Every `&` opens an entity XML actually defines.
|
||||
for (i, _) in xhtml.match_indices('&') {
|
||||
let tail = &xhtml[i + 1..];
|
||||
assert!(
|
||||
is_defined_entity(tail),
|
||||
"bare `&` at byte {i} in {}: {:?}",
|
||||
chapter.id,
|
||||
&xhtml[i..(i + 24).min(xhtml.len())]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// XML predefines only these five names; everything else must be numeric.
|
||||
fn is_defined_entity(after_ampersand: &str) -> bool {
|
||||
let Some(end) = after_ampersand.find(';') else {
|
||||
return false;
|
||||
};
|
||||
let name = &after_ampersand[..end];
|
||||
if matches!(name, "amp" | "lt" | "gt" | "quot" | "apos") {
|
||||
return true;
|
||||
}
|
||||
match name.strip_prefix('#') {
|
||||
Some(rest) => match rest.strip_prefix('x').or_else(|| rest.strip_prefix('X')) {
|
||||
Some(hex) => !hex.is_empty() && hex.chars().all(|c| c.is_ascii_hexdigit()),
|
||||
None => !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit()),
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rating_links_carry_the_spec_token() {
|
||||
let issue = fixtures::issue();
|
||||
let chapter = build::render_all(
|
||||
&issue,
|
||||
Edition::Standard,
|
||||
&[],
|
||||
"https://daily.hallada.net",
|
||||
Some("integration-secret"),
|
||||
)
|
||||
.expect("render")
|
||||
.into_iter()
|
||||
.find(|c| c.id == "art-1001")
|
||||
.expect("article chapter");
|
||||
|
||||
let date = issue.meta.date;
|
||||
let up = build::rating_token("integration-secret", date, 1, Vote::Up);
|
||||
let down = build::rating_token("integration-secret", date, 1, Vote::Down);
|
||||
assert_eq!(up.len(), 16);
|
||||
assert_ne!(up, down);
|
||||
assert!(chapter.xhtml.contains(&format!(
|
||||
"https://daily.hallada.net/r/2026-08-15/1/up?t={up}"
|
||||
)));
|
||||
assert!(chapter.xhtml.contains(&format!(
|
||||
"https://daily.hallada.net/r/2026-08-15/1/down?t={down}"
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_and_world_fixtures_feed_real_chapters() {
|
||||
let hn: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/hn_item.json"
|
||||
))
|
||||
.expect("fixture"),
|
||||
)
|
||||
.expect("json");
|
||||
let thread = comments::parse_hn(&hn).expect("thread");
|
||||
assert_eq!(thread.total_comments, 4);
|
||||
|
||||
let html = std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/wikipedia_current_events.html"
|
||||
))
|
||||
.expect("fixture");
|
||||
let body = world::extract_events(&html).expect("events");
|
||||
assert!(body.contains("<li>"));
|
||||
assert!(!body.contains("<a "));
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//! M7/M8 integration: drive the real `daily-epub serve` binary over TCP.
|
||||
//!
|
||||
//! The crate has no library target, so this file cannot link the crate's modules
|
||||
//! (the endpoint-level tests over `server::router` live in `src/server.rs`).
|
||||
//! What it *can* do — and what nothing else covers — is prove that the shipped
|
||||
//! binary boots from `DAILY_EPUB_*` configuration, migrates its database, and
|
||||
//! answers the spec's routes on a real socket, exactly as the systemd unit runs it
|
||||
//! (spec §3.12, §3.15).
|
||||
//!
|
||||
//! Only std + dev-dependencies are available here, so the HTTP client below is a
|
||||
//! hand-rolled `GET`.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Shared fixture vector, mirrored by the unit tests in `src/server.rs`:
|
||||
/// `hex(hmac_sha256("test-secret", "2026-08-15/42/up"))[..16]`.
|
||||
const SECRET: &str = "test-secret";
|
||||
const TOKEN_UP_ARTICLE_42: &str = "3b314cf7e6d8f50f";
|
||||
/// base64("opds:hunter2")
|
||||
const BASIC_AUTH: &str = "b3BkczpodW50ZXIy";
|
||||
|
||||
struct Server {
|
||||
child: Child,
|
||||
port: u16,
|
||||
dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl Server {
|
||||
fn start(basic_auth: bool) -> Server {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let xtc_dir = dir.path().join("xtc");
|
||||
std::fs::create_dir_all(&xtc_dir).expect("xtc dir");
|
||||
let log = std::fs::File::create(dir.path().join("server.log")).expect("log file");
|
||||
|
||||
let port = free_port();
|
||||
let mut cmd = Command::new(env!("CARGO_BIN_EXE_daily-epub"));
|
||||
cmd.arg("serve")
|
||||
// cwd must not contain the repo's config.toml.
|
||||
.current_dir(dir.path())
|
||||
.env_clear()
|
||||
.env("PATH", std::env::var("PATH").unwrap_or_default())
|
||||
.env("RUST_LOG", "warn")
|
||||
.env("DAILY_EPUB_DATABASE_PATH", dir.path().join("daily-epub.db"))
|
||||
.env("DAILY_EPUB_SERVER__BIND", format!("127.0.0.1:{port}"))
|
||||
.env("DAILY_EPUB_SERVER__HMAC_SECRET", SECRET)
|
||||
.env(
|
||||
"DAILY_EPUB_SERVER__PUBLIC_URL",
|
||||
format!("http://127.0.0.1:{port}"),
|
||||
)
|
||||
.env("DAILY_EPUB_PUBLISH__XTC_DIR", &xtc_dir)
|
||||
.env(
|
||||
"DAILY_EPUB_PUBLISH__BOOKORBIT_DIR",
|
||||
dir.path().join("bookorbit"),
|
||||
)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::from(log));
|
||||
if basic_auth {
|
||||
cmd.env("DAILY_EPUB_SERVER__BASIC_AUTH_USER", "opds")
|
||||
.env("DAILY_EPUB_SERVER__BASIC_AUTH_PASS", "hunter2");
|
||||
}
|
||||
let child = cmd.spawn().expect("spawning daily-epub serve");
|
||||
|
||||
let server = Server { child, port, dir };
|
||||
server.wait_until_ready();
|
||||
server
|
||||
}
|
||||
|
||||
fn xtc_dir(&self) -> std::path::PathBuf {
|
||||
self.dir.path().join("xtc")
|
||||
}
|
||||
|
||||
fn wait_until_ready(&self) {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
while Instant::now() < deadline {
|
||||
if let Some(res) = try_get(self.port, "/healthz", None)
|
||||
&& res.status == 200
|
||||
{
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
let log = std::fs::read_to_string(self.dir.path().join("server.log")).unwrap_or_default();
|
||||
panic!(
|
||||
"daily-epub serve never became healthy on port {}:\n{log}",
|
||||
self.port
|
||||
);
|
||||
}
|
||||
|
||||
fn get(&self, path: &str) -> HttpResponse {
|
||||
try_get(self.port, path, None).expect("request failed")
|
||||
}
|
||||
|
||||
fn get_auth(&self, path: &str, credentials: &str) -> HttpResponse {
|
||||
try_get(self.port, path, Some(credentials)).expect("request failed")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HttpResponse {
|
||||
status: u16,
|
||||
headers: String,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl HttpResponse {
|
||||
fn header(&self, name: &str) -> Option<&str> {
|
||||
let name = format!("{}:", name.to_ascii_lowercase());
|
||||
self.headers
|
||||
.lines()
|
||||
.find(|l| l.to_ascii_lowercase().starts_with(&name))
|
||||
.and_then(|l| l.split_once(':'))
|
||||
.map(|(_, v)| v.trim())
|
||||
}
|
||||
}
|
||||
|
||||
fn free_port() -> u16 {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("ephemeral port");
|
||||
let port = listener.local_addr().expect("local addr").port();
|
||||
drop(listener);
|
||||
port
|
||||
}
|
||||
|
||||
/// A minimal HTTP/1.1 `GET`; `None` when the connection could not be made.
|
||||
fn try_get(port: u16, path: &str, credentials: Option<&str>) -> Option<HttpResponse> {
|
||||
let mut stream = TcpStream::connect(("127.0.0.1", port)).ok()?;
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.ok()?;
|
||||
let auth = credentials
|
||||
.map(|c| format!("Authorization: Basic {c}\r\n"))
|
||||
.unwrap_or_default();
|
||||
let request =
|
||||
format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n{auth}\r\n");
|
||||
stream.write_all(request.as_bytes()).ok()?;
|
||||
stream.flush().ok()?;
|
||||
// No half-close here: hyper drops a connection whose peer has shut down its
|
||||
// write side before the response is written. `Connection: close` is enough.
|
||||
|
||||
let mut raw = Vec::new();
|
||||
stream.read_to_end(&mut raw).ok()?;
|
||||
let text = String::from_utf8_lossy(&raw).into_owned();
|
||||
let (head, body) = text.split_once("\r\n\r\n")?;
|
||||
let status = head
|
||||
.lines()
|
||||
.next()?
|
||||
.split_whitespace()
|
||||
.nth(1)?
|
||||
.parse()
|
||||
.ok()?;
|
||||
Some(HttpResponse {
|
||||
status,
|
||||
headers: head.to_string(),
|
||||
body: body.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_serves_health_issues_and_rating_endpoints() {
|
||||
let server = Server::start(false);
|
||||
|
||||
let res = server.get("/healthz");
|
||||
assert_eq!(res.status, 200);
|
||||
assert_eq!(res.body, "ok");
|
||||
|
||||
// Migrations ran on startup, so `issues.json` answers with an empty list.
|
||||
let res = server.get("/issues.json");
|
||||
assert_eq!(res.status, 200);
|
||||
assert_eq!(
|
||||
res.header("content-type"),
|
||||
Some("application/json"),
|
||||
"{}",
|
||||
res.headers
|
||||
);
|
||||
assert_eq!(res.body.trim(), "[]");
|
||||
|
||||
// A tampered token never reaches the database.
|
||||
let res = server.get("/r/2026-08-15/42/up?t=deadbeefdeadbeef");
|
||||
assert_eq!(res.status, 403);
|
||||
assert!(res.body.contains("Invalid link"), "{}", res.body);
|
||||
|
||||
// The shared HMAC vector verifies, but article 42 does not exist here.
|
||||
let res = server.get(&format!("/r/2026-08-15/42/up?t={TOKEN_UP_ARTICLE_42}"));
|
||||
assert_eq!(res.status, 404, "token vector no longer verifies: {res:?}");
|
||||
assert!(res.body.contains("Unknown article"), "{}", res.body);
|
||||
|
||||
// The confirmation pages stay e-ink sized and self contained (§3.9).
|
||||
assert!(res.body.len() < 1024, "page is {} bytes", res.body.len());
|
||||
assert!(!res.body.contains("<link"));
|
||||
|
||||
// Malformed date / vote are rejected before any lookup.
|
||||
assert_eq!(
|
||||
server
|
||||
.get(&format!("/r/nope/42/up?t={TOKEN_UP_ARTICLE_42}"))
|
||||
.status,
|
||||
400
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.get(&format!("/r/2026-08-15/42/maybe?t={TOKEN_UP_ARTICLE_42}"))
|
||||
.status,
|
||||
400
|
||||
);
|
||||
assert_eq!(server.get("/nope").status, 404);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_serves_opds_and_files_behind_basic_auth() {
|
||||
let server = Server::start(true);
|
||||
let name = "The Daily EPUB - 2026-08-15 (X4).xtch";
|
||||
write(&server.xtc_dir().join("xtc.xml"), FEED);
|
||||
write(&server.xtc_dir().join(name), "XTCH");
|
||||
write(&server.dir.path().join("secret"), "top secret");
|
||||
|
||||
// No credentials → challenge.
|
||||
let res = server.get("/opds/xtc.xml");
|
||||
assert_eq!(res.status, 401);
|
||||
assert_eq!(
|
||||
res.header("www-authenticate"),
|
||||
Some("Basic realm=\"The Daily EPUB\", charset=\"UTF-8\"")
|
||||
);
|
||||
assert_eq!(server.get_auth("/opds/xtc.xml", "bm9wZTpub3Bl").status, 401);
|
||||
|
||||
// Correct credentials → the feed, typed as OPDS.
|
||||
let res = server.get_auth("/opds/xtc.xml", BASIC_AUTH);
|
||||
assert_eq!(res.status, 200);
|
||||
assert!(
|
||||
res.header("content-type")
|
||||
.unwrap_or_default()
|
||||
.starts_with("application/atom+xml"),
|
||||
"{}",
|
||||
res.headers
|
||||
);
|
||||
assert!(res.body.contains("opds-spec.org/acquisition"));
|
||||
|
||||
// The acquisition link in the feed resolves to the file itself.
|
||||
let res = server.get_auth(
|
||||
"/files/xtc/The%20Daily%20EPUB%20-%202026-08-15%20%28X4%29.xtch",
|
||||
BASIC_AUTH,
|
||||
);
|
||||
assert_eq!(res.status, 200);
|
||||
assert_eq!(res.body, "XTCH");
|
||||
assert_eq!(res.header("content-type"), Some("application/octet-stream"));
|
||||
|
||||
// Path traversal, percent-encoded so the URL parser cannot normalize it away.
|
||||
for attack in [
|
||||
"/files/xtc/..%2Fsecret",
|
||||
"/files/xtc/%2e%2e%2fsecret",
|
||||
"/files/xtc/%2Fetc%2Fpasswd",
|
||||
] {
|
||||
let res = server.get_auth(attack, BASIC_AUTH);
|
||||
assert_eq!(res.status, 400, "{attack} was not rejected: {res:?}");
|
||||
assert!(!res.body.contains("top secret"));
|
||||
}
|
||||
// Unauthenticated traversal is refused before the path is even looked at.
|
||||
assert_eq!(server.get("/files/xtc/..%2Fsecret").status, 401);
|
||||
|
||||
// /healthz stays open so the reverse proxy can probe it.
|
||||
assert_eq!(server.get("/healthz").status, 200);
|
||||
}
|
||||
|
||||
fn write(path: &Path, body: &str) {
|
||||
std::fs::write(path, body).unwrap_or_else(|e| panic!("writing {}: {e}", path.display()));
|
||||
}
|
||||
|
||||
/// A feed shaped like the one `publish::write_xtc_opds` generates.
|
||||
const FEED: &str = r#"<?xml version="1.0" encoding="utf-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<id>urn:daily-epub:xtc</id>
|
||||
<title>The Daily EPUB — XTC editions</title>
|
||||
<updated>2026-08-15T05:40:00Z</updated>
|
||||
<entry>
|
||||
<title>The Daily EPUB — 2026-08-15</title>
|
||||
<id>urn:daily-epub:xtc:x</id>
|
||||
<updated>2026-08-15T05:40:00Z</updated>
|
||||
<link rel="http://opds-spec.org/acquisition" href="http://127.0.0.1/files/xtc/x.xtch" type="application/octet-stream" length="4"/>
|
||||
</entry>
|
||||
</feed>
|
||||
"#;
|
||||
Reference in New Issue
Block a user