diff --git a/README.md b/README.md
index 1bed58e..87b1bd0 100644
--- a/README.md
+++ b/README.md
@@ -399,10 +399,32 @@ extraction → prefilter → selection (both the `--skip-llm` route and a
`MockBackend` DeepSeek route) → editorial → both EPUB editions → publish → OPDS
and database rows, with no network access anywhere.
-Layout: `src/pipeline.rs` wires the stages; `src/{dedupe,extract,social,curate,
-comments,world,epub,publish,server}` implement them; `src/auth.rs` owns the rating
-token formula used by both the EPUB writer and the server; `src/types.rs` is the
-contract between stages.
+### Layout
+
+`src/pipeline.rs` wires the stages; `src/types.rs` is the contract between them;
+`src/auth.rs` owns the rating token formula, shared by the EPUB writer and the
+server. The stages themselves:
+
+```text
+miniflux.rs ingest curate/ scoring and selection
+dedupe.rs clustering prefilter, llm, score, select, editorial
+extract.rs body text profile/ the reader's taste profile
+images/ article images comments.rs discussion chapters
+ normalize usable ![]()
world.rs the world briefing
+ refs what's there epub/ the two editions
+ fetch download chapters, cover, build, x4
+ encode re-encode publish.rs BookOrbit + XTC
+ embed into the page server.rs ratings, OPDS
+html.rs markup helpers db.rs SQLite
+```
+
+Two modules are worth knowing about before you go looking for their contents.
+`src/html.rs` holds the generic markup helpers — tag scanning, escaping, entity
+decoding, XHTML fixups — that extraction, images, comments and the world briefing
+all need; put anything that works on markup without caring what the markup is
+*about* there. `src/images/` owns every stage of an article's images, which is
+otherwise the kind of concern that smears itself across extraction and EPUB
+building; see its module docs for the order the stages run in.
### Auditing images against real articles
@@ -419,7 +441,9 @@ cargo run --release --example image_audit -- \
It prints per-article `refs / embedded / shown / placeholders`, a tally of loss
reasons, and totals. Pages are cached on first run, so a change can be measured
against byte-identical input; `--dump ` lists the URLs one
-article resolved to. It needs the network and is not part of `cargo test`.
+article resolved to, and `--epub-out DIR` writes a readable EPUB of the audited
+articles so the images can be looked at on a device rather than counted in a
+table. It needs the network and is not part of `cargo test`.
---
diff --git a/examples/image_audit.rs b/examples/image_audit.rs
index aa73217..cc37b8e 100644
--- a/examples/image_audit.rs
+++ b/examples/image_audit.rs
@@ -20,9 +20,14 @@ use std::collections::BTreeMap;
use std::io::Read;
use std::path::{Path, PathBuf};
-use daily_epub::epub::{build, images};
+use daily_epub::epub::build;
use daily_epub::extract::{self, Extractor};
-use daily_epub::types::{Article, EntryId, ExtractMethod, ImageAsset, Pick, SourceKind, SourceRef};
+use daily_epub::html;
+use daily_epub::images;
+use daily_epub::types::{
+ Article, Colophon, Editorial, EntryId, ExtractMethod, ImageAsset, Issue, IssueMeta, Lineup,
+ Pick, SourceKind, SourceRef,
+};
/// One article we are auditing, pulled back out of a published EPUB.
#[derive(Debug, Clone)]
@@ -60,17 +65,23 @@ async fn main() {
let mut cache = PathBuf::from("/tmp/daily-epub-audit-cache");
let mut dump: Option<String> = None;
+ let mut epub_out: Option<PathBuf> = None;
let mut epubs: Vec<PathBuf> = Vec::new();
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--cache" => cache = PathBuf::from(args.next().expect("--cache needs a path")),
"--dump" => dump = Some(args.next().expect("--dump needs a title substring")),
+ "--epub-out" => {
+ epub_out = Some(PathBuf::from(args.next().expect("--epub-out needs a path")))
+ }
other => epubs.push(PathBuf::from(other)),
}
}
if epubs.is_empty() {
- eprintln!("usage: image_audit [--cache DIR] <issue.epub>...");
+ eprintln!(
+ "usage: image_audit [--cache DIR] [--dump TITLE] [--epub-out DIR] <issue.epub>..."
+ );
std::process::exit(2);
}
std::fs::create_dir_all(&cache).expect("cache dir");
@@ -212,6 +223,10 @@ async fn main() {
for (kind, n) in &kinds {
println!(" {n:>4} {kind}");
}
+ if let Some(dir) = &epub_out {
+ write_audit_epub(dir, &picks, &assets);
+ }
+
if !skipped.is_empty() {
println!("\n== unfetchable ({})", skipped.len());
for (t, e) in &skipped {
@@ -286,7 +301,7 @@ async fn body_for(
let _ = extractor;
let readable = extract::readability(&html, &base).map_err(|e| e.to_string())?;
Ok(extract::sanitize_with_base(
- &extract::normalize_img_tags(&readable),
+ &images::normalize_img_tags(&readable),
&base,
))
}
@@ -320,8 +335,8 @@ async fn raw_fetch(url: &str) -> Result<(Vec<u8>, String), String> {
}
fn pick_for(target: &Target, content_html: String) -> Pick {
- let word_count = extract::word_count(&content_html);
- let image_urls = extract::collect_image_urls(&content_html, &target.url);
+ let word_count = html::word_count(&content_html);
+ let image_urls = images::collect_image_urls(&content_html, &target.url);
Pick {
article: Article {
id: target.entry_id,
@@ -336,7 +351,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
sources: vec![SourceRef {
entry_id: target.entry_id,
feed_id: 1,
- feed_title: "Feed".into(),
+ feed_title: target.issue.clone(),
category: None,
kind: SourceKind::Feed,
}],
@@ -344,14 +359,14 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
url: target.url.clone(),
author: None,
feed_id: 1,
- feed_title: "Feed".into(),
+ feed_title: host(&target.url),
category: None,
published_at: None,
comments_url: None,
social: vec![],
extract_method: ExtractMethod::Readability,
},
- section: "Audit".into(),
+ section: target.issue.clone(),
position: 0,
is_lead: false,
summary: None,
@@ -360,6 +375,67 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
}
}
+/// Build a readable EPUB out of the audited articles (`--epub-out DIR`).
+///
+/// Not a regenerated issue — there is no editorial, no discussion chapters and
+/// no world briefing here, and the real thing needs the database. It exists so
+/// the images can be looked at on a device rather than counted in a table.
+fn write_audit_epub(out_dir: &Path, picks: &[Pick], assets: &[ImageAsset]) {
+ let sections: Vec<String> = {
+ let mut seen: Vec<String> = Vec::new();
+ for p in picks {
+ if !seen.contains(&p.section) {
+ seen.push(p.section.clone());
+ }
+ }
+ seen
+ };
+ let issue = Issue {
+ meta: IssueMeta {
+ date: "2026-08-16".parse().unwrap(),
+ issue_number: 0,
+ generated_at: jiff::Timestamp::now(),
+ display_date: "Image audit rebuild".into(),
+ article_count: picks.len() as i64,
+ section_count: sections.len() as i64,
+ total_words: picks.iter().map(|p| p.article.word_count).sum(),
+ reading_minutes: picks.iter().map(|p| p.article.reading_minutes()).sum(),
+ },
+ lineup: Lineup {
+ date: "2026-08-16".parse().unwrap(),
+ picks: picks.to_vec(),
+ section_order: sections,
+ },
+ editorial: Editorial {
+ front_page_html: "<p>Rebuilt from published issues by \
+ <code>examples/image_audit.rs</code> to check image handling. \
+ Articles are re-extracted live; editorial, discussions and the \
+ world briefing are absent by design.</p>"
+ .into(),
+ section_intros: Default::default(),
+ summaries: Default::default(),
+ },
+ world_briefing: None,
+ colophon: Colophon::default(),
+ };
+ let cfg = daily_epub::config::Config::default();
+ match daily_epub::epub::build_edition_with_images(
+ &issue,
+ daily_epub::types::Edition::Standard,
+ &cfg,
+ out_dir,
+ assets,
+ ) {
+ Ok(artifact) => println!(
+ "\nwrote {} ({:.1} MB, {} images)",
+ artifact.path.display(),
+ artifact.bytes as f64 / 1_048_576.0,
+ assets.len()
+ ),
+ Err(e) => eprintln!("epub build failed: {e}"),
+ }
+}
+
/// Pull `(entry id, title, url)` out of every article chapter in an EPUB.
fn targets_from_epub(path: &Path) -> Vec<Target> {
let issue = path
diff --git a/src/comments.rs b/src/comments.rs
index 8093e9e..d66a663 100644
--- a/src/comments.rs
+++ b/src/comments.rs
@@ -9,7 +9,7 @@ use std::collections::HashSet;
use futures::StreamExt;
use serde_json::Value;
-use crate::epub::images::{text_escape, to_xhtml};
+use crate::html::{text_escape, to_xhtml};
use crate::types::{Comment, CommentThread, Discussion, Pick, SocialSource};
/// Top-level threads kept per source (§3.7).
diff --git a/src/curate/editorial.rs b/src/curate/editorial.rs
index 1109806..35ce406 100644
--- a/src/curate/editorial.rs
+++ b/src/curate/editorial.rs
@@ -14,7 +14,7 @@ use std::fmt::Write as _;
use serde::{Deserialize, Serialize};
use super::llm::{LlmClient, LlmError};
-use super::{escape_html, html_to_text, text_to_paragraphs, truncate_tokens, truncate_words};
+use super::{escape_html, prompt_text, text_to_paragraphs, truncate_tokens, truncate_words};
use crate::types::{ArticleId, Editorial, Lineup, Pick};
/// Article text is truncated to roughly this many tokens per summary call (§3.6).
@@ -116,7 +116,7 @@ pub async fn summarize_article(
temperature: f32,
) -> Result<String, LlmError> {
llm.meter.check_budget()?;
- let body = truncate_tokens(&html_to_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET);
+ let body = truncate_tokens(&prompt_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET);
let mut prompt = String::with_capacity(body.len() + SUMMARY_INSTRUCTIONS.len() + 256);
prompt.push_str(SUMMARY_INSTRUCTIONS);
let _ = write!(
@@ -309,7 +309,7 @@ fn social_note(pick: &Pick) -> String {
/// The article's own opening words, used when no LLM summary exists (§3.6).
pub fn excerpt_summary(pick: &Pick) -> String {
let text = truncate_words(
- &html_to_text(&pick.article.content_html),
+ &prompt_text(&pick.article.content_html),
FALLBACK_SUMMARY_WORDS,
);
if text.is_empty() {
diff --git a/src/curate/mod.rs b/src/curate/mod.rs
index 43ca2ac..93c4130 100644
--- a/src/curate/mod.rs
+++ b/src/curate/mod.rs
@@ -164,7 +164,11 @@ pub fn approx_tokens(text: &str) -> usize {
/// Strip markup and collapse whitespace, so article bodies can go into prompts
/// as plain text (cheaper and less confusing for the model than raw HTML).
-pub fn html_to_text(html: &str) -> String {
+///
+/// Deliberately not [`crate::html::html_to_text`]: this one collapses runs of
+/// whitespace and never builds a DOM, because it runs over every candidate
+/// body on every run and only has to be good enough to size a prompt.
+pub fn prompt_text(html: &str) -> String {
/// Does `tail` open the named element, i.e. `<name` or `</name`?
fn opens(tail: &str, name: &str) -> bool {
let bytes = tail.as_bytes();
@@ -272,11 +276,11 @@ mod tests {
fn html_becomes_readable_text() {
let html = "<h1>Title</h1><p>First & best.</p><script>alert('x')</script>\
<p>Second<br/>line</p><style>p{color:red}</style>";
- assert_eq!(html_to_text(html), "Title First & best. Second line");
- assert_eq!(html_to_text(""), "");
- assert_eq!(html_to_text("no markup at all"), "no markup at all");
+ assert_eq!(prompt_text(html), "Title First & best. Second line");
+ assert_eq!(prompt_text(""), "");
+ assert_eq!(prompt_text("no markup at all"), "no markup at all");
// Unicode survives byte-wise walking.
- assert_eq!(html_to_text("<p>café — naïve</p>"), "café — naïve");
+ assert_eq!(prompt_text("<p>café — naïve</p>"), "café — naïve");
}
#[test]
diff --git a/src/curate/profile.rs b/src/curate/profile/mod.rs
similarity index 79%
rename from src/curate/profile.rs
rename to src/curate/profile/mod.rs
index 162b6f9..a7491b1 100644
--- a/src/curate/profile.rs
+++ b/src/curate/profile/mod.rs
@@ -99,303 +99,9 @@ fn xml_unescape(s: &str) -> String {
.replace("&", "&")
}
-// ---------------------------------------------------------------------------
-// Theming (§3.6a)
-// ---------------------------------------------------------------------------
+pub mod themes;
-/// Theme name → lowercase keywords, in priority order. The first theme whose
-/// keyword appears in the interest name wins, so the table is ordered from the
-/// most specific bucket to the most general.
-const THEMES: &[(&str, &[&str])] = &[
- (
- "Systems & languages",
- &[
- "rust",
- "zig",
- "lua",
- "assembly",
- "compiler",
- "systems programming",
- "concurrency",
- "async",
- "memory",
- "simd",
- "zero-copy",
- "parser",
- "fuzz",
- "static analysis",
- "functional programming",
- "data structures",
- "algorithm",
- "performance",
- "profil",
- "python",
- "typescript",
- "node.js",
- "wasm",
- "webassembly",
- "reactive programming",
- "lsp",
- ],
- ),
- (
- "Databases & data",
- &[
- "database",
- "sql",
- "postgres",
- "query",
- "vector databases",
- "bloom",
- "compression",
- "data engineering",
- "knowledge graph",
- "message queue",
- "distributed systems",
- "crdt",
- "microservices",
- "system design",
- ],
- ),
- (
- "AI & machine learning",
- &[
- "ai",
- "llm",
- "machine learning",
- "nlp",
- "rag",
- "prompt",
- "agent",
- "claude",
- "eval",
- ],
- ),
- (
- "Web, infra & devtools",
- &[
- "web",
- "css",
- "htmx",
- "svelte",
- "react",
- "pwa",
- "api",
- "developer",
- "devops",
- "docker",
- "git",
- "observability",
- "cloud",
- "cdn",
- "edge",
- "networking",
- "mesh",
- "ipfs",
- "activitypub",
- "atproto",
- "search engines",
- "tauri",
- "design systems",
- "cybersecurity",
- "cryptography",
- "monitoring",
- "spatial computing",
- "webrtc",
- "webgpu",
- "webgl",
- "webcodecs",
- "android",
- "mobile",
- "code generation",
- ],
- ),
- (
- "Self-hosting, RSS & the indie web",
- &[
- "self-host",
- "self host",
- "homelab",
- "rss",
- "feed reader",
- "indie web",
- "personal website",
- "personal wiki",
- "digital garden",
- "static site",
- "static sites",
- "personal archiving",
- "offline-first",
- "privacy",
- "open source",
- "licensing",
- "side projects",
- "digital nomad",
- "remote living",
- "off-grid",
- "quantified self",
- "cloudflare workers",
- ],
- ),
- (
- "E-ink, terminals & hardware",
- &[
- "e-ink",
- "eink",
- "writerdeck",
- "tmux",
- "neovim",
- "terminal",
- "text editor",
- "editor",
- "window manager",
- "keyboard",
- "hardware",
- "electronics",
- "calibre",
- "ghostty",
- "retro computing",
- "low-tech",
- "manufacturing",
- "typography",
- "linux",
- ],
- ),
- (
- "Games & interactive fiction",
- &[
- "game",
- "gaming",
- "minecraft",
- "mud",
- "interactive fiction",
- "inform7",
- "bevy",
- "sim racing",
- "modding",
- ],
- ),
- (
- "Creative coding & digital art",
- &[
- "generative",
- "creative coding",
- "creative automation",
- "glitch",
- "procedural",
- "digital art",
- "gaussian splatting",
- "cellular automata",
- "sonification",
- "code visualization",
- "photo",
- "photography",
- "agent-based",
- ],
- ),
- (
- "Writing, books & PKM",
- &[
- "writing",
- "write",
- "reading",
- "book",
- "essay",
- "poetry",
- "literature",
- "note-taking",
- "journaling",
- "pkm",
- "knowledge management",
- "obsidian",
- "markdown",
- "commonplace",
- "longform",
- "long-form",
- "fiction",
- "worldbuilding",
- "sci-fi",
- "science fiction",
- "speculative",
- "podcast",
- "narrative",
- "choice architecture",
- ],
- ),
- (
- "Science, space & nature",
- &[
- "space",
- "aerospace",
- "aviation",
- "science",
- "neuroscience",
- "bioinformatics",
- "nature",
- "cognitive",
- "social science",
- ],
- ),
- (
- "History, policy & culture",
- &[
- "history",
- "policy",
- "culture",
- "lgbt",
- "queer",
- "startups",
- "apple",
- "boston",
- "criticism",
- "internet history",
- "computing history",
- ],
- ),
- (
- "Outdoors, coffee & everyday life",
- &[
- "hiking",
- "camping",
- "running",
- "trail",
- "coffee",
- "cats",
- "films",
- "music",
- "engineering",
- ],
- ),
-];
-
-/// Fallback bucket for interests no keyword claims.
-const OTHER_THEME: &str = "Other standing interests";
-
-/// Group ~220 interests into readable themes for the prompt (§3.6).
-///
-/// Deterministic: theme order follows [`THEMES`], members are sorted
-/// case-insensitively, and empty themes are omitted.
-pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> {
- let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1];
- for interest in interests {
- let lower = interest.to_lowercase();
- let idx = THEMES
- .iter()
- .position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k)))
- .unwrap_or(THEMES.len());
- buckets[idx].push(interest.clone());
- }
- let mut out = Vec::new();
- for (idx, mut members) in buckets.into_iter().enumerate() {
- if members.is_empty() {
- continue;
- }
- members.sort_by_key(|m| (m.to_lowercase(), m.clone()));
- let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME);
- out.push((name.to_string(), members));
- }
- out
-}
+pub use themes::group_into_themes;
// ---------------------------------------------------------------------------
// Document assembly (§3.6)
@@ -870,27 +576,6 @@ mod tests {
);
}
- #[test]
- fn theming_is_deterministic_and_total() {
- let list = interests();
- let a = group_into_themes(&list);
- let b = group_into_themes(&list);
- assert_eq!(a, b);
- let grouped: usize = a.iter().map(|(_, m)| m.len()).sum();
- assert_eq!(
- grouped,
- list.len(),
- "every interest lands in exactly one theme"
- );
- assert!(a.len() >= 6, "expected several themes, got {}", a.len());
- // Members are sorted within a theme.
- for (_, members) in &a {
- let mut sorted = members.clone();
- sorted.sort_by_key(|m| (m.to_lowercase(), m.clone()));
- assert_eq!(members, &sorted);
- }
- }
-
#[test]
fn profile_document_is_deterministic_and_complete() {
let list = interests();
diff --git a/src/curate/profile/themes.rs b/src/curate/profile/themes.rs
new file mode 100644
index 0000000..f2d5483
--- /dev/null
+++ b/src/curate/profile/themes.rs
@@ -0,0 +1,354 @@
+//! Grouping the reader's ~220 Scour interests into readable themes (§3.6a).
+//!
+//! A lookup table and one function over it. It lives in its own file because the
+//! table is long and almost never changes, while the profile logic around it
+//! does — and a 260-line const in the middle of that logic is a wall to scroll
+//! past, not something to read.
+
+/// Theme name → lowercase keywords, in priority order. The first theme whose
+/// keyword appears in the interest name wins, so the table is ordered from the
+/// most specific bucket to the most general.
+const THEMES: &[(&str, &[&str])] = &[
+ (
+ "Systems & languages",
+ &[
+ "rust",
+ "zig",
+ "lua",
+ "assembly",
+ "compiler",
+ "systems programming",
+ "concurrency",
+ "async",
+ "memory",
+ "simd",
+ "zero-copy",
+ "parser",
+ "fuzz",
+ "static analysis",
+ "functional programming",
+ "data structures",
+ "algorithm",
+ "performance",
+ "profil",
+ "python",
+ "typescript",
+ "node.js",
+ "wasm",
+ "webassembly",
+ "reactive programming",
+ "lsp",
+ ],
+ ),
+ (
+ "Databases & data",
+ &[
+ "database",
+ "sql",
+ "postgres",
+ "query",
+ "vector databases",
+ "bloom",
+ "compression",
+ "data engineering",
+ "knowledge graph",
+ "message queue",
+ "distributed systems",
+ "crdt",
+ "microservices",
+ "system design",
+ ],
+ ),
+ (
+ "AI & machine learning",
+ &[
+ "ai",
+ "llm",
+ "machine learning",
+ "nlp",
+ "rag",
+ "prompt",
+ "agent",
+ "claude",
+ "eval",
+ ],
+ ),
+ (
+ "Web, infra & devtools",
+ &[
+ "web",
+ "css",
+ "htmx",
+ "svelte",
+ "react",
+ "pwa",
+ "api",
+ "developer",
+ "devops",
+ "docker",
+ "git",
+ "observability",
+ "cloud",
+ "cdn",
+ "edge",
+ "networking",
+ "mesh",
+ "ipfs",
+ "activitypub",
+ "atproto",
+ "search engines",
+ "tauri",
+ "design systems",
+ "cybersecurity",
+ "cryptography",
+ "monitoring",
+ "spatial computing",
+ "webrtc",
+ "webgpu",
+ "webgl",
+ "webcodecs",
+ "android",
+ "mobile",
+ "code generation",
+ ],
+ ),
+ (
+ "Self-hosting, RSS & the indie web",
+ &[
+ "self-host",
+ "self host",
+ "homelab",
+ "rss",
+ "feed reader",
+ "indie web",
+ "personal website",
+ "personal wiki",
+ "digital garden",
+ "static site",
+ "static sites",
+ "personal archiving",
+ "offline-first",
+ "privacy",
+ "open source",
+ "licensing",
+ "side projects",
+ "digital nomad",
+ "remote living",
+ "off-grid",
+ "quantified self",
+ "cloudflare workers",
+ ],
+ ),
+ (
+ "E-ink, terminals & hardware",
+ &[
+ "e-ink",
+ "eink",
+ "writerdeck",
+ "tmux",
+ "neovim",
+ "terminal",
+ "text editor",
+ "editor",
+ "window manager",
+ "keyboard",
+ "hardware",
+ "electronics",
+ "calibre",
+ "ghostty",
+ "retro computing",
+ "low-tech",
+ "manufacturing",
+ "typography",
+ "linux",
+ ],
+ ),
+ (
+ "Games & interactive fiction",
+ &[
+ "game",
+ "gaming",
+ "minecraft",
+ "mud",
+ "interactive fiction",
+ "inform7",
+ "bevy",
+ "sim racing",
+ "modding",
+ ],
+ ),
+ (
+ "Creative coding & digital art",
+ &[
+ "generative",
+ "creative coding",
+ "creative automation",
+ "glitch",
+ "procedural",
+ "digital art",
+ "gaussian splatting",
+ "cellular automata",
+ "sonification",
+ "code visualization",
+ "photo",
+ "photography",
+ "agent-based",
+ ],
+ ),
+ (
+ "Writing, books & PKM",
+ &[
+ "writing",
+ "write",
+ "reading",
+ "book",
+ "essay",
+ "poetry",
+ "literature",
+ "note-taking",
+ "journaling",
+ "pkm",
+ "knowledge management",
+ "obsidian",
+ "markdown",
+ "commonplace",
+ "longform",
+ "long-form",
+ "fiction",
+ "worldbuilding",
+ "sci-fi",
+ "science fiction",
+ "speculative",
+ "podcast",
+ "narrative",
+ "choice architecture",
+ ],
+ ),
+ (
+ "Science, space & nature",
+ &[
+ "space",
+ "aerospace",
+ "aviation",
+ "science",
+ "neuroscience",
+ "bioinformatics",
+ "nature",
+ "cognitive",
+ "social science",
+ ],
+ ),
+ (
+ "History, policy & culture",
+ &[
+ "history",
+ "policy",
+ "culture",
+ "lgbt",
+ "queer",
+ "startups",
+ "apple",
+ "boston",
+ "criticism",
+ "internet history",
+ "computing history",
+ ],
+ ),
+ (
+ "Outdoors, coffee & everyday life",
+ &[
+ "hiking",
+ "camping",
+ "running",
+ "trail",
+ "coffee",
+ "cats",
+ "films",
+ "music",
+ "engineering",
+ ],
+ ),
+];
+
+/// Fallback bucket for interests no keyword claims.
+const OTHER_THEME: &str = "Other standing interests";
+
+/// Group ~220 interests into readable themes for the prompt (§3.6).
+///
+/// Deterministic: theme order follows [`THEMES`], members are sorted
+/// case-insensitively, and empty themes are omitted.
+pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> {
+ let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1];
+ for interest in interests {
+ let lower = interest.to_lowercase();
+ let idx = THEMES
+ .iter()
+ .position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k)))
+ .unwrap_or(THEMES.len());
+ buckets[idx].push(interest.clone());
+ }
+ let mut out = Vec::new();
+ for (idx, mut members) in buckets.into_iter().enumerate() {
+ if members.is_empty() {
+ continue;
+ }
+ members.sort_by_key(|m| (m.to_lowercase(), m.clone()));
+ let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME);
+ out.push((name.to_string(), members));
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn interests() -> Vec<String> {
+ [
+ "Rust",
+ "Zig",
+ "Postgres",
+ "SQLite",
+ "Kubernetes",
+ "E-ink",
+ "Keyboards",
+ "Retrocomputing",
+ "Bicycles",
+ "Trail running",
+ "Coffee",
+ "Bread baking",
+ "Cartography",
+ "Typography",
+ "Ambient music",
+ "Science fiction",
+ "Local politics",
+ "Boston",
+ "Machine learning",
+ "Self-hosting",
+ ]
+ .iter()
+ .map(|s| s.to_string())
+ .collect()
+ }
+
+ #[test]
+ fn theming_is_deterministic_and_total() {
+ let list = interests();
+ let a = group_into_themes(&list);
+ let b = group_into_themes(&list);
+ assert_eq!(a, b);
+ let grouped: usize = a.iter().map(|(_, m)| m.len()).sum();
+ assert_eq!(
+ grouped,
+ list.len(),
+ "every interest lands in exactly one theme"
+ );
+ assert!(a.len() >= 6, "expected several themes, got {}", a.len());
+ // Members are sorted within a theme.
+ for (_, members) in &a {
+ let mut sorted = members.clone();
+ sorted.sort_by_key(|m| (m.to_lowercase(), m.clone()));
+ assert_eq!(members, &sorted);
+ }
+ }
+}
diff --git a/src/curate/score.rs b/src/curate/score.rs
index 9636ce7..49be464 100644
--- a/src/curate/score.rs
+++ b/src/curate/score.rs
@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::llm::{LlmClient, LlmError, strip_code_fence};
-use super::{html_to_text, truncate_words};
+use super::{prompt_text, truncate_words};
use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind};
/// Words of article text sent per candidate in stage A (§3.6).
@@ -155,7 +155,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String {
);
let _ = writeln!(block, "social: {}", social_line(candidate));
let _ = writeln!(block, "came via: {}", sources_line(candidate));
- let excerpt = truncate_words(&html_to_text(&a.content_html), EXCERPT_WORDS);
+ let excerpt = truncate_words(&prompt_text(&a.content_html), EXCERPT_WORDS);
let _ = writeln!(
block,
"excerpt: {}",
diff --git a/src/curate/select.rs b/src/curate/select.rs
index 4ab7dda..75d3db3 100644
--- a/src/curate/select.rs
+++ b/src/curate/select.rs
@@ -16,7 +16,7 @@ use jiff::civil::Date;
use serde::{Deserialize, Serialize};
use super::llm::{LlmClient, LlmError, strip_code_fence};
-use super::{html_to_text, truncate_words};
+use super::{prompt_text, truncate_words};
use crate::types::{ArticleId, Lineup, Pick, ScoredArticle, SourceKind, WORLD_BRIEFING_SECTION};
/// How many candidates are offered to stage B (§3.6).
@@ -154,7 +154,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String {
""
}
);
- let blurb = truncate_words(&html_to_text(&a.content_html), BLURB_WORDS);
+ let blurb = truncate_words(&prompt_text(&a.content_html), BLURB_WORDS);
if !blurb.is_empty() {
let _ = writeln!(block, "opening: {blurb}");
}
diff --git a/src/dedupe.rs b/src/dedupe.rs
index 77fd486..8517708 100644
--- a/src/dedupe.rs
+++ b/src/dedupe.rs
@@ -162,7 +162,7 @@ fn is_enclosure_only(raw_content: &str) -> bool {
|| lower.contains("<video")
|| lower.contains("<embed")
|| lower.contains("<iframe");
- embeds && crate::extract::word_count(raw_content) < 25
+ embeds && crate::html::word_count(raw_content) < 25
}
/// Classify which kind of feed an entry arrived through, for the sources list (§3.2, §3.5).
@@ -345,7 +345,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
let best = members
.iter()
.enumerate()
- .max_by_key(|(_, (entry, _))| (crate::extract::word_count(&entry.raw_content), -(entry.id)))
+ .max_by_key(|(_, (entry, _))| (crate::html::word_count(&entry.raw_content), -(entry.id)))
.map(|(i, _)| i)
.unwrap_or(0);
let (best_entry, canonical) = &members[best];
@@ -385,7 +385,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
.filter_map(|(e, _)| e.author.clone())
.find(|a| !a.trim().is_empty());
- let word_count = crate::extract::word_count(&best_entry.raw_content);
+ let word_count = crate::html::word_count(&best_entry.raw_content);
Article {
id: 0,
diff --git a/src/epub/build.rs b/src/epub/build.rs
index 2bb1e82..8e4575d 100644
--- a/src/epub/build.rs
+++ b/src/epub/build.rs
@@ -1,22 +1,34 @@
-//! `epub-builder` assembly and askama chapter rendering (spec §3.10).
+//! Chapter ordering and `epub-builder` assembly (spec §3.10).
//!
//! Structure: cover → From the Editor → In This Issue → sections (title page,
-//! article chapters, discussion chapters) → World Briefing → colophon.
+//! article chapters, discussion chapters) → World Briefing → colophon. The
+//! chapters themselves are rendered by [`super::chapters`] and the cover by
+//! [`super::cover`]; this module decides the order and zips the result.
-use askama::Template;
use epub_builder::{
EpubBuilder, EpubContent, EpubVersion, MetadataOpfV3, ReferenceType, ZipLibrary,
};
use jiff::civil::Date;
-use crate::comments;
-use crate::types::{Edition, ImageAsset, Issue, Pick, SocialRef, Vote, WORLD_BRIEFING_SECTION};
-use crate::world;
+use crate::types::{Edition, ImageAsset, Issue};
use super::EpubError;
-use super::images;
+use super::chapters::{
+ render_colophon, render_front_page, render_in_this_issue, render_section_page,
+ render_world_briefing, section_names,
+};
+use super::cover::{CoverAsset, render_cover_page};
use super::x4;
+// Re-exported so callers keep one import path for "everything about building an
+// issue"; the definitions live in the modules that own them.
+pub use super::chapters::{
+ TOKEN_LEN, prepare_body, rating_message, rating_token, rating_url, render_article,
+ render_discussion, social_line,
+};
+pub use super::cover::{cover_badge, cover_size, render_cover};
+pub use super::fixtures;
+
/// EPUB3 `belongs-to-collection` name (§3.10).
pub const COLLECTION_NAME: &str = "The Daily EPUB";
/// `id` the collection refinements point at (§3.10).
@@ -25,21 +37,6 @@ pub const COLLECTION_ID: &str = "daily-epub-collection";
pub const CREATOR: &str = "The Daily EPUB";
/// `dc:language` (§3.10).
pub const LANGUAGE: &str = "en";
-/// Characters of the hex HMAC kept in rating links (§3.9).
-pub const TOKEN_LEN: usize = crate::auth::TOKEN_LEN;
-#[derive(Debug, Clone, PartialEq)]
-pub struct CoverAsset {
- pub bytes: Vec<u8>,
- pub filename: &'static str,
- pub mime: &'static str,
-}
-
-pub fn cover_href(edition: Edition) -> &'static str {
- match edition {
- Edition::Standard => "cover.png",
- Edition::X4 => "cover.jpg",
- }
-}
/// One rendered chapter ready to be added to the EPUB (§3.10).
#[derive(Debug, Clone, PartialEq)]
@@ -54,742 +51,7 @@ pub struct Chapter {
pub toc_level: u8,
}
-// ---------------------------------------------------------------------------
-// Rating links (§3.9)
-// ---------------------------------------------------------------------------
-
-// One implementation, shared with the verifying side in [`crate::server`]:
-// see [`crate::auth`] for the formula and the pinned test vector (§3.9).
-pub use crate::auth::{rating_message, rating_token, rating_url};
-
-// ---------------------------------------------------------------------------
-// Templates
-// ---------------------------------------------------------------------------
-
-#[derive(Template)]
-#[template(path = "cover.svg", escape = "html")]
-struct CoverSvg {
- width: i64,
- height: i64,
- margin: i64,
- inner_width: i64,
- inner_height: i64,
- border: i64,
- hairline: i64,
- center_x: i64,
- masthead_y: i64,
- masthead_size: i64,
- rule_x1: i64,
- rule_x2: i64,
- rule_y: i64,
- rule2_y: i64,
- weekday: String,
- weekday_y: i64,
- weekday_size: i64,
- long_date: String,
- date_y: i64,
- date_size: i64,
- issue_number: i64,
- issue_y: i64,
- issue_size: i64,
- stats_line: String,
- stats_y: i64,
- stats_size: i64,
- /// Empty for the standard edition — see [`cover_badge`].
- edition_tag: String,
- badge_x: i64,
- badge_y: i64,
- badge_width: i64,
- badge_height: i64,
- badge_radius: i64,
- badge_text_y: i64,
- badge_size: i64,
- footer: String,
- footer_y: i64,
- footer_size: i64,
-}
-
-#[derive(Template)]
-#[template(path = "cover_page.xhtml", escape = "html")]
-struct CoverPage {
- title: String,
- alt: String,
- cover_href: &'static str,
-}
-
-#[derive(Template)]
-#[template(path = "front_page.xhtml", escape = "html")]
-struct FrontPage {
- title: String,
- display_date: String,
- issue_number: i64,
- stats_line: String,
- body_html: String,
-}
-
-struct IndexEntry {
- href: String,
- title: String,
- source: String,
- reading_minutes: i64,
- summary: String,
-}
-
-struct IndexSection {
- name: String,
- entries: Vec<IndexEntry>,
-}
-
-#[derive(Template)]
-#[template(path = "in_this_issue.xhtml", escape = "html")]
-struct InThisIssue {
- title: String,
- stats_line: String,
- sections: Vec<IndexSection>,
-}
-
-#[derive(Template)]
-#[template(path = "section.xhtml", escape = "html")]
-struct SectionPage {
- title: String,
- name: String,
- intro: Option<String>,
-}
-
-struct RatingLinks {
- up_url: String,
- down_url: String,
-}
-
-#[derive(Template)]
-#[template(path = "chapter.xhtml", escape = "html")]
-struct ArticleChapter {
- title: String,
- article_title: String,
- byline: Option<String>,
- meta_line: String,
- social_line: Option<String>,
- summary: Option<String>,
- excerpt_only: bool,
- body_html: String,
- rating: Option<RatingLinks>,
- read_online_url: String,
- discussion_href: Option<String>,
-}
-
-#[derive(Template)]
-#[template(path = "discussion.xhtml", escape = "html")]
-struct DiscussionChapter {
- title: String,
- heading: String,
- body_html: String,
- article_href: String,
-}
-
-#[derive(Template)]
-#[template(path = "world_briefing.xhtml", escape = "html")]
-struct WorldBriefingChapter {
- title: String,
- display_date: String,
- body_html: String,
-}
-
-#[derive(Template)]
-#[template(path = "colophon.xhtml", escape = "html")]
-struct ColophonChapter {
- title: String,
- issue_number: i64,
- display_date: String,
- generated_at: String,
- model: String,
- entries_fetched: i64,
- feeds_seen: i64,
- candidates: i64,
- article_count: i64,
- section_count: i64,
- total_words: i64,
- reading_line: String,
- cost_usd: String,
- generator_version: String,
-}
-
-// ---------------------------------------------------------------------------
-// Cover (§3.10)
-// ---------------------------------------------------------------------------
-
-/// Cover pixel size per edition: 1200×1600 standard, 480×800 X4 (§3.10).
-pub fn cover_size(edition: Edition) -> (u32, u32) {
- match edition {
- Edition::Standard => (1200, 1600),
- Edition::X4 => x4::X4_SCREEN,
- }
-}
-
-fn reading_line(minutes: i64) -> String {
- let (h, m) = (minutes / 60, minutes % 60);
- if h > 0 {
- format!("~{h}h {m}m read")
- } else {
- format!("~{m}m read")
- }
-}
-
-/// Split "Friday, August 15, 2026" into ("Friday", "August 15, 2026").
-fn split_display_date(display: &str) -> (String, String) {
- match display.split_once(", ") {
- Some((weekday, rest)) => (weekday.to_string(), rest.to_string()),
- None => (String::new(), display.to_string()),
- }
-}
-
-/// Reversed badge printed on the cover so the two editions are told apart at
-/// thumbnail size, where the title is unreadable (§3.10).
-///
-/// Empty for the standard edition: an unmarked cover is the default, and the
-/// presence of the slab is itself the signal.
-pub fn cover_badge(edition: Edition) -> &'static str {
- match edition {
- Edition::Standard => "",
- Edition::X4 => "X4 EDITION",
- }
-}
-
-fn cover_svg(
- issue: &Issue,
- edition: Edition,
- width: u32,
- height: u32,
-) -> Result<String, EpubError> {
- let w = i64::from(width);
- let h = i64::from(height);
- let margin = w / 15;
- let (weekday, long_date) = split_display_date(&issue.meta.display_date);
- // A solid bar between the stats line and the footer: at 100px wide in a
- // library grid the black slab is the only thing still legible.
- let badge_height = h / 16;
- let badge_width = w * 2 / 5;
- let tpl = CoverSvg {
- width: w,
- height: h,
- margin,
- inner_width: w - 2 * margin,
- inner_height: h - 2 * margin,
- border: (w / 300).max(2),
- hairline: (w / 600).max(1),
- center_x: w / 2,
- masthead_y: h * 30 / 100,
- masthead_size: w / 9,
- rule_x1: margin + w / 12,
- rule_x2: w - margin - w / 12,
- rule_y: h * 34 / 100,
- rule2_y: h * 53 / 100,
- weekday,
- weekday_y: h * 42 / 100,
- weekday_size: w / 28,
- long_date,
- date_y: h * 47 / 100,
- date_size: w / 22,
- issue_number: issue.meta.issue_number,
- issue_y: h * 60 / 100,
- issue_size: w / 18,
- stats_line: issue.meta.stats_line(),
- stats_y: h * 66 / 100,
- stats_size: w / 32,
- edition_tag: cover_badge(edition).to_string(),
- badge_x: (w - badge_width) / 2,
- badge_y: h * 73 / 100,
- badge_width,
- badge_height,
- badge_radius: badge_height / 2,
- badge_text_y: h * 73 / 100 + badge_height * 7 / 10,
- badge_size: badge_height * 11 / 20,
- footer: "Assembled overnight \u{00b7} read offline".to_string(),
- footer_y: h - margin - h / 25,
- footer_size: w / 40,
- };
- Ok(tpl.render()?)
-}
-
-/// Render the standard cover as RGB PNG and the X4 cover as baseline RGB JPEG.
-pub fn render_cover(issue: &Issue, edition: Edition) -> Result<CoverAsset, EpubError> {
- let (width, height) = cover_size(edition);
- let svg = cover_svg(issue, edition, width, height)?;
- let pixmap = match rasterize(&svg, width, height) {
- Some(pixmap) => pixmap,
- None => {
- tracing::warn!("no usable system fonts: falling back to a geometric cover");
- draw_fallback_cover(width, height, edition)
- .ok_or_else(|| EpubError::Build("could not allocate the cover".into()))?
- }
- };
- Ok(CoverAsset {
- bytes: encode_cover(pixmap, edition)?,
- filename: cover_href(edition),
- mime: if edition == Edition::X4 {
- "image/jpeg"
- } else {
- "image/png"
- },
- })
-}
-
-fn rasterize(svg: &str, width: u32, height: u32) -> Option<tiny_skia::Pixmap> {
- let mut options = resvg::usvg::Options::default();
- options.fontdb_mut().load_system_fonts();
- if options.fontdb.is_empty() {
- return None;
- }
- let tree = resvg::usvg::Tree::from_str(svg, &options)
- .map_err(|e| tracing::warn!("cover svg did not parse: {e}"))
- .ok()?;
- let mut pixmap = tiny_skia::Pixmap::new(width, height)?;
- pixmap.fill(tiny_skia::Color::WHITE);
- let size = tree.size();
- let transform = tiny_skia::Transform::from_scale(
- width as f32 / size.width(),
- height as f32 / size.height(),
- );
- resvg::render(&tree, transform, &mut pixmap.as_mut());
- Some(pixmap)
-}
-
-/// Text-free cover used when font resolution fails — the build never fails (§3.10).
-///
-/// The badge cannot carry its lettering here, but the slab itself still keeps
-/// the editions apart in a thumbnail grid.
-fn draw_fallback_cover(width: u32, height: u32, edition: Edition) -> Option<tiny_skia::Pixmap> {
- let mut pixmap = tiny_skia::Pixmap::new(width, height)?;
- pixmap.fill(tiny_skia::Color::WHITE);
- let mut paint = tiny_skia::Paint::default();
- paint.set_color(tiny_skia::Color::BLACK);
- paint.anti_alias = true;
-
- let w = width as f32;
- let h = height as f32;
- let margin = w / 15.0;
- let rect = |x: f32, y: f32, rw: f32, rh: f32, pixmap: &mut tiny_skia::Pixmap| {
- if let Some(r) = tiny_skia::Rect::from_xywh(x, y, rw, rh) {
- let path = tiny_skia::PathBuilder::from_rect(r);
- pixmap.fill_path(
- &path,
- &paint,
- tiny_skia::FillRule::Winding,
- tiny_skia::Transform::identity(),
- None,
- );
- }
- };
- // Frame.
- let border = (w / 300.0).max(2.0);
- rect(margin, margin, w - 2.0 * margin, border, &mut pixmap);
- rect(
- margin,
- h - margin - border,
- w - 2.0 * margin,
- border,
- &mut pixmap,
- );
- rect(margin, margin, border, h - 2.0 * margin, &mut pixmap);
- rect(
- w - margin - border,
- margin,
- border,
- h - 2.0 * margin,
- &mut pixmap,
- );
- // Masthead slab plus body rules — a newspaper silhouette.
- rect(
- margin * 2.0,
- h * 0.26,
- w - 4.0 * margin,
- h * 0.035,
- &mut pixmap,
- );
- for i in 0..8 {
- let y = h * 0.42 + (i as f32) * h * 0.045;
- let inset = if i % 3 == 2 {
- margin * 4.0
- } else {
- margin * 2.0
- };
- rect(inset, y, w - 2.0 * inset, border, &mut pixmap);
- }
- if !cover_badge(edition).is_empty() {
- let badge_height = h / 16.0;
- let badge_width = w * 2.0 / 5.0;
- rect(
- (w - badge_width) / 2.0,
- h * 0.73,
- badge_width,
- badge_height,
- &mut pixmap,
- );
- }
- Some(pixmap)
-}
-
-fn encode_cover(pixmap: tiny_skia::Pixmap, edition: Edition) -> Result<Vec<u8>, EpubError> {
- let (w, h) = (pixmap.width(), pixmap.height());
- let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())
- .ok_or_else(|| EpubError::Build("cover pixel buffer had the wrong size".into()))?;
- let rgb = image::DynamicImage::ImageRgba8(rgba).to_rgb8();
- let mut bytes = Vec::new();
- match edition {
- Edition::Standard => image::DynamicImage::ImageRgb8(rgb).write_to(
- &mut std::io::Cursor::new(&mut bytes),
- image::ImageFormat::Png,
- ),
- Edition::X4 => image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 92)
- .encode_image(&image::DynamicImage::ImageRgb8(rgb)),
- }
- .map_err(|e| EpubError::Build(format!("cover encoding failed: {e}")))?;
- Ok(bytes)
-}
-
-// ---------------------------------------------------------------------------
-// Chapters (§3.10)
-// ---------------------------------------------------------------------------
-
-/// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids.
-pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String {
- // Sanitize first: image rewriting emits our own trusted markup (including the
- // `image-placeholder` class, which ammonia would otherwise strip).
- let cleaned = ammonia::clean(html);
- let rewritten = images::rewrite_img_srcs(&cleaned, images_);
- images::to_xhtml(&rewritten)
-}
-
-/// "▲ 342 on HN · 210 comments" (§3.10).
-pub fn social_line(social: &[SocialRef]) -> Option<String> {
- let mut parts: Vec<String> = Vec::new();
- let mut comments = 0i64;
- for entry in social {
- if entry.score > 0 {
- parts.push(format!(
- "\u{25b2} {} on {}",
- entry.score,
- entry.source.display_name()
- ));
- }
- comments += entry.num_comments.max(0);
- }
- if comments > 0 {
- let noun = if comments == 1 { "comment" } else { "comments" };
- parts.push(format!("{comments} {noun}"));
- }
- if parts.is_empty() {
- None
- } else {
- Some(parts.join(" \u{00b7} "))
- }
-}
-
-fn article_href(pick: &Pick) -> String {
- format!("{}.xhtml", pick.article.chapter_id())
-}
-
-fn discussion_href(pick: &Pick) -> String {
- format!("disc-{}.xhtml", pick.article.best_entry_id)
-}
-
-fn section_href(name: &str) -> String {
- let slug: String = name
- .chars()
- .map(|c| {
- if c.is_ascii_alphanumeric() {
- c.to_ascii_lowercase()
- } else {
- '-'
- }
- })
- .collect();
- let slug = slug
- .split('-')
- .filter(|s| !s.is_empty())
- .collect::<Vec<_>>()
- .join("-");
- format!("sec-{slug}.xhtml")
-}
-
-fn summary_for<'a>(issue: &'a Issue, pick: &'a Pick) -> Option<&'a str> {
- pick.summary
- .as_deref()
- .or_else(|| {
- issue
- .editorial
- .summaries
- .get(&pick.article.id)
- .map(|s| s.as_str())
- })
- .filter(|s| !s.trim().is_empty())
-}
-
-fn published_display(pick: &Pick) -> Option<String> {
- pick.article
- .published_at
- .map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string())
-}
-
-/// "From the Editor" front page plus the issue stats line (§3.10).
-pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
- let body = issue.editorial.front_page_html.trim();
- let body_html = if body.is_empty() {
- format!(
- "<p>{} of reading, chosen overnight.</p>",
- images::text_escape(&issue.meta.stats_line())
- )
- } else {
- images::to_xhtml(&ammonia::clean(body))
- };
- let tpl = FrontPage {
- title: "From the Editor".into(),
- display_date: issue.meta.display_date.clone(),
- issue_number: issue.meta.issue_number,
- stats_line: issue.meta.stats_line(),
- body_html,
- };
- Ok(Chapter {
- id: "front".into(),
- href: "front.xhtml".into(),
- title: "From the Editor".into(),
- xhtml: tpl.render()?,
- toc_level: 1,
- })
-}
-
-/// Section names in issue order, with the reserved world section removed (§3.6).
-pub fn section_names(issue: &Issue) -> Vec<String> {
- let mut names: Vec<String> = issue
- .lineup
- .section_order
- .iter()
- .filter(|s| s.as_str() != WORLD_BRIEFING_SECTION)
- .cloned()
- .collect();
- for pick in &issue.lineup.picks {
- if pick.section != WORLD_BRIEFING_SECTION && !names.contains(&pick.section) {
- names.push(pick.section.clone());
- }
- }
- names.retain(|n| issue.lineup.picks.iter().any(|p| &p.section == n));
- names
-}
-
-/// "In This Issue": per section, each article's title, source, reading time and
-/// summary, linked to its chapter (§3.10).
-pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> {
- let mut sections = Vec::new();
- for name in section_names(issue) {
- let entries = issue
- .lineup
- .section_picks(&name)
- .into_iter()
- .map(|pick| IndexEntry {
- href: article_href(pick),
- title: pick.article.title.clone(),
- source: pick.article.feed_title.clone(),
- reading_minutes: pick.article.reading_minutes(),
- summary: summary_for(issue, pick).unwrap_or_default().to_string(),
- })
- .collect();
- sections.push(IndexSection { name, entries });
- }
- if issue.world_briefing.is_some() {
- sections.push(IndexSection {
- name: WORLD_BRIEFING_SECTION.to_string(),
- entries: vec![IndexEntry {
- href: "world.xhtml".into(),
- title: "World Briefing".into(),
- source: "Wikipedia Current Events".into(),
- reading_minutes: 3,
- summary: "The day's events, as recorded by the Current Events portal.".into(),
- }],
- });
- }
- let tpl = InThisIssue {
- title: "In This Issue".into(),
- stats_line: issue.meta.stats_line(),
- sections,
- };
- Ok(Chapter {
- id: "in-this-issue".into(),
- href: "in-this-issue.xhtml".into(),
- title: "In This Issue".into(),
- xhtml: tpl.render()?,
- toc_level: 1,
- })
-}
-
-/// A section title page: name + LLM intro (§3.10).
-pub fn render_section_page(name: &str, intro: Option<&str>) -> Result<Chapter, EpubError> {
- let tpl = SectionPage {
- title: name.to_string(),
- name: name.to_string(),
- intro: intro
- .map(|s| s.trim().to_string())
- .filter(|s| !s.is_empty()),
- };
- Ok(Chapter {
- id: format!("sec-{name}"),
- href: section_href(name),
- title: name.to_string(),
- xhtml: tpl.render()?,
- toc_level: 1,
- })
-}
-
-/// One article chapter: header, cleaned body with embedded images, rating footer (§3.10).
-pub fn render_article(
- issue: &Issue,
- pick: &Pick,
- images_: &[ImageAsset],
- edition: Edition,
- public_url: &str,
- hmac_secret: Option<&str>,
-) -> Result<Chapter, EpubError> {
- let article = &pick.article;
- let mut meta_parts = vec![article.feed_title.clone()];
- if let Some(date) = published_display(pick) {
- meta_parts.push(date);
- }
- meta_parts.push(format!(
- "{} min read \u{00b7} {} words",
- article.reading_minutes(),
- article.word_count
- ));
-
- // The X4 has no browser, so rating links are pointless there (§7).
- let rating = match (hmac_secret, edition) {
- (Some(secret), Edition::Standard) if !secret.is_empty() => Some(RatingLinks {
- up_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Up),
- down_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Down),
- }),
- _ => None,
- };
-
- let tpl = ArticleChapter {
- title: article.title.clone(),
- article_title: article.title.clone(),
- byline: article.author.as_ref().map(|a| format!("By {a}")),
- meta_line: meta_parts.join(" \u{00b7} "),
- social_line: social_line(&article.social),
- summary: summary_for(issue, pick).map(str::to_string),
- excerpt_only: article.excerpt_only,
- body_html: prepare_body(&article.content_html, images_),
- rating,
- read_online_url: article.url.clone(),
- discussion_href: pick.discussion.as_ref().map(|_| discussion_href(pick)),
- };
- Ok(Chapter {
- id: article.chapter_id(),
- href: article_href(pick),
- title: article.title.clone(),
- xhtml: tpl.render()?,
- toc_level: 2,
- })
-}
-
-/// The discussion chapter that follows an article when it has comments (§3.7).
-pub fn render_discussion(pick: &Pick) -> Result<Option<Chapter>, EpubError> {
- let Some(discussion) = &pick.discussion else {
- return Ok(None);
- };
- if discussion.threads.is_empty() {
- return Ok(None);
- }
- let title = comments::chapter_title(&pick.article.title, discussion);
- let tpl = DiscussionChapter {
- title: title.clone(),
- heading: title.clone(),
- body_html: comments::render_xhtml(discussion, &pick.article.title),
- article_href: article_href(pick),
- };
- Ok(Some(Chapter {
- id: format!("disc-{}", pick.article.best_entry_id),
- href: discussion_href(pick),
- title,
- xhtml: tpl.render()?,
- toc_level: 3,
- }))
-}
-
-/// The Wikipedia Current Events section chapter (§3.8).
-pub fn render_world_briefing(issue: &Issue) -> Result<Option<Chapter>, EpubError> {
- let Some(briefing) = &issue.world_briefing else {
- return Ok(None);
- };
- // The briefing may cover an earlier day than the issue: the portal page for
- // the issue's own date is still an empty stub at 05:30 (§3.8). Dateline the
- // section with the day it actually reports on, not the masthead date.
- let tpl = WorldBriefingChapter {
- title: WORLD_BRIEFING_SECTION.to_string(),
- display_date: crate::pipeline::display_date(briefing.date),
- body_html: world::render_xhtml(briefing),
- };
- Ok(Some(Chapter {
- id: "world".into(),
- href: "world.xhtml".into(),
- title: WORLD_BRIEFING_SECTION.to_string(),
- xhtml: tpl.render()?,
- toc_level: 1,
- }))
-}
-
-/// Colophon: generation timestamp, models used, token cost, feed counts (§3.10).
-pub fn render_colophon(issue: &Issue) -> Result<Chapter, EpubError> {
- let colophon = &issue.colophon;
- let tpl = ColophonChapter {
- title: "Colophon".into(),
- issue_number: issue.meta.issue_number,
- display_date: issue.meta.display_date.clone(),
- generated_at: issue.meta.generated_at.to_string(),
- model: if colophon.model.is_empty() {
- "none (heuristic selection)".into()
- } else {
- colophon.model.clone()
- },
- entries_fetched: colophon.entries_fetched,
- feeds_seen: colophon.feeds_seen,
- candidates: colophon.candidates,
- article_count: issue.meta.article_count,
- section_count: issue.meta.section_count,
- total_words: issue.meta.total_words,
- reading_line: reading_line(issue.meta.reading_minutes),
- cost_usd: format!("${:.4}", colophon.cost_usd),
- generator_version: if colophon.generator_version.is_empty() {
- format!("daily-epub {}", env!("CARGO_PKG_VERSION"))
- } else {
- colophon.generator_version.clone()
- },
- };
- Ok(Chapter {
- id: "colophon".into(),
- href: "colophon.xhtml".into(),
- title: "Colophon".into(),
- xhtml: tpl.render()?,
- toc_level: 1,
- })
-}
-
-fn render_cover_page(issue: &Issue, edition: Edition) -> Result<Chapter, EpubError> {
- let tpl = CoverPage {
- title: issue.meta.title_for(edition),
- alt: format!(
- "The Daily EPUB, {} \u{2014} No. {}",
- issue.meta.display_date, issue.meta.issue_number
- ),
- cover_href: cover_href(edition),
- };
- Ok(Chapter {
- id: "cover".into(),
- href: "cover.xhtml".into(),
- title: "Cover".into(),
- xhtml: tpl.render()?,
- toc_level: 1,
- })
-}
-
-/// Render every chapter for an edition, in issue order (§3.10).
+/// Render every chapter of one edition, in issue order (§3.10).
pub fn render_all(
issue: &Issue,
edition: Edition,
@@ -993,376 +255,10 @@ pub fn assemble(
Ok(out)
}
-/// A synthetic, fully offline [`Issue`] used by the unit tests here and by the
-/// integration tests in `tests/` (which can only see the public API).
-pub mod fixtures {
- use std::collections::BTreeMap;
-
- use jiff::Timestamp;
-
- use crate::types::*;
-
- pub fn timestamp() -> Timestamp {
- "2026-08-15T05:30:00Z".parse().expect("fixed timestamp")
- }
-
- pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article {
- Article {
- id,
- canonical_url: format!("https://example.com/{entry_id}"),
- title: title.to_string(),
- best_entry_id: entry_id,
- content_html: format!(
- "<p>Body of <em>{title}</em> with an image.</p><img src=\"https://img.example/{entry_id}.png\" alt=\"A chart of the daily figures\"><p>More words & things.</p>"
- ),
- word_count: 1200,
- excerpt_only: false,
- image_count: 1,
- sources: vec![SourceRef {
- entry_id,
- feed_id: 7,
- feed_title: "Example Feed".into(),
- category: Some("Tech".into()),
- kind: SourceKind::Feed,
- }],
- first_seen: timestamp(),
- url: format!("https://example.com/{entry_id}"),
- author: Some("A. Writer".into()),
- feed_id: 7,
- feed_title: "Example Feed".into(),
- category: Some("Tech".into()),
- published_at: Some(timestamp()),
- comments_url: None,
- image_urls: vec![format!("https://img.example/{entry_id}.png")],
- social: vec![SocialRef {
- article_id: id,
- source: SocialSource::Hn,
- item_id: Some("40100000".into()),
- score: 342,
- num_comments: 210,
- item_url: Some("https://news.ycombinator.com/item?id=40100000".into()),
- fetched_at: timestamp(),
- }],
- extract_method: ExtractMethod::Readability,
- }
- }
-
- pub fn discussion(article_id: ArticleId, entry_id: EntryId) -> Discussion {
- Discussion {
- article_id,
- chapter_id: format!("disc-{entry_id}"),
- threads: vec![CommentThread {
- source: SocialSource::Hn,
- item_url: "https://news.ycombinator.com/item?id=40100000".into(),
- total_comments: 210,
- comments: vec![Comment {
- author: "alice".into(),
- points: Some(61),
- text_html: "<p>The write path is the interesting part.</p>".into(),
- depth: 0,
- children: vec![Comment {
- author: "bob".into(),
- points: Some(24),
- text_html: "<p>Agreed.</p>".into(),
- depth: 1,
- children: vec![],
- }],
- }],
- }],
- }
- }
-
- /// A synthetic two-article issue, one of them carrying a discussion.
- pub fn issue() -> Issue {
- let lead = Pick {
- article: article(1, 1001, "The Lead Story"),
- section: "Top Stories".into(),
- position: 0,
- is_lead: true,
- summary: Some("What it argues, and why it is worth the time.".into()),
- llm: None,
- discussion: Some(discussion(1, 1001)),
- };
- let second = Pick {
- article: article(2, 1002, "A Niche Delight & Other Tales"),
- section: "Niche Corner".into(),
- position: 0,
- is_lead: false,
- summary: None,
- llm: None,
- discussion: None,
- };
- let mut section_intros = BTreeMap::new();
- section_intros.insert("Top Stories".to_string(), "The day in brief.".to_string());
- let mut summaries = BTreeMap::new();
- summaries.insert(2, "A short abstract for the second piece.".to_string());
-
- Issue {
- meta: IssueMeta {
- date: "2026-08-15".parse().expect("fixed date"),
- issue_number: 42,
- generated_at: timestamp(),
- display_date: "Friday, August 15, 2026".into(),
- article_count: 2,
- section_count: 2,
- total_words: 2400,
- reading_minutes: 11,
- },
- lineup: Lineup {
- date: "2026-08-15".parse().expect("fixed date"),
- picks: vec![lead, second],
- section_order: vec!["Top Stories".into(), "Niche Corner".into()],
- },
- editorial: Editorial {
- front_page_html: "<p>Two stories today, both worth your coffee.</p>".into(),
- section_intros,
- summaries,
- },
- world_briefing: Some(WorldBriefing {
- date: "2026-08-15".parse().expect("fixed date"),
- source_url: "https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15"
- .into(),
- overview: Some("A concise view of the day.".into()),
- sections: vec![WorldBriefingSection {
- title: "Top Stories".into(),
- events: vec![WorldEvent {
- id: "s1-e1".into(),
- source_text: "Something happened somewhere.".into(),
- links: vec![],
- children: vec![],
- summary: Some("The event in context.".into()),
- }],
- }],
- }),
- colophon: Colophon {
- model: "deepseek-v4-flash".into(),
- entries_fetched: 431,
- feeds_seen: 92,
- candidates: 120,
- cost_usd: 0.0731,
- generator_version: "daily-epub 0.1.0".into(),
- },
- }
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
- use fixtures::issue;
- use hmac::{Hmac, KeyInit};
- use sha2::Sha256;
-
- #[test]
- fn rating_token_matches_the_spec_vector() {
- let date: Date = "2026-08-15".parse().unwrap();
- assert_eq!(rating_message(date, 1234, Vote::Up), "2026-08-15/1234/up");
- // hex(hmac_sha256("test-secret", "2026-08-15/1234/up"))[..16]
- let token = rating_token("test-secret", date, 1234, Vote::Up);
- assert_eq!(token.len(), TOKEN_LEN);
- assert!(token.chars().all(|c| c.is_ascii_hexdigit()));
-
- // Independently computed reference value.
- use hmac::Mac;
- let mut mac = Hmac::<Sha256>::new_from_slice(b"test-secret").unwrap();
- mac.update(b"2026-08-15/1234/up");
- let expected: String = hex::encode(mac.finalize().into_bytes())
- .chars()
- .take(16)
- .collect();
- assert_eq!(token, expected);
-
- // Different vote, article and secret all change the token.
- assert_ne!(token, rating_token("test-secret", date, 1234, Vote::Down));
- assert_ne!(token, rating_token("test-secret", date, 1235, Vote::Up));
- assert_ne!(token, rating_token("other-secret", date, 1234, Vote::Up));
- }
-
- /// The EPUB signs the links and `server.rs` verifies them: one formula, or no
- /// rating ever lands. This is the vector `server::tests` pins from its side
- /// (`VECTOR_SECRET` / `VECTOR_TOKEN_UP`) — change one, change both (§3.9).
- #[test]
- fn epub_and_server_share_one_token_vector() {
- let date: Date = "2026-08-15".parse().unwrap();
- assert_eq!(
- rating_token("test-secret", date, 42, Vote::Up),
- "3b314cf7e6d8f50f"
- );
- }
-
- #[test]
- fn rating_url_has_the_spec_shape() {
- let date: Date = "2026-08-15".parse().unwrap();
- let url = rating_url("https://daily.hallada.net/", "s3cret", date, 99, Vote::Down);
- let token = rating_token("s3cret", date, 99, Vote::Down);
- assert_eq!(
- url,
- format!("https://daily.hallada.net/r/2026-08-15/99/down?t={token}")
- );
- }
-
- #[test]
- fn social_line_matches_the_spec_example() {
- let refs = vec![SocialRef {
- article_id: 1,
- source: crate::types::SocialSource::Hn,
- item_id: None,
- score: 342,
- num_comments: 210,
- item_url: None,
- fetched_at: fixtures::timestamp(),
- }];
- assert_eq!(
- social_line(&refs).as_deref(),
- Some("\u{25b2} 342 on HN \u{00b7} 210 comments")
- );
- assert!(social_line(&[]).is_none());
- }
-
- #[test]
- fn hrefs_are_deterministic() {
- let issue = issue();
- let pick = &issue.lineup.picks[0];
- assert_eq!(article_href(pick), "art-1001.xhtml");
- assert_eq!(discussion_href(pick), "disc-1001.xhtml");
- assert_eq!(
- section_href("Tech & Engineering"),
- "sec-tech-engineering.xhtml"
- );
- }
-
- fn assert_xml_ok(xhtml: &str) {
- // A crude well-formedness check: the document parses as XML only if every
- // tag is closed, so compare open/close counts for the elements we emit.
- assert!(xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>"));
- assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\""));
- assert!(xhtml.trim_end().ends_with("</html>"));
- for tag in ["html", "head", "body", "div", "p"] {
- let opens = xhtml.matches(&format!("<{tag}")).count();
- let closes = xhtml.matches(&format!("</{tag}>")).count();
- assert_eq!(opens, closes, "unbalanced <{tag}> in\n{xhtml}");
- }
- assert!(!xhtml.contains(" "));
- for void in ["<br>", "<hr>", "<img "] {
- if void == "<img " {
- for (i, _) in xhtml.match_indices("<img ") {
- let tail = &xhtml[i..];
- let end = tail.find('>').unwrap_or(0);
- assert!(tail[..end].ends_with('/'), "unclosed <img> in\n{xhtml}");
- }
- } else {
- assert!(!xhtml.contains(void), "unclosed {void} in\n{xhtml}");
- }
- }
- }
-
- #[test]
- fn front_page_renders_stats_and_editorial() {
- let issue = issue();
- let chapter = render_front_page(&issue).unwrap();
- assert_eq!(chapter.href, "front.xhtml");
- assert!(chapter.xhtml.contains("From the Editor"));
- assert!(chapter.xhtml.contains("2 articles"));
- assert!(chapter.xhtml.contains("both worth your coffee"));
- assert!(chapter.xhtml.contains("No. 42"));
- assert_xml_ok(&chapter.xhtml);
- }
-
- #[test]
- fn in_this_issue_links_every_pick() {
- let issue = issue();
- let chapter = render_in_this_issue(&issue).unwrap();
- assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\""));
- assert!(chapter.xhtml.contains("href=\"art-1002.xhtml\""));
- assert!(chapter.xhtml.contains("Example Feed"));
- assert!(chapter.xhtml.contains("6 min read"));
- assert!(chapter.xhtml.contains("What it argues"));
- assert!(chapter.xhtml.contains("A short abstract"));
- // Titles are escaped (askama emits numeric references), never injected raw.
- assert!(chapter.xhtml.contains("A Niche Delight & Other Tales"));
- assert_xml_ok(&chapter.xhtml);
- }
-
- #[test]
- fn article_chapter_has_header_body_and_footer() {
- let issue = issue();
- let pick = &issue.lineup.picks[0];
- let chapter = render_article(
- &issue,
- pick,
- &[],
- Edition::Standard,
- "https://daily.hallada.net",
- Some("s3cret"),
- )
- .unwrap();
- assert_eq!(chapter.id, "art-1001");
- assert_eq!(chapter.toc_level, 2);
- assert!(chapter.xhtml.contains("By A. Writer"));
- assert!(chapter.xhtml.contains("Example Feed"));
- assert!(chapter.xhtml.contains("6 min read"));
- assert!(chapter.xhtml.contains("\u{25b2} 342 on HN"));
- assert!(chapter.xhtml.contains("/r/2026-08-15/1/up?t="));
- assert!(chapter.xhtml.contains("/r/2026-08-15/1/down?t="));
- assert!(chapter.xhtml.contains("Read online"));
- assert!(chapter.xhtml.contains("href=\"disc-1001.xhtml\""));
- // The un-downloaded image degrades to a placeholder.
- assert!(
- chapter
- .xhtml
- .contains("[image: A chart of the daily figures]")
- );
- assert_xml_ok(&chapter.xhtml);
- }
-
- #[test]
- fn x4_articles_omit_rating_links() {
- let issue = issue();
- let chapter = render_article(
- &issue,
- &issue.lineup.picks[0],
- &[],
- Edition::X4,
- "https://daily.hallada.net",
- Some("s3cret"),
- )
- .unwrap();
- assert!(!chapter.xhtml.contains("/r/2026-08-15/"));
- assert!(chapter.xhtml.contains("Read online"));
- }
-
- #[test]
- fn discussion_chapter_nests_under_its_article() {
- let issue = issue();
- let chapter = render_discussion(&issue.lineup.picks[0]).unwrap().unwrap();
- assert_eq!(chapter.id, "disc-1001");
- assert_eq!(chapter.toc_level, 3);
- assert!(
- chapter
- .title
- .starts_with("\u{1f4ac} Discussion: The Lead Story")
- );
- assert!(chapter.xhtml.contains("alice"));
- assert!(chapter.xhtml.contains("blockquote class=\"comment\""));
- assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\""));
- assert_xml_ok(&chapter.xhtml);
- assert!(render_discussion(&issue.lineup.picks[1]).unwrap().is_none());
- }
-
- #[test]
- fn world_and_colophon_chapters_render() {
- let issue = issue();
- let world = render_world_briefing(&issue).unwrap().unwrap();
- assert!(world.xhtml.contains("Something happened somewhere"));
- assert!(world.xhtml.contains("CC BY-SA"));
- assert_xml_ok(&world.xhtml);
-
- let colophon = render_colophon(&issue).unwrap();
- assert!(colophon.xhtml.contains("deepseek-v4-flash"));
- assert!(colophon.xhtml.contains("431"));
- assert!(colophon.xhtml.contains("$0.0731"));
- assert_xml_ok(&colophon.xhtml);
- }
+ use crate::epub::fixtures::{assert_xml_ok, issue};
#[test]
fn render_all_orders_the_issue() {
@@ -1390,100 +286,4 @@ mod tests {
assert_xml_ok(&chapter.xhtml);
}
}
-
- #[test]
- fn covers_rasterize_for_both_editions() {
- let issue = issue();
- for edition in [Edition::Standard, Edition::X4] {
- let cover = render_cover(&issue, edition).expect("cover");
- let decoded = image::load_from_memory(&cover.bytes).expect("cover is a valid image");
- assert_eq!(
- (decoded.width(), decoded.height()),
- cover_size(edition),
- "cover size for {edition:?}"
- );
- assert_eq!(decoded.color(), image::ColorType::Rgb8);
- if edition == Edition::X4 {
- assert_eq!(cover.filename, "cover.jpg");
- assert_eq!(cover.mime, "image/jpeg");
- assert!(
- cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc0]),
- "baseline SOF0 missing"
- );
- assert!(
- !cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc2]),
- "progressive SOF2 present"
- );
- } else {
- assert_eq!(cover.filename, "cover.png");
- assert_eq!(cover.mime, "image/png");
- }
- }
- }
-
- #[test]
- fn fallback_cover_is_drawn_without_fonts() {
- let png = encode_cover(
- draw_fallback_cover(480, 800, Edition::X4).unwrap(),
- Edition::X4,
- )
- .unwrap();
- let decoded = image::load_from_memory(&png).unwrap();
- assert_eq!((decoded.width(), decoded.height()), (480, 800));
- // Some ink actually landed on the page.
- let gray = decoded.to_luma8();
- assert!(gray.pixels().any(|p| p[0] < 32));
- assert!(gray.pixels().any(|p| p[0] > 224));
-
- // The badge slab is the only per-edition mark the text-free fallback can
- // draw, so the two editions still differ without any fonts installed.
- let standard = draw_fallback_cover(480, 800, Edition::Standard).unwrap();
- let x4 = draw_fallback_cover(480, 800, Edition::X4).unwrap();
- assert_ne!(standard.data(), x4.data());
- assert!(badge_ink(&x4) > badge_ink(&standard));
- }
-
- /// Dark pixels inside the badge rectangle.
- fn badge_ink(pixmap: &tiny_skia::Pixmap) -> usize {
- let (w, h) = (pixmap.width() as usize, pixmap.height() as usize);
- let (x0, x1) = (w * 3 / 10, w * 7 / 10);
- let (y0, y1) = (h * 74 / 100, h * 78 / 100);
- let mut dark = 0;
- for y in y0..y1 {
- for x in x0..x1 {
- if pixmap.data()[(y * w + x) * 4] < 32 {
- dark += 1;
- }
- }
- }
- dark
- }
-
- /// At thumbnail size the title is unreadable, so the cover itself has to
- /// say which edition it is (§3.10).
- #[test]
- fn only_the_x4_cover_carries_the_edition_badge() {
- let issue = fixtures::issue();
- let x4 = cover_svg(&issue, Edition::X4, 480, 800).unwrap();
- assert!(x4.contains(">X4 EDITION<"), "{x4}");
- // White appears twice: the page ground, and the badge text reversed out
- // of the slab.
- assert_eq!(x4.matches("fill=\"#ffffff\"").count(), 2, "{x4}");
-
- let standard = cover_svg(&issue, Edition::Standard, 1200, 1600).unwrap();
- assert!(!standard.contains("X4"));
- assert_eq!(standard.matches("fill=\"#ffffff\"").count(), 1);
-
- // The badge sits between the stats line and the footer, inside the frame.
- let cover = render_cover(&issue, Edition::X4).unwrap();
- let gray = image::load_from_memory(&cover.bytes).unwrap().to_luma8();
- let dark_in_badge = (584..634)
- .flat_map(|y| (144..336).map(move |x| (x, y)))
- .filter(|&(x, y)| gray.get_pixel(x, y)[0] < 32)
- .count();
- assert!(
- dark_in_badge > 4000,
- "badge slab is missing: {dark_in_badge}"
- );
- }
}
diff --git a/src/epub/chapters.rs b/src/epub/chapters.rs
new file mode 100644
index 0000000..db0ad3e
--- /dev/null
+++ b/src/epub/chapters.rs
@@ -0,0 +1,651 @@
+//! Rendering one chapter of XHTML at a time (spec §3.10).
+//!
+//! Each function here turns part of an [`Issue`] into a [`Chapter`]: the front
+//! page, the index, section title pages, articles, discussions, the world
+//! briefing and the colophon. [`super::build::render_all`] puts them in order;
+//! [`super::build::assemble`] zips them.
+
+use askama::Template;
+
+use crate::comments;
+use crate::html::{text_escape, to_xhtml};
+use crate::images;
+use crate::types::{Edition, ImageAsset, Issue, Pick, SocialRef, Vote, WORLD_BRIEFING_SECTION};
+use crate::world;
+
+use super::EpubError;
+use super::build::Chapter;
+
+/// Characters of the hex HMAC kept in rating links (§3.9).
+pub const TOKEN_LEN: usize = crate::auth::TOKEN_LEN;
+
+// One implementation, shared with the verifying side in [`crate::server`]:
+// see [`crate::auth`] for the formula and the pinned test vector (§3.9).
+pub use crate::auth::{rating_message, rating_token, rating_url};
+
+#[derive(Template)]
+#[template(path = "front_page.xhtml", escape = "html")]
+struct FrontPage {
+ title: String,
+ display_date: String,
+ issue_number: i64,
+ stats_line: String,
+ body_html: String,
+}
+
+struct IndexEntry {
+ href: String,
+ title: String,
+ source: String,
+ reading_minutes: i64,
+ summary: String,
+}
+
+struct IndexSection {
+ name: String,
+ entries: Vec<IndexEntry>,
+}
+
+#[derive(Template)]
+#[template(path = "in_this_issue.xhtml", escape = "html")]
+struct InThisIssue {
+ title: String,
+ stats_line: String,
+ sections: Vec<IndexSection>,
+}
+
+#[derive(Template)]
+#[template(path = "section.xhtml", escape = "html")]
+struct SectionPage {
+ title: String,
+ name: String,
+ intro: Option<String>,
+}
+
+struct RatingLinks {
+ up_url: String,
+ down_url: String,
+}
+
+#[derive(Template)]
+#[template(path = "chapter.xhtml", escape = "html")]
+struct ArticleChapter {
+ title: String,
+ article_title: String,
+ byline: Option<String>,
+ meta_line: String,
+ social_line: Option<String>,
+ summary: Option<String>,
+ excerpt_only: bool,
+ body_html: String,
+ rating: Option<RatingLinks>,
+ read_online_url: String,
+ discussion_href: Option<String>,
+}
+
+#[derive(Template)]
+#[template(path = "discussion.xhtml", escape = "html")]
+struct DiscussionChapter {
+ title: String,
+ heading: String,
+ body_html: String,
+ article_href: String,
+}
+
+#[derive(Template)]
+#[template(path = "world_briefing.xhtml", escape = "html")]
+struct WorldBriefingChapter {
+ title: String,
+ display_date: String,
+ body_html: String,
+}
+
+#[derive(Template)]
+#[template(path = "colophon.xhtml", escape = "html")]
+struct ColophonChapter {
+ title: String,
+ issue_number: i64,
+ display_date: String,
+ generated_at: String,
+ model: String,
+ entries_fetched: i64,
+ feeds_seen: i64,
+ candidates: i64,
+ article_count: i64,
+ section_count: i64,
+ total_words: i64,
+ reading_line: String,
+ cost_usd: String,
+ generator_version: String,
+}
+
+// ---------------------------------------------------------------------------
+// Chapters (§3.10)
+// ---------------------------------------------------------------------------
+
+/// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids.
+pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String {
+ // Sanitize first: image rewriting emits our own trusted markup (including the
+ // `image-placeholder` class, which ammonia would otherwise strip).
+ let cleaned = ammonia::clean(html);
+ let rewritten = images::rewrite_img_srcs(&cleaned, images_);
+ to_xhtml(&rewritten)
+}
+
+/// "~2h 15m read" for the colophon (§3.10).
+fn reading_line(minutes: i64) -> String {
+ let (h, m) = (minutes / 60, minutes % 60);
+ if h > 0 {
+ format!("~{h}h {m}m read")
+ } else {
+ format!("~{m}m read")
+ }
+}
+
+/// "▲ 342 on HN · 210 comments" (§3.10).
+pub fn social_line(social: &[SocialRef]) -> Option<String> {
+ let mut parts: Vec<String> = Vec::new();
+ let mut comments = 0i64;
+ for entry in social {
+ if entry.score > 0 {
+ parts.push(format!(
+ "\u{25b2} {} on {}",
+ entry.score,
+ entry.source.display_name()
+ ));
+ }
+ comments += entry.num_comments.max(0);
+ }
+ if comments > 0 {
+ let noun = if comments == 1 { "comment" } else { "comments" };
+ parts.push(format!("{comments} {noun}"));
+ }
+ if parts.is_empty() {
+ None
+ } else {
+ Some(parts.join(" \u{00b7} "))
+ }
+}
+
+fn article_href(pick: &Pick) -> String {
+ format!("{}.xhtml", pick.article.chapter_id())
+}
+
+fn discussion_href(pick: &Pick) -> String {
+ format!("disc-{}.xhtml", pick.article.best_entry_id)
+}
+
+fn section_href(name: &str) -> String {
+ let slug: String = name
+ .chars()
+ .map(|c| {
+ if c.is_ascii_alphanumeric() {
+ c.to_ascii_lowercase()
+ } else {
+ '-'
+ }
+ })
+ .collect();
+ let slug = slug
+ .split('-')
+ .filter(|s| !s.is_empty())
+ .collect::<Vec<_>>()
+ .join("-");
+ format!("sec-{slug}.xhtml")
+}
+
+fn summary_for<'a>(issue: &'a Issue, pick: &'a Pick) -> Option<&'a str> {
+ pick.summary
+ .as_deref()
+ .or_else(|| {
+ issue
+ .editorial
+ .summaries
+ .get(&pick.article.id)
+ .map(|s| s.as_str())
+ })
+ .filter(|s| !s.trim().is_empty())
+}
+
+fn published_display(pick: &Pick) -> Option<String> {
+ pick.article
+ .published_at
+ .map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string())
+}
+
+/// "From the Editor" front page plus the issue stats line (§3.10).
+pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> {
+ let body = issue.editorial.front_page_html.trim();
+ let body_html = if body.is_empty() {
+ format!(
+ "<p>{} of reading, chosen overnight.</p>",
+ text_escape(&issue.meta.stats_line())
+ )
+ } else {
+ to_xhtml(&ammonia::clean(body))
+ };
+ let tpl = FrontPage {
+ title: "From the Editor".into(),
+ display_date: issue.meta.display_date.clone(),
+ issue_number: issue.meta.issue_number,
+ stats_line: issue.meta.stats_line(),
+ body_html,
+ };
+ Ok(Chapter {
+ id: "front".into(),
+ href: "front.xhtml".into(),
+ title: "From the Editor".into(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ })
+}
+
+/// Section names in issue order, with the reserved world section removed (§3.6).
+pub fn section_names(issue: &Issue) -> Vec<String> {
+ let mut names: Vec<String> = issue
+ .lineup
+ .section_order
+ .iter()
+ .filter(|s| s.as_str() != WORLD_BRIEFING_SECTION)
+ .cloned()
+ .collect();
+ for pick in &issue.lineup.picks {
+ if pick.section != WORLD_BRIEFING_SECTION && !names.contains(&pick.section) {
+ names.push(pick.section.clone());
+ }
+ }
+ names.retain(|n| issue.lineup.picks.iter().any(|p| &p.section == n));
+ names
+}
+
+/// "In This Issue": per section, each article's title, source, reading time and
+/// summary, linked to its chapter (§3.10).
+pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> {
+ let mut sections = Vec::new();
+ for name in section_names(issue) {
+ let entries = issue
+ .lineup
+ .section_picks(&name)
+ .into_iter()
+ .map(|pick| IndexEntry {
+ href: article_href(pick),
+ title: pick.article.title.clone(),
+ source: pick.article.feed_title.clone(),
+ reading_minutes: pick.article.reading_minutes(),
+ summary: summary_for(issue, pick).unwrap_or_default().to_string(),
+ })
+ .collect();
+ sections.push(IndexSection { name, entries });
+ }
+ if issue.world_briefing.is_some() {
+ sections.push(IndexSection {
+ name: WORLD_BRIEFING_SECTION.to_string(),
+ entries: vec![IndexEntry {
+ href: "world.xhtml".into(),
+ title: "World Briefing".into(),
+ source: "Wikipedia Current Events".into(),
+ reading_minutes: 3,
+ summary: "The day's events, as recorded by the Current Events portal.".into(),
+ }],
+ });
+ }
+ let tpl = InThisIssue {
+ title: "In This Issue".into(),
+ stats_line: issue.meta.stats_line(),
+ sections,
+ };
+ Ok(Chapter {
+ id: "in-this-issue".into(),
+ href: "in-this-issue.xhtml".into(),
+ title: "In This Issue".into(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ })
+}
+
+/// A section title page: name + LLM intro (§3.10).
+pub fn render_section_page(name: &str, intro: Option<&str>) -> Result<Chapter, EpubError> {
+ let tpl = SectionPage {
+ title: name.to_string(),
+ name: name.to_string(),
+ intro: intro
+ .map(|s| s.trim().to_string())
+ .filter(|s| !s.is_empty()),
+ };
+ Ok(Chapter {
+ id: format!("sec-{name}"),
+ href: section_href(name),
+ title: name.to_string(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ })
+}
+
+/// One article chapter: header, cleaned body with embedded images, rating footer (§3.10).
+pub fn render_article(
+ issue: &Issue,
+ pick: &Pick,
+ images_: &[ImageAsset],
+ edition: Edition,
+ public_url: &str,
+ hmac_secret: Option<&str>,
+) -> Result<Chapter, EpubError> {
+ let article = &pick.article;
+ let mut meta_parts = vec![article.feed_title.clone()];
+ if let Some(date) = published_display(pick) {
+ meta_parts.push(date);
+ }
+ meta_parts.push(format!(
+ "{} min read \u{00b7} {} words",
+ article.reading_minutes(),
+ article.word_count
+ ));
+
+ // The X4 has no browser, so rating links are pointless there (§7).
+ let rating = match (hmac_secret, edition) {
+ (Some(secret), Edition::Standard) if !secret.is_empty() => Some(RatingLinks {
+ up_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Up),
+ down_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Down),
+ }),
+ _ => None,
+ };
+
+ let tpl = ArticleChapter {
+ title: article.title.clone(),
+ article_title: article.title.clone(),
+ byline: article.author.as_ref().map(|a| format!("By {a}")),
+ meta_line: meta_parts.join(" \u{00b7} "),
+ social_line: social_line(&article.social),
+ summary: summary_for(issue, pick).map(str::to_string),
+ excerpt_only: article.excerpt_only,
+ body_html: prepare_body(&article.content_html, images_),
+ rating,
+ read_online_url: article.url.clone(),
+ discussion_href: pick.discussion.as_ref().map(|_| discussion_href(pick)),
+ };
+ Ok(Chapter {
+ id: article.chapter_id(),
+ href: article_href(pick),
+ title: article.title.clone(),
+ xhtml: tpl.render()?,
+ toc_level: 2,
+ })
+}
+
+/// The discussion chapter that follows an article when it has comments (§3.7).
+pub fn render_discussion(pick: &Pick) -> Result<Option<Chapter>, EpubError> {
+ let Some(discussion) = &pick.discussion else {
+ return Ok(None);
+ };
+ if discussion.threads.is_empty() {
+ return Ok(None);
+ }
+ let title = comments::chapter_title(&pick.article.title, discussion);
+ let tpl = DiscussionChapter {
+ title: title.clone(),
+ heading: title.clone(),
+ body_html: comments::render_xhtml(discussion, &pick.article.title),
+ article_href: article_href(pick),
+ };
+ Ok(Some(Chapter {
+ id: format!("disc-{}", pick.article.best_entry_id),
+ href: discussion_href(pick),
+ title,
+ xhtml: tpl.render()?,
+ toc_level: 3,
+ }))
+}
+
+/// The Wikipedia Current Events section chapter (§3.8).
+pub fn render_world_briefing(issue: &Issue) -> Result<Option<Chapter>, EpubError> {
+ let Some(briefing) = &issue.world_briefing else {
+ return Ok(None);
+ };
+ // The briefing may cover an earlier day than the issue: the portal page for
+ // the issue's own date is still an empty stub at 05:30 (§3.8). Dateline the
+ // section with the day it actually reports on, not the masthead date.
+ let tpl = WorldBriefingChapter {
+ title: WORLD_BRIEFING_SECTION.to_string(),
+ display_date: crate::pipeline::display_date(briefing.date),
+ body_html: world::render_xhtml(briefing),
+ };
+ Ok(Some(Chapter {
+ id: "world".into(),
+ href: "world.xhtml".into(),
+ title: WORLD_BRIEFING_SECTION.to_string(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ }))
+}
+
+/// Colophon: generation timestamp, models used, token cost, feed counts (§3.10).
+pub fn render_colophon(issue: &Issue) -> Result<Chapter, EpubError> {
+ let colophon = &issue.colophon;
+ let tpl = ColophonChapter {
+ title: "Colophon".into(),
+ issue_number: issue.meta.issue_number,
+ display_date: issue.meta.display_date.clone(),
+ generated_at: issue.meta.generated_at.to_string(),
+ model: if colophon.model.is_empty() {
+ "none (heuristic selection)".into()
+ } else {
+ colophon.model.clone()
+ },
+ entries_fetched: colophon.entries_fetched,
+ feeds_seen: colophon.feeds_seen,
+ candidates: colophon.candidates,
+ article_count: issue.meta.article_count,
+ section_count: issue.meta.section_count,
+ total_words: issue.meta.total_words,
+ reading_line: reading_line(issue.meta.reading_minutes),
+ cost_usd: format!("${:.4}", colophon.cost_usd),
+ generator_version: if colophon.generator_version.is_empty() {
+ format!("daily-epub {}", env!("CARGO_PKG_VERSION"))
+ } else {
+ colophon.generator_version.clone()
+ },
+ };
+ Ok(Chapter {
+ id: "colophon".into(),
+ href: "colophon.xhtml".into(),
+ title: "Colophon".into(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::epub::fixtures::{self, assert_xml_ok, issue};
+ use crate::types::{SocialRef, Vote};
+ use hmac::{Hmac, KeyInit};
+ use jiff::civil::Date;
+ use sha2::Sha256;
+
+ #[test]
+ fn rating_token_matches_the_spec_vector() {
+ let date: Date = "2026-08-15".parse().unwrap();
+ assert_eq!(rating_message(date, 1234, Vote::Up), "2026-08-15/1234/up");
+ // hex(hmac_sha256("test-secret", "2026-08-15/1234/up"))[..16]
+ let token = rating_token("test-secret", date, 1234, Vote::Up);
+ assert_eq!(token.len(), TOKEN_LEN);
+ assert!(token.chars().all(|c| c.is_ascii_hexdigit()));
+
+ // Independently computed reference value.
+ use hmac::Mac;
+ let mut mac = Hmac::<Sha256>::new_from_slice(b"test-secret").unwrap();
+ mac.update(b"2026-08-15/1234/up");
+ let expected: String = hex::encode(mac.finalize().into_bytes())
+ .chars()
+ .take(16)
+ .collect();
+ assert_eq!(token, expected);
+
+ // Different vote, article and secret all change the token.
+ assert_ne!(token, rating_token("test-secret", date, 1234, Vote::Down));
+ assert_ne!(token, rating_token("test-secret", date, 1235, Vote::Up));
+ assert_ne!(token, rating_token("other-secret", date, 1234, Vote::Up));
+ }
+
+ /// The EPUB signs the links and `server.rs` verifies them: one formula, or no
+ /// rating ever lands. This is the vector `server::tests` pins from its side
+ /// (`VECTOR_SECRET` / `VECTOR_TOKEN_UP`) — change one, change both (§3.9).
+ #[test]
+ fn epub_and_server_share_one_token_vector() {
+ let date: Date = "2026-08-15".parse().unwrap();
+ assert_eq!(
+ rating_token("test-secret", date, 42, Vote::Up),
+ "3b314cf7e6d8f50f"
+ );
+ }
+
+ #[test]
+ fn rating_url_has_the_spec_shape() {
+ let date: Date = "2026-08-15".parse().unwrap();
+ let url = rating_url("https://daily.hallada.net/", "s3cret", date, 99, Vote::Down);
+ let token = rating_token("s3cret", date, 99, Vote::Down);
+ assert_eq!(
+ url,
+ format!("https://daily.hallada.net/r/2026-08-15/99/down?t={token}")
+ );
+ }
+
+ #[test]
+ fn social_line_matches_the_spec_example() {
+ let refs = vec![SocialRef {
+ article_id: 1,
+ source: crate::types::SocialSource::Hn,
+ item_id: None,
+ score: 342,
+ num_comments: 210,
+ item_url: None,
+ fetched_at: fixtures::timestamp(),
+ }];
+ assert_eq!(
+ social_line(&refs).as_deref(),
+ Some("\u{25b2} 342 on HN \u{00b7} 210 comments")
+ );
+ assert!(social_line(&[]).is_none());
+ }
+
+ #[test]
+ fn hrefs_are_deterministic() {
+ let issue = issue();
+ let pick = &issue.lineup.picks[0];
+ assert_eq!(article_href(pick), "art-1001.xhtml");
+ assert_eq!(discussion_href(pick), "disc-1001.xhtml");
+ assert_eq!(
+ section_href("Tech & Engineering"),
+ "sec-tech-engineering.xhtml"
+ );
+ }
+
+ #[test]
+ fn front_page_renders_stats_and_editorial() {
+ let issue = issue();
+ let chapter = render_front_page(&issue).unwrap();
+ assert_eq!(chapter.href, "front.xhtml");
+ assert!(chapter.xhtml.contains("From the Editor"));
+ assert!(chapter.xhtml.contains("2 articles"));
+ assert!(chapter.xhtml.contains("both worth your coffee"));
+ assert!(chapter.xhtml.contains("No. 42"));
+ assert_xml_ok(&chapter.xhtml);
+ }
+
+ #[test]
+ fn in_this_issue_links_every_pick() {
+ let issue = issue();
+ let chapter = render_in_this_issue(&issue).unwrap();
+ assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\""));
+ assert!(chapter.xhtml.contains("href=\"art-1002.xhtml\""));
+ assert!(chapter.xhtml.contains("Example Feed"));
+ assert!(chapter.xhtml.contains("6 min read"));
+ assert!(chapter.xhtml.contains("What it argues"));
+ assert!(chapter.xhtml.contains("A short abstract"));
+ // Titles are escaped (askama emits numeric references), never injected raw.
+ assert!(chapter.xhtml.contains("A Niche Delight & Other Tales"));
+ assert_xml_ok(&chapter.xhtml);
+ }
+
+ #[test]
+ fn article_chapter_has_header_body_and_footer() {
+ let issue = issue();
+ let pick = &issue.lineup.picks[0];
+ let chapter = render_article(
+ &issue,
+ pick,
+ &[],
+ Edition::Standard,
+ "https://daily.hallada.net",
+ Some("s3cret"),
+ )
+ .unwrap();
+ assert_eq!(chapter.id, "art-1001");
+ assert_eq!(chapter.toc_level, 2);
+ assert!(chapter.xhtml.contains("By A. Writer"));
+ assert!(chapter.xhtml.contains("Example Feed"));
+ assert!(chapter.xhtml.contains("6 min read"));
+ assert!(chapter.xhtml.contains("\u{25b2} 342 on HN"));
+ assert!(chapter.xhtml.contains("/r/2026-08-15/1/up?t="));
+ assert!(chapter.xhtml.contains("/r/2026-08-15/1/down?t="));
+ assert!(chapter.xhtml.contains("Read online"));
+ assert!(chapter.xhtml.contains("href=\"disc-1001.xhtml\""));
+ // The un-downloaded image degrades to a placeholder.
+ assert!(
+ chapter
+ .xhtml
+ .contains("[image: A chart of the daily figures]")
+ );
+ assert_xml_ok(&chapter.xhtml);
+ }
+
+ #[test]
+ fn x4_articles_omit_rating_links() {
+ let issue = issue();
+ let chapter = render_article(
+ &issue,
+ &issue.lineup.picks[0],
+ &[],
+ Edition::X4,
+ "https://daily.hallada.net",
+ Some("s3cret"),
+ )
+ .unwrap();
+ assert!(!chapter.xhtml.contains("/r/2026-08-15/"));
+ assert!(chapter.xhtml.contains("Read online"));
+ }
+
+ #[test]
+ fn discussion_chapter_nests_under_its_article() {
+ let issue = issue();
+ let chapter = render_discussion(&issue.lineup.picks[0]).unwrap().unwrap();
+ assert_eq!(chapter.id, "disc-1001");
+ assert_eq!(chapter.toc_level, 3);
+ assert!(
+ chapter
+ .title
+ .starts_with("\u{1f4ac} Discussion: The Lead Story")
+ );
+ assert!(chapter.xhtml.contains("alice"));
+ assert!(chapter.xhtml.contains("blockquote class=\"comment\""));
+ assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\""));
+ assert_xml_ok(&chapter.xhtml);
+ assert!(render_discussion(&issue.lineup.picks[1]).unwrap().is_none());
+ }
+
+ #[test]
+ fn world_and_colophon_chapters_render() {
+ let issue = issue();
+ let world = render_world_briefing(&issue).unwrap().unwrap();
+ assert!(world.xhtml.contains("Something happened somewhere"));
+ assert!(world.xhtml.contains("CC BY-SA"));
+ assert_xml_ok(&world.xhtml);
+
+ let colophon = render_colophon(&issue).unwrap();
+ assert!(colophon.xhtml.contains("deepseek-v4-flash"));
+ assert!(colophon.xhtml.contains("431"));
+ assert!(colophon.xhtml.contains("$0.0731"));
+ assert_xml_ok(&colophon.xhtml);
+ }
+}
diff --git a/src/epub/cover.rs b/src/epub/cover.rs
new file mode 100644
index 0000000..c7d71f3
--- /dev/null
+++ b/src/epub/cover.rs
@@ -0,0 +1,423 @@
+//! The generated cover image and cover page (spec §3.10).
+//!
+//! The cover is drawn as an SVG from a template, rasterized with resvg and
+//! encoded per edition. If font resolution fails the run still gets a cover —
+//! a text-free variant that keeps the editions apart in a library thumbnail
+//! grid (notes §3: nothing here fails a build).
+
+use askama::Template;
+
+use crate::types::{Edition, Issue};
+
+use super::EpubError;
+use super::build::Chapter;
+use super::x4;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct CoverAsset {
+ pub bytes: Vec<u8>,
+ pub filename: &'static str,
+ pub mime: &'static str,
+}
+
+pub fn cover_href(edition: Edition) -> &'static str {
+ match edition {
+ Edition::Standard => "cover.png",
+ Edition::X4 => "cover.jpg",
+ }
+}
+
+#[derive(Template)]
+#[template(path = "cover.svg", escape = "html")]
+struct CoverSvg {
+ width: i64,
+ height: i64,
+ margin: i64,
+ inner_width: i64,
+ inner_height: i64,
+ border: i64,
+ hairline: i64,
+ center_x: i64,
+ masthead_y: i64,
+ masthead_size: i64,
+ rule_x1: i64,
+ rule_x2: i64,
+ rule_y: i64,
+ rule2_y: i64,
+ weekday: String,
+ weekday_y: i64,
+ weekday_size: i64,
+ long_date: String,
+ date_y: i64,
+ date_size: i64,
+ issue_number: i64,
+ issue_y: i64,
+ issue_size: i64,
+ stats_line: String,
+ stats_y: i64,
+ stats_size: i64,
+ /// Empty for the standard edition — see [`cover_badge`].
+ edition_tag: String,
+ badge_x: i64,
+ badge_y: i64,
+ badge_width: i64,
+ badge_height: i64,
+ badge_radius: i64,
+ badge_text_y: i64,
+ badge_size: i64,
+ footer: String,
+ footer_y: i64,
+ footer_size: i64,
+}
+
+#[derive(Template)]
+#[template(path = "cover_page.xhtml", escape = "html")]
+struct CoverPage {
+ title: String,
+ alt: String,
+ cover_href: &'static str,
+}
+
+// ---------------------------------------------------------------------------
+// Cover (§3.10)
+// ---------------------------------------------------------------------------
+
+/// Cover pixel size per edition: 1200×1600 standard, 480×800 X4 (§3.10).
+pub fn cover_size(edition: Edition) -> (u32, u32) {
+ match edition {
+ Edition::Standard => (1200, 1600),
+ Edition::X4 => x4::X4_SCREEN,
+ }
+}
+
+/// Split "Friday, August 15, 2026" into ("Friday", "August 15, 2026").
+fn split_display_date(display: &str) -> (String, String) {
+ match display.split_once(", ") {
+ Some((weekday, rest)) => (weekday.to_string(), rest.to_string()),
+ None => (String::new(), display.to_string()),
+ }
+}
+
+/// Reversed badge printed on the cover so the two editions are told apart at
+/// thumbnail size, where the title is unreadable (§3.10).
+///
+/// Empty for the standard edition: an unmarked cover is the default, and the
+/// presence of the slab is itself the signal.
+pub fn cover_badge(edition: Edition) -> &'static str {
+ match edition {
+ Edition::Standard => "",
+ Edition::X4 => "X4 EDITION",
+ }
+}
+
+fn cover_svg(
+ issue: &Issue,
+ edition: Edition,
+ width: u32,
+ height: u32,
+) -> Result<String, EpubError> {
+ let w = i64::from(width);
+ let h = i64::from(height);
+ let margin = w / 15;
+ let (weekday, long_date) = split_display_date(&issue.meta.display_date);
+ // A solid bar between the stats line and the footer: at 100px wide in a
+ // library grid the black slab is the only thing still legible.
+ let badge_height = h / 16;
+ let badge_width = w * 2 / 5;
+ let tpl = CoverSvg {
+ width: w,
+ height: h,
+ margin,
+ inner_width: w - 2 * margin,
+ inner_height: h - 2 * margin,
+ border: (w / 300).max(2),
+ hairline: (w / 600).max(1),
+ center_x: w / 2,
+ masthead_y: h * 30 / 100,
+ masthead_size: w / 9,
+ rule_x1: margin + w / 12,
+ rule_x2: w - margin - w / 12,
+ rule_y: h * 34 / 100,
+ rule2_y: h * 53 / 100,
+ weekday,
+ weekday_y: h * 42 / 100,
+ weekday_size: w / 28,
+ long_date,
+ date_y: h * 47 / 100,
+ date_size: w / 22,
+ issue_number: issue.meta.issue_number,
+ issue_y: h * 60 / 100,
+ issue_size: w / 18,
+ stats_line: issue.meta.stats_line(),
+ stats_y: h * 66 / 100,
+ stats_size: w / 32,
+ edition_tag: cover_badge(edition).to_string(),
+ badge_x: (w - badge_width) / 2,
+ badge_y: h * 73 / 100,
+ badge_width,
+ badge_height,
+ badge_radius: badge_height / 2,
+ badge_text_y: h * 73 / 100 + badge_height * 7 / 10,
+ badge_size: badge_height * 11 / 20,
+ footer: "Assembled overnight \u{00b7} read offline".to_string(),
+ footer_y: h - margin - h / 25,
+ footer_size: w / 40,
+ };
+ Ok(tpl.render()?)
+}
+
+/// Render the standard cover as RGB PNG and the X4 cover as baseline RGB JPEG.
+pub fn render_cover(issue: &Issue, edition: Edition) -> Result<CoverAsset, EpubError> {
+ let (width, height) = cover_size(edition);
+ let svg = cover_svg(issue, edition, width, height)?;
+ let pixmap = match rasterize(&svg, width, height) {
+ Some(pixmap) => pixmap,
+ None => {
+ tracing::warn!("no usable system fonts: falling back to a geometric cover");
+ draw_fallback_cover(width, height, edition)
+ .ok_or_else(|| EpubError::Build("could not allocate the cover".into()))?
+ }
+ };
+ Ok(CoverAsset {
+ bytes: encode_cover(pixmap, edition)?,
+ filename: cover_href(edition),
+ mime: if edition == Edition::X4 {
+ "image/jpeg"
+ } else {
+ "image/png"
+ },
+ })
+}
+
+fn rasterize(svg: &str, width: u32, height: u32) -> Option<tiny_skia::Pixmap> {
+ let mut options = resvg::usvg::Options::default();
+ options.fontdb_mut().load_system_fonts();
+ if options.fontdb.is_empty() {
+ return None;
+ }
+ let tree = resvg::usvg::Tree::from_str(svg, &options)
+ .map_err(|e| tracing::warn!("cover svg did not parse: {e}"))
+ .ok()?;
+ let mut pixmap = tiny_skia::Pixmap::new(width, height)?;
+ pixmap.fill(tiny_skia::Color::WHITE);
+ let size = tree.size();
+ let transform = tiny_skia::Transform::from_scale(
+ width as f32 / size.width(),
+ height as f32 / size.height(),
+ );
+ resvg::render(&tree, transform, &mut pixmap.as_mut());
+ Some(pixmap)
+}
+
+/// Text-free cover used when font resolution fails — the build never fails (§3.10).
+///
+/// The badge cannot carry its lettering here, but the slab itself still keeps
+/// the editions apart in a thumbnail grid.
+fn draw_fallback_cover(width: u32, height: u32, edition: Edition) -> Option<tiny_skia::Pixmap> {
+ let mut pixmap = tiny_skia::Pixmap::new(width, height)?;
+ pixmap.fill(tiny_skia::Color::WHITE);
+ let mut paint = tiny_skia::Paint::default();
+ paint.set_color(tiny_skia::Color::BLACK);
+ paint.anti_alias = true;
+
+ let w = width as f32;
+ let h = height as f32;
+ let margin = w / 15.0;
+ let rect = |x: f32, y: f32, rw: f32, rh: f32, pixmap: &mut tiny_skia::Pixmap| {
+ if let Some(r) = tiny_skia::Rect::from_xywh(x, y, rw, rh) {
+ let path = tiny_skia::PathBuilder::from_rect(r);
+ pixmap.fill_path(
+ &path,
+ &paint,
+ tiny_skia::FillRule::Winding,
+ tiny_skia::Transform::identity(),
+ None,
+ );
+ }
+ };
+ // Frame.
+ let border = (w / 300.0).max(2.0);
+ rect(margin, margin, w - 2.0 * margin, border, &mut pixmap);
+ rect(
+ margin,
+ h - margin - border,
+ w - 2.0 * margin,
+ border,
+ &mut pixmap,
+ );
+ rect(margin, margin, border, h - 2.0 * margin, &mut pixmap);
+ rect(
+ w - margin - border,
+ margin,
+ border,
+ h - 2.0 * margin,
+ &mut pixmap,
+ );
+ // Masthead slab plus body rules — a newspaper silhouette.
+ rect(
+ margin * 2.0,
+ h * 0.26,
+ w - 4.0 * margin,
+ h * 0.035,
+ &mut pixmap,
+ );
+ for i in 0..8 {
+ let y = h * 0.42 + (i as f32) * h * 0.045;
+ let inset = if i % 3 == 2 {
+ margin * 4.0
+ } else {
+ margin * 2.0
+ };
+ rect(inset, y, w - 2.0 * inset, border, &mut pixmap);
+ }
+ if !cover_badge(edition).is_empty() {
+ let badge_height = h / 16.0;
+ let badge_width = w * 2.0 / 5.0;
+ rect(
+ (w - badge_width) / 2.0,
+ h * 0.73,
+ badge_width,
+ badge_height,
+ &mut pixmap,
+ );
+ }
+ Some(pixmap)
+}
+
+fn encode_cover(pixmap: tiny_skia::Pixmap, edition: Edition) -> Result<Vec<u8>, EpubError> {
+ let (w, h) = (pixmap.width(), pixmap.height());
+ let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())
+ .ok_or_else(|| EpubError::Build("cover pixel buffer had the wrong size".into()))?;
+ let rgb = image::DynamicImage::ImageRgba8(rgba).to_rgb8();
+ let mut bytes = Vec::new();
+ match edition {
+ Edition::Standard => image::DynamicImage::ImageRgb8(rgb).write_to(
+ &mut std::io::Cursor::new(&mut bytes),
+ image::ImageFormat::Png,
+ ),
+ Edition::X4 => image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 92)
+ .encode_image(&image::DynamicImage::ImageRgb8(rgb)),
+ }
+ .map_err(|e| EpubError::Build(format!("cover encoding failed: {e}")))?;
+ Ok(bytes)
+}
+
+pub fn render_cover_page(issue: &Issue, edition: Edition) -> Result<Chapter, EpubError> {
+ let tpl = CoverPage {
+ title: issue.meta.title_for(edition),
+ alt: format!(
+ "The Daily EPUB, {} \u{2014} No. {}",
+ issue.meta.display_date, issue.meta.issue_number
+ ),
+ cover_href: cover_href(edition),
+ };
+ Ok(Chapter {
+ id: "cover".into(),
+ href: "cover.xhtml".into(),
+ title: "Cover".into(),
+ xhtml: tpl.render()?,
+ toc_level: 1,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::epub::fixtures::{self, issue};
+
+ #[test]
+ fn covers_rasterize_for_both_editions() {
+ let issue = issue();
+ for edition in [Edition::Standard, Edition::X4] {
+ let cover = render_cover(&issue, edition).expect("cover");
+ let decoded = image::load_from_memory(&cover.bytes).expect("cover is a valid image");
+ assert_eq!(
+ (decoded.width(), decoded.height()),
+ cover_size(edition),
+ "cover size for {edition:?}"
+ );
+ assert_eq!(decoded.color(), image::ColorType::Rgb8);
+ if edition == Edition::X4 {
+ assert_eq!(cover.filename, "cover.jpg");
+ assert_eq!(cover.mime, "image/jpeg");
+ assert!(
+ cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc0]),
+ "baseline SOF0 missing"
+ );
+ assert!(
+ !cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc2]),
+ "progressive SOF2 present"
+ );
+ } else {
+ assert_eq!(cover.filename, "cover.png");
+ assert_eq!(cover.mime, "image/png");
+ }
+ }
+ }
+
+ #[test]
+ fn fallback_cover_is_drawn_without_fonts() {
+ let png = encode_cover(
+ draw_fallback_cover(480, 800, Edition::X4).unwrap(),
+ Edition::X4,
+ )
+ .unwrap();
+ let decoded = image::load_from_memory(&png).unwrap();
+ assert_eq!((decoded.width(), decoded.height()), (480, 800));
+ // Some ink actually landed on the page.
+ let gray = decoded.to_luma8();
+ assert!(gray.pixels().any(|p| p[0] < 32));
+ assert!(gray.pixels().any(|p| p[0] > 224));
+
+ // The badge slab is the only per-edition mark the text-free fallback can
+ // draw, so the two editions still differ without any fonts installed.
+ let standard = draw_fallback_cover(480, 800, Edition::Standard).unwrap();
+ let x4 = draw_fallback_cover(480, 800, Edition::X4).unwrap();
+ assert_ne!(standard.data(), x4.data());
+ assert!(badge_ink(&x4) > badge_ink(&standard));
+ }
+
+ /// Dark pixels inside the badge rectangle.
+ fn badge_ink(pixmap: &tiny_skia::Pixmap) -> usize {
+ let (w, h) = (pixmap.width() as usize, pixmap.height() as usize);
+ let (x0, x1) = (w * 3 / 10, w * 7 / 10);
+ let (y0, y1) = (h * 74 / 100, h * 78 / 100);
+ let mut dark = 0;
+ for y in y0..y1 {
+ for x in x0..x1 {
+ if pixmap.data()[(y * w + x) * 4] < 32 {
+ dark += 1;
+ }
+ }
+ }
+ dark
+ }
+
+ /// At thumbnail size the title is unreadable, so the cover itself has to
+ /// say which edition it is (§3.10).
+ #[test]
+ fn only_the_x4_cover_carries_the_edition_badge() {
+ let issue = fixtures::issue();
+ let x4 = cover_svg(&issue, Edition::X4, 480, 800).unwrap();
+ assert!(x4.contains(">X4 EDITION<"), "{x4}");
+ // White appears twice: the page ground, and the badge text reversed out
+ // of the slab.
+ assert_eq!(x4.matches("fill=\"#ffffff\"").count(), 2, "{x4}");
+
+ let standard = cover_svg(&issue, Edition::Standard, 1200, 1600).unwrap();
+ assert!(!standard.contains("X4"));
+ assert_eq!(standard.matches("fill=\"#ffffff\"").count(), 1);
+
+ // The badge sits between the stats line and the footer, inside the frame.
+ let cover = render_cover(&issue, Edition::X4).unwrap();
+ let gray = image::load_from_memory(&cover.bytes).unwrap().to_luma8();
+ let dark_in_badge = (584..634)
+ .flat_map(|y| (144..336).map(move |x| (x, y)))
+ .filter(|&(x, y)| gray.get_pixel(x, y)[0] < 32)
+ .count();
+ assert!(
+ dark_in_badge > 4000,
+ "badge slab is missing: {dark_in_badge}"
+ );
+ }
+}
diff --git a/src/epub/fixtures.rs b/src/epub/fixtures.rs
new file mode 100644
index 0000000..9a35e91
--- /dev/null
+++ b/src/epub/fixtures.rs
@@ -0,0 +1,182 @@
+//! A synthetic, fully offline [`crate::types::Issue`] for tests.
+//!
+//! Lives here rather than inside a `#[cfg(test)]` block because the integration
+//! tests in `tests/` can only see the public API.
+
+use std::collections::BTreeMap;
+
+use jiff::Timestamp;
+
+use crate::types::*;
+
+pub fn timestamp() -> Timestamp {
+ "2026-08-15T05:30:00Z".parse().expect("fixed timestamp")
+}
+
+pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article {
+ Article {
+ id,
+ canonical_url: format!("https://example.com/{entry_id}"),
+ title: title.to_string(),
+ best_entry_id: entry_id,
+ content_html: format!(
+ "<p>Body of <em>{title}</em> with an image.</p><img src=\"https://img.example/{entry_id}.png\" alt=\"A chart of the daily figures\"><p>More words & things.</p>"
+ ),
+ word_count: 1200,
+ excerpt_only: false,
+ image_count: 1,
+ sources: vec![SourceRef {
+ entry_id,
+ feed_id: 7,
+ feed_title: "Example Feed".into(),
+ category: Some("Tech".into()),
+ kind: SourceKind::Feed,
+ }],
+ first_seen: timestamp(),
+ url: format!("https://example.com/{entry_id}"),
+ author: Some("A. Writer".into()),
+ feed_id: 7,
+ feed_title: "Example Feed".into(),
+ category: Some("Tech".into()),
+ published_at: Some(timestamp()),
+ comments_url: None,
+ image_urls: vec![format!("https://img.example/{entry_id}.png")],
+ social: vec![SocialRef {
+ article_id: id,
+ source: SocialSource::Hn,
+ item_id: Some("40100000".into()),
+ score: 342,
+ num_comments: 210,
+ item_url: Some("https://news.ycombinator.com/item?id=40100000".into()),
+ fetched_at: timestamp(),
+ }],
+ extract_method: ExtractMethod::Readability,
+ }
+}
+
+pub fn discussion(article_id: ArticleId, entry_id: EntryId) -> Discussion {
+ Discussion {
+ article_id,
+ chapter_id: format!("disc-{entry_id}"),
+ threads: vec![CommentThread {
+ source: SocialSource::Hn,
+ item_url: "https://news.ycombinator.com/item?id=40100000".into(),
+ total_comments: 210,
+ comments: vec![Comment {
+ author: "alice".into(),
+ points: Some(61),
+ text_html: "<p>The write path is the interesting part.</p>".into(),
+ depth: 0,
+ children: vec![Comment {
+ author: "bob".into(),
+ points: Some(24),
+ text_html: "<p>Agreed.</p>".into(),
+ depth: 1,
+ children: vec![],
+ }],
+ }],
+ }],
+ }
+}
+
+/// A synthetic two-article issue, one of them carrying a discussion.
+pub fn issue() -> Issue {
+ let lead = Pick {
+ article: article(1, 1001, "The Lead Story"),
+ section: "Top Stories".into(),
+ position: 0,
+ is_lead: true,
+ summary: Some("What it argues, and why it is worth the time.".into()),
+ llm: None,
+ discussion: Some(discussion(1, 1001)),
+ };
+ let second = Pick {
+ article: article(2, 1002, "A Niche Delight & Other Tales"),
+ section: "Niche Corner".into(),
+ position: 0,
+ is_lead: false,
+ summary: None,
+ llm: None,
+ discussion: None,
+ };
+ let mut section_intros = BTreeMap::new();
+ section_intros.insert("Top Stories".to_string(), "The day in brief.".to_string());
+ let mut summaries = BTreeMap::new();
+ summaries.insert(2, "A short abstract for the second piece.".to_string());
+
+ Issue {
+ meta: IssueMeta {
+ date: "2026-08-15".parse().expect("fixed date"),
+ issue_number: 42,
+ generated_at: timestamp(),
+ display_date: "Friday, August 15, 2026".into(),
+ article_count: 2,
+ section_count: 2,
+ total_words: 2400,
+ reading_minutes: 11,
+ },
+ lineup: Lineup {
+ date: "2026-08-15".parse().expect("fixed date"),
+ picks: vec![lead, second],
+ section_order: vec!["Top Stories".into(), "Niche Corner".into()],
+ },
+ editorial: Editorial {
+ front_page_html: "<p>Two stories today, both worth your coffee.</p>".into(),
+ section_intros,
+ summaries,
+ },
+ world_briefing: Some(WorldBriefing {
+ date: "2026-08-15".parse().expect("fixed date"),
+ source_url: "https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15".into(),
+ overview: Some("A concise view of the day.".into()),
+ sections: vec![WorldBriefingSection {
+ title: "Top Stories".into(),
+ events: vec![WorldEvent {
+ id: "s1-e1".into(),
+ source_text: "Something happened somewhere.".into(),
+ links: vec![],
+ children: vec![],
+ summary: Some("The event in context.".into()),
+ }],
+ }],
+ }),
+ colophon: Colophon {
+ model: "deepseek-v4-flash".into(),
+ entries_fetched: 431,
+ feeds_seen: 92,
+ candidates: 120,
+ cost_usd: 0.0731,
+ generator_version: "daily-epub 0.1.0".into(),
+ },
+ }
+}
+
+/// A crude XHTML well-formedness check for rendered chapters.
+///
+/// The real proof is `epubcheck`, which cannot run in a unit test; this catches
+/// the mistakes that actually happen — an unclosed void element or a raw
+/// ` `, both of which make an EPUB3 content document unparseable.
+pub fn assert_xml_ok(xhtml: &str) {
+ // A crude well-formedness check: the document parses as XML only if every
+ // tag is closed, so compare open/close counts for the elements we emit.
+ assert!(xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>"));
+ assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\""));
+ assert!(xhtml.trim_end().ends_with("</html>"));
+ for tag in ["html", "head", "body", "div", "p"] {
+ let opens = xhtml.matches(&format!("<{tag}")).count();
+ let closes = xhtml.matches(&format!("</{tag}>")).count();
+ assert_eq!(opens, closes, "unbalanced <{tag}> in\n{xhtml}");
+ }
+ assert!(!xhtml.contains(" "));
+ for void in ["<br>", "<hr>", "<img "] {
+ if void == "<img " {
+ for (i, _) in xhtml.match_indices("<img ") {
+ let tail = &xhtml[i..];
+ let end = tail.find('>').unwrap_or(0);
+ assert!(tail[..end].ends_with('/'), "unclosed <img> in\n{xhtml}");
+ }
+ } else {
+ assert!(!xhtml.contains(void), "unclosed {void} in\n{xhtml}");
+ }
+ }
+}
diff --git a/src/epub/images.rs b/src/epub/images.rs
deleted file mode 100644
index e6a4c11..0000000
--- a/src/epub/images.rs
+++ /dev/null
@@ -1,963 +0,0 @@
-//! Image download and re-encoding (spec §3.10 "Images").
-//!
-//! Failed downloads degrade to a `[image: alt text]` placeholder paragraph — the
-//! run never fails because of an image (notes §3).
-
-use std::collections::HashMap;
-use std::io::Cursor;
-use std::time::Duration;
-
-use futures::StreamExt;
-use image::{DynamicImage, GenericImageView, ImageFormat};
-
-use crate::types::{Edition, ImageAsset, Pick};
-
-/// Per-image download timeout (§3.10).
-pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
-/// Per-image size cap (§3.10).
-pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
-/// Concurrent downloads (§3.10).
-pub const CONCURRENCY: usize = 8;
-/// Whole-issue asset budget (§3.10).
-pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
-/// Images smaller than this in either dimension are decorative — skipped (§3.10).
-pub const MIN_DIMENSION_PX: u32 = 24;
-/// Width/height an SVG is rasterized to when it declares no intrinsic size.
-const SVG_FALLBACK_SIZE: u32 = 1000;
-
-/// HTML void elements: XHTML requires them self-closed (§3.10 "valid XHTML").
-pub const VOID_ELEMENTS: &[&str] = &[
- "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source",
- "track", "wbr",
-];
-
-/// Per-edition re-encoding parameters (§3.10).
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct ImageProfile {
- pub max_width: u32,
- pub max_height: u32,
- pub jpeg_quality: u8,
- pub grayscale: bool,
-}
-
-impl ImageProfile {
- /// Standard edition: max width 1200px, JPEG q80, color (§3.10).
- pub const STANDARD: ImageProfile = ImageProfile {
- max_width: 1200,
- max_height: 4000,
- jpeg_quality: 80,
- grayscale: false,
- };
-
- /// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10).
- pub const X4: ImageProfile = ImageProfile {
- max_width: 480,
- max_height: 800,
- jpeg_quality: 70,
- grayscale: true,
- };
-
- pub fn for_edition(edition: Edition) -> Self {
- match edition {
- Edition::Standard => Self::STANDARD,
- Edition::X4 => Self::X4,
- }
- }
-}
-
-/// One `<img>` found in article markup, with the caption of its `<figure>` if any.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ImgRef {
- pub src: String,
- pub alt: String,
- pub caption: Option<String>,
-}
-
-/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup.
-pub fn extract_img_refs(html: &str) -> Vec<ImgRef> {
- let doc = scraper::Html::parse_fragment(html);
- let Ok(img_sel) = scraper::Selector::parse("img") else {
- return Vec::new();
- };
- let cap_sel = scraper::Selector::parse("figcaption").ok();
-
- let mut out = Vec::new();
- let mut seen: Vec<String> = Vec::new();
- for el in doc.select(&img_sel) {
- let Some(src) = el.value().attr("src") else {
- continue;
- };
- let src = src.trim();
- if src.is_empty() || src.starts_with("data:") {
- continue;
- }
- if seen.iter().any(|s| s == src) {
- continue;
- }
- seen.push(src.to_string());
- let alt = el
- .value()
- .attr("alt")
- .unwrap_or_default()
- .trim()
- .to_string();
- // Walk up to an enclosing <figure> and take its caption, if any.
- let mut caption = None;
- if let Some(cap_sel) = &cap_sel {
- let mut cursor = el.parent();
- while let Some(node) = cursor {
- if let Some(elem) = scraper::ElementRef::wrap(node) {
- if elem.value().name() == "figure" {
- caption = elem.select(cap_sel).next().map(|c| {
- c.text()
- .collect::<String>()
- .split_whitespace()
- .collect::<Vec<_>>()
- .join(" ")
- });
- break;
- }
- cursor = elem.parent();
- } else {
- break;
- }
- }
- }
- out.push(ImgRef {
- src: src.to_string(),
- alt,
- caption: caption.filter(|c| !c.is_empty()),
- });
- }
- out
-}
-
-/// Download one image, honoring the timeout and size cap (§3.10).
-pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
- let resp = http
- .get(url)
- // Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10).
- .header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8")
- .timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
- .send()
- .await
- .map_err(|e| tracing::debug!(url, "image download failed: {e}"))
- .ok()?;
- if !resp.status().is_success() {
- tracing::debug!(url, status = %resp.status(), "image download rejected");
- return None;
- }
- if let Some(len) = resp.content_length()
- && len as usize > MAX_IMAGE_BYTES
- {
- tracing::debug!(url, len, "image exceeds the size cap");
- return None;
- }
- let mut resp = resp;
- let mut buf: Vec<u8> = Vec::new();
- loop {
- match resp.chunk().await {
- Ok(Some(chunk)) => {
- if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
- tracing::debug!(url, "image exceeds the size cap mid-stream");
- return None;
- }
- buf.extend_from_slice(&chunk);
- }
- Ok(None) => break,
- Err(e) => {
- tracing::debug!(url, "image download interrupted: {e}");
- return None;
- }
- }
- }
- if buf.is_empty() { None } else { Some(buf) }
-}
-
-/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10).
-///
-/// Line art with transparency is kept as PNG after flattening; everything else
-/// becomes JPEG. SVG is rasterized first — charts and diagrams are frequently
-/// vector-only, and dropping them loses the point of the article. Returns `None`
-/// for sources no decoder handles.
-pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> {
- if looks_like_svg(bytes) {
- let raster = rasterize_svg(bytes, profile)?;
- return reencode(&raster, profile);
- }
- let format = image::guess_format(bytes).ok();
- let decoded = image::load_from_memory(bytes)
- .map_err(|e| tracing::debug!("undecodable image: {e}"))
- .ok()?;
-
- let (w, h) = decoded.dimensions();
- if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX {
- tracing::debug!(w, h, "skipping decorative image");
- return None;
- }
-
- let has_alpha = decoded.color().has_alpha();
- let flattened = if has_alpha {
- flatten_to_white(&decoded)
- } else {
- decoded
- };
-
- let resized = if w > profile.max_width || h > profile.max_height {
- flattened.resize(
- profile.max_width,
- profile.max_height,
- image::imageops::FilterType::Lanczos3,
- )
- } else {
- flattened
- };
-
- // Keep line art (PNG source, few distinct tones) lossless; everything else
- // becomes JPEG, which is far smaller for photographs (§3.10).
- let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized);
-
- let mut out = Cursor::new(Vec::new());
- // NB: encode the concrete buffer, not the `DynamicImage` — the latter always
- // reports RGBA pixels, which would silently re-colorize a grayscale image.
- if profile.grayscale {
- let gray = resized.to_luma8();
- if keep_png {
- DynamicImage::ImageLuma8(gray)
- .write_to(&mut out, ImageFormat::Png)
- .ok()?;
- return Some((out.into_inner(), "image/png"));
- }
- let mut enc =
- image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
- enc.encode_image(&gray).ok()?;
- return Some((out.into_inner(), "image/jpeg"));
- }
-
- let rgb = resized.to_rgb8();
- if keep_png {
- DynamicImage::ImageRgb8(rgb)
- .write_to(&mut out, ImageFormat::Png)
- .ok()?;
- return Some((out.into_inner(), "image/png"));
- }
- let mut enc =
- image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
- enc.encode_image(&rgb).ok()?;
- Some((out.into_inner(), "image/jpeg"))
-}
-
-/// True when `bytes` are an SVG document (possibly behind an XML prolog or BOM).
-fn looks_like_svg(bytes: &[u8]) -> bool {
- let head = &bytes[..bytes.len().min(1024)];
- let text = String::from_utf8_lossy(head);
- let text = text.trim_start_matches('\u{feff}').trim_start();
- text.starts_with("<svg")
- || (text.starts_with("<?xml") || text.starts_with("<!DOCTYPE svg")) && text.contains("<svg")
-}
-
-/// Rasterize an SVG to a PNG at the profile's target width (§3.10).
-///
-/// The profile's own resize pass then handles the height cap, so this only has
-/// to land in the right ballpark.
-fn rasterize_svg(bytes: &[u8], profile: ImageProfile) -> Option<Vec<u8>> {
- let mut options = resvg::usvg::Options::default();
- options.fontdb_mut().load_system_fonts();
- let tree = resvg::usvg::Tree::from_data(bytes, &options)
- .map_err(|e| tracing::debug!("svg did not parse: {e}"))
- .ok()?;
-
- // An `<svg>` that parses but draws nothing is not an image, it is a stray
- // tag: rasterizing it would embed a blank rectangle.
- if tree.root().children().is_empty() {
- tracing::debug!("svg has nothing to draw");
- return None;
- }
- let size = tree.size();
- let (sw, sh) = (size.width(), size.height());
- if !(sw.is_finite() && sh.is_finite()) || sw <= 0.0 || sh <= 0.0 {
- return None;
- }
- // Judge "decorative" by the declared size, before scaling: a 16×16 icon is
- // an icon however large we choose to draw it.
- if sw < MIN_DIMENSION_PX as f32 || sh < MIN_DIMENSION_PX as f32 {
- tracing::debug!(sw, sh, "skipping decorative svg");
- return None;
- }
- // Vector art has no native resolution, so render straight at the edition's
- // target width — upscaling a rasterized copy afterwards would only blur it.
- let target_w = profile
- .max_width
- .max(SVG_FALLBACK_SIZE.min(profile.max_width));
- let scale = (target_w as f32 / sw).min(profile.max_height as f32 / sh);
- let scale = if scale.is_finite() && scale > 0.0 {
- scale
- } else {
- 1.0
- };
- let (w, h) = (
- (sw * scale).round().max(1.0) as u32,
- (sh * scale).round().max(1.0) as u32,
- );
- let mut pixmap = tiny_skia::Pixmap::new(w, h)?;
- // E-ink has no transparency; render onto white so alpha never becomes black.
- pixmap.fill(tiny_skia::Color::WHITE);
- resvg::render(
- &tree,
- tiny_skia::Transform::from_scale(scale, scale),
- &mut pixmap.as_mut(),
- );
- let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())?;
- let mut png = Cursor::new(Vec::new());
- DynamicImage::ImageRgba8(rgba)
- .write_to(&mut png, ImageFormat::Png)
- .ok()?;
- Some(png.into_inner())
-}
-
-/// Composite over an opaque white page — e-ink has no transparency (§3.10).
-fn flatten_to_white(img: &DynamicImage) -> DynamicImage {
- let rgba = img.to_rgba8();
- let mut rgb = image::RgbImage::new(rgba.width(), rgba.height());
- for (x, y, px) in rgba.enumerate_pixels() {
- let a = f32::from(px[3]) / 255.0;
- let blend = |c: u8| {
- ((f32::from(c) * a) + 255.0 * (1.0 - a))
- .round()
- .clamp(0.0, 255.0) as u8
- };
- rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])]));
- }
- DynamicImage::ImageRgb8(rgb)
-}
-
-/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text).
-fn is_line_art(img: &DynamicImage) -> bool {
- const SAMPLE_LIMIT: usize = 20_000;
- const DISTINCT_LIMIT: usize = 64;
- let rgb = img.to_rgb8();
- let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1);
- for (i, px) in rgb.pixels().enumerate() {
- if i >= SAMPLE_LIMIT {
- break;
- }
- let c = [px[0], px[1], px[2]];
- if !distinct.contains(&c) {
- distinct.push(c);
- if distinct.len() > DISTINCT_LIMIT {
- return false;
- }
- }
- }
- true
-}
-
-/// Everything needed to fetch one image, in deterministic issue order.
-#[derive(Debug, Clone)]
-struct PendingImage {
- id: String,
- url: String,
- alt: String,
- caption: Option<String>,
-}
-
-fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> {
- let entry_id = pick.article.best_entry_id;
- let mut refs = extract_img_refs(&pick.article.content_html);
- if refs.is_empty() {
- refs = pick
- .article
- .image_urls
- .iter()
- .map(|u| ImgRef {
- src: u.clone(),
- alt: String::new(),
- caption: None,
- })
- .collect();
- }
- refs.into_iter()
- .filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://"))
- .enumerate()
- .map(|(i, r)| PendingImage {
- id: format!("img-{entry_id}-{i}"),
- url: r.src,
- alt: r.alt,
- caption: r.caption,
- })
- .collect()
-}
-
-/// Download and re-encode every image referenced by the lineup for one edition,
-/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10).
-pub async fn collect_for_issue(
- http: &reqwest::Client,
- picks: &[Pick],
- edition: Edition,
-) -> Vec<ImageAsset> {
- let profile = ImageProfile::for_edition(edition);
- let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect();
- if pending.is_empty() {
- return Vec::new();
- }
- tracing::info!(count = pending.len(), ?edition, "downloading issue images");
-
- let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> =
- futures::stream::iter(pending.into_iter().map(|p| {
- let http = http.clone();
- async move {
- let raw = download(&http, &p.url).await?;
- let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile))
- .await
- .ok()
- .flatten()?;
- Some((p, bytes, mime))
- }
- }))
- .buffered(CONCURRENCY)
- .collect()
- .await;
-
- let mut assets = Vec::new();
- let mut budget_used = 0usize;
- let mut skipped = 0usize;
- for result in results.into_iter().flatten() {
- let (pending, bytes, mime) = result;
- if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES {
- skipped += 1;
- continue;
- }
- budget_used += bytes.len();
- let ext = if mime == "image/png" { "png" } else { "jpg" };
- assets.push(ImageAsset {
- href: format!("images/{}.{ext}", pending.id),
- id: pending.id,
- mime: mime.to_string(),
- data: bytes,
- alt: pending.alt,
- caption: pending.caption,
- source_url: pending.url,
- });
- }
- if skipped > 0 {
- tracing::warn!(skipped, budget_used, "issue image budget exhausted");
- }
- tracing::info!(
- embedded = assets.len(),
- bytes = budget_used,
- "issue images ready"
- );
- assets
-}
-
-// ---------------------------------------------------------------------------
-// Markup rewriting
-// ---------------------------------------------------------------------------
-
-/// End index (exclusive) of the tag starting at `start` (`html[start] == '<'`),
-/// respecting quoted attribute values and comments.
-pub(crate) fn tag_end(html: &str, start: usize) -> Option<usize> {
- let rest = &html[start..];
- if rest.starts_with("<!--") {
- return rest.find("-->").map(|i| start + i + 3);
- }
- let mut quote: Option<char> = None;
- for (i, c) in rest.char_indices().skip(1) {
- match (quote, c) {
- (Some(q), c) if c == q => quote = None,
- (Some(_), _) => {}
- (None, '"') | (None, '\'') => quote = Some(c),
- (None, '>') => return Some(start + i + c.len_utf8()),
- (None, _) => {}
- }
- }
- None
-}
-
-/// Lowercased element name of a tag body such as `img src="…"`.
-pub(crate) fn tag_name(inner: &str) -> String {
- inner
- .trim_start_matches('/')
- .split(|c: char| c.is_whitespace() || c == '/' || c == '>')
- .next()
- .unwrap_or("")
- .to_ascii_lowercase()
-}
-
-/// Parse `name="value"` pairs out of a tag body, with entity-decoded values.
-///
-/// Decoding matters: this scanner reads markup that ammonia has serialized, and
-/// ammonia writes `&` in a URL as `&`. A raw comparison against a URL that
-/// came out of a real HTML parser would never match (§3.10).
-pub(crate) fn parse_attrs(inner: &str) -> Vec<(String, String)> {
- let mut attrs = Vec::new();
- let bytes: Vec<char> = inner.chars().collect();
- let mut i = 0;
- // Skip the element name.
- while i < bytes.len() && !bytes[i].is_whitespace() {
- i += 1;
- }
- while i < bytes.len() {
- while i < bytes.len() && (bytes[i].is_whitespace() || bytes[i] == '/') {
- i += 1;
- }
- let name_start = i;
- while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '=' && bytes[i] != '/' {
- i += 1;
- }
- if i == name_start {
- break;
- }
- let name: String = bytes[name_start..i]
- .iter()
- .collect::<String>()
- .to_ascii_lowercase();
- while i < bytes.len() && bytes[i].is_whitespace() {
- i += 1;
- }
- let mut value = String::new();
- if i < bytes.len() && bytes[i] == '=' {
- i += 1;
- while i < bytes.len() && bytes[i].is_whitespace() {
- i += 1;
- }
- if i < bytes.len() && (bytes[i] == '"' || bytes[i] == '\'') {
- let quote = bytes[i];
- i += 1;
- while i < bytes.len() && bytes[i] != quote {
- value.push(bytes[i]);
- i += 1;
- }
- i += 1;
- } else {
- while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '>' {
- value.push(bytes[i]);
- i += 1;
- }
- }
- }
- attrs.push((name, decode_entities(&value)));
- }
- attrs
-}
-
-/// Decode the handful of entities an HTML serializer emits inside attributes.
-///
-/// Numeric forms are included because feeds and WordPress write `&` for
-/// `&`; anything else is left alone rather than guessed at.
-pub(crate) fn decode_entities(s: &str) -> String {
- if !s.contains('&') {
- return s.to_string();
- }
- let mut out = String::with_capacity(s.len());
- let mut rest = s;
- while let Some(i) = rest.find('&') {
- out.push_str(&rest[..i]);
- let tail = &rest[i..];
- let Some(end) = tail[..tail.len().min(12)].find(';') else {
- out.push('&');
- rest = &tail[1..];
- continue;
- };
- let entity = &tail[1..end];
- let decoded = match entity {
- "amp" => Some('&'),
- "lt" => Some('<'),
- "gt" => Some('>'),
- "quot" => Some('"'),
- "apos" | "#39" => Some('\''),
- "nbsp" => Some('\u{a0}'),
- _ => entity
- .strip_prefix('#')
- .and_then(|n| match n.strip_prefix(['x', 'X']) {
- Some(hex) => u32::from_str_radix(hex, 16).ok(),
- None => n.parse::<u32>().ok(),
- })
- .and_then(char::from_u32),
- };
- match decoded {
- Some(c) => {
- out.push(c);
- rest = &tail[end + 1..];
- }
- None => {
- out.push('&');
- rest = &tail[1..];
- }
- }
- }
- out.push_str(rest);
- out
-}
-
-/// Escape a string for use inside a double-quoted XML attribute.
-fn attr_escape(s: &str) -> String {
- let mut out = String::with_capacity(s.len());
- for c in s.chars() {
- match c {
- '&' => out.push_str("&"),
- '<' => out.push_str("<"),
- '>' => out.push_str(">"),
- '"' => out.push_str("""),
- _ => out.push(c),
- }
- }
- out
-}
-
-/// Escape a string for XML text content.
-pub fn text_escape(s: &str) -> String {
- let mut out = String::with_capacity(s.len());
- for c in s.chars() {
- match c {
- '&' => out.push_str("&"),
- '<' => out.push_str("<"),
- '>' => out.push_str(">"),
- _ => out.push(c),
- }
- }
- out
-}
-
-/// Whether an image we could not embed is worth telling the reader about.
-///
-/// Only descriptive alt text qualifies. A filename, a bare label like `red line`
-/// on a divider rule, or no alt at all carries nothing the reader loses by not
-/// seeing the picture — announcing those turns every decorative graphic and
-/// dead link into a line of clutter, which is how the placeholders got out of
-/// hand in the first place.
-fn alt_is_worth_announcing(alt: &str) -> bool {
- const MIN_DESCRIPTIVE_WORDS: usize = 4;
- !alt.is_empty()
- && !is_filename_alt(alt)
- && alt.split_whitespace().count() >= MIN_DESCRIPTIVE_WORDS
-}
-
-/// True for alt text that is really just the uploaded filename — `IMG_0808.JPG`,
-/// `cut pieces v01.JPG`, `chart-final-2.png`.
-fn is_filename_alt(alt: &str) -> bool {
- let alt = alt.trim();
- if alt.contains(' ') && alt.split_whitespace().count() > 4 {
- return false;
- }
- let Some((stem, ext)) = alt.rsplit_once('.') else {
- return false;
- };
- let ext = ext.to_ascii_lowercase();
- matches!(
- ext.as_str(),
- "jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" | "avif" | "bmp" | "heic"
- ) && !stem.is_empty()
- && stem
- .chars()
- .all(|c| c.is_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.'))
-}
-
-/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the
-/// `[image: alt]` placeholder paragraph (§3.10).
-pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String {
- let by_url: HashMap<&str, &ImageAsset> =
- assets.iter().map(|a| (a.source_url.as_str(), a)).collect();
- let mut out = String::with_capacity(html.len());
- let mut cursor = 0usize;
- while let Some(rel) = html[cursor..].find('<') {
- let start = cursor + rel;
- out.push_str(&html[cursor..start]);
- let Some(end) = tag_end(html, start) else {
- out.push_str(&html[start..]);
- return out;
- };
- let raw = &html[start..end];
- let inner = raw
- .trim_start_matches('<')
- .trim_end_matches('>')
- .trim_end_matches('/');
- if tag_name(inner) == "img" {
- let attrs = parse_attrs(inner);
- let src = attrs
- .iter()
- .find(|(k, _)| k == "src")
- .map(|(_, v)| v.trim().to_string())
- .unwrap_or_default();
- let alt = attrs
- .iter()
- .find(|(k, _)| k == "alt")
- .map(|(_, v)| v.trim().to_string())
- .unwrap_or_default();
- match by_url.get(src.as_str()) {
- Some(asset) => {
- let alt = if alt.is_empty() { &asset.alt } else { &alt };
- out.push_str(&format!(
- "<img src=\"{}\" alt=\"{}\"/>",
- attr_escape(&asset.href),
- attr_escape(alt)
- ));
- }
- // An image we could not embed is only worth announcing when its
- // alt text tells the reader something; otherwise the `<img>`
- // just goes away. That covers the decorative graphics the
- // re-encoder deliberately skips as well as genuine misses (§3.10).
- None if !alt_is_worth_announcing(&alt) => {}
- None => {
- out.push_str(&format!(
- "<p class=\"image-placeholder\">[image: {}]</p>",
- text_escape(&alt)
- ));
- }
- }
- } else {
- out.push_str(raw);
- }
- cursor = end;
- }
- out.push_str(&html[cursor..]);
- out
-}
-
-/// Self-close HTML void elements and normalize ` ` so the markup parses as
-/// XML — EPUB3 content documents are XHTML (§3.10).
-pub fn to_xhtml(html: &str) -> String {
- let mut out = String::with_capacity(html.len());
- let mut cursor = 0usize;
- while let Some(rel) = html[cursor..].find('<') {
- let start = cursor + rel;
- out.push_str(&html[cursor..start]);
- let Some(end) = tag_end(html, start) else {
- out.push_str(&html[start..]);
- cursor = html.len();
- break;
- };
- let raw = &html[start..end];
- let inner = raw.trim_start_matches('<').trim_end_matches('>');
- let name = tag_name(inner);
- if VOID_ELEMENTS.contains(&name.as_str()) && !inner.trim_end().ends_with('/') {
- out.push('<');
- out.push_str(inner.trim_end());
- out.push_str("/>");
- } else {
- out.push_str(raw);
- }
- cursor = end;
- }
- out.push_str(&html[cursor..]);
- // html5ever (via ammonia) emits ` `, which is undefined in XML.
- out.replace(" ", " ")
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn asset(url: &str, href: &str) -> ImageAsset {
- ImageAsset {
- id: "img-1-0".into(),
- href: href.into(),
- mime: "image/jpeg".into(),
- data: vec![1, 2, 3],
- alt: "fallback alt".into(),
- caption: None,
- source_url: url.into(),
- }
- }
-
- #[test]
- fn extracts_img_refs_with_captions() {
- let html = r#"<p>hi</p>
- <figure><img src="https://e.g/a.png" alt="A diagram"/>
- <figcaption>Figure 1: the thing</figcaption></figure>
- <img src="https://e.g/b.jpg"/>
- <img src="data:image/png;base64,zz"/>
- <img src="https://e.g/a.png" alt="dupe"/>"#;
- let refs = extract_img_refs(html);
- assert_eq!(refs.len(), 2);
- assert_eq!(refs[0].src, "https://e.g/a.png");
- assert_eq!(refs[0].alt, "A diagram");
- assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing"));
- assert_eq!(refs[1].src, "https://e.g/b.jpg");
- assert!(refs[1].caption.is_none());
- }
-
- #[test]
- fn rewrites_hits_and_placeholders_misses() {
- let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")];
- let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="A chart of missing things">"#;
- let out = rewrite_img_srcs(html, &assets);
- // The alt round-trips through one level of escaping, not two.
- assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt & more"/>"#));
- assert!(
- out.contains(r#"<p class="image-placeholder">[image: A chart of missing things]</p>"#)
- );
- assert!(!out.contains("gone.png"));
- }
-
- /// The whole point of the fix: ammonia writes `&` into the markup, and
- /// the asset was keyed on the URL a real parser produced.
- #[test]
- fn entity_encoded_urls_still_match_their_asset() {
- let assets = vec![asset(
- "https://e.g/a.jpg?id=1&width=980",
- "images/img-1-0.jpg",
- )];
- let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
- assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
- // The numeric spelling WordPress emits works too.
- let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
- assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
- }
-
- #[test]
- fn unembeddable_images_only_speak_up_when_the_alt_says_something() {
- // No alt at all: the image simply disappears.
- assert_eq!(
- rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]),
- ""
- );
- // A filename is not a description.
- assert_eq!(
- rewrite_img_srcs(r#"<img src="https://e.g/x.png" alt="IMG_0808.JPG">"#, &[]),
- ""
- );
- // Neither is the label on a decorative divider rule.
- assert_eq!(
- rewrite_img_srcs(r#"<img src="https://e.g/rule.png" alt="red line">"#, &[]),
- ""
- );
- // A real description is worth keeping.
- assert!(
- rewrite_img_srcs(
- r#"<img src="https://e.g/x.png" alt="A man in a hard hat stands over a well hole">"#,
- &[]
- )
- .contains("[image: A man in a hard hat stands over a well hole]")
- );
- }
-
- #[test]
- fn filename_alt_detection() {
- for yes in [
- "IMG_0808.JPG",
- "cut pieces v01.JPG",
- "chart-final-2.png",
- "diagram.svg",
- ] {
- assert!(is_filename_alt(yes), "{yes} should read as a filename");
- }
- for no in [
- "A hydrogen well head",
- "",
- "Fig. 3",
- "The lion-man of Hohlenstein-Stadel, carved from mammoth ivory.",
- ] {
- assert!(!is_filename_alt(no), "{no} should read as a description");
- }
- }
-
- #[test]
- fn to_xhtml_self_closes_voids_and_entities() {
- let html = "<p>a<br>b<hr>c d<img src=\"x.png\" alt=\"y\"></p><p>e<br/></p>";
- let out = to_xhtml(html);
- assert!(out.contains("<br/>"));
- assert!(out.contains("<hr/>"));
- assert!(out.contains("<img src=\"x.png\" alt=\"y\"/>"));
- assert!(out.contains(" "));
- assert!(!out.contains(" "));
- assert!(!out.contains("<br/ >"));
- // Already-closed voids are left alone (no double slash).
- assert_eq!(out.matches("<br/>").count(), 2);
- }
-
- #[test]
- fn tag_scanner_ignores_angle_brackets_in_attributes() {
- let html = r#"<a title="a > b">x</a>"#;
- assert_eq!(to_xhtml(html), html);
- }
-
- #[test]
- fn reencode_resizes_grayscales_and_encodes() {
- let mut img = image::RgbaImage::new(200, 100);
- for (x, y, px) in img.enumerate_pixels_mut() {
- *px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]);
- }
- let mut png = Cursor::new(Vec::new());
- DynamicImage::ImageRgba8(img)
- .write_to(&mut png, ImageFormat::Png)
- .unwrap();
- let raw = png.into_inner();
-
- let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap();
- assert_eq!(std_mime, "image/jpeg");
- let decoded = image::load_from_memory(&std_bytes).unwrap();
- assert_eq!(decoded.dimensions(), (200, 100), "no upscaling");
-
- let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap();
- let x4 = image::load_from_memory(&x4_bytes).unwrap();
- assert!(x4.width() <= 480 && x4.height() <= 800);
- assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8");
- }
-
- #[test]
- fn reencode_skips_decorative_images_and_junk() {
- let tiny = image::RgbaImage::new(8, 8);
- let mut png = Cursor::new(Vec::new());
- DynamicImage::ImageRgba8(tiny)
- .write_to(&mut png, ImageFormat::Png)
- .unwrap();
- assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none());
- // A bare `<svg>` tag with nothing to draw is markup, not a picture.
- assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none());
- assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none());
- }
-
- #[test]
- fn svg_charts_are_rasterized_rather_than_dropped() {
- let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300">
- <rect x="10" y="10" width="380" height="280" fill="#3355bb"/>
- <circle cx="200" cy="150" r="60" fill="#ffcc00"/>
- </svg>"##;
- let (bytes, mime) = reencode(svg, ImageProfile::STANDARD).expect("svg rasterizes");
- assert_eq!(mime, "image/png", "flat colour art stays lossless");
- let decoded = image::load_from_memory(&bytes).expect("decodable output");
- // Drawn at the edition's target width, not at the SVG's nominal size.
- assert_eq!(decoded.dimensions(), (1200, 900));
- assert!(!decoded.color().has_alpha(), "rendered onto white");
-
- // An XML prolog and a leading BOM must not hide the format.
- let with_prolog = format!(
- "\u{feff}<?xml version=\"1.0\"?>{}",
- String::from_utf8_lossy(svg)
- );
- let (x4, _) = reencode(with_prolog.as_bytes(), ImageProfile::X4).expect("x4 rasterizes");
- let x4 = image::load_from_memory(&x4).unwrap();
- assert!(x4.width() <= 480 && x4.height() <= 800);
-
- // A 16×16 icon is decorative however large we could draw it.
- let icon = br#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
- <rect width="16" height="16"/></svg>"#;
- assert!(reencode(icon, ImageProfile::STANDARD).is_none());
- }
-
- #[test]
- fn line_art_png_stays_png_and_is_flattened() {
- let mut img = image::RgbaImage::new(120, 60);
- for (x, _y, px) in img.enumerate_pixels_mut() {
- *px = if x % 12 == 0 {
- image::Rgba([0, 0, 0, 255])
- } else {
- image::Rgba([255, 255, 255, 0])
- };
- }
- let mut png = Cursor::new(Vec::new());
- DynamicImage::ImageRgba8(img)
- .write_to(&mut png, ImageFormat::Png)
- .unwrap();
- let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap();
- assert_eq!(mime, "image/png");
- let decoded = image::load_from_memory(&bytes).unwrap();
- assert!(!decoded.color().has_alpha(), "transparency is flattened");
- // Transparent pixels became white.
- assert_eq!(
- decoded.to_rgb8().get_pixel(1, 1),
- &image::Rgb([255, 255, 255])
- );
- }
-}
diff --git a/src/epub/mod.rs b/src/epub/mod.rs
index 24a88e5..82805e8 100644
--- a/src/epub/mod.rs
+++ b/src/epub/mod.rs
@@ -5,12 +5,15 @@
//! `art-{entry_id}` so rating links stay stable across regenerations.
pub mod build;
-pub mod images;
+pub mod chapters;
+pub mod cover;
+pub mod fixtures;
pub mod x4;
use std::path::{Path, PathBuf};
use crate::config::Config;
+use crate::images;
use crate::types::{Artifact, Edition, ImageAsset, Issue};
/// Chapter order inside an issue (§3.10).
diff --git a/src/epub/x4.rs b/src/epub/x4.rs
index 390e4f7..e0e936a 100644
--- a/src/epub/x4.rs
+++ b/src/epub/x4.rs
@@ -10,7 +10,7 @@ use std::path::{Path, PathBuf};
use crate::config::XtcConfig;
-use super::images::tag_end;
+use crate::html::tag_end;
/// Native X4 screen size, used for the cover and image fitting (§3.10).
pub const X4_SCREEN: (u32, u32) = (480, 800);
diff --git a/src/extract.rs b/src/extract.rs
index 894398a..4f84359 100644
--- a/src/extract.rs
+++ b/src/extract.rs
@@ -1,17 +1,23 @@
-//! Content extraction, sanitization and word counting (spec §3.3).
+//! Getting an article's body text (spec §3.3).
//!
//! Priority order per article: Miniflux content if it looks like full text →
//! fetch + `dom_smoothie` readability → feed excerpt with a "(excerpt only)" note.
+//! Whichever wins is sanitized down to the tag subset the EPUB templates accept.
+//!
+//! Image handling lives in [`crate::images`]; this module calls into
+//! [`crate::images::normalize`] before readability and before sanitizing, and
+//! reads image URLs back out with [`crate::images::collect_image_urls`].
use std::collections::HashSet;
use std::sync::Arc;
use std::time::Duration;
use futures::StreamExt;
-use scraper::{Html, Node, Selector};
use url::Url;
-use crate::epub::images::{tag_end, tag_name};
+use crate::html::word_count;
+use crate::images::normalize::{normalize_img_tags, prepare_for_readability};
+use crate::images::refs::collect_image_urls;
use crate::types::{Article, ExtractMethod, Extracted};
/// Word count at or above which Miniflux content is treated as full text (§3.3).
@@ -343,303 +349,6 @@ fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
domains
}
-// ---------------------------------------------------------------------------
-// Image normalization (§3.3)
-// ---------------------------------------------------------------------------
-
-/// Elements that readability deletes outright, and which a page may nevertheless
-/// have wrapped around an image (lightbox triggers, mostly).
-const IMAGE_WRAPPER_TAGS: &[&str] = &["button", "form", "fieldset", "object"];
-
-/// Attributes lazy-loading libraries use for the real image URL.
-///
-/// These outrank `src`, because a page only sets them when `src` is a stand-in:
-/// a transparent GIF, a blurred thumbnail, an inline SVG spacer. Attributes that
-/// merely *look* image-ish (`data-template`, `data-attrs`, `data-orig-file`) are
-/// deliberately absent — those hold templates and metadata, and preferring them
-/// is exactly the mistake readability's own heuristic makes.
-const LAZY_SRC_ATTRS: &[&str] = &[
- "data-src",
- "data-lazy-src",
- "data-original",
- "data-runner-src",
- "data-full-src",
- "data-hi-res-src",
- "data-image-src",
-];
-
-/// Widest `srcset` candidate we will pick; above this we are downloading pixels
-/// the re-encoder immediately throws away.
-const MAX_SRCSET_WIDTH: u32 = 2000;
-
-/// Make a fetched page safe to hand to readability (§3.3).
-///
-/// Two passes, both about images: unwrap the elements that would take an image
-/// with them when readability deletes them, then reduce every `<img>` to a plain
-/// `src`/`alt`/`title` triple. The second pass is what stops readability's own
-/// lazy-image heuristic from replacing a working `src` — with no `srcset`,
-/// `loading` or `data-*` attributes left on the element, it has nothing to
-/// substitute and leaves the image alone.
-pub fn prepare_for_readability(html: &str) -> String {
- normalize_img_tags(&unwrap_image_wrappers(html))
-}
-
-/// Replace image-only `<button>`/`<form>`/`<fieldset>`/`<object>` wrappers with
-/// their contents (§3.3).
-///
-/// A `<button>` holding nothing but an image is a lightbox trigger, not a
-/// control: the image is the content. Wrappers that also carry text are left
-/// alone, because those really are interface.
-pub fn unwrap_image_wrappers(html: &str) -> String {
- let mut out = String::with_capacity(html.len());
- let mut cursor = 0usize;
- while let Some(rel) = html[cursor..].find('<') {
- let start = cursor + rel;
- out.push_str(&html[cursor..start]);
- let Some(end) = tag_end(html, start) else {
- out.push_str(&html[start..]);
- return out;
- };
- let raw = &html[start..end];
- let inner = raw.trim_start_matches('<').trim_end_matches('>');
- let name = tag_name(inner);
-
- if IMAGE_WRAPPER_TAGS.contains(&name.as_str())
- && !inner.trim_end().ends_with('/')
- && let Some((content, after)) = element_content(html, end, &name)
- && content.contains("<img")
- && html_to_text(content).trim().is_empty()
- {
- out.push_str(&unwrap_image_wrappers(content));
- cursor = after;
- continue;
- }
-
- out.push_str(raw);
- cursor = end;
- }
- out.push_str(&html[cursor..]);
- out
-}
-
-/// The content of an element whose open tag ended at `body_start`, plus the
-/// offset just past its close tag. `None` when the element is never closed.
-fn element_content<'a>(html: &'a str, body_start: usize, name: &str) -> Option<(&'a str, usize)> {
- let open = format!("<{name}");
- let close = format!("</{name}");
- let mut depth = 1usize;
- let mut cursor = body_start;
- while let Some(rel) = html[cursor..].find('<') {
- let start = cursor + rel;
- let end = tag_end(html, start)?;
- let tag = &html[start..end];
- let lower = tag.to_ascii_lowercase();
- if lower.starts_with(&close) {
- depth -= 1;
- if depth == 0 {
- return Some((&html[body_start..start], end));
- }
- } else if lower.starts_with(&open)
- && !lower[open.len()..].starts_with(|c: char| c.is_alphanumeric() || c == '-')
- && !tag.trim_end().ends_with("/>")
- {
- depth += 1;
- }
- cursor = end;
- }
- None
-}
-
-/// Reduce every `<img>` to `<img src alt title>` with a usable URL (§3.3).
-///
-/// The `src` a page ships is not automatically the one to use: it can be a lazy
-/// placeholder (`data:image/svg+xml,…`), an unresolved template
-/// (`…/resize/{width}/…`), a JSON blob a framework parked there, or an entire
-/// `srcset` string. Candidates are tried in order and the first plausible one
-/// wins; an image with no plausible candidate is dropped, because a broken
-/// `<img>` only becomes clutter further down the pipeline.
-pub fn normalize_img_tags(html: &str) -> String {
- let mut out = String::with_capacity(html.len());
- let mut cursor = 0usize;
- // `<picture>` puts the real candidates on sibling `<source>` elements.
- let mut picture_srcset: Option<String> = None;
-
- while let Some(rel) = html[cursor..].find('<') {
- let start = cursor + rel;
- out.push_str(&html[cursor..start]);
- let Some(end) = tag_end(html, start) else {
- out.push_str(&html[start..]);
- return out;
- };
- let raw = &html[start..end];
- let inner = raw
- .trim_start_matches('<')
- .trim_end_matches('>')
- .trim_end_matches('/');
- let name = tag_name(inner);
-
- match name.as_str() {
- "picture" => {
- picture_srcset = None;
- out.push_str(raw);
- }
- "source" => {
- let attrs = crate::epub::images::parse_attrs(inner);
- if picture_srcset.is_none()
- && let Some(set) =
- attr(&attrs, "srcset").or_else(|| attr(&attrs, "data-srcset"))
- {
- picture_srcset = Some(set.to_string());
- }
- out.push_str(raw);
- }
- "img" => {
- let attrs = crate::epub::images::parse_attrs(inner);
- if let Some(src) = best_img_src(&attrs, picture_srcset.as_deref()) {
- out.push_str("<img src=\"");
- out.push_str(&escape_attr(&src));
- out.push('"');
- for key in ["alt", "title"] {
- if let Some(v) = attr(&attrs, key) {
- out.push(' ');
- out.push_str(key);
- out.push_str("=\"");
- out.push_str(&escape_attr(v));
- out.push('"');
- }
- }
- out.push_str("/>");
- } else {
- tracing::debug!(tag = %&raw[..raw.len().min(120)], "dropping unusable img");
- }
- }
- _ => out.push_str(raw),
- }
- if name == "picture" && inner.starts_with('/') {
- picture_srcset = None;
- }
- cursor = end;
- }
- out.push_str(&html[cursor..]);
- out
-}
-
-fn attr<'a>(attrs: &'a [(String, String)], name: &str) -> Option<&'a str> {
- attrs
- .iter()
- .find(|(k, _)| k == name)
- .map(|(_, v)| v.trim())
- .filter(|v| !v.is_empty())
-}
-
-/// Pick the best URL for one `<img>` from everything the element carries.
-fn best_img_src(attrs: &[(String, String)], picture_srcset: Option<&str>) -> Option<String> {
- for key in LAZY_SRC_ATTRS {
- if let Some(v) = attr(attrs, key).filter(|s| plausible_url(s)) {
- return Some(v.to_string());
- }
- }
- if let Some(src) = attr(attrs, "src").filter(|s| plausible_url(s)) {
- return Some(src.to_string());
- }
- if let Some(from_set) = attr(attrs, "srcset").and_then(best_from_srcset) {
- return Some(from_set);
- }
- if let Some(from_set) = attr(attrs, "data-srcset").and_then(best_from_srcset) {
- return Some(from_set);
- }
- picture_srcset.and_then(best_from_srcset)
-}
-
-/// The widest candidate in a `srcset` that is still worth downloading.
-///
-/// Parsed by whitespace rather than by comma: the URLs of several image CDNs
-/// contain commas of their own, and splitting on those shreds them.
-fn best_from_srcset(srcset: &str) -> Option<String> {
- let mut best: Option<(u32, String)> = None;
- let mut smallest: Option<(u32, String)> = None;
- let mut pending: Option<String> = None;
-
- let mut consider = |url: String, width: u32| {
- if width <= MAX_SRCSET_WIDTH && best.as_ref().is_none_or(|(w, _)| width > *w) {
- best = Some((width, url.clone()));
- }
- if smallest.as_ref().is_none_or(|(w, _)| width < *w) {
- smallest = Some((width, url));
- }
- };
-
- for token in srcset.split_whitespace() {
- let token = token.trim_end_matches(',');
- if token.is_empty() {
- continue;
- }
- match parse_descriptor(token) {
- Some(width) => {
- if let Some(url) = pending.take() {
- consider(url, width);
- }
- }
- None => {
- // A URL with no descriptor of its own still counts, at width 1x.
- if let Some(url) = pending.replace(token.to_string()) {
- consider(url, 1);
- }
- }
- }
- }
- if let Some(url) = pending.take() {
- consider(url, 1);
- }
-
- best.or(smallest)
- .map(|(_, url)| url)
- .filter(|u| plausible_url(u))
-}
-
-/// `800w` → 800, `2x` → a synthetic width so density candidates sort sensibly.
-fn parse_descriptor(token: &str) -> Option<u32> {
- let (value, unit) = token.split_at(token.len().checked_sub(1)?);
- match unit {
- "w" => value.parse::<u32>().ok(),
- "x" => value
- .parse::<f32>()
- .ok()
- .map(|d| (d * 1000.0).round().clamp(1.0, f32::from(u16::MAX)) as u32),
- _ => None,
- }
-}
-
-/// Whether a string can serve as an image URL at all.
-///
-/// This is deliberately about shape, not about the host: whitespace, braces and
-/// quotes mean we are looking at a `srcset` blob, an unfilled URL template or a
-/// serialized object, none of which will ever resolve.
-fn plausible_url(candidate: &str) -> bool {
- let candidate = candidate.trim();
- if candidate.is_empty() || candidate.len() > 2048 {
- return false;
- }
- if candidate.starts_with("data:") || candidate.starts_with("about:") {
- return false;
- }
- if candidate
- .chars()
- .any(|c| c.is_whitespace() || matches!(c, '{' | '}' | '"' | '\'' | '<' | '>' | '\\'))
- {
- return false;
- }
- // `%20` is a space that survived encoding — same blob, different spelling.
- !candidate.contains("%20")
-}
-
-fn escape_attr(s: &str) -> String {
- s.replace('&', "&")
- .replace('<', "<")
- .replace('>', ">")
- .replace('"', """)
-}
-
// ---------------------------------------------------------------------------
// Sanitization (§3.3)
// ---------------------------------------------------------------------------
@@ -711,74 +420,6 @@ pub fn sanitize_with_base(html: &str, base_url: &str) -> String {
}
}
-// ---------------------------------------------------------------------------
-// Text measurement (§3.3)
-// ---------------------------------------------------------------------------
-
-/// Visible text of an HTML fragment, entities decoded, `script`/`style` skipped.
-pub fn html_to_text(html: &str) -> String {
- let document = Html::parse_fragment(html);
- let mut out = String::with_capacity(html.len() / 2);
- for node in document.tree.nodes() {
- let Node::Text(text) = node.value() else {
- continue;
- };
- let hidden = node.ancestors().any(|a| match a.value() {
- Node::Element(e) => matches!(e.name(), "script" | "style" | "noscript"),
- _ => false,
- });
- if hidden {
- continue;
- }
- out.push_str(text);
- out.push(' ');
- }
- out
-}
-
-/// Count words in rendered text (tags stripped) (§3.3).
-pub fn word_count(html: &str) -> i64 {
- html_to_text(html)
- .split_whitespace()
- .filter(|w| w.chars().any(char::is_alphanumeric))
- .count() as i64
-}
-
-/// Absolute image URLs referenced by `html`, resolved against `base_url` (§3.3).
-///
-/// Every image an article carries is kept: a photo essay with thirty pictures is
-/// a photo essay, and the issue-wide byte budget is the real backstop.
-pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> {
- let Ok(selector) = Selector::parse("img") else {
- return Vec::new();
- };
- let base = Url::parse(base_url).ok();
- let document = Html::parse_fragment(html);
- let mut seen: HashSet<String> = HashSet::new();
- let mut out: Vec<String> = Vec::new();
- for element in document.select(&selector) {
- let raw = element
- .value()
- .attr("src")
- .or_else(|| element.value().attr("data-src"))
- .map(str::trim)
- .filter(|s| !s.is_empty());
- let Some(raw) = raw else { continue };
- let resolved = match Url::parse(raw) {
- Ok(u) => Some(u),
- Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()),
- };
- let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else {
- continue;
- };
- let url = url.to_string();
- if seen.insert(url.clone()) {
- out.push(url);
- }
- }
- out
-}
-
/// Heuristic paywall detection: very short text on a known paywall domain (§3.3).
///
/// Two rules: anything under [`EXCERPT_MAX_WORDS`] is a stub whatever the host,
@@ -927,238 +568,10 @@ mod tests {
assert!(sanitize_with_base(html, "not a url").contains("/next"));
}
- #[test]
- fn word_count_ignores_markup_and_script() {
- assert_eq!(word_count("<p>one two three</p>"), 3);
- assert_eq!(word_count("<p>a</p><script>b c d e</script>"), 1);
- assert_eq!(word_count("<p>& — ok</p>"), 1);
- assert_eq!(word_count(""), 0);
- assert_eq!(word_count("<p></p>"), 0);
- }
-
- #[test]
- fn image_collection_resolves_and_keeps_every_image() {
- let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#);
- html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#);
- for i in 0..20 {
- html.push_str(&format!(r#"<img src="/n{i}.png">"#));
- }
- let urls = collect_image_urls(&html, "https://blog.dev/posts/one");
- // Two named images, the data-src one, and all twenty of the rest: no cap.
- assert_eq!(urls.len(), 23);
- assert_eq!(urls[0], "https://blog.dev/a.png");
- assert_eq!(urls[1], "https://cdn.dev/b.png");
- assert_eq!(urls[2], "https://blog.dev/posts/c.png");
- // Duplicates and data: URIs never appear.
- assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1);
- assert!(!urls.iter().any(|u| u.starts_with("data:")));
- assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty());
- }
-
- // -----------------------------------------------------------------------
- // Image normalization — each case is a page shape seen in a real issue.
- // -----------------------------------------------------------------------
-
- fn src_of(html: &str) -> Vec<String> {
- Html::parse_fragment(html)
- .select(&Selector::parse("img").unwrap())
- .filter_map(|e| e.value().attr("src").map(str::to_string))
- .collect()
- }
-
- #[test]
- fn lazy_placeholder_src_gives_way_to_the_real_url() {
- // IEEE Spectrum: an inline SVG spacer with the URL parked on data-runner-src.
- let html = r#"<img alt="A well head" lazy-loadable="true"
- src="data:image/svg+xml,%3Csvg%20xmlns=%27http://www.w3.org/2000/svg%27%3E%3C/svg%3E"
- data-runner-src="https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"/>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"]
- );
- }
-
- #[test]
- fn unresolved_url_templates_fall_through_to_a_real_candidate() {
- // NPR: readability copies data-template over a perfectly good src.
- let html = r#"<img alt="Meghan Cliffel"
- src="https://npr.brightspotcdn.com/resize/{width}/quality/{quality}/x.jpg"
- srcset="https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg 1100w"/>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg"]
- );
- // With nothing usable anywhere, the image goes rather than becoming a
- // request for a picture that says "Image".
- let only_template =
- r#"<img alt="x" src="https://cdn.dev/resize/{width}/quality/{quality}/x.jpg"/>"#;
- assert!(src_of(&normalize_img_tags(only_template)).is_empty());
- }
-
- #[test]
- fn a_srcset_blob_parked_in_src_is_rejected_and_reparsed() {
- // dfarq: readability copies the entire srcset string into src.
- let blob = "https://i0.wp.com/x.jpg?resize=300%2C158&ssl=1 300w, \
- https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1 1024w, \
- https://i0.wp.com/x.jpg?w=3000&ssl=1 3000w";
- let html = format!(r#"<img alt="printer" src="{blob}" srcset="{blob}"/>"#);
- // The widest candidate under the download ceiling wins; 3000w does not.
- assert_eq!(
- src_of(&normalize_img_tags(&html)),
- ["https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1"]
- );
- }
-
- #[test]
- fn a_json_blob_parked_in_src_is_rejected() {
- // Substack: readability copies data-attrs (JSON) over src.
- let html = r#"<img alt="" src="{"src":"https://s3.dev/a.jpeg","width":1000}"
- srcset="https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg 1456w"/>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg"]
- );
- }
-
- #[test]
- fn picture_sources_back_up_an_empty_img() {
- let html = r#"<picture>
- <source srcset="https://cdn.dev/a.avif 800w" type="image/avif"/>
- <source srcset="https://cdn.dev/a.webp 800w" type="image/webp"/>
- <img alt="A photo"/>
- </picture>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://cdn.dev/a.avif"]
- );
- }
-
- #[test]
- fn a_good_src_is_never_traded_for_a_metadata_attribute() {
- // `data-template`, `data-attrs` and friends hold templates and JSON, not
- // URLs — trading a working src for one of those is the original sin.
- let html = r#"<img alt="A photo" loading="lazy" class="lazyload"
- src="https://cdn.dev/real.jpg"
- data-template="https://cdn.dev/{width}/real.jpg"
- data-orig-file="https://cdn.dev/orig.jpg"/>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://cdn.dev/real.jpg"]
- );
- }
-
- #[test]
- fn a_lazy_loader_placeholder_loses_to_its_data_src() {
- // iRunFar (a3-lazy-load): src is a shared 1×1 spacer that would download
- // and re-encode perfectly happily, and be the wrong picture.
- let html = r#"<img class="lazy lazy-hidden" alt="Brooks Cascadia 20"
- src="//www.irunfar.com/wp-content/plugins/a3-lazy-load/assets/images/lazy_placeholder.gif"
- data-src="https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"/>"#;
- assert_eq!(
- src_of(&normalize_img_tags(html)),
- ["https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"]
- );
- }
-
- #[test]
- fn normalization_keeps_alt_and_title_and_drops_the_rest() {
- let html = r#"<img src="/a.png" alt="An & alt" title="T" class="x" width="900"
- onerror="evil()" srcset="/b.png 2x"/>"#;
- let out = normalize_img_tags(html);
- assert!(out.contains(r#"alt="An & alt""#), "{out}");
- assert!(out.contains(r#"title="T""#), "{out}");
- for gone in ["class=", "width=", "onerror", "srcset="] {
- assert!(
- !out.contains(gone),
- "expected {gone} to be dropped from {out}"
- );
- }
- // Relative URLs survive: sanitize_with_base absolutizes them later.
- assert_eq!(src_of(&out), ["/a.png"]);
- }
-
- #[test]
- fn lightbox_buttons_no_longer_take_their_image_with_them() {
- // Nautilus: readability deletes <button> and everything inside it.
- let html = r#"<figure class="wp-block-image">
- <button type="button" aria-label="Enlarge image">
- <img class="wp-image-1" src="https://cdn.dev/flower.png?w=710" alt=""/>
- </button>
- <figcaption>BEAUTIFUL DANGER: a belladonna flower.</figcaption>
- </figure>"#;
- let out = unwrap_image_wrappers(html);
- assert!(!out.contains("<button"), "{out}");
- assert!(!out.contains("</button>"), "{out}");
- assert!(out.contains("flower.png"), "{out}");
- assert!(out.contains("BEAUTIFUL DANGER"), "caption survives: {out}");
- }
-
- #[test]
- fn buttons_that_are_really_buttons_are_left_alone() {
- let html = r#"<button class="subscribe">Subscribe <img src="/icon.png" alt=""/></button>"#;
- assert_eq!(unwrap_image_wrappers(html), html);
- // And an image-free control is untouched too.
- let plain = r#"<button>Share</button>"#;
- assert_eq!(unwrap_image_wrappers(plain), plain);
- }
-
- #[test]
- fn nested_and_unclosed_wrappers_do_not_derail_the_scan() {
- let nested = r#"<button><button><img src="/a.png"/></button></button><p>after</p>"#;
- let out = unwrap_image_wrappers(nested);
- assert!(!out.contains("button"), "{out}");
- assert!(out.contains("/a.png") && out.contains("after"), "{out}");
- // An open tag that never closes is passed through rather than eating
- // the rest of the document.
- let unclosed = r#"<button><img src="/a.png"/><p>rest</p>"#;
- assert!(unwrap_image_wrappers(unclosed).contains("rest"));
- }
-
- #[test]
- fn srcset_descriptors_pick_the_widest_usable_candidate() {
- assert_eq!(
- best_from_srcset("/a.png 150w, /b.png 800w, /c.png 4000w").as_deref(),
- Some("/b.png")
- );
- // Density descriptors work as an ordering too.
- assert_eq!(
- best_from_srcset("/a.png 1x, /b.png 2x").as_deref(),
- Some("/b.png")
- );
- // A bare URL with no descriptor is still a candidate.
- assert_eq!(best_from_srcset("/only.png").as_deref(), Some("/only.png"));
- // Every candidate too wide: take the narrowest rather than nothing.
- assert_eq!(
- best_from_srcset("/big.png 3000w, /huge.png 5000w").as_deref(),
- Some("/big.png")
- );
- assert_eq!(best_from_srcset(" ").as_deref(), None);
- }
-
- #[test]
- fn plausibility_is_about_shape_not_host() {
- for good in [
- "https://cdn.dev/a.jpg?id=1&width=980",
- "/_next/image?url=https%3A%2F%2Fx.dev%2Fa.png&w=3840&q=75",
- "https://substackcdn.com/image/fetch/$s_!y1,w_1456/a.jpeg",
- ] {
- assert!(plausible_url(good), "{good} should be usable");
- }
- for bad in [
- "",
- "data:image/svg+xml,%3Csvg%3E%3C/svg%3E",
- "https://cdn.dev/resize/{width}/a.jpg",
- r#"{"src":"https://cdn.dev/a.jpg"}"#,
- "https://cdn.dev/a.jpg 300w, https://cdn.dev/b.jpg 600w",
- "https://cdn.dev/a.jpg%20300w",
- ] {
- assert!(!plausible_url(bad), "{bad} should be rejected");
- }
- }
-
+ /// Feeds carry the same lazy markup pages do, so the same normalization runs
+ /// on Miniflux content before it is sanitized.
#[tokio::test]
async fn feed_content_is_normalized_too() {
- // Feeds carry the same lazy markup pages do.
let extractor = Extractor::offline(vec![]);
let body = format!(
r#"<p>{}</p><img src="data:image/gif;base64,zz" data-src="https://cdn.dev/real.jpg"/>"#,
diff --git a/src/html.rs b/src/html.rs
new file mode 100644
index 0000000..2c5ce84
--- /dev/null
+++ b/src/html.rs
@@ -0,0 +1,356 @@
+//! Generic HTML markup utilities shared across the pipeline.
+//!
+//! Everything here is about *markup as text*: scanning tags without building a
+//! DOM, escaping for XML output, and reading a fragment as prose. It knows
+//! nothing about articles, images or EPUBs — those modules build on top of it.
+//!
+//! There are two ways to look at HTML in this crate. `scraper` parses a real
+//! DOM and is the right tool when structure matters (walking up to an enclosing
+//! `<figure>`, say). The scanners here walk the string instead, which is what
+//! you want when the job is to rewrite tags in place and hand back markup that
+//! is otherwise byte-identical.
+
+use scraper::{Html, Node};
+
+/// HTML void elements: XHTML requires them self-closed (§3.10 "valid XHTML").
+pub const VOID_ELEMENTS: &[&str] = &[
+ "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source",
+ "track", "wbr",
+];
+
+// ---------------------------------------------------------------------------
+// Tag scanning
+// ---------------------------------------------------------------------------
+
+/// End index (exclusive) of the tag starting at `start` (`html[start] == '<'`),
+/// respecting quoted attribute values and comments.
+pub fn tag_end(html: &str, start: usize) -> Option<usize> {
+ let rest = &html[start..];
+ if rest.starts_with("<!--") {
+ return rest.find("-->").map(|i| start + i + 3);
+ }
+ let mut quote: Option<char> = None;
+ for (i, c) in rest.char_indices().skip(1) {
+ match (quote, c) {
+ (Some(q), c) if c == q => quote = None,
+ (Some(_), _) => {}
+ (None, '"') | (None, '\'') => quote = Some(c),
+ (None, '>') => return Some(start + i + c.len_utf8()),
+ (None, _) => {}
+ }
+ }
+ None
+}
+
+/// Lowercased element name of a tag body such as `img src="…"`.
+pub fn tag_name(inner: &str) -> String {
+ inner
+ .trim_start_matches('/')
+ .split(|c: char| c.is_whitespace() || c == '/' || c == '>')
+ .next()
+ .unwrap_or("")
+ .to_ascii_lowercase()
+}
+
+/// Parse `name="value"` pairs out of a tag body, with entity-decoded values.
+///
+/// Decoding matters: this scanner reads markup that ammonia has serialized, and
+/// ammonia writes `&` in a URL as `&`. A raw comparison against a URL that
+/// came out of a real HTML parser would never match (§3.10).
+pub fn parse_attrs(inner: &str) -> Vec<(String, String)> {
+ let mut attrs = Vec::new();
+ let bytes: Vec<char> = inner.chars().collect();
+ let mut i = 0;
+ // Skip the element name.
+ while i < bytes.len() && !bytes[i].is_whitespace() {
+ i += 1;
+ }
+ while i < bytes.len() {
+ while i < bytes.len() && (bytes[i].is_whitespace() || bytes[i] == '/') {
+ i += 1;
+ }
+ let name_start = i;
+ while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '=' && bytes[i] != '/' {
+ i += 1;
+ }
+ if i == name_start {
+ break;
+ }
+ let name: String = bytes[name_start..i]
+ .iter()
+ .collect::<String>()
+ .to_ascii_lowercase();
+ while i < bytes.len() && bytes[i].is_whitespace() {
+ i += 1;
+ }
+ let mut value = String::new();
+ if i < bytes.len() && bytes[i] == '=' {
+ i += 1;
+ while i < bytes.len() && bytes[i].is_whitespace() {
+ i += 1;
+ }
+ if i < bytes.len() && (bytes[i] == '"' || bytes[i] == '\'') {
+ let quote = bytes[i];
+ i += 1;
+ while i < bytes.len() && bytes[i] != quote {
+ value.push(bytes[i]);
+ i += 1;
+ }
+ i += 1;
+ } else {
+ while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '>' {
+ value.push(bytes[i]);
+ i += 1;
+ }
+ }
+ }
+ attrs.push((name, decode_entities(&value)));
+ }
+ attrs
+}
+
+/// Decode the handful of entities an HTML serializer emits inside attributes.
+///
+/// Numeric forms are included because feeds and WordPress write `&` for
+/// `&`; anything else is left alone rather than guessed at.
+pub fn decode_entities(s: &str) -> String {
+ if !s.contains('&') {
+ return s.to_string();
+ }
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(i) = rest.find('&') {
+ out.push_str(&rest[..i]);
+ let tail = &rest[i..];
+ let Some(end) = tail[..tail.len().min(12)].find(';') else {
+ out.push('&');
+ rest = &tail[1..];
+ continue;
+ };
+ let entity = &tail[1..end];
+ let decoded = match entity {
+ "amp" => Some('&'),
+ "lt" => Some('<'),
+ "gt" => Some('>'),
+ "quot" => Some('"'),
+ "apos" | "#39" => Some('\''),
+ "nbsp" => Some('\u{a0}'),
+ _ => entity
+ .strip_prefix('#')
+ .and_then(|n| match n.strip_prefix(['x', 'X']) {
+ Some(hex) => u32::from_str_radix(hex, 16).ok(),
+ None => n.parse::<u32>().ok(),
+ })
+ .and_then(char::from_u32),
+ };
+ match decoded {
+ Some(c) => {
+ out.push(c);
+ rest = &tail[end + 1..];
+ }
+ None => {
+ out.push('&');
+ rest = &tail[1..];
+ }
+ }
+ }
+ out.push_str(rest);
+ out
+}
+
+/// Escape a string for use inside a double-quoted XML attribute.
+pub fn attr_escape(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ for c in s.chars() {
+ match c {
+ '&' => out.push_str("&"),
+ '<' => out.push_str("<"),
+ '>' => out.push_str(">"),
+ '"' => out.push_str("""),
+ _ => out.push(c),
+ }
+ }
+ out
+}
+
+/// Escape a string for XML text content.
+pub fn text_escape(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ for c in s.chars() {
+ match c {
+ '&' => out.push_str("&"),
+ '<' => out.push_str("<"),
+ '>' => out.push_str(">"),
+ _ => out.push(c),
+ }
+ }
+ out
+}
+
+// ---------------------------------------------------------------------------
+// XHTML output
+// ---------------------------------------------------------------------------
+
+/// Self-close HTML void elements and normalize ` ` so the markup parses as
+/// XML — EPUB3 content documents are XHTML (§3.10).
+pub fn to_xhtml(html: &str) -> String {
+ let mut out = String::with_capacity(html.len());
+ let mut cursor = 0usize;
+ while let Some(rel) = html[cursor..].find('<') {
+ let start = cursor + rel;
+ out.push_str(&html[cursor..start]);
+ let Some(end) = tag_end(html, start) else {
+ out.push_str(&html[start..]);
+ cursor = html.len();
+ break;
+ };
+ let raw = &html[start..end];
+ let inner = raw.trim_start_matches('<').trim_end_matches('>');
+ let name = tag_name(inner);
+ if VOID_ELEMENTS.contains(&name.as_str()) && !inner.trim_end().ends_with('/') {
+ out.push('<');
+ out.push_str(inner.trim_end());
+ out.push_str("/>");
+ } else {
+ out.push_str(raw);
+ }
+ cursor = end;
+ }
+ out.push_str(&html[cursor..]);
+ // html5ever (via ammonia) emits ` `, which is undefined in XML.
+ out.replace(" ", " ")
+}
+
+// ---------------------------------------------------------------------------
+// Reading markup as text
+// ---------------------------------------------------------------------------
+
+/// Visible text of an HTML fragment, entities decoded, `script`/`style` skipped.
+pub fn html_to_text(html: &str) -> String {
+ let document = Html::parse_fragment(html);
+ let mut out = String::with_capacity(html.len() / 2);
+ for node in document.tree.nodes() {
+ let Node::Text(text) = node.value() else {
+ continue;
+ };
+ let hidden = node.ancestors().any(|a| match a.value() {
+ Node::Element(e) => matches!(e.name(), "script" | "style" | "noscript"),
+ _ => false,
+ });
+ if hidden {
+ continue;
+ }
+ out.push_str(text);
+ out.push(' ');
+ }
+ out
+}
+
+/// Count words in rendered text (tags stripped) (§3.3).
+pub fn word_count(html: &str) -> i64 {
+ html_to_text(html)
+ .split_whitespace()
+ .filter(|w| w.chars().any(char::is_alphanumeric))
+ .count() as i64
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn tag_scanner_finds_the_end_of_awkward_tags() {
+ // A `>` inside a quoted attribute is not the end of the tag.
+ let html = r#"<a title="a > b">x</a>"#;
+ assert_eq!(
+ tag_end(html, 0),
+ Some(17),
+ "past the closing `>`, not the one in the title"
+ );
+ // Comments end at `-->`, not at the first `>`.
+ let comment = "<!-- a > b -->rest";
+ assert_eq!(tag_end(comment, 0), Some(14));
+ // An unterminated tag has no end.
+ assert_eq!(tag_end("<p class=\"x", 0), None);
+ }
+
+ #[test]
+ fn tag_names_are_lowercased_and_stripped() {
+ assert_eq!(tag_name("IMG src=\"x\""), "img");
+ assert_eq!(tag_name("/DIV"), "div");
+ assert_eq!(tag_name("br/"), "br");
+ assert_eq!(tag_name(""), "");
+ }
+
+ #[test]
+ fn attributes_parse_in_every_spelling() {
+ let attrs = parse_attrs(r#"img src="a.png" ALT='an alt' loading=lazy hidden"#);
+ let get = |k: &str| attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str());
+ assert_eq!(get("src"), Some("a.png"));
+ assert_eq!(
+ get("alt"),
+ Some("an alt"),
+ "names lowercase, quotes either way"
+ );
+ assert_eq!(get("loading"), Some("lazy"), "unquoted values work");
+ assert_eq!(get("hidden"), Some(""), "valueless attributes are empty");
+ }
+
+ #[test]
+ fn attribute_values_come_back_entity_decoded() {
+ // The reason this matters: a serializer writes `&` in a URL as `&`,
+ // and callers compare against URLs a real parser produced.
+ let attrs = parse_attrs(r#"img src="a.jpg?id=1&w=9&h=2""#);
+ assert_eq!(attrs[0].1, "a.jpg?id=1&w=9&h=2");
+ }
+
+ #[test]
+ fn entity_decoding_covers_named_decimal_and_hex() {
+ assert_eq!(decode_entities("a & b"), "a & b");
+ assert_eq!(decode_entities("&&"), "&&");
+ assert_eq!(decode_entities("<p>"), "<p>");
+ assert_eq!(decode_entities("café"), "café");
+ // Nothing to do, and nothing invented for what we do not know.
+ assert_eq!(decode_entities("plain"), "plain");
+ assert_eq!(decode_entities("&unknown; &"), "&unknown; &");
+ }
+
+ #[test]
+ fn escaping_is_the_inverse_that_output_needs() {
+ assert_eq!(
+ attr_escape(r#"a & "b" <c>"#),
+ "a & "b" <c>"
+ );
+ // Text content keeps quotes as they are.
+ assert_eq!(text_escape(r#"a & "b" <c>"#), r#"a & "b" <c>"#);
+ }
+
+ #[test]
+ fn to_xhtml_self_closes_voids_and_entities() {
+ let html = "<p>a<br>b<hr>c d<img src=\"x.png\" alt=\"y\"></p><p>e<br/></p>";
+ let out = to_xhtml(html);
+ assert!(out.contains("<br/>"));
+ assert!(out.contains("<hr/>"));
+ assert!(out.contains("<img src=\"x.png\" alt=\"y\"/>"));
+ assert!(out.contains(" "));
+ assert!(!out.contains(" "));
+ assert!(!out.contains("<br/ >"));
+ // Already-closed voids are left alone (no double slash).
+ assert_eq!(out.matches("<br/>").count(), 2);
+ }
+
+ #[test]
+ fn to_xhtml_ignores_angle_brackets_in_attributes() {
+ let html = r#"<a title="a > b">x</a>"#;
+ assert_eq!(to_xhtml(html), html);
+ }
+
+ #[test]
+ fn word_count_ignores_markup_and_script() {
+ assert_eq!(word_count("<p>one two three</p>"), 3);
+ assert_eq!(word_count("<p>a</p><script>b c d e</script>"), 1);
+ assert_eq!(word_count("<p>& — ok</p>"), 1);
+ assert_eq!(word_count(""), 0);
+ assert_eq!(word_count("<p></p>"), 0);
+ }
+}
diff --git a/src/images/embed.rs b/src/images/embed.rs
new file mode 100644
index 0000000..d55cc8a
--- /dev/null
+++ b/src/images/embed.rs
@@ -0,0 +1,193 @@
+//! Pointing article markup at the images actually embedded in the EPUB.
+
+use std::collections::HashMap;
+
+use crate::html::{attr_escape, parse_attrs, tag_end, tag_name, text_escape};
+use crate::types::ImageAsset;
+
+/// Whether an image we could not embed is worth telling the reader about.
+///
+/// Only descriptive alt text qualifies. A filename, a bare label like `red line`
+/// on a divider rule, or no alt at all carries nothing the reader loses by not
+/// seeing the picture — announcing those turns every decorative graphic and
+/// dead link into a line of clutter, which is how the placeholders got out of
+/// hand in the first place.
+fn alt_is_worth_announcing(alt: &str) -> bool {
+ const MIN_DESCRIPTIVE_WORDS: usize = 4;
+ !alt.is_empty()
+ && !is_filename_alt(alt)
+ && alt.split_whitespace().count() >= MIN_DESCRIPTIVE_WORDS
+}
+
+/// True for alt text that is really just the uploaded filename — `IMG_0808.JPG`,
+/// `cut pieces v01.JPG`, `chart-final-2.png`.
+fn is_filename_alt(alt: &str) -> bool {
+ let alt = alt.trim();
+ if alt.contains(' ') && alt.split_whitespace().count() > 4 {
+ return false;
+ }
+ let Some((stem, ext)) = alt.rsplit_once('.') else {
+ return false;
+ };
+ let ext = ext.to_ascii_lowercase();
+ matches!(
+ ext.as_str(),
+ "jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" | "avif" | "bmp" | "heic"
+ ) && !stem.is_empty()
+ && stem
+ .chars()
+ .all(|c| c.is_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.'))
+}
+
+/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the
+/// `[image: alt]` placeholder paragraph (§3.10).
+pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String {
+ let by_url: HashMap<&str, &ImageAsset> =
+ assets.iter().map(|a| (a.source_url.as_str(), a)).collect();
+ let mut out = String::with_capacity(html.len());
+ let mut cursor = 0usize;
+ while let Some(rel) = html[cursor..].find('<') {
+ let start = cursor + rel;
+ out.push_str(&html[cursor..start]);
+ let Some(end) = tag_end(html, start) else {
+ out.push_str(&html[start..]);
+ return out;
+ };
+ let raw = &html[start..end];
+ let inner = raw
+ .trim_start_matches('<')
+ .trim_end_matches('>')
+ .trim_end_matches('/');
+ if tag_name(inner) == "img" {
+ let attrs = parse_attrs(inner);
+ let src = attrs
+ .iter()
+ .find(|(k, _)| k == "src")
+ .map(|(_, v)| v.trim().to_string())
+ .unwrap_or_default();
+ let alt = attrs
+ .iter()
+ .find(|(k, _)| k == "alt")
+ .map(|(_, v)| v.trim().to_string())
+ .unwrap_or_default();
+ match by_url.get(src.as_str()) {
+ Some(asset) => {
+ let alt = if alt.is_empty() { &asset.alt } else { &alt };
+ out.push_str(&format!(
+ "<img src=\"{}\" alt=\"{}\"/>",
+ attr_escape(&asset.href),
+ attr_escape(alt)
+ ));
+ }
+ // An image we could not embed is only worth announcing when its
+ // alt text tells the reader something; otherwise the `<img>`
+ // just goes away. That covers the decorative graphics the
+ // re-encoder deliberately skips as well as genuine misses (§3.10).
+ None if !alt_is_worth_announcing(&alt) => {}
+ None => {
+ out.push_str(&format!(
+ "<p class=\"image-placeholder\">[image: {}]</p>",
+ text_escape(&alt)
+ ));
+ }
+ }
+ } else {
+ out.push_str(raw);
+ }
+ cursor = end;
+ }
+ out.push_str(&html[cursor..]);
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn asset(url: &str, href: &str) -> ImageAsset {
+ ImageAsset {
+ id: "img-1-0".into(),
+ href: href.into(),
+ mime: "image/jpeg".into(),
+ data: vec![1, 2, 3],
+ alt: "fallback alt".into(),
+ caption: None,
+ source_url: url.into(),
+ }
+ }
+
+ #[test]
+ fn rewrites_hits_and_placeholders_misses() {
+ let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")];
+ let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="A chart of missing things">"#;
+ let out = rewrite_img_srcs(html, &assets);
+ // The alt round-trips through one level of escaping, not two.
+ assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt & more"/>"#));
+ assert!(
+ out.contains(r#"<p class="image-placeholder">[image: A chart of missing things]</p>"#)
+ );
+ assert!(!out.contains("gone.png"));
+ }
+
+ /// The whole point of the fix: ammonia writes `&` into the markup, and
+ /// the asset was keyed on the URL a real parser produced.
+ #[test]
+ fn entity_encoded_urls_still_match_their_asset() {
+ let assets = vec![asset(
+ "https://e.g/a.jpg?id=1&width=980",
+ "images/img-1-0.jpg",
+ )];
+ let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
+ assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
+ // The numeric spelling WordPress emits works too.
+ let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
+ assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
+ }
+
+ #[test]
+ fn unembeddable_images_only_speak_up_when_the_alt_says_something() {
+ // No alt at all: the image simply disappears.
+ assert_eq!(
+ rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]),
+ ""
+ );
+ // A filename is not a description.
+ assert_eq!(
+ rewrite_img_srcs(r#"<img src="https://e.g/x.png" alt="IMG_0808.JPG">"#, &[]),
+ ""
+ );
+ // Neither is the label on a decorative divider rule.
+ assert_eq!(
+ rewrite_img_srcs(r#"<img src="https://e.g/rule.png" alt="red line">"#, &[]),
+ ""
+ );
+ // A real description is worth keeping.
+ assert!(
+ rewrite_img_srcs(
+ r#"<img src="https://e.g/x.png" alt="A man in a hard hat stands over a well hole">"#,
+ &[]
+ )
+ .contains("[image: A man in a hard hat stands over a well hole]")
+ );
+ }
+
+ #[test]
+ fn filename_alt_detection() {
+ for yes in [
+ "IMG_0808.JPG",
+ "cut pieces v01.JPG",
+ "chart-final-2.png",
+ "diagram.svg",
+ ] {
+ assert!(is_filename_alt(yes), "{yes} should read as a filename");
+ }
+ for no in [
+ "A hydrogen well head",
+ "",
+ "Fig. 3",
+ "The lion-man of Hohlenstein-Stadel, carved from mammoth ivory.",
+ ] {
+ assert!(!is_filename_alt(no), "{no} should read as a description");
+ }
+ }
+}
diff --git a/src/images/encode.rs b/src/images/encode.rs
new file mode 100644
index 0000000..14108eb
--- /dev/null
+++ b/src/images/encode.rs
@@ -0,0 +1,322 @@
+//! Turning downloaded bytes into something an e-reader can display.
+//!
+//! Every image is decoded, fitted to the edition's profile, flattened onto white
+//! (e-ink has no transparency) and re-encoded. SVG is rasterized on the way in,
+//! because charts and diagrams are frequently vector-only.
+
+use std::io::Cursor;
+
+use image::{DynamicImage, GenericImageView, ImageFormat};
+
+use crate::types::Edition;
+
+/// Images smaller than this in either dimension are decorative — skipped (§3.10).
+pub const MIN_DIMENSION_PX: u32 = 24;
+/// Width an SVG is rendered at when the profile asks for less than this.
+const SVG_FALLBACK_SIZE: u32 = 1000;
+
+/// Per-edition re-encoding parameters (§3.10).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ImageProfile {
+ pub max_width: u32,
+ pub max_height: u32,
+ pub jpeg_quality: u8,
+ pub grayscale: bool,
+}
+
+impl ImageProfile {
+ /// Standard edition: max width 1200px, JPEG q80, color (§3.10).
+ pub const STANDARD: ImageProfile = ImageProfile {
+ max_width: 1200,
+ max_height: 4000,
+ jpeg_quality: 80,
+ grayscale: false,
+ };
+
+ /// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10).
+ pub const X4: ImageProfile = ImageProfile {
+ max_width: 480,
+ max_height: 800,
+ jpeg_quality: 70,
+ grayscale: true,
+ };
+
+ pub fn for_edition(edition: Edition) -> Self {
+ match edition {
+ Edition::Standard => Self::STANDARD,
+ Edition::X4 => Self::X4,
+ }
+ }
+}
+
+/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10).
+///
+/// Line art with transparency is kept as PNG after flattening; everything else
+/// becomes JPEG. SVG is rasterized first — charts and diagrams are frequently
+/// vector-only, and dropping them loses the point of the article. Returns `None`
+/// for sources no decoder handles.
+pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> {
+ if looks_like_svg(bytes) {
+ let raster = rasterize_svg(bytes, profile)?;
+ return reencode(&raster, profile);
+ }
+ let format = image::guess_format(bytes).ok();
+ let decoded = image::load_from_memory(bytes)
+ .map_err(|e| tracing::debug!("undecodable image: {e}"))
+ .ok()?;
+
+ let (w, h) = decoded.dimensions();
+ if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX {
+ tracing::debug!(w, h, "skipping decorative image");
+ return None;
+ }
+
+ let has_alpha = decoded.color().has_alpha();
+ let flattened = if has_alpha {
+ flatten_to_white(&decoded)
+ } else {
+ decoded
+ };
+
+ let resized = if w > profile.max_width || h > profile.max_height {
+ flattened.resize(
+ profile.max_width,
+ profile.max_height,
+ image::imageops::FilterType::Lanczos3,
+ )
+ } else {
+ flattened
+ };
+
+ // Keep line art (PNG source, few distinct tones) lossless; everything else
+ // becomes JPEG, which is far smaller for photographs (§3.10).
+ let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized);
+
+ let mut out = Cursor::new(Vec::new());
+ // NB: encode the concrete buffer, not the `DynamicImage` — the latter always
+ // reports RGBA pixels, which would silently re-colorize a grayscale image.
+ if profile.grayscale {
+ let gray = resized.to_luma8();
+ if keep_png {
+ DynamicImage::ImageLuma8(gray)
+ .write_to(&mut out, ImageFormat::Png)
+ .ok()?;
+ return Some((out.into_inner(), "image/png"));
+ }
+ let mut enc =
+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
+ enc.encode_image(&gray).ok()?;
+ return Some((out.into_inner(), "image/jpeg"));
+ }
+
+ let rgb = resized.to_rgb8();
+ if keep_png {
+ DynamicImage::ImageRgb8(rgb)
+ .write_to(&mut out, ImageFormat::Png)
+ .ok()?;
+ return Some((out.into_inner(), "image/png"));
+ }
+ let mut enc =
+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
+ enc.encode_image(&rgb).ok()?;
+ Some((out.into_inner(), "image/jpeg"))
+}
+
+/// True when `bytes` are an SVG document (possibly behind an XML prolog or BOM).
+fn looks_like_svg(bytes: &[u8]) -> bool {
+ let head = &bytes[..bytes.len().min(1024)];
+ let text = String::from_utf8_lossy(head);
+ let text = text.trim_start_matches('\u{feff}').trim_start();
+ text.starts_with("<svg")
+ || (text.starts_with("<?xml") || text.starts_with("<!DOCTYPE svg")) && text.contains("<svg")
+}
+
+/// Rasterize an SVG to a PNG at the profile's target width (§3.10).
+///
+/// The profile's own resize pass then handles the height cap, so this only has
+/// to land in the right ballpark.
+fn rasterize_svg(bytes: &[u8], profile: ImageProfile) -> Option<Vec<u8>> {
+ let mut options = resvg::usvg::Options::default();
+ options.fontdb_mut().load_system_fonts();
+ let tree = resvg::usvg::Tree::from_data(bytes, &options)
+ .map_err(|e| tracing::debug!("svg did not parse: {e}"))
+ .ok()?;
+
+ // An `<svg>` that parses but draws nothing is not an image, it is a stray
+ // tag: rasterizing it would embed a blank rectangle.
+ if tree.root().children().is_empty() {
+ tracing::debug!("svg has nothing to draw");
+ return None;
+ }
+ let size = tree.size();
+ let (sw, sh) = (size.width(), size.height());
+ if !(sw.is_finite() && sh.is_finite()) || sw <= 0.0 || sh <= 0.0 {
+ return None;
+ }
+ // Judge "decorative" by the declared size, before scaling: a 16×16 icon is
+ // an icon however large we choose to draw it.
+ if sw < MIN_DIMENSION_PX as f32 || sh < MIN_DIMENSION_PX as f32 {
+ tracing::debug!(sw, sh, "skipping decorative svg");
+ return None;
+ }
+ // Vector art has no native resolution, so render straight at the edition's
+ // target width — upscaling a rasterized copy afterwards would only blur it.
+ let target_w = profile
+ .max_width
+ .max(SVG_FALLBACK_SIZE.min(profile.max_width));
+ let scale = (target_w as f32 / sw).min(profile.max_height as f32 / sh);
+ let scale = if scale.is_finite() && scale > 0.0 {
+ scale
+ } else {
+ 1.0
+ };
+ let (w, h) = (
+ (sw * scale).round().max(1.0) as u32,
+ (sh * scale).round().max(1.0) as u32,
+ );
+ let mut pixmap = tiny_skia::Pixmap::new(w, h)?;
+ // E-ink has no transparency; render onto white so alpha never becomes black.
+ pixmap.fill(tiny_skia::Color::WHITE);
+ resvg::render(
+ &tree,
+ tiny_skia::Transform::from_scale(scale, scale),
+ &mut pixmap.as_mut(),
+ );
+ let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())?;
+ let mut png = Cursor::new(Vec::new());
+ DynamicImage::ImageRgba8(rgba)
+ .write_to(&mut png, ImageFormat::Png)
+ .ok()?;
+ Some(png.into_inner())
+}
+
+/// Composite over an opaque white page — e-ink has no transparency (§3.10).
+fn flatten_to_white(img: &DynamicImage) -> DynamicImage {
+ let rgba = img.to_rgba8();
+ let mut rgb = image::RgbImage::new(rgba.width(), rgba.height());
+ for (x, y, px) in rgba.enumerate_pixels() {
+ let a = f32::from(px[3]) / 255.0;
+ let blend = |c: u8| {
+ ((f32::from(c) * a) + 255.0 * (1.0 - a))
+ .round()
+ .clamp(0.0, 255.0) as u8
+ };
+ rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])]));
+ }
+ DynamicImage::ImageRgb8(rgb)
+}
+
+/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text).
+fn is_line_art(img: &DynamicImage) -> bool {
+ const SAMPLE_LIMIT: usize = 20_000;
+ const DISTINCT_LIMIT: usize = 64;
+ let rgb = img.to_rgb8();
+ let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1);
+ for (i, px) in rgb.pixels().enumerate() {
+ if i >= SAMPLE_LIMIT {
+ break;
+ }
+ let c = [px[0], px[1], px[2]];
+ if !distinct.contains(&c) {
+ distinct.push(c);
+ if distinct.len() > DISTINCT_LIMIT {
+ return false;
+ }
+ }
+ }
+ true
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn reencode_resizes_grayscales_and_encodes() {
+ let mut img = image::RgbaImage::new(200, 100);
+ for (x, y, px) in img.enumerate_pixels_mut() {
+ *px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]);
+ }
+ let mut png = Cursor::new(Vec::new());
+ DynamicImage::ImageRgba8(img)
+ .write_to(&mut png, ImageFormat::Png)
+ .unwrap();
+ let raw = png.into_inner();
+
+ let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap();
+ assert_eq!(std_mime, "image/jpeg");
+ let decoded = image::load_from_memory(&std_bytes).unwrap();
+ assert_eq!(decoded.dimensions(), (200, 100), "no upscaling");
+
+ let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap();
+ let x4 = image::load_from_memory(&x4_bytes).unwrap();
+ assert!(x4.width() <= 480 && x4.height() <= 800);
+ assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8");
+ }
+
+ #[test]
+ fn reencode_skips_decorative_images_and_junk() {
+ let tiny = image::RgbaImage::new(8, 8);
+ let mut png = Cursor::new(Vec::new());
+ DynamicImage::ImageRgba8(tiny)
+ .write_to(&mut png, ImageFormat::Png)
+ .unwrap();
+ assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none());
+ // A bare `<svg>` tag with nothing to draw is markup, not a picture.
+ assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none());
+ assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none());
+ }
+
+ #[test]
+ fn svg_charts_are_rasterized_rather_than_dropped() {
+ let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300">
+ <rect x="10" y="10" width="380" height="280" fill="#3355bb"/>
+ <circle cx="200" cy="150" r="60" fill="#ffcc00"/>
+ </svg>"##;
+ let (bytes, mime) = reencode(svg, ImageProfile::STANDARD).expect("svg rasterizes");
+ assert_eq!(mime, "image/png", "flat colour art stays lossless");
+ let decoded = image::load_from_memory(&bytes).expect("decodable output");
+ // Drawn at the edition's target width, not at the SVG's nominal size.
+ assert_eq!(decoded.dimensions(), (1200, 900));
+ assert!(!decoded.color().has_alpha(), "rendered onto white");
+
+ // An XML prolog and a leading BOM must not hide the format.
+ let with_prolog = format!(
+ "\u{feff}<?xml version=\"1.0\"?>{}",
+ String::from_utf8_lossy(svg)
+ );
+ let (x4, _) = reencode(with_prolog.as_bytes(), ImageProfile::X4).expect("x4 rasterizes");
+ let x4 = image::load_from_memory(&x4).unwrap();
+ assert!(x4.width() <= 480 && x4.height() <= 800);
+
+ // A 16×16 icon is decorative however large we could draw it.
+ let icon = br#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
+ <rect width="16" height="16"/></svg>"#;
+ assert!(reencode(icon, ImageProfile::STANDARD).is_none());
+ }
+
+ #[test]
+ fn line_art_png_stays_png_and_is_flattened() {
+ let mut img = image::RgbaImage::new(120, 60);
+ for (x, _y, px) in img.enumerate_pixels_mut() {
+ *px = if x % 12 == 0 {
+ image::Rgba([0, 0, 0, 255])
+ } else {
+ image::Rgba([255, 255, 255, 0])
+ };
+ }
+ let mut png = Cursor::new(Vec::new());
+ DynamicImage::ImageRgba8(img)
+ .write_to(&mut png, ImageFormat::Png)
+ .unwrap();
+ let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap();
+ assert_eq!(mime, "image/png");
+ let decoded = image::load_from_memory(&bytes).unwrap();
+ assert!(!decoded.color().has_alpha(), "transparency is flattened");
+ // Transparent pixels became white.
+ assert_eq!(
+ decoded.to_rgb8().get_pixel(1, 1),
+ &image::Rgb([255, 255, 255])
+ );
+ }
+}
diff --git a/src/images/fetch.rs b/src/images/fetch.rs
new file mode 100644
index 0000000..ef7681b
--- /dev/null
+++ b/src/images/fetch.rs
@@ -0,0 +1,159 @@
+//! Downloading an issue's images and packing them into EPUB assets.
+
+use std::time::Duration;
+
+use futures::StreamExt;
+
+use crate::types::{Edition, ImageAsset, Pick};
+
+use super::encode::{ImageProfile, reencode};
+use super::refs::{ImgRef, extract_img_refs};
+
+/// Per-image download timeout (§3.10).
+pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
+/// Per-image size cap (§3.10).
+pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
+/// Concurrent downloads (§3.10).
+pub const CONCURRENCY: usize = 8;
+/// Whole-issue asset budget (§3.10).
+pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
+
+/// Download one image, honoring the timeout and size cap (§3.10).
+pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
+ let resp = http
+ .get(url)
+ // Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10).
+ .header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8")
+ .timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
+ .send()
+ .await
+ .map_err(|e| tracing::debug!(url, "image download failed: {e}"))
+ .ok()?;
+ if !resp.status().is_success() {
+ tracing::debug!(url, status = %resp.status(), "image download rejected");
+ return None;
+ }
+ if let Some(len) = resp.content_length()
+ && len as usize > MAX_IMAGE_BYTES
+ {
+ tracing::debug!(url, len, "image exceeds the size cap");
+ return None;
+ }
+ let mut resp = resp;
+ let mut buf: Vec<u8> = Vec::new();
+ loop {
+ match resp.chunk().await {
+ Ok(Some(chunk)) => {
+ if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
+ tracing::debug!(url, "image exceeds the size cap mid-stream");
+ return None;
+ }
+ buf.extend_from_slice(&chunk);
+ }
+ Ok(None) => break,
+ Err(e) => {
+ tracing::debug!(url, "image download interrupted: {e}");
+ return None;
+ }
+ }
+ }
+ if buf.is_empty() { None } else { Some(buf) }
+}
+
+/// Everything needed to fetch one image, in deterministic issue order.
+#[derive(Debug, Clone)]
+struct PendingImage {
+ id: String,
+ url: String,
+ alt: String,
+ caption: Option<String>,
+}
+
+fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> {
+ let entry_id = pick.article.best_entry_id;
+ let mut refs = extract_img_refs(&pick.article.content_html);
+ if refs.is_empty() {
+ refs = pick
+ .article
+ .image_urls
+ .iter()
+ .map(|u| ImgRef {
+ src: u.clone(),
+ alt: String::new(),
+ caption: None,
+ })
+ .collect();
+ }
+ refs.into_iter()
+ .filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://"))
+ .enumerate()
+ .map(|(i, r)| PendingImage {
+ id: format!("img-{entry_id}-{i}"),
+ url: r.src,
+ alt: r.alt,
+ caption: r.caption,
+ })
+ .collect()
+}
+
+/// Download and re-encode every image referenced by the lineup for one edition,
+/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10).
+pub async fn collect_for_issue(
+ http: &reqwest::Client,
+ picks: &[Pick],
+ edition: Edition,
+) -> Vec<ImageAsset> {
+ let profile = ImageProfile::for_edition(edition);
+ let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect();
+ if pending.is_empty() {
+ return Vec::new();
+ }
+ tracing::info!(count = pending.len(), ?edition, "downloading issue images");
+
+ let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> =
+ futures::stream::iter(pending.into_iter().map(|p| {
+ let http = http.clone();
+ async move {
+ let raw = download(&http, &p.url).await?;
+ let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile))
+ .await
+ .ok()
+ .flatten()?;
+ Some((p, bytes, mime))
+ }
+ }))
+ .buffered(CONCURRENCY)
+ .collect()
+ .await;
+
+ let mut assets = Vec::new();
+ let mut budget_used = 0usize;
+ let mut skipped = 0usize;
+ for result in results.into_iter().flatten() {
+ let (pending, bytes, mime) = result;
+ if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES {
+ skipped += 1;
+ continue;
+ }
+ budget_used += bytes.len();
+ let ext = if mime == "image/png" { "png" } else { "jpg" };
+ assets.push(ImageAsset {
+ href: format!("images/{}.{ext}", pending.id),
+ id: pending.id,
+ mime: mime.to_string(),
+ data: bytes,
+ alt: pending.alt,
+ caption: pending.caption,
+ source_url: pending.url,
+ });
+ }
+ if skipped > 0 {
+ tracing::warn!(skipped, budget_used, "issue image budget exhausted");
+ }
+ tracing::info!(
+ embedded = assets.len(),
+ bytes = budget_used,
+ "issue images ready"
+ );
+ assets
+}
diff --git a/src/images/mod.rs b/src/images/mod.rs
new file mode 100644
index 0000000..e3656c0
--- /dev/null
+++ b/src/images/mod.rs
@@ -0,0 +1,25 @@
+//! Article images, end to end (spec §3.3 extraction, §3.10 "Images").
+//!
+//! The stages run in this order, and each submodule owns one of them:
+//!
+//! | stage | module | when |
+//! |---|---|---|
+//! | make a page's `<img>` elements usable | [`normalize`] | extraction, before readability |
+//! | find what an article references | [`refs`] | extraction and issue build |
+//! | download and re-encode per edition | [`fetch`], [`encode`] | issue build |
+//! | point the markup at the embedded files | [`embed`] | chapter rendering |
+//!
+//! Nothing here ever fails a run: an image that cannot be fetched, decoded or
+//! resolved is dropped, and the article is rendered without it (notes §3).
+
+pub mod embed;
+pub mod encode;
+pub mod fetch;
+pub mod normalize;
+pub mod refs;
+
+pub use embed::rewrite_img_srcs;
+pub use encode::{ImageProfile, MIN_DIMENSION_PX, reencode};
+pub use fetch::{ISSUE_ASSET_BUDGET_BYTES, collect_for_issue, download};
+pub use normalize::{normalize_img_tags, prepare_for_readability, unwrap_image_wrappers};
+pub use refs::{ImgRef, collect_image_urls, extract_img_refs};
diff --git a/src/images/normalize.rs b/src/images/normalize.rs
new file mode 100644
index 0000000..687d8fd
--- /dev/null
+++ b/src/images/normalize.rs
@@ -0,0 +1,516 @@
+//! Making a page's `<img>` elements usable before anything else touches them.
+//!
+//! Publishers ship images in a dozen incompatible shapes: lazy placeholders,
+//! `srcset` lists, `<picture>` sources, URLs parked in `data-*` attributes, and
+//! `src` values that are not URLs at all (unfilled templates, JSON blobs, whole
+//! `srcset` strings). On top of that, readability actively damages images while
+//! it works — it deletes `<button>` subtrees, taking lightbox images with them,
+//! and its own lazy-image heuristic overwrites a working `src` with any
+//! attribute whose value happens to contain `.jpg`.
+//!
+//! [`prepare_for_readability`] runs before readability and leaves every `<img>`
+//! as a plain `src`/`alt`/`title` triple, which both fixes the input and denies
+//! readability the raw material for its substitution. [`normalize_img_tags`] is
+//! also applied to feed content, which carries the same markup.
+//!
+//! Candidates are judged by *shape*, never by publisher: a string with braces,
+//! whitespace or quotes in it cannot resolve, whoever wrote it.
+
+use crate::html::{html_to_text, parse_attrs, tag_end, tag_name};
+
+/// Elements that readability deletes outright, and which a page may nevertheless
+/// have wrapped around an image (lightbox triggers, mostly).
+const IMAGE_WRAPPER_TAGS: &[&str] = &["button", "form", "fieldset", "object"];
+
+/// Attributes lazy-loading libraries use for the real image URL.
+///
+/// These outrank `src`, because a page only sets them when `src` is a stand-in:
+/// a transparent GIF, a blurred thumbnail, an inline SVG spacer. Attributes that
+/// merely *look* image-ish (`data-template`, `data-attrs`, `data-orig-file`) are
+/// deliberately absent — those hold templates and metadata, and preferring them
+/// is exactly the mistake readability's own heuristic makes.
+const LAZY_SRC_ATTRS: &[&str] = &[
+ "data-src",
+ "data-lazy-src",
+ "data-original",
+ "data-runner-src",
+ "data-full-src",
+ "data-hi-res-src",
+ "data-image-src",
+];
+
+/// Widest `srcset` candidate we will pick; above this we are downloading pixels
+/// the re-encoder immediately throws away.
+const MAX_SRCSET_WIDTH: u32 = 2000;
+
+/// Make a fetched page safe to hand to readability (§3.3).
+///
+/// Two passes, both about images: unwrap the elements that would take an image
+/// with them when readability deletes them, then reduce every `<img>` to a plain
+/// `src`/`alt`/`title` triple. The second pass is what stops readability's own
+/// lazy-image heuristic from replacing a working `src` — with no `srcset`,
+/// `loading` or `data-*` attributes left on the element, it has nothing to
+/// substitute and leaves the image alone.
+pub fn prepare_for_readability(html: &str) -> String {
+ normalize_img_tags(&unwrap_image_wrappers(html))
+}
+
+/// Replace image-only `<button>`/`<form>`/`<fieldset>`/`<object>` wrappers with
+/// their contents (§3.3).
+///
+/// A `<button>` holding nothing but an image is a lightbox trigger, not a
+/// control: the image is the content. Wrappers that also carry text are left
+/// alone, because those really are interface.
+pub fn unwrap_image_wrappers(html: &str) -> String {
+ let mut out = String::with_capacity(html.len());
+ let mut cursor = 0usize;
+ while let Some(rel) = html[cursor..].find('<') {
+ let start = cursor + rel;
+ out.push_str(&html[cursor..start]);
+ let Some(end) = tag_end(html, start) else {
+ out.push_str(&html[start..]);
+ return out;
+ };
+ let raw = &html[start..end];
+ let inner = raw.trim_start_matches('<').trim_end_matches('>');
+ let name = tag_name(inner);
+
+ if IMAGE_WRAPPER_TAGS.contains(&name.as_str())
+ && !inner.trim_end().ends_with('/')
+ && let Some((content, after)) = element_content(html, end, &name)
+ && content.contains("<img")
+ && html_to_text(content).trim().is_empty()
+ {
+ out.push_str(&unwrap_image_wrappers(content));
+ cursor = after;
+ continue;
+ }
+
+ out.push_str(raw);
+ cursor = end;
+ }
+ out.push_str(&html[cursor..]);
+ out
+}
+
+/// The content of an element whose open tag ended at `body_start`, plus the
+/// offset just past its close tag. `None` when the element is never closed.
+fn element_content<'a>(html: &'a str, body_start: usize, name: &str) -> Option<(&'a str, usize)> {
+ let open = format!("<{name}");
+ let close = format!("</{name}");
+ let mut depth = 1usize;
+ let mut cursor = body_start;
+ while let Some(rel) = html[cursor..].find('<') {
+ let start = cursor + rel;
+ let end = tag_end(html, start)?;
+ let tag = &html[start..end];
+ let lower = tag.to_ascii_lowercase();
+ if lower.starts_with(&close) {
+ depth -= 1;
+ if depth == 0 {
+ return Some((&html[body_start..start], end));
+ }
+ } else if lower.starts_with(&open)
+ && !lower[open.len()..].starts_with(|c: char| c.is_alphanumeric() || c == '-')
+ && !tag.trim_end().ends_with("/>")
+ {
+ depth += 1;
+ }
+ cursor = end;
+ }
+ None
+}
+
+/// Reduce every `<img>` to `<img src alt title>` with a usable URL (§3.3).
+///
+/// The `src` a page ships is not automatically the one to use: it can be a lazy
+/// placeholder (`data:image/svg+xml,…`), an unresolved template
+/// (`…/resize/{width}/…`), a JSON blob a framework parked there, or an entire
+/// `srcset` string. Candidates are tried in order and the first plausible one
+/// wins; an image with no plausible candidate is dropped, because a broken
+/// `<img>` only becomes clutter further down the pipeline.
+pub fn normalize_img_tags(html: &str) -> String {
+ let mut out = String::with_capacity(html.len());
+ let mut cursor = 0usize;
+ // `<picture>` puts the real candidates on sibling `<source>` elements.
+ let mut picture_srcset: Option<String> = None;
+
+ while let Some(rel) = html[cursor..].find('<') {
+ let start = cursor + rel;
+ out.push_str(&html[cursor..start]);
+ let Some(end) = tag_end(html, start) else {
+ out.push_str(&html[start..]);
+ return out;
+ };
+ let raw = &html[start..end];
+ let inner = raw
+ .trim_start_matches('<')
+ .trim_end_matches('>')
+ .trim_end_matches('/');
+ let name = tag_name(inner);
+
+ match name.as_str() {
+ "picture" => {
+ picture_srcset = None;
+ out.push_str(raw);
+ }
+ "source" => {
+ let attrs = parse_attrs(inner);
+ if picture_srcset.is_none()
+ && let Some(set) =
+ attr(&attrs, "srcset").or_else(|| attr(&attrs, "data-srcset"))
+ {
+ picture_srcset = Some(set.to_string());
+ }
+ out.push_str(raw);
+ }
+ "img" => {
+ let attrs = parse_attrs(inner);
+ if let Some(src) = best_img_src(&attrs, picture_srcset.as_deref()) {
+ out.push_str("<img src=\"");
+ out.push_str(&escape_attr(&src));
+ out.push('"');
+ for key in ["alt", "title"] {
+ if let Some(v) = attr(&attrs, key) {
+ out.push(' ');
+ out.push_str(key);
+ out.push_str("=\"");
+ out.push_str(&escape_attr(v));
+ out.push('"');
+ }
+ }
+ out.push_str("/>");
+ } else {
+ tracing::debug!(tag = %&raw[..raw.len().min(120)], "dropping unusable img");
+ }
+ }
+ _ => out.push_str(raw),
+ }
+ if name == "picture" && inner.starts_with('/') {
+ picture_srcset = None;
+ }
+ cursor = end;
+ }
+ out.push_str(&html[cursor..]);
+ out
+}
+
+fn attr<'a>(attrs: &'a [(String, String)], name: &str) -> Option<&'a str> {
+ attrs
+ .iter()
+ .find(|(k, _)| k == name)
+ .map(|(_, v)| v.trim())
+ .filter(|v| !v.is_empty())
+}
+
+/// Pick the best URL for one `<img>` from everything the element carries.
+fn best_img_src(attrs: &[(String, String)], picture_srcset: Option<&str>) -> Option<String> {
+ for key in LAZY_SRC_ATTRS {
+ if let Some(v) = attr(attrs, key).filter(|s| plausible_url(s)) {
+ return Some(v.to_string());
+ }
+ }
+ if let Some(src) = attr(attrs, "src").filter(|s| plausible_url(s)) {
+ return Some(src.to_string());
+ }
+ if let Some(from_set) = attr(attrs, "srcset").and_then(best_from_srcset) {
+ return Some(from_set);
+ }
+ if let Some(from_set) = attr(attrs, "data-srcset").and_then(best_from_srcset) {
+ return Some(from_set);
+ }
+ picture_srcset.and_then(best_from_srcset)
+}
+
+/// The widest candidate in a `srcset` that is still worth downloading.
+///
+/// Parsed by whitespace rather than by comma: the URLs of several image CDNs
+/// contain commas of their own, and splitting on those shreds them.
+fn best_from_srcset(srcset: &str) -> Option<String> {
+ let mut best: Option<(u32, String)> = None;
+ let mut smallest: Option<(u32, String)> = None;
+ let mut pending: Option<String> = None;
+
+ let mut consider = |url: String, width: u32| {
+ if width <= MAX_SRCSET_WIDTH && best.as_ref().is_none_or(|(w, _)| width > *w) {
+ best = Some((width, url.clone()));
+ }
+ if smallest.as_ref().is_none_or(|(w, _)| width < *w) {
+ smallest = Some((width, url));
+ }
+ };
+
+ for token in srcset.split_whitespace() {
+ let token = token.trim_end_matches(',');
+ if token.is_empty() {
+ continue;
+ }
+ match parse_descriptor(token) {
+ Some(width) => {
+ if let Some(url) = pending.take() {
+ consider(url, width);
+ }
+ }
+ None => {
+ // A URL with no descriptor of its own still counts, at width 1x.
+ if let Some(url) = pending.replace(token.to_string()) {
+ consider(url, 1);
+ }
+ }
+ }
+ }
+ if let Some(url) = pending.take() {
+ consider(url, 1);
+ }
+
+ best.or(smallest)
+ .map(|(_, url)| url)
+ .filter(|u| plausible_url(u))
+}
+
+/// `800w` → 800, `2x` → a synthetic width so density candidates sort sensibly.
+fn parse_descriptor(token: &str) -> Option<u32> {
+ let (value, unit) = token.split_at(token.len().checked_sub(1)?);
+ match unit {
+ "w" => value.parse::<u32>().ok(),
+ "x" => value
+ .parse::<f32>()
+ .ok()
+ .map(|d| (d * 1000.0).round().clamp(1.0, f32::from(u16::MAX)) as u32),
+ _ => None,
+ }
+}
+
+/// Whether a string can serve as an image URL at all.
+///
+/// This is deliberately about shape, not about the host: whitespace, braces and
+/// quotes mean we are looking at a `srcset` blob, an unfilled URL template or a
+/// serialized object, none of which will ever resolve.
+fn plausible_url(candidate: &str) -> bool {
+ let candidate = candidate.trim();
+ if candidate.is_empty() || candidate.len() > 2048 {
+ return false;
+ }
+ if candidate.starts_with("data:") || candidate.starts_with("about:") {
+ return false;
+ }
+ if candidate
+ .chars()
+ .any(|c| c.is_whitespace() || matches!(c, '{' | '}' | '"' | '\'' | '<' | '>' | '\\'))
+ {
+ return false;
+ }
+ // `%20` is a space that survived encoding — same blob, different spelling.
+ !candidate.contains("%20")
+}
+
+fn escape_attr(s: &str) -> String {
+ s.replace('&', "&")
+ .replace('<', "<")
+ .replace('>', ">")
+ .replace('"', """)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use scraper::{Html, Selector};
+
+ /// The `src` values that survive normalization, read back with a real parser.
+ fn src_of(html: &str) -> Vec<String> {
+ Html::parse_fragment(html)
+ .select(&Selector::parse("img").unwrap())
+ .filter_map(|e| e.value().attr("src").map(str::to_string))
+ .collect()
+ }
+
+ #[test]
+ fn lazy_placeholder_src_gives_way_to_the_real_url() {
+ // IEEE Spectrum: an inline SVG spacer with the URL parked on data-runner-src.
+ let html = r#"<img alt="A well head" lazy-loadable="true"
+ src="data:image/svg+xml,%3Csvg%20xmlns=%27http://www.w3.org/2000/svg%27%3E%3C/svg%3E"
+ data-runner-src="https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"/>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"]
+ );
+ }
+
+ #[test]
+ fn unresolved_url_templates_fall_through_to_a_real_candidate() {
+ // NPR: readability copies data-template over a perfectly good src.
+ let html = r#"<img alt="Meghan Cliffel"
+ src="https://npr.brightspotcdn.com/resize/{width}/quality/{quality}/x.jpg"
+ srcset="https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg 1100w"/>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg"]
+ );
+ // With nothing usable anywhere, the image goes rather than becoming a
+ // request for a picture that says "Image".
+ let only_template =
+ r#"<img alt="x" src="https://cdn.dev/resize/{width}/quality/{quality}/x.jpg"/>"#;
+ assert!(src_of(&normalize_img_tags(only_template)).is_empty());
+ }
+
+ #[test]
+ fn a_srcset_blob_parked_in_src_is_rejected_and_reparsed() {
+ // dfarq: readability copies the entire srcset string into src.
+ let blob = "https://i0.wp.com/x.jpg?resize=300%2C158&ssl=1 300w, \
+ https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1 1024w, \
+ https://i0.wp.com/x.jpg?w=3000&ssl=1 3000w";
+ let html = format!(r#"<img alt="printer" src="{blob}" srcset="{blob}"/>"#);
+ // The widest candidate under the download ceiling wins; 3000w does not.
+ assert_eq!(
+ src_of(&normalize_img_tags(&html)),
+ ["https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1"]
+ );
+ }
+
+ #[test]
+ fn a_json_blob_parked_in_src_is_rejected() {
+ // Substack: readability copies data-attrs (JSON) over src.
+ let html = r#"<img alt="" src="{"src":"https://s3.dev/a.jpeg","width":1000}"
+ srcset="https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg 1456w"/>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg"]
+ );
+ }
+
+ #[test]
+ fn picture_sources_back_up_an_empty_img() {
+ let html = r#"<picture>
+ <source srcset="https://cdn.dev/a.avif 800w" type="image/avif"/>
+ <source srcset="https://cdn.dev/a.webp 800w" type="image/webp"/>
+ <img alt="A photo"/>
+ </picture>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://cdn.dev/a.avif"]
+ );
+ }
+
+ #[test]
+ fn a_good_src_is_never_traded_for_a_metadata_attribute() {
+ // `data-template`, `data-attrs` and friends hold templates and JSON, not
+ // URLs — trading a working src for one of those is the original sin.
+ let html = r#"<img alt="A photo" loading="lazy" class="lazyload"
+ src="https://cdn.dev/real.jpg"
+ data-template="https://cdn.dev/{width}/real.jpg"
+ data-orig-file="https://cdn.dev/orig.jpg"/>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://cdn.dev/real.jpg"]
+ );
+ }
+
+ #[test]
+ fn a_lazy_loader_placeholder_loses_to_its_data_src() {
+ // iRunFar (a3-lazy-load): src is a shared 1×1 spacer that would download
+ // and re-encode perfectly happily, and be the wrong picture.
+ let html = r#"<img class="lazy lazy-hidden" alt="Brooks Cascadia 20"
+ src="//www.irunfar.com/wp-content/plugins/a3-lazy-load/assets/images/lazy_placeholder.gif"
+ data-src="https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"/>"#;
+ assert_eq!(
+ src_of(&normalize_img_tags(html)),
+ ["https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"]
+ );
+ }
+
+ #[test]
+ fn normalization_keeps_alt_and_title_and_drops_the_rest() {
+ let html = r#"<img src="/a.png" alt="An & alt" title="T" class="x" width="900"
+ onerror="evil()" srcset="/b.png 2x"/>"#;
+ let out = normalize_img_tags(html);
+ assert!(out.contains(r#"alt="An & alt""#), "{out}");
+ assert!(out.contains(r#"title="T""#), "{out}");
+ for gone in ["class=", "width=", "onerror", "srcset="] {
+ assert!(
+ !out.contains(gone),
+ "expected {gone} to be dropped from {out}"
+ );
+ }
+ // Relative URLs survive: sanitize_with_base absolutizes them later.
+ assert_eq!(src_of(&out), ["/a.png"]);
+ }
+
+ #[test]
+ fn lightbox_buttons_no_longer_take_their_image_with_them() {
+ // Nautilus: readability deletes <button> and everything inside it.
+ let html = r#"<figure class="wp-block-image">
+ <button type="button" aria-label="Enlarge image">
+ <img class="wp-image-1" src="https://cdn.dev/flower.png?w=710" alt=""/>
+ </button>
+ <figcaption>BEAUTIFUL DANGER: a belladonna flower.</figcaption>
+ </figure>"#;
+ let out = unwrap_image_wrappers(html);
+ assert!(!out.contains("<button"), "{out}");
+ assert!(!out.contains("</button>"), "{out}");
+ assert!(out.contains("flower.png"), "{out}");
+ assert!(out.contains("BEAUTIFUL DANGER"), "caption survives: {out}");
+ }
+
+ #[test]
+ fn buttons_that_are_really_buttons_are_left_alone() {
+ let html = r#"<button class="subscribe">Subscribe <img src="/icon.png" alt=""/></button>"#;
+ assert_eq!(unwrap_image_wrappers(html), html);
+ // And an image-free control is untouched too.
+ let plain = r#"<button>Share</button>"#;
+ assert_eq!(unwrap_image_wrappers(plain), plain);
+ }
+
+ #[test]
+ fn nested_and_unclosed_wrappers_do_not_derail_the_scan() {
+ let nested = r#"<button><button><img src="/a.png"/></button></button><p>after</p>"#;
+ let out = unwrap_image_wrappers(nested);
+ assert!(!out.contains("button"), "{out}");
+ assert!(out.contains("/a.png") && out.contains("after"), "{out}");
+ // An open tag that never closes is passed through rather than eating
+ // the rest of the document.
+ let unclosed = r#"<button><img src="/a.png"/><p>rest</p>"#;
+ assert!(unwrap_image_wrappers(unclosed).contains("rest"));
+ }
+
+ #[test]
+ fn srcset_descriptors_pick_the_widest_usable_candidate() {
+ assert_eq!(
+ best_from_srcset("/a.png 150w, /b.png 800w, /c.png 4000w").as_deref(),
+ Some("/b.png")
+ );
+ // Density descriptors work as an ordering too.
+ assert_eq!(
+ best_from_srcset("/a.png 1x, /b.png 2x").as_deref(),
+ Some("/b.png")
+ );
+ // A bare URL with no descriptor is still a candidate.
+ assert_eq!(best_from_srcset("/only.png").as_deref(), Some("/only.png"));
+ // Every candidate too wide: take the narrowest rather than nothing.
+ assert_eq!(
+ best_from_srcset("/big.png 3000w, /huge.png 5000w").as_deref(),
+ Some("/big.png")
+ );
+ assert_eq!(best_from_srcset(" ").as_deref(), None);
+ }
+
+ #[test]
+ fn plausibility_is_about_shape_not_host() {
+ for good in [
+ "https://cdn.dev/a.jpg?id=1&width=980",
+ "/_next/image?url=https%3A%2F%2Fx.dev%2Fa.png&w=3840&q=75",
+ "https://substackcdn.com/image/fetch/$s_!y1,w_1456/a.jpeg",
+ ] {
+ assert!(plausible_url(good), "{good} should be usable");
+ }
+ for bad in [
+ "",
+ "data:image/svg+xml,%3Csvg%3E%3C/svg%3E",
+ "https://cdn.dev/resize/{width}/a.jpg",
+ r#"{"src":"https://cdn.dev/a.jpg"}"#,
+ "https://cdn.dev/a.jpg 300w, https://cdn.dev/b.jpg 600w",
+ "https://cdn.dev/a.jpg%20300w",
+ ] {
+ assert!(!plausible_url(bad), "{bad} should be rejected");
+ }
+ }
+}
diff --git a/src/images/refs.rs b/src/images/refs.rs
new file mode 100644
index 0000000..7e36c4f
--- /dev/null
+++ b/src/images/refs.rs
@@ -0,0 +1,156 @@
+//! Finding the images an article references.
+//!
+//! The input here is already-normalized markup (see [`super::normalize`]), so
+//! every `<img>` is expected to carry a plain, usable `src`.
+
+use std::collections::HashSet;
+
+use scraper::{Html, Selector};
+use url::Url;
+
+/// One `<img>` found in article markup, with the caption of its `<figure>` if any.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ImgRef {
+ pub src: String,
+ pub alt: String,
+ pub caption: Option<String>,
+}
+
+/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup.
+pub fn extract_img_refs(html: &str) -> Vec<ImgRef> {
+ let doc = scraper::Html::parse_fragment(html);
+ let Ok(img_sel) = scraper::Selector::parse("img") else {
+ return Vec::new();
+ };
+ let cap_sel = scraper::Selector::parse("figcaption").ok();
+
+ let mut out = Vec::new();
+ let mut seen: Vec<String> = Vec::new();
+ for el in doc.select(&img_sel) {
+ let Some(src) = el.value().attr("src") else {
+ continue;
+ };
+ let src = src.trim();
+ if src.is_empty() || src.starts_with("data:") {
+ continue;
+ }
+ if seen.iter().any(|s| s == src) {
+ continue;
+ }
+ seen.push(src.to_string());
+ let alt = el
+ .value()
+ .attr("alt")
+ .unwrap_or_default()
+ .trim()
+ .to_string();
+ // Walk up to an enclosing <figure> and take its caption, if any.
+ let mut caption = None;
+ if let Some(cap_sel) = &cap_sel {
+ let mut cursor = el.parent();
+ while let Some(node) = cursor {
+ if let Some(elem) = scraper::ElementRef::wrap(node) {
+ if elem.value().name() == "figure" {
+ caption = elem.select(cap_sel).next().map(|c| {
+ c.text()
+ .collect::<String>()
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ")
+ });
+ break;
+ }
+ cursor = elem.parent();
+ } else {
+ break;
+ }
+ }
+ }
+ out.push(ImgRef {
+ src: src.to_string(),
+ alt,
+ caption: caption.filter(|c| !c.is_empty()),
+ });
+ }
+ out
+}
+
+/// Absolute image URLs referenced by `html`, resolved against `base_url` (§3.3).
+///
+/// Every image an article carries is kept: a photo essay with thirty pictures is
+/// a photo essay, and the issue-wide byte budget is the real backstop.
+pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> {
+ let Ok(selector) = Selector::parse("img") else {
+ return Vec::new();
+ };
+ let base = Url::parse(base_url).ok();
+ let document = Html::parse_fragment(html);
+ let mut seen: HashSet<String> = HashSet::new();
+ let mut out: Vec<String> = Vec::new();
+ for element in document.select(&selector) {
+ let raw = element
+ .value()
+ .attr("src")
+ .or_else(|| element.value().attr("data-src"))
+ .map(str::trim)
+ .filter(|s| !s.is_empty());
+ let Some(raw) = raw else { continue };
+ let resolved = match Url::parse(raw) {
+ Ok(u) => Some(u),
+ Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()),
+ };
+ let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else {
+ continue;
+ };
+ let url = url.to_string();
+ if seen.insert(url.clone()) {
+ out.push(url);
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn image_collection_resolves_and_keeps_every_image() {
+ let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#);
+ html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#);
+ for i in 0..20 {
+ html.push_str(&format!(r#"<img src="/n{i}.png">"#));
+ }
+ let urls = collect_image_urls(&html, "https://blog.dev/posts/one");
+ // Two named images, the data-src one, and all twenty of the rest: no cap.
+ assert_eq!(urls.len(), 23);
+ assert_eq!(urls[0], "https://blog.dev/a.png");
+ assert_eq!(urls[1], "https://cdn.dev/b.png");
+ assert_eq!(urls[2], "https://blog.dev/posts/c.png");
+ // Duplicates and data: URIs never appear.
+ assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1);
+ assert!(!urls.iter().any(|u| u.starts_with("data:")));
+ assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty());
+ }
+
+ // -----------------------------------------------------------------------
+ // Image normalization — each case is a page shape seen in a real issue.
+ // -----------------------------------------------------------------------
+
+ #[test]
+ fn extracts_img_refs_with_captions() {
+ let html = r#"<p>hi</p>
+ <figure><img src="https://e.g/a.png" alt="A diagram"/>
+ <figcaption>Figure 1: the thing</figcaption></figure>
+ <img src="https://e.g/b.jpg"/>
+ <img src="data:image/png;base64,zz"/>
+ <img src="https://e.g/a.png" alt="dupe"/>"#;
+ let refs = extract_img_refs(html);
+ assert_eq!(refs.len(), 2);
+ assert_eq!(refs[0].src, "https://e.g/a.png");
+ assert_eq!(refs[0].alt, "A diagram");
+ assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing"));
+ assert_eq!(refs[1].src, "https://e.g/b.jpg");
+ assert!(refs[1].caption.is_none());
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index c74f837..5fd4c4a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -20,7 +20,9 @@ pub mod db;
pub mod dedupe;
pub mod epub;
pub mod extract;
+pub mod html;
pub mod http;
+pub mod images;
pub mod miniflux;
pub mod pipeline;
pub mod publish;
diff --git a/src/world.rs b/src/world.rs
index a72121e..fa87175 100644
--- a/src/world.rs
+++ b/src/world.rs
@@ -9,7 +9,7 @@ use serde::Deserialize;
use url::Url;
use crate::curate::llm::{LlmClient, LlmError};
-use crate::epub::images::text_escape;
+use crate::html::text_escape;
use crate::types::{WorldBriefing, WorldBriefingSection, WorldEvent};
type NodeRef<'a> = <scraper::ElementRef<'a> as std::ops::Deref>::Target;