From db19d082579e8ad25cff154bdf7126d444d6d449 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Tue, 18 Aug 2026 03:35:21 +0000 Subject: [PATCH] Break up the image and EPUB god files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image fixes left two problems of navigation. Image handling was spread across `extract.rs` (300 lines of normalization) and `epub/images.rs` (download, re-encode, markup rewriting, plus generic HTML helpers that comments, world and x4 were all reaching into a module named "images" to borrow). And `epub/build.rs` had grown to 1,148 lines of code covering cover rendering, ten askama templates, every chapter renderer and the zip assembly. New homes: - `src/html.rs` — markup helpers that do not care what the markup is about: tag scanning, attribute parsing, entity decoding, escaping, XHTML fixups, reading a fragment as text. Previously scattered between `epub/images.rs` and `extract.rs`. - `src/images/` — one module per stage of an article's images, in the order they run: `normalize` (make `` usable, pre-readability), `refs` (what an article references), `fetch` + `encode` (download and re-encode per edition), `embed` (point the markup at what shipped). - `src/epub/{cover,chapters,build}.rs` — the cover, the chapter renderers, and the ordering plus assembly that puts them together. `epub/fixtures.rs` takes the shared test issue, which was a public module wedged inside `build.rs`. - `src/curate/profile/themes.rs` — a 260-line keyword table that sat in the middle of the profile logic. `curate::html_to_text` is renamed `prompt_text`: it is a different function from `html::html_to_text` (collapses whitespace, no DOM, sized for prompt budgets) and sharing a name with it was a trap. Largest module drops from 1,148 code lines to 828, and no file mixes two subjects. Behaviour is unchanged: 231 lib tests plus 25 integration tests green, and the real-world audit over issues 1–3 still reports 214 images referenced, 214 shown, 0 placeholders, 0 orphaned assets. `image_audit` gains `--epub-out DIR`, which writes a readable EPUB of the audited articles so images can be checked on a device. Co-Authored-By: Claude Opus 5 --- README.md | 34 +- examples/image_audit.rs | 94 +- src/comments.rs | 2 +- src/curate/editorial.rs | 6 +- src/curate/mod.rs | 14 +- src/curate/{profile.rs => profile/mod.rs} | 319 +----- src/curate/profile/themes.rs | 354 ++++++ src/curate/score.rs | 4 +- src/curate/select.rs | 4 +- src/dedupe.rs | 6 +- src/epub/build.rs | 1242 +-------------------- src/epub/chapters.rs | 651 +++++++++++ src/epub/cover.rs | 423 +++++++ src/epub/fixtures.rs | 182 +++ src/epub/images.rs | 963 ---------------- src/epub/mod.rs | 5 +- src/epub/x4.rs | 2 +- src/extract.rs | 609 +--------- src/html.rs | 356 ++++++ src/images/embed.rs | 193 ++++ src/images/encode.rs | 322 ++++++ src/images/fetch.rs | 159 +++ src/images/mod.rs | 25 + src/images/normalize.rs | 516 +++++++++ src/images/refs.rs | 156 +++ src/lib.rs | 2 + src/world.rs | 2 +- 27 files changed, 3513 insertions(+), 3132 deletions(-) rename src/curate/{profile.rs => profile/mod.rs} (79%) create mode 100644 src/curate/profile/themes.rs create mode 100644 src/epub/chapters.rs create mode 100644 src/epub/cover.rs create mode 100644 src/epub/fixtures.rs delete mode 100644 src/epub/images.rs create mode 100644 src/html.rs create mode 100644 src/images/embed.rs create mode 100644 src/images/encode.rs create mode 100644 src/images/fetch.rs create mode 100644 src/images/mod.rs create mode 100644 src/images/normalize.rs create mode 100644 src/images/refs.rs diff --git a/README.md b/README.md index 1bed58e..87b1bd0 100644 --- a/README.md +++ b/README.md @@ -399,10 +399,32 @@ extraction → prefilter → selection (both the `--skip-llm` route and a `MockBackend` DeepSeek route) → editorial → both EPUB editions → publish → OPDS and database rows, with no network access anywhere. -Layout: `src/pipeline.rs` wires the stages; `src/{dedupe,extract,social,curate, -comments,world,epub,publish,server}` implement them; `src/auth.rs` owns the rating -token formula used by both the EPUB writer and the server; `src/types.rs` is the -contract between stages. +### Layout + +`src/pipeline.rs` wires the stages; `src/types.rs` is the contract between them; +`src/auth.rs` owns the rating token formula, shared by the EPUB writer and the +server. The stages themselves: + +```text +miniflux.rs ingest curate/ scoring and selection +dedupe.rs clustering prefilter, llm, score, select, editorial +extract.rs body text profile/ the reader's taste profile +images/ article images comments.rs discussion chapters + normalize usable world.rs the world briefing + refs what's there epub/ the two editions + fetch download chapters, cover, build, x4 + encode re-encode publish.rs BookOrbit + XTC + embed into the page server.rs ratings, OPDS +html.rs markup helpers db.rs SQLite +``` + +Two modules are worth knowing about before you go looking for their contents. +`src/html.rs` holds the generic markup helpers — tag scanning, escaping, entity +decoding, XHTML fixups — that extraction, images, comments and the world briefing +all need; put anything that works on markup without caring what the markup is +*about* there. `src/images/` owns every stage of an article's images, which is +otherwise the kind of concern that smears itself across extraction and EPUB +building; see its module docs for the order the stages run in. ### Auditing images against real articles @@ -419,7 +441,9 @@ cargo run --release --example image_audit -- \ It prints per-article `refs / embedded / shown / placeholders`, a tally of loss reasons, and totals. Pages are cached on first run, so a change can be measured against byte-identical input; `--dump ` lists the URLs one -article resolved to. It needs the network and is not part of `cargo test`. +article resolved to, and `--epub-out DIR` writes a readable EPUB of the audited +articles so the images can be looked at on a device rather than counted in a +table. It needs the network and is not part of `cargo test`. --- diff --git a/examples/image_audit.rs b/examples/image_audit.rs index aa73217..cc37b8e 100644 --- a/examples/image_audit.rs +++ b/examples/image_audit.rs @@ -20,9 +20,14 @@ use std::collections::BTreeMap; use std::io::Read; use std::path::{Path, PathBuf}; -use daily_epub::epub::{build, images}; +use daily_epub::epub::build; use daily_epub::extract::{self, Extractor}; -use daily_epub::types::{Article, EntryId, ExtractMethod, ImageAsset, Pick, SourceKind, SourceRef}; +use daily_epub::html; +use daily_epub::images; +use daily_epub::types::{ + Article, Colophon, Editorial, EntryId, ExtractMethod, ImageAsset, Issue, IssueMeta, Lineup, + Pick, SourceKind, SourceRef, +}; /// One article we are auditing, pulled back out of a published EPUB. #[derive(Debug, Clone)] @@ -60,17 +65,23 @@ async fn main() { let mut cache = PathBuf::from("/tmp/daily-epub-audit-cache"); let mut dump: Option<String> = None; + let mut epub_out: Option<PathBuf> = None; let mut epubs: Vec<PathBuf> = Vec::new(); let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { match arg.as_str() { "--cache" => cache = PathBuf::from(args.next().expect("--cache needs a path")), "--dump" => dump = Some(args.next().expect("--dump needs a title substring")), + "--epub-out" => { + epub_out = Some(PathBuf::from(args.next().expect("--epub-out needs a path"))) + } other => epubs.push(PathBuf::from(other)), } } if epubs.is_empty() { - eprintln!("usage: image_audit [--cache DIR] <issue.epub>..."); + eprintln!( + "usage: image_audit [--cache DIR] [--dump TITLE] [--epub-out DIR] <issue.epub>..." + ); std::process::exit(2); } std::fs::create_dir_all(&cache).expect("cache dir"); @@ -212,6 +223,10 @@ async fn main() { for (kind, n) in &kinds { println!(" {n:>4} {kind}"); } + if let Some(dir) = &epub_out { + write_audit_epub(dir, &picks, &assets); + } + if !skipped.is_empty() { println!("\n== unfetchable ({})", skipped.len()); for (t, e) in &skipped { @@ -286,7 +301,7 @@ async fn body_for( let _ = extractor; let readable = extract::readability(&html, &base).map_err(|e| e.to_string())?; Ok(extract::sanitize_with_base( - &extract::normalize_img_tags(&readable), + &images::normalize_img_tags(&readable), &base, )) } @@ -320,8 +335,8 @@ async fn raw_fetch(url: &str) -> Result<(Vec<u8>, String), String> { } fn pick_for(target: &Target, content_html: String) -> Pick { - let word_count = extract::word_count(&content_html); - let image_urls = extract::collect_image_urls(&content_html, &target.url); + let word_count = html::word_count(&content_html); + let image_urls = images::collect_image_urls(&content_html, &target.url); Pick { article: Article { id: target.entry_id, @@ -336,7 +351,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick { sources: vec![SourceRef { entry_id: target.entry_id, feed_id: 1, - feed_title: "Feed".into(), + feed_title: target.issue.clone(), category: None, kind: SourceKind::Feed, }], @@ -344,14 +359,14 @@ fn pick_for(target: &Target, content_html: String) -> Pick { url: target.url.clone(), author: None, feed_id: 1, - feed_title: "Feed".into(), + feed_title: host(&target.url), category: None, published_at: None, comments_url: None, social: vec![], extract_method: ExtractMethod::Readability, }, - section: "Audit".into(), + section: target.issue.clone(), position: 0, is_lead: false, summary: None, @@ -360,6 +375,67 @@ fn pick_for(target: &Target, content_html: String) -> Pick { } } +/// Build a readable EPUB out of the audited articles (`--epub-out DIR`). +/// +/// Not a regenerated issue — there is no editorial, no discussion chapters and +/// no world briefing here, and the real thing needs the database. It exists so +/// the images can be looked at on a device rather than counted in a table. +fn write_audit_epub(out_dir: &Path, picks: &[Pick], assets: &[ImageAsset]) { + let sections: Vec<String> = { + let mut seen: Vec<String> = Vec::new(); + for p in picks { + if !seen.contains(&p.section) { + seen.push(p.section.clone()); + } + } + seen + }; + let issue = Issue { + meta: IssueMeta { + date: "2026-08-16".parse().unwrap(), + issue_number: 0, + generated_at: jiff::Timestamp::now(), + display_date: "Image audit rebuild".into(), + article_count: picks.len() as i64, + section_count: sections.len() as i64, + total_words: picks.iter().map(|p| p.article.word_count).sum(), + reading_minutes: picks.iter().map(|p| p.article.reading_minutes()).sum(), + }, + lineup: Lineup { + date: "2026-08-16".parse().unwrap(), + picks: picks.to_vec(), + section_order: sections, + }, + editorial: Editorial { + front_page_html: "<p>Rebuilt from published issues by \ + <code>examples/image_audit.rs</code> to check image handling. \ + Articles are re-extracted live; editorial, discussions and the \ + world briefing are absent by design.</p>" + .into(), + section_intros: Default::default(), + summaries: Default::default(), + }, + world_briefing: None, + colophon: Colophon::default(), + }; + let cfg = daily_epub::config::Config::default(); + match daily_epub::epub::build_edition_with_images( + &issue, + daily_epub::types::Edition::Standard, + &cfg, + out_dir, + assets, + ) { + Ok(artifact) => println!( + "\nwrote {} ({:.1} MB, {} images)", + artifact.path.display(), + artifact.bytes as f64 / 1_048_576.0, + assets.len() + ), + Err(e) => eprintln!("epub build failed: {e}"), + } +} + /// Pull `(entry id, title, url)` out of every article chapter in an EPUB. fn targets_from_epub(path: &Path) -> Vec<Target> { let issue = path diff --git a/src/comments.rs b/src/comments.rs index 8093e9e..d66a663 100644 --- a/src/comments.rs +++ b/src/comments.rs @@ -9,7 +9,7 @@ use std::collections::HashSet; use futures::StreamExt; use serde_json::Value; -use crate::epub::images::{text_escape, to_xhtml}; +use crate::html::{text_escape, to_xhtml}; use crate::types::{Comment, CommentThread, Discussion, Pick, SocialSource}; /// Top-level threads kept per source (§3.7). diff --git a/src/curate/editorial.rs b/src/curate/editorial.rs index 1109806..35ce406 100644 --- a/src/curate/editorial.rs +++ b/src/curate/editorial.rs @@ -14,7 +14,7 @@ use std::fmt::Write as _; use serde::{Deserialize, Serialize}; use super::llm::{LlmClient, LlmError}; -use super::{escape_html, html_to_text, text_to_paragraphs, truncate_tokens, truncate_words}; +use super::{escape_html, prompt_text, text_to_paragraphs, truncate_tokens, truncate_words}; use crate::types::{ArticleId, Editorial, Lineup, Pick}; /// Article text is truncated to roughly this many tokens per summary call (§3.6). @@ -116,7 +116,7 @@ pub async fn summarize_article( temperature: f32, ) -> Result<String, LlmError> { llm.meter.check_budget()?; - let body = truncate_tokens(&html_to_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET); + let body = truncate_tokens(&prompt_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET); let mut prompt = String::with_capacity(body.len() + SUMMARY_INSTRUCTIONS.len() + 256); prompt.push_str(SUMMARY_INSTRUCTIONS); let _ = write!( @@ -309,7 +309,7 @@ fn social_note(pick: &Pick) -> String { /// The article's own opening words, used when no LLM summary exists (§3.6). pub fn excerpt_summary(pick: &Pick) -> String { let text = truncate_words( - &html_to_text(&pick.article.content_html), + &prompt_text(&pick.article.content_html), FALLBACK_SUMMARY_WORDS, ); if text.is_empty() { diff --git a/src/curate/mod.rs b/src/curate/mod.rs index 43ca2ac..93c4130 100644 --- a/src/curate/mod.rs +++ b/src/curate/mod.rs @@ -164,7 +164,11 @@ pub fn approx_tokens(text: &str) -> usize { /// Strip markup and collapse whitespace, so article bodies can go into prompts /// as plain text (cheaper and less confusing for the model than raw HTML). -pub fn html_to_text(html: &str) -> String { +/// +/// Deliberately not [`crate::html::html_to_text`]: this one collapses runs of +/// whitespace and never builds a DOM, because it runs over every candidate +/// body on every run and only has to be good enough to size a prompt. +pub fn prompt_text(html: &str) -> String { /// Does `tail` open the named element, i.e. `<name` or `</name`? fn opens(tail: &str, name: &str) -> bool { let bytes = tail.as_bytes(); @@ -272,11 +276,11 @@ mod tests { fn html_becomes_readable_text() { let html = "<h1>Title</h1><p>First & best.</p><script>alert('x')</script>\ <p>Second<br/>line</p><style>p{color:red}</style>"; - assert_eq!(html_to_text(html), "Title First & best. Second line"); - assert_eq!(html_to_text(""), ""); - assert_eq!(html_to_text("no markup at all"), "no markup at all"); + assert_eq!(prompt_text(html), "Title First & best. Second line"); + assert_eq!(prompt_text(""), ""); + assert_eq!(prompt_text("no markup at all"), "no markup at all"); // Unicode survives byte-wise walking. - assert_eq!(html_to_text("<p>café — naïve</p>"), "café — naïve"); + assert_eq!(prompt_text("<p>café — naïve</p>"), "café — naïve"); } #[test] diff --git a/src/curate/profile.rs b/src/curate/profile/mod.rs similarity index 79% rename from src/curate/profile.rs rename to src/curate/profile/mod.rs index 162b6f9..a7491b1 100644 --- a/src/curate/profile.rs +++ b/src/curate/profile/mod.rs @@ -99,303 +99,9 @@ fn xml_unescape(s: &str) -> String { .replace("&", "&") } -// --------------------------------------------------------------------------- -// Theming (§3.6a) -// --------------------------------------------------------------------------- +pub mod themes; -/// Theme name → lowercase keywords, in priority order. The first theme whose -/// keyword appears in the interest name wins, so the table is ordered from the -/// most specific bucket to the most general. -const THEMES: &[(&str, &[&str])] = &[ - ( - "Systems & languages", - &[ - "rust", - "zig", - "lua", - "assembly", - "compiler", - "systems programming", - "concurrency", - "async", - "memory", - "simd", - "zero-copy", - "parser", - "fuzz", - "static analysis", - "functional programming", - "data structures", - "algorithm", - "performance", - "profil", - "python", - "typescript", - "node.js", - "wasm", - "webassembly", - "reactive programming", - "lsp", - ], - ), - ( - "Databases & data", - &[ - "database", - "sql", - "postgres", - "query", - "vector databases", - "bloom", - "compression", - "data engineering", - "knowledge graph", - "message queue", - "distributed systems", - "crdt", - "microservices", - "system design", - ], - ), - ( - "AI & machine learning", - &[ - "ai", - "llm", - "machine learning", - "nlp", - "rag", - "prompt", - "agent", - "claude", - "eval", - ], - ), - ( - "Web, infra & devtools", - &[ - "web", - "css", - "htmx", - "svelte", - "react", - "pwa", - "api", - "developer", - "devops", - "docker", - "git", - "observability", - "cloud", - "cdn", - "edge", - "networking", - "mesh", - "ipfs", - "activitypub", - "atproto", - "search engines", - "tauri", - "design systems", - "cybersecurity", - "cryptography", - "monitoring", - "spatial computing", - "webrtc", - "webgpu", - "webgl", - "webcodecs", - "android", - "mobile", - "code generation", - ], - ), - ( - "Self-hosting, RSS & the indie web", - &[ - "self-host", - "self host", - "homelab", - "rss", - "feed reader", - "indie web", - "personal website", - "personal wiki", - "digital garden", - "static site", - "static sites", - "personal archiving", - "offline-first", - "privacy", - "open source", - "licensing", - "side projects", - "digital nomad", - "remote living", - "off-grid", - "quantified self", - "cloudflare workers", - ], - ), - ( - "E-ink, terminals & hardware", - &[ - "e-ink", - "eink", - "writerdeck", - "tmux", - "neovim", - "terminal", - "text editor", - "editor", - "window manager", - "keyboard", - "hardware", - "electronics", - "calibre", - "ghostty", - "retro computing", - "low-tech", - "manufacturing", - "typography", - "linux", - ], - ), - ( - "Games & interactive fiction", - &[ - "game", - "gaming", - "minecraft", - "mud", - "interactive fiction", - "inform7", - "bevy", - "sim racing", - "modding", - ], - ), - ( - "Creative coding & digital art", - &[ - "generative", - "creative coding", - "creative automation", - "glitch", - "procedural", - "digital art", - "gaussian splatting", - "cellular automata", - "sonification", - "code visualization", - "photo", - "photography", - "agent-based", - ], - ), - ( - "Writing, books & PKM", - &[ - "writing", - "write", - "reading", - "book", - "essay", - "poetry", - "literature", - "note-taking", - "journaling", - "pkm", - "knowledge management", - "obsidian", - "markdown", - "commonplace", - "longform", - "long-form", - "fiction", - "worldbuilding", - "sci-fi", - "science fiction", - "speculative", - "podcast", - "narrative", - "choice architecture", - ], - ), - ( - "Science, space & nature", - &[ - "space", - "aerospace", - "aviation", - "science", - "neuroscience", - "bioinformatics", - "nature", - "cognitive", - "social science", - ], - ), - ( - "History, policy & culture", - &[ - "history", - "policy", - "culture", - "lgbt", - "queer", - "startups", - "apple", - "boston", - "criticism", - "internet history", - "computing history", - ], - ), - ( - "Outdoors, coffee & everyday life", - &[ - "hiking", - "camping", - "running", - "trail", - "coffee", - "cats", - "films", - "music", - "engineering", - ], - ), -]; - -/// Fallback bucket for interests no keyword claims. -const OTHER_THEME: &str = "Other standing interests"; - -/// Group ~220 interests into readable themes for the prompt (§3.6). -/// -/// Deterministic: theme order follows [`THEMES`], members are sorted -/// case-insensitively, and empty themes are omitted. -pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> { - let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1]; - for interest in interests { - let lower = interest.to_lowercase(); - let idx = THEMES - .iter() - .position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k))) - .unwrap_or(THEMES.len()); - buckets[idx].push(interest.clone()); - } - let mut out = Vec::new(); - for (idx, mut members) in buckets.into_iter().enumerate() { - if members.is_empty() { - continue; - } - members.sort_by_key(|m| (m.to_lowercase(), m.clone())); - let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME); - out.push((name.to_string(), members)); - } - out -} +pub use themes::group_into_themes; // --------------------------------------------------------------------------- // Document assembly (§3.6) @@ -870,27 +576,6 @@ mod tests { ); } - #[test] - fn theming_is_deterministic_and_total() { - let list = interests(); - let a = group_into_themes(&list); - let b = group_into_themes(&list); - assert_eq!(a, b); - let grouped: usize = a.iter().map(|(_, m)| m.len()).sum(); - assert_eq!( - grouped, - list.len(), - "every interest lands in exactly one theme" - ); - assert!(a.len() >= 6, "expected several themes, got {}", a.len()); - // Members are sorted within a theme. - for (_, members) in &a { - let mut sorted = members.clone(); - sorted.sort_by_key(|m| (m.to_lowercase(), m.clone())); - assert_eq!(members, &sorted); - } - } - #[test] fn profile_document_is_deterministic_and_complete() { let list = interests(); diff --git a/src/curate/profile/themes.rs b/src/curate/profile/themes.rs new file mode 100644 index 0000000..f2d5483 --- /dev/null +++ b/src/curate/profile/themes.rs @@ -0,0 +1,354 @@ +//! Grouping the reader's ~220 Scour interests into readable themes (§3.6a). +//! +//! A lookup table and one function over it. It lives in its own file because the +//! table is long and almost never changes, while the profile logic around it +//! does — and a 260-line const in the middle of that logic is a wall to scroll +//! past, not something to read. + +/// Theme name → lowercase keywords, in priority order. The first theme whose +/// keyword appears in the interest name wins, so the table is ordered from the +/// most specific bucket to the most general. +const THEMES: &[(&str, &[&str])] = &[ + ( + "Systems & languages", + &[ + "rust", + "zig", + "lua", + "assembly", + "compiler", + "systems programming", + "concurrency", + "async", + "memory", + "simd", + "zero-copy", + "parser", + "fuzz", + "static analysis", + "functional programming", + "data structures", + "algorithm", + "performance", + "profil", + "python", + "typescript", + "node.js", + "wasm", + "webassembly", + "reactive programming", + "lsp", + ], + ), + ( + "Databases & data", + &[ + "database", + "sql", + "postgres", + "query", + "vector databases", + "bloom", + "compression", + "data engineering", + "knowledge graph", + "message queue", + "distributed systems", + "crdt", + "microservices", + "system design", + ], + ), + ( + "AI & machine learning", + &[ + "ai", + "llm", + "machine learning", + "nlp", + "rag", + "prompt", + "agent", + "claude", + "eval", + ], + ), + ( + "Web, infra & devtools", + &[ + "web", + "css", + "htmx", + "svelte", + "react", + "pwa", + "api", + "developer", + "devops", + "docker", + "git", + "observability", + "cloud", + "cdn", + "edge", + "networking", + "mesh", + "ipfs", + "activitypub", + "atproto", + "search engines", + "tauri", + "design systems", + "cybersecurity", + "cryptography", + "monitoring", + "spatial computing", + "webrtc", + "webgpu", + "webgl", + "webcodecs", + "android", + "mobile", + "code generation", + ], + ), + ( + "Self-hosting, RSS & the indie web", + &[ + "self-host", + "self host", + "homelab", + "rss", + "feed reader", + "indie web", + "personal website", + "personal wiki", + "digital garden", + "static site", + "static sites", + "personal archiving", + "offline-first", + "privacy", + "open source", + "licensing", + "side projects", + "digital nomad", + "remote living", + "off-grid", + "quantified self", + "cloudflare workers", + ], + ), + ( + "E-ink, terminals & hardware", + &[ + "e-ink", + "eink", + "writerdeck", + "tmux", + "neovim", + "terminal", + "text editor", + "editor", + "window manager", + "keyboard", + "hardware", + "electronics", + "calibre", + "ghostty", + "retro computing", + "low-tech", + "manufacturing", + "typography", + "linux", + ], + ), + ( + "Games & interactive fiction", + &[ + "game", + "gaming", + "minecraft", + "mud", + "interactive fiction", + "inform7", + "bevy", + "sim racing", + "modding", + ], + ), + ( + "Creative coding & digital art", + &[ + "generative", + "creative coding", + "creative automation", + "glitch", + "procedural", + "digital art", + "gaussian splatting", + "cellular automata", + "sonification", + "code visualization", + "photo", + "photography", + "agent-based", + ], + ), + ( + "Writing, books & PKM", + &[ + "writing", + "write", + "reading", + "book", + "essay", + "poetry", + "literature", + "note-taking", + "journaling", + "pkm", + "knowledge management", + "obsidian", + "markdown", + "commonplace", + "longform", + "long-form", + "fiction", + "worldbuilding", + "sci-fi", + "science fiction", + "speculative", + "podcast", + "narrative", + "choice architecture", + ], + ), + ( + "Science, space & nature", + &[ + "space", + "aerospace", + "aviation", + "science", + "neuroscience", + "bioinformatics", + "nature", + "cognitive", + "social science", + ], + ), + ( + "History, policy & culture", + &[ + "history", + "policy", + "culture", + "lgbt", + "queer", + "startups", + "apple", + "boston", + "criticism", + "internet history", + "computing history", + ], + ), + ( + "Outdoors, coffee & everyday life", + &[ + "hiking", + "camping", + "running", + "trail", + "coffee", + "cats", + "films", + "music", + "engineering", + ], + ), +]; + +/// Fallback bucket for interests no keyword claims. +const OTHER_THEME: &str = "Other standing interests"; + +/// Group ~220 interests into readable themes for the prompt (§3.6). +/// +/// Deterministic: theme order follows [`THEMES`], members are sorted +/// case-insensitively, and empty themes are omitted. +pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> { + let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1]; + for interest in interests { + let lower = interest.to_lowercase(); + let idx = THEMES + .iter() + .position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k))) + .unwrap_or(THEMES.len()); + buckets[idx].push(interest.clone()); + } + let mut out = Vec::new(); + for (idx, mut members) in buckets.into_iter().enumerate() { + if members.is_empty() { + continue; + } + members.sort_by_key(|m| (m.to_lowercase(), m.clone())); + let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME); + out.push((name.to_string(), members)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn interests() -> Vec<String> { + [ + "Rust", + "Zig", + "Postgres", + "SQLite", + "Kubernetes", + "E-ink", + "Keyboards", + "Retrocomputing", + "Bicycles", + "Trail running", + "Coffee", + "Bread baking", + "Cartography", + "Typography", + "Ambient music", + "Science fiction", + "Local politics", + "Boston", + "Machine learning", + "Self-hosting", + ] + .iter() + .map(|s| s.to_string()) + .collect() + } + + #[test] + fn theming_is_deterministic_and_total() { + let list = interests(); + let a = group_into_themes(&list); + let b = group_into_themes(&list); + assert_eq!(a, b); + let grouped: usize = a.iter().map(|(_, m)| m.len()).sum(); + assert_eq!( + grouped, + list.len(), + "every interest lands in exactly one theme" + ); + assert!(a.len() >= 6, "expected several themes, got {}", a.len()); + // Members are sorted within a theme. + for (_, members) in &a { + let mut sorted = members.clone(); + sorted.sort_by_key(|m| (m.to_lowercase(), m.clone())); + assert_eq!(members, &sorted); + } + } +} diff --git a/src/curate/score.rs b/src/curate/score.rs index 9636ce7..49be464 100644 --- a/src/curate/score.rs +++ b/src/curate/score.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use super::llm::{LlmClient, LlmError, strip_code_fence}; -use super::{html_to_text, truncate_words}; +use super::{prompt_text, truncate_words}; use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind}; /// Words of article text sent per candidate in stage A (§3.6). @@ -155,7 +155,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String { ); let _ = writeln!(block, "social: {}", social_line(candidate)); let _ = writeln!(block, "came via: {}", sources_line(candidate)); - let excerpt = truncate_words(&html_to_text(&a.content_html), EXCERPT_WORDS); + let excerpt = truncate_words(&prompt_text(&a.content_html), EXCERPT_WORDS); let _ = writeln!( block, "excerpt: {}", diff --git a/src/curate/select.rs b/src/curate/select.rs index 4ab7dda..75d3db3 100644 --- a/src/curate/select.rs +++ b/src/curate/select.rs @@ -16,7 +16,7 @@ use jiff::civil::Date; use serde::{Deserialize, Serialize}; use super::llm::{LlmClient, LlmError, strip_code_fence}; -use super::{html_to_text, truncate_words}; +use super::{prompt_text, truncate_words}; use crate::types::{ArticleId, Lineup, Pick, ScoredArticle, SourceKind, WORLD_BRIEFING_SECTION}; /// How many candidates are offered to stage B (§3.6). @@ -154,7 +154,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String { "" } ); - let blurb = truncate_words(&html_to_text(&a.content_html), BLURB_WORDS); + let blurb = truncate_words(&prompt_text(&a.content_html), BLURB_WORDS); if !blurb.is_empty() { let _ = writeln!(block, "opening: {blurb}"); } diff --git a/src/dedupe.rs b/src/dedupe.rs index 77fd486..8517708 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -162,7 +162,7 @@ fn is_enclosure_only(raw_content: &str) -> bool { || lower.contains("<video") || lower.contains("<embed") || lower.contains("<iframe"); - embeds && crate::extract::word_count(raw_content) < 25 + embeds && crate::html::word_count(raw_content) < 25 } /// Classify which kind of feed an entry arrived through, for the sources list (§3.2, §3.5). @@ -345,7 +345,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article let best = members .iter() .enumerate() - .max_by_key(|(_, (entry, _))| (crate::extract::word_count(&entry.raw_content), -(entry.id))) + .max_by_key(|(_, (entry, _))| (crate::html::word_count(&entry.raw_content), -(entry.id))) .map(|(i, _)| i) .unwrap_or(0); let (best_entry, canonical) = &members[best]; @@ -385,7 +385,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article .filter_map(|(e, _)| e.author.clone()) .find(|a| !a.trim().is_empty()); - let word_count = crate::extract::word_count(&best_entry.raw_content); + let word_count = crate::html::word_count(&best_entry.raw_content); Article { id: 0, diff --git a/src/epub/build.rs b/src/epub/build.rs index 2bb1e82..8e4575d 100644 --- a/src/epub/build.rs +++ b/src/epub/build.rs @@ -1,22 +1,34 @@ -//! `epub-builder` assembly and askama chapter rendering (spec §3.10). +//! Chapter ordering and `epub-builder` assembly (spec §3.10). //! //! Structure: cover → From the Editor → In This Issue → sections (title page, -//! article chapters, discussion chapters) → World Briefing → colophon. +//! article chapters, discussion chapters) → World Briefing → colophon. The +//! chapters themselves are rendered by [`super::chapters`] and the cover by +//! [`super::cover`]; this module decides the order and zips the result. -use askama::Template; use epub_builder::{ EpubBuilder, EpubContent, EpubVersion, MetadataOpfV3, ReferenceType, ZipLibrary, }; use jiff::civil::Date; -use crate::comments; -use crate::types::{Edition, ImageAsset, Issue, Pick, SocialRef, Vote, WORLD_BRIEFING_SECTION}; -use crate::world; +use crate::types::{Edition, ImageAsset, Issue}; use super::EpubError; -use super::images; +use super::chapters::{ + render_colophon, render_front_page, render_in_this_issue, render_section_page, + render_world_briefing, section_names, +}; +use super::cover::{CoverAsset, render_cover_page}; use super::x4; +// Re-exported so callers keep one import path for "everything about building an +// issue"; the definitions live in the modules that own them. +pub use super::chapters::{ + TOKEN_LEN, prepare_body, rating_message, rating_token, rating_url, render_article, + render_discussion, social_line, +}; +pub use super::cover::{cover_badge, cover_size, render_cover}; +pub use super::fixtures; + /// EPUB3 `belongs-to-collection` name (§3.10). pub const COLLECTION_NAME: &str = "The Daily EPUB"; /// `id` the collection refinements point at (§3.10). @@ -25,21 +37,6 @@ pub const COLLECTION_ID: &str = "daily-epub-collection"; pub const CREATOR: &str = "The Daily EPUB"; /// `dc:language` (§3.10). pub const LANGUAGE: &str = "en"; -/// Characters of the hex HMAC kept in rating links (§3.9). -pub const TOKEN_LEN: usize = crate::auth::TOKEN_LEN; -#[derive(Debug, Clone, PartialEq)] -pub struct CoverAsset { - pub bytes: Vec<u8>, - pub filename: &'static str, - pub mime: &'static str, -} - -pub fn cover_href(edition: Edition) -> &'static str { - match edition { - Edition::Standard => "cover.png", - Edition::X4 => "cover.jpg", - } -} /// One rendered chapter ready to be added to the EPUB (§3.10). #[derive(Debug, Clone, PartialEq)] @@ -54,742 +51,7 @@ pub struct Chapter { pub toc_level: u8, } -// --------------------------------------------------------------------------- -// Rating links (§3.9) -// --------------------------------------------------------------------------- - -// One implementation, shared with the verifying side in [`crate::server`]: -// see [`crate::auth`] for the formula and the pinned test vector (§3.9). -pub use crate::auth::{rating_message, rating_token, rating_url}; - -// --------------------------------------------------------------------------- -// Templates -// --------------------------------------------------------------------------- - -#[derive(Template)] -#[template(path = "cover.svg", escape = "html")] -struct CoverSvg { - width: i64, - height: i64, - margin: i64, - inner_width: i64, - inner_height: i64, - border: i64, - hairline: i64, - center_x: i64, - masthead_y: i64, - masthead_size: i64, - rule_x1: i64, - rule_x2: i64, - rule_y: i64, - rule2_y: i64, - weekday: String, - weekday_y: i64, - weekday_size: i64, - long_date: String, - date_y: i64, - date_size: i64, - issue_number: i64, - issue_y: i64, - issue_size: i64, - stats_line: String, - stats_y: i64, - stats_size: i64, - /// Empty for the standard edition — see [`cover_badge`]. - edition_tag: String, - badge_x: i64, - badge_y: i64, - badge_width: i64, - badge_height: i64, - badge_radius: i64, - badge_text_y: i64, - badge_size: i64, - footer: String, - footer_y: i64, - footer_size: i64, -} - -#[derive(Template)] -#[template(path = "cover_page.xhtml", escape = "html")] -struct CoverPage { - title: String, - alt: String, - cover_href: &'static str, -} - -#[derive(Template)] -#[template(path = "front_page.xhtml", escape = "html")] -struct FrontPage { - title: String, - display_date: String, - issue_number: i64, - stats_line: String, - body_html: String, -} - -struct IndexEntry { - href: String, - title: String, - source: String, - reading_minutes: i64, - summary: String, -} - -struct IndexSection { - name: String, - entries: Vec<IndexEntry>, -} - -#[derive(Template)] -#[template(path = "in_this_issue.xhtml", escape = "html")] -struct InThisIssue { - title: String, - stats_line: String, - sections: Vec<IndexSection>, -} - -#[derive(Template)] -#[template(path = "section.xhtml", escape = "html")] -struct SectionPage { - title: String, - name: String, - intro: Option<String>, -} - -struct RatingLinks { - up_url: String, - down_url: String, -} - -#[derive(Template)] -#[template(path = "chapter.xhtml", escape = "html")] -struct ArticleChapter { - title: String, - article_title: String, - byline: Option<String>, - meta_line: String, - social_line: Option<String>, - summary: Option<String>, - excerpt_only: bool, - body_html: String, - rating: Option<RatingLinks>, - read_online_url: String, - discussion_href: Option<String>, -} - -#[derive(Template)] -#[template(path = "discussion.xhtml", escape = "html")] -struct DiscussionChapter { - title: String, - heading: String, - body_html: String, - article_href: String, -} - -#[derive(Template)] -#[template(path = "world_briefing.xhtml", escape = "html")] -struct WorldBriefingChapter { - title: String, - display_date: String, - body_html: String, -} - -#[derive(Template)] -#[template(path = "colophon.xhtml", escape = "html")] -struct ColophonChapter { - title: String, - issue_number: i64, - display_date: String, - generated_at: String, - model: String, - entries_fetched: i64, - feeds_seen: i64, - candidates: i64, - article_count: i64, - section_count: i64, - total_words: i64, - reading_line: String, - cost_usd: String, - generator_version: String, -} - -// --------------------------------------------------------------------------- -// Cover (§3.10) -// --------------------------------------------------------------------------- - -/// Cover pixel size per edition: 1200×1600 standard, 480×800 X4 (§3.10). -pub fn cover_size(edition: Edition) -> (u32, u32) { - match edition { - Edition::Standard => (1200, 1600), - Edition::X4 => x4::X4_SCREEN, - } -} - -fn reading_line(minutes: i64) -> String { - let (h, m) = (minutes / 60, minutes % 60); - if h > 0 { - format!("~{h}h {m}m read") - } else { - format!("~{m}m read") - } -} - -/// Split "Friday, August 15, 2026" into ("Friday", "August 15, 2026"). -fn split_display_date(display: &str) -> (String, String) { - match display.split_once(", ") { - Some((weekday, rest)) => (weekday.to_string(), rest.to_string()), - None => (String::new(), display.to_string()), - } -} - -/// Reversed badge printed on the cover so the two editions are told apart at -/// thumbnail size, where the title is unreadable (§3.10). -/// -/// Empty for the standard edition: an unmarked cover is the default, and the -/// presence of the slab is itself the signal. -pub fn cover_badge(edition: Edition) -> &'static str { - match edition { - Edition::Standard => "", - Edition::X4 => "X4 EDITION", - } -} - -fn cover_svg( - issue: &Issue, - edition: Edition, - width: u32, - height: u32, -) -> Result<String, EpubError> { - let w = i64::from(width); - let h = i64::from(height); - let margin = w / 15; - let (weekday, long_date) = split_display_date(&issue.meta.display_date); - // A solid bar between the stats line and the footer: at 100px wide in a - // library grid the black slab is the only thing still legible. - let badge_height = h / 16; - let badge_width = w * 2 / 5; - let tpl = CoverSvg { - width: w, - height: h, - margin, - inner_width: w - 2 * margin, - inner_height: h - 2 * margin, - border: (w / 300).max(2), - hairline: (w / 600).max(1), - center_x: w / 2, - masthead_y: h * 30 / 100, - masthead_size: w / 9, - rule_x1: margin + w / 12, - rule_x2: w - margin - w / 12, - rule_y: h * 34 / 100, - rule2_y: h * 53 / 100, - weekday, - weekday_y: h * 42 / 100, - weekday_size: w / 28, - long_date, - date_y: h * 47 / 100, - date_size: w / 22, - issue_number: issue.meta.issue_number, - issue_y: h * 60 / 100, - issue_size: w / 18, - stats_line: issue.meta.stats_line(), - stats_y: h * 66 / 100, - stats_size: w / 32, - edition_tag: cover_badge(edition).to_string(), - badge_x: (w - badge_width) / 2, - badge_y: h * 73 / 100, - badge_width, - badge_height, - badge_radius: badge_height / 2, - badge_text_y: h * 73 / 100 + badge_height * 7 / 10, - badge_size: badge_height * 11 / 20, - footer: "Assembled overnight \u{00b7} read offline".to_string(), - footer_y: h - margin - h / 25, - footer_size: w / 40, - }; - Ok(tpl.render()?) -} - -/// Render the standard cover as RGB PNG and the X4 cover as baseline RGB JPEG. -pub fn render_cover(issue: &Issue, edition: Edition) -> Result<CoverAsset, EpubError> { - let (width, height) = cover_size(edition); - let svg = cover_svg(issue, edition, width, height)?; - let pixmap = match rasterize(&svg, width, height) { - Some(pixmap) => pixmap, - None => { - tracing::warn!("no usable system fonts: falling back to a geometric cover"); - draw_fallback_cover(width, height, edition) - .ok_or_else(|| EpubError::Build("could not allocate the cover".into()))? - } - }; - Ok(CoverAsset { - bytes: encode_cover(pixmap, edition)?, - filename: cover_href(edition), - mime: if edition == Edition::X4 { - "image/jpeg" - } else { - "image/png" - }, - }) -} - -fn rasterize(svg: &str, width: u32, height: u32) -> Option<tiny_skia::Pixmap> { - let mut options = resvg::usvg::Options::default(); - options.fontdb_mut().load_system_fonts(); - if options.fontdb.is_empty() { - return None; - } - let tree = resvg::usvg::Tree::from_str(svg, &options) - .map_err(|e| tracing::warn!("cover svg did not parse: {e}")) - .ok()?; - let mut pixmap = tiny_skia::Pixmap::new(width, height)?; - pixmap.fill(tiny_skia::Color::WHITE); - let size = tree.size(); - let transform = tiny_skia::Transform::from_scale( - width as f32 / size.width(), - height as f32 / size.height(), - ); - resvg::render(&tree, transform, &mut pixmap.as_mut()); - Some(pixmap) -} - -/// Text-free cover used when font resolution fails — the build never fails (§3.10). -/// -/// The badge cannot carry its lettering here, but the slab itself still keeps -/// the editions apart in a thumbnail grid. -fn draw_fallback_cover(width: u32, height: u32, edition: Edition) -> Option<tiny_skia::Pixmap> { - let mut pixmap = tiny_skia::Pixmap::new(width, height)?; - pixmap.fill(tiny_skia::Color::WHITE); - let mut paint = tiny_skia::Paint::default(); - paint.set_color(tiny_skia::Color::BLACK); - paint.anti_alias = true; - - let w = width as f32; - let h = height as f32; - let margin = w / 15.0; - let rect = |x: f32, y: f32, rw: f32, rh: f32, pixmap: &mut tiny_skia::Pixmap| { - if let Some(r) = tiny_skia::Rect::from_xywh(x, y, rw, rh) { - let path = tiny_skia::PathBuilder::from_rect(r); - pixmap.fill_path( - &path, - &paint, - tiny_skia::FillRule::Winding, - tiny_skia::Transform::identity(), - None, - ); - } - }; - // Frame. - let border = (w / 300.0).max(2.0); - rect(margin, margin, w - 2.0 * margin, border, &mut pixmap); - rect( - margin, - h - margin - border, - w - 2.0 * margin, - border, - &mut pixmap, - ); - rect(margin, margin, border, h - 2.0 * margin, &mut pixmap); - rect( - w - margin - border, - margin, - border, - h - 2.0 * margin, - &mut pixmap, - ); - // Masthead slab plus body rules — a newspaper silhouette. - rect( - margin * 2.0, - h * 0.26, - w - 4.0 * margin, - h * 0.035, - &mut pixmap, - ); - for i in 0..8 { - let y = h * 0.42 + (i as f32) * h * 0.045; - let inset = if i % 3 == 2 { - margin * 4.0 - } else { - margin * 2.0 - }; - rect(inset, y, w - 2.0 * inset, border, &mut pixmap); - } - if !cover_badge(edition).is_empty() { - let badge_height = h / 16.0; - let badge_width = w * 2.0 / 5.0; - rect( - (w - badge_width) / 2.0, - h * 0.73, - badge_width, - badge_height, - &mut pixmap, - ); - } - Some(pixmap) -} - -fn encode_cover(pixmap: tiny_skia::Pixmap, edition: Edition) -> Result<Vec<u8>, EpubError> { - let (w, h) = (pixmap.width(), pixmap.height()); - let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied()) - .ok_or_else(|| EpubError::Build("cover pixel buffer had the wrong size".into()))?; - let rgb = image::DynamicImage::ImageRgba8(rgba).to_rgb8(); - let mut bytes = Vec::new(); - match edition { - Edition::Standard => image::DynamicImage::ImageRgb8(rgb).write_to( - &mut std::io::Cursor::new(&mut bytes), - image::ImageFormat::Png, - ), - Edition::X4 => image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 92) - .encode_image(&image::DynamicImage::ImageRgb8(rgb)), - } - .map_err(|e| EpubError::Build(format!("cover encoding failed: {e}")))?; - Ok(bytes) -} - -// --------------------------------------------------------------------------- -// Chapters (§3.10) -// --------------------------------------------------------------------------- - -/// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids. -pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String { - // Sanitize first: image rewriting emits our own trusted markup (including the - // `image-placeholder` class, which ammonia would otherwise strip). - let cleaned = ammonia::clean(html); - let rewritten = images::rewrite_img_srcs(&cleaned, images_); - images::to_xhtml(&rewritten) -} - -/// "▲ 342 on HN · 210 comments" (§3.10). -pub fn social_line(social: &[SocialRef]) -> Option<String> { - let mut parts: Vec<String> = Vec::new(); - let mut comments = 0i64; - for entry in social { - if entry.score > 0 { - parts.push(format!( - "\u{25b2} {} on {}", - entry.score, - entry.source.display_name() - )); - } - comments += entry.num_comments.max(0); - } - if comments > 0 { - let noun = if comments == 1 { "comment" } else { "comments" }; - parts.push(format!("{comments} {noun}")); - } - if parts.is_empty() { - None - } else { - Some(parts.join(" \u{00b7} ")) - } -} - -fn article_href(pick: &Pick) -> String { - format!("{}.xhtml", pick.article.chapter_id()) -} - -fn discussion_href(pick: &Pick) -> String { - format!("disc-{}.xhtml", pick.article.best_entry_id) -} - -fn section_href(name: &str) -> String { - let slug: String = name - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() { - c.to_ascii_lowercase() - } else { - '-' - } - }) - .collect(); - let slug = slug - .split('-') - .filter(|s| !s.is_empty()) - .collect::<Vec<_>>() - .join("-"); - format!("sec-{slug}.xhtml") -} - -fn summary_for<'a>(issue: &'a Issue, pick: &'a Pick) -> Option<&'a str> { - pick.summary - .as_deref() - .or_else(|| { - issue - .editorial - .summaries - .get(&pick.article.id) - .map(|s| s.as_str()) - }) - .filter(|s| !s.trim().is_empty()) -} - -fn published_display(pick: &Pick) -> Option<String> { - pick.article - .published_at - .map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string()) -} - -/// "From the Editor" front page plus the issue stats line (§3.10). -pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> { - let body = issue.editorial.front_page_html.trim(); - let body_html = if body.is_empty() { - format!( - "<p>{} of reading, chosen overnight.</p>", - images::text_escape(&issue.meta.stats_line()) - ) - } else { - images::to_xhtml(&ammonia::clean(body)) - }; - let tpl = FrontPage { - title: "From the Editor".into(), - display_date: issue.meta.display_date.clone(), - issue_number: issue.meta.issue_number, - stats_line: issue.meta.stats_line(), - body_html, - }; - Ok(Chapter { - id: "front".into(), - href: "front.xhtml".into(), - title: "From the Editor".into(), - xhtml: tpl.render()?, - toc_level: 1, - }) -} - -/// Section names in issue order, with the reserved world section removed (§3.6). -pub fn section_names(issue: &Issue) -> Vec<String> { - let mut names: Vec<String> = issue - .lineup - .section_order - .iter() - .filter(|s| s.as_str() != WORLD_BRIEFING_SECTION) - .cloned() - .collect(); - for pick in &issue.lineup.picks { - if pick.section != WORLD_BRIEFING_SECTION && !names.contains(&pick.section) { - names.push(pick.section.clone()); - } - } - names.retain(|n| issue.lineup.picks.iter().any(|p| &p.section == n)); - names -} - -/// "In This Issue": per section, each article's title, source, reading time and -/// summary, linked to its chapter (§3.10). -pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> { - let mut sections = Vec::new(); - for name in section_names(issue) { - let entries = issue - .lineup - .section_picks(&name) - .into_iter() - .map(|pick| IndexEntry { - href: article_href(pick), - title: pick.article.title.clone(), - source: pick.article.feed_title.clone(), - reading_minutes: pick.article.reading_minutes(), - summary: summary_for(issue, pick).unwrap_or_default().to_string(), - }) - .collect(); - sections.push(IndexSection { name, entries }); - } - if issue.world_briefing.is_some() { - sections.push(IndexSection { - name: WORLD_BRIEFING_SECTION.to_string(), - entries: vec![IndexEntry { - href: "world.xhtml".into(), - title: "World Briefing".into(), - source: "Wikipedia Current Events".into(), - reading_minutes: 3, - summary: "The day's events, as recorded by the Current Events portal.".into(), - }], - }); - } - let tpl = InThisIssue { - title: "In This Issue".into(), - stats_line: issue.meta.stats_line(), - sections, - }; - Ok(Chapter { - id: "in-this-issue".into(), - href: "in-this-issue.xhtml".into(), - title: "In This Issue".into(), - xhtml: tpl.render()?, - toc_level: 1, - }) -} - -/// A section title page: name + LLM intro (§3.10). -pub fn render_section_page(name: &str, intro: Option<&str>) -> Result<Chapter, EpubError> { - let tpl = SectionPage { - title: name.to_string(), - name: name.to_string(), - intro: intro - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()), - }; - Ok(Chapter { - id: format!("sec-{name}"), - href: section_href(name), - title: name.to_string(), - xhtml: tpl.render()?, - toc_level: 1, - }) -} - -/// One article chapter: header, cleaned body with embedded images, rating footer (§3.10). -pub fn render_article( - issue: &Issue, - pick: &Pick, - images_: &[ImageAsset], - edition: Edition, - public_url: &str, - hmac_secret: Option<&str>, -) -> Result<Chapter, EpubError> { - let article = &pick.article; - let mut meta_parts = vec![article.feed_title.clone()]; - if let Some(date) = published_display(pick) { - meta_parts.push(date); - } - meta_parts.push(format!( - "{} min read \u{00b7} {} words", - article.reading_minutes(), - article.word_count - )); - - // The X4 has no browser, so rating links are pointless there (§7). - let rating = match (hmac_secret, edition) { - (Some(secret), Edition::Standard) if !secret.is_empty() => Some(RatingLinks { - up_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Up), - down_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Down), - }), - _ => None, - }; - - let tpl = ArticleChapter { - title: article.title.clone(), - article_title: article.title.clone(), - byline: article.author.as_ref().map(|a| format!("By {a}")), - meta_line: meta_parts.join(" \u{00b7} "), - social_line: social_line(&article.social), - summary: summary_for(issue, pick).map(str::to_string), - excerpt_only: article.excerpt_only, - body_html: prepare_body(&article.content_html, images_), - rating, - read_online_url: article.url.clone(), - discussion_href: pick.discussion.as_ref().map(|_| discussion_href(pick)), - }; - Ok(Chapter { - id: article.chapter_id(), - href: article_href(pick), - title: article.title.clone(), - xhtml: tpl.render()?, - toc_level: 2, - }) -} - -/// The discussion chapter that follows an article when it has comments (§3.7). -pub fn render_discussion(pick: &Pick) -> Result<Option<Chapter>, EpubError> { - let Some(discussion) = &pick.discussion else { - return Ok(None); - }; - if discussion.threads.is_empty() { - return Ok(None); - } - let title = comments::chapter_title(&pick.article.title, discussion); - let tpl = DiscussionChapter { - title: title.clone(), - heading: title.clone(), - body_html: comments::render_xhtml(discussion, &pick.article.title), - article_href: article_href(pick), - }; - Ok(Some(Chapter { - id: format!("disc-{}", pick.article.best_entry_id), - href: discussion_href(pick), - title, - xhtml: tpl.render()?, - toc_level: 3, - })) -} - -/// The Wikipedia Current Events section chapter (§3.8). -pub fn render_world_briefing(issue: &Issue) -> Result<Option<Chapter>, EpubError> { - let Some(briefing) = &issue.world_briefing else { - return Ok(None); - }; - // The briefing may cover an earlier day than the issue: the portal page for - // the issue's own date is still an empty stub at 05:30 (§3.8). Dateline the - // section with the day it actually reports on, not the masthead date. - let tpl = WorldBriefingChapter { - title: WORLD_BRIEFING_SECTION.to_string(), - display_date: crate::pipeline::display_date(briefing.date), - body_html: world::render_xhtml(briefing), - }; - Ok(Some(Chapter { - id: "world".into(), - href: "world.xhtml".into(), - title: WORLD_BRIEFING_SECTION.to_string(), - xhtml: tpl.render()?, - toc_level: 1, - })) -} - -/// Colophon: generation timestamp, models used, token cost, feed counts (§3.10). -pub fn render_colophon(issue: &Issue) -> Result<Chapter, EpubError> { - let colophon = &issue.colophon; - let tpl = ColophonChapter { - title: "Colophon".into(), - issue_number: issue.meta.issue_number, - display_date: issue.meta.display_date.clone(), - generated_at: issue.meta.generated_at.to_string(), - model: if colophon.model.is_empty() { - "none (heuristic selection)".into() - } else { - colophon.model.clone() - }, - entries_fetched: colophon.entries_fetched, - feeds_seen: colophon.feeds_seen, - candidates: colophon.candidates, - article_count: issue.meta.article_count, - section_count: issue.meta.section_count, - total_words: issue.meta.total_words, - reading_line: reading_line(issue.meta.reading_minutes), - cost_usd: format!("${:.4}", colophon.cost_usd), - generator_version: if colophon.generator_version.is_empty() { - format!("daily-epub {}", env!("CARGO_PKG_VERSION")) - } else { - colophon.generator_version.clone() - }, - }; - Ok(Chapter { - id: "colophon".into(), - href: "colophon.xhtml".into(), - title: "Colophon".into(), - xhtml: tpl.render()?, - toc_level: 1, - }) -} - -fn render_cover_page(issue: &Issue, edition: Edition) -> Result<Chapter, EpubError> { - let tpl = CoverPage { - title: issue.meta.title_for(edition), - alt: format!( - "The Daily EPUB, {} \u{2014} No. {}", - issue.meta.display_date, issue.meta.issue_number - ), - cover_href: cover_href(edition), - }; - Ok(Chapter { - id: "cover".into(), - href: "cover.xhtml".into(), - title: "Cover".into(), - xhtml: tpl.render()?, - toc_level: 1, - }) -} - -/// Render every chapter for an edition, in issue order (§3.10). +/// Render every chapter of one edition, in issue order (§3.10). pub fn render_all( issue: &Issue, edition: Edition, @@ -993,376 +255,10 @@ pub fn assemble( Ok(out) } -/// A synthetic, fully offline [`Issue`] used by the unit tests here and by the -/// integration tests in `tests/` (which can only see the public API). -pub mod fixtures { - use std::collections::BTreeMap; - - use jiff::Timestamp; - - use crate::types::*; - - pub fn timestamp() -> Timestamp { - "2026-08-15T05:30:00Z".parse().expect("fixed timestamp") - } - - pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article { - Article { - id, - canonical_url: format!("https://example.com/{entry_id}"), - title: title.to_string(), - best_entry_id: entry_id, - content_html: format!( - "<p>Body of <em>{title}</em> with an image.</p><img src=\"https://img.example/{entry_id}.png\" alt=\"A chart of the daily figures\"><p>More words & things.</p>" - ), - word_count: 1200, - excerpt_only: false, - image_count: 1, - sources: vec![SourceRef { - entry_id, - feed_id: 7, - feed_title: "Example Feed".into(), - category: Some("Tech".into()), - kind: SourceKind::Feed, - }], - first_seen: timestamp(), - url: format!("https://example.com/{entry_id}"), - author: Some("A. Writer".into()), - feed_id: 7, - feed_title: "Example Feed".into(), - category: Some("Tech".into()), - published_at: Some(timestamp()), - comments_url: None, - image_urls: vec![format!("https://img.example/{entry_id}.png")], - social: vec![SocialRef { - article_id: id, - source: SocialSource::Hn, - item_id: Some("40100000".into()), - score: 342, - num_comments: 210, - item_url: Some("https://news.ycombinator.com/item?id=40100000".into()), - fetched_at: timestamp(), - }], - extract_method: ExtractMethod::Readability, - } - } - - pub fn discussion(article_id: ArticleId, entry_id: EntryId) -> Discussion { - Discussion { - article_id, - chapter_id: format!("disc-{entry_id}"), - threads: vec![CommentThread { - source: SocialSource::Hn, - item_url: "https://news.ycombinator.com/item?id=40100000".into(), - total_comments: 210, - comments: vec![Comment { - author: "alice".into(), - points: Some(61), - text_html: "<p>The write path is the interesting part.</p>".into(), - depth: 0, - children: vec![Comment { - author: "bob".into(), - points: Some(24), - text_html: "<p>Agreed.</p>".into(), - depth: 1, - children: vec![], - }], - }], - }], - } - } - - /// A synthetic two-article issue, one of them carrying a discussion. - pub fn issue() -> Issue { - let lead = Pick { - article: article(1, 1001, "The Lead Story"), - section: "Top Stories".into(), - position: 0, - is_lead: true, - summary: Some("What it argues, and why it is worth the time.".into()), - llm: None, - discussion: Some(discussion(1, 1001)), - }; - let second = Pick { - article: article(2, 1002, "A Niche Delight & Other Tales"), - section: "Niche Corner".into(), - position: 0, - is_lead: false, - summary: None, - llm: None, - discussion: None, - }; - let mut section_intros = BTreeMap::new(); - section_intros.insert("Top Stories".to_string(), "The day in brief.".to_string()); - let mut summaries = BTreeMap::new(); - summaries.insert(2, "A short abstract for the second piece.".to_string()); - - Issue { - meta: IssueMeta { - date: "2026-08-15".parse().expect("fixed date"), - issue_number: 42, - generated_at: timestamp(), - display_date: "Friday, August 15, 2026".into(), - article_count: 2, - section_count: 2, - total_words: 2400, - reading_minutes: 11, - }, - lineup: Lineup { - date: "2026-08-15".parse().expect("fixed date"), - picks: vec![lead, second], - section_order: vec!["Top Stories".into(), "Niche Corner".into()], - }, - editorial: Editorial { - front_page_html: "<p>Two stories today, both worth your coffee.</p>".into(), - section_intros, - summaries, - }, - world_briefing: Some(WorldBriefing { - date: "2026-08-15".parse().expect("fixed date"), - source_url: "https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15" - .into(), - overview: Some("A concise view of the day.".into()), - sections: vec![WorldBriefingSection { - title: "Top Stories".into(), - events: vec![WorldEvent { - id: "s1-e1".into(), - source_text: "Something happened somewhere.".into(), - links: vec![], - children: vec![], - summary: Some("The event in context.".into()), - }], - }], - }), - colophon: Colophon { - model: "deepseek-v4-flash".into(), - entries_fetched: 431, - feeds_seen: 92, - candidates: 120, - cost_usd: 0.0731, - generator_version: "daily-epub 0.1.0".into(), - }, - } - } -} - #[cfg(test)] mod tests { use super::*; - use fixtures::issue; - use hmac::{Hmac, KeyInit}; - use sha2::Sha256; - - #[test] - fn rating_token_matches_the_spec_vector() { - let date: Date = "2026-08-15".parse().unwrap(); - assert_eq!(rating_message(date, 1234, Vote::Up), "2026-08-15/1234/up"); - // hex(hmac_sha256("test-secret", "2026-08-15/1234/up"))[..16] - let token = rating_token("test-secret", date, 1234, Vote::Up); - assert_eq!(token.len(), TOKEN_LEN); - assert!(token.chars().all(|c| c.is_ascii_hexdigit())); - - // Independently computed reference value. - use hmac::Mac; - let mut mac = Hmac::<Sha256>::new_from_slice(b"test-secret").unwrap(); - mac.update(b"2026-08-15/1234/up"); - let expected: String = hex::encode(mac.finalize().into_bytes()) - .chars() - .take(16) - .collect(); - assert_eq!(token, expected); - - // Different vote, article and secret all change the token. - assert_ne!(token, rating_token("test-secret", date, 1234, Vote::Down)); - assert_ne!(token, rating_token("test-secret", date, 1235, Vote::Up)); - assert_ne!(token, rating_token("other-secret", date, 1234, Vote::Up)); - } - - /// The EPUB signs the links and `server.rs` verifies them: one formula, or no - /// rating ever lands. This is the vector `server::tests` pins from its side - /// (`VECTOR_SECRET` / `VECTOR_TOKEN_UP`) — change one, change both (§3.9). - #[test] - fn epub_and_server_share_one_token_vector() { - let date: Date = "2026-08-15".parse().unwrap(); - assert_eq!( - rating_token("test-secret", date, 42, Vote::Up), - "3b314cf7e6d8f50f" - ); - } - - #[test] - fn rating_url_has_the_spec_shape() { - let date: Date = "2026-08-15".parse().unwrap(); - let url = rating_url("https://daily.hallada.net/", "s3cret", date, 99, Vote::Down); - let token = rating_token("s3cret", date, 99, Vote::Down); - assert_eq!( - url, - format!("https://daily.hallada.net/r/2026-08-15/99/down?t={token}") - ); - } - - #[test] - fn social_line_matches_the_spec_example() { - let refs = vec![SocialRef { - article_id: 1, - source: crate::types::SocialSource::Hn, - item_id: None, - score: 342, - num_comments: 210, - item_url: None, - fetched_at: fixtures::timestamp(), - }]; - assert_eq!( - social_line(&refs).as_deref(), - Some("\u{25b2} 342 on HN \u{00b7} 210 comments") - ); - assert!(social_line(&[]).is_none()); - } - - #[test] - fn hrefs_are_deterministic() { - let issue = issue(); - let pick = &issue.lineup.picks[0]; - assert_eq!(article_href(pick), "art-1001.xhtml"); - assert_eq!(discussion_href(pick), "disc-1001.xhtml"); - assert_eq!( - section_href("Tech & Engineering"), - "sec-tech-engineering.xhtml" - ); - } - - fn assert_xml_ok(xhtml: &str) { - // A crude well-formedness check: the document parses as XML only if every - // tag is closed, so compare open/close counts for the elements we emit. - assert!(xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>")); - assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\"")); - assert!(xhtml.trim_end().ends_with("</html>")); - for tag in ["html", "head", "body", "div", "p"] { - let opens = xhtml.matches(&format!("<{tag}")).count(); - let closes = xhtml.matches(&format!("</{tag}>")).count(); - assert_eq!(opens, closes, "unbalanced <{tag}> in\n{xhtml}"); - } - assert!(!xhtml.contains(" ")); - for void in ["<br>", "<hr>", "<img "] { - if void == "<img " { - for (i, _) in xhtml.match_indices("<img ") { - let tail = &xhtml[i..]; - let end = tail.find('>').unwrap_or(0); - assert!(tail[..end].ends_with('/'), "unclosed <img> in\n{xhtml}"); - } - } else { - assert!(!xhtml.contains(void), "unclosed {void} in\n{xhtml}"); - } - } - } - - #[test] - fn front_page_renders_stats_and_editorial() { - let issue = issue(); - let chapter = render_front_page(&issue).unwrap(); - assert_eq!(chapter.href, "front.xhtml"); - assert!(chapter.xhtml.contains("From the Editor")); - assert!(chapter.xhtml.contains("2 articles")); - assert!(chapter.xhtml.contains("both worth your coffee")); - assert!(chapter.xhtml.contains("No. 42")); - assert_xml_ok(&chapter.xhtml); - } - - #[test] - fn in_this_issue_links_every_pick() { - let issue = issue(); - let chapter = render_in_this_issue(&issue).unwrap(); - assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\"")); - assert!(chapter.xhtml.contains("href=\"art-1002.xhtml\"")); - assert!(chapter.xhtml.contains("Example Feed")); - assert!(chapter.xhtml.contains("6 min read")); - assert!(chapter.xhtml.contains("What it argues")); - assert!(chapter.xhtml.contains("A short abstract")); - // Titles are escaped (askama emits numeric references), never injected raw. - assert!(chapter.xhtml.contains("A Niche Delight & Other Tales")); - assert_xml_ok(&chapter.xhtml); - } - - #[test] - fn article_chapter_has_header_body_and_footer() { - let issue = issue(); - let pick = &issue.lineup.picks[0]; - let chapter = render_article( - &issue, - pick, - &[], - Edition::Standard, - "https://daily.hallada.net", - Some("s3cret"), - ) - .unwrap(); - assert_eq!(chapter.id, "art-1001"); - assert_eq!(chapter.toc_level, 2); - assert!(chapter.xhtml.contains("By A. Writer")); - assert!(chapter.xhtml.contains("Example Feed")); - assert!(chapter.xhtml.contains("6 min read")); - assert!(chapter.xhtml.contains("\u{25b2} 342 on HN")); - assert!(chapter.xhtml.contains("/r/2026-08-15/1/up?t=")); - assert!(chapter.xhtml.contains("/r/2026-08-15/1/down?t=")); - assert!(chapter.xhtml.contains("Read online")); - assert!(chapter.xhtml.contains("href=\"disc-1001.xhtml\"")); - // The un-downloaded image degrades to a placeholder. - assert!( - chapter - .xhtml - .contains("[image: A chart of the daily figures]") - ); - assert_xml_ok(&chapter.xhtml); - } - - #[test] - fn x4_articles_omit_rating_links() { - let issue = issue(); - let chapter = render_article( - &issue, - &issue.lineup.picks[0], - &[], - Edition::X4, - "https://daily.hallada.net", - Some("s3cret"), - ) - .unwrap(); - assert!(!chapter.xhtml.contains("/r/2026-08-15/")); - assert!(chapter.xhtml.contains("Read online")); - } - - #[test] - fn discussion_chapter_nests_under_its_article() { - let issue = issue(); - let chapter = render_discussion(&issue.lineup.picks[0]).unwrap().unwrap(); - assert_eq!(chapter.id, "disc-1001"); - assert_eq!(chapter.toc_level, 3); - assert!( - chapter - .title - .starts_with("\u{1f4ac} Discussion: The Lead Story") - ); - assert!(chapter.xhtml.contains("alice")); - assert!(chapter.xhtml.contains("blockquote class=\"comment\"")); - assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\"")); - assert_xml_ok(&chapter.xhtml); - assert!(render_discussion(&issue.lineup.picks[1]).unwrap().is_none()); - } - - #[test] - fn world_and_colophon_chapters_render() { - let issue = issue(); - let world = render_world_briefing(&issue).unwrap().unwrap(); - assert!(world.xhtml.contains("Something happened somewhere")); - assert!(world.xhtml.contains("CC BY-SA")); - assert_xml_ok(&world.xhtml); - - let colophon = render_colophon(&issue).unwrap(); - assert!(colophon.xhtml.contains("deepseek-v4-flash")); - assert!(colophon.xhtml.contains("431")); - assert!(colophon.xhtml.contains("$0.0731")); - assert_xml_ok(&colophon.xhtml); - } + use crate::epub::fixtures::{assert_xml_ok, issue}; #[test] fn render_all_orders_the_issue() { @@ -1390,100 +286,4 @@ mod tests { assert_xml_ok(&chapter.xhtml); } } - - #[test] - fn covers_rasterize_for_both_editions() { - let issue = issue(); - for edition in [Edition::Standard, Edition::X4] { - let cover = render_cover(&issue, edition).expect("cover"); - let decoded = image::load_from_memory(&cover.bytes).expect("cover is a valid image"); - assert_eq!( - (decoded.width(), decoded.height()), - cover_size(edition), - "cover size for {edition:?}" - ); - assert_eq!(decoded.color(), image::ColorType::Rgb8); - if edition == Edition::X4 { - assert_eq!(cover.filename, "cover.jpg"); - assert_eq!(cover.mime, "image/jpeg"); - assert!( - cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc0]), - "baseline SOF0 missing" - ); - assert!( - !cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc2]), - "progressive SOF2 present" - ); - } else { - assert_eq!(cover.filename, "cover.png"); - assert_eq!(cover.mime, "image/png"); - } - } - } - - #[test] - fn fallback_cover_is_drawn_without_fonts() { - let png = encode_cover( - draw_fallback_cover(480, 800, Edition::X4).unwrap(), - Edition::X4, - ) - .unwrap(); - let decoded = image::load_from_memory(&png).unwrap(); - assert_eq!((decoded.width(), decoded.height()), (480, 800)); - // Some ink actually landed on the page. - let gray = decoded.to_luma8(); - assert!(gray.pixels().any(|p| p[0] < 32)); - assert!(gray.pixels().any(|p| p[0] > 224)); - - // The badge slab is the only per-edition mark the text-free fallback can - // draw, so the two editions still differ without any fonts installed. - let standard = draw_fallback_cover(480, 800, Edition::Standard).unwrap(); - let x4 = draw_fallback_cover(480, 800, Edition::X4).unwrap(); - assert_ne!(standard.data(), x4.data()); - assert!(badge_ink(&x4) > badge_ink(&standard)); - } - - /// Dark pixels inside the badge rectangle. - fn badge_ink(pixmap: &tiny_skia::Pixmap) -> usize { - let (w, h) = (pixmap.width() as usize, pixmap.height() as usize); - let (x0, x1) = (w * 3 / 10, w * 7 / 10); - let (y0, y1) = (h * 74 / 100, h * 78 / 100); - let mut dark = 0; - for y in y0..y1 { - for x in x0..x1 { - if pixmap.data()[(y * w + x) * 4] < 32 { - dark += 1; - } - } - } - dark - } - - /// At thumbnail size the title is unreadable, so the cover itself has to - /// say which edition it is (§3.10). - #[test] - fn only_the_x4_cover_carries_the_edition_badge() { - let issue = fixtures::issue(); - let x4 = cover_svg(&issue, Edition::X4, 480, 800).unwrap(); - assert!(x4.contains(">X4 EDITION<"), "{x4}"); - // White appears twice: the page ground, and the badge text reversed out - // of the slab. - assert_eq!(x4.matches("fill=\"#ffffff\"").count(), 2, "{x4}"); - - let standard = cover_svg(&issue, Edition::Standard, 1200, 1600).unwrap(); - assert!(!standard.contains("X4")); - assert_eq!(standard.matches("fill=\"#ffffff\"").count(), 1); - - // The badge sits between the stats line and the footer, inside the frame. - let cover = render_cover(&issue, Edition::X4).unwrap(); - let gray = image::load_from_memory(&cover.bytes).unwrap().to_luma8(); - let dark_in_badge = (584..634) - .flat_map(|y| (144..336).map(move |x| (x, y))) - .filter(|&(x, y)| gray.get_pixel(x, y)[0] < 32) - .count(); - assert!( - dark_in_badge > 4000, - "badge slab is missing: {dark_in_badge}" - ); - } } diff --git a/src/epub/chapters.rs b/src/epub/chapters.rs new file mode 100644 index 0000000..db0ad3e --- /dev/null +++ b/src/epub/chapters.rs @@ -0,0 +1,651 @@ +//! Rendering one chapter of XHTML at a time (spec §3.10). +//! +//! Each function here turns part of an [`Issue`] into a [`Chapter`]: the front +//! page, the index, section title pages, articles, discussions, the world +//! briefing and the colophon. [`super::build::render_all`] puts them in order; +//! [`super::build::assemble`] zips them. + +use askama::Template; + +use crate::comments; +use crate::html::{text_escape, to_xhtml}; +use crate::images; +use crate::types::{Edition, ImageAsset, Issue, Pick, SocialRef, Vote, WORLD_BRIEFING_SECTION}; +use crate::world; + +use super::EpubError; +use super::build::Chapter; + +/// Characters of the hex HMAC kept in rating links (§3.9). +pub const TOKEN_LEN: usize = crate::auth::TOKEN_LEN; + +// One implementation, shared with the verifying side in [`crate::server`]: +// see [`crate::auth`] for the formula and the pinned test vector (§3.9). +pub use crate::auth::{rating_message, rating_token, rating_url}; + +#[derive(Template)] +#[template(path = "front_page.xhtml", escape = "html")] +struct FrontPage { + title: String, + display_date: String, + issue_number: i64, + stats_line: String, + body_html: String, +} + +struct IndexEntry { + href: String, + title: String, + source: String, + reading_minutes: i64, + summary: String, +} + +struct IndexSection { + name: String, + entries: Vec<IndexEntry>, +} + +#[derive(Template)] +#[template(path = "in_this_issue.xhtml", escape = "html")] +struct InThisIssue { + title: String, + stats_line: String, + sections: Vec<IndexSection>, +} + +#[derive(Template)] +#[template(path = "section.xhtml", escape = "html")] +struct SectionPage { + title: String, + name: String, + intro: Option<String>, +} + +struct RatingLinks { + up_url: String, + down_url: String, +} + +#[derive(Template)] +#[template(path = "chapter.xhtml", escape = "html")] +struct ArticleChapter { + title: String, + article_title: String, + byline: Option<String>, + meta_line: String, + social_line: Option<String>, + summary: Option<String>, + excerpt_only: bool, + body_html: String, + rating: Option<RatingLinks>, + read_online_url: String, + discussion_href: Option<String>, +} + +#[derive(Template)] +#[template(path = "discussion.xhtml", escape = "html")] +struct DiscussionChapter { + title: String, + heading: String, + body_html: String, + article_href: String, +} + +#[derive(Template)] +#[template(path = "world_briefing.xhtml", escape = "html")] +struct WorldBriefingChapter { + title: String, + display_date: String, + body_html: String, +} + +#[derive(Template)] +#[template(path = "colophon.xhtml", escape = "html")] +struct ColophonChapter { + title: String, + issue_number: i64, + display_date: String, + generated_at: String, + model: String, + entries_fetched: i64, + feeds_seen: i64, + candidates: i64, + article_count: i64, + section_count: i64, + total_words: i64, + reading_line: String, + cost_usd: String, + generator_version: String, +} + +// --------------------------------------------------------------------------- +// Chapters (§3.10) +// --------------------------------------------------------------------------- + +/// Prepare article markup for XHTML: rewrite images, sanitize, self-close voids. +pub fn prepare_body(html: &str, images_: &[ImageAsset]) -> String { + // Sanitize first: image rewriting emits our own trusted markup (including the + // `image-placeholder` class, which ammonia would otherwise strip). + let cleaned = ammonia::clean(html); + let rewritten = images::rewrite_img_srcs(&cleaned, images_); + to_xhtml(&rewritten) +} + +/// "~2h 15m read" for the colophon (§3.10). +fn reading_line(minutes: i64) -> String { + let (h, m) = (minutes / 60, minutes % 60); + if h > 0 { + format!("~{h}h {m}m read") + } else { + format!("~{m}m read") + } +} + +/// "▲ 342 on HN · 210 comments" (§3.10). +pub fn social_line(social: &[SocialRef]) -> Option<String> { + let mut parts: Vec<String> = Vec::new(); + let mut comments = 0i64; + for entry in social { + if entry.score > 0 { + parts.push(format!( + "\u{25b2} {} on {}", + entry.score, + entry.source.display_name() + )); + } + comments += entry.num_comments.max(0); + } + if comments > 0 { + let noun = if comments == 1 { "comment" } else { "comments" }; + parts.push(format!("{comments} {noun}")); + } + if parts.is_empty() { + None + } else { + Some(parts.join(" \u{00b7} ")) + } +} + +fn article_href(pick: &Pick) -> String { + format!("{}.xhtml", pick.article.chapter_id()) +} + +fn discussion_href(pick: &Pick) -> String { + format!("disc-{}.xhtml", pick.article.best_entry_id) +} + +fn section_href(name: &str) -> String { + let slug: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + } + }) + .collect(); + let slug = slug + .split('-') + .filter(|s| !s.is_empty()) + .collect::<Vec<_>>() + .join("-"); + format!("sec-{slug}.xhtml") +} + +fn summary_for<'a>(issue: &'a Issue, pick: &'a Pick) -> Option<&'a str> { + pick.summary + .as_deref() + .or_else(|| { + issue + .editorial + .summaries + .get(&pick.article.id) + .map(|s| s.as_str()) + }) + .filter(|s| !s.trim().is_empty()) +} + +fn published_display(pick: &Pick) -> Option<String> { + pick.article + .published_at + .map(|ts| ts.to_zoned(jiff::tz::TimeZone::UTC).date().to_string()) +} + +/// "From the Editor" front page plus the issue stats line (§3.10). +pub fn render_front_page(issue: &Issue) -> Result<Chapter, EpubError> { + let body = issue.editorial.front_page_html.trim(); + let body_html = if body.is_empty() { + format!( + "<p>{} of reading, chosen overnight.</p>", + text_escape(&issue.meta.stats_line()) + ) + } else { + to_xhtml(&ammonia::clean(body)) + }; + let tpl = FrontPage { + title: "From the Editor".into(), + display_date: issue.meta.display_date.clone(), + issue_number: issue.meta.issue_number, + stats_line: issue.meta.stats_line(), + body_html, + }; + Ok(Chapter { + id: "front".into(), + href: "front.xhtml".into(), + title: "From the Editor".into(), + xhtml: tpl.render()?, + toc_level: 1, + }) +} + +/// Section names in issue order, with the reserved world section removed (§3.6). +pub fn section_names(issue: &Issue) -> Vec<String> { + let mut names: Vec<String> = issue + .lineup + .section_order + .iter() + .filter(|s| s.as_str() != WORLD_BRIEFING_SECTION) + .cloned() + .collect(); + for pick in &issue.lineup.picks { + if pick.section != WORLD_BRIEFING_SECTION && !names.contains(&pick.section) { + names.push(pick.section.clone()); + } + } + names.retain(|n| issue.lineup.picks.iter().any(|p| &p.section == n)); + names +} + +/// "In This Issue": per section, each article's title, source, reading time and +/// summary, linked to its chapter (§3.10). +pub fn render_in_this_issue(issue: &Issue) -> Result<Chapter, EpubError> { + let mut sections = Vec::new(); + for name in section_names(issue) { + let entries = issue + .lineup + .section_picks(&name) + .into_iter() + .map(|pick| IndexEntry { + href: article_href(pick), + title: pick.article.title.clone(), + source: pick.article.feed_title.clone(), + reading_minutes: pick.article.reading_minutes(), + summary: summary_for(issue, pick).unwrap_or_default().to_string(), + }) + .collect(); + sections.push(IndexSection { name, entries }); + } + if issue.world_briefing.is_some() { + sections.push(IndexSection { + name: WORLD_BRIEFING_SECTION.to_string(), + entries: vec![IndexEntry { + href: "world.xhtml".into(), + title: "World Briefing".into(), + source: "Wikipedia Current Events".into(), + reading_minutes: 3, + summary: "The day's events, as recorded by the Current Events portal.".into(), + }], + }); + } + let tpl = InThisIssue { + title: "In This Issue".into(), + stats_line: issue.meta.stats_line(), + sections, + }; + Ok(Chapter { + id: "in-this-issue".into(), + href: "in-this-issue.xhtml".into(), + title: "In This Issue".into(), + xhtml: tpl.render()?, + toc_level: 1, + }) +} + +/// A section title page: name + LLM intro (§3.10). +pub fn render_section_page(name: &str, intro: Option<&str>) -> Result<Chapter, EpubError> { + let tpl = SectionPage { + title: name.to_string(), + name: name.to_string(), + intro: intro + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()), + }; + Ok(Chapter { + id: format!("sec-{name}"), + href: section_href(name), + title: name.to_string(), + xhtml: tpl.render()?, + toc_level: 1, + }) +} + +/// One article chapter: header, cleaned body with embedded images, rating footer (§3.10). +pub fn render_article( + issue: &Issue, + pick: &Pick, + images_: &[ImageAsset], + edition: Edition, + public_url: &str, + hmac_secret: Option<&str>, +) -> Result<Chapter, EpubError> { + let article = &pick.article; + let mut meta_parts = vec![article.feed_title.clone()]; + if let Some(date) = published_display(pick) { + meta_parts.push(date); + } + meta_parts.push(format!( + "{} min read \u{00b7} {} words", + article.reading_minutes(), + article.word_count + )); + + // The X4 has no browser, so rating links are pointless there (§7). + let rating = match (hmac_secret, edition) { + (Some(secret), Edition::Standard) if !secret.is_empty() => Some(RatingLinks { + up_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Up), + down_url: rating_url(public_url, secret, issue.meta.date, article.id, Vote::Down), + }), + _ => None, + }; + + let tpl = ArticleChapter { + title: article.title.clone(), + article_title: article.title.clone(), + byline: article.author.as_ref().map(|a| format!("By {a}")), + meta_line: meta_parts.join(" \u{00b7} "), + social_line: social_line(&article.social), + summary: summary_for(issue, pick).map(str::to_string), + excerpt_only: article.excerpt_only, + body_html: prepare_body(&article.content_html, images_), + rating, + read_online_url: article.url.clone(), + discussion_href: pick.discussion.as_ref().map(|_| discussion_href(pick)), + }; + Ok(Chapter { + id: article.chapter_id(), + href: article_href(pick), + title: article.title.clone(), + xhtml: tpl.render()?, + toc_level: 2, + }) +} + +/// The discussion chapter that follows an article when it has comments (§3.7). +pub fn render_discussion(pick: &Pick) -> Result<Option<Chapter>, EpubError> { + let Some(discussion) = &pick.discussion else { + return Ok(None); + }; + if discussion.threads.is_empty() { + return Ok(None); + } + let title = comments::chapter_title(&pick.article.title, discussion); + let tpl = DiscussionChapter { + title: title.clone(), + heading: title.clone(), + body_html: comments::render_xhtml(discussion, &pick.article.title), + article_href: article_href(pick), + }; + Ok(Some(Chapter { + id: format!("disc-{}", pick.article.best_entry_id), + href: discussion_href(pick), + title, + xhtml: tpl.render()?, + toc_level: 3, + })) +} + +/// The Wikipedia Current Events section chapter (§3.8). +pub fn render_world_briefing(issue: &Issue) -> Result<Option<Chapter>, EpubError> { + let Some(briefing) = &issue.world_briefing else { + return Ok(None); + }; + // The briefing may cover an earlier day than the issue: the portal page for + // the issue's own date is still an empty stub at 05:30 (§3.8). Dateline the + // section with the day it actually reports on, not the masthead date. + let tpl = WorldBriefingChapter { + title: WORLD_BRIEFING_SECTION.to_string(), + display_date: crate::pipeline::display_date(briefing.date), + body_html: world::render_xhtml(briefing), + }; + Ok(Some(Chapter { + id: "world".into(), + href: "world.xhtml".into(), + title: WORLD_BRIEFING_SECTION.to_string(), + xhtml: tpl.render()?, + toc_level: 1, + })) +} + +/// Colophon: generation timestamp, models used, token cost, feed counts (§3.10). +pub fn render_colophon(issue: &Issue) -> Result<Chapter, EpubError> { + let colophon = &issue.colophon; + let tpl = ColophonChapter { + title: "Colophon".into(), + issue_number: issue.meta.issue_number, + display_date: issue.meta.display_date.clone(), + generated_at: issue.meta.generated_at.to_string(), + model: if colophon.model.is_empty() { + "none (heuristic selection)".into() + } else { + colophon.model.clone() + }, + entries_fetched: colophon.entries_fetched, + feeds_seen: colophon.feeds_seen, + candidates: colophon.candidates, + article_count: issue.meta.article_count, + section_count: issue.meta.section_count, + total_words: issue.meta.total_words, + reading_line: reading_line(issue.meta.reading_minutes), + cost_usd: format!("${:.4}", colophon.cost_usd), + generator_version: if colophon.generator_version.is_empty() { + format!("daily-epub {}", env!("CARGO_PKG_VERSION")) + } else { + colophon.generator_version.clone() + }, + }; + Ok(Chapter { + id: "colophon".into(), + href: "colophon.xhtml".into(), + title: "Colophon".into(), + xhtml: tpl.render()?, + toc_level: 1, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::epub::fixtures::{self, assert_xml_ok, issue}; + use crate::types::{SocialRef, Vote}; + use hmac::{Hmac, KeyInit}; + use jiff::civil::Date; + use sha2::Sha256; + + #[test] + fn rating_token_matches_the_spec_vector() { + let date: Date = "2026-08-15".parse().unwrap(); + assert_eq!(rating_message(date, 1234, Vote::Up), "2026-08-15/1234/up"); + // hex(hmac_sha256("test-secret", "2026-08-15/1234/up"))[..16] + let token = rating_token("test-secret", date, 1234, Vote::Up); + assert_eq!(token.len(), TOKEN_LEN); + assert!(token.chars().all(|c| c.is_ascii_hexdigit())); + + // Independently computed reference value. + use hmac::Mac; + let mut mac = Hmac::<Sha256>::new_from_slice(b"test-secret").unwrap(); + mac.update(b"2026-08-15/1234/up"); + let expected: String = hex::encode(mac.finalize().into_bytes()) + .chars() + .take(16) + .collect(); + assert_eq!(token, expected); + + // Different vote, article and secret all change the token. + assert_ne!(token, rating_token("test-secret", date, 1234, Vote::Down)); + assert_ne!(token, rating_token("test-secret", date, 1235, Vote::Up)); + assert_ne!(token, rating_token("other-secret", date, 1234, Vote::Up)); + } + + /// The EPUB signs the links and `server.rs` verifies them: one formula, or no + /// rating ever lands. This is the vector `server::tests` pins from its side + /// (`VECTOR_SECRET` / `VECTOR_TOKEN_UP`) — change one, change both (§3.9). + #[test] + fn epub_and_server_share_one_token_vector() { + let date: Date = "2026-08-15".parse().unwrap(); + assert_eq!( + rating_token("test-secret", date, 42, Vote::Up), + "3b314cf7e6d8f50f" + ); + } + + #[test] + fn rating_url_has_the_spec_shape() { + let date: Date = "2026-08-15".parse().unwrap(); + let url = rating_url("https://daily.hallada.net/", "s3cret", date, 99, Vote::Down); + let token = rating_token("s3cret", date, 99, Vote::Down); + assert_eq!( + url, + format!("https://daily.hallada.net/r/2026-08-15/99/down?t={token}") + ); + } + + #[test] + fn social_line_matches_the_spec_example() { + let refs = vec![SocialRef { + article_id: 1, + source: crate::types::SocialSource::Hn, + item_id: None, + score: 342, + num_comments: 210, + item_url: None, + fetched_at: fixtures::timestamp(), + }]; + assert_eq!( + social_line(&refs).as_deref(), + Some("\u{25b2} 342 on HN \u{00b7} 210 comments") + ); + assert!(social_line(&[]).is_none()); + } + + #[test] + fn hrefs_are_deterministic() { + let issue = issue(); + let pick = &issue.lineup.picks[0]; + assert_eq!(article_href(pick), "art-1001.xhtml"); + assert_eq!(discussion_href(pick), "disc-1001.xhtml"); + assert_eq!( + section_href("Tech & Engineering"), + "sec-tech-engineering.xhtml" + ); + } + + #[test] + fn front_page_renders_stats_and_editorial() { + let issue = issue(); + let chapter = render_front_page(&issue).unwrap(); + assert_eq!(chapter.href, "front.xhtml"); + assert!(chapter.xhtml.contains("From the Editor")); + assert!(chapter.xhtml.contains("2 articles")); + assert!(chapter.xhtml.contains("both worth your coffee")); + assert!(chapter.xhtml.contains("No. 42")); + assert_xml_ok(&chapter.xhtml); + } + + #[test] + fn in_this_issue_links_every_pick() { + let issue = issue(); + let chapter = render_in_this_issue(&issue).unwrap(); + assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\"")); + assert!(chapter.xhtml.contains("href=\"art-1002.xhtml\"")); + assert!(chapter.xhtml.contains("Example Feed")); + assert!(chapter.xhtml.contains("6 min read")); + assert!(chapter.xhtml.contains("What it argues")); + assert!(chapter.xhtml.contains("A short abstract")); + // Titles are escaped (askama emits numeric references), never injected raw. + assert!(chapter.xhtml.contains("A Niche Delight & Other Tales")); + assert_xml_ok(&chapter.xhtml); + } + + #[test] + fn article_chapter_has_header_body_and_footer() { + let issue = issue(); + let pick = &issue.lineup.picks[0]; + let chapter = render_article( + &issue, + pick, + &[], + Edition::Standard, + "https://daily.hallada.net", + Some("s3cret"), + ) + .unwrap(); + assert_eq!(chapter.id, "art-1001"); + assert_eq!(chapter.toc_level, 2); + assert!(chapter.xhtml.contains("By A. Writer")); + assert!(chapter.xhtml.contains("Example Feed")); + assert!(chapter.xhtml.contains("6 min read")); + assert!(chapter.xhtml.contains("\u{25b2} 342 on HN")); + assert!(chapter.xhtml.contains("/r/2026-08-15/1/up?t=")); + assert!(chapter.xhtml.contains("/r/2026-08-15/1/down?t=")); + assert!(chapter.xhtml.contains("Read online")); + assert!(chapter.xhtml.contains("href=\"disc-1001.xhtml\"")); + // The un-downloaded image degrades to a placeholder. + assert!( + chapter + .xhtml + .contains("[image: A chart of the daily figures]") + ); + assert_xml_ok(&chapter.xhtml); + } + + #[test] + fn x4_articles_omit_rating_links() { + let issue = issue(); + let chapter = render_article( + &issue, + &issue.lineup.picks[0], + &[], + Edition::X4, + "https://daily.hallada.net", + Some("s3cret"), + ) + .unwrap(); + assert!(!chapter.xhtml.contains("/r/2026-08-15/")); + assert!(chapter.xhtml.contains("Read online")); + } + + #[test] + fn discussion_chapter_nests_under_its_article() { + let issue = issue(); + let chapter = render_discussion(&issue.lineup.picks[0]).unwrap().unwrap(); + assert_eq!(chapter.id, "disc-1001"); + assert_eq!(chapter.toc_level, 3); + assert!( + chapter + .title + .starts_with("\u{1f4ac} Discussion: The Lead Story") + ); + assert!(chapter.xhtml.contains("alice")); + assert!(chapter.xhtml.contains("blockquote class=\"comment\"")); + assert!(chapter.xhtml.contains("href=\"art-1001.xhtml\"")); + assert_xml_ok(&chapter.xhtml); + assert!(render_discussion(&issue.lineup.picks[1]).unwrap().is_none()); + } + + #[test] + fn world_and_colophon_chapters_render() { + let issue = issue(); + let world = render_world_briefing(&issue).unwrap().unwrap(); + assert!(world.xhtml.contains("Something happened somewhere")); + assert!(world.xhtml.contains("CC BY-SA")); + assert_xml_ok(&world.xhtml); + + let colophon = render_colophon(&issue).unwrap(); + assert!(colophon.xhtml.contains("deepseek-v4-flash")); + assert!(colophon.xhtml.contains("431")); + assert!(colophon.xhtml.contains("$0.0731")); + assert_xml_ok(&colophon.xhtml); + } +} diff --git a/src/epub/cover.rs b/src/epub/cover.rs new file mode 100644 index 0000000..c7d71f3 --- /dev/null +++ b/src/epub/cover.rs @@ -0,0 +1,423 @@ +//! The generated cover image and cover page (spec §3.10). +//! +//! The cover is drawn as an SVG from a template, rasterized with resvg and +//! encoded per edition. If font resolution fails the run still gets a cover — +//! a text-free variant that keeps the editions apart in a library thumbnail +//! grid (notes §3: nothing here fails a build). + +use askama::Template; + +use crate::types::{Edition, Issue}; + +use super::EpubError; +use super::build::Chapter; +use super::x4; + +#[derive(Debug, Clone, PartialEq)] +pub struct CoverAsset { + pub bytes: Vec<u8>, + pub filename: &'static str, + pub mime: &'static str, +} + +pub fn cover_href(edition: Edition) -> &'static str { + match edition { + Edition::Standard => "cover.png", + Edition::X4 => "cover.jpg", + } +} + +#[derive(Template)] +#[template(path = "cover.svg", escape = "html")] +struct CoverSvg { + width: i64, + height: i64, + margin: i64, + inner_width: i64, + inner_height: i64, + border: i64, + hairline: i64, + center_x: i64, + masthead_y: i64, + masthead_size: i64, + rule_x1: i64, + rule_x2: i64, + rule_y: i64, + rule2_y: i64, + weekday: String, + weekday_y: i64, + weekday_size: i64, + long_date: String, + date_y: i64, + date_size: i64, + issue_number: i64, + issue_y: i64, + issue_size: i64, + stats_line: String, + stats_y: i64, + stats_size: i64, + /// Empty for the standard edition — see [`cover_badge`]. + edition_tag: String, + badge_x: i64, + badge_y: i64, + badge_width: i64, + badge_height: i64, + badge_radius: i64, + badge_text_y: i64, + badge_size: i64, + footer: String, + footer_y: i64, + footer_size: i64, +} + +#[derive(Template)] +#[template(path = "cover_page.xhtml", escape = "html")] +struct CoverPage { + title: String, + alt: String, + cover_href: &'static str, +} + +// --------------------------------------------------------------------------- +// Cover (§3.10) +// --------------------------------------------------------------------------- + +/// Cover pixel size per edition: 1200×1600 standard, 480×800 X4 (§3.10). +pub fn cover_size(edition: Edition) -> (u32, u32) { + match edition { + Edition::Standard => (1200, 1600), + Edition::X4 => x4::X4_SCREEN, + } +} + +/// Split "Friday, August 15, 2026" into ("Friday", "August 15, 2026"). +fn split_display_date(display: &str) -> (String, String) { + match display.split_once(", ") { + Some((weekday, rest)) => (weekday.to_string(), rest.to_string()), + None => (String::new(), display.to_string()), + } +} + +/// Reversed badge printed on the cover so the two editions are told apart at +/// thumbnail size, where the title is unreadable (§3.10). +/// +/// Empty for the standard edition: an unmarked cover is the default, and the +/// presence of the slab is itself the signal. +pub fn cover_badge(edition: Edition) -> &'static str { + match edition { + Edition::Standard => "", + Edition::X4 => "X4 EDITION", + } +} + +fn cover_svg( + issue: &Issue, + edition: Edition, + width: u32, + height: u32, +) -> Result<String, EpubError> { + let w = i64::from(width); + let h = i64::from(height); + let margin = w / 15; + let (weekday, long_date) = split_display_date(&issue.meta.display_date); + // A solid bar between the stats line and the footer: at 100px wide in a + // library grid the black slab is the only thing still legible. + let badge_height = h / 16; + let badge_width = w * 2 / 5; + let tpl = CoverSvg { + width: w, + height: h, + margin, + inner_width: w - 2 * margin, + inner_height: h - 2 * margin, + border: (w / 300).max(2), + hairline: (w / 600).max(1), + center_x: w / 2, + masthead_y: h * 30 / 100, + masthead_size: w / 9, + rule_x1: margin + w / 12, + rule_x2: w - margin - w / 12, + rule_y: h * 34 / 100, + rule2_y: h * 53 / 100, + weekday, + weekday_y: h * 42 / 100, + weekday_size: w / 28, + long_date, + date_y: h * 47 / 100, + date_size: w / 22, + issue_number: issue.meta.issue_number, + issue_y: h * 60 / 100, + issue_size: w / 18, + stats_line: issue.meta.stats_line(), + stats_y: h * 66 / 100, + stats_size: w / 32, + edition_tag: cover_badge(edition).to_string(), + badge_x: (w - badge_width) / 2, + badge_y: h * 73 / 100, + badge_width, + badge_height, + badge_radius: badge_height / 2, + badge_text_y: h * 73 / 100 + badge_height * 7 / 10, + badge_size: badge_height * 11 / 20, + footer: "Assembled overnight \u{00b7} read offline".to_string(), + footer_y: h - margin - h / 25, + footer_size: w / 40, + }; + Ok(tpl.render()?) +} + +/// Render the standard cover as RGB PNG and the X4 cover as baseline RGB JPEG. +pub fn render_cover(issue: &Issue, edition: Edition) -> Result<CoverAsset, EpubError> { + let (width, height) = cover_size(edition); + let svg = cover_svg(issue, edition, width, height)?; + let pixmap = match rasterize(&svg, width, height) { + Some(pixmap) => pixmap, + None => { + tracing::warn!("no usable system fonts: falling back to a geometric cover"); + draw_fallback_cover(width, height, edition) + .ok_or_else(|| EpubError::Build("could not allocate the cover".into()))? + } + }; + Ok(CoverAsset { + bytes: encode_cover(pixmap, edition)?, + filename: cover_href(edition), + mime: if edition == Edition::X4 { + "image/jpeg" + } else { + "image/png" + }, + }) +} + +fn rasterize(svg: &str, width: u32, height: u32) -> Option<tiny_skia::Pixmap> { + let mut options = resvg::usvg::Options::default(); + options.fontdb_mut().load_system_fonts(); + if options.fontdb.is_empty() { + return None; + } + let tree = resvg::usvg::Tree::from_str(svg, &options) + .map_err(|e| tracing::warn!("cover svg did not parse: {e}")) + .ok()?; + let mut pixmap = tiny_skia::Pixmap::new(width, height)?; + pixmap.fill(tiny_skia::Color::WHITE); + let size = tree.size(); + let transform = tiny_skia::Transform::from_scale( + width as f32 / size.width(), + height as f32 / size.height(), + ); + resvg::render(&tree, transform, &mut pixmap.as_mut()); + Some(pixmap) +} + +/// Text-free cover used when font resolution fails — the build never fails (§3.10). +/// +/// The badge cannot carry its lettering here, but the slab itself still keeps +/// the editions apart in a thumbnail grid. +fn draw_fallback_cover(width: u32, height: u32, edition: Edition) -> Option<tiny_skia::Pixmap> { + let mut pixmap = tiny_skia::Pixmap::new(width, height)?; + pixmap.fill(tiny_skia::Color::WHITE); + let mut paint = tiny_skia::Paint::default(); + paint.set_color(tiny_skia::Color::BLACK); + paint.anti_alias = true; + + let w = width as f32; + let h = height as f32; + let margin = w / 15.0; + let rect = |x: f32, y: f32, rw: f32, rh: f32, pixmap: &mut tiny_skia::Pixmap| { + if let Some(r) = tiny_skia::Rect::from_xywh(x, y, rw, rh) { + let path = tiny_skia::PathBuilder::from_rect(r); + pixmap.fill_path( + &path, + &paint, + tiny_skia::FillRule::Winding, + tiny_skia::Transform::identity(), + None, + ); + } + }; + // Frame. + let border = (w / 300.0).max(2.0); + rect(margin, margin, w - 2.0 * margin, border, &mut pixmap); + rect( + margin, + h - margin - border, + w - 2.0 * margin, + border, + &mut pixmap, + ); + rect(margin, margin, border, h - 2.0 * margin, &mut pixmap); + rect( + w - margin - border, + margin, + border, + h - 2.0 * margin, + &mut pixmap, + ); + // Masthead slab plus body rules — a newspaper silhouette. + rect( + margin * 2.0, + h * 0.26, + w - 4.0 * margin, + h * 0.035, + &mut pixmap, + ); + for i in 0..8 { + let y = h * 0.42 + (i as f32) * h * 0.045; + let inset = if i % 3 == 2 { + margin * 4.0 + } else { + margin * 2.0 + }; + rect(inset, y, w - 2.0 * inset, border, &mut pixmap); + } + if !cover_badge(edition).is_empty() { + let badge_height = h / 16.0; + let badge_width = w * 2.0 / 5.0; + rect( + (w - badge_width) / 2.0, + h * 0.73, + badge_width, + badge_height, + &mut pixmap, + ); + } + Some(pixmap) +} + +fn encode_cover(pixmap: tiny_skia::Pixmap, edition: Edition) -> Result<Vec<u8>, EpubError> { + let (w, h) = (pixmap.width(), pixmap.height()); + let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied()) + .ok_or_else(|| EpubError::Build("cover pixel buffer had the wrong size".into()))?; + let rgb = image::DynamicImage::ImageRgba8(rgba).to_rgb8(); + let mut bytes = Vec::new(); + match edition { + Edition::Standard => image::DynamicImage::ImageRgb8(rgb).write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ), + Edition::X4 => image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, 92) + .encode_image(&image::DynamicImage::ImageRgb8(rgb)), + } + .map_err(|e| EpubError::Build(format!("cover encoding failed: {e}")))?; + Ok(bytes) +} + +pub fn render_cover_page(issue: &Issue, edition: Edition) -> Result<Chapter, EpubError> { + let tpl = CoverPage { + title: issue.meta.title_for(edition), + alt: format!( + "The Daily EPUB, {} \u{2014} No. {}", + issue.meta.display_date, issue.meta.issue_number + ), + cover_href: cover_href(edition), + }; + Ok(Chapter { + id: "cover".into(), + href: "cover.xhtml".into(), + title: "Cover".into(), + xhtml: tpl.render()?, + toc_level: 1, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::epub::fixtures::{self, issue}; + + #[test] + fn covers_rasterize_for_both_editions() { + let issue = issue(); + for edition in [Edition::Standard, Edition::X4] { + let cover = render_cover(&issue, edition).expect("cover"); + let decoded = image::load_from_memory(&cover.bytes).expect("cover is a valid image"); + assert_eq!( + (decoded.width(), decoded.height()), + cover_size(edition), + "cover size for {edition:?}" + ); + assert_eq!(decoded.color(), image::ColorType::Rgb8); + if edition == Edition::X4 { + assert_eq!(cover.filename, "cover.jpg"); + assert_eq!(cover.mime, "image/jpeg"); + assert!( + cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc0]), + "baseline SOF0 missing" + ); + assert!( + !cover.bytes.windows(2).any(|marker| marker == [0xff, 0xc2]), + "progressive SOF2 present" + ); + } else { + assert_eq!(cover.filename, "cover.png"); + assert_eq!(cover.mime, "image/png"); + } + } + } + + #[test] + fn fallback_cover_is_drawn_without_fonts() { + let png = encode_cover( + draw_fallback_cover(480, 800, Edition::X4).unwrap(), + Edition::X4, + ) + .unwrap(); + let decoded = image::load_from_memory(&png).unwrap(); + assert_eq!((decoded.width(), decoded.height()), (480, 800)); + // Some ink actually landed on the page. + let gray = decoded.to_luma8(); + assert!(gray.pixels().any(|p| p[0] < 32)); + assert!(gray.pixels().any(|p| p[0] > 224)); + + // The badge slab is the only per-edition mark the text-free fallback can + // draw, so the two editions still differ without any fonts installed. + let standard = draw_fallback_cover(480, 800, Edition::Standard).unwrap(); + let x4 = draw_fallback_cover(480, 800, Edition::X4).unwrap(); + assert_ne!(standard.data(), x4.data()); + assert!(badge_ink(&x4) > badge_ink(&standard)); + } + + /// Dark pixels inside the badge rectangle. + fn badge_ink(pixmap: &tiny_skia::Pixmap) -> usize { + let (w, h) = (pixmap.width() as usize, pixmap.height() as usize); + let (x0, x1) = (w * 3 / 10, w * 7 / 10); + let (y0, y1) = (h * 74 / 100, h * 78 / 100); + let mut dark = 0; + for y in y0..y1 { + for x in x0..x1 { + if pixmap.data()[(y * w + x) * 4] < 32 { + dark += 1; + } + } + } + dark + } + + /// At thumbnail size the title is unreadable, so the cover itself has to + /// say which edition it is (§3.10). + #[test] + fn only_the_x4_cover_carries_the_edition_badge() { + let issue = fixtures::issue(); + let x4 = cover_svg(&issue, Edition::X4, 480, 800).unwrap(); + assert!(x4.contains(">X4 EDITION<"), "{x4}"); + // White appears twice: the page ground, and the badge text reversed out + // of the slab. + assert_eq!(x4.matches("fill=\"#ffffff\"").count(), 2, "{x4}"); + + let standard = cover_svg(&issue, Edition::Standard, 1200, 1600).unwrap(); + assert!(!standard.contains("X4")); + assert_eq!(standard.matches("fill=\"#ffffff\"").count(), 1); + + // The badge sits between the stats line and the footer, inside the frame. + let cover = render_cover(&issue, Edition::X4).unwrap(); + let gray = image::load_from_memory(&cover.bytes).unwrap().to_luma8(); + let dark_in_badge = (584..634) + .flat_map(|y| (144..336).map(move |x| (x, y))) + .filter(|&(x, y)| gray.get_pixel(x, y)[0] < 32) + .count(); + assert!( + dark_in_badge > 4000, + "badge slab is missing: {dark_in_badge}" + ); + } +} diff --git a/src/epub/fixtures.rs b/src/epub/fixtures.rs new file mode 100644 index 0000000..9a35e91 --- /dev/null +++ b/src/epub/fixtures.rs @@ -0,0 +1,182 @@ +//! A synthetic, fully offline [`crate::types::Issue`] for tests. +//! +//! Lives here rather than inside a `#[cfg(test)]` block because the integration +//! tests in `tests/` can only see the public API. + +use std::collections::BTreeMap; + +use jiff::Timestamp; + +use crate::types::*; + +pub fn timestamp() -> Timestamp { + "2026-08-15T05:30:00Z".parse().expect("fixed timestamp") +} + +pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article { + Article { + id, + canonical_url: format!("https://example.com/{entry_id}"), + title: title.to_string(), + best_entry_id: entry_id, + content_html: format!( + "<p>Body of <em>{title}</em> with an image.</p><img src=\"https://img.example/{entry_id}.png\" alt=\"A chart of the daily figures\"><p>More words & things.</p>" + ), + word_count: 1200, + excerpt_only: false, + image_count: 1, + sources: vec![SourceRef { + entry_id, + feed_id: 7, + feed_title: "Example Feed".into(), + category: Some("Tech".into()), + kind: SourceKind::Feed, + }], + first_seen: timestamp(), + url: format!("https://example.com/{entry_id}"), + author: Some("A. Writer".into()), + feed_id: 7, + feed_title: "Example Feed".into(), + category: Some("Tech".into()), + published_at: Some(timestamp()), + comments_url: None, + image_urls: vec![format!("https://img.example/{entry_id}.png")], + social: vec![SocialRef { + article_id: id, + source: SocialSource::Hn, + item_id: Some("40100000".into()), + score: 342, + num_comments: 210, + item_url: Some("https://news.ycombinator.com/item?id=40100000".into()), + fetched_at: timestamp(), + }], + extract_method: ExtractMethod::Readability, + } +} + +pub fn discussion(article_id: ArticleId, entry_id: EntryId) -> Discussion { + Discussion { + article_id, + chapter_id: format!("disc-{entry_id}"), + threads: vec![CommentThread { + source: SocialSource::Hn, + item_url: "https://news.ycombinator.com/item?id=40100000".into(), + total_comments: 210, + comments: vec![Comment { + author: "alice".into(), + points: Some(61), + text_html: "<p>The write path is the interesting part.</p>".into(), + depth: 0, + children: vec![Comment { + author: "bob".into(), + points: Some(24), + text_html: "<p>Agreed.</p>".into(), + depth: 1, + children: vec![], + }], + }], + }], + } +} + +/// A synthetic two-article issue, one of them carrying a discussion. +pub fn issue() -> Issue { + let lead = Pick { + article: article(1, 1001, "The Lead Story"), + section: "Top Stories".into(), + position: 0, + is_lead: true, + summary: Some("What it argues, and why it is worth the time.".into()), + llm: None, + discussion: Some(discussion(1, 1001)), + }; + let second = Pick { + article: article(2, 1002, "A Niche Delight & Other Tales"), + section: "Niche Corner".into(), + position: 0, + is_lead: false, + summary: None, + llm: None, + discussion: None, + }; + let mut section_intros = BTreeMap::new(); + section_intros.insert("Top Stories".to_string(), "The day in brief.".to_string()); + let mut summaries = BTreeMap::new(); + summaries.insert(2, "A short abstract for the second piece.".to_string()); + + Issue { + meta: IssueMeta { + date: "2026-08-15".parse().expect("fixed date"), + issue_number: 42, + generated_at: timestamp(), + display_date: "Friday, August 15, 2026".into(), + article_count: 2, + section_count: 2, + total_words: 2400, + reading_minutes: 11, + }, + lineup: Lineup { + date: "2026-08-15".parse().expect("fixed date"), + picks: vec![lead, second], + section_order: vec!["Top Stories".into(), "Niche Corner".into()], + }, + editorial: Editorial { + front_page_html: "<p>Two stories today, both worth your coffee.</p>".into(), + section_intros, + summaries, + }, + world_briefing: Some(WorldBriefing { + date: "2026-08-15".parse().expect("fixed date"), + source_url: "https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15".into(), + overview: Some("A concise view of the day.".into()), + sections: vec![WorldBriefingSection { + title: "Top Stories".into(), + events: vec![WorldEvent { + id: "s1-e1".into(), + source_text: "Something happened somewhere.".into(), + links: vec![], + children: vec![], + summary: Some("The event in context.".into()), + }], + }], + }), + colophon: Colophon { + model: "deepseek-v4-flash".into(), + entries_fetched: 431, + feeds_seen: 92, + candidates: 120, + cost_usd: 0.0731, + generator_version: "daily-epub 0.1.0".into(), + }, + } +} + +/// A crude XHTML well-formedness check for rendered chapters. +/// +/// The real proof is `epubcheck`, which cannot run in a unit test; this catches +/// the mistakes that actually happen — an unclosed void element or a raw +/// ` `, both of which make an EPUB3 content document unparseable. +pub fn assert_xml_ok(xhtml: &str) { + // A crude well-formedness check: the document parses as XML only if every + // tag is closed, so compare open/close counts for the elements we emit. + assert!(xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>")); + assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\"")); + assert!(xhtml.trim_end().ends_with("</html>")); + for tag in ["html", "head", "body", "div", "p"] { + let opens = xhtml.matches(&format!("<{tag}")).count(); + let closes = xhtml.matches(&format!("</{tag}>")).count(); + assert_eq!(opens, closes, "unbalanced <{tag}> in\n{xhtml}"); + } + assert!(!xhtml.contains(" ")); + for void in ["<br>", "<hr>", "<img "] { + if void == "<img " { + for (i, _) in xhtml.match_indices("<img ") { + let tail = &xhtml[i..]; + let end = tail.find('>').unwrap_or(0); + assert!(tail[..end].ends_with('/'), "unclosed <img> in\n{xhtml}"); + } + } else { + assert!(!xhtml.contains(void), "unclosed {void} in\n{xhtml}"); + } + } +} diff --git a/src/epub/images.rs b/src/epub/images.rs deleted file mode 100644 index e6a4c11..0000000 --- a/src/epub/images.rs +++ /dev/null @@ -1,963 +0,0 @@ -//! Image download and re-encoding (spec §3.10 "Images"). -//! -//! Failed downloads degrade to a `[image: alt text]` placeholder paragraph — the -//! run never fails because of an image (notes §3). - -use std::collections::HashMap; -use std::io::Cursor; -use std::time::Duration; - -use futures::StreamExt; -use image::{DynamicImage, GenericImageView, ImageFormat}; - -use crate::types::{Edition, ImageAsset, Pick}; - -/// Per-image download timeout (§3.10). -pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10; -/// Per-image size cap (§3.10). -pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; -/// Concurrent downloads (§3.10). -pub const CONCURRENCY: usize = 8; -/// Whole-issue asset budget (§3.10). -pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024; -/// Images smaller than this in either dimension are decorative — skipped (§3.10). -pub const MIN_DIMENSION_PX: u32 = 24; -/// Width/height an SVG is rasterized to when it declares no intrinsic size. -const SVG_FALLBACK_SIZE: u32 = 1000; - -/// HTML void elements: XHTML requires them self-closed (§3.10 "valid XHTML"). -pub const VOID_ELEMENTS: &[&str] = &[ - "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", - "track", "wbr", -]; - -/// Per-edition re-encoding parameters (§3.10). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ImageProfile { - pub max_width: u32, - pub max_height: u32, - pub jpeg_quality: u8, - pub grayscale: bool, -} - -impl ImageProfile { - /// Standard edition: max width 1200px, JPEG q80, color (§3.10). - pub const STANDARD: ImageProfile = ImageProfile { - max_width: 1200, - max_height: 4000, - jpeg_quality: 80, - grayscale: false, - }; - - /// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10). - pub const X4: ImageProfile = ImageProfile { - max_width: 480, - max_height: 800, - jpeg_quality: 70, - grayscale: true, - }; - - pub fn for_edition(edition: Edition) -> Self { - match edition { - Edition::Standard => Self::STANDARD, - Edition::X4 => Self::X4, - } - } -} - -/// One `<img>` found in article markup, with the caption of its `<figure>` if any. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ImgRef { - pub src: String, - pub alt: String, - pub caption: Option<String>, -} - -/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup. -pub fn extract_img_refs(html: &str) -> Vec<ImgRef> { - let doc = scraper::Html::parse_fragment(html); - let Ok(img_sel) = scraper::Selector::parse("img") else { - return Vec::new(); - }; - let cap_sel = scraper::Selector::parse("figcaption").ok(); - - let mut out = Vec::new(); - let mut seen: Vec<String> = Vec::new(); - for el in doc.select(&img_sel) { - let Some(src) = el.value().attr("src") else { - continue; - }; - let src = src.trim(); - if src.is_empty() || src.starts_with("data:") { - continue; - } - if seen.iter().any(|s| s == src) { - continue; - } - seen.push(src.to_string()); - let alt = el - .value() - .attr("alt") - .unwrap_or_default() - .trim() - .to_string(); - // Walk up to an enclosing <figure> and take its caption, if any. - let mut caption = None; - if let Some(cap_sel) = &cap_sel { - let mut cursor = el.parent(); - while let Some(node) = cursor { - if let Some(elem) = scraper::ElementRef::wrap(node) { - if elem.value().name() == "figure" { - caption = elem.select(cap_sel).next().map(|c| { - c.text() - .collect::<String>() - .split_whitespace() - .collect::<Vec<_>>() - .join(" ") - }); - break; - } - cursor = elem.parent(); - } else { - break; - } - } - } - out.push(ImgRef { - src: src.to_string(), - alt, - caption: caption.filter(|c| !c.is_empty()), - }); - } - out -} - -/// Download one image, honoring the timeout and size cap (§3.10). -pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> { - let resp = http - .get(url) - // Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10). - .header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8") - .timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS)) - .send() - .await - .map_err(|e| tracing::debug!(url, "image download failed: {e}")) - .ok()?; - if !resp.status().is_success() { - tracing::debug!(url, status = %resp.status(), "image download rejected"); - return None; - } - if let Some(len) = resp.content_length() - && len as usize > MAX_IMAGE_BYTES - { - tracing::debug!(url, len, "image exceeds the size cap"); - return None; - } - let mut resp = resp; - let mut buf: Vec<u8> = Vec::new(); - loop { - match resp.chunk().await { - Ok(Some(chunk)) => { - if buf.len() + chunk.len() > MAX_IMAGE_BYTES { - tracing::debug!(url, "image exceeds the size cap mid-stream"); - return None; - } - buf.extend_from_slice(&chunk); - } - Ok(None) => break, - Err(e) => { - tracing::debug!(url, "image download interrupted: {e}"); - return None; - } - } - } - if buf.is_empty() { None } else { Some(buf) } -} - -/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10). -/// -/// Line art with transparency is kept as PNG after flattening; everything else -/// becomes JPEG. SVG is rasterized first — charts and diagrams are frequently -/// vector-only, and dropping them loses the point of the article. Returns `None` -/// for sources no decoder handles. -pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> { - if looks_like_svg(bytes) { - let raster = rasterize_svg(bytes, profile)?; - return reencode(&raster, profile); - } - let format = image::guess_format(bytes).ok(); - let decoded = image::load_from_memory(bytes) - .map_err(|e| tracing::debug!("undecodable image: {e}")) - .ok()?; - - let (w, h) = decoded.dimensions(); - if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX { - tracing::debug!(w, h, "skipping decorative image"); - return None; - } - - let has_alpha = decoded.color().has_alpha(); - let flattened = if has_alpha { - flatten_to_white(&decoded) - } else { - decoded - }; - - let resized = if w > profile.max_width || h > profile.max_height { - flattened.resize( - profile.max_width, - profile.max_height, - image::imageops::FilterType::Lanczos3, - ) - } else { - flattened - }; - - // Keep line art (PNG source, few distinct tones) lossless; everything else - // becomes JPEG, which is far smaller for photographs (§3.10). - let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized); - - let mut out = Cursor::new(Vec::new()); - // NB: encode the concrete buffer, not the `DynamicImage` — the latter always - // reports RGBA pixels, which would silently re-colorize a grayscale image. - if profile.grayscale { - let gray = resized.to_luma8(); - if keep_png { - DynamicImage::ImageLuma8(gray) - .write_to(&mut out, ImageFormat::Png) - .ok()?; - return Some((out.into_inner(), "image/png")); - } - let mut enc = - image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality); - enc.encode_image(&gray).ok()?; - return Some((out.into_inner(), "image/jpeg")); - } - - let rgb = resized.to_rgb8(); - if keep_png { - DynamicImage::ImageRgb8(rgb) - .write_to(&mut out, ImageFormat::Png) - .ok()?; - return Some((out.into_inner(), "image/png")); - } - let mut enc = - image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality); - enc.encode_image(&rgb).ok()?; - Some((out.into_inner(), "image/jpeg")) -} - -/// True when `bytes` are an SVG document (possibly behind an XML prolog or BOM). -fn looks_like_svg(bytes: &[u8]) -> bool { - let head = &bytes[..bytes.len().min(1024)]; - let text = String::from_utf8_lossy(head); - let text = text.trim_start_matches('\u{feff}').trim_start(); - text.starts_with("<svg") - || (text.starts_with("<?xml") || text.starts_with("<!DOCTYPE svg")) && text.contains("<svg") -} - -/// Rasterize an SVG to a PNG at the profile's target width (§3.10). -/// -/// The profile's own resize pass then handles the height cap, so this only has -/// to land in the right ballpark. -fn rasterize_svg(bytes: &[u8], profile: ImageProfile) -> Option<Vec<u8>> { - let mut options = resvg::usvg::Options::default(); - options.fontdb_mut().load_system_fonts(); - let tree = resvg::usvg::Tree::from_data(bytes, &options) - .map_err(|e| tracing::debug!("svg did not parse: {e}")) - .ok()?; - - // An `<svg>` that parses but draws nothing is not an image, it is a stray - // tag: rasterizing it would embed a blank rectangle. - if tree.root().children().is_empty() { - tracing::debug!("svg has nothing to draw"); - return None; - } - let size = tree.size(); - let (sw, sh) = (size.width(), size.height()); - if !(sw.is_finite() && sh.is_finite()) || sw <= 0.0 || sh <= 0.0 { - return None; - } - // Judge "decorative" by the declared size, before scaling: a 16×16 icon is - // an icon however large we choose to draw it. - if sw < MIN_DIMENSION_PX as f32 || sh < MIN_DIMENSION_PX as f32 { - tracing::debug!(sw, sh, "skipping decorative svg"); - return None; - } - // Vector art has no native resolution, so render straight at the edition's - // target width — upscaling a rasterized copy afterwards would only blur it. - let target_w = profile - .max_width - .max(SVG_FALLBACK_SIZE.min(profile.max_width)); - let scale = (target_w as f32 / sw).min(profile.max_height as f32 / sh); - let scale = if scale.is_finite() && scale > 0.0 { - scale - } else { - 1.0 - }; - let (w, h) = ( - (sw * scale).round().max(1.0) as u32, - (sh * scale).round().max(1.0) as u32, - ); - let mut pixmap = tiny_skia::Pixmap::new(w, h)?; - // E-ink has no transparency; render onto white so alpha never becomes black. - pixmap.fill(tiny_skia::Color::WHITE); - resvg::render( - &tree, - tiny_skia::Transform::from_scale(scale, scale), - &mut pixmap.as_mut(), - ); - let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())?; - let mut png = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(rgba) - .write_to(&mut png, ImageFormat::Png) - .ok()?; - Some(png.into_inner()) -} - -/// Composite over an opaque white page — e-ink has no transparency (§3.10). -fn flatten_to_white(img: &DynamicImage) -> DynamicImage { - let rgba = img.to_rgba8(); - let mut rgb = image::RgbImage::new(rgba.width(), rgba.height()); - for (x, y, px) in rgba.enumerate_pixels() { - let a = f32::from(px[3]) / 255.0; - let blend = |c: u8| { - ((f32::from(c) * a) + 255.0 * (1.0 - a)) - .round() - .clamp(0.0, 255.0) as u8 - }; - rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])])); - } - DynamicImage::ImageRgb8(rgb) -} - -/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text). -fn is_line_art(img: &DynamicImage) -> bool { - const SAMPLE_LIMIT: usize = 20_000; - const DISTINCT_LIMIT: usize = 64; - let rgb = img.to_rgb8(); - let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1); - for (i, px) in rgb.pixels().enumerate() { - if i >= SAMPLE_LIMIT { - break; - } - let c = [px[0], px[1], px[2]]; - if !distinct.contains(&c) { - distinct.push(c); - if distinct.len() > DISTINCT_LIMIT { - return false; - } - } - } - true -} - -/// Everything needed to fetch one image, in deterministic issue order. -#[derive(Debug, Clone)] -struct PendingImage { - id: String, - url: String, - alt: String, - caption: Option<String>, -} - -fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> { - let entry_id = pick.article.best_entry_id; - let mut refs = extract_img_refs(&pick.article.content_html); - if refs.is_empty() { - refs = pick - .article - .image_urls - .iter() - .map(|u| ImgRef { - src: u.clone(), - alt: String::new(), - caption: None, - }) - .collect(); - } - refs.into_iter() - .filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://")) - .enumerate() - .map(|(i, r)| PendingImage { - id: format!("img-{entry_id}-{i}"), - url: r.src, - alt: r.alt, - caption: r.caption, - }) - .collect() -} - -/// Download and re-encode every image referenced by the lineup for one edition, -/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10). -pub async fn collect_for_issue( - http: &reqwest::Client, - picks: &[Pick], - edition: Edition, -) -> Vec<ImageAsset> { - let profile = ImageProfile::for_edition(edition); - let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect(); - if pending.is_empty() { - return Vec::new(); - } - tracing::info!(count = pending.len(), ?edition, "downloading issue images"); - - let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> = - futures::stream::iter(pending.into_iter().map(|p| { - let http = http.clone(); - async move { - let raw = download(&http, &p.url).await?; - let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile)) - .await - .ok() - .flatten()?; - Some((p, bytes, mime)) - } - })) - .buffered(CONCURRENCY) - .collect() - .await; - - let mut assets = Vec::new(); - let mut budget_used = 0usize; - let mut skipped = 0usize; - for result in results.into_iter().flatten() { - let (pending, bytes, mime) = result; - if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES { - skipped += 1; - continue; - } - budget_used += bytes.len(); - let ext = if mime == "image/png" { "png" } else { "jpg" }; - assets.push(ImageAsset { - href: format!("images/{}.{ext}", pending.id), - id: pending.id, - mime: mime.to_string(), - data: bytes, - alt: pending.alt, - caption: pending.caption, - source_url: pending.url, - }); - } - if skipped > 0 { - tracing::warn!(skipped, budget_used, "issue image budget exhausted"); - } - tracing::info!( - embedded = assets.len(), - bytes = budget_used, - "issue images ready" - ); - assets -} - -// --------------------------------------------------------------------------- -// Markup rewriting -// --------------------------------------------------------------------------- - -/// End index (exclusive) of the tag starting at `start` (`html[start] == '<'`), -/// respecting quoted attribute values and comments. -pub(crate) fn tag_end(html: &str, start: usize) -> Option<usize> { - let rest = &html[start..]; - if rest.starts_with("<!--") { - return rest.find("-->").map(|i| start + i + 3); - } - let mut quote: Option<char> = None; - for (i, c) in rest.char_indices().skip(1) { - match (quote, c) { - (Some(q), c) if c == q => quote = None, - (Some(_), _) => {} - (None, '"') | (None, '\'') => quote = Some(c), - (None, '>') => return Some(start + i + c.len_utf8()), - (None, _) => {} - } - } - None -} - -/// Lowercased element name of a tag body such as `img src="…"`. -pub(crate) fn tag_name(inner: &str) -> String { - inner - .trim_start_matches('/') - .split(|c: char| c.is_whitespace() || c == '/' || c == '>') - .next() - .unwrap_or("") - .to_ascii_lowercase() -} - -/// Parse `name="value"` pairs out of a tag body, with entity-decoded values. -/// -/// Decoding matters: this scanner reads markup that ammonia has serialized, and -/// ammonia writes `&` in a URL as `&`. A raw comparison against a URL that -/// came out of a real HTML parser would never match (§3.10). -pub(crate) fn parse_attrs(inner: &str) -> Vec<(String, String)> { - let mut attrs = Vec::new(); - let bytes: Vec<char> = inner.chars().collect(); - let mut i = 0; - // Skip the element name. - while i < bytes.len() && !bytes[i].is_whitespace() { - i += 1; - } - while i < bytes.len() { - while i < bytes.len() && (bytes[i].is_whitespace() || bytes[i] == '/') { - i += 1; - } - let name_start = i; - while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '=' && bytes[i] != '/' { - i += 1; - } - if i == name_start { - break; - } - let name: String = bytes[name_start..i] - .iter() - .collect::<String>() - .to_ascii_lowercase(); - while i < bytes.len() && bytes[i].is_whitespace() { - i += 1; - } - let mut value = String::new(); - if i < bytes.len() && bytes[i] == '=' { - i += 1; - while i < bytes.len() && bytes[i].is_whitespace() { - i += 1; - } - if i < bytes.len() && (bytes[i] == '"' || bytes[i] == '\'') { - let quote = bytes[i]; - i += 1; - while i < bytes.len() && bytes[i] != quote { - value.push(bytes[i]); - i += 1; - } - i += 1; - } else { - while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '>' { - value.push(bytes[i]); - i += 1; - } - } - } - attrs.push((name, decode_entities(&value))); - } - attrs -} - -/// Decode the handful of entities an HTML serializer emits inside attributes. -/// -/// Numeric forms are included because feeds and WordPress write `&` for -/// `&`; anything else is left alone rather than guessed at. -pub(crate) fn decode_entities(s: &str) -> String { - if !s.contains('&') { - return s.to_string(); - } - let mut out = String::with_capacity(s.len()); - let mut rest = s; - while let Some(i) = rest.find('&') { - out.push_str(&rest[..i]); - let tail = &rest[i..]; - let Some(end) = tail[..tail.len().min(12)].find(';') else { - out.push('&'); - rest = &tail[1..]; - continue; - }; - let entity = &tail[1..end]; - let decoded = match entity { - "amp" => Some('&'), - "lt" => Some('<'), - "gt" => Some('>'), - "quot" => Some('"'), - "apos" | "#39" => Some('\''), - "nbsp" => Some('\u{a0}'), - _ => entity - .strip_prefix('#') - .and_then(|n| match n.strip_prefix(['x', 'X']) { - Some(hex) => u32::from_str_radix(hex, 16).ok(), - None => n.parse::<u32>().ok(), - }) - .and_then(char::from_u32), - }; - match decoded { - Some(c) => { - out.push(c); - rest = &tail[end + 1..]; - } - None => { - out.push('&'); - rest = &tail[1..]; - } - } - } - out.push_str(rest); - out -} - -/// Escape a string for use inside a double-quoted XML attribute. -fn attr_escape(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for c in s.chars() { - match c { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - _ => out.push(c), - } - } - out -} - -/// Escape a string for XML text content. -pub fn text_escape(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for c in s.chars() { - match c { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - _ => out.push(c), - } - } - out -} - -/// Whether an image we could not embed is worth telling the reader about. -/// -/// Only descriptive alt text qualifies. A filename, a bare label like `red line` -/// on a divider rule, or no alt at all carries nothing the reader loses by not -/// seeing the picture — announcing those turns every decorative graphic and -/// dead link into a line of clutter, which is how the placeholders got out of -/// hand in the first place. -fn alt_is_worth_announcing(alt: &str) -> bool { - const MIN_DESCRIPTIVE_WORDS: usize = 4; - !alt.is_empty() - && !is_filename_alt(alt) - && alt.split_whitespace().count() >= MIN_DESCRIPTIVE_WORDS -} - -/// True for alt text that is really just the uploaded filename — `IMG_0808.JPG`, -/// `cut pieces v01.JPG`, `chart-final-2.png`. -fn is_filename_alt(alt: &str) -> bool { - let alt = alt.trim(); - if alt.contains(' ') && alt.split_whitespace().count() > 4 { - return false; - } - let Some((stem, ext)) = alt.rsplit_once('.') else { - return false; - }; - let ext = ext.to_ascii_lowercase(); - matches!( - ext.as_str(), - "jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" | "avif" | "bmp" | "heic" - ) && !stem.is_empty() - && stem - .chars() - .all(|c| c.is_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.')) -} - -/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the -/// `[image: alt]` placeholder paragraph (§3.10). -pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String { - let by_url: HashMap<&str, &ImageAsset> = - assets.iter().map(|a| (a.source_url.as_str(), a)).collect(); - let mut out = String::with_capacity(html.len()); - let mut cursor = 0usize; - while let Some(rel) = html[cursor..].find('<') { - let start = cursor + rel; - out.push_str(&html[cursor..start]); - let Some(end) = tag_end(html, start) else { - out.push_str(&html[start..]); - return out; - }; - let raw = &html[start..end]; - let inner = raw - .trim_start_matches('<') - .trim_end_matches('>') - .trim_end_matches('/'); - if tag_name(inner) == "img" { - let attrs = parse_attrs(inner); - let src = attrs - .iter() - .find(|(k, _)| k == "src") - .map(|(_, v)| v.trim().to_string()) - .unwrap_or_default(); - let alt = attrs - .iter() - .find(|(k, _)| k == "alt") - .map(|(_, v)| v.trim().to_string()) - .unwrap_or_default(); - match by_url.get(src.as_str()) { - Some(asset) => { - let alt = if alt.is_empty() { &asset.alt } else { &alt }; - out.push_str(&format!( - "<img src=\"{}\" alt=\"{}\"/>", - attr_escape(&asset.href), - attr_escape(alt) - )); - } - // An image we could not embed is only worth announcing when its - // alt text tells the reader something; otherwise the `<img>` - // just goes away. That covers the decorative graphics the - // re-encoder deliberately skips as well as genuine misses (§3.10). - None if !alt_is_worth_announcing(&alt) => {} - None => { - out.push_str(&format!( - "<p class=\"image-placeholder\">[image: {}]</p>", - text_escape(&alt) - )); - } - } - } else { - out.push_str(raw); - } - cursor = end; - } - out.push_str(&html[cursor..]); - out -} - -/// Self-close HTML void elements and normalize ` ` so the markup parses as -/// XML — EPUB3 content documents are XHTML (§3.10). -pub fn to_xhtml(html: &str) -> String { - let mut out = String::with_capacity(html.len()); - let mut cursor = 0usize; - while let Some(rel) = html[cursor..].find('<') { - let start = cursor + rel; - out.push_str(&html[cursor..start]); - let Some(end) = tag_end(html, start) else { - out.push_str(&html[start..]); - cursor = html.len(); - break; - }; - let raw = &html[start..end]; - let inner = raw.trim_start_matches('<').trim_end_matches('>'); - let name = tag_name(inner); - if VOID_ELEMENTS.contains(&name.as_str()) && !inner.trim_end().ends_with('/') { - out.push('<'); - out.push_str(inner.trim_end()); - out.push_str("/>"); - } else { - out.push_str(raw); - } - cursor = end; - } - out.push_str(&html[cursor..]); - // html5ever (via ammonia) emits ` `, which is undefined in XML. - out.replace(" ", " ") -} - -#[cfg(test)] -mod tests { - use super::*; - - fn asset(url: &str, href: &str) -> ImageAsset { - ImageAsset { - id: "img-1-0".into(), - href: href.into(), - mime: "image/jpeg".into(), - data: vec![1, 2, 3], - alt: "fallback alt".into(), - caption: None, - source_url: url.into(), - } - } - - #[test] - fn extracts_img_refs_with_captions() { - let html = r#"<p>hi</p> - <figure><img src="https://e.g/a.png" alt="A diagram"/> - <figcaption>Figure 1: the thing</figcaption></figure> - <img src="https://e.g/b.jpg"/> - <img src="data:image/png;base64,zz"/> - <img src="https://e.g/a.png" alt="dupe"/>"#; - let refs = extract_img_refs(html); - assert_eq!(refs.len(), 2); - assert_eq!(refs[0].src, "https://e.g/a.png"); - assert_eq!(refs[0].alt, "A diagram"); - assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing")); - assert_eq!(refs[1].src, "https://e.g/b.jpg"); - assert!(refs[1].caption.is_none()); - } - - #[test] - fn rewrites_hits_and_placeholders_misses() { - let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")]; - let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="A chart of missing things">"#; - let out = rewrite_img_srcs(html, &assets); - // The alt round-trips through one level of escaping, not two. - assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt & more"/>"#)); - assert!( - out.contains(r#"<p class="image-placeholder">[image: A chart of missing things]</p>"#) - ); - assert!(!out.contains("gone.png")); - } - - /// The whole point of the fix: ammonia writes `&` into the markup, and - /// the asset was keyed on the URL a real parser produced. - #[test] - fn entity_encoded_urls_still_match_their_asset() { - let assets = vec![asset( - "https://e.g/a.jpg?id=1&width=980", - "images/img-1-0.jpg", - )]; - let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#; - assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#)); - // The numeric spelling WordPress emits works too. - let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#; - assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#)); - } - - #[test] - fn unembeddable_images_only_speak_up_when_the_alt_says_something() { - // No alt at all: the image simply disappears. - assert_eq!( - rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]), - "" - ); - // A filename is not a description. - assert_eq!( - rewrite_img_srcs(r#"<img src="https://e.g/x.png" alt="IMG_0808.JPG">"#, &[]), - "" - ); - // Neither is the label on a decorative divider rule. - assert_eq!( - rewrite_img_srcs(r#"<img src="https://e.g/rule.png" alt="red line">"#, &[]), - "" - ); - // A real description is worth keeping. - assert!( - rewrite_img_srcs( - r#"<img src="https://e.g/x.png" alt="A man in a hard hat stands over a well hole">"#, - &[] - ) - .contains("[image: A man in a hard hat stands over a well hole]") - ); - } - - #[test] - fn filename_alt_detection() { - for yes in [ - "IMG_0808.JPG", - "cut pieces v01.JPG", - "chart-final-2.png", - "diagram.svg", - ] { - assert!(is_filename_alt(yes), "{yes} should read as a filename"); - } - for no in [ - "A hydrogen well head", - "", - "Fig. 3", - "The lion-man of Hohlenstein-Stadel, carved from mammoth ivory.", - ] { - assert!(!is_filename_alt(no), "{no} should read as a description"); - } - } - - #[test] - fn to_xhtml_self_closes_voids_and_entities() { - let html = "<p>a<br>b<hr>c d<img src=\"x.png\" alt=\"y\"></p><p>e<br/></p>"; - let out = to_xhtml(html); - assert!(out.contains("<br/>")); - assert!(out.contains("<hr/>")); - assert!(out.contains("<img src=\"x.png\" alt=\"y\"/>")); - assert!(out.contains(" ")); - assert!(!out.contains(" ")); - assert!(!out.contains("<br/ >")); - // Already-closed voids are left alone (no double slash). - assert_eq!(out.matches("<br/>").count(), 2); - } - - #[test] - fn tag_scanner_ignores_angle_brackets_in_attributes() { - let html = r#"<a title="a > b">x</a>"#; - assert_eq!(to_xhtml(html), html); - } - - #[test] - fn reencode_resizes_grayscales_and_encodes() { - let mut img = image::RgbaImage::new(200, 100); - for (x, y, px) in img.enumerate_pixels_mut() { - *px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]); - } - let mut png = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(img) - .write_to(&mut png, ImageFormat::Png) - .unwrap(); - let raw = png.into_inner(); - - let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap(); - assert_eq!(std_mime, "image/jpeg"); - let decoded = image::load_from_memory(&std_bytes).unwrap(); - assert_eq!(decoded.dimensions(), (200, 100), "no upscaling"); - - let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap(); - let x4 = image::load_from_memory(&x4_bytes).unwrap(); - assert!(x4.width() <= 480 && x4.height() <= 800); - assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8"); - } - - #[test] - fn reencode_skips_decorative_images_and_junk() { - let tiny = image::RgbaImage::new(8, 8); - let mut png = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(tiny) - .write_to(&mut png, ImageFormat::Png) - .unwrap(); - assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none()); - // A bare `<svg>` tag with nothing to draw is markup, not a picture. - assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none()); - assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none()); - } - - #[test] - fn svg_charts_are_rasterized_rather_than_dropped() { - let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300"> - <rect x="10" y="10" width="380" height="280" fill="#3355bb"/> - <circle cx="200" cy="150" r="60" fill="#ffcc00"/> - </svg>"##; - let (bytes, mime) = reencode(svg, ImageProfile::STANDARD).expect("svg rasterizes"); - assert_eq!(mime, "image/png", "flat colour art stays lossless"); - let decoded = image::load_from_memory(&bytes).expect("decodable output"); - // Drawn at the edition's target width, not at the SVG's nominal size. - assert_eq!(decoded.dimensions(), (1200, 900)); - assert!(!decoded.color().has_alpha(), "rendered onto white"); - - // An XML prolog and a leading BOM must not hide the format. - let with_prolog = format!( - "\u{feff}<?xml version=\"1.0\"?>{}", - String::from_utf8_lossy(svg) - ); - let (x4, _) = reencode(with_prolog.as_bytes(), ImageProfile::X4).expect("x4 rasterizes"); - let x4 = image::load_from_memory(&x4).unwrap(); - assert!(x4.width() <= 480 && x4.height() <= 800); - - // A 16×16 icon is decorative however large we could draw it. - let icon = br#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"> - <rect width="16" height="16"/></svg>"#; - assert!(reencode(icon, ImageProfile::STANDARD).is_none()); - } - - #[test] - fn line_art_png_stays_png_and_is_flattened() { - let mut img = image::RgbaImage::new(120, 60); - for (x, _y, px) in img.enumerate_pixels_mut() { - *px = if x % 12 == 0 { - image::Rgba([0, 0, 0, 255]) - } else { - image::Rgba([255, 255, 255, 0]) - }; - } - let mut png = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(img) - .write_to(&mut png, ImageFormat::Png) - .unwrap(); - let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap(); - assert_eq!(mime, "image/png"); - let decoded = image::load_from_memory(&bytes).unwrap(); - assert!(!decoded.color().has_alpha(), "transparency is flattened"); - // Transparent pixels became white. - assert_eq!( - decoded.to_rgb8().get_pixel(1, 1), - &image::Rgb([255, 255, 255]) - ); - } -} diff --git a/src/epub/mod.rs b/src/epub/mod.rs index 24a88e5..82805e8 100644 --- a/src/epub/mod.rs +++ b/src/epub/mod.rs @@ -5,12 +5,15 @@ //! `art-{entry_id}` so rating links stay stable across regenerations. pub mod build; -pub mod images; +pub mod chapters; +pub mod cover; +pub mod fixtures; pub mod x4; use std::path::{Path, PathBuf}; use crate::config::Config; +use crate::images; use crate::types::{Artifact, Edition, ImageAsset, Issue}; /// Chapter order inside an issue (§3.10). diff --git a/src/epub/x4.rs b/src/epub/x4.rs index 390e4f7..e0e936a 100644 --- a/src/epub/x4.rs +++ b/src/epub/x4.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use crate::config::XtcConfig; -use super::images::tag_end; +use crate::html::tag_end; /// Native X4 screen size, used for the cover and image fitting (§3.10). pub const X4_SCREEN: (u32, u32) = (480, 800); diff --git a/src/extract.rs b/src/extract.rs index 894398a..4f84359 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -1,17 +1,23 @@ -//! Content extraction, sanitization and word counting (spec §3.3). +//! Getting an article's body text (spec §3.3). //! //! Priority order per article: Miniflux content if it looks like full text → //! fetch + `dom_smoothie` readability → feed excerpt with a "(excerpt only)" note. +//! Whichever wins is sanitized down to the tag subset the EPUB templates accept. +//! +//! Image handling lives in [`crate::images`]; this module calls into +//! [`crate::images::normalize`] before readability and before sanitizing, and +//! reads image URLs back out with [`crate::images::collect_image_urls`]. use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use futures::StreamExt; -use scraper::{Html, Node, Selector}; use url::Url; -use crate::epub::images::{tag_end, tag_name}; +use crate::html::word_count; +use crate::images::normalize::{normalize_img_tags, prepare_for_readability}; +use crate::images::refs::collect_image_urls; use crate::types::{Article, ExtractMethod, Extracted}; /// Word count at or above which Miniflux content is treated as full text (§3.3). @@ -343,303 +349,6 @@ fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> { domains } -// --------------------------------------------------------------------------- -// Image normalization (§3.3) -// --------------------------------------------------------------------------- - -/// Elements that readability deletes outright, and which a page may nevertheless -/// have wrapped around an image (lightbox triggers, mostly). -const IMAGE_WRAPPER_TAGS: &[&str] = &["button", "form", "fieldset", "object"]; - -/// Attributes lazy-loading libraries use for the real image URL. -/// -/// These outrank `src`, because a page only sets them when `src` is a stand-in: -/// a transparent GIF, a blurred thumbnail, an inline SVG spacer. Attributes that -/// merely *look* image-ish (`data-template`, `data-attrs`, `data-orig-file`) are -/// deliberately absent — those hold templates and metadata, and preferring them -/// is exactly the mistake readability's own heuristic makes. -const LAZY_SRC_ATTRS: &[&str] = &[ - "data-src", - "data-lazy-src", - "data-original", - "data-runner-src", - "data-full-src", - "data-hi-res-src", - "data-image-src", -]; - -/// Widest `srcset` candidate we will pick; above this we are downloading pixels -/// the re-encoder immediately throws away. -const MAX_SRCSET_WIDTH: u32 = 2000; - -/// Make a fetched page safe to hand to readability (§3.3). -/// -/// Two passes, both about images: unwrap the elements that would take an image -/// with them when readability deletes them, then reduce every `<img>` to a plain -/// `src`/`alt`/`title` triple. The second pass is what stops readability's own -/// lazy-image heuristic from replacing a working `src` — with no `srcset`, -/// `loading` or `data-*` attributes left on the element, it has nothing to -/// substitute and leaves the image alone. -pub fn prepare_for_readability(html: &str) -> String { - normalize_img_tags(&unwrap_image_wrappers(html)) -} - -/// Replace image-only `<button>`/`<form>`/`<fieldset>`/`<object>` wrappers with -/// their contents (§3.3). -/// -/// A `<button>` holding nothing but an image is a lightbox trigger, not a -/// control: the image is the content. Wrappers that also carry text are left -/// alone, because those really are interface. -pub fn unwrap_image_wrappers(html: &str) -> String { - let mut out = String::with_capacity(html.len()); - let mut cursor = 0usize; - while let Some(rel) = html[cursor..].find('<') { - let start = cursor + rel; - out.push_str(&html[cursor..start]); - let Some(end) = tag_end(html, start) else { - out.push_str(&html[start..]); - return out; - }; - let raw = &html[start..end]; - let inner = raw.trim_start_matches('<').trim_end_matches('>'); - let name = tag_name(inner); - - if IMAGE_WRAPPER_TAGS.contains(&name.as_str()) - && !inner.trim_end().ends_with('/') - && let Some((content, after)) = element_content(html, end, &name) - && content.contains("<img") - && html_to_text(content).trim().is_empty() - { - out.push_str(&unwrap_image_wrappers(content)); - cursor = after; - continue; - } - - out.push_str(raw); - cursor = end; - } - out.push_str(&html[cursor..]); - out -} - -/// The content of an element whose open tag ended at `body_start`, plus the -/// offset just past its close tag. `None` when the element is never closed. -fn element_content<'a>(html: &'a str, body_start: usize, name: &str) -> Option<(&'a str, usize)> { - let open = format!("<{name}"); - let close = format!("</{name}"); - let mut depth = 1usize; - let mut cursor = body_start; - while let Some(rel) = html[cursor..].find('<') { - let start = cursor + rel; - let end = tag_end(html, start)?; - let tag = &html[start..end]; - let lower = tag.to_ascii_lowercase(); - if lower.starts_with(&close) { - depth -= 1; - if depth == 0 { - return Some((&html[body_start..start], end)); - } - } else if lower.starts_with(&open) - && !lower[open.len()..].starts_with(|c: char| c.is_alphanumeric() || c == '-') - && !tag.trim_end().ends_with("/>") - { - depth += 1; - } - cursor = end; - } - None -} - -/// Reduce every `<img>` to `<img src alt title>` with a usable URL (§3.3). -/// -/// The `src` a page ships is not automatically the one to use: it can be a lazy -/// placeholder (`data:image/svg+xml,…`), an unresolved template -/// (`…/resize/{width}/…`), a JSON blob a framework parked there, or an entire -/// `srcset` string. Candidates are tried in order and the first plausible one -/// wins; an image with no plausible candidate is dropped, because a broken -/// `<img>` only becomes clutter further down the pipeline. -pub fn normalize_img_tags(html: &str) -> String { - let mut out = String::with_capacity(html.len()); - let mut cursor = 0usize; - // `<picture>` puts the real candidates on sibling `<source>` elements. - let mut picture_srcset: Option<String> = None; - - while let Some(rel) = html[cursor..].find('<') { - let start = cursor + rel; - out.push_str(&html[cursor..start]); - let Some(end) = tag_end(html, start) else { - out.push_str(&html[start..]); - return out; - }; - let raw = &html[start..end]; - let inner = raw - .trim_start_matches('<') - .trim_end_matches('>') - .trim_end_matches('/'); - let name = tag_name(inner); - - match name.as_str() { - "picture" => { - picture_srcset = None; - out.push_str(raw); - } - "source" => { - let attrs = crate::epub::images::parse_attrs(inner); - if picture_srcset.is_none() - && let Some(set) = - attr(&attrs, "srcset").or_else(|| attr(&attrs, "data-srcset")) - { - picture_srcset = Some(set.to_string()); - } - out.push_str(raw); - } - "img" => { - let attrs = crate::epub::images::parse_attrs(inner); - if let Some(src) = best_img_src(&attrs, picture_srcset.as_deref()) { - out.push_str("<img src=\""); - out.push_str(&escape_attr(&src)); - out.push('"'); - for key in ["alt", "title"] { - if let Some(v) = attr(&attrs, key) { - out.push(' '); - out.push_str(key); - out.push_str("=\""); - out.push_str(&escape_attr(v)); - out.push('"'); - } - } - out.push_str("/>"); - } else { - tracing::debug!(tag = %&raw[..raw.len().min(120)], "dropping unusable img"); - } - } - _ => out.push_str(raw), - } - if name == "picture" && inner.starts_with('/') { - picture_srcset = None; - } - cursor = end; - } - out.push_str(&html[cursor..]); - out -} - -fn attr<'a>(attrs: &'a [(String, String)], name: &str) -> Option<&'a str> { - attrs - .iter() - .find(|(k, _)| k == name) - .map(|(_, v)| v.trim()) - .filter(|v| !v.is_empty()) -} - -/// Pick the best URL for one `<img>` from everything the element carries. -fn best_img_src(attrs: &[(String, String)], picture_srcset: Option<&str>) -> Option<String> { - for key in LAZY_SRC_ATTRS { - if let Some(v) = attr(attrs, key).filter(|s| plausible_url(s)) { - return Some(v.to_string()); - } - } - if let Some(src) = attr(attrs, "src").filter(|s| plausible_url(s)) { - return Some(src.to_string()); - } - if let Some(from_set) = attr(attrs, "srcset").and_then(best_from_srcset) { - return Some(from_set); - } - if let Some(from_set) = attr(attrs, "data-srcset").and_then(best_from_srcset) { - return Some(from_set); - } - picture_srcset.and_then(best_from_srcset) -} - -/// The widest candidate in a `srcset` that is still worth downloading. -/// -/// Parsed by whitespace rather than by comma: the URLs of several image CDNs -/// contain commas of their own, and splitting on those shreds them. -fn best_from_srcset(srcset: &str) -> Option<String> { - let mut best: Option<(u32, String)> = None; - let mut smallest: Option<(u32, String)> = None; - let mut pending: Option<String> = None; - - let mut consider = |url: String, width: u32| { - if width <= MAX_SRCSET_WIDTH && best.as_ref().is_none_or(|(w, _)| width > *w) { - best = Some((width, url.clone())); - } - if smallest.as_ref().is_none_or(|(w, _)| width < *w) { - smallest = Some((width, url)); - } - }; - - for token in srcset.split_whitespace() { - let token = token.trim_end_matches(','); - if token.is_empty() { - continue; - } - match parse_descriptor(token) { - Some(width) => { - if let Some(url) = pending.take() { - consider(url, width); - } - } - None => { - // A URL with no descriptor of its own still counts, at width 1x. - if let Some(url) = pending.replace(token.to_string()) { - consider(url, 1); - } - } - } - } - if let Some(url) = pending.take() { - consider(url, 1); - } - - best.or(smallest) - .map(|(_, url)| url) - .filter(|u| plausible_url(u)) -} - -/// `800w` → 800, `2x` → a synthetic width so density candidates sort sensibly. -fn parse_descriptor(token: &str) -> Option<u32> { - let (value, unit) = token.split_at(token.len().checked_sub(1)?); - match unit { - "w" => value.parse::<u32>().ok(), - "x" => value - .parse::<f32>() - .ok() - .map(|d| (d * 1000.0).round().clamp(1.0, f32::from(u16::MAX)) as u32), - _ => None, - } -} - -/// Whether a string can serve as an image URL at all. -/// -/// This is deliberately about shape, not about the host: whitespace, braces and -/// quotes mean we are looking at a `srcset` blob, an unfilled URL template or a -/// serialized object, none of which will ever resolve. -fn plausible_url(candidate: &str) -> bool { - let candidate = candidate.trim(); - if candidate.is_empty() || candidate.len() > 2048 { - return false; - } - if candidate.starts_with("data:") || candidate.starts_with("about:") { - return false; - } - if candidate - .chars() - .any(|c| c.is_whitespace() || matches!(c, '{' | '}' | '"' | '\'' | '<' | '>' | '\\')) - { - return false; - } - // `%20` is a space that survived encoding — same blob, different spelling. - !candidate.contains("%20") -} - -fn escape_attr(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) -} - // --------------------------------------------------------------------------- // Sanitization (§3.3) // --------------------------------------------------------------------------- @@ -711,74 +420,6 @@ pub fn sanitize_with_base(html: &str, base_url: &str) -> String { } } -// --------------------------------------------------------------------------- -// Text measurement (§3.3) -// --------------------------------------------------------------------------- - -/// Visible text of an HTML fragment, entities decoded, `script`/`style` skipped. -pub fn html_to_text(html: &str) -> String { - let document = Html::parse_fragment(html); - let mut out = String::with_capacity(html.len() / 2); - for node in document.tree.nodes() { - let Node::Text(text) = node.value() else { - continue; - }; - let hidden = node.ancestors().any(|a| match a.value() { - Node::Element(e) => matches!(e.name(), "script" | "style" | "noscript"), - _ => false, - }); - if hidden { - continue; - } - out.push_str(text); - out.push(' '); - } - out -} - -/// Count words in rendered text (tags stripped) (§3.3). -pub fn word_count(html: &str) -> i64 { - html_to_text(html) - .split_whitespace() - .filter(|w| w.chars().any(char::is_alphanumeric)) - .count() as i64 -} - -/// Absolute image URLs referenced by `html`, resolved against `base_url` (§3.3). -/// -/// Every image an article carries is kept: a photo essay with thirty pictures is -/// a photo essay, and the issue-wide byte budget is the real backstop. -pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> { - let Ok(selector) = Selector::parse("img") else { - return Vec::new(); - }; - let base = Url::parse(base_url).ok(); - let document = Html::parse_fragment(html); - let mut seen: HashSet<String> = HashSet::new(); - let mut out: Vec<String> = Vec::new(); - for element in document.select(&selector) { - let raw = element - .value() - .attr("src") - .or_else(|| element.value().attr("data-src")) - .map(str::trim) - .filter(|s| !s.is_empty()); - let Some(raw) = raw else { continue }; - let resolved = match Url::parse(raw) { - Ok(u) => Some(u), - Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()), - }; - let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else { - continue; - }; - let url = url.to_string(); - if seen.insert(url.clone()) { - out.push(url); - } - } - out -} - /// Heuristic paywall detection: very short text on a known paywall domain (§3.3). /// /// Two rules: anything under [`EXCERPT_MAX_WORDS`] is a stub whatever the host, @@ -927,238 +568,10 @@ mod tests { assert!(sanitize_with_base(html, "not a url").contains("/next")); } - #[test] - fn word_count_ignores_markup_and_script() { - assert_eq!(word_count("<p>one two three</p>"), 3); - assert_eq!(word_count("<p>a</p><script>b c d e</script>"), 1); - assert_eq!(word_count("<p>& — ok</p>"), 1); - assert_eq!(word_count(""), 0); - assert_eq!(word_count("<p></p>"), 0); - } - - #[test] - fn image_collection_resolves_and_keeps_every_image() { - let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#); - html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#); - for i in 0..20 { - html.push_str(&format!(r#"<img src="/n{i}.png">"#)); - } - let urls = collect_image_urls(&html, "https://blog.dev/posts/one"); - // Two named images, the data-src one, and all twenty of the rest: no cap. - assert_eq!(urls.len(), 23); - assert_eq!(urls[0], "https://blog.dev/a.png"); - assert_eq!(urls[1], "https://cdn.dev/b.png"); - assert_eq!(urls[2], "https://blog.dev/posts/c.png"); - // Duplicates and data: URIs never appear. - assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1); - assert!(!urls.iter().any(|u| u.starts_with("data:"))); - assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty()); - } - - // ----------------------------------------------------------------------- - // Image normalization — each case is a page shape seen in a real issue. - // ----------------------------------------------------------------------- - - fn src_of(html: &str) -> Vec<String> { - Html::parse_fragment(html) - .select(&Selector::parse("img").unwrap()) - .filter_map(|e| e.value().attr("src").map(str::to_string)) - .collect() - } - - #[test] - fn lazy_placeholder_src_gives_way_to_the_real_url() { - // IEEE Spectrum: an inline SVG spacer with the URL parked on data-runner-src. - let html = r#"<img alt="A well head" lazy-loadable="true" - src="data:image/svg+xml,%3Csvg%20xmlns=%27http://www.w3.org/2000/svg%27%3E%3C/svg%3E" - data-runner-src="https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"/>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"] - ); - } - - #[test] - fn unresolved_url_templates_fall_through_to_a_real_candidate() { - // NPR: readability copies data-template over a perfectly good src. - let html = r#"<img alt="Meghan Cliffel" - src="https://npr.brightspotcdn.com/resize/{width}/quality/{quality}/x.jpg" - srcset="https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg 1100w"/>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg"] - ); - // With nothing usable anywhere, the image goes rather than becoming a - // request for a picture that says "Image". - let only_template = - r#"<img alt="x" src="https://cdn.dev/resize/{width}/quality/{quality}/x.jpg"/>"#; - assert!(src_of(&normalize_img_tags(only_template)).is_empty()); - } - - #[test] - fn a_srcset_blob_parked_in_src_is_rejected_and_reparsed() { - // dfarq: readability copies the entire srcset string into src. - let blob = "https://i0.wp.com/x.jpg?resize=300%2C158&ssl=1 300w, \ - https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1 1024w, \ - https://i0.wp.com/x.jpg?w=3000&ssl=1 3000w"; - let html = format!(r#"<img alt="printer" src="{blob}" srcset="{blob}"/>"#); - // The widest candidate under the download ceiling wins; 3000w does not. - assert_eq!( - src_of(&normalize_img_tags(&html)), - ["https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1"] - ); - } - - #[test] - fn a_json_blob_parked_in_src_is_rejected() { - // Substack: readability copies data-attrs (JSON) over src. - let html = r#"<img alt="" src="{"src":"https://s3.dev/a.jpeg","width":1000}" - srcset="https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg 1456w"/>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg"] - ); - } - - #[test] - fn picture_sources_back_up_an_empty_img() { - let html = r#"<picture> - <source srcset="https://cdn.dev/a.avif 800w" type="image/avif"/> - <source srcset="https://cdn.dev/a.webp 800w" type="image/webp"/> - <img alt="A photo"/> - </picture>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://cdn.dev/a.avif"] - ); - } - - #[test] - fn a_good_src_is_never_traded_for_a_metadata_attribute() { - // `data-template`, `data-attrs` and friends hold templates and JSON, not - // URLs — trading a working src for one of those is the original sin. - let html = r#"<img alt="A photo" loading="lazy" class="lazyload" - src="https://cdn.dev/real.jpg" - data-template="https://cdn.dev/{width}/real.jpg" - data-orig-file="https://cdn.dev/orig.jpg"/>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://cdn.dev/real.jpg"] - ); - } - - #[test] - fn a_lazy_loader_placeholder_loses_to_its_data_src() { - // iRunFar (a3-lazy-load): src is a shared 1×1 spacer that would download - // and re-encode perfectly happily, and be the wrong picture. - let html = r#"<img class="lazy lazy-hidden" alt="Brooks Cascadia 20" - src="//www.irunfar.com/wp-content/plugins/a3-lazy-load/assets/images/lazy_placeholder.gif" - data-src="https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"/>"#; - assert_eq!( - src_of(&normalize_img_tags(html)), - ["https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"] - ); - } - - #[test] - fn normalization_keeps_alt_and_title_and_drops_the_rest() { - let html = r#"<img src="/a.png" alt="An & alt" title="T" class="x" width="900" - onerror="evil()" srcset="/b.png 2x"/>"#; - let out = normalize_img_tags(html); - assert!(out.contains(r#"alt="An & alt""#), "{out}"); - assert!(out.contains(r#"title="T""#), "{out}"); - for gone in ["class=", "width=", "onerror", "srcset="] { - assert!( - !out.contains(gone), - "expected {gone} to be dropped from {out}" - ); - } - // Relative URLs survive: sanitize_with_base absolutizes them later. - assert_eq!(src_of(&out), ["/a.png"]); - } - - #[test] - fn lightbox_buttons_no_longer_take_their_image_with_them() { - // Nautilus: readability deletes <button> and everything inside it. - let html = r#"<figure class="wp-block-image"> - <button type="button" aria-label="Enlarge image"> - <img class="wp-image-1" src="https://cdn.dev/flower.png?w=710" alt=""/> - </button> - <figcaption>BEAUTIFUL DANGER: a belladonna flower.</figcaption> - </figure>"#; - let out = unwrap_image_wrappers(html); - assert!(!out.contains("<button"), "{out}"); - assert!(!out.contains("</button>"), "{out}"); - assert!(out.contains("flower.png"), "{out}"); - assert!(out.contains("BEAUTIFUL DANGER"), "caption survives: {out}"); - } - - #[test] - fn buttons_that_are_really_buttons_are_left_alone() { - let html = r#"<button class="subscribe">Subscribe <img src="/icon.png" alt=""/></button>"#; - assert_eq!(unwrap_image_wrappers(html), html); - // And an image-free control is untouched too. - let plain = r#"<button>Share</button>"#; - assert_eq!(unwrap_image_wrappers(plain), plain); - } - - #[test] - fn nested_and_unclosed_wrappers_do_not_derail_the_scan() { - let nested = r#"<button><button><img src="/a.png"/></button></button><p>after</p>"#; - let out = unwrap_image_wrappers(nested); - assert!(!out.contains("button"), "{out}"); - assert!(out.contains("/a.png") && out.contains("after"), "{out}"); - // An open tag that never closes is passed through rather than eating - // the rest of the document. - let unclosed = r#"<button><img src="/a.png"/><p>rest</p>"#; - assert!(unwrap_image_wrappers(unclosed).contains("rest")); - } - - #[test] - fn srcset_descriptors_pick_the_widest_usable_candidate() { - assert_eq!( - best_from_srcset("/a.png 150w, /b.png 800w, /c.png 4000w").as_deref(), - Some("/b.png") - ); - // Density descriptors work as an ordering too. - assert_eq!( - best_from_srcset("/a.png 1x, /b.png 2x").as_deref(), - Some("/b.png") - ); - // A bare URL with no descriptor is still a candidate. - assert_eq!(best_from_srcset("/only.png").as_deref(), Some("/only.png")); - // Every candidate too wide: take the narrowest rather than nothing. - assert_eq!( - best_from_srcset("/big.png 3000w, /huge.png 5000w").as_deref(), - Some("/big.png") - ); - assert_eq!(best_from_srcset(" ").as_deref(), None); - } - - #[test] - fn plausibility_is_about_shape_not_host() { - for good in [ - "https://cdn.dev/a.jpg?id=1&width=980", - "/_next/image?url=https%3A%2F%2Fx.dev%2Fa.png&w=3840&q=75", - "https://substackcdn.com/image/fetch/$s_!y1,w_1456/a.jpeg", - ] { - assert!(plausible_url(good), "{good} should be usable"); - } - for bad in [ - "", - "data:image/svg+xml,%3Csvg%3E%3C/svg%3E", - "https://cdn.dev/resize/{width}/a.jpg", - r#"{"src":"https://cdn.dev/a.jpg"}"#, - "https://cdn.dev/a.jpg 300w, https://cdn.dev/b.jpg 600w", - "https://cdn.dev/a.jpg%20300w", - ] { - assert!(!plausible_url(bad), "{bad} should be rejected"); - } - } - + /// Feeds carry the same lazy markup pages do, so the same normalization runs + /// on Miniflux content before it is sanitized. #[tokio::test] async fn feed_content_is_normalized_too() { - // Feeds carry the same lazy markup pages do. let extractor = Extractor::offline(vec![]); let body = format!( r#"<p>{}</p><img src="data:image/gif;base64,zz" data-src="https://cdn.dev/real.jpg"/>"#, diff --git a/src/html.rs b/src/html.rs new file mode 100644 index 0000000..2c5ce84 --- /dev/null +++ b/src/html.rs @@ -0,0 +1,356 @@ +//! Generic HTML markup utilities shared across the pipeline. +//! +//! Everything here is about *markup as text*: scanning tags without building a +//! DOM, escaping for XML output, and reading a fragment as prose. It knows +//! nothing about articles, images or EPUBs — those modules build on top of it. +//! +//! There are two ways to look at HTML in this crate. `scraper` parses a real +//! DOM and is the right tool when structure matters (walking up to an enclosing +//! `<figure>`, say). The scanners here walk the string instead, which is what +//! you want when the job is to rewrite tags in place and hand back markup that +//! is otherwise byte-identical. + +use scraper::{Html, Node}; + +/// HTML void elements: XHTML requires them self-closed (§3.10 "valid XHTML"). +pub const VOID_ELEMENTS: &[&str] = &[ + "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", + "track", "wbr", +]; + +// --------------------------------------------------------------------------- +// Tag scanning +// --------------------------------------------------------------------------- + +/// End index (exclusive) of the tag starting at `start` (`html[start] == '<'`), +/// respecting quoted attribute values and comments. +pub fn tag_end(html: &str, start: usize) -> Option<usize> { + let rest = &html[start..]; + if rest.starts_with("<!--") { + return rest.find("-->").map(|i| start + i + 3); + } + let mut quote: Option<char> = None; + for (i, c) in rest.char_indices().skip(1) { + match (quote, c) { + (Some(q), c) if c == q => quote = None, + (Some(_), _) => {} + (None, '"') | (None, '\'') => quote = Some(c), + (None, '>') => return Some(start + i + c.len_utf8()), + (None, _) => {} + } + } + None +} + +/// Lowercased element name of a tag body such as `img src="…"`. +pub fn tag_name(inner: &str) -> String { + inner + .trim_start_matches('/') + .split(|c: char| c.is_whitespace() || c == '/' || c == '>') + .next() + .unwrap_or("") + .to_ascii_lowercase() +} + +/// Parse `name="value"` pairs out of a tag body, with entity-decoded values. +/// +/// Decoding matters: this scanner reads markup that ammonia has serialized, and +/// ammonia writes `&` in a URL as `&`. A raw comparison against a URL that +/// came out of a real HTML parser would never match (§3.10). +pub fn parse_attrs(inner: &str) -> Vec<(String, String)> { + let mut attrs = Vec::new(); + let bytes: Vec<char> = inner.chars().collect(); + let mut i = 0; + // Skip the element name. + while i < bytes.len() && !bytes[i].is_whitespace() { + i += 1; + } + while i < bytes.len() { + while i < bytes.len() && (bytes[i].is_whitespace() || bytes[i] == '/') { + i += 1; + } + let name_start = i; + while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '=' && bytes[i] != '/' { + i += 1; + } + if i == name_start { + break; + } + let name: String = bytes[name_start..i] + .iter() + .collect::<String>() + .to_ascii_lowercase(); + while i < bytes.len() && bytes[i].is_whitespace() { + i += 1; + } + let mut value = String::new(); + if i < bytes.len() && bytes[i] == '=' { + i += 1; + while i < bytes.len() && bytes[i].is_whitespace() { + i += 1; + } + if i < bytes.len() && (bytes[i] == '"' || bytes[i] == '\'') { + let quote = bytes[i]; + i += 1; + while i < bytes.len() && bytes[i] != quote { + value.push(bytes[i]); + i += 1; + } + i += 1; + } else { + while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '>' { + value.push(bytes[i]); + i += 1; + } + } + } + attrs.push((name, decode_entities(&value))); + } + attrs +} + +/// Decode the handful of entities an HTML serializer emits inside attributes. +/// +/// Numeric forms are included because feeds and WordPress write `&` for +/// `&`; anything else is left alone rather than guessed at. +pub fn decode_entities(s: &str) -> String { + if !s.contains('&') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + let tail = &rest[i..]; + let Some(end) = tail[..tail.len().min(12)].find(';') else { + out.push('&'); + rest = &tail[1..]; + continue; + }; + let entity = &tail[1..end]; + let decoded = match entity { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" | "#39" => Some('\''), + "nbsp" => Some('\u{a0}'), + _ => entity + .strip_prefix('#') + .and_then(|n| match n.strip_prefix(['x', 'X']) { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => n.parse::<u32>().ok(), + }) + .and_then(char::from_u32), + }; + match decoded { + Some(c) => { + out.push(c); + rest = &tail[end + 1..]; + } + None => { + out.push('&'); + rest = &tail[1..]; + } + } + } + out.push_str(rest); + out +} + +/// Escape a string for use inside a double-quoted XML attribute. +pub fn attr_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + _ => out.push(c), + } + } + out +} + +/// Escape a string for XML text content. +pub fn text_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + _ => out.push(c), + } + } + out +} + +// --------------------------------------------------------------------------- +// XHTML output +// --------------------------------------------------------------------------- + +/// Self-close HTML void elements and normalize ` ` so the markup parses as +/// XML — EPUB3 content documents are XHTML (§3.10). +pub fn to_xhtml(html: &str) -> String { + let mut out = String::with_capacity(html.len()); + let mut cursor = 0usize; + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + out.push_str(&html[cursor..start]); + let Some(end) = tag_end(html, start) else { + out.push_str(&html[start..]); + cursor = html.len(); + break; + }; + let raw = &html[start..end]; + let inner = raw.trim_start_matches('<').trim_end_matches('>'); + let name = tag_name(inner); + if VOID_ELEMENTS.contains(&name.as_str()) && !inner.trim_end().ends_with('/') { + out.push('<'); + out.push_str(inner.trim_end()); + out.push_str("/>"); + } else { + out.push_str(raw); + } + cursor = end; + } + out.push_str(&html[cursor..]); + // html5ever (via ammonia) emits ` `, which is undefined in XML. + out.replace(" ", " ") +} + +// --------------------------------------------------------------------------- +// Reading markup as text +// --------------------------------------------------------------------------- + +/// Visible text of an HTML fragment, entities decoded, `script`/`style` skipped. +pub fn html_to_text(html: &str) -> String { + let document = Html::parse_fragment(html); + let mut out = String::with_capacity(html.len() / 2); + for node in document.tree.nodes() { + let Node::Text(text) = node.value() else { + continue; + }; + let hidden = node.ancestors().any(|a| match a.value() { + Node::Element(e) => matches!(e.name(), "script" | "style" | "noscript"), + _ => false, + }); + if hidden { + continue; + } + out.push_str(text); + out.push(' '); + } + out +} + +/// Count words in rendered text (tags stripped) (§3.3). +pub fn word_count(html: &str) -> i64 { + html_to_text(html) + .split_whitespace() + .filter(|w| w.chars().any(char::is_alphanumeric)) + .count() as i64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tag_scanner_finds_the_end_of_awkward_tags() { + // A `>` inside a quoted attribute is not the end of the tag. + let html = r#"<a title="a > b">x</a>"#; + assert_eq!( + tag_end(html, 0), + Some(17), + "past the closing `>`, not the one in the title" + ); + // Comments end at `-->`, not at the first `>`. + let comment = "<!-- a > b -->rest"; + assert_eq!(tag_end(comment, 0), Some(14)); + // An unterminated tag has no end. + assert_eq!(tag_end("<p class=\"x", 0), None); + } + + #[test] + fn tag_names_are_lowercased_and_stripped() { + assert_eq!(tag_name("IMG src=\"x\""), "img"); + assert_eq!(tag_name("/DIV"), "div"); + assert_eq!(tag_name("br/"), "br"); + assert_eq!(tag_name(""), ""); + } + + #[test] + fn attributes_parse_in_every_spelling() { + let attrs = parse_attrs(r#"img src="a.png" ALT='an alt' loading=lazy hidden"#); + let get = |k: &str| attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str()); + assert_eq!(get("src"), Some("a.png")); + assert_eq!( + get("alt"), + Some("an alt"), + "names lowercase, quotes either way" + ); + assert_eq!(get("loading"), Some("lazy"), "unquoted values work"); + assert_eq!(get("hidden"), Some(""), "valueless attributes are empty"); + } + + #[test] + fn attribute_values_come_back_entity_decoded() { + // The reason this matters: a serializer writes `&` in a URL as `&`, + // and callers compare against URLs a real parser produced. + let attrs = parse_attrs(r#"img src="a.jpg?id=1&w=9&h=2""#); + assert_eq!(attrs[0].1, "a.jpg?id=1&w=9&h=2"); + } + + #[test] + fn entity_decoding_covers_named_decimal_and_hex() { + assert_eq!(decode_entities("a & b"), "a & b"); + assert_eq!(decode_entities("&&"), "&&"); + assert_eq!(decode_entities("<p>"), "<p>"); + assert_eq!(decode_entities("café"), "café"); + // Nothing to do, and nothing invented for what we do not know. + assert_eq!(decode_entities("plain"), "plain"); + assert_eq!(decode_entities("&unknown; &"), "&unknown; &"); + } + + #[test] + fn escaping_is_the_inverse_that_output_needs() { + assert_eq!( + attr_escape(r#"a & "b" <c>"#), + "a & "b" <c>" + ); + // Text content keeps quotes as they are. + assert_eq!(text_escape(r#"a & "b" <c>"#), r#"a & "b" <c>"#); + } + + #[test] + fn to_xhtml_self_closes_voids_and_entities() { + let html = "<p>a<br>b<hr>c d<img src=\"x.png\" alt=\"y\"></p><p>e<br/></p>"; + let out = to_xhtml(html); + assert!(out.contains("<br/>")); + assert!(out.contains("<hr/>")); + assert!(out.contains("<img src=\"x.png\" alt=\"y\"/>")); + assert!(out.contains(" ")); + assert!(!out.contains(" ")); + assert!(!out.contains("<br/ >")); + // Already-closed voids are left alone (no double slash). + assert_eq!(out.matches("<br/>").count(), 2); + } + + #[test] + fn to_xhtml_ignores_angle_brackets_in_attributes() { + let html = r#"<a title="a > b">x</a>"#; + assert_eq!(to_xhtml(html), html); + } + + #[test] + fn word_count_ignores_markup_and_script() { + assert_eq!(word_count("<p>one two three</p>"), 3); + assert_eq!(word_count("<p>a</p><script>b c d e</script>"), 1); + assert_eq!(word_count("<p>& — ok</p>"), 1); + assert_eq!(word_count(""), 0); + assert_eq!(word_count("<p></p>"), 0); + } +} diff --git a/src/images/embed.rs b/src/images/embed.rs new file mode 100644 index 0000000..d55cc8a --- /dev/null +++ b/src/images/embed.rs @@ -0,0 +1,193 @@ +//! Pointing article markup at the images actually embedded in the EPUB. + +use std::collections::HashMap; + +use crate::html::{attr_escape, parse_attrs, tag_end, tag_name, text_escape}; +use crate::types::ImageAsset; + +/// Whether an image we could not embed is worth telling the reader about. +/// +/// Only descriptive alt text qualifies. A filename, a bare label like `red line` +/// on a divider rule, or no alt at all carries nothing the reader loses by not +/// seeing the picture — announcing those turns every decorative graphic and +/// dead link into a line of clutter, which is how the placeholders got out of +/// hand in the first place. +fn alt_is_worth_announcing(alt: &str) -> bool { + const MIN_DESCRIPTIVE_WORDS: usize = 4; + !alt.is_empty() + && !is_filename_alt(alt) + && alt.split_whitespace().count() >= MIN_DESCRIPTIVE_WORDS +} + +/// True for alt text that is really just the uploaded filename — `IMG_0808.JPG`, +/// `cut pieces v01.JPG`, `chart-final-2.png`. +fn is_filename_alt(alt: &str) -> bool { + let alt = alt.trim(); + if alt.contains(' ') && alt.split_whitespace().count() > 4 { + return false; + } + let Some((stem, ext)) = alt.rsplit_once('.') else { + return false; + }; + let ext = ext.to_ascii_lowercase(); + matches!( + ext.as_str(), + "jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" | "avif" | "bmp" | "heic" + ) && !stem.is_empty() + && stem + .chars() + .all(|c| c.is_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.')) +} + +/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the +/// `[image: alt]` placeholder paragraph (§3.10). +pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String { + let by_url: HashMap<&str, &ImageAsset> = + assets.iter().map(|a| (a.source_url.as_str(), a)).collect(); + let mut out = String::with_capacity(html.len()); + let mut cursor = 0usize; + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + out.push_str(&html[cursor..start]); + let Some(end) = tag_end(html, start) else { + out.push_str(&html[start..]); + return out; + }; + let raw = &html[start..end]; + let inner = raw + .trim_start_matches('<') + .trim_end_matches('>') + .trim_end_matches('/'); + if tag_name(inner) == "img" { + let attrs = parse_attrs(inner); + let src = attrs + .iter() + .find(|(k, _)| k == "src") + .map(|(_, v)| v.trim().to_string()) + .unwrap_or_default(); + let alt = attrs + .iter() + .find(|(k, _)| k == "alt") + .map(|(_, v)| v.trim().to_string()) + .unwrap_or_default(); + match by_url.get(src.as_str()) { + Some(asset) => { + let alt = if alt.is_empty() { &asset.alt } else { &alt }; + out.push_str(&format!( + "<img src=\"{}\" alt=\"{}\"/>", + attr_escape(&asset.href), + attr_escape(alt) + )); + } + // An image we could not embed is only worth announcing when its + // alt text tells the reader something; otherwise the `<img>` + // just goes away. That covers the decorative graphics the + // re-encoder deliberately skips as well as genuine misses (§3.10). + None if !alt_is_worth_announcing(&alt) => {} + None => { + out.push_str(&format!( + "<p class=\"image-placeholder\">[image: {}]</p>", + text_escape(&alt) + )); + } + } + } else { + out.push_str(raw); + } + cursor = end; + } + out.push_str(&html[cursor..]); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset(url: &str, href: &str) -> ImageAsset { + ImageAsset { + id: "img-1-0".into(), + href: href.into(), + mime: "image/jpeg".into(), + data: vec![1, 2, 3], + alt: "fallback alt".into(), + caption: None, + source_url: url.into(), + } + } + + #[test] + fn rewrites_hits_and_placeholders_misses() { + let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")]; + let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="A chart of missing things">"#; + let out = rewrite_img_srcs(html, &assets); + // The alt round-trips through one level of escaping, not two. + assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt & more"/>"#)); + assert!( + out.contains(r#"<p class="image-placeholder">[image: A chart of missing things]</p>"#) + ); + assert!(!out.contains("gone.png")); + } + + /// The whole point of the fix: ammonia writes `&` into the markup, and + /// the asset was keyed on the URL a real parser produced. + #[test] + fn entity_encoded_urls_still_match_their_asset() { + let assets = vec![asset( + "https://e.g/a.jpg?id=1&width=980", + "images/img-1-0.jpg", + )]; + let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#; + assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#)); + // The numeric spelling WordPress emits works too. + let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#; + assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#)); + } + + #[test] + fn unembeddable_images_only_speak_up_when_the_alt_says_something() { + // No alt at all: the image simply disappears. + assert_eq!( + rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]), + "" + ); + // A filename is not a description. + assert_eq!( + rewrite_img_srcs(r#"<img src="https://e.g/x.png" alt="IMG_0808.JPG">"#, &[]), + "" + ); + // Neither is the label on a decorative divider rule. + assert_eq!( + rewrite_img_srcs(r#"<img src="https://e.g/rule.png" alt="red line">"#, &[]), + "" + ); + // A real description is worth keeping. + assert!( + rewrite_img_srcs( + r#"<img src="https://e.g/x.png" alt="A man in a hard hat stands over a well hole">"#, + &[] + ) + .contains("[image: A man in a hard hat stands over a well hole]") + ); + } + + #[test] + fn filename_alt_detection() { + for yes in [ + "IMG_0808.JPG", + "cut pieces v01.JPG", + "chart-final-2.png", + "diagram.svg", + ] { + assert!(is_filename_alt(yes), "{yes} should read as a filename"); + } + for no in [ + "A hydrogen well head", + "", + "Fig. 3", + "The lion-man of Hohlenstein-Stadel, carved from mammoth ivory.", + ] { + assert!(!is_filename_alt(no), "{no} should read as a description"); + } + } +} diff --git a/src/images/encode.rs b/src/images/encode.rs new file mode 100644 index 0000000..14108eb --- /dev/null +++ b/src/images/encode.rs @@ -0,0 +1,322 @@ +//! Turning downloaded bytes into something an e-reader can display. +//! +//! Every image is decoded, fitted to the edition's profile, flattened onto white +//! (e-ink has no transparency) and re-encoded. SVG is rasterized on the way in, +//! because charts and diagrams are frequently vector-only. + +use std::io::Cursor; + +use image::{DynamicImage, GenericImageView, ImageFormat}; + +use crate::types::Edition; + +/// Images smaller than this in either dimension are decorative — skipped (§3.10). +pub const MIN_DIMENSION_PX: u32 = 24; +/// Width an SVG is rendered at when the profile asks for less than this. +const SVG_FALLBACK_SIZE: u32 = 1000; + +/// Per-edition re-encoding parameters (§3.10). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ImageProfile { + pub max_width: u32, + pub max_height: u32, + pub jpeg_quality: u8, + pub grayscale: bool, +} + +impl ImageProfile { + /// Standard edition: max width 1200px, JPEG q80, color (§3.10). + pub const STANDARD: ImageProfile = ImageProfile { + max_width: 1200, + max_height: 4000, + jpeg_quality: 80, + grayscale: false, + }; + + /// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10). + pub const X4: ImageProfile = ImageProfile { + max_width: 480, + max_height: 800, + jpeg_quality: 70, + grayscale: true, + }; + + pub fn for_edition(edition: Edition) -> Self { + match edition { + Edition::Standard => Self::STANDARD, + Edition::X4 => Self::X4, + } + } +} + +/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10). +/// +/// Line art with transparency is kept as PNG after flattening; everything else +/// becomes JPEG. SVG is rasterized first — charts and diagrams are frequently +/// vector-only, and dropping them loses the point of the article. Returns `None` +/// for sources no decoder handles. +pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> { + if looks_like_svg(bytes) { + let raster = rasterize_svg(bytes, profile)?; + return reencode(&raster, profile); + } + let format = image::guess_format(bytes).ok(); + let decoded = image::load_from_memory(bytes) + .map_err(|e| tracing::debug!("undecodable image: {e}")) + .ok()?; + + let (w, h) = decoded.dimensions(); + if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX { + tracing::debug!(w, h, "skipping decorative image"); + return None; + } + + let has_alpha = decoded.color().has_alpha(); + let flattened = if has_alpha { + flatten_to_white(&decoded) + } else { + decoded + }; + + let resized = if w > profile.max_width || h > profile.max_height { + flattened.resize( + profile.max_width, + profile.max_height, + image::imageops::FilterType::Lanczos3, + ) + } else { + flattened + }; + + // Keep line art (PNG source, few distinct tones) lossless; everything else + // becomes JPEG, which is far smaller for photographs (§3.10). + let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized); + + let mut out = Cursor::new(Vec::new()); + // NB: encode the concrete buffer, not the `DynamicImage` — the latter always + // reports RGBA pixels, which would silently re-colorize a grayscale image. + if profile.grayscale { + let gray = resized.to_luma8(); + if keep_png { + DynamicImage::ImageLuma8(gray) + .write_to(&mut out, ImageFormat::Png) + .ok()?; + return Some((out.into_inner(), "image/png")); + } + let mut enc = + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality); + enc.encode_image(&gray).ok()?; + return Some((out.into_inner(), "image/jpeg")); + } + + let rgb = resized.to_rgb8(); + if keep_png { + DynamicImage::ImageRgb8(rgb) + .write_to(&mut out, ImageFormat::Png) + .ok()?; + return Some((out.into_inner(), "image/png")); + } + let mut enc = + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality); + enc.encode_image(&rgb).ok()?; + Some((out.into_inner(), "image/jpeg")) +} + +/// True when `bytes` are an SVG document (possibly behind an XML prolog or BOM). +fn looks_like_svg(bytes: &[u8]) -> bool { + let head = &bytes[..bytes.len().min(1024)]; + let text = String::from_utf8_lossy(head); + let text = text.trim_start_matches('\u{feff}').trim_start(); + text.starts_with("<svg") + || (text.starts_with("<?xml") || text.starts_with("<!DOCTYPE svg")) && text.contains("<svg") +} + +/// Rasterize an SVG to a PNG at the profile's target width (§3.10). +/// +/// The profile's own resize pass then handles the height cap, so this only has +/// to land in the right ballpark. +fn rasterize_svg(bytes: &[u8], profile: ImageProfile) -> Option<Vec<u8>> { + let mut options = resvg::usvg::Options::default(); + options.fontdb_mut().load_system_fonts(); + let tree = resvg::usvg::Tree::from_data(bytes, &options) + .map_err(|e| tracing::debug!("svg did not parse: {e}")) + .ok()?; + + // An `<svg>` that parses but draws nothing is not an image, it is a stray + // tag: rasterizing it would embed a blank rectangle. + if tree.root().children().is_empty() { + tracing::debug!("svg has nothing to draw"); + return None; + } + let size = tree.size(); + let (sw, sh) = (size.width(), size.height()); + if !(sw.is_finite() && sh.is_finite()) || sw <= 0.0 || sh <= 0.0 { + return None; + } + // Judge "decorative" by the declared size, before scaling: a 16×16 icon is + // an icon however large we choose to draw it. + if sw < MIN_DIMENSION_PX as f32 || sh < MIN_DIMENSION_PX as f32 { + tracing::debug!(sw, sh, "skipping decorative svg"); + return None; + } + // Vector art has no native resolution, so render straight at the edition's + // target width — upscaling a rasterized copy afterwards would only blur it. + let target_w = profile + .max_width + .max(SVG_FALLBACK_SIZE.min(profile.max_width)); + let scale = (target_w as f32 / sw).min(profile.max_height as f32 / sh); + let scale = if scale.is_finite() && scale > 0.0 { + scale + } else { + 1.0 + }; + let (w, h) = ( + (sw * scale).round().max(1.0) as u32, + (sh * scale).round().max(1.0) as u32, + ); + let mut pixmap = tiny_skia::Pixmap::new(w, h)?; + // E-ink has no transparency; render onto white so alpha never becomes black. + pixmap.fill(tiny_skia::Color::WHITE); + resvg::render( + &tree, + tiny_skia::Transform::from_scale(scale, scale), + &mut pixmap.as_mut(), + ); + let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())?; + let mut png = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(rgba) + .write_to(&mut png, ImageFormat::Png) + .ok()?; + Some(png.into_inner()) +} + +/// Composite over an opaque white page — e-ink has no transparency (§3.10). +fn flatten_to_white(img: &DynamicImage) -> DynamicImage { + let rgba = img.to_rgba8(); + let mut rgb = image::RgbImage::new(rgba.width(), rgba.height()); + for (x, y, px) in rgba.enumerate_pixels() { + let a = f32::from(px[3]) / 255.0; + let blend = |c: u8| { + ((f32::from(c) * a) + 255.0 * (1.0 - a)) + .round() + .clamp(0.0, 255.0) as u8 + }; + rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])])); + } + DynamicImage::ImageRgb8(rgb) +} + +/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text). +fn is_line_art(img: &DynamicImage) -> bool { + const SAMPLE_LIMIT: usize = 20_000; + const DISTINCT_LIMIT: usize = 64; + let rgb = img.to_rgb8(); + let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1); + for (i, px) in rgb.pixels().enumerate() { + if i >= SAMPLE_LIMIT { + break; + } + let c = [px[0], px[1], px[2]]; + if !distinct.contains(&c) { + distinct.push(c); + if distinct.len() > DISTINCT_LIMIT { + return false; + } + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reencode_resizes_grayscales_and_encodes() { + let mut img = image::RgbaImage::new(200, 100); + for (x, y, px) in img.enumerate_pixels_mut() { + *px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]); + } + let mut png = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(img) + .write_to(&mut png, ImageFormat::Png) + .unwrap(); + let raw = png.into_inner(); + + let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap(); + assert_eq!(std_mime, "image/jpeg"); + let decoded = image::load_from_memory(&std_bytes).unwrap(); + assert_eq!(decoded.dimensions(), (200, 100), "no upscaling"); + + let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap(); + let x4 = image::load_from_memory(&x4_bytes).unwrap(); + assert!(x4.width() <= 480 && x4.height() <= 800); + assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8"); + } + + #[test] + fn reencode_skips_decorative_images_and_junk() { + let tiny = image::RgbaImage::new(8, 8); + let mut png = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(tiny) + .write_to(&mut png, ImageFormat::Png) + .unwrap(); + assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none()); + // A bare `<svg>` tag with nothing to draw is markup, not a picture. + assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none()); + assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none()); + } + + #[test] + fn svg_charts_are_rasterized_rather_than_dropped() { + let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300"> + <rect x="10" y="10" width="380" height="280" fill="#3355bb"/> + <circle cx="200" cy="150" r="60" fill="#ffcc00"/> + </svg>"##; + let (bytes, mime) = reencode(svg, ImageProfile::STANDARD).expect("svg rasterizes"); + assert_eq!(mime, "image/png", "flat colour art stays lossless"); + let decoded = image::load_from_memory(&bytes).expect("decodable output"); + // Drawn at the edition's target width, not at the SVG's nominal size. + assert_eq!(decoded.dimensions(), (1200, 900)); + assert!(!decoded.color().has_alpha(), "rendered onto white"); + + // An XML prolog and a leading BOM must not hide the format. + let with_prolog = format!( + "\u{feff}<?xml version=\"1.0\"?>{}", + String::from_utf8_lossy(svg) + ); + let (x4, _) = reencode(with_prolog.as_bytes(), ImageProfile::X4).expect("x4 rasterizes"); + let x4 = image::load_from_memory(&x4).unwrap(); + assert!(x4.width() <= 480 && x4.height() <= 800); + + // A 16×16 icon is decorative however large we could draw it. + let icon = br#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"> + <rect width="16" height="16"/></svg>"#; + assert!(reencode(icon, ImageProfile::STANDARD).is_none()); + } + + #[test] + fn line_art_png_stays_png_and_is_flattened() { + let mut img = image::RgbaImage::new(120, 60); + for (x, _y, px) in img.enumerate_pixels_mut() { + *px = if x % 12 == 0 { + image::Rgba([0, 0, 0, 255]) + } else { + image::Rgba([255, 255, 255, 0]) + }; + } + let mut png = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(img) + .write_to(&mut png, ImageFormat::Png) + .unwrap(); + let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap(); + assert_eq!(mime, "image/png"); + let decoded = image::load_from_memory(&bytes).unwrap(); + assert!(!decoded.color().has_alpha(), "transparency is flattened"); + // Transparent pixels became white. + assert_eq!( + decoded.to_rgb8().get_pixel(1, 1), + &image::Rgb([255, 255, 255]) + ); + } +} diff --git a/src/images/fetch.rs b/src/images/fetch.rs new file mode 100644 index 0000000..ef7681b --- /dev/null +++ b/src/images/fetch.rs @@ -0,0 +1,159 @@ +//! Downloading an issue's images and packing them into EPUB assets. + +use std::time::Duration; + +use futures::StreamExt; + +use crate::types::{Edition, ImageAsset, Pick}; + +use super::encode::{ImageProfile, reencode}; +use super::refs::{ImgRef, extract_img_refs}; + +/// Per-image download timeout (§3.10). +pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10; +/// Per-image size cap (§3.10). +pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; +/// Concurrent downloads (§3.10). +pub const CONCURRENCY: usize = 8; +/// Whole-issue asset budget (§3.10). +pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024; + +/// Download one image, honoring the timeout and size cap (§3.10). +pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> { + let resp = http + .get(url) + // Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10). + .header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8") + .timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS)) + .send() + .await + .map_err(|e| tracing::debug!(url, "image download failed: {e}")) + .ok()?; + if !resp.status().is_success() { + tracing::debug!(url, status = %resp.status(), "image download rejected"); + return None; + } + if let Some(len) = resp.content_length() + && len as usize > MAX_IMAGE_BYTES + { + tracing::debug!(url, len, "image exceeds the size cap"); + return None; + } + let mut resp = resp; + let mut buf: Vec<u8> = Vec::new(); + loop { + match resp.chunk().await { + Ok(Some(chunk)) => { + if buf.len() + chunk.len() > MAX_IMAGE_BYTES { + tracing::debug!(url, "image exceeds the size cap mid-stream"); + return None; + } + buf.extend_from_slice(&chunk); + } + Ok(None) => break, + Err(e) => { + tracing::debug!(url, "image download interrupted: {e}"); + return None; + } + } + } + if buf.is_empty() { None } else { Some(buf) } +} + +/// Everything needed to fetch one image, in deterministic issue order. +#[derive(Debug, Clone)] +struct PendingImage { + id: String, + url: String, + alt: String, + caption: Option<String>, +} + +fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> { + let entry_id = pick.article.best_entry_id; + let mut refs = extract_img_refs(&pick.article.content_html); + if refs.is_empty() { + refs = pick + .article + .image_urls + .iter() + .map(|u| ImgRef { + src: u.clone(), + alt: String::new(), + caption: None, + }) + .collect(); + } + refs.into_iter() + .filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://")) + .enumerate() + .map(|(i, r)| PendingImage { + id: format!("img-{entry_id}-{i}"), + url: r.src, + alt: r.alt, + caption: r.caption, + }) + .collect() +} + +/// Download and re-encode every image referenced by the lineup for one edition, +/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10). +pub async fn collect_for_issue( + http: &reqwest::Client, + picks: &[Pick], + edition: Edition, +) -> Vec<ImageAsset> { + let profile = ImageProfile::for_edition(edition); + let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect(); + if pending.is_empty() { + return Vec::new(); + } + tracing::info!(count = pending.len(), ?edition, "downloading issue images"); + + let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> = + futures::stream::iter(pending.into_iter().map(|p| { + let http = http.clone(); + async move { + let raw = download(&http, &p.url).await?; + let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile)) + .await + .ok() + .flatten()?; + Some((p, bytes, mime)) + } + })) + .buffered(CONCURRENCY) + .collect() + .await; + + let mut assets = Vec::new(); + let mut budget_used = 0usize; + let mut skipped = 0usize; + for result in results.into_iter().flatten() { + let (pending, bytes, mime) = result; + if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES { + skipped += 1; + continue; + } + budget_used += bytes.len(); + let ext = if mime == "image/png" { "png" } else { "jpg" }; + assets.push(ImageAsset { + href: format!("images/{}.{ext}", pending.id), + id: pending.id, + mime: mime.to_string(), + data: bytes, + alt: pending.alt, + caption: pending.caption, + source_url: pending.url, + }); + } + if skipped > 0 { + tracing::warn!(skipped, budget_used, "issue image budget exhausted"); + } + tracing::info!( + embedded = assets.len(), + bytes = budget_used, + "issue images ready" + ); + assets +} diff --git a/src/images/mod.rs b/src/images/mod.rs new file mode 100644 index 0000000..e3656c0 --- /dev/null +++ b/src/images/mod.rs @@ -0,0 +1,25 @@ +//! Article images, end to end (spec §3.3 extraction, §3.10 "Images"). +//! +//! The stages run in this order, and each submodule owns one of them: +//! +//! | stage | module | when | +//! |---|---|---| +//! | make a page's `<img>` elements usable | [`normalize`] | extraction, before readability | +//! | find what an article references | [`refs`] | extraction and issue build | +//! | download and re-encode per edition | [`fetch`], [`encode`] | issue build | +//! | point the markup at the embedded files | [`embed`] | chapter rendering | +//! +//! Nothing here ever fails a run: an image that cannot be fetched, decoded or +//! resolved is dropped, and the article is rendered without it (notes §3). + +pub mod embed; +pub mod encode; +pub mod fetch; +pub mod normalize; +pub mod refs; + +pub use embed::rewrite_img_srcs; +pub use encode::{ImageProfile, MIN_DIMENSION_PX, reencode}; +pub use fetch::{ISSUE_ASSET_BUDGET_BYTES, collect_for_issue, download}; +pub use normalize::{normalize_img_tags, prepare_for_readability, unwrap_image_wrappers}; +pub use refs::{ImgRef, collect_image_urls, extract_img_refs}; diff --git a/src/images/normalize.rs b/src/images/normalize.rs new file mode 100644 index 0000000..687d8fd --- /dev/null +++ b/src/images/normalize.rs @@ -0,0 +1,516 @@ +//! Making a page's `<img>` elements usable before anything else touches them. +//! +//! Publishers ship images in a dozen incompatible shapes: lazy placeholders, +//! `srcset` lists, `<picture>` sources, URLs parked in `data-*` attributes, and +//! `src` values that are not URLs at all (unfilled templates, JSON blobs, whole +//! `srcset` strings). On top of that, readability actively damages images while +//! it works — it deletes `<button>` subtrees, taking lightbox images with them, +//! and its own lazy-image heuristic overwrites a working `src` with any +//! attribute whose value happens to contain `.jpg`. +//! +//! [`prepare_for_readability`] runs before readability and leaves every `<img>` +//! as a plain `src`/`alt`/`title` triple, which both fixes the input and denies +//! readability the raw material for its substitution. [`normalize_img_tags`] is +//! also applied to feed content, which carries the same markup. +//! +//! Candidates are judged by *shape*, never by publisher: a string with braces, +//! whitespace or quotes in it cannot resolve, whoever wrote it. + +use crate::html::{html_to_text, parse_attrs, tag_end, tag_name}; + +/// Elements that readability deletes outright, and which a page may nevertheless +/// have wrapped around an image (lightbox triggers, mostly). +const IMAGE_WRAPPER_TAGS: &[&str] = &["button", "form", "fieldset", "object"]; + +/// Attributes lazy-loading libraries use for the real image URL. +/// +/// These outrank `src`, because a page only sets them when `src` is a stand-in: +/// a transparent GIF, a blurred thumbnail, an inline SVG spacer. Attributes that +/// merely *look* image-ish (`data-template`, `data-attrs`, `data-orig-file`) are +/// deliberately absent — those hold templates and metadata, and preferring them +/// is exactly the mistake readability's own heuristic makes. +const LAZY_SRC_ATTRS: &[&str] = &[ + "data-src", + "data-lazy-src", + "data-original", + "data-runner-src", + "data-full-src", + "data-hi-res-src", + "data-image-src", +]; + +/// Widest `srcset` candidate we will pick; above this we are downloading pixels +/// the re-encoder immediately throws away. +const MAX_SRCSET_WIDTH: u32 = 2000; + +/// Make a fetched page safe to hand to readability (§3.3). +/// +/// Two passes, both about images: unwrap the elements that would take an image +/// with them when readability deletes them, then reduce every `<img>` to a plain +/// `src`/`alt`/`title` triple. The second pass is what stops readability's own +/// lazy-image heuristic from replacing a working `src` — with no `srcset`, +/// `loading` or `data-*` attributes left on the element, it has nothing to +/// substitute and leaves the image alone. +pub fn prepare_for_readability(html: &str) -> String { + normalize_img_tags(&unwrap_image_wrappers(html)) +} + +/// Replace image-only `<button>`/`<form>`/`<fieldset>`/`<object>` wrappers with +/// their contents (§3.3). +/// +/// A `<button>` holding nothing but an image is a lightbox trigger, not a +/// control: the image is the content. Wrappers that also carry text are left +/// alone, because those really are interface. +pub fn unwrap_image_wrappers(html: &str) -> String { + let mut out = String::with_capacity(html.len()); + let mut cursor = 0usize; + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + out.push_str(&html[cursor..start]); + let Some(end) = tag_end(html, start) else { + out.push_str(&html[start..]); + return out; + }; + let raw = &html[start..end]; + let inner = raw.trim_start_matches('<').trim_end_matches('>'); + let name = tag_name(inner); + + if IMAGE_WRAPPER_TAGS.contains(&name.as_str()) + && !inner.trim_end().ends_with('/') + && let Some((content, after)) = element_content(html, end, &name) + && content.contains("<img") + && html_to_text(content).trim().is_empty() + { + out.push_str(&unwrap_image_wrappers(content)); + cursor = after; + continue; + } + + out.push_str(raw); + cursor = end; + } + out.push_str(&html[cursor..]); + out +} + +/// The content of an element whose open tag ended at `body_start`, plus the +/// offset just past its close tag. `None` when the element is never closed. +fn element_content<'a>(html: &'a str, body_start: usize, name: &str) -> Option<(&'a str, usize)> { + let open = format!("<{name}"); + let close = format!("</{name}"); + let mut depth = 1usize; + let mut cursor = body_start; + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + let end = tag_end(html, start)?; + let tag = &html[start..end]; + let lower = tag.to_ascii_lowercase(); + if lower.starts_with(&close) { + depth -= 1; + if depth == 0 { + return Some((&html[body_start..start], end)); + } + } else if lower.starts_with(&open) + && !lower[open.len()..].starts_with(|c: char| c.is_alphanumeric() || c == '-') + && !tag.trim_end().ends_with("/>") + { + depth += 1; + } + cursor = end; + } + None +} + +/// Reduce every `<img>` to `<img src alt title>` with a usable URL (§3.3). +/// +/// The `src` a page ships is not automatically the one to use: it can be a lazy +/// placeholder (`data:image/svg+xml,…`), an unresolved template +/// (`…/resize/{width}/…`), a JSON blob a framework parked there, or an entire +/// `srcset` string. Candidates are tried in order and the first plausible one +/// wins; an image with no plausible candidate is dropped, because a broken +/// `<img>` only becomes clutter further down the pipeline. +pub fn normalize_img_tags(html: &str) -> String { + let mut out = String::with_capacity(html.len()); + let mut cursor = 0usize; + // `<picture>` puts the real candidates on sibling `<source>` elements. + let mut picture_srcset: Option<String> = None; + + while let Some(rel) = html[cursor..].find('<') { + let start = cursor + rel; + out.push_str(&html[cursor..start]); + let Some(end) = tag_end(html, start) else { + out.push_str(&html[start..]); + return out; + }; + let raw = &html[start..end]; + let inner = raw + .trim_start_matches('<') + .trim_end_matches('>') + .trim_end_matches('/'); + let name = tag_name(inner); + + match name.as_str() { + "picture" => { + picture_srcset = None; + out.push_str(raw); + } + "source" => { + let attrs = parse_attrs(inner); + if picture_srcset.is_none() + && let Some(set) = + attr(&attrs, "srcset").or_else(|| attr(&attrs, "data-srcset")) + { + picture_srcset = Some(set.to_string()); + } + out.push_str(raw); + } + "img" => { + let attrs = parse_attrs(inner); + if let Some(src) = best_img_src(&attrs, picture_srcset.as_deref()) { + out.push_str("<img src=\""); + out.push_str(&escape_attr(&src)); + out.push('"'); + for key in ["alt", "title"] { + if let Some(v) = attr(&attrs, key) { + out.push(' '); + out.push_str(key); + out.push_str("=\""); + out.push_str(&escape_attr(v)); + out.push('"'); + } + } + out.push_str("/>"); + } else { + tracing::debug!(tag = %&raw[..raw.len().min(120)], "dropping unusable img"); + } + } + _ => out.push_str(raw), + } + if name == "picture" && inner.starts_with('/') { + picture_srcset = None; + } + cursor = end; + } + out.push_str(&html[cursor..]); + out +} + +fn attr<'a>(attrs: &'a [(String, String)], name: &str) -> Option<&'a str> { + attrs + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.trim()) + .filter(|v| !v.is_empty()) +} + +/// Pick the best URL for one `<img>` from everything the element carries. +fn best_img_src(attrs: &[(String, String)], picture_srcset: Option<&str>) -> Option<String> { + for key in LAZY_SRC_ATTRS { + if let Some(v) = attr(attrs, key).filter(|s| plausible_url(s)) { + return Some(v.to_string()); + } + } + if let Some(src) = attr(attrs, "src").filter(|s| plausible_url(s)) { + return Some(src.to_string()); + } + if let Some(from_set) = attr(attrs, "srcset").and_then(best_from_srcset) { + return Some(from_set); + } + if let Some(from_set) = attr(attrs, "data-srcset").and_then(best_from_srcset) { + return Some(from_set); + } + picture_srcset.and_then(best_from_srcset) +} + +/// The widest candidate in a `srcset` that is still worth downloading. +/// +/// Parsed by whitespace rather than by comma: the URLs of several image CDNs +/// contain commas of their own, and splitting on those shreds them. +fn best_from_srcset(srcset: &str) -> Option<String> { + let mut best: Option<(u32, String)> = None; + let mut smallest: Option<(u32, String)> = None; + let mut pending: Option<String> = None; + + let mut consider = |url: String, width: u32| { + if width <= MAX_SRCSET_WIDTH && best.as_ref().is_none_or(|(w, _)| width > *w) { + best = Some((width, url.clone())); + } + if smallest.as_ref().is_none_or(|(w, _)| width < *w) { + smallest = Some((width, url)); + } + }; + + for token in srcset.split_whitespace() { + let token = token.trim_end_matches(','); + if token.is_empty() { + continue; + } + match parse_descriptor(token) { + Some(width) => { + if let Some(url) = pending.take() { + consider(url, width); + } + } + None => { + // A URL with no descriptor of its own still counts, at width 1x. + if let Some(url) = pending.replace(token.to_string()) { + consider(url, 1); + } + } + } + } + if let Some(url) = pending.take() { + consider(url, 1); + } + + best.or(smallest) + .map(|(_, url)| url) + .filter(|u| plausible_url(u)) +} + +/// `800w` → 800, `2x` → a synthetic width so density candidates sort sensibly. +fn parse_descriptor(token: &str) -> Option<u32> { + let (value, unit) = token.split_at(token.len().checked_sub(1)?); + match unit { + "w" => value.parse::<u32>().ok(), + "x" => value + .parse::<f32>() + .ok() + .map(|d| (d * 1000.0).round().clamp(1.0, f32::from(u16::MAX)) as u32), + _ => None, + } +} + +/// Whether a string can serve as an image URL at all. +/// +/// This is deliberately about shape, not about the host: whitespace, braces and +/// quotes mean we are looking at a `srcset` blob, an unfilled URL template or a +/// serialized object, none of which will ever resolve. +fn plausible_url(candidate: &str) -> bool { + let candidate = candidate.trim(); + if candidate.is_empty() || candidate.len() > 2048 { + return false; + } + if candidate.starts_with("data:") || candidate.starts_with("about:") { + return false; + } + if candidate + .chars() + .any(|c| c.is_whitespace() || matches!(c, '{' | '}' | '"' | '\'' | '<' | '>' | '\\')) + { + return false; + } + // `%20` is a space that survived encoding — same blob, different spelling. + !candidate.contains("%20") +} + +fn escape_attr(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +#[cfg(test)] +mod tests { + use super::*; + use scraper::{Html, Selector}; + + /// The `src` values that survive normalization, read back with a real parser. + fn src_of(html: &str) -> Vec<String> { + Html::parse_fragment(html) + .select(&Selector::parse("img").unwrap()) + .filter_map(|e| e.value().attr("src").map(str::to_string)) + .collect() + } + + #[test] + fn lazy_placeholder_src_gives_way_to_the_real_url() { + // IEEE Spectrum: an inline SVG spacer with the URL parked on data-runner-src. + let html = r#"<img alt="A well head" lazy-loadable="true" + src="data:image/svg+xml,%3Csvg%20xmlns=%27http://www.w3.org/2000/svg%27%3E%3C/svg%3E" + data-runner-src="https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"/>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"] + ); + } + + #[test] + fn unresolved_url_templates_fall_through_to_a_real_candidate() { + // NPR: readability copies data-template over a perfectly good src. + let html = r#"<img alt="Meghan Cliffel" + src="https://npr.brightspotcdn.com/resize/{width}/quality/{quality}/x.jpg" + srcset="https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg 1100w"/>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg"] + ); + // With nothing usable anywhere, the image goes rather than becoming a + // request for a picture that says "Image". + let only_template = + r#"<img alt="x" src="https://cdn.dev/resize/{width}/quality/{quality}/x.jpg"/>"#; + assert!(src_of(&normalize_img_tags(only_template)).is_empty()); + } + + #[test] + fn a_srcset_blob_parked_in_src_is_rejected_and_reparsed() { + // dfarq: readability copies the entire srcset string into src. + let blob = "https://i0.wp.com/x.jpg?resize=300%2C158&ssl=1 300w, \ + https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1 1024w, \ + https://i0.wp.com/x.jpg?w=3000&ssl=1 3000w"; + let html = format!(r#"<img alt="printer" src="{blob}" srcset="{blob}"/>"#); + // The widest candidate under the download ceiling wins; 3000w does not. + assert_eq!( + src_of(&normalize_img_tags(&html)), + ["https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1"] + ); + } + + #[test] + fn a_json_blob_parked_in_src_is_rejected() { + // Substack: readability copies data-attrs (JSON) over src. + let html = r#"<img alt="" src="{"src":"https://s3.dev/a.jpeg","width":1000}" + srcset="https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg 1456w"/>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg"] + ); + } + + #[test] + fn picture_sources_back_up_an_empty_img() { + let html = r#"<picture> + <source srcset="https://cdn.dev/a.avif 800w" type="image/avif"/> + <source srcset="https://cdn.dev/a.webp 800w" type="image/webp"/> + <img alt="A photo"/> + </picture>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://cdn.dev/a.avif"] + ); + } + + #[test] + fn a_good_src_is_never_traded_for_a_metadata_attribute() { + // `data-template`, `data-attrs` and friends hold templates and JSON, not + // URLs — trading a working src for one of those is the original sin. + let html = r#"<img alt="A photo" loading="lazy" class="lazyload" + src="https://cdn.dev/real.jpg" + data-template="https://cdn.dev/{width}/real.jpg" + data-orig-file="https://cdn.dev/orig.jpg"/>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://cdn.dev/real.jpg"] + ); + } + + #[test] + fn a_lazy_loader_placeholder_loses_to_its_data_src() { + // iRunFar (a3-lazy-load): src is a shared 1×1 spacer that would download + // and re-encode perfectly happily, and be the wrong picture. + let html = r#"<img class="lazy lazy-hidden" alt="Brooks Cascadia 20" + src="//www.irunfar.com/wp-content/plugins/a3-lazy-load/assets/images/lazy_placeholder.gif" + data-src="https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"/>"#; + assert_eq!( + src_of(&normalize_img_tags(html)), + ["https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"] + ); + } + + #[test] + fn normalization_keeps_alt_and_title_and_drops_the_rest() { + let html = r#"<img src="/a.png" alt="An & alt" title="T" class="x" width="900" + onerror="evil()" srcset="/b.png 2x"/>"#; + let out = normalize_img_tags(html); + assert!(out.contains(r#"alt="An & alt""#), "{out}"); + assert!(out.contains(r#"title="T""#), "{out}"); + for gone in ["class=", "width=", "onerror", "srcset="] { + assert!( + !out.contains(gone), + "expected {gone} to be dropped from {out}" + ); + } + // Relative URLs survive: sanitize_with_base absolutizes them later. + assert_eq!(src_of(&out), ["/a.png"]); + } + + #[test] + fn lightbox_buttons_no_longer_take_their_image_with_them() { + // Nautilus: readability deletes <button> and everything inside it. + let html = r#"<figure class="wp-block-image"> + <button type="button" aria-label="Enlarge image"> + <img class="wp-image-1" src="https://cdn.dev/flower.png?w=710" alt=""/> + </button> + <figcaption>BEAUTIFUL DANGER: a belladonna flower.</figcaption> + </figure>"#; + let out = unwrap_image_wrappers(html); + assert!(!out.contains("<button"), "{out}"); + assert!(!out.contains("</button>"), "{out}"); + assert!(out.contains("flower.png"), "{out}"); + assert!(out.contains("BEAUTIFUL DANGER"), "caption survives: {out}"); + } + + #[test] + fn buttons_that_are_really_buttons_are_left_alone() { + let html = r#"<button class="subscribe">Subscribe <img src="/icon.png" alt=""/></button>"#; + assert_eq!(unwrap_image_wrappers(html), html); + // And an image-free control is untouched too. + let plain = r#"<button>Share</button>"#; + assert_eq!(unwrap_image_wrappers(plain), plain); + } + + #[test] + fn nested_and_unclosed_wrappers_do_not_derail_the_scan() { + let nested = r#"<button><button><img src="/a.png"/></button></button><p>after</p>"#; + let out = unwrap_image_wrappers(nested); + assert!(!out.contains("button"), "{out}"); + assert!(out.contains("/a.png") && out.contains("after"), "{out}"); + // An open tag that never closes is passed through rather than eating + // the rest of the document. + let unclosed = r#"<button><img src="/a.png"/><p>rest</p>"#; + assert!(unwrap_image_wrappers(unclosed).contains("rest")); + } + + #[test] + fn srcset_descriptors_pick_the_widest_usable_candidate() { + assert_eq!( + best_from_srcset("/a.png 150w, /b.png 800w, /c.png 4000w").as_deref(), + Some("/b.png") + ); + // Density descriptors work as an ordering too. + assert_eq!( + best_from_srcset("/a.png 1x, /b.png 2x").as_deref(), + Some("/b.png") + ); + // A bare URL with no descriptor is still a candidate. + assert_eq!(best_from_srcset("/only.png").as_deref(), Some("/only.png")); + // Every candidate too wide: take the narrowest rather than nothing. + assert_eq!( + best_from_srcset("/big.png 3000w, /huge.png 5000w").as_deref(), + Some("/big.png") + ); + assert_eq!(best_from_srcset(" ").as_deref(), None); + } + + #[test] + fn plausibility_is_about_shape_not_host() { + for good in [ + "https://cdn.dev/a.jpg?id=1&width=980", + "/_next/image?url=https%3A%2F%2Fx.dev%2Fa.png&w=3840&q=75", + "https://substackcdn.com/image/fetch/$s_!y1,w_1456/a.jpeg", + ] { + assert!(plausible_url(good), "{good} should be usable"); + } + for bad in [ + "", + "data:image/svg+xml,%3Csvg%3E%3C/svg%3E", + "https://cdn.dev/resize/{width}/a.jpg", + r#"{"src":"https://cdn.dev/a.jpg"}"#, + "https://cdn.dev/a.jpg 300w, https://cdn.dev/b.jpg 600w", + "https://cdn.dev/a.jpg%20300w", + ] { + assert!(!plausible_url(bad), "{bad} should be rejected"); + } + } +} diff --git a/src/images/refs.rs b/src/images/refs.rs new file mode 100644 index 0000000..7e36c4f --- /dev/null +++ b/src/images/refs.rs @@ -0,0 +1,156 @@ +//! Finding the images an article references. +//! +//! The input here is already-normalized markup (see [`super::normalize`]), so +//! every `<img>` is expected to carry a plain, usable `src`. + +use std::collections::HashSet; + +use scraper::{Html, Selector}; +use url::Url; + +/// One `<img>` found in article markup, with the caption of its `<figure>` if any. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImgRef { + pub src: String, + pub alt: String, + pub caption: Option<String>, +} + +/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup. +pub fn extract_img_refs(html: &str) -> Vec<ImgRef> { + let doc = scraper::Html::parse_fragment(html); + let Ok(img_sel) = scraper::Selector::parse("img") else { + return Vec::new(); + }; + let cap_sel = scraper::Selector::parse("figcaption").ok(); + + let mut out = Vec::new(); + let mut seen: Vec<String> = Vec::new(); + for el in doc.select(&img_sel) { + let Some(src) = el.value().attr("src") else { + continue; + }; + let src = src.trim(); + if src.is_empty() || src.starts_with("data:") { + continue; + } + if seen.iter().any(|s| s == src) { + continue; + } + seen.push(src.to_string()); + let alt = el + .value() + .attr("alt") + .unwrap_or_default() + .trim() + .to_string(); + // Walk up to an enclosing <figure> and take its caption, if any. + let mut caption = None; + if let Some(cap_sel) = &cap_sel { + let mut cursor = el.parent(); + while let Some(node) = cursor { + if let Some(elem) = scraper::ElementRef::wrap(node) { + if elem.value().name() == "figure" { + caption = elem.select(cap_sel).next().map(|c| { + c.text() + .collect::<String>() + .split_whitespace() + .collect::<Vec<_>>() + .join(" ") + }); + break; + } + cursor = elem.parent(); + } else { + break; + } + } + } + out.push(ImgRef { + src: src.to_string(), + alt, + caption: caption.filter(|c| !c.is_empty()), + }); + } + out +} + +/// Absolute image URLs referenced by `html`, resolved against `base_url` (§3.3). +/// +/// Every image an article carries is kept: a photo essay with thirty pictures is +/// a photo essay, and the issue-wide byte budget is the real backstop. +pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> { + let Ok(selector) = Selector::parse("img") else { + return Vec::new(); + }; + let base = Url::parse(base_url).ok(); + let document = Html::parse_fragment(html); + let mut seen: HashSet<String> = HashSet::new(); + let mut out: Vec<String> = Vec::new(); + for element in document.select(&selector) { + let raw = element + .value() + .attr("src") + .or_else(|| element.value().attr("data-src")) + .map(str::trim) + .filter(|s| !s.is_empty()); + let Some(raw) = raw else { continue }; + let resolved = match Url::parse(raw) { + Ok(u) => Some(u), + Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()), + }; + let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else { + continue; + }; + let url = url.to_string(); + if seen.insert(url.clone()) { + out.push(url); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn image_collection_resolves_and_keeps_every_image() { + let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#); + html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#); + for i in 0..20 { + html.push_str(&format!(r#"<img src="/n{i}.png">"#)); + } + let urls = collect_image_urls(&html, "https://blog.dev/posts/one"); + // Two named images, the data-src one, and all twenty of the rest: no cap. + assert_eq!(urls.len(), 23); + assert_eq!(urls[0], "https://blog.dev/a.png"); + assert_eq!(urls[1], "https://cdn.dev/b.png"); + assert_eq!(urls[2], "https://blog.dev/posts/c.png"); + // Duplicates and data: URIs never appear. + assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1); + assert!(!urls.iter().any(|u| u.starts_with("data:"))); + assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty()); + } + + // ----------------------------------------------------------------------- + // Image normalization — each case is a page shape seen in a real issue. + // ----------------------------------------------------------------------- + + #[test] + fn extracts_img_refs_with_captions() { + let html = r#"<p>hi</p> + <figure><img src="https://e.g/a.png" alt="A diagram"/> + <figcaption>Figure 1: the thing</figcaption></figure> + <img src="https://e.g/b.jpg"/> + <img src="data:image/png;base64,zz"/> + <img src="https://e.g/a.png" alt="dupe"/>"#; + let refs = extract_img_refs(html); + assert_eq!(refs.len(), 2); + assert_eq!(refs[0].src, "https://e.g/a.png"); + assert_eq!(refs[0].alt, "A diagram"); + assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing")); + assert_eq!(refs[1].src, "https://e.g/b.jpg"); + assert!(refs[1].caption.is_none()); + } +} diff --git a/src/lib.rs b/src/lib.rs index c74f837..5fd4c4a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,9 @@ pub mod db; pub mod dedupe; pub mod epub; pub mod extract; +pub mod html; pub mod http; +pub mod images; pub mod miniflux; pub mod pipeline; pub mod publish; diff --git a/src/world.rs b/src/world.rs index a72121e..fa87175 100644 --- a/src/world.rs +++ b/src/world.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use url::Url; use crate::curate::llm::{LlmClient, LlmError}; -use crate::epub::images::text_escape; +use crate::html::text_escape; use crate::types::{WorldBriefing, WorldBriefingSection, WorldEvent}; type NodeRef<'a> = <scraper::ElementRef<'a> as std::ops::Deref>::Target;