thallada/the-daily-epub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break up the image and EPUB god files

Browse Source 
The image fixes left two problems of navigation. Image handling was
spread across `extract.rs` (300 lines of normalization) and
`epub/images.rs` (download, re-encode, markup rewriting, plus generic
HTML helpers that comments, world and x4 were all reaching into a
module named "images" to borrow). And `epub/build.rs` had grown to
1,148 lines of code covering cover rendering, ten askama templates,
every chapter renderer and the zip assembly.

New homes:

- `src/html.rs` — markup helpers that do not care what the markup is
  about: tag scanning, attribute parsing, entity decoding, escaping,
  XHTML fixups, reading a fragment as text. Previously scattered between
  `epub/images.rs` and `extract.rs`.
- `src/images/` — one module per stage of an article's images, in the
  order they run: `normalize` (make `<img>` usable, pre-readability),
  `refs` (what an article references), `fetch` + `encode` (download and
  re-encode per edition), `embed` (point the markup at what shipped).
- `src/epub/{cover,chapters,build}.rs` — the cover, the chapter
  renderers, and the ordering plus assembly that puts them together.
  `epub/fixtures.rs` takes the shared test issue, which was a public
  module wedged inside `build.rs`.
- `src/curate/profile/themes.rs` — a 260-line keyword table that sat in
  the middle of the profile logic.

`curate::html_to_text` is renamed `prompt_text`: it is a different
function from `html::html_to_text` (collapses whitespace, no DOM, sized
for prompt budgets) and sharing a name with it was a trap.

Largest module drops from 1,148 code lines to 828, and no file mixes
two subjects. Behaviour is unchanged: 231 lib tests plus 25 integration
tests green, and the real-world audit over issues 1–3 still reports 214
images referenced, 214 shown, 0 placeholders, 0 orphaned assets.

`image_audit` gains `--epub-out DIR`, which writes a readable EPUB of
the audited articles so images can be checked on a device.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
thallada
2026-08-18 03:35:21 +00:00
co-authored by Claude Opus 5
parent 254eaeb713
commit db19d08257
27 changed files with 3513 additions and 3132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/curate/editorial.rs
+3 -3
View File
@@ -14,7 +14,7 @@ use std::fmt::Write as _;
use serde::{Deserialize, Serialize};
use super::llm::{LlmClient, LlmError};
use super::{escape_html, html_to_text, text_to_paragraphs, truncate_tokens, truncate_words};
use super::{escape_html, prompt_text, text_to_paragraphs, truncate_tokens, truncate_words};
use crate::types::{ArticleId, Editorial, Lineup, Pick};
/// Article text is truncated to roughly this many tokens per summary call (§3.6).
@@ -116,7 +116,7 @@ pub async fn summarize_article(
temperature: f32,
) -> Result<String, LlmError> {
llm.meter.check_budget()?;
let body = truncate_tokens(&html_to_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET);
let body = truncate_tokens(&prompt_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET);
let mut prompt = String::with_capacity(body.len() + SUMMARY_INSTRUCTIONS.len() + 256);
prompt.push_str(SUMMARY_INSTRUCTIONS);
let _ = write!(
@@ -309,7 +309,7 @@ fn social_note(pick: &Pick) -> String {
/// The article's own opening words, used when no LLM summary exists (§3.6).
pub fn excerpt_summary(pick: &Pick) -> String {
let text = truncate_words(
&html_to_text(&pick.article.content_html),
&prompt_text(&pick.article.content_html),
FALLBACK_SUMMARY_WORDS,
);
if text.is_empty() {
src/curate/mod.rs
+9 -5
View File
@@ -164,7 +164,11 @@ pub fn approx_tokens(text: &str) -> usize {
/// Strip markup and collapse whitespace, so article bodies can go into prompts
/// as plain text (cheaper and less confusing for the model than raw HTML).
pub fn html_to_text(html: &str) -> String {
///
/// Deliberately not [`crate::html::html_to_text`]: this one collapses runs of
/// whitespace and never builds a DOM, because it runs over every candidate
/// body on every run and only has to be good enough to size a prompt.
pub fn prompt_text(html: &str) -> String {
/// Does `tail` open the named element, i.e. `<name` or `</name`?
fn opens(tail: &str, name: &str) -> bool {
let bytes = tail.as_bytes();
@@ -272,11 +276,11 @@ mod tests {
fn html_becomes_readable_text() {
let html = "<h1>Title</h1><p>First &amp; best.</p><script>alert('x')</script>\
<p>Second<br/>line</p><style>p{color:red}</style>";
assert_eq!(html_to_text(html), "Title First & best. Second line");
assert_eq!(html_to_text(""), "");
assert_eq!(html_to_text("no markup at all"), "no markup at all");
assert_eq!(prompt_text(html), "Title First & best. Second line");
assert_eq!(prompt_text(""), "");
assert_eq!(prompt_text("no markup at all"), "no markup at all");
// Unicode survives byte-wise walking.
assert_eq!(html_to_text("<p>café — naïve</p>"), "café — naïve");
assert_eq!(prompt_text("<p>café — naïve</p>"), "café — naïve");
}
#[test]
src/curate/profile.rs → src/curate/profile/mod.rs
+2 -317
View File
@@ -99,303 +99,9 @@ fn xml_unescape(s: &str) -> String {
.replace("&amp;", "&")
}
// ---------------------------------------------------------------------------
// Theming (§3.6a)
// ---------------------------------------------------------------------------
pub mod themes;
/// Theme name → lowercase keywords, in priority order. The first theme whose
/// keyword appears in the interest name wins, so the table is ordered from the
/// most specific bucket to the most general.
const THEMES: &[(&str, &[&str])] = &[
(
"Systems & languages",
&[
"rust",
"zig",
"lua",
"assembly",
"compiler",
"systems programming",
"concurrency",
"async",
"memory",
"simd",
"zero-copy",
"parser",
"fuzz",
"static analysis",
"functional programming",
"data structures",
"algorithm",
"performance",
"profil",
"python",
"typescript",
"node.js",
"wasm",
"webassembly",
"reactive programming",
"lsp",
],
),
(
"Databases & data",
&[
"database",
"sql",
"postgres",
"query",
"vector databases",
"bloom",
"compression",
"data engineering",
"knowledge graph",
"message queue",
"distributed systems",
"crdt",
"microservices",
"system design",
],
),
(
"AI & machine learning",
&[
"ai",
"llm",
"machine learning",
"nlp",
"rag",
"prompt",
"agent",
"claude",
"eval",
],
),
(
"Web, infra & devtools",
&[
"web",
"css",
"htmx",
"svelte",
"react",
"pwa",
"api",
"developer",
"devops",
"docker",
"git",
"observability",
"cloud",
"cdn",
"edge",
"networking",
"mesh",
"ipfs",
"activitypub",
"atproto",
"search engines",
"tauri",
"design systems",
"cybersecurity",
"cryptography",
"monitoring",
"spatial computing",
"webrtc",
"webgpu",
"webgl",
"webcodecs",
"android",
"mobile",
"code generation",
],
),
(
"Self-hosting, RSS & the indie web",
&[
"self-host",
"self host",
"homelab",
"rss",
"feed reader",
"indie web",
"personal website",
"personal wiki",
"digital garden",
"static site",
"static sites",
"personal archiving",
"offline-first",
"privacy",
"open source",
"licensing",
"side projects",
"digital nomad",
"remote living",
"off-grid",
"quantified self",
"cloudflare workers",
],
),
(
"E-ink, terminals & hardware",
&[
"e-ink",
"eink",
"writerdeck",
"tmux",
"neovim",
"terminal",
"text editor",
"editor",
"window manager",
"keyboard",
"hardware",
"electronics",
"calibre",
"ghostty",
"retro computing",
"low-tech",
"manufacturing",
"typography",
"linux",
],
),
(
"Games & interactive fiction",
&[
"game",
"gaming",
"minecraft",
"mud",
"interactive fiction",
"inform7",
"bevy",
"sim racing",
"modding",
],
),
(
"Creative coding & digital art",
&[
"generative",
"creative coding",
"creative automation",
"glitch",
"procedural",
"digital art",
"gaussian splatting",
"cellular automata",
"sonification",
"code visualization",
"photo",
"photography",
"agent-based",
],
),
(
"Writing, books & PKM",
&[
"writing",
"write",
"reading",
"book",
"essay",
"poetry",
"literature",
"note-taking",
"journaling",
"pkm",
"knowledge management",
"obsidian",
"markdown",
"commonplace",
"longform",
"long-form",
"fiction",
"worldbuilding",
"sci-fi",
"science fiction",
"speculative",
"podcast",
"narrative",
"choice architecture",
],
),
(
"Science, space & nature",
&[
"space",
"aerospace",
"aviation",
"science",
"neuroscience",
"bioinformatics",
"nature",
"cognitive",
"social science",
],
),
(
"History, policy & culture",
&[
"history",
"policy",
"culture",
"lgbt",
"queer",
"startups",
"apple",
"boston",
"criticism",
"internet history",
"computing history",
],
),
(
"Outdoors, coffee & everyday life",
&[
"hiking",
"camping",
"running",
"trail",
"coffee",
"cats",
"films",
"music",
"engineering",
],
),
];
/// Fallback bucket for interests no keyword claims.
const OTHER_THEME: &str = "Other standing interests";
/// Group ~220 interests into readable themes for the prompt (§3.6).
///
/// Deterministic: theme order follows [`THEMES`], members are sorted
/// case-insensitively, and empty themes are omitted.
pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> {
let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1];
for interest in interests {
let lower = interest.to_lowercase();
let idx = THEMES
.iter()
.position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k)))
.unwrap_or(THEMES.len());
buckets[idx].push(interest.clone());
}
let mut out = Vec::new();
for (idx, mut members) in buckets.into_iter().enumerate() {
if members.is_empty() {
continue;
}
members.sort_by_key(|m| (m.to_lowercase(), m.clone()));
let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME);
out.push((name.to_string(), members));
}
out
}
pub use themes::group_into_themes;
// ---------------------------------------------------------------------------
// Document assembly (§3.6)
@@ -870,27 +576,6 @@ mod tests {
);
}
#[test]
fn theming_is_deterministic_and_total() {
let list = interests();
let a = group_into_themes(&list);
let b = group_into_themes(&list);
assert_eq!(a, b);
let grouped: usize = a.iter().map(|(_, m)| m.len()).sum();
assert_eq!(
grouped,
list.len(),
"every interest lands in exactly one theme"
);
assert!(a.len() >= 6, "expected several themes, got {}", a.len());
// Members are sorted within a theme.
for (_, members) in &a {
let mut sorted = members.clone();
sorted.sort_by_key(|m| (m.to_lowercase(), m.clone()));
assert_eq!(members, &sorted);
}
}
#[test]
fn profile_document_is_deterministic_and_complete() {
let list = interests();
src/curate/profile/themes.rs
+354
View File
@@ -0,0 +1,354 @@
//! Grouping the reader's ~220 Scour interests into readable themes (§3.6a).
//!
//! A lookup table and one function over it. It lives in its own file because the
//! table is long and almost never changes, while the profile logic around it
//! does — and a 260-line const in the middle of that logic is a wall to scroll
//! past, not something to read.
/// Theme name → lowercase keywords, in priority order. The first theme whose
/// keyword appears in the interest name wins, so the table is ordered from the
/// most specific bucket to the most general.
const THEMES: &[(&str, &[&str])] = &[
(
"Systems & languages",
&[
"rust",
"zig",
"lua",
"assembly",
"compiler",
"systems programming",
"concurrency",
"async",
"memory",
"simd",
"zero-copy",
"parser",
"fuzz",
"static analysis",
"functional programming",
"data structures",
"algorithm",
"performance",
"profil",
"python",
"typescript",
"node.js",
"wasm",
"webassembly",
"reactive programming",
"lsp",
],
),
(
"Databases & data",
&[
"database",
"sql",
"postgres",
"query",
"vector databases",
"bloom",
"compression",
"data engineering",
"knowledge graph",
"message queue",
"distributed systems",
"crdt",
"microservices",
"system design",
],
),
(
"AI & machine learning",
&[
"ai",
"llm",
"machine learning",
"nlp",
"rag",
"prompt",
"agent",
"claude",
"eval",
],
),
(
"Web, infra & devtools",
&[
"web",
"css",
"htmx",
"svelte",
"react",
"pwa",
"api",
"developer",
"devops",
"docker",
"git",
"observability",
"cloud",
"cdn",
"edge",
"networking",
"mesh",
"ipfs",
"activitypub",
"atproto",
"search engines",
"tauri",
"design systems",
"cybersecurity",
"cryptography",
"monitoring",
"spatial computing",
"webrtc",
"webgpu",
"webgl",
"webcodecs",
"android",
"mobile",
"code generation",
],
),
(
"Self-hosting, RSS & the indie web",
&[
"self-host",
"self host",
"homelab",
"rss",
"feed reader",
"indie web",
"personal website",
"personal wiki",
"digital garden",
"static site",
"static sites",
"personal archiving",
"offline-first",
"privacy",
"open source",
"licensing",
"side projects",
"digital nomad",
"remote living",
"off-grid",
"quantified self",
"cloudflare workers",
],
),
(
"E-ink, terminals & hardware",
&[
"e-ink",
"eink",
"writerdeck",
"tmux",
"neovim",
"terminal",
"text editor",
"editor",
"window manager",
"keyboard",
"hardware",
"electronics",
"calibre",
"ghostty",
"retro computing",
"low-tech",
"manufacturing",
"typography",
"linux",
],
),
(
"Games & interactive fiction",
&[
"game",
"gaming",
"minecraft",
"mud",
"interactive fiction",
"inform7",
"bevy",
"sim racing",
"modding",
],
),
(
"Creative coding & digital art",
&[
"generative",
"creative coding",
"creative automation",
"glitch",
"procedural",
"digital art",
"gaussian splatting",
"cellular automata",
"sonification",
"code visualization",
"photo",
"photography",
"agent-based",
],
),
(
"Writing, books & PKM",
&[
"writing",
"write",
"reading",
"book",
"essay",
"poetry",
"literature",
"note-taking",
"journaling",
"pkm",
"knowledge management",
"obsidian",
"markdown",
"commonplace",
"longform",
"long-form",
"fiction",
"worldbuilding",
"sci-fi",
"science fiction",
"speculative",
"podcast",
"narrative",
"choice architecture",
],
),
(
"Science, space & nature",
&[
"space",
"aerospace",
"aviation",
"science",
"neuroscience",
"bioinformatics",
"nature",
"cognitive",
"social science",
],
),
(
"History, policy & culture",
&[
"history",
"policy",
"culture",
"lgbt",
"queer",
"startups",
"apple",
"boston",
"criticism",
"internet history",
"computing history",
],
),
(
"Outdoors, coffee & everyday life",
&[
"hiking",
"camping",
"running",
"trail",
"coffee",
"cats",
"films",
"music",
"engineering",
],
),
];
/// Fallback bucket for interests no keyword claims.
const OTHER_THEME: &str = "Other standing interests";
/// Group ~220 interests into readable themes for the prompt (§3.6).
///
/// Deterministic: theme order follows [`THEMES`], members are sorted
/// case-insensitively, and empty themes are omitted.
pub fn group_into_themes(interests: &[String]) -> Vec<(String, Vec<String>)> {
let mut buckets: Vec<Vec<String>> = vec![Vec::new(); THEMES.len() + 1];
for interest in interests {
let lower = interest.to_lowercase();
let idx = THEMES
.iter()
.position(|(_, keywords)| keywords.iter().any(|k| lower.contains(k)))
.unwrap_or(THEMES.len());
buckets[idx].push(interest.clone());
}
let mut out = Vec::new();
for (idx, mut members) in buckets.into_iter().enumerate() {
if members.is_empty() {
continue;
}
members.sort_by_key(|m| (m.to_lowercase(), m.clone()));
let name = THEMES.get(idx).map(|(n, _)| *n).unwrap_or(OTHER_THEME);
out.push((name.to_string(), members));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn interests() -> Vec<String> {
[
"Rust",
"Zig",
"Postgres",
"SQLite",
"Kubernetes",
"E-ink",
"Keyboards",
"Retrocomputing",
"Bicycles",
"Trail running",
"Coffee",
"Bread baking",
"Cartography",
"Typography",
"Ambient music",
"Science fiction",
"Local politics",
"Boston",
"Machine learning",
"Self-hosting",
]
.iter()
.map(|s| s.to_string())
.collect()
}
#[test]
fn theming_is_deterministic_and_total() {
let list = interests();
let a = group_into_themes(&list);
let b = group_into_themes(&list);
assert_eq!(a, b);
let grouped: usize = a.iter().map(|(_, m)| m.len()).sum();
assert_eq!(
grouped,
list.len(),
"every interest lands in exactly one theme"
);
assert!(a.len() >= 6, "expected several themes, got {}", a.len());
// Members are sorted within a theme.
for (_, members) in &a {
let mut sorted = members.clone();
sorted.sort_by_key(|m| (m.to_lowercase(), m.clone()));
assert_eq!(members, &sorted);
}
}
}
src/curate/score.rs
+2 -2
View File
@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::llm::{LlmClient, LlmError, strip_code_fence};
use super::{html_to_text, truncate_words};
use super::{prompt_text, truncate_words};
use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind};
/// Words of article text sent per candidate in stage A (§3.6).
@@ -155,7 +155,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String {
);
let _ = writeln!(block, "social: {}", social_line(candidate));
let _ = writeln!(block, "came via: {}", sources_line(candidate));
let excerpt = truncate_words(&html_to_text(&a.content_html), EXCERPT_WORDS);
let excerpt = truncate_words(&prompt_text(&a.content_html), EXCERPT_WORDS);
let _ = writeln!(
block,
"excerpt: {}",
src/curate/select.rs
+2 -2
View File
@@ -16,7 +16,7 @@ use jiff::civil::Date;
use serde::{Deserialize, Serialize};
use super::llm::{LlmClient, LlmError, strip_code_fence};
use super::{html_to_text, truncate_words};
use super::{prompt_text, truncate_words};
use crate::types::{ArticleId, Lineup, Pick, ScoredArticle, SourceKind, WORLD_BRIEFING_SECTION};
/// How many candidates are offered to stage B (§3.6).
@@ -154,7 +154,7 @@ fn render_candidate(candidate: &ScoredArticle) -> String {
""
}
);
let blurb = truncate_words(&html_to_text(&a.content_html), BLURB_WORDS);
let blurb = truncate_words(&prompt_text(&a.content_html), BLURB_WORDS);
if !blurb.is_empty() {
let _ = writeln!(block, "opening: {blurb}");
}