thallada/the-daily-epub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break up the image and EPUB god files

Browse Source 
The image fixes left two problems of navigation. Image handling was
spread across `extract.rs` (300 lines of normalization) and
`epub/images.rs` (download, re-encode, markup rewriting, plus generic
HTML helpers that comments, world and x4 were all reaching into a
module named "images" to borrow). And `epub/build.rs` had grown to
1,148 lines of code covering cover rendering, ten askama templates,
every chapter renderer and the zip assembly.

New homes:

- `src/html.rs` — markup helpers that do not care what the markup is
  about: tag scanning, attribute parsing, entity decoding, escaping,
  XHTML fixups, reading a fragment as text. Previously scattered between
  `epub/images.rs` and `extract.rs`.
- `src/images/` — one module per stage of an article's images, in the
  order they run: `normalize` (make `<img>` usable, pre-readability),
  `refs` (what an article references), `fetch` + `encode` (download and
  re-encode per edition), `embed` (point the markup at what shipped).
- `src/epub/{cover,chapters,build}.rs` — the cover, the chapter
  renderers, and the ordering plus assembly that puts them together.
  `epub/fixtures.rs` takes the shared test issue, which was a public
  module wedged inside `build.rs`.
- `src/curate/profile/themes.rs` — a 260-line keyword table that sat in
  the middle of the profile logic.

`curate::html_to_text` is renamed `prompt_text`: it is a different
function from `html::html_to_text` (collapses whitespace, no DOM, sized
for prompt budgets) and sharing a name with it was a trap.

Largest module drops from 1,148 code lines to 828, and no file mixes
two subjects. Behaviour is unchanged: 231 lib tests plus 25 integration
tests green, and the real-world audit over issues 1–3 still reports 214
images referenced, 214 shown, 0 placeholders, 0 orphaned assets.

`image_audit` gains `--epub-out DIR`, which writes a readable EPUB of
the audited articles so images can be checked on a device.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
thallada
2026-08-18 03:35:21 +00:00
co-authored by Claude Opus 5
parent 254eaeb713
commit db19d08257
27 changed files with 3513 additions and 3132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/image_audit.rs
+85 -9
View File
@@ -20,9 +20,14 @@ use std::collections::BTreeMap;
use std::io::Read;
use std::path::{Path, PathBuf};
use daily_epub::epub::{build, images};
use daily_epub::epub::build;
use daily_epub::extract::{self, Extractor};
use daily_epub::types::{Article, EntryId, ExtractMethod, ImageAsset, Pick, SourceKind, SourceRef};
use daily_epub::html;
use daily_epub::images;
use daily_epub::types::{
Article, Colophon, Editorial, EntryId, ExtractMethod, ImageAsset, Issue, IssueMeta, Lineup,
Pick, SourceKind, SourceRef,
};
/// One article we are auditing, pulled back out of a published EPUB.
#[derive(Debug, Clone)]
@@ -60,17 +65,23 @@ async fn main() {
let mut cache = PathBuf::from("/tmp/daily-epub-audit-cache");
let mut dump: Option<String> = None;
let mut epub_out: Option<PathBuf> = None;
let mut epubs: Vec<PathBuf> = Vec::new();
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--cache" => cache = PathBuf::from(args.next().expect("--cache needs a path")),
"--dump" => dump = Some(args.next().expect("--dump needs a title substring")),
"--epub-out" => {
epub_out = Some(PathBuf::from(args.next().expect("--epub-out needs a path")))
}
other => epubs.push(PathBuf::from(other)),
}
}
if epubs.is_empty() {
eprintln!("usage: image_audit [--cache DIR] <issue.epub>...");
eprintln!(
"usage: image_audit [--cache DIR] [--dump TITLE] [--epub-out DIR] <issue.epub>..."
);
std::process::exit(2);
}
std::fs::create_dir_all(&cache).expect("cache dir");
@@ -212,6 +223,10 @@ async fn main() {
for (kind, n) in &kinds {
println!(" {n:>4} {kind}");
}
if let Some(dir) = &epub_out {
write_audit_epub(dir, &picks, &assets);
}
if !skipped.is_empty() {
println!("\n== unfetchable ({})", skipped.len());
for (t, e) in &skipped {
@@ -286,7 +301,7 @@ async fn body_for(
let _ = extractor;
let readable = extract::readability(&html, &base).map_err(|e| e.to_string())?;
Ok(extract::sanitize_with_base(
&extract::normalize_img_tags(&readable),
&images::normalize_img_tags(&readable),
&base,
))
}
@@ -320,8 +335,8 @@ async fn raw_fetch(url: &str) -> Result<(Vec<u8>, String), String> {
}
fn pick_for(target: &Target, content_html: String) -> Pick {
let word_count = extract::word_count(&content_html);
let image_urls = extract::collect_image_urls(&content_html, &target.url);
let word_count = html::word_count(&content_html);
let image_urls = images::collect_image_urls(&content_html, &target.url);
Pick {
article: Article {
id: target.entry_id,
@@ -336,7 +351,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
sources: vec![SourceRef {
entry_id: target.entry_id,
feed_id: 1,
feed_title: "Feed".into(),
feed_title: target.issue.clone(),
category: None,
kind: SourceKind::Feed,
}],
@@ -344,14 +359,14 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
url: target.url.clone(),
author: None,
feed_id: 1,
feed_title: "Feed".into(),
feed_title: host(&target.url),
category: None,
published_at: None,
comments_url: None,
social: vec![],
extract_method: ExtractMethod::Readability,
},
section: "Audit".into(),
section: target.issue.clone(),
position: 0,
is_lead: false,
summary: None,
@@ -360,6 +375,67 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
}
}
/// Build a readable EPUB out of the audited articles (`--epub-out DIR`).
///
/// Not a regenerated issue — there is no editorial, no discussion chapters and
/// no world briefing here, and the real thing needs the database. It exists so
/// the images can be looked at on a device rather than counted in a table.
fn write_audit_epub(out_dir: &Path, picks: &[Pick], assets: &[ImageAsset]) {
let sections: Vec<String> = {
let mut seen: Vec<String> = Vec::new();
for p in picks {
if !seen.contains(&p.section) {
seen.push(p.section.clone());
}
}
seen
};
let issue = Issue {
meta: IssueMeta {
date: "2026-08-16".parse().unwrap(),
issue_number: 0,
generated_at: jiff::Timestamp::now(),
display_date: "Image audit rebuild".into(),
article_count: picks.len() as i64,
section_count: sections.len() as i64,
total_words: picks.iter().map(|p| p.article.word_count).sum(),
reading_minutes: picks.iter().map(|p| p.article.reading_minutes()).sum(),
},
lineup: Lineup {
date: "2026-08-16".parse().unwrap(),
picks: picks.to_vec(),
section_order: sections,
},
editorial: Editorial {
front_page_html: "<p>Rebuilt from published issues by \
<code>examples/image_audit.rs</code> to check image handling. \
Articles are re-extracted live; editorial, discussions and the \
world briefing are absent by design.</p>"
.into(),
section_intros: Default::default(),
summaries: Default::default(),
},
world_briefing: None,
colophon: Colophon::default(),
};
let cfg = daily_epub::config::Config::default();
match daily_epub::epub::build_edition_with_images(
&issue,
daily_epub::types::Edition::Standard,
&cfg,
out_dir,
assets,
) {
Ok(artifact) => println!(
"\nwrote {} ({:.1} MB, {} images)",
artifact.path.display(),
artifact.bytes as f64 / 1_048_576.0,
assets.len()
),
Err(e) => eprintln!("epub build failed: {e}"),
}
}
/// Pull `(entry id, title, url)` out of every article chapter in an EPUB.
fn targets_from_epub(path: &Path) -> Vec<Target> {
let issue = path