Break up the image and EPUB god files
The image fixes left two problems of navigation. Image handling was
spread across `extract.rs` (300 lines of normalization) and
`epub/images.rs` (download, re-encode, markup rewriting, plus generic
HTML helpers that comments, world and x4 were all reaching into a
module named "images" to borrow). And `epub/build.rs` had grown to
1,148 lines of code covering cover rendering, ten askama templates,
every chapter renderer and the zip assembly.
New homes:
- `src/html.rs` — markup helpers that do not care what the markup is
about: tag scanning, attribute parsing, entity decoding, escaping,
XHTML fixups, reading a fragment as text. Previously scattered between
`epub/images.rs` and `extract.rs`.
- `src/images/` — one module per stage of an article's images, in the
order they run: `normalize` (make `<img>` usable, pre-readability),
`refs` (what an article references), `fetch` + `encode` (download and
re-encode per edition), `embed` (point the markup at what shipped).
- `src/epub/{cover,chapters,build}.rs` — the cover, the chapter
renderers, and the ordering plus assembly that puts them together.
`epub/fixtures.rs` takes the shared test issue, which was a public
module wedged inside `build.rs`.
- `src/curate/profile/themes.rs` — a 260-line keyword table that sat in
the middle of the profile logic.
`curate::html_to_text` is renamed `prompt_text`: it is a different
function from `html::html_to_text` (collapses whitespace, no DOM, sized
for prompt budgets) and sharing a name with it was a trap.
Largest module drops from 1,148 code lines to 828, and no file mixes
two subjects. Behaviour is unchanged: 231 lib tests plus 25 integration
tests green, and the real-world audit over issues 1–3 still reports 214
images referenced, 214 shown, 0 placeholders, 0 orphaned assets.
`image_audit` gains `--epub-out DIR`, which writes a readable EPUB of
the audited articles so images can be checked on a device.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
//! Downloading an issue's images and packing them into EPUB assets.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
|
||||
use crate::types::{Edition, ImageAsset, Pick};
|
||||
|
||||
use super::encode::{ImageProfile, reencode};
|
||||
use super::refs::{ImgRef, extract_img_refs};
|
||||
|
||||
/// Per-image download timeout (§3.10).
|
||||
pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
|
||||
/// Per-image size cap (§3.10).
|
||||
pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
|
||||
/// Concurrent downloads (§3.10).
|
||||
pub const CONCURRENCY: usize = 8;
|
||||
/// Whole-issue asset budget (§3.10).
|
||||
pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
|
||||
|
||||
/// Download one image, honoring the timeout and size cap (§3.10).
|
||||
pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
||||
let resp = http
|
||||
.get(url)
|
||||
// Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10).
|
||||
.header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8")
|
||||
.timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::debug!(url, "image download failed: {e}"))
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
tracing::debug!(url, status = %resp.status(), "image download rejected");
|
||||
return None;
|
||||
}
|
||||
if let Some(len) = resp.content_length()
|
||||
&& len as usize > MAX_IMAGE_BYTES
|
||||
{
|
||||
tracing::debug!(url, len, "image exceeds the size cap");
|
||||
return None;
|
||||
}
|
||||
let mut resp = resp;
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
loop {
|
||||
match resp.chunk().await {
|
||||
Ok(Some(chunk)) => {
|
||||
if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
|
||||
tracing::debug!(url, "image exceeds the size cap mid-stream");
|
||||
return None;
|
||||
}
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::debug!(url, "image download interrupted: {e}");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf.is_empty() { None } else { Some(buf) }
|
||||
}
|
||||
|
||||
/// Everything needed to fetch one image, in deterministic issue order.
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingImage {
|
||||
id: String,
|
||||
url: String,
|
||||
alt: String,
|
||||
caption: Option<String>,
|
||||
}
|
||||
|
||||
fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> {
|
||||
let entry_id = pick.article.best_entry_id;
|
||||
let mut refs = extract_img_refs(&pick.article.content_html);
|
||||
if refs.is_empty() {
|
||||
refs = pick
|
||||
.article
|
||||
.image_urls
|
||||
.iter()
|
||||
.map(|u| ImgRef {
|
||||
src: u.clone(),
|
||||
alt: String::new(),
|
||||
caption: None,
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
refs.into_iter()
|
||||
.filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://"))
|
||||
.enumerate()
|
||||
.map(|(i, r)| PendingImage {
|
||||
id: format!("img-{entry_id}-{i}"),
|
||||
url: r.src,
|
||||
alt: r.alt,
|
||||
caption: r.caption,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Download and re-encode every image referenced by the lineup for one edition,
|
||||
/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10).
|
||||
pub async fn collect_for_issue(
|
||||
http: &reqwest::Client,
|
||||
picks: &[Pick],
|
||||
edition: Edition,
|
||||
) -> Vec<ImageAsset> {
|
||||
let profile = ImageProfile::for_edition(edition);
|
||||
let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect();
|
||||
if pending.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
tracing::info!(count = pending.len(), ?edition, "downloading issue images");
|
||||
|
||||
let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> =
|
||||
futures::stream::iter(pending.into_iter().map(|p| {
|
||||
let http = http.clone();
|
||||
async move {
|
||||
let raw = download(&http, &p.url).await?;
|
||||
let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile))
|
||||
.await
|
||||
.ok()
|
||||
.flatten()?;
|
||||
Some((p, bytes, mime))
|
||||
}
|
||||
}))
|
||||
.buffered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut assets = Vec::new();
|
||||
let mut budget_used = 0usize;
|
||||
let mut skipped = 0usize;
|
||||
for result in results.into_iter().flatten() {
|
||||
let (pending, bytes, mime) = result;
|
||||
if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
budget_used += bytes.len();
|
||||
let ext = if mime == "image/png" { "png" } else { "jpg" };
|
||||
assets.push(ImageAsset {
|
||||
href: format!("images/{}.{ext}", pending.id),
|
||||
id: pending.id,
|
||||
mime: mime.to_string(),
|
||||
data: bytes,
|
||||
alt: pending.alt,
|
||||
caption: pending.caption,
|
||||
source_url: pending.url,
|
||||
});
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, budget_used, "issue image budget exhausted");
|
||||
}
|
||||
tracing::info!(
|
||||
embedded = assets.len(),
|
||||
bytes = budget_used,
|
||||
"issue images ready"
|
||||
);
|
||||
assets
|
||||
}
|
||||
Reference in New Issue
Block a user