Break up the image and EPUB god files
The image fixes left two problems of navigation. Image handling was
spread across `extract.rs` (300 lines of normalization) and
`epub/images.rs` (download, re-encode, markup rewriting, plus generic
HTML helpers that comments, world and x4 were all reaching into a
module named "images" to borrow). And `epub/build.rs` had grown to
1,148 lines of code covering cover rendering, ten askama templates,
every chapter renderer and the zip assembly.
New homes:
- `src/html.rs` — markup helpers that do not care what the markup is
about: tag scanning, attribute parsing, entity decoding, escaping,
XHTML fixups, reading a fragment as text. Previously scattered between
`epub/images.rs` and `extract.rs`.
- `src/images/` — one module per stage of an article's images, in the
order they run: `normalize` (make `<img>` usable, pre-readability),
`refs` (what an article references), `fetch` + `encode` (download and
re-encode per edition), `embed` (point the markup at what shipped).
- `src/epub/{cover,chapters,build}.rs` — the cover, the chapter
renderers, and the ordering plus assembly that puts them together.
`epub/fixtures.rs` takes the shared test issue, which was a public
module wedged inside `build.rs`.
- `src/curate/profile/themes.rs` — a 260-line keyword table that sat in
the middle of the profile logic.
`curate::html_to_text` is renamed `prompt_text`: it is a different
function from `html::html_to_text` (collapses whitespace, no DOM, sized
for prompt budgets) and sharing a name with it was a trap.
Largest module drops from 1,148 code lines to 828, and no file mixes
two subjects. Behaviour is unchanged: 231 lib tests plus 25 integration
tests green, and the real-world audit over issues 1–3 still reports 214
images referenced, 214 shown, 0 placeholders, 0 orphaned assets.
`image_audit` gains `--epub-out DIR`, which writes a readable EPUB of
the audited articles so images can be checked on a device.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,193 @@
|
||||
//! Pointing article markup at the images actually embedded in the EPUB.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::html::{attr_escape, parse_attrs, tag_end, tag_name, text_escape};
|
||||
use crate::types::ImageAsset;
|
||||
|
||||
/// Whether an image we could not embed is worth telling the reader about.
|
||||
///
|
||||
/// Only descriptive alt text qualifies. A filename, a bare label like `red line`
|
||||
/// on a divider rule, or no alt at all carries nothing the reader loses by not
|
||||
/// seeing the picture — announcing those turns every decorative graphic and
|
||||
/// dead link into a line of clutter, which is how the placeholders got out of
|
||||
/// hand in the first place.
|
||||
fn alt_is_worth_announcing(alt: &str) -> bool {
|
||||
const MIN_DESCRIPTIVE_WORDS: usize = 4;
|
||||
!alt.is_empty()
|
||||
&& !is_filename_alt(alt)
|
||||
&& alt.split_whitespace().count() >= MIN_DESCRIPTIVE_WORDS
|
||||
}
|
||||
|
||||
/// True for alt text that is really just the uploaded filename — `IMG_0808.JPG`,
|
||||
/// `cut pieces v01.JPG`, `chart-final-2.png`.
|
||||
fn is_filename_alt(alt: &str) -> bool {
|
||||
let alt = alt.trim();
|
||||
if alt.contains(' ') && alt.split_whitespace().count() > 4 {
|
||||
return false;
|
||||
}
|
||||
let Some((stem, ext)) = alt.rsplit_once('.') else {
|
||||
return false;
|
||||
};
|
||||
let ext = ext.to_ascii_lowercase();
|
||||
matches!(
|
||||
ext.as_str(),
|
||||
"jpg" | "jpeg" | "png" | "gif" | "webp" | "svg" | "avif" | "bmp" | "heic"
|
||||
) && !stem.is_empty()
|
||||
&& stem
|
||||
.chars()
|
||||
.all(|c| c.is_alphanumeric() || matches!(c, ' ' | '_' | '-' | '.'))
|
||||
}
|
||||
|
||||
/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the
|
||||
/// `[image: alt]` placeholder paragraph (§3.10).
|
||||
pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String {
|
||||
let by_url: HashMap<&str, &ImageAsset> =
|
||||
assets.iter().map(|a| (a.source_url.as_str(), a)).collect();
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
let inner = raw
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>')
|
||||
.trim_end_matches('/');
|
||||
if tag_name(inner) == "img" {
|
||||
let attrs = parse_attrs(inner);
|
||||
let src = attrs
|
||||
.iter()
|
||||
.find(|(k, _)| k == "src")
|
||||
.map(|(_, v)| v.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
let alt = attrs
|
||||
.iter()
|
||||
.find(|(k, _)| k == "alt")
|
||||
.map(|(_, v)| v.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
match by_url.get(src.as_str()) {
|
||||
Some(asset) => {
|
||||
let alt = if alt.is_empty() { &asset.alt } else { &alt };
|
||||
out.push_str(&format!(
|
||||
"<img src=\"{}\" alt=\"{}\"/>",
|
||||
attr_escape(&asset.href),
|
||||
attr_escape(alt)
|
||||
));
|
||||
}
|
||||
// An image we could not embed is only worth announcing when its
|
||||
// alt text tells the reader something; otherwise the `<img>`
|
||||
// just goes away. That covers the decorative graphics the
|
||||
// re-encoder deliberately skips as well as genuine misses (§3.10).
|
||||
None if !alt_is_worth_announcing(&alt) => {}
|
||||
None => {
|
||||
out.push_str(&format!(
|
||||
"<p class=\"image-placeholder\">[image: {}]</p>",
|
||||
text_escape(&alt)
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.push_str(raw);
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn asset(url: &str, href: &str) -> ImageAsset {
|
||||
ImageAsset {
|
||||
id: "img-1-0".into(),
|
||||
href: href.into(),
|
||||
mime: "image/jpeg".into(),
|
||||
data: vec![1, 2, 3],
|
||||
alt: "fallback alt".into(),
|
||||
caption: None,
|
||||
source_url: url.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_hits_and_placeholders_misses() {
|
||||
let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")];
|
||||
let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="A chart of missing things">"#;
|
||||
let out = rewrite_img_srcs(html, &assets);
|
||||
// The alt round-trips through one level of escaping, not two.
|
||||
assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt & more"/>"#));
|
||||
assert!(
|
||||
out.contains(r#"<p class="image-placeholder">[image: A chart of missing things]</p>"#)
|
||||
);
|
||||
assert!(!out.contains("gone.png"));
|
||||
}
|
||||
|
||||
/// The whole point of the fix: ammonia writes `&` into the markup, and
|
||||
/// the asset was keyed on the URL a real parser produced.
|
||||
#[test]
|
||||
fn entity_encoded_urls_still_match_their_asset() {
|
||||
let assets = vec![asset(
|
||||
"https://e.g/a.jpg?id=1&width=980",
|
||||
"images/img-1-0.jpg",
|
||||
)];
|
||||
let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
|
||||
assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
|
||||
// The numeric spelling WordPress emits works too.
|
||||
let html = r#"<img src="https://e.g/a.jpg?id=1&width=980" alt="Chart"/>"#;
|
||||
assert!(rewrite_img_srcs(html, &assets).contains(r#"src="images/img-1-0.jpg""#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unembeddable_images_only_speak_up_when_the_alt_says_something() {
|
||||
// No alt at all: the image simply disappears.
|
||||
assert_eq!(
|
||||
rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]),
|
||||
""
|
||||
);
|
||||
// A filename is not a description.
|
||||
assert_eq!(
|
||||
rewrite_img_srcs(r#"<img src="https://e.g/x.png" alt="IMG_0808.JPG">"#, &[]),
|
||||
""
|
||||
);
|
||||
// Neither is the label on a decorative divider rule.
|
||||
assert_eq!(
|
||||
rewrite_img_srcs(r#"<img src="https://e.g/rule.png" alt="red line">"#, &[]),
|
||||
""
|
||||
);
|
||||
// A real description is worth keeping.
|
||||
assert!(
|
||||
rewrite_img_srcs(
|
||||
r#"<img src="https://e.g/x.png" alt="A man in a hard hat stands over a well hole">"#,
|
||||
&[]
|
||||
)
|
||||
.contains("[image: A man in a hard hat stands over a well hole]")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filename_alt_detection() {
|
||||
for yes in [
|
||||
"IMG_0808.JPG",
|
||||
"cut pieces v01.JPG",
|
||||
"chart-final-2.png",
|
||||
"diagram.svg",
|
||||
] {
|
||||
assert!(is_filename_alt(yes), "{yes} should read as a filename");
|
||||
}
|
||||
for no in [
|
||||
"A hydrogen well head",
|
||||
"",
|
||||
"Fig. 3",
|
||||
"The lion-man of Hohlenstein-Stadel, carved from mammoth ivory.",
|
||||
] {
|
||||
assert!(!is_filename_alt(no), "{no} should read as a description");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
//! Turning downloaded bytes into something an e-reader can display.
|
||||
//!
|
||||
//! Every image is decoded, fitted to the edition's profile, flattened onto white
|
||||
//! (e-ink has no transparency) and re-encoded. SVG is rasterized on the way in,
|
||||
//! because charts and diagrams are frequently vector-only.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use image::{DynamicImage, GenericImageView, ImageFormat};
|
||||
|
||||
use crate::types::Edition;
|
||||
|
||||
/// Images smaller than this in either dimension are decorative — skipped (§3.10).
|
||||
pub const MIN_DIMENSION_PX: u32 = 24;
|
||||
/// Width an SVG is rendered at when the profile asks for less than this.
|
||||
const SVG_FALLBACK_SIZE: u32 = 1000;
|
||||
|
||||
/// Per-edition re-encoding parameters (§3.10).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ImageProfile {
|
||||
pub max_width: u32,
|
||||
pub max_height: u32,
|
||||
pub jpeg_quality: u8,
|
||||
pub grayscale: bool,
|
||||
}
|
||||
|
||||
impl ImageProfile {
|
||||
/// Standard edition: max width 1200px, JPEG q80, color (§3.10).
|
||||
pub const STANDARD: ImageProfile = ImageProfile {
|
||||
max_width: 1200,
|
||||
max_height: 4000,
|
||||
jpeg_quality: 80,
|
||||
grayscale: false,
|
||||
};
|
||||
|
||||
/// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10).
|
||||
pub const X4: ImageProfile = ImageProfile {
|
||||
max_width: 480,
|
||||
max_height: 800,
|
||||
jpeg_quality: 70,
|
||||
grayscale: true,
|
||||
};
|
||||
|
||||
pub fn for_edition(edition: Edition) -> Self {
|
||||
match edition {
|
||||
Edition::Standard => Self::STANDARD,
|
||||
Edition::X4 => Self::X4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10).
|
||||
///
|
||||
/// Line art with transparency is kept as PNG after flattening; everything else
|
||||
/// becomes JPEG. SVG is rasterized first — charts and diagrams are frequently
|
||||
/// vector-only, and dropping them loses the point of the article. Returns `None`
|
||||
/// for sources no decoder handles.
|
||||
pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> {
|
||||
if looks_like_svg(bytes) {
|
||||
let raster = rasterize_svg(bytes, profile)?;
|
||||
return reencode(&raster, profile);
|
||||
}
|
||||
let format = image::guess_format(bytes).ok();
|
||||
let decoded = image::load_from_memory(bytes)
|
||||
.map_err(|e| tracing::debug!("undecodable image: {e}"))
|
||||
.ok()?;
|
||||
|
||||
let (w, h) = decoded.dimensions();
|
||||
if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX {
|
||||
tracing::debug!(w, h, "skipping decorative image");
|
||||
return None;
|
||||
}
|
||||
|
||||
let has_alpha = decoded.color().has_alpha();
|
||||
let flattened = if has_alpha {
|
||||
flatten_to_white(&decoded)
|
||||
} else {
|
||||
decoded
|
||||
};
|
||||
|
||||
let resized = if w > profile.max_width || h > profile.max_height {
|
||||
flattened.resize(
|
||||
profile.max_width,
|
||||
profile.max_height,
|
||||
image::imageops::FilterType::Lanczos3,
|
||||
)
|
||||
} else {
|
||||
flattened
|
||||
};
|
||||
|
||||
// Keep line art (PNG source, few distinct tones) lossless; everything else
|
||||
// becomes JPEG, which is far smaller for photographs (§3.10).
|
||||
let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized);
|
||||
|
||||
let mut out = Cursor::new(Vec::new());
|
||||
// NB: encode the concrete buffer, not the `DynamicImage` — the latter always
|
||||
// reports RGBA pixels, which would silently re-colorize a grayscale image.
|
||||
if profile.grayscale {
|
||||
let gray = resized.to_luma8();
|
||||
if keep_png {
|
||||
DynamicImage::ImageLuma8(gray)
|
||||
.write_to(&mut out, ImageFormat::Png)
|
||||
.ok()?;
|
||||
return Some((out.into_inner(), "image/png"));
|
||||
}
|
||||
let mut enc =
|
||||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
|
||||
enc.encode_image(&gray).ok()?;
|
||||
return Some((out.into_inner(), "image/jpeg"));
|
||||
}
|
||||
|
||||
let rgb = resized.to_rgb8();
|
||||
if keep_png {
|
||||
DynamicImage::ImageRgb8(rgb)
|
||||
.write_to(&mut out, ImageFormat::Png)
|
||||
.ok()?;
|
||||
return Some((out.into_inner(), "image/png"));
|
||||
}
|
||||
let mut enc =
|
||||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
|
||||
enc.encode_image(&rgb).ok()?;
|
||||
Some((out.into_inner(), "image/jpeg"))
|
||||
}
|
||||
|
||||
/// True when `bytes` are an SVG document (possibly behind an XML prolog or BOM).
|
||||
fn looks_like_svg(bytes: &[u8]) -> bool {
|
||||
let head = &bytes[..bytes.len().min(1024)];
|
||||
let text = String::from_utf8_lossy(head);
|
||||
let text = text.trim_start_matches('\u{feff}').trim_start();
|
||||
text.starts_with("<svg")
|
||||
|| (text.starts_with("<?xml") || text.starts_with("<!DOCTYPE svg")) && text.contains("<svg")
|
||||
}
|
||||
|
||||
/// Rasterize an SVG to a PNG at the profile's target width (§3.10).
|
||||
///
|
||||
/// The profile's own resize pass then handles the height cap, so this only has
|
||||
/// to land in the right ballpark.
|
||||
fn rasterize_svg(bytes: &[u8], profile: ImageProfile) -> Option<Vec<u8>> {
|
||||
let mut options = resvg::usvg::Options::default();
|
||||
options.fontdb_mut().load_system_fonts();
|
||||
let tree = resvg::usvg::Tree::from_data(bytes, &options)
|
||||
.map_err(|e| tracing::debug!("svg did not parse: {e}"))
|
||||
.ok()?;
|
||||
|
||||
// An `<svg>` that parses but draws nothing is not an image, it is a stray
|
||||
// tag: rasterizing it would embed a blank rectangle.
|
||||
if tree.root().children().is_empty() {
|
||||
tracing::debug!("svg has nothing to draw");
|
||||
return None;
|
||||
}
|
||||
let size = tree.size();
|
||||
let (sw, sh) = (size.width(), size.height());
|
||||
if !(sw.is_finite() && sh.is_finite()) || sw <= 0.0 || sh <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
// Judge "decorative" by the declared size, before scaling: a 16×16 icon is
|
||||
// an icon however large we choose to draw it.
|
||||
if sw < MIN_DIMENSION_PX as f32 || sh < MIN_DIMENSION_PX as f32 {
|
||||
tracing::debug!(sw, sh, "skipping decorative svg");
|
||||
return None;
|
||||
}
|
||||
// Vector art has no native resolution, so render straight at the edition's
|
||||
// target width — upscaling a rasterized copy afterwards would only blur it.
|
||||
let target_w = profile
|
||||
.max_width
|
||||
.max(SVG_FALLBACK_SIZE.min(profile.max_width));
|
||||
let scale = (target_w as f32 / sw).min(profile.max_height as f32 / sh);
|
||||
let scale = if scale.is_finite() && scale > 0.0 {
|
||||
scale
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
let (w, h) = (
|
||||
(sw * scale).round().max(1.0) as u32,
|
||||
(sh * scale).round().max(1.0) as u32,
|
||||
);
|
||||
let mut pixmap = tiny_skia::Pixmap::new(w, h)?;
|
||||
// E-ink has no transparency; render onto white so alpha never becomes black.
|
||||
pixmap.fill(tiny_skia::Color::WHITE);
|
||||
resvg::render(
|
||||
&tree,
|
||||
tiny_skia::Transform::from_scale(scale, scale),
|
||||
&mut pixmap.as_mut(),
|
||||
);
|
||||
let rgba = image::RgbaImage::from_raw(w, h, pixmap.take_demultiplied())?;
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(rgba)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.ok()?;
|
||||
Some(png.into_inner())
|
||||
}
|
||||
|
||||
/// Composite over an opaque white page — e-ink has no transparency (§3.10).
|
||||
fn flatten_to_white(img: &DynamicImage) -> DynamicImage {
|
||||
let rgba = img.to_rgba8();
|
||||
let mut rgb = image::RgbImage::new(rgba.width(), rgba.height());
|
||||
for (x, y, px) in rgba.enumerate_pixels() {
|
||||
let a = f32::from(px[3]) / 255.0;
|
||||
let blend = |c: u8| {
|
||||
((f32::from(c) * a) + 255.0 * (1.0 - a))
|
||||
.round()
|
||||
.clamp(0.0, 255.0) as u8
|
||||
};
|
||||
rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])]));
|
||||
}
|
||||
DynamicImage::ImageRgb8(rgb)
|
||||
}
|
||||
|
||||
/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text).
|
||||
fn is_line_art(img: &DynamicImage) -> bool {
|
||||
const SAMPLE_LIMIT: usize = 20_000;
|
||||
const DISTINCT_LIMIT: usize = 64;
|
||||
let rgb = img.to_rgb8();
|
||||
let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1);
|
||||
for (i, px) in rgb.pixels().enumerate() {
|
||||
if i >= SAMPLE_LIMIT {
|
||||
break;
|
||||
}
|
||||
let c = [px[0], px[1], px[2]];
|
||||
if !distinct.contains(&c) {
|
||||
distinct.push(c);
|
||||
if distinct.len() > DISTINCT_LIMIT {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reencode_resizes_grayscales_and_encodes() {
|
||||
let mut img = image::RgbaImage::new(200, 100);
|
||||
for (x, y, px) in img.enumerate_pixels_mut() {
|
||||
*px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]);
|
||||
}
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(img)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let raw = png.into_inner();
|
||||
|
||||
let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap();
|
||||
assert_eq!(std_mime, "image/jpeg");
|
||||
let decoded = image::load_from_memory(&std_bytes).unwrap();
|
||||
assert_eq!(decoded.dimensions(), (200, 100), "no upscaling");
|
||||
|
||||
let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap();
|
||||
let x4 = image::load_from_memory(&x4_bytes).unwrap();
|
||||
assert!(x4.width() <= 480 && x4.height() <= 800);
|
||||
assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reencode_skips_decorative_images_and_junk() {
|
||||
let tiny = image::RgbaImage::new(8, 8);
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(tiny)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none());
|
||||
// A bare `<svg>` tag with nothing to draw is markup, not a picture.
|
||||
assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none());
|
||||
assert!(reencode(b"not an image at all", ImageProfile::STANDARD).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn svg_charts_are_rasterized_rather_than_dropped() {
|
||||
let svg = br##"<svg xmlns="http://www.w3.org/2000/svg" width="400" height="300">
|
||||
<rect x="10" y="10" width="380" height="280" fill="#3355bb"/>
|
||||
<circle cx="200" cy="150" r="60" fill="#ffcc00"/>
|
||||
</svg>"##;
|
||||
let (bytes, mime) = reencode(svg, ImageProfile::STANDARD).expect("svg rasterizes");
|
||||
assert_eq!(mime, "image/png", "flat colour art stays lossless");
|
||||
let decoded = image::load_from_memory(&bytes).expect("decodable output");
|
||||
// Drawn at the edition's target width, not at the SVG's nominal size.
|
||||
assert_eq!(decoded.dimensions(), (1200, 900));
|
||||
assert!(!decoded.color().has_alpha(), "rendered onto white");
|
||||
|
||||
// An XML prolog and a leading BOM must not hide the format.
|
||||
let with_prolog = format!(
|
||||
"\u{feff}<?xml version=\"1.0\"?>{}",
|
||||
String::from_utf8_lossy(svg)
|
||||
);
|
||||
let (x4, _) = reencode(with_prolog.as_bytes(), ImageProfile::X4).expect("x4 rasterizes");
|
||||
let x4 = image::load_from_memory(&x4).unwrap();
|
||||
assert!(x4.width() <= 480 && x4.height() <= 800);
|
||||
|
||||
// A 16×16 icon is decorative however large we could draw it.
|
||||
let icon = br#"<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16">
|
||||
<rect width="16" height="16"/></svg>"#;
|
||||
assert!(reencode(icon, ImageProfile::STANDARD).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_art_png_stays_png_and_is_flattened() {
|
||||
let mut img = image::RgbaImage::new(120, 60);
|
||||
for (x, _y, px) in img.enumerate_pixels_mut() {
|
||||
*px = if x % 12 == 0 {
|
||||
image::Rgba([0, 0, 0, 255])
|
||||
} else {
|
||||
image::Rgba([255, 255, 255, 0])
|
||||
};
|
||||
}
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(img)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap();
|
||||
assert_eq!(mime, "image/png");
|
||||
let decoded = image::load_from_memory(&bytes).unwrap();
|
||||
assert!(!decoded.color().has_alpha(), "transparency is flattened");
|
||||
// Transparent pixels became white.
|
||||
assert_eq!(
|
||||
decoded.to_rgb8().get_pixel(1, 1),
|
||||
&image::Rgb([255, 255, 255])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
//! Downloading an issue's images and packing them into EPUB assets.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
|
||||
use crate::types::{Edition, ImageAsset, Pick};
|
||||
|
||||
use super::encode::{ImageProfile, reencode};
|
||||
use super::refs::{ImgRef, extract_img_refs};
|
||||
|
||||
/// Per-image download timeout (§3.10).
|
||||
pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
|
||||
/// Per-image size cap (§3.10).
|
||||
pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
|
||||
/// Concurrent downloads (§3.10).
|
||||
pub const CONCURRENCY: usize = 8;
|
||||
/// Whole-issue asset budget (§3.10).
|
||||
pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
|
||||
|
||||
/// Download one image, honoring the timeout and size cap (§3.10).
|
||||
pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
||||
let resp = http
|
||||
.get(url)
|
||||
// Some CDNs answer `Accept: */*` with an HTML interstitial (§3.10).
|
||||
.header(reqwest::header::ACCEPT, "image/*,*/*;q=0.8")
|
||||
.timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::debug!(url, "image download failed: {e}"))
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
tracing::debug!(url, status = %resp.status(), "image download rejected");
|
||||
return None;
|
||||
}
|
||||
if let Some(len) = resp.content_length()
|
||||
&& len as usize > MAX_IMAGE_BYTES
|
||||
{
|
||||
tracing::debug!(url, len, "image exceeds the size cap");
|
||||
return None;
|
||||
}
|
||||
let mut resp = resp;
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
loop {
|
||||
match resp.chunk().await {
|
||||
Ok(Some(chunk)) => {
|
||||
if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
|
||||
tracing::debug!(url, "image exceeds the size cap mid-stream");
|
||||
return None;
|
||||
}
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::debug!(url, "image download interrupted: {e}");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf.is_empty() { None } else { Some(buf) }
|
||||
}
|
||||
|
||||
/// Everything needed to fetch one image, in deterministic issue order.
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingImage {
|
||||
id: String,
|
||||
url: String,
|
||||
alt: String,
|
||||
caption: Option<String>,
|
||||
}
|
||||
|
||||
fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> {
|
||||
let entry_id = pick.article.best_entry_id;
|
||||
let mut refs = extract_img_refs(&pick.article.content_html);
|
||||
if refs.is_empty() {
|
||||
refs = pick
|
||||
.article
|
||||
.image_urls
|
||||
.iter()
|
||||
.map(|u| ImgRef {
|
||||
src: u.clone(),
|
||||
alt: String::new(),
|
||||
caption: None,
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
refs.into_iter()
|
||||
.filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://"))
|
||||
.enumerate()
|
||||
.map(|(i, r)| PendingImage {
|
||||
id: format!("img-{entry_id}-{i}"),
|
||||
url: r.src,
|
||||
alt: r.alt,
|
||||
caption: r.caption,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Download and re-encode every image referenced by the lineup for one edition,
|
||||
/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10).
|
||||
pub async fn collect_for_issue(
|
||||
http: &reqwest::Client,
|
||||
picks: &[Pick],
|
||||
edition: Edition,
|
||||
) -> Vec<ImageAsset> {
|
||||
let profile = ImageProfile::for_edition(edition);
|
||||
let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect();
|
||||
if pending.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
tracing::info!(count = pending.len(), ?edition, "downloading issue images");
|
||||
|
||||
let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> =
|
||||
futures::stream::iter(pending.into_iter().map(|p| {
|
||||
let http = http.clone();
|
||||
async move {
|
||||
let raw = download(&http, &p.url).await?;
|
||||
let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile))
|
||||
.await
|
||||
.ok()
|
||||
.flatten()?;
|
||||
Some((p, bytes, mime))
|
||||
}
|
||||
}))
|
||||
.buffered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut assets = Vec::new();
|
||||
let mut budget_used = 0usize;
|
||||
let mut skipped = 0usize;
|
||||
for result in results.into_iter().flatten() {
|
||||
let (pending, bytes, mime) = result;
|
||||
if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
budget_used += bytes.len();
|
||||
let ext = if mime == "image/png" { "png" } else { "jpg" };
|
||||
assets.push(ImageAsset {
|
||||
href: format!("images/{}.{ext}", pending.id),
|
||||
id: pending.id,
|
||||
mime: mime.to_string(),
|
||||
data: bytes,
|
||||
alt: pending.alt,
|
||||
caption: pending.caption,
|
||||
source_url: pending.url,
|
||||
});
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, budget_used, "issue image budget exhausted");
|
||||
}
|
||||
tracing::info!(
|
||||
embedded = assets.len(),
|
||||
bytes = budget_used,
|
||||
"issue images ready"
|
||||
);
|
||||
assets
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//! Article images, end to end (spec §3.3 extraction, §3.10 "Images").
|
||||
//!
|
||||
//! The stages run in this order, and each submodule owns one of them:
|
||||
//!
|
||||
//! | stage | module | when |
|
||||
//! |---|---|---|
|
||||
//! | make a page's `<img>` elements usable | [`normalize`] | extraction, before readability |
|
||||
//! | find what an article references | [`refs`] | extraction and issue build |
|
||||
//! | download and re-encode per edition | [`fetch`], [`encode`] | issue build |
|
||||
//! | point the markup at the embedded files | [`embed`] | chapter rendering |
|
||||
//!
|
||||
//! Nothing here ever fails a run: an image that cannot be fetched, decoded or
|
||||
//! resolved is dropped, and the article is rendered without it (notes §3).
|
||||
|
||||
pub mod embed;
|
||||
pub mod encode;
|
||||
pub mod fetch;
|
||||
pub mod normalize;
|
||||
pub mod refs;
|
||||
|
||||
pub use embed::rewrite_img_srcs;
|
||||
pub use encode::{ImageProfile, MIN_DIMENSION_PX, reencode};
|
||||
pub use fetch::{ISSUE_ASSET_BUDGET_BYTES, collect_for_issue, download};
|
||||
pub use normalize::{normalize_img_tags, prepare_for_readability, unwrap_image_wrappers};
|
||||
pub use refs::{ImgRef, collect_image_urls, extract_img_refs};
|
||||
@@ -0,0 +1,516 @@
|
||||
//! Making a page's `<img>` elements usable before anything else touches them.
|
||||
//!
|
||||
//! Publishers ship images in a dozen incompatible shapes: lazy placeholders,
|
||||
//! `srcset` lists, `<picture>` sources, URLs parked in `data-*` attributes, and
|
||||
//! `src` values that are not URLs at all (unfilled templates, JSON blobs, whole
|
||||
//! `srcset` strings). On top of that, readability actively damages images while
|
||||
//! it works — it deletes `<button>` subtrees, taking lightbox images with them,
|
||||
//! and its own lazy-image heuristic overwrites a working `src` with any
|
||||
//! attribute whose value happens to contain `.jpg`.
|
||||
//!
|
||||
//! [`prepare_for_readability`] runs before readability and leaves every `<img>`
|
||||
//! as a plain `src`/`alt`/`title` triple, which both fixes the input and denies
|
||||
//! readability the raw material for its substitution. [`normalize_img_tags`] is
|
||||
//! also applied to feed content, which carries the same markup.
|
||||
//!
|
||||
//! Candidates are judged by *shape*, never by publisher: a string with braces,
|
||||
//! whitespace or quotes in it cannot resolve, whoever wrote it.
|
||||
|
||||
use crate::html::{html_to_text, parse_attrs, tag_end, tag_name};
|
||||
|
||||
/// Elements that readability deletes outright, and which a page may nevertheless
|
||||
/// have wrapped around an image (lightbox triggers, mostly).
|
||||
const IMAGE_WRAPPER_TAGS: &[&str] = &["button", "form", "fieldset", "object"];
|
||||
|
||||
/// Attributes lazy-loading libraries use for the real image URL.
|
||||
///
|
||||
/// These outrank `src`, because a page only sets them when `src` is a stand-in:
|
||||
/// a transparent GIF, a blurred thumbnail, an inline SVG spacer. Attributes that
|
||||
/// merely *look* image-ish (`data-template`, `data-attrs`, `data-orig-file`) are
|
||||
/// deliberately absent — those hold templates and metadata, and preferring them
|
||||
/// is exactly the mistake readability's own heuristic makes.
|
||||
const LAZY_SRC_ATTRS: &[&str] = &[
|
||||
"data-src",
|
||||
"data-lazy-src",
|
||||
"data-original",
|
||||
"data-runner-src",
|
||||
"data-full-src",
|
||||
"data-hi-res-src",
|
||||
"data-image-src",
|
||||
];
|
||||
|
||||
/// Widest `srcset` candidate we will pick; above this we are downloading pixels
|
||||
/// the re-encoder immediately throws away.
|
||||
const MAX_SRCSET_WIDTH: u32 = 2000;
|
||||
|
||||
/// Make a fetched page safe to hand to readability (§3.3).
|
||||
///
|
||||
/// Two passes, both about images: unwrap the elements that would take an image
|
||||
/// with them when readability deletes them, then reduce every `<img>` to a plain
|
||||
/// `src`/`alt`/`title` triple. The second pass is what stops readability's own
|
||||
/// lazy-image heuristic from replacing a working `src` — with no `srcset`,
|
||||
/// `loading` or `data-*` attributes left on the element, it has nothing to
|
||||
/// substitute and leaves the image alone.
|
||||
pub fn prepare_for_readability(html: &str) -> String {
|
||||
normalize_img_tags(&unwrap_image_wrappers(html))
|
||||
}
|
||||
|
||||
/// Replace image-only `<button>`/`<form>`/`<fieldset>`/`<object>` wrappers with
|
||||
/// their contents (§3.3).
|
||||
///
|
||||
/// A `<button>` holding nothing but an image is a lightbox trigger, not a
|
||||
/// control: the image is the content. Wrappers that also carry text are left
|
||||
/// alone, because those really are interface.
|
||||
pub fn unwrap_image_wrappers(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
let inner = raw.trim_start_matches('<').trim_end_matches('>');
|
||||
let name = tag_name(inner);
|
||||
|
||||
if IMAGE_WRAPPER_TAGS.contains(&name.as_str())
|
||||
&& !inner.trim_end().ends_with('/')
|
||||
&& let Some((content, after)) = element_content(html, end, &name)
|
||||
&& content.contains("<img")
|
||||
&& html_to_text(content).trim().is_empty()
|
||||
{
|
||||
out.push_str(&unwrap_image_wrappers(content));
|
||||
cursor = after;
|
||||
continue;
|
||||
}
|
||||
|
||||
out.push_str(raw);
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The content of an element whose open tag ended at `body_start`, plus the
|
||||
/// offset just past its close tag. `None` when the element is never closed.
|
||||
fn element_content<'a>(html: &'a str, body_start: usize, name: &str) -> Option<(&'a str, usize)> {
|
||||
let open = format!("<{name}");
|
||||
let close = format!("</{name}");
|
||||
let mut depth = 1usize;
|
||||
let mut cursor = body_start;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
let end = tag_end(html, start)?;
|
||||
let tag = &html[start..end];
|
||||
let lower = tag.to_ascii_lowercase();
|
||||
if lower.starts_with(&close) {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return Some((&html[body_start..start], end));
|
||||
}
|
||||
} else if lower.starts_with(&open)
|
||||
&& !lower[open.len()..].starts_with(|c: char| c.is_alphanumeric() || c == '-')
|
||||
&& !tag.trim_end().ends_with("/>")
|
||||
{
|
||||
depth += 1;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Reduce every `<img>` to `<img src alt title>` with a usable URL (§3.3).
|
||||
///
|
||||
/// The `src` a page ships is not automatically the one to use: it can be a lazy
|
||||
/// placeholder (`data:image/svg+xml,…`), an unresolved template
|
||||
/// (`…/resize/{width}/…`), a JSON blob a framework parked there, or an entire
|
||||
/// `srcset` string. Candidates are tried in order and the first plausible one
|
||||
/// wins; an image with no plausible candidate is dropped, because a broken
|
||||
/// `<img>` only becomes clutter further down the pipeline.
|
||||
pub fn normalize_img_tags(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
// `<picture>` puts the real candidates on sibling `<source>` elements.
|
||||
let mut picture_srcset: Option<String> = None;
|
||||
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
let inner = raw
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>')
|
||||
.trim_end_matches('/');
|
||||
let name = tag_name(inner);
|
||||
|
||||
match name.as_str() {
|
||||
"picture" => {
|
||||
picture_srcset = None;
|
||||
out.push_str(raw);
|
||||
}
|
||||
"source" => {
|
||||
let attrs = parse_attrs(inner);
|
||||
if picture_srcset.is_none()
|
||||
&& let Some(set) =
|
||||
attr(&attrs, "srcset").or_else(|| attr(&attrs, "data-srcset"))
|
||||
{
|
||||
picture_srcset = Some(set.to_string());
|
||||
}
|
||||
out.push_str(raw);
|
||||
}
|
||||
"img" => {
|
||||
let attrs = parse_attrs(inner);
|
||||
if let Some(src) = best_img_src(&attrs, picture_srcset.as_deref()) {
|
||||
out.push_str("<img src=\"");
|
||||
out.push_str(&escape_attr(&src));
|
||||
out.push('"');
|
||||
for key in ["alt", "title"] {
|
||||
if let Some(v) = attr(&attrs, key) {
|
||||
out.push(' ');
|
||||
out.push_str(key);
|
||||
out.push_str("=\"");
|
||||
out.push_str(&escape_attr(v));
|
||||
out.push('"');
|
||||
}
|
||||
}
|
||||
out.push_str("/>");
|
||||
} else {
|
||||
tracing::debug!(tag = %&raw[..raw.len().min(120)], "dropping unusable img");
|
||||
}
|
||||
}
|
||||
_ => out.push_str(raw),
|
||||
}
|
||||
if name == "picture" && inner.starts_with('/') {
|
||||
picture_srcset = None;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
fn attr<'a>(attrs: &'a [(String, String)], name: &str) -> Option<&'a str> {
|
||||
attrs
|
||||
.iter()
|
||||
.find(|(k, _)| k == name)
|
||||
.map(|(_, v)| v.trim())
|
||||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
/// Pick the best URL for one `<img>` from everything the element carries.
|
||||
fn best_img_src(attrs: &[(String, String)], picture_srcset: Option<&str>) -> Option<String> {
|
||||
for key in LAZY_SRC_ATTRS {
|
||||
if let Some(v) = attr(attrs, key).filter(|s| plausible_url(s)) {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
}
|
||||
if let Some(src) = attr(attrs, "src").filter(|s| plausible_url(s)) {
|
||||
return Some(src.to_string());
|
||||
}
|
||||
if let Some(from_set) = attr(attrs, "srcset").and_then(best_from_srcset) {
|
||||
return Some(from_set);
|
||||
}
|
||||
if let Some(from_set) = attr(attrs, "data-srcset").and_then(best_from_srcset) {
|
||||
return Some(from_set);
|
||||
}
|
||||
picture_srcset.and_then(best_from_srcset)
|
||||
}
|
||||
|
||||
/// The widest candidate in a `srcset` that is still worth downloading.
|
||||
///
|
||||
/// Parsed by whitespace rather than by comma: the URLs of several image CDNs
|
||||
/// contain commas of their own, and splitting on those shreds them.
|
||||
fn best_from_srcset(srcset: &str) -> Option<String> {
|
||||
let mut best: Option<(u32, String)> = None;
|
||||
let mut smallest: Option<(u32, String)> = None;
|
||||
let mut pending: Option<String> = None;
|
||||
|
||||
let mut consider = |url: String, width: u32| {
|
||||
if width <= MAX_SRCSET_WIDTH && best.as_ref().is_none_or(|(w, _)| width > *w) {
|
||||
best = Some((width, url.clone()));
|
||||
}
|
||||
if smallest.as_ref().is_none_or(|(w, _)| width < *w) {
|
||||
smallest = Some((width, url));
|
||||
}
|
||||
};
|
||||
|
||||
for token in srcset.split_whitespace() {
|
||||
let token = token.trim_end_matches(',');
|
||||
if token.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match parse_descriptor(token) {
|
||||
Some(width) => {
|
||||
if let Some(url) = pending.take() {
|
||||
consider(url, width);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// A URL with no descriptor of its own still counts, at width 1x.
|
||||
if let Some(url) = pending.replace(token.to_string()) {
|
||||
consider(url, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(url) = pending.take() {
|
||||
consider(url, 1);
|
||||
}
|
||||
|
||||
best.or(smallest)
|
||||
.map(|(_, url)| url)
|
||||
.filter(|u| plausible_url(u))
|
||||
}
|
||||
|
||||
/// `800w` → 800, `2x` → a synthetic width so density candidates sort sensibly.
|
||||
fn parse_descriptor(token: &str) -> Option<u32> {
|
||||
let (value, unit) = token.split_at(token.len().checked_sub(1)?);
|
||||
match unit {
|
||||
"w" => value.parse::<u32>().ok(),
|
||||
"x" => value
|
||||
.parse::<f32>()
|
||||
.ok()
|
||||
.map(|d| (d * 1000.0).round().clamp(1.0, f32::from(u16::MAX)) as u32),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a string can serve as an image URL at all.
|
||||
///
|
||||
/// This is deliberately about shape, not about the host: whitespace, braces and
|
||||
/// quotes mean we are looking at a `srcset` blob, an unfilled URL template or a
|
||||
/// serialized object, none of which will ever resolve.
|
||||
fn plausible_url(candidate: &str) -> bool {
|
||||
let candidate = candidate.trim();
|
||||
if candidate.is_empty() || candidate.len() > 2048 {
|
||||
return false;
|
||||
}
|
||||
if candidate.starts_with("data:") || candidate.starts_with("about:") {
|
||||
return false;
|
||||
}
|
||||
if candidate
|
||||
.chars()
|
||||
.any(|c| c.is_whitespace() || matches!(c, '{' | '}' | '"' | '\'' | '<' | '>' | '\\'))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// `%20` is a space that survived encoding — same blob, different spelling.
|
||||
!candidate.contains("%20")
|
||||
}
|
||||
|
||||
fn escape_attr(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
/// The `src` values that survive normalization, read back with a real parser.
|
||||
fn src_of(html: &str) -> Vec<String> {
|
||||
Html::parse_fragment(html)
|
||||
.select(&Selector::parse("img").unwrap())
|
||||
.filter_map(|e| e.value().attr("src").map(str::to_string))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lazy_placeholder_src_gives_way_to_the_real_url() {
|
||||
// IEEE Spectrum: an inline SVG spacer with the URL parked on data-runner-src.
|
||||
let html = r#"<img alt="A well head" lazy-loadable="true"
|
||||
src="data:image/svg+xml,%3Csvg%20xmlns=%27http://www.w3.org/2000/svg%27%3E%3C/svg%3E"
|
||||
data-runner-src="https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"/>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://spectrum.ieee.org/media-library/well.jpg?id=675&width=980"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unresolved_url_templates_fall_through_to_a_real_candidate() {
|
||||
// NPR: readability copies data-template over a perfectly good src.
|
||||
let html = r#"<img alt="Meghan Cliffel"
|
||||
src="https://npr.brightspotcdn.com/resize/{width}/quality/{quality}/x.jpg"
|
||||
srcset="https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg 1100w"/>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://npr.brightspotcdn.com/resize/1100/quality/50/x.jpg"]
|
||||
);
|
||||
// With nothing usable anywhere, the image goes rather than becoming a
|
||||
// request for a picture that says "Image".
|
||||
let only_template =
|
||||
r#"<img alt="x" src="https://cdn.dev/resize/{width}/quality/{quality}/x.jpg"/>"#;
|
||||
assert!(src_of(&normalize_img_tags(only_template)).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_srcset_blob_parked_in_src_is_rejected_and_reparsed() {
|
||||
// dfarq: readability copies the entire srcset string into src.
|
||||
let blob = "https://i0.wp.com/x.jpg?resize=300%2C158&ssl=1 300w, \
|
||||
https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1 1024w, \
|
||||
https://i0.wp.com/x.jpg?w=3000&ssl=1 3000w";
|
||||
let html = format!(r#"<img alt="printer" src="{blob}" srcset="{blob}"/>"#);
|
||||
// The widest candidate under the download ceiling wins; 3000w does not.
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(&html)),
|
||||
["https://i0.wp.com/x.jpg?resize=1024%2C540&ssl=1"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_json_blob_parked_in_src_is_rejected() {
|
||||
// Substack: readability copies data-attrs (JSON) over src.
|
||||
let html = r#"<img alt="" src="{"src":"https://s3.dev/a.jpeg","width":1000}"
|
||||
srcset="https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg 1456w"/>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://substackcdn.com/image/fetch/$s_!y1,w_1456,c_limit/a.jpeg"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn picture_sources_back_up_an_empty_img() {
|
||||
let html = r#"<picture>
|
||||
<source srcset="https://cdn.dev/a.avif 800w" type="image/avif"/>
|
||||
<source srcset="https://cdn.dev/a.webp 800w" type="image/webp"/>
|
||||
<img alt="A photo"/>
|
||||
</picture>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://cdn.dev/a.avif"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_good_src_is_never_traded_for_a_metadata_attribute() {
|
||||
// `data-template`, `data-attrs` and friends hold templates and JSON, not
|
||||
// URLs — trading a working src for one of those is the original sin.
|
||||
let html = r#"<img alt="A photo" loading="lazy" class="lazyload"
|
||||
src="https://cdn.dev/real.jpg"
|
||||
data-template="https://cdn.dev/{width}/real.jpg"
|
||||
data-orig-file="https://cdn.dev/orig.jpg"/>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://cdn.dev/real.jpg"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lazy_loader_placeholder_loses_to_its_data_src() {
|
||||
// iRunFar (a3-lazy-load): src is a shared 1×1 spacer that would download
|
||||
// and re-encode perfectly happily, and be the wrong picture.
|
||||
let html = r#"<img class="lazy lazy-hidden" alt="Brooks Cascadia 20"
|
||||
src="//www.irunfar.com/wp-content/plugins/a3-lazy-load/assets/images/lazy_placeholder.gif"
|
||||
data-src="https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"/>"#;
|
||||
assert_eq!(
|
||||
src_of(&normalize_img_tags(html)),
|
||||
["https://s3.amazonaws.com/www.irunfar.com/uploads/Brooks-Cascadia-20.jpg"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_keeps_alt_and_title_and_drops_the_rest() {
|
||||
let html = r#"<img src="/a.png" alt="An & alt" title="T" class="x" width="900"
|
||||
onerror="evil()" srcset="/b.png 2x"/>"#;
|
||||
let out = normalize_img_tags(html);
|
||||
assert!(out.contains(r#"alt="An & alt""#), "{out}");
|
||||
assert!(out.contains(r#"title="T""#), "{out}");
|
||||
for gone in ["class=", "width=", "onerror", "srcset="] {
|
||||
assert!(
|
||||
!out.contains(gone),
|
||||
"expected {gone} to be dropped from {out}"
|
||||
);
|
||||
}
|
||||
// Relative URLs survive: sanitize_with_base absolutizes them later.
|
||||
assert_eq!(src_of(&out), ["/a.png"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lightbox_buttons_no_longer_take_their_image_with_them() {
|
||||
// Nautilus: readability deletes <button> and everything inside it.
|
||||
let html = r#"<figure class="wp-block-image">
|
||||
<button type="button" aria-label="Enlarge image">
|
||||
<img class="wp-image-1" src="https://cdn.dev/flower.png?w=710" alt=""/>
|
||||
</button>
|
||||
<figcaption>BEAUTIFUL DANGER: a belladonna flower.</figcaption>
|
||||
</figure>"#;
|
||||
let out = unwrap_image_wrappers(html);
|
||||
assert!(!out.contains("<button"), "{out}");
|
||||
assert!(!out.contains("</button>"), "{out}");
|
||||
assert!(out.contains("flower.png"), "{out}");
|
||||
assert!(out.contains("BEAUTIFUL DANGER"), "caption survives: {out}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buttons_that_are_really_buttons_are_left_alone() {
|
||||
let html = r#"<button class="subscribe">Subscribe <img src="/icon.png" alt=""/></button>"#;
|
||||
assert_eq!(unwrap_image_wrappers(html), html);
|
||||
// And an image-free control is untouched too.
|
||||
let plain = r#"<button>Share</button>"#;
|
||||
assert_eq!(unwrap_image_wrappers(plain), plain);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_and_unclosed_wrappers_do_not_derail_the_scan() {
|
||||
let nested = r#"<button><button><img src="/a.png"/></button></button><p>after</p>"#;
|
||||
let out = unwrap_image_wrappers(nested);
|
||||
assert!(!out.contains("button"), "{out}");
|
||||
assert!(out.contains("/a.png") && out.contains("after"), "{out}");
|
||||
// An open tag that never closes is passed through rather than eating
|
||||
// the rest of the document.
|
||||
let unclosed = r#"<button><img src="/a.png"/><p>rest</p>"#;
|
||||
assert!(unwrap_image_wrappers(unclosed).contains("rest"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srcset_descriptors_pick_the_widest_usable_candidate() {
|
||||
assert_eq!(
|
||||
best_from_srcset("/a.png 150w, /b.png 800w, /c.png 4000w").as_deref(),
|
||||
Some("/b.png")
|
||||
);
|
||||
// Density descriptors work as an ordering too.
|
||||
assert_eq!(
|
||||
best_from_srcset("/a.png 1x, /b.png 2x").as_deref(),
|
||||
Some("/b.png")
|
||||
);
|
||||
// A bare URL with no descriptor is still a candidate.
|
||||
assert_eq!(best_from_srcset("/only.png").as_deref(), Some("/only.png"));
|
||||
// Every candidate too wide: take the narrowest rather than nothing.
|
||||
assert_eq!(
|
||||
best_from_srcset("/big.png 3000w, /huge.png 5000w").as_deref(),
|
||||
Some("/big.png")
|
||||
);
|
||||
assert_eq!(best_from_srcset(" ").as_deref(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plausibility_is_about_shape_not_host() {
|
||||
for good in [
|
||||
"https://cdn.dev/a.jpg?id=1&width=980",
|
||||
"/_next/image?url=https%3A%2F%2Fx.dev%2Fa.png&w=3840&q=75",
|
||||
"https://substackcdn.com/image/fetch/$s_!y1,w_1456/a.jpeg",
|
||||
] {
|
||||
assert!(plausible_url(good), "{good} should be usable");
|
||||
}
|
||||
for bad in [
|
||||
"",
|
||||
"data:image/svg+xml,%3Csvg%3E%3C/svg%3E",
|
||||
"https://cdn.dev/resize/{width}/a.jpg",
|
||||
r#"{"src":"https://cdn.dev/a.jpg"}"#,
|
||||
"https://cdn.dev/a.jpg 300w, https://cdn.dev/b.jpg 600w",
|
||||
"https://cdn.dev/a.jpg%20300w",
|
||||
] {
|
||||
assert!(!plausible_url(bad), "{bad} should be rejected");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//! Finding the images an article references.
|
||||
//!
|
||||
//! The input here is already-normalized markup (see [`super::normalize`]), so
|
||||
//! every `<img>` is expected to carry a plain, usable `src`.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use scraper::{Html, Selector};
|
||||
use url::Url;
|
||||
|
||||
/// One `<img>` found in article markup, with the caption of its `<figure>` if any.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImgRef {
|
||||
pub src: String,
|
||||
pub alt: String,
|
||||
pub caption: Option<String>,
|
||||
}
|
||||
|
||||
/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup.
|
||||
pub fn extract_img_refs(html: &str) -> Vec<ImgRef> {
|
||||
let doc = scraper::Html::parse_fragment(html);
|
||||
let Ok(img_sel) = scraper::Selector::parse("img") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let cap_sel = scraper::Selector::parse("figcaption").ok();
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
for el in doc.select(&img_sel) {
|
||||
let Some(src) = el.value().attr("src") else {
|
||||
continue;
|
||||
};
|
||||
let src = src.trim();
|
||||
if src.is_empty() || src.starts_with("data:") {
|
||||
continue;
|
||||
}
|
||||
if seen.iter().any(|s| s == src) {
|
||||
continue;
|
||||
}
|
||||
seen.push(src.to_string());
|
||||
let alt = el
|
||||
.value()
|
||||
.attr("alt")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
// Walk up to an enclosing <figure> and take its caption, if any.
|
||||
let mut caption = None;
|
||||
if let Some(cap_sel) = &cap_sel {
|
||||
let mut cursor = el.parent();
|
||||
while let Some(node) = cursor {
|
||||
if let Some(elem) = scraper::ElementRef::wrap(node) {
|
||||
if elem.value().name() == "figure" {
|
||||
caption = elem.select(cap_sel).next().map(|c| {
|
||||
c.text()
|
||||
.collect::<String>()
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
});
|
||||
break;
|
||||
}
|
||||
cursor = elem.parent();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(ImgRef {
|
||||
src: src.to_string(),
|
||||
alt,
|
||||
caption: caption.filter(|c| !c.is_empty()),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Absolute image URLs referenced by `html`, resolved against `base_url` (§3.3).
|
||||
///
|
||||
/// Every image an article carries is kept: a photo essay with thirty pictures is
|
||||
/// a photo essay, and the issue-wide byte budget is the real backstop.
|
||||
pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> {
|
||||
let Ok(selector) = Selector::parse("img") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let base = Url::parse(base_url).ok();
|
||||
let document = Html::parse_fragment(html);
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for element in document.select(&selector) {
|
||||
let raw = element
|
||||
.value()
|
||||
.attr("src")
|
||||
.or_else(|| element.value().attr("data-src"))
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let Some(raw) = raw else { continue };
|
||||
let resolved = match Url::parse(raw) {
|
||||
Ok(u) => Some(u),
|
||||
Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()),
|
||||
};
|
||||
let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else {
|
||||
continue;
|
||||
};
|
||||
let url = url.to_string();
|
||||
if seen.insert(url.clone()) {
|
||||
out.push(url);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn image_collection_resolves_and_keeps_every_image() {
|
||||
let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#);
|
||||
html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#);
|
||||
for i in 0..20 {
|
||||
html.push_str(&format!(r#"<img src="/n{i}.png">"#));
|
||||
}
|
||||
let urls = collect_image_urls(&html, "https://blog.dev/posts/one");
|
||||
// Two named images, the data-src one, and all twenty of the rest: no cap.
|
||||
assert_eq!(urls.len(), 23);
|
||||
assert_eq!(urls[0], "https://blog.dev/a.png");
|
||||
assert_eq!(urls[1], "https://cdn.dev/b.png");
|
||||
assert_eq!(urls[2], "https://blog.dev/posts/c.png");
|
||||
// Duplicates and data: URIs never appear.
|
||||
assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1);
|
||||
assert!(!urls.iter().any(|u| u.starts_with("data:")));
|
||||
assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty());
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Image normalization — each case is a page shape seen in a real issue.
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn extracts_img_refs_with_captions() {
|
||||
let html = r#"<p>hi</p>
|
||||
<figure><img src="https://e.g/a.png" alt="A diagram"/>
|
||||
<figcaption>Figure 1: the thing</figcaption></figure>
|
||||
<img src="https://e.g/b.jpg"/>
|
||||
<img src="data:image/png;base64,zz"/>
|
||||
<img src="https://e.g/a.png" alt="dupe"/>"#;
|
||||
let refs = extract_img_refs(html);
|
||||
assert_eq!(refs.len(), 2);
|
||||
assert_eq!(refs[0].src, "https://e.g/a.png");
|
||||
assert_eq!(refs[0].alt, "A diagram");
|
||||
assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing"));
|
||||
assert_eq!(refs[1].src, "https://e.g/b.jpg");
|
||||
assert!(refs[1].caption.is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user