Initial commit: The Daily EPUB full implementation
Full implementation of a personalized daily newspaper delivered as an EPUB. Articles are pulled from a local self-hosted Miniflux instance, enriched with comments, summarized and filtered by DeepSeek AI, and then assembled into two EPUB editions: standard and optimized for the Xteink X4 e-ink reader. Both are served by the local self-hosted BookOrbit OPDS server in a separate library. Then the X4 edition is futher converted to XTC format and served over a separate OPDS server hosted by the Rust binary. Runs are tracked in a local SQLite database so runs are idempotent per date. Full documentation of the plan is in docs/plans and setup and install instructions are in the README.md file.
This commit is contained in:
@@ -0,0 +1,278 @@
|
||||
//! M4 — a complete issue EPUB, built offline (spec §3.10, §4 M4).
|
||||
//!
|
||||
//! Everything here is offline: the synthetic issue's images are never
|
||||
//! downloaded, so the chapters exercise the placeholder path.
|
||||
|
||||
use daily_epub::config::{self, Config};
|
||||
use daily_epub::epub::build::fixtures;
|
||||
use daily_epub::epub::{self, build};
|
||||
use daily_epub::types::{Edition, Issue, Vote};
|
||||
use daily_epub::{comments, world};
|
||||
|
||||
/// Local file headers store entry names verbatim, so a byte search over the
|
||||
/// archive is enough to assert its contents without a zip reader.
|
||||
fn contains_entry(zip: &[u8], name: &str) -> bool {
|
||||
zip.windows(name.len()).any(|w| w == name.as_bytes())
|
||||
}
|
||||
|
||||
/// Read one entry out of the archive, inflating it.
|
||||
fn read_entry(zip: &[u8], name: &str) -> String {
|
||||
use std::io::Read as _;
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip)).expect("zip opens");
|
||||
let mut file = archive.by_name(name).expect("entry exists");
|
||||
let mut out = String::new();
|
||||
file.read_to_string(&mut out).expect("entry is text");
|
||||
out
|
||||
}
|
||||
|
||||
/// Builds one edition into a temporary directory; the directory is cleaned up
|
||||
/// when the returned `TempDir` is dropped.
|
||||
fn build_edition_to_bytes(
|
||||
issue: &Issue,
|
||||
edition: Edition,
|
||||
) -> (tempfile::TempDir, std::path::PathBuf, Vec<u8>) {
|
||||
let cfg = Config {
|
||||
server: config::ServerConfig {
|
||||
public_url: "https://daily.hallada.net".into(),
|
||||
hmac_secret: Some("integration-secret".into()),
|
||||
..config::ServerConfig::default()
|
||||
},
|
||||
..Config::default()
|
||||
};
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let artifact = epub::build_edition_with_images(issue, edition, &cfg, dir.path(), &[])
|
||||
.expect("edition builds");
|
||||
let bytes = std::fs::read(&artifact.path).expect("read epub");
|
||||
assert_eq!(artifact.bytes as usize, bytes.len());
|
||||
(dir, artifact.path, bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_edition_is_a_well_formed_epub3_archive() {
|
||||
let issue = fixtures::issue();
|
||||
let (_dir, path, zip) = build_edition_to_bytes(&issue, Edition::Standard);
|
||||
|
||||
assert_eq!(
|
||||
path.file_name().and_then(|n| n.to_str()),
|
||||
Some("The Daily EPUB - 2026-08-15.epub")
|
||||
);
|
||||
assert_eq!(&zip[0..4], b"PK\x03\x04", "starts with a zip local header");
|
||||
assert_eq!(
|
||||
&zip[30..38],
|
||||
b"mimetype",
|
||||
"`mimetype` must be the first entry"
|
||||
);
|
||||
assert_eq!(&zip[38..58], b"application/epub+zip");
|
||||
|
||||
for entry in [
|
||||
"META-INF/container.xml",
|
||||
"OEBPS/content.opf",
|
||||
"OEBPS/toc.ncx",
|
||||
"OEBPS/nav.xhtml",
|
||||
"OEBPS/stylesheet.css",
|
||||
"OEBPS/cover.png",
|
||||
"OEBPS/cover.xhtml",
|
||||
"OEBPS/front.xhtml",
|
||||
"OEBPS/in-this-issue.xhtml",
|
||||
"OEBPS/sec-top-stories.xhtml",
|
||||
"OEBPS/art-1001.xhtml",
|
||||
"OEBPS/disc-1001.xhtml",
|
||||
"OEBPS/sec-niche-corner.xhtml",
|
||||
"OEBPS/art-1002.xhtml",
|
||||
"OEBPS/world.xhtml",
|
||||
"OEBPS/colophon.xhtml",
|
||||
] {
|
||||
assert!(contains_entry(&zip, entry), "missing {entry}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn x4_edition_is_built_alongside_the_standard_one() {
|
||||
let issue = fixtures::issue();
|
||||
let (_dir, path, zip) = build_edition_to_bytes(&issue, Edition::X4);
|
||||
assert_eq!(
|
||||
path.file_name().and_then(|n| n.to_str()),
|
||||
Some("The Daily EPUB - 2026-08-15 (X4).epub")
|
||||
);
|
||||
assert_eq!(&zip[30..38], b"mimetype");
|
||||
assert!(contains_entry(&zip, "OEBPS/art-1001.xhtml"));
|
||||
assert!(contains_entry(&zip, "OEBPS/cover.png"));
|
||||
}
|
||||
|
||||
/// Both editions land in the same BookOrbit library, which lists books by
|
||||
/// `dc:title` — so the edition has to be in the title, not just the filename
|
||||
/// (§3.10). Without this the two are indistinguishable in the library UI and
|
||||
/// over OPDS.
|
||||
#[test]
|
||||
fn the_two_editions_have_distinct_titles_in_the_opf() {
|
||||
let issue = fixtures::issue();
|
||||
let (_d1, _, standard) = build_edition_to_bytes(&issue, Edition::Standard);
|
||||
let (_d2, _, x4) = build_edition_to_bytes(&issue, Edition::X4);
|
||||
|
||||
let standard_opf = read_entry(&standard, "OEBPS/content.opf");
|
||||
let x4_opf = read_entry(&x4, "OEBPS/content.opf");
|
||||
|
||||
assert!(
|
||||
standard_opf.contains("<dc:title>The Daily EPUB \u{2014} 2026-08-15</dc:title>"),
|
||||
"{standard_opf}"
|
||||
);
|
||||
assert!(
|
||||
x4_opf.contains("<dc:title>The Daily EPUB \u{2014} 2026-08-15 (X4)</dc:title>"),
|
||||
"{x4_opf}"
|
||||
);
|
||||
|
||||
// The series metadata still groups them: same collection, same position, so
|
||||
// they sort together rather than as two unrelated books.
|
||||
for opf in [&standard_opf, &x4_opf] {
|
||||
assert!(opf.contains("belongs-to-collection"), "{opf}");
|
||||
assert!(opf.contains("<dc:date>2026-08-15</dc:date>"), "{opf}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapter_ids_hrefs_and_toc_levels_are_stable() {
|
||||
let issue = fixtures::issue();
|
||||
let first =
|
||||
build::render_all(&issue, Edition::Standard, &[], "https://x.test", None).expect("render");
|
||||
let second =
|
||||
build::render_all(&issue, Edition::Standard, &[], "https://x.test", None).expect("render");
|
||||
assert_eq!(first, second, "rendering is deterministic");
|
||||
|
||||
let map: Vec<(String, String, u8)> = first
|
||||
.iter()
|
||||
.map(|c| (c.id.clone(), c.href.clone(), c.toc_level))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
map,
|
||||
vec![
|
||||
("cover".into(), "cover.xhtml".into(), 1),
|
||||
("front".into(), "front.xhtml".into(), 1),
|
||||
("in-this-issue".into(), "in-this-issue.xhtml".into(), 1),
|
||||
("sec-Top Stories".into(), "sec-top-stories.xhtml".into(), 1),
|
||||
("art-1001".into(), "art-1001.xhtml".into(), 2),
|
||||
("disc-1001".into(), "disc-1001.xhtml".into(), 3),
|
||||
(
|
||||
"sec-Niche Corner".into(),
|
||||
"sec-niche-corner.xhtml".into(),
|
||||
1
|
||||
),
|
||||
("art-1002".into(), "art-1002.xhtml".into(), 2),
|
||||
("world".into(), "world.xhtml".into(), 1),
|
||||
("colophon".into(), "colophon.xhtml".into(), 1),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_chapter_is_parseable_xhtml() {
|
||||
let issue = fixtures::issue();
|
||||
let chapters = build::render_all(
|
||||
&issue,
|
||||
Edition::Standard,
|
||||
&[],
|
||||
"https://daily.hallada.net",
|
||||
Some("integration-secret"),
|
||||
)
|
||||
.expect("render");
|
||||
|
||||
for chapter in &chapters {
|
||||
let xhtml = &chapter.xhtml;
|
||||
assert!(
|
||||
xhtml.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>"),
|
||||
"{} lacks an XML prologue",
|
||||
chapter.id
|
||||
);
|
||||
assert!(xhtml.contains("xmlns=\"http://www.w3.org/1999/xhtml\""));
|
||||
assert!(xhtml.trim_end().ends_with("</html>"));
|
||||
// Undefined XML entities (html5ever's ` `) would break XML parsers.
|
||||
assert!(!xhtml.contains(" "), "{} has ", chapter.id);
|
||||
for tag in ["html", "head", "body", "div", "p", "a", "blockquote"] {
|
||||
let opens = xhtml.matches(&format!("<{tag}")).count();
|
||||
let closes = xhtml.matches(&format!("</{tag}>")).count();
|
||||
assert_eq!(opens, closes, "unbalanced <{tag}> in {}", chapter.id);
|
||||
}
|
||||
// Void elements are self-closed.
|
||||
for void in ["<br>", "<hr>", "<link ", "<meta charset=\"utf-8\">"] {
|
||||
assert!(!xhtml.contains(void) || xhtml.contains("/>"), "{void}");
|
||||
}
|
||||
// Every `&` opens an entity XML actually defines.
|
||||
for (i, _) in xhtml.match_indices('&') {
|
||||
let tail = &xhtml[i + 1..];
|
||||
assert!(
|
||||
is_defined_entity(tail),
|
||||
"bare `&` at byte {i} in {}: {:?}",
|
||||
chapter.id,
|
||||
&xhtml[i..(i + 24).min(xhtml.len())]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// XML predefines only these five names; everything else must be numeric.
|
||||
fn is_defined_entity(after_ampersand: &str) -> bool {
|
||||
let Some(end) = after_ampersand.find(';') else {
|
||||
return false;
|
||||
};
|
||||
let name = &after_ampersand[..end];
|
||||
if matches!(name, "amp" | "lt" | "gt" | "quot" | "apos") {
|
||||
return true;
|
||||
}
|
||||
match name.strip_prefix('#') {
|
||||
Some(rest) => match rest.strip_prefix('x').or_else(|| rest.strip_prefix('X')) {
|
||||
Some(hex) => !hex.is_empty() && hex.chars().all(|c| c.is_ascii_hexdigit()),
|
||||
None => !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit()),
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rating_links_carry_the_spec_token() {
|
||||
let issue = fixtures::issue();
|
||||
let chapter = build::render_all(
|
||||
&issue,
|
||||
Edition::Standard,
|
||||
&[],
|
||||
"https://daily.hallada.net",
|
||||
Some("integration-secret"),
|
||||
)
|
||||
.expect("render")
|
||||
.into_iter()
|
||||
.find(|c| c.id == "art-1001")
|
||||
.expect("article chapter");
|
||||
|
||||
let date = issue.meta.date;
|
||||
let up = build::rating_token("integration-secret", date, 1, Vote::Up);
|
||||
let down = build::rating_token("integration-secret", date, 1, Vote::Down);
|
||||
assert_eq!(up.len(), 16);
|
||||
assert_ne!(up, down);
|
||||
assert!(chapter.xhtml.contains(&format!(
|
||||
"https://daily.hallada.net/r/2026-08-15/1/up?t={up}"
|
||||
)));
|
||||
assert!(chapter.xhtml.contains(&format!(
|
||||
"https://daily.hallada.net/r/2026-08-15/1/down?t={down}"
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_and_world_fixtures_feed_real_chapters() {
|
||||
let hn: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/hn_item.json"
|
||||
))
|
||||
.expect("fixture"),
|
||||
)
|
||||
.expect("json");
|
||||
let thread = comments::parse_hn(&hn).expect("thread");
|
||||
assert_eq!(thread.total_comments, 4);
|
||||
|
||||
let html = std::fs::read_to_string(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/wikipedia_current_events.html"
|
||||
))
|
||||
.expect("fixture");
|
||||
let body = world::extract_events(&html).expect("events");
|
||||
assert!(body.contains("<li>"));
|
||||
assert!(!body.contains("<a "));
|
||||
}
|
||||
Reference in New Issue
Block a user