Initial commit: The Daily EPUB full implementation

Full implementation of a personalized daily newspaper delivered as an
EPUB.

Articles are pulled from a local self-hosted Miniflux instance, enriched
with comments, summarized and filtered by DeepSeek AI, and then
assembled into two EPUB editions: standard and optimized for the Xteink
X4 e-ink reader. Both are served by the local self-hosted BookOrbit OPDS
server in a separate library. Then the X4 edition is futher converted to
XTC format and served over a separate OPDS server hosted by the Rust
binary. Runs are tracked in a local SQLite database so runs are
idempotent per date.

Full documentation of the plan is in docs/plans and setup and install
instructions are in the README.md file.
This commit is contained in:
2026-08-15 17:46:19 +00:00
commit 9e30c1dcdf
80 changed files with 27578 additions and 0 deletions
+702
View File
@@ -0,0 +1,702 @@
//! Shared domain types — the contract between pipeline stages (spec §2, §3.13).
//!
//! Every stage module (`dedupe`, `extract`, `social`, `curate`, `comments`, `epub`,
//! `publish`, `server`, `world`) codes against the types defined here so that the
//! stages can be implemented independently. Keep this module free of I/O.
use std::collections::BTreeMap;
use std::fmt;
use std::path::PathBuf;
use jiff::Timestamp;
use jiff::civil::Date;
use serde::{Deserialize, Serialize};
/// Miniflux entry id (also our `entries.id`).
pub type EntryId = i64;
/// Row id of a deduped article cluster (`articles.id`).
pub type ArticleId = i64;
/// Miniflux feed id.
pub type FeedId = i64;
// ---------------------------------------------------------------------------
// Ingest (§3.1)
// ---------------------------------------------------------------------------
/// A raw Miniflux entry as persisted in the `entries` table (§3.1, §3.13).
///
/// `canonical_url` is `None` at ingest time; the dedupe stage
/// ([`crate::dedupe::canonical_url`], §3.2) fills it in.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Entry {
pub id: EntryId,
pub feed_id: FeedId,
pub feed_title: Option<String>,
pub category: Option<String>,
pub title: String,
pub url: String,
pub canonical_url: Option<String>,
pub author: Option<String>,
pub published_at: Option<Timestamp>,
pub comments_url: Option<String>,
pub raw_content: String,
pub fetched_at: Timestamp,
}
/// Where an article reached us from — a curation signal in its own right (§3.2, §3.5).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceKind {
/// Arrived via a Scour interest feed — it already matched a stated interest.
Scour,
/// Arrived via the Hacker News frontpage feed (hnrss et al).
HnFrontpage,
/// Arrived via a lobste.rs feed.
Lobsters,
/// Arrived via a Reddit feed.
Reddit,
/// A plain blog/publication feed.
Feed,
}
/// One feed that carried this story; an article cluster keeps the union of them (§3.2).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SourceRef {
pub entry_id: EntryId,
pub feed_id: FeedId,
pub feed_title: String,
pub category: Option<String>,
pub kind: SourceKind,
}
// ---------------------------------------------------------------------------
// Dedupe + extraction (§3.2, §3.3)
// ---------------------------------------------------------------------------
/// How an article's body text was obtained (§3.3).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ExtractMethod {
/// Miniflux's stored content already looked like full text.
Miniflux,
/// Fetched the article URL and ran readability over it.
Readability,
/// Only a feed summary/excerpt was available.
Excerpt,
}
/// Result of the content-extraction stage for one article (§3.3).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Extracted {
/// Sanitized XHTML-safe body markup.
pub content_html: String,
pub word_count: i64,
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
pub excerpt_only: bool,
/// Absolute image URLs referenced by the body, capped at 12 (§3.3).
pub image_urls: Vec<String>,
pub method: ExtractMethod,
}
/// A deduped story cluster: the unit everything downstream operates on (§3.2).
///
/// Persisted fields map to the `articles` table; the remaining fields are
/// denormalized from the best entry / `social` table for convenience and are
/// re-hydrated by [`crate::db`] when an article is loaded.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Article {
/// Zero until the row has been inserted.
pub id: ArticleId,
pub canonical_url: String,
pub title: String,
/// The entry whose content we kept (the richest one).
pub best_entry_id: EntryId,
pub content_html: String,
pub word_count: i64,
pub excerpt_only: bool,
pub image_count: i64,
/// Union of the feeds that carried this story (`articles.sources_json`).
pub sources: Vec<SourceRef>,
pub first_seen: Timestamp,
// --- denormalized, not stored on `articles` ---
pub url: String,
pub author: Option<String>,
pub feed_id: FeedId,
pub feed_title: String,
pub category: Option<String>,
pub published_at: Option<Timestamp>,
pub comments_url: Option<String>,
pub image_urls: Vec<String>,
pub social: Vec<SocialRef>,
pub extract_method: ExtractMethod,
}
impl Article {
/// Estimated reading time at 220 wpm, minimum one minute (§3.10).
pub fn reading_minutes(&self) -> i64 {
reading_minutes(self.word_count)
}
/// Composite social proof across all sources (§3.4).
pub fn social_score(&self) -> f64 {
composite_social_score(&self.social)
}
/// True when this story arrived via a feed of the given kind (§3.5).
pub fn came_via(&self, kind: SourceKind) -> bool {
self.sources.iter().any(|s| s.kind == kind)
}
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
pub fn chapter_id(&self) -> String {
format!("art-{}", self.best_entry_id)
}
}
/// Estimated reading time at 220 wpm, minimum one minute.
pub fn reading_minutes(word_count: i64) -> i64 {
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
}
// ---------------------------------------------------------------------------
// Social proof (§3.4)
// ---------------------------------------------------------------------------
/// Social platforms we look up. `X` is reserved: no free API today (§3.4, §7).
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum SocialSource {
Hn,
Lobsters,
Reddit,
X,
}
impl SocialSource {
/// Value stored in `social.source` (matches the CHECK constraint).
pub fn as_str(self) -> &'static str {
match self {
SocialSource::Hn => "hn",
SocialSource::Lobsters => "lobsters",
SocialSource::Reddit => "reddit",
SocialSource::X => "x",
}
}
/// Human-readable label used in chapter titles and stat lines (§3.7, §3.10).
pub fn display_name(self) -> &'static str {
match self {
SocialSource::Hn => "HN",
SocialSource::Lobsters => "Lobsters",
SocialSource::Reddit => "Reddit",
SocialSource::X => "X",
}
}
pub fn parse(s: &str) -> Option<Self> {
match s {
"hn" => Some(SocialSource::Hn),
"lobsters" => Some(SocialSource::Lobsters),
"reddit" => Some(SocialSource::Reddit),
"x" => Some(SocialSource::X),
_ => None,
}
}
}
impl fmt::Display for SocialSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
/// A cached social-proof lookup for one article on one platform (`social` table).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SocialRef {
pub article_id: ArticleId,
pub source: SocialSource,
/// Platform item id: HN `objectID`, lobsters story id, reddit fullname.
pub item_id: Option<String>,
pub score: i64,
pub num_comments: i64,
/// Link a human can open (HN item page, lobsters story, reddit permalink).
pub item_url: Option<String>,
pub fetched_at: Timestamp,
}
/// `log10(1+hn) + 0.7*log10(1+reddit) + log10(1+lobsters) + 0.5*log10(1+comments)` (§3.4).
///
/// Lives here rather than in `social/` because both the pre-filter and the EPUB
/// stat line need it.
pub fn composite_social_score(refs: &[SocialRef]) -> f64 {
let mut score = 0.0;
let mut comments = 0i64;
for r in refs {
let points = (r.score.max(0)) as f64;
let weight = match r.source {
SocialSource::Hn | SocialSource::Lobsters => 1.0,
SocialSource::Reddit => 0.7,
SocialSource::X => 0.0,
};
score += weight * (1.0 + points).log10();
comments += r.num_comments.max(0);
}
score + 0.5 * (1.0 + comments as f64).log10()
}
// ---------------------------------------------------------------------------
// Curation (§3.5, §3.6)
// ---------------------------------------------------------------------------
/// DeepSeek stage-A output for one article (§3.6).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct LlmScore {
/// 0–10.
pub score: f64,
pub category: String,
/// ≤ 20 words.
pub rationale: String,
#[serde(default)]
pub is_paywalled_guess: bool,
}
/// An article carrying every ranking signal computed so far (§3.5, §3.6).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ScoredArticle {
pub article: Article,
/// Heuristic pre-filter score, 0–100 (§3.5).
pub prefilter_score: f64,
/// Cached [`composite_social_score`] for the article.
pub social_score: f64,
/// Beta-smoothed per-feed upvote rate applied by the pre-filter (§3.9).
pub feed_prior: f64,
/// `None` until stage A has run (or when `--skip-llm`).
pub llm: Option<LlmScore>,
/// From `curation.always_include_feeds`: may be scored but never dropped (§3.5).
pub auto_include: bool,
}
impl ScoredArticle {
/// Ranking key for stage B: LLM score weighted with social proof and priors (§3.6).
pub fn combined_score(&self) -> f64 {
let llm = self.llm.as_ref().map(|l| l.score).unwrap_or(0.0);
llm * 10.0 + self.social_score * 4.0 + self.feed_prior * 10.0 + self.prefilter_score * 0.1
}
}
/// One selected article with its section placement (§3.6 stage B).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Pick {
pub article: Article,
/// One of `curation.sections` (or the reserved `World Briefing`).
pub section: String,
/// Order within the section, ascending.
pub position: i64,
pub is_lead: bool,
/// Newspaper-abstract summary from stage C; `None` until editorial runs.
pub summary: Option<String>,
pub llm: Option<LlmScore>,
/// Rendered comment chapter, when the article had social refs (§3.7).
pub discussion: Option<Discussion>,
}
/// The day's final lineup: 15–25 picks grouped into sections (§3.6 stage B).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Lineup {
pub date: Date,
/// Sorted by (section order, position).
pub picks: Vec<Pick>,
/// Section names in issue order; empty sections are omitted (§3.6).
pub section_order: Vec<String>,
}
impl Lineup {
/// Picks belonging to `section`, in position order.
pub fn section_picks(&self, section: &str) -> Vec<&Pick> {
let mut v: Vec<&Pick> = self.picks.iter().filter(|p| p.section == section).collect();
v.sort_by_key(|p| p.position);
v
}
pub fn lead(&self) -> Option<&Pick> {
self.picks.iter().find(|p| p.is_lead)
}
pub fn total_words(&self) -> i64 {
self.picks.iter().map(|p| p.article.word_count).sum()
}
}
/// Stage-C editorial output (§3.6).
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Editorial {
/// "From the Editor", 250–400 words, already sanitized XHTML.
pub front_page_html: String,
/// Section name → 2–3 sentence intro.
pub section_intros: BTreeMap<String, String>,
/// Article id → 2–3 sentence newspaper abstract.
pub summaries: BTreeMap<ArticleId, String>,
}
/// The taste profile that forms the DeepSeek system prompt (§3.6, `kv`).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TasteProfile {
/// Full ~600-word prompt document.
pub text: String,
pub version: i64,
pub built_at: Timestamp,
}
// ---------------------------------------------------------------------------
// Comments (§3.7)
// ---------------------------------------------------------------------------
/// One comment node in a discussion tree (§3.7).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Comment {
pub author: String,
pub points: Option<i64>,
/// Sanitized comment body, ellipsized to 1,200 chars.
pub text_html: String,
/// 0 for top-level; rendering stops at depth 3.
pub depth: usize,
pub children: Vec<Comment>,
}
/// The comment tree fetched from one platform for one article (§3.7).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CommentThread {
pub source: SocialSource,
pub item_url: String,
pub total_comments: i64,
/// Top ~8 top-level threads by score.
pub comments: Vec<Comment>,
}
/// A rendered discussion chapter: one per article, HN → Lobsters → Reddit (§3.7).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Discussion {
pub article_id: ArticleId,
/// Chapter id, `disc-{entry_id}` (implementation notes §12).
pub chapter_id: String,
pub threads: Vec<CommentThread>,
}
impl Discussion {
pub fn total_comments(&self) -> i64 {
self.threads.iter().map(|t| t.total_comments).sum()
}
}
// ---------------------------------------------------------------------------
// World briefing (§3.8)
// ---------------------------------------------------------------------------
/// Wikipedia Current Events portal digest for one day (§3.8).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct WorldBriefing {
pub date: Date,
/// Portal URL the content came from (also used for CC BY-SA attribution).
pub source_url: String,
/// Sanitized `<ul>`-style markup of the day's events.
pub body_html: String,
}
/// Reserved section name for [`WorldBriefing`] — never offered to the LLM (§3.6).
pub const WORLD_BRIEFING_SECTION: &str = "World Briefing";
// ---------------------------------------------------------------------------
// Issue assembly (§3.10)
// ---------------------------------------------------------------------------
/// Which of the two editions is being built (§3.10).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Edition {
/// 1200px images, full CSS.
Standard,
/// Grayscale, 480×800, simplified CSS — input for the XTC converter.
X4,
}
impl Edition {
/// Filename suffix: `""` / `" (X4)"` (§3.11).
pub fn file_suffix(self) -> &'static str {
match self {
Edition::Standard => "",
Edition::X4 => " (X4)",
}
}
}
/// Issue-level metadata rendered on the cover, front page and OPF (§3.10).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct IssueMeta {
pub date: Date,
/// Days since the first issue; EPUB3 `group-position`.
pub issue_number: i64,
pub generated_at: Timestamp,
/// "Friday, August 15, 2026".
pub display_date: String,
pub article_count: i64,
pub section_count: i64,
pub total_words: i64,
pub reading_minutes: i64,
}
impl IssueMeta {
/// The issue's name, without an edition tag: "The Daily EPUB — 2026-08-15".
pub fn title(&self) -> String {
format!("The Daily EPUB — {}", self.date)
}
/// `dc:title` for one edition: [`title`](Self::title) plus the edition tag
/// (§3.10).
///
/// Both editions land in the same BookOrbit library, and BookOrbit — like
/// every OPDS client — lists books by `dc:title`. Carrying the distinction
/// only in the filename makes them indistinguishable everywhere except the
/// per-book file listing, so the title and the filename share one suffix.
pub fn title_for(&self, edition: Edition) -> String {
format!("{}{}", self.title(), edition.file_suffix())
}
/// "22 articles · ~1h 45m read · 6 sections" (§3.10).
pub fn stats_line(&self) -> String {
let (h, m) = (self.reading_minutes / 60, self.reading_minutes % 60);
let time = if h > 0 {
format!("~{h}h {m}m read")
} else {
format!("~{m}m read")
};
format!(
"{} articles · {} · {} sections",
self.article_count, time, self.section_count
)
}
}
/// Everything the EPUB builder needs; fully materialized before rendering (§3.10).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Issue {
pub meta: IssueMeta,
pub lineup: Lineup,
pub editorial: Editorial,
pub world_briefing: Option<WorldBriefing>,
/// Colophon facts: models used, token cost, feed counts (§3.10).
pub colophon: Colophon,
}
/// Back-matter facts printed in the colophon chapter (§3.10).
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Colophon {
pub model: String,
pub entries_fetched: i64,
pub feeds_seen: i64,
pub candidates: i64,
pub cost_usd: f64,
pub generator_version: String,
}
/// A downloaded, re-encoded image embedded in an edition (§3.10 images).
#[derive(Debug, Clone, PartialEq)]
pub struct ImageAsset {
/// Manifest id, unique within the issue.
pub id: String,
/// Path inside the EPUB, e.g. `images/art-1234-0.jpg`.
pub href: String,
pub mime: String,
pub data: Vec<u8>,
pub alt: String,
pub caption: Option<String>,
/// The original remote URL, used to rewrite `<img src>`.
pub source_url: String,
}
/// A file produced by the build/publish stages (§3.11).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Artifact {
pub edition: Edition,
pub path: PathBuf,
pub bytes: u64,
}
// ---------------------------------------------------------------------------
// Feedback (§3.9)
// ---------------------------------------------------------------------------
/// 👍 / 👎 stored as `+1` / `-1` in `ratings.vote` (§3.9).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Vote {
Up,
Down,
}
impl Vote {
pub fn as_i64(self) -> i64 {
match self {
Vote::Up => 1,
Vote::Down => -1,
}
}
/// Path segment used in rating links: `up` / `down` (§3.9).
pub fn as_str(self) -> &'static str {
match self {
Vote::Up => "up",
Vote::Down => "down",
}
}
pub fn parse(s: &str) -> Option<Self> {
match s {
"up" => Some(Vote::Up),
"down" => Some(Vote::Down),
_ => None,
}
}
}
/// A recorded reader vote (`ratings` table, §3.9).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Rating {
pub issue_date: Date,
pub article_id: ArticleId,
pub vote: Vote,
pub rated_at: Timestamp,
}
/// Beta-smoothed per-feed upvote rate used by the pre-filter (`feed_priors`, §3.9).
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
pub struct FeedPrior {
pub feed_id: FeedId,
pub upvotes: i64,
pub downvotes: i64,
pub included: i64,
}
impl FeedPrior {
/// `(up + 1) / (up + down + 2)` — 0.5 with no evidence (§3.9).
pub fn rate(&self) -> f64 {
(self.upvotes + 1) as f64 / (self.upvotes + self.downvotes + 2) as f64
}
}
// ---------------------------------------------------------------------------
// LLM accounting (§3.6 cost guardrail)
// ---------------------------------------------------------------------------
/// Token counters accumulated across every DeepSeek call in a run (§3.6).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct TokenUsage {
/// Cache-miss input tokens (billed at the full input rate).
pub input_tokens: i64,
/// Prefix-cache hits (billed at the cached rate).
pub cached_tokens: i64,
pub output_tokens: i64,
}
impl TokenUsage {
pub fn add(&mut self, other: TokenUsage) {
self.input_tokens += other.input_tokens;
self.cached_tokens += other.cached_tokens;
self.output_tokens += other.output_tokens;
}
/// USD cost given the per-1M-token prices from `[deepseek]` config (§3.6).
pub fn cost_usd(&self, price_input: f64, price_cached: f64, price_output: f64) -> f64 {
(self.input_tokens as f64 * price_input
+ self.cached_tokens as f64 * price_cached
+ self.output_tokens as f64 * price_output)
/ 1_000_000.0
}
}
#[cfg(test)]
mod tests {
use super::*;
fn ts() -> Timestamp {
"2026-08-15T05:30:00Z".parse().unwrap()
}
fn social(source: SocialSource, score: i64, comments: i64) -> SocialRef {
SocialRef {
article_id: 1,
source,
item_id: Some("1".into()),
score,
num_comments: comments,
item_url: None,
fetched_at: ts(),
}
}
#[test]
fn composite_social_score_matches_spec_formula() {
let refs = vec![
social(SocialSource::Hn, 342, 210),
social(SocialSource::Reddit, 99, 40),
];
let expected = (343f64).log10() + 0.7 * (100f64).log10() + 0.5 * (251f64).log10();
assert!((composite_social_score(&refs) - expected).abs() < 1e-9);
assert_eq!(composite_social_score(&[]), 0.0);
}
#[test]
fn feed_prior_is_beta_smoothed() {
assert_eq!(FeedPrior::default().rate(), 0.5);
let p = FeedPrior {
feed_id: 1,
upvotes: 3,
downvotes: 1,
included: 4,
};
assert!((p.rate() - 4.0 / 6.0).abs() < 1e-12);
}
#[test]
fn stats_line_and_reading_time() {
assert_eq!(reading_minutes(0), 1);
assert_eq!(reading_minutes(440), 2);
let meta = IssueMeta {
date: "2026-08-15".parse().unwrap(),
issue_number: 1,
generated_at: ts(),
display_date: "Friday, August 15, 2026".into(),
article_count: 22,
section_count: 6,
total_words: 23_000,
reading_minutes: 105,
};
assert_eq!(meta.stats_line(), "22 articles · ~1h 45m read · 6 sections");
assert_eq!(meta.title(), "The Daily EPUB — 2026-08-15");
assert_eq!(
meta.title_for(Edition::Standard),
"The Daily EPUB — 2026-08-15"
);
assert_eq!(
meta.title_for(Edition::X4),
"The Daily EPUB — 2026-08-15 (X4)"
);
// Title and filename carry the same tag, so a book found in the library
// maps back to a file without guessing.
assert!(
meta.title_for(Edition::X4)
.ends_with(Edition::X4.file_suffix())
);
}
#[test]
fn vote_and_social_source_round_trip() {
assert_eq!(Vote::parse("up"), Some(Vote::Up));
assert_eq!(Vote::Down.as_i64(), -1);
assert_eq!(
SocialSource::parse("lobsters"),
Some(SocialSource::Lobsters)
);
assert_eq!(SocialSource::Hn.as_str(), "hn");
}
}