Initial commit: The Daily EPUB full implementation
Full implementation of a personalized daily newspaper delivered as an EPUB. Articles are pulled from a local self-hosted Miniflux instance, enriched with comments, summarized and filtered by DeepSeek AI, and then assembled into two EPUB editions: standard and optimized for the Xteink X4 e-ink reader. Both are served by the local self-hosted BookOrbit OPDS server in a separate library. Then the X4 edition is futher converted to XTC format and served over a separate OPDS server hosted by the Rust binary. Runs are tracked in a local SQLite database so runs are idempotent per date. Full documentation of the plan is in docs/plans and setup and install instructions are in the README.md file.
This commit is contained in:
+141
@@ -0,0 +1,141 @@
|
||||
//! Rating-link signing — the single source of truth for the HMAC token (spec §3.9).
|
||||
//!
|
||||
//! The EPUB article footer ([`crate::epub::build`]) mints the links and the rating
|
||||
//! endpoint ([`crate::server`]) verifies them, so the formula must be identical on
|
||||
//! both sides. It lives here and nowhere else:
|
||||
//!
|
||||
//! ```text
|
||||
//! message = "{issue_date}/{article_id}/{up|down}"
|
||||
//! token = hex(hmac_sha256(secret, message))[..16]
|
||||
//! link = {public_url}/r/{issue_date}/{article_id}/{vote}?t={token}
|
||||
//! ```
|
||||
//!
|
||||
//! Pinned test vector, asserted from three places (here, `epub::build`,
|
||||
//! `tests/m7_server.rs`): `secret = "test-secret"`, date `2026-08-15`,
|
||||
//! article `42`, `up` → `3b314cf7e6d8f50f`.
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use jiff::civil::Date;
|
||||
use sha2::Sha256;
|
||||
|
||||
use crate::types::{ArticleId, Vote};
|
||||
|
||||
/// Characters of the hex HMAC kept in rating links (§3.9).
|
||||
pub const TOKEN_LEN: usize = 16;
|
||||
|
||||
/// The exact signed string: `{issue_date}/{article_id}/{up|down}` (§3.9).
|
||||
pub fn rating_message(issue_date: Date, article_id: ArticleId, vote: Vote) -> String {
|
||||
format!("{issue_date}/{article_id}/{}", vote.as_str())
|
||||
}
|
||||
|
||||
/// `hex(hmac_sha256(secret, "{issue_date}/{article_id}/{vote}"))[..16]` (§3.9).
|
||||
pub fn rating_token(secret: &str, issue_date: Date, article_id: ArticleId, vote: Vote) -> String {
|
||||
// `Hmac` derives a fixed-size key from any input length, so this never fails.
|
||||
let mut mac = <Hmac<Sha256> as KeyInit>::new_from_slice(secret.as_bytes())
|
||||
.expect("HMAC accepts keys of any length");
|
||||
mac.update(rating_message(issue_date, article_id, vote).as_bytes());
|
||||
let digest = hex::encode(mac.finalize().into_bytes());
|
||||
digest[..TOKEN_LEN].to_string()
|
||||
}
|
||||
|
||||
/// Constant-time comparison of a supplied token against the expected one (§3.9).
|
||||
pub fn verify_token(
|
||||
secret: &str,
|
||||
issue_date: Date,
|
||||
article_id: ArticleId,
|
||||
vote: Vote,
|
||||
token: &str,
|
||||
) -> bool {
|
||||
constant_time_eq(
|
||||
rating_token(secret, issue_date, article_id, vote).as_bytes(),
|
||||
token.as_bytes(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Length-independent, data-independent byte comparison.
|
||||
///
|
||||
/// A tiny local implementation so the crate does not need `subtle` directly;
|
||||
/// `black_box` keeps the optimizer from short-circuiting the accumulate.
|
||||
pub fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
|
||||
let mut diff = (a.len() ^ b.len()) as u8;
|
||||
for i in 0..a.len().max(b.len()) {
|
||||
let x = a.get(i).copied().unwrap_or(0);
|
||||
let y = b.get(i).copied().unwrap_or(0);
|
||||
diff |= x ^ y;
|
||||
}
|
||||
std::hint::black_box(diff) == 0
|
||||
}
|
||||
|
||||
/// Full rating URL embedded in an article footer:
|
||||
/// `{public_url}/r/{date}/{article_id}/{up|down}?t={token}` (§3.9).
|
||||
pub fn rating_url(
|
||||
public_url: &str,
|
||||
secret: &str,
|
||||
issue_date: Date,
|
||||
article_id: ArticleId,
|
||||
vote: Vote,
|
||||
) -> String {
|
||||
let token = rating_token(secret, issue_date, article_id, vote);
|
||||
format!(
|
||||
"{}/r/{issue_date}/{article_id}/{}?t={token}",
|
||||
public_url.trim_end_matches('/'),
|
||||
vote.as_str()
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn date() -> Date {
|
||||
"2026-08-15".parse().expect("date")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_matches_the_shared_vector() {
|
||||
assert_eq!(rating_message(date(), 42, Vote::Up), "2026-08-15/42/up");
|
||||
assert_eq!(
|
||||
rating_token("test-secret", date(), 42, Vote::Up),
|
||||
"3b314cf7e6d8f50f"
|
||||
);
|
||||
assert_eq!(rating_token("test-secret", date(), 42, Vote::Up).len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_are_per_article_and_per_vote() {
|
||||
let up = rating_token("s", date(), 42, Vote::Up);
|
||||
assert_ne!(up, rating_token("s", date(), 42, Vote::Down));
|
||||
assert_ne!(up, rating_token("s", date(), 43, Vote::Up));
|
||||
assert_ne!(up, rating_token("other", date(), 42, Vote::Up));
|
||||
let tomorrow: Date = "2026-08-16".parse().unwrap();
|
||||
assert_ne!(up, rating_token("s", tomorrow, 42, Vote::Up));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_is_exact() {
|
||||
assert!(verify_token(
|
||||
"s",
|
||||
date(),
|
||||
42,
|
||||
Vote::Up,
|
||||
&rating_token("s", date(), 42, Vote::Up)
|
||||
));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, "deadbeefdeadbeef"));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, ""));
|
||||
assert!(!verify_token("s", date(), 42, Vote::Up, "short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_shape_matches_the_spec() {
|
||||
assert_eq!(
|
||||
rating_url(
|
||||
"https://daily.hallada.net/",
|
||||
"test-secret",
|
||||
date(),
|
||||
42,
|
||||
Vote::Up
|
||||
),
|
||||
"https://daily.hallada.net/r/2026-08-15/42/up?t=3b314cf7e6d8f50f"
|
||||
);
|
||||
}
|
||||
}
|
||||
+832
@@ -0,0 +1,832 @@
|
||||
//! Comment chapters: fetch trees for selected articles and render them (spec §3.7).
|
||||
//!
|
||||
//! Heuristic only, no LLM. Rendered as nested border-left indentation that reads
|
||||
//! well on e-ink (no color). Every fetch is best-effort: a platform that errors
|
||||
//! out is simply left out of the chapter (notes §3).
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use futures::StreamExt;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::epub::images::{text_escape, to_xhtml};
|
||||
use crate::types::{Comment, CommentThread, Discussion, Pick, SocialSource};
|
||||
|
||||
/// Top-level threads kept per source (§3.7).
|
||||
pub const MAX_TOP_LEVEL: usize = 8;
|
||||
/// Maximum nesting depth rendered: depths 0, 1 and 2 (§3.7).
|
||||
pub const MAX_DEPTH: usize = 3;
|
||||
/// Maximum children rendered per node (§3.7).
|
||||
pub const MAX_CHILDREN: usize = 4;
|
||||
/// Per-comment character cap before ellipsizing (§3.7).
|
||||
pub const MAX_COMMENT_CHARS: usize = 1200;
|
||||
/// Whole-chapter word cap (§3.7).
|
||||
pub const MAX_CHAPTER_WORDS: usize = 4000;
|
||||
/// Concurrent discussion fetches.
|
||||
pub const CONCURRENCY: usize = 4;
|
||||
|
||||
/// Platform order inside a discussion chapter: HN → Lobsters → Reddit (§3.7).
|
||||
pub const SOURCE_ORDER: &[SocialSource] = &[
|
||||
SocialSource::Hn,
|
||||
SocialSource::Lobsters,
|
||||
SocialSource::Reddit,
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fetching
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `https://hn.algolia.com/api/v1/items/{objectID}` (§3.7).
|
||||
pub fn hn_items_url(item_id: &str) -> String {
|
||||
format!("https://hn.algolia.com/api/v1/items/{item_id}")
|
||||
}
|
||||
|
||||
/// `https://lobste.rs/s/{id}.json` (§3.7).
|
||||
pub fn lobsters_url(item_id: &str) -> String {
|
||||
format!("https://lobste.rs/s/{item_id}.json")
|
||||
}
|
||||
|
||||
/// `https://www.reddit.com{permalink}.json?limit=100&depth=3&sort=top` (§3.7).
|
||||
pub fn reddit_url(permalink_or_url: &str) -> String {
|
||||
let base = permalink_or_url.trim_end_matches('/');
|
||||
let base = if base.starts_with("http://") || base.starts_with("https://") {
|
||||
base.to_string()
|
||||
} else if base.starts_with('/') {
|
||||
format!("https://www.reddit.com{base}")
|
||||
} else {
|
||||
format!("https://www.reddit.com/{base}")
|
||||
};
|
||||
let base = base.trim_end_matches(".json").to_string();
|
||||
format!("{base}.json?limit=100&depth=3&sort=top")
|
||||
}
|
||||
|
||||
async fn get_json(http: &reqwest::Client, url: &str) -> Option<Value> {
|
||||
let resp = http
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::debug!(url, "comment fetch failed: {e}"))
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
tracing::debug!(url, status = %resp.status(), "comment fetch rejected");
|
||||
return None;
|
||||
}
|
||||
resp.json::<Value>()
|
||||
.await
|
||||
.map_err(|e| tracing::debug!(url, "comment payload was not json: {e}"))
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Fetch and assemble the discussion for one selected article, ordered
|
||||
/// HN → Lobsters → Reddit (§3.7). Best-effort: returns `None` on failure.
|
||||
pub async fn fetch_discussion(http: &reqwest::Client, pick: &Pick) -> Option<Discussion> {
|
||||
let mut threads: Vec<CommentThread> = Vec::new();
|
||||
for source in SOURCE_ORDER {
|
||||
let Some(social) = pick.article.social.iter().find(|s| s.source == *source) else {
|
||||
continue;
|
||||
};
|
||||
let thread = match source {
|
||||
SocialSource::Hn => match social.item_id.as_deref() {
|
||||
Some(id) => get_json(http, &hn_items_url(id))
|
||||
.await
|
||||
.and_then(|v| parse_hn(&v)),
|
||||
None => None,
|
||||
},
|
||||
SocialSource::Lobsters => match social.item_id.as_deref() {
|
||||
Some(id) => get_json(http, &lobsters_url(id))
|
||||
.await
|
||||
.and_then(|v| parse_lobsters(&v)),
|
||||
None => None,
|
||||
},
|
||||
SocialSource::Reddit => {
|
||||
let target = social
|
||||
.item_url
|
||||
.as_deref()
|
||||
.or(social.item_id.as_deref())
|
||||
.map(reddit_url);
|
||||
match target {
|
||||
Some(url) => get_json(http, &url)
|
||||
.await
|
||||
.and_then(|v| parse_reddit(&v, "")),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
SocialSource::X => None,
|
||||
};
|
||||
match thread {
|
||||
Some(mut t) => {
|
||||
t.comments = truncate(t.comments);
|
||||
if !t.comments.is_empty() {
|
||||
threads.push(t);
|
||||
}
|
||||
}
|
||||
None => tracing::debug!(
|
||||
source = %source,
|
||||
article = pick.article.id,
|
||||
"no comment tree for this source"
|
||||
),
|
||||
}
|
||||
}
|
||||
if threads.is_empty() {
|
||||
return None;
|
||||
}
|
||||
enforce_chapter_budget(&mut threads);
|
||||
Some(Discussion {
|
||||
article_id: pick.article.id,
|
||||
chapter_id: format!("disc-{}", pick.article.best_entry_id),
|
||||
threads,
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetch discussions for every pick in parallel, filling [`Pick::discussion`] (§3.7).
|
||||
pub async fn fetch_all(http: &reqwest::Client, picks: &mut [Pick]) -> usize {
|
||||
let fetched: Vec<Option<Discussion>> = futures::stream::iter(picks.iter().map(|pick| {
|
||||
let http = http.clone();
|
||||
async move { fetch_discussion(&http, pick).await }
|
||||
}))
|
||||
.buffered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut count = 0;
|
||||
for (pick, discussion) in picks.iter_mut().zip(fetched) {
|
||||
if discussion.is_some() {
|
||||
count += 1;
|
||||
}
|
||||
pick.discussion = discussion;
|
||||
}
|
||||
tracing::info!(count, of = picks.len(), "fetched discussion chapters");
|
||||
count
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsing (pure — fixtures cover these, no network in tests)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse the Algolia `items/{id}` tree (§3.7).
|
||||
pub fn parse_hn(v: &Value) -> Option<CommentThread> {
|
||||
let id = v.get("id")?.as_i64()?;
|
||||
let mut comments = Vec::new();
|
||||
let mut total = 0i64;
|
||||
for child in v.get("children")?.as_array()?.iter() {
|
||||
if let Some(c) = hn_node(child, 0, &mut total) {
|
||||
comments.push(c);
|
||||
}
|
||||
}
|
||||
sort_by_points(&mut comments);
|
||||
Some(CommentThread {
|
||||
source: SocialSource::Hn,
|
||||
item_url: format!("https://news.ycombinator.com/item?id={id}"),
|
||||
total_comments: total,
|
||||
comments,
|
||||
})
|
||||
}
|
||||
|
||||
fn hn_node(v: &Value, depth: usize, total: &mut i64) -> Option<Comment> {
|
||||
let text = v.get("text").and_then(|t| t.as_str()).unwrap_or("");
|
||||
let author = v.get("author").and_then(|a| a.as_str()).unwrap_or("");
|
||||
let mut children = Vec::new();
|
||||
if let Some(kids) = v.get("children").and_then(|c| c.as_array()) {
|
||||
for kid in kids {
|
||||
if let Some(c) = hn_node(kid, depth + 1, total) {
|
||||
children.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
if text.is_empty() || author.is_empty() {
|
||||
// Dead/deleted node: keep its (live) replies by lifting them up.
|
||||
return children.into_iter().next();
|
||||
}
|
||||
*total += 1;
|
||||
sort_by_points(&mut children);
|
||||
Some(Comment {
|
||||
author: author.to_string(),
|
||||
points: v.get("points").and_then(|p| p.as_i64()),
|
||||
text_html: sanitize_comment(text),
|
||||
depth,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse `https://lobste.rs/s/{id}.json` — a flat list keyed by `indent_level` (§3.7).
|
||||
pub fn parse_lobsters(v: &Value) -> Option<CommentThread> {
|
||||
let short_id = v.get("short_id").and_then(|s| s.as_str()).unwrap_or("");
|
||||
let item_url = v
|
||||
.get("short_id_url")
|
||||
.and_then(|s| s.as_str())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| format!("https://lobste.rs/s/{short_id}"));
|
||||
let raw = v.get("comments").and_then(|c| c.as_array())?;
|
||||
|
||||
// Rebuild the tree from indent_level (1 = top level).
|
||||
let mut roots: Vec<Comment> = Vec::new();
|
||||
// Path of indices into the tree for the current branch.
|
||||
let mut path: Vec<usize> = Vec::new();
|
||||
let mut total = 0i64;
|
||||
for item in raw {
|
||||
let text = item
|
||||
.get("comment")
|
||||
.and_then(|c| c.as_str())
|
||||
.or_else(|| item.get("comment_plain").and_then(|c| c.as_str()))
|
||||
.unwrap_or("");
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let author = match item.get("commenting_user") {
|
||||
Some(Value::String(s)) => s.clone(),
|
||||
Some(Value::Object(o)) => o
|
||||
.get("username")
|
||||
.and_then(|u| u.as_str())
|
||||
.unwrap_or("someone")
|
||||
.to_string(),
|
||||
_ => "someone".to_string(),
|
||||
};
|
||||
let indent = item
|
||||
.get("indent_level")
|
||||
.and_then(|i| i.as_i64())
|
||||
.unwrap_or(1)
|
||||
.max(1) as usize;
|
||||
let depth = indent - 1;
|
||||
total += 1;
|
||||
let comment = Comment {
|
||||
author,
|
||||
points: item.get("score").and_then(|s| s.as_i64()),
|
||||
text_html: sanitize_comment(text),
|
||||
depth,
|
||||
children: Vec::new(),
|
||||
};
|
||||
path.truncate(depth);
|
||||
if depth == 0 || path.len() < depth {
|
||||
path.clear();
|
||||
roots.push(comment);
|
||||
path.push(roots.len() - 1);
|
||||
} else {
|
||||
let mut node = &mut roots[path[0]];
|
||||
for idx in &path[1..] {
|
||||
node = &mut node.children[*idx];
|
||||
}
|
||||
node.children.push(comment);
|
||||
let child_idx = node.children.len() - 1;
|
||||
path.push(child_idx);
|
||||
}
|
||||
}
|
||||
sort_by_points(&mut roots);
|
||||
let total_comments = v
|
||||
.get("comment_count")
|
||||
.and_then(|c| c.as_i64())
|
||||
.unwrap_or(total);
|
||||
Some(CommentThread {
|
||||
source: SocialSource::Lobsters,
|
||||
item_url,
|
||||
total_comments,
|
||||
comments: roots,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse `{permalink}.json` — `[post listing, comment listing]` (§3.7).
|
||||
pub fn parse_reddit(v: &Value, fallback_url: &str) -> Option<CommentThread> {
|
||||
let listings = v.as_array()?;
|
||||
let post = listings.first();
|
||||
let permalink = post
|
||||
.and_then(|l| l.pointer("/data/children/0/data/permalink"))
|
||||
.and_then(|p| p.as_str())
|
||||
.map(|p| format!("https://www.reddit.com{p}"))
|
||||
.unwrap_or_else(|| fallback_url.to_string());
|
||||
let declared = post
|
||||
.and_then(|l| l.pointer("/data/children/0/data/num_comments"))
|
||||
.and_then(|n| n.as_i64());
|
||||
|
||||
let children = listings
|
||||
.get(1)
|
||||
.and_then(|l| l.pointer("/data/children"))
|
||||
.and_then(|c| c.as_array())?;
|
||||
|
||||
let mut comments = Vec::new();
|
||||
let mut total = 0i64;
|
||||
for child in children {
|
||||
if let Some(c) = reddit_node(child, 0, &mut total) {
|
||||
comments.push(c);
|
||||
}
|
||||
}
|
||||
sort_by_points(&mut comments);
|
||||
Some(CommentThread {
|
||||
source: SocialSource::Reddit,
|
||||
item_url: permalink,
|
||||
total_comments: declared.unwrap_or(total),
|
||||
comments,
|
||||
})
|
||||
}
|
||||
|
||||
fn reddit_node(child: &Value, depth: usize, total: &mut i64) -> Option<Comment> {
|
||||
if child.get("kind").and_then(|k| k.as_str()) != Some("t1") {
|
||||
return None; // "more" placeholders and the post itself
|
||||
}
|
||||
let data = child.get("data")?;
|
||||
let author = data.get("author").and_then(|a| a.as_str()).unwrap_or("");
|
||||
let body = data
|
||||
.get("body_html")
|
||||
.and_then(|b| b.as_str())
|
||||
.map(unescape_entities)
|
||||
.or_else(|| {
|
||||
data.get("body")
|
||||
.and_then(|b| b.as_str())
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if author.is_empty() || author == "[deleted]" || body.is_empty() {
|
||||
return None;
|
||||
}
|
||||
*total += 1;
|
||||
let mut children = Vec::new();
|
||||
if let Some(replies) = data
|
||||
.get("replies")
|
||||
.and_then(|r| r.pointer("/data/children"))
|
||||
&& let Some(list) = replies.as_array()
|
||||
{
|
||||
for reply in list {
|
||||
if let Some(c) = reddit_node(reply, depth + 1, total) {
|
||||
children.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
sort_by_points(&mut children);
|
||||
Some(Comment {
|
||||
author: author.to_string(),
|
||||
points: data.get("score").and_then(|s| s.as_i64()),
|
||||
text_html: sanitize_comment(&body),
|
||||
depth,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
fn sort_by_points(comments: &mut [Comment]) {
|
||||
// Stable: platform ordering survives when scores are missing or equal.
|
||||
comments.sort_by_key(|c| std::cmp::Reverse(c.points.unwrap_or(0)));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sanitization and pruning
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Sanitize a comment body down to the small tag set the EPUB CSS styles (§3.7).
|
||||
pub fn sanitize_comment(html: &str) -> String {
|
||||
let tags: HashSet<&str> = [
|
||||
"p",
|
||||
"a",
|
||||
"em",
|
||||
"i",
|
||||
"strong",
|
||||
"b",
|
||||
"code",
|
||||
"pre",
|
||||
"blockquote",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"br",
|
||||
"del",
|
||||
"sup",
|
||||
"sub",
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let cleaned = ammonia::Builder::new()
|
||||
.tags(tags)
|
||||
.link_rel(None)
|
||||
.clean(html)
|
||||
.to_string();
|
||||
let trimmed = cleaned.trim();
|
||||
if trimmed.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
if trimmed.starts_with('<') {
|
||||
trimmed.to_string()
|
||||
} else {
|
||||
// HN comment bodies start with a bare text run.
|
||||
format!("<p>{trimmed}</p>")
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal HTML entity decode — Reddit double-escapes `body_html`.
|
||||
pub fn unescape_entities(s: &str) -> String {
|
||||
s.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("​", "")
|
||||
.replace(" ", " ")
|
||||
.replace("&", "&")
|
||||
}
|
||||
|
||||
/// Plain text of a markup fragment, used for length and word budgeting.
|
||||
pub fn strip_tags(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut in_tag = false;
|
||||
for c in html.chars() {
|
||||
match c {
|
||||
'<' => in_tag = true,
|
||||
'>' => in_tag = false,
|
||||
_ if !in_tag => out.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
unescape_entities(&out)
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
/// Ellipsize a comment body to `max_chars` of visible text (§3.7).
|
||||
pub fn ellipsize_html(html: &str, max_chars: usize) -> String {
|
||||
let text = strip_tags(html);
|
||||
if text.chars().count() <= max_chars {
|
||||
return html.to_string();
|
||||
}
|
||||
let mut kept: String = text.chars().take(max_chars).collect();
|
||||
// Prefer cutting on a word boundary.
|
||||
if let Some(idx) = kept.rfind(' ')
|
||||
&& idx > max_chars * 3 / 4
|
||||
{
|
||||
kept.truncate(idx);
|
||||
}
|
||||
format!("<p>{}…</p>", text_escape(kept.trim_end()))
|
||||
}
|
||||
|
||||
fn word_count(html: &str) -> usize {
|
||||
strip_tags(html).split_whitespace().count()
|
||||
}
|
||||
|
||||
/// Truncate a tree to the §3.7 limits: top threads by score, depth, children,
|
||||
/// per-comment length and whole-chapter word budget.
|
||||
pub fn truncate(comments: Vec<Comment>) -> Vec<Comment> {
|
||||
let mut roots: Vec<Comment> = comments;
|
||||
sort_by_points(&mut roots);
|
||||
roots.truncate(MAX_TOP_LEVEL);
|
||||
let mut pruned: Vec<Comment> = roots
|
||||
.into_iter()
|
||||
.map(|c| prune_node(c, 0))
|
||||
.filter(|c| !c.text_html.is_empty())
|
||||
.collect();
|
||||
let mut budget = MAX_CHAPTER_WORDS;
|
||||
trim_to_budget(&mut pruned, &mut budget);
|
||||
pruned
|
||||
}
|
||||
|
||||
fn prune_node(mut comment: Comment, depth: usize) -> Comment {
|
||||
comment.depth = depth;
|
||||
comment.text_html = ellipsize_html(&comment.text_html, MAX_COMMENT_CHARS);
|
||||
if depth + 1 >= MAX_DEPTH {
|
||||
comment.children = Vec::new();
|
||||
return comment;
|
||||
}
|
||||
let mut children = std::mem::take(&mut comment.children);
|
||||
sort_by_points(&mut children);
|
||||
children.truncate(MAX_CHILDREN);
|
||||
comment.children = children
|
||||
.into_iter()
|
||||
.map(|c| prune_node(c, depth + 1))
|
||||
.filter(|c| !c.text_html.is_empty())
|
||||
.collect();
|
||||
comment
|
||||
}
|
||||
|
||||
/// Drop comments (depth-first, in render order) once the word budget runs out.
|
||||
fn trim_to_budget(comments: &mut Vec<Comment>, budget: &mut usize) {
|
||||
let mut kept = Vec::with_capacity(comments.len());
|
||||
for mut comment in std::mem::take(comments) {
|
||||
let cost = word_count(&comment.text_html);
|
||||
if cost > *budget {
|
||||
break;
|
||||
}
|
||||
*budget -= cost;
|
||||
trim_to_budget(&mut comment.children, budget);
|
||||
kept.push(comment);
|
||||
}
|
||||
*comments = kept;
|
||||
}
|
||||
|
||||
/// Apply the whole-chapter word cap across every source in the chapter (§3.7).
|
||||
pub fn enforce_chapter_budget(threads: &mut Vec<CommentThread>) {
|
||||
let mut budget = MAX_CHAPTER_WORDS;
|
||||
for thread in threads.iter_mut() {
|
||||
trim_to_budget(&mut thread.comments, &mut budget);
|
||||
}
|
||||
threads.retain(|t| !t.comments.is_empty());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rendering
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Chapter title: "💬 Discussion: {title} ({N} comments on {source})" (§3.7).
|
||||
pub fn chapter_title(article_title: &str, discussion: &Discussion) -> String {
|
||||
let sources: Vec<&str> = discussion
|
||||
.threads
|
||||
.iter()
|
||||
.map(|t| t.source.display_name())
|
||||
.collect();
|
||||
let sources = if sources.is_empty() {
|
||||
"the web".to_string()
|
||||
} else {
|
||||
sources.join(", ")
|
||||
};
|
||||
let n = discussion.total_comments();
|
||||
let noun = if n == 1 { "comment" } else { "comments" };
|
||||
format!("\u{1f4ac} Discussion: {article_title} ({n} {noun} on {sources})")
|
||||
}
|
||||
|
||||
/// Render a discussion to sanitized XHTML for the EPUB (§3.7).
|
||||
pub fn render_xhtml(discussion: &Discussion, article_title: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for thread in &discussion.threads {
|
||||
out.push_str(&format!(
|
||||
" <h2 class=\"discussion-source\">{}</h2>\n",
|
||||
text_escape(&thread_heading(thread))
|
||||
));
|
||||
out.push_str(&format!(
|
||||
" <p class=\"discussion-link\"><a href=\"{}\">View the thread \u{2197}</a></p>\n",
|
||||
text_escape(&thread.item_url)
|
||||
));
|
||||
for comment in &thread.comments {
|
||||
render_comment(comment, 3, 0, &mut out);
|
||||
}
|
||||
}
|
||||
if out.is_empty() {
|
||||
out.push_str(&format!(
|
||||
" <p>No comments were available for {}.</p>\n",
|
||||
text_escape(article_title)
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn thread_heading(thread: &CommentThread) -> String {
|
||||
let noun = if thread.total_comments == 1 {
|
||||
"comment"
|
||||
} else {
|
||||
"comments"
|
||||
};
|
||||
format!(
|
||||
"{} \u{00b7} {} {}",
|
||||
thread.source.display_name(),
|
||||
thread.total_comments,
|
||||
noun
|
||||
)
|
||||
}
|
||||
|
||||
/// Tag a comment's paragraphs so the X4 can style them without a descendant
|
||||
/// selector (§3.10). [`sanitize_comment`] allows no attributes on `p`, so every
|
||||
/// paragraph in a comment body is exactly `<p>`.
|
||||
fn class_comment_paragraphs(html: &str) -> String {
|
||||
html.replace("<p>", "<p class=\"comment-line\">")
|
||||
}
|
||||
|
||||
/// `indent` is cosmetic whitespace; `depth` is the reply nesting level, 0 for a
|
||||
/// thread's top-level comments.
|
||||
fn render_comment(comment: &Comment, indent: usize, depth: usize, out: &mut String) {
|
||||
let pad = " ".repeat(indent * 2);
|
||||
// Nesting is carried as a class rather than left to a descendant selector:
|
||||
// the X4's CSS engine only understands `tag`, `.class` and `tag.class`
|
||||
// (§3.10), so `blockquote.comment blockquote.comment` never matches there.
|
||||
let class = if depth > 0 {
|
||||
"comment reply"
|
||||
} else {
|
||||
"comment"
|
||||
};
|
||||
out.push_str(&format!("{pad}<blockquote class=\"{class}\">\n"));
|
||||
let points = match comment.points {
|
||||
Some(p) => format!(" \u{00b7} {p} points"),
|
||||
None => String::new(),
|
||||
};
|
||||
out.push_str(&format!(
|
||||
"{pad} <p class=\"comment-meta\">{}{}</p>\n",
|
||||
text_escape(&comment.author),
|
||||
text_escape(&points)
|
||||
));
|
||||
out.push_str(&format!(
|
||||
"{pad} <div class=\"comment-body\">{}</div>\n",
|
||||
class_comment_paragraphs(&to_xhtml(&comment.text_html))
|
||||
));
|
||||
for child in &comment.children {
|
||||
render_comment(child, indent + 1, depth + 1, out);
|
||||
}
|
||||
out.push_str(&format!("{pad}</blockquote>\n"));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn leaf(author: &str, points: i64, text: &str) -> Comment {
|
||||
Comment {
|
||||
author: author.into(),
|
||||
points: Some(points),
|
||||
text_html: format!("<p>{text}</p>"),
|
||||
depth: 0,
|
||||
children: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(name: &str) -> Value {
|
||||
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(name);
|
||||
let raw = std::fs::read_to_string(&path).expect("fixture must exist");
|
||||
serde_json::from_str(&raw).expect("fixture must be json")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_the_hn_item_tree() {
|
||||
let thread = parse_hn(&fixture("hn_item.json")).expect("hn tree");
|
||||
assert_eq!(thread.source, SocialSource::Hn);
|
||||
assert_eq!(
|
||||
thread.item_url,
|
||||
"https://news.ycombinator.com/item?id=40100000"
|
||||
);
|
||||
assert_eq!(thread.total_comments, 4);
|
||||
// Highest-scoring root first.
|
||||
assert_eq!(thread.comments[0].author, "alice");
|
||||
assert_eq!(thread.comments[0].children.len(), 1);
|
||||
assert_eq!(thread.comments[0].children[0].author, "bob");
|
||||
assert!(thread.comments[0].text_html.contains("<p>"));
|
||||
// The deleted node is dropped but its live reply survives.
|
||||
assert!(thread.comments.iter().all(|c| !c.author.is_empty()));
|
||||
assert!(thread.comments.iter().any(|c| c.author == "dana"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_lobsters_indent_levels_into_a_tree() {
|
||||
let thread = parse_lobsters(&fixture("lobsters_story.json")).expect("lobsters tree");
|
||||
assert_eq!(thread.source, SocialSource::Lobsters);
|
||||
assert_eq!(thread.item_url, "https://lobste.rs/s/abcdef");
|
||||
assert_eq!(thread.total_comments, 4);
|
||||
assert_eq!(thread.comments.len(), 2);
|
||||
let top = &thread.comments[0];
|
||||
assert_eq!(top.author, "pushcx");
|
||||
assert_eq!(top.children.len(), 1);
|
||||
assert_eq!(top.children[0].children.len(), 1);
|
||||
assert_eq!(top.children[0].children[0].author, "third");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_reddit_listings_and_skips_more_stubs() {
|
||||
let thread = parse_reddit(&fixture("reddit_comments.json"), "").expect("reddit tree");
|
||||
assert_eq!(thread.source, SocialSource::Reddit);
|
||||
assert_eq!(
|
||||
thread.item_url,
|
||||
"https://www.reddit.com/r/rust/comments/abc/title/"
|
||||
);
|
||||
assert_eq!(thread.total_comments, 87);
|
||||
assert_eq!(thread.comments.len(), 2);
|
||||
assert_eq!(thread.comments[0].author, "ferris");
|
||||
assert_eq!(thread.comments[0].children.len(), 1);
|
||||
// body_html arrives entity-escaped and must decode into real markup.
|
||||
assert!(thread.comments[0].text_html.contains("<p>"));
|
||||
assert!(thread.comments[0].text_html.contains("borrow checker"));
|
||||
assert!(!thread.comments[0].text_html.contains("<p>"));
|
||||
assert!(thread.comments.iter().all(|c| c.author != "[deleted]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reddit_url_normalizes_permalinks() {
|
||||
assert_eq!(
|
||||
reddit_url("/r/rust/comments/abc/title/"),
|
||||
"https://www.reddit.com/r/rust/comments/abc/title.json?limit=100&depth=3&sort=top"
|
||||
);
|
||||
assert_eq!(
|
||||
reddit_url("https://www.reddit.com/r/rust/comments/abc/title"),
|
||||
"https://www.reddit.com/r/rust/comments/abc/title.json?limit=100&depth=3&sort=top"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizer_strips_scripts_and_wraps_bare_text() {
|
||||
let out = sanitize_comment("hello <script>alert(1)</script><b>world</b>");
|
||||
assert!(out.starts_with("<p>"));
|
||||
assert!(!out.contains("script"));
|
||||
assert!(out.contains("<b>world</b>"));
|
||||
assert_eq!(sanitize_comment("<p>kept</p>"), "<p>kept</p>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_applies_every_spec_limit() {
|
||||
let mut roots: Vec<Comment> = (0..12)
|
||||
.map(|i| leaf(&format!("u{i}"), i as i64, "word ".repeat(10).trim()))
|
||||
.collect();
|
||||
// Give the top root six children, each with children of their own.
|
||||
let mut deep = leaf("deep0", 100, "one");
|
||||
for i in 0..6 {
|
||||
let mut child = leaf(&format!("c{i}"), i as i64, "two");
|
||||
child.children.push(leaf("grandchild", 1, "three"));
|
||||
child.children[0].children.push(leaf("too-deep", 1, "four"));
|
||||
deep.children.push(child);
|
||||
}
|
||||
roots.push(deep);
|
||||
|
||||
let out = truncate(roots);
|
||||
assert_eq!(out.len(), MAX_TOP_LEVEL, "top-level threads capped");
|
||||
assert_eq!(out[0].author, "deep0", "sorted by score, best first");
|
||||
assert_eq!(out[0].children.len(), MAX_CHILDREN, "children capped");
|
||||
assert_eq!(out[0].children[0].depth, 1);
|
||||
assert_eq!(out[0].children[0].children.len(), 1);
|
||||
assert_eq!(out[0].children[0].children[0].depth, 2);
|
||||
assert!(
|
||||
out[0].children[0].children[0].children.is_empty(),
|
||||
"rendering stops at depth {MAX_DEPTH}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_comment_text_is_ellipsized() {
|
||||
let long = "lorem ipsum ".repeat(200);
|
||||
let comment = leaf("verbose", 5, &long);
|
||||
let out = truncate(vec![comment]);
|
||||
let text = strip_tags(&out[0].text_html);
|
||||
assert!(text.chars().count() <= MAX_COMMENT_CHARS + 1);
|
||||
assert!(out[0].text_html.ends_with("…</p>"));
|
||||
// Short comments are left untouched.
|
||||
assert_eq!(ellipsize_html("<p>short</p>", 100), "<p>short</p>");
|
||||
}
|
||||
|
||||
fn tree_words(comments: &[Comment]) -> usize {
|
||||
comments
|
||||
.iter()
|
||||
.map(|c| word_count(&c.text_html) + tree_words(&c.children))
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn tree_len(comments: &[Comment]) -> usize {
|
||||
comments.iter().map(|c| 1 + tree_len(&c.children)).sum()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapter_word_budget_is_enforced() {
|
||||
// Every comment ellipsizes to ~240 words, so a full 8×4×4 tree is far
|
||||
// over the 4,000-word chapter budget.
|
||||
let long = "word ".repeat(400);
|
||||
let roots: Vec<Comment> = (0..MAX_TOP_LEVEL)
|
||||
.map(|i| {
|
||||
let mut root = leaf(&format!("u{i}"), 100 - i as i64, &long);
|
||||
for j in 0..MAX_CHILDREN {
|
||||
let mut child = leaf(&format!("c{i}{j}"), 10, &long);
|
||||
child.children.push(leaf("grandchild", 1, &long));
|
||||
root.children.push(child);
|
||||
}
|
||||
root
|
||||
})
|
||||
.collect();
|
||||
let full = tree_len(&roots);
|
||||
|
||||
let out = truncate(roots);
|
||||
let total = tree_words(&out);
|
||||
assert!(total <= MAX_CHAPTER_WORDS, "{total} words is over budget");
|
||||
assert!(!out.is_empty());
|
||||
assert!(
|
||||
tree_len(&out) < full,
|
||||
"comments past the budget are dropped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_nested_blockquotes_and_a_title() {
|
||||
let mut root = leaf("alice", 42, "top level");
|
||||
root.children.push(leaf("bob", 3, "reply"));
|
||||
let discussion = Discussion {
|
||||
article_id: 7,
|
||||
chapter_id: "disc-1001".into(),
|
||||
threads: vec![CommentThread {
|
||||
source: SocialSource::Hn,
|
||||
item_url: "https://news.ycombinator.com/item?id=1".into(),
|
||||
total_comments: 210,
|
||||
comments: vec![root],
|
||||
}],
|
||||
};
|
||||
let xhtml = render_xhtml(&discussion, "A Title");
|
||||
assert!(xhtml.contains("HN \u{00b7} 210 comments"));
|
||||
assert!(xhtml.contains("alice \u{00b7} 42 points"));
|
||||
// Top-level comments and replies are distinguishable by class alone, so
|
||||
// the X4 needs no descendant selector to indent them (§3.10).
|
||||
assert_eq!(xhtml.matches("<blockquote class=\"comment\">").count(), 1);
|
||||
assert_eq!(
|
||||
xhtml
|
||||
.matches("<blockquote class=\"comment reply\">")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
xhtml.matches("</blockquote>").count(),
|
||||
2,
|
||||
"every blockquote is closed"
|
||||
);
|
||||
// Comment paragraphs carry their own class for the same reason.
|
||||
assert!(
|
||||
xhtml.contains("<p class=\"comment-line\">top level</p>"),
|
||||
"{xhtml}"
|
||||
);
|
||||
assert!(!xhtml.contains("<p>"), "an unclassed paragraph survived");
|
||||
assert_eq!(
|
||||
chapter_title("A Title", &discussion),
|
||||
"\u{1f4ac} Discussion: A Title (210 comments on HN)"
|
||||
);
|
||||
}
|
||||
}
|
||||
+471
@@ -0,0 +1,471 @@
|
||||
//! Typed configuration (spec §3.14).
|
||||
//!
|
||||
//! Load order, later wins: built-in defaults ← `config.toml` (path from `--config`,
|
||||
//! else `./config.toml` if present) ← `DAILY_EPUB_*` environment variables, where
|
||||
//! nesting is expressed with a double underscore (`DAILY_EPUB_MINIFLUX__API_KEY`).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use figment::Figment;
|
||||
use figment::providers::{Env, Format, Serialized, Toml};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Environment-variable prefix for every override (§3.14).
|
||||
pub const ENV_PREFIX: &str = "DAILY_EPUB_";
|
||||
/// Nesting separator inside env var names.
|
||||
pub const ENV_SPLIT: &str = "__";
|
||||
/// Default config file looked up when `--config` is not given.
|
||||
pub const DEFAULT_CONFIG_FILE: &str = "config.toml";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("failed to load configuration: {0}")]
|
||||
Figment(#[from] Box<figment::Error>),
|
||||
#[error("config file not found: {0}")]
|
||||
Missing(PathBuf),
|
||||
#[error("invalid configuration: {0}")]
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
impl From<figment::Error> for ConfigError {
|
||||
fn from(e: figment::Error) -> Self {
|
||||
ConfigError::Figment(Box::new(e))
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy/alternate env var for the rating-link HMAC key (spec §1).
|
||||
pub const ENV_SECRET_ALIAS: &str = "DAILY_EPUB_SECRET";
|
||||
|
||||
/// Root configuration document (§3.14).
|
||||
///
|
||||
/// Unknown *top-level* keys are ignored on purpose: the prefix `DAILY_EPUB_` is
|
||||
/// shared with plain operator env vars such as [`ENV_SECRET_ALIAS`].
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct Config {
|
||||
/// IANA tz used for day boundaries and `--date` (§3.14, notes §2).
|
||||
pub timezone: String,
|
||||
/// Ingest window size in hours (§3.1).
|
||||
pub lookback_hours: u32,
|
||||
/// How many articles the lineup should contain (§3.6 stage B).
|
||||
pub target_article_count: usize,
|
||||
/// How many articles survive the heuristic pre-filter (§3.5).
|
||||
pub prefilter_keep: usize,
|
||||
/// Days of published files kept in the publish dirs (§3.11).
|
||||
pub retention_days: u32,
|
||||
/// Hard cost ceiling per run (§3.6 guardrail).
|
||||
pub max_daily_usd: f64,
|
||||
/// Include the Wikipedia Current Events section (§3.8).
|
||||
pub world_briefing: bool,
|
||||
|
||||
/// SQLite file; parent dirs are created on open.
|
||||
pub database_path: PathBuf,
|
||||
/// Default artifact output directory (overridden by `generate --out`).
|
||||
pub out_dir: PathBuf,
|
||||
/// Scour interests OPML used to seed the taste profile (§3.6).
|
||||
pub interests_opml: PathBuf,
|
||||
|
||||
pub miniflux: MinifluxConfig,
|
||||
pub deepseek: DeepseekConfig,
|
||||
pub curation: CurationConfig,
|
||||
pub publish: PublishConfig,
|
||||
pub xtc: XtcConfig,
|
||||
pub server: ServerConfig,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
timezone: "America/New_York".into(),
|
||||
lookback_hours: 26,
|
||||
target_article_count: 20,
|
||||
prefilter_keep: 120,
|
||||
retention_days: 21,
|
||||
max_daily_usd: 2.0,
|
||||
world_briefing: true,
|
||||
database_path: PathBuf::from("/var/lib/daily-epub/daily-epub.db"),
|
||||
out_dir: PathBuf::from("/var/lib/daily-epub/out"),
|
||||
interests_opml: PathBuf::from("data/scour-interests.opml"),
|
||||
miniflux: MinifluxConfig::default(),
|
||||
deepseek: DeepseekConfig::default(),
|
||||
curation: CurationConfig::default(),
|
||||
publish: PublishConfig::default(),
|
||||
xtc: XtcConfig::default(),
|
||||
server: ServerConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[miniflux]` — API client settings (§3.1).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct MinifluxConfig {
|
||||
pub base_url: String,
|
||||
/// `X-Auth-Token`; supply via `DAILY_EPUB_MINIFLUX__API_KEY`.
|
||||
pub api_key: Option<String>,
|
||||
/// Page size for `GET /v1/entries` (Miniflux caps this at 250).
|
||||
pub page_limit: u32,
|
||||
}
|
||||
|
||||
impl Default for MinifluxConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base_url: "http://127.0.0.1:8082".into(),
|
||||
api_key: None,
|
||||
page_limit: 250,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[deepseek]` — LLM endpoint, model and pricing (§3.6, notes "verified facts").
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct DeepseekConfig {
|
||||
pub base_url: String,
|
||||
pub model: String,
|
||||
/// Supply via `DAILY_EPUB_DEEPSEEK__API_KEY`.
|
||||
pub api_key: Option<String>,
|
||||
/// Articles per stage-A scoring request (§3.6).
|
||||
pub score_batch_size: usize,
|
||||
pub score_temperature: f32,
|
||||
pub editorial_temperature: f32,
|
||||
/// USD per 1M cache-miss input tokens.
|
||||
pub price_input_per_mtok: f64,
|
||||
/// USD per 1M prefix-cache-hit input tokens.
|
||||
pub price_cached_input_per_mtok: f64,
|
||||
/// USD per 1M output tokens.
|
||||
pub price_output_per_mtok: f64,
|
||||
}
|
||||
|
||||
impl Default for DeepseekConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
base_url: "https://api.deepseek.com/v1".into(),
|
||||
model: "deepseek-v4-flash".into(),
|
||||
api_key: None,
|
||||
score_batch_size: 12,
|
||||
score_temperature: 0.3,
|
||||
editorial_temperature: 0.8,
|
||||
price_input_per_mtok: 0.14,
|
||||
price_cached_input_per_mtok: 0.0028,
|
||||
price_output_per_mtok: 0.28,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[curation]` — pre-filter and section palette (§3.5, §3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct CurationConfig {
|
||||
/// Miniflux feed ids or site URLs that can never be dropped (§3.5).
|
||||
pub always_include_feeds: Vec<String>,
|
||||
/// Hosts excluded outright (§3.5).
|
||||
pub blocked_domains: Vec<String>,
|
||||
/// Extra paywalled hosts, merged with [`crate::extract::DEFAULT_PAYWALL_DOMAINS`]
|
||||
/// by the extraction stage's `excerpt_only` heuristic (§3.3).
|
||||
pub paywall_domains: Vec<String>,
|
||||
/// The only section names the LLM may use (§3.6 stage B).
|
||||
pub sections: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for CurationConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
always_include_feeds: Vec::new(),
|
||||
blocked_domains: Vec::new(),
|
||||
paywall_domains: Vec::new(),
|
||||
sections: [
|
||||
"Top Stories",
|
||||
"Tech & Engineering",
|
||||
"Science & Space",
|
||||
"AI & Machine Learning",
|
||||
"Culture & Essays",
|
||||
"Boston & Local",
|
||||
"Niche Corner",
|
||||
"From the Blogroll",
|
||||
]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[publish]` — where finished artifacts land (§3.11).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct PublishConfig {
|
||||
/// BookOrbit "The Daily EPUB" library watched folder.
|
||||
pub bookorbit_dir: PathBuf,
|
||||
/// Directory served at `/files/xtc/`.
|
||||
pub xtc_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Default for PublishConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bookorbit_dir: PathBuf::from("/srv/bookorbit/libraries/daily-epub"),
|
||||
xtc_dir: PathBuf::from("/var/lib/daily-epub/xtc"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// XTC output flavour (§3.11).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum XtcFormat {
|
||||
/// 1-bit.
|
||||
Xtc,
|
||||
/// 2-bit grayscale — the default (better image quality).
|
||||
Xtch,
|
||||
}
|
||||
|
||||
impl XtcFormat {
|
||||
/// Value passed to the converter's `-f` flag.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
XtcFormat::Xtc => "xtc",
|
||||
XtcFormat::Xtch => "xtch",
|
||||
}
|
||||
}
|
||||
|
||||
/// File extension of the produced artifact.
|
||||
pub fn extension(self) -> &'static str {
|
||||
self.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// `[xtc]` — invocation of `epub-to-xtc-converter` (§3.11, notes "verified facts").
|
||||
///
|
||||
/// The converter has no global npm bin, so `command` + `args` form the prefix and
|
||||
/// the code appends `<input.epub> -o <output> -f <format>` (plus `-c <settings>`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct XtcConfig {
|
||||
pub enabled: bool,
|
||||
pub command: String,
|
||||
pub args: Vec<String>,
|
||||
pub format: XtcFormat,
|
||||
/// Optional settings JSON passed as `-c`.
|
||||
pub settings: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Default for XtcConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
command: "node".into(),
|
||||
args: vec![
|
||||
"/opt/epub-to-xtc-converter/cli/index.js".into(),
|
||||
"convert".into(),
|
||||
],
|
||||
format: XtcFormat::Xtch,
|
||||
settings: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `[server]` — axum listener and rating-link signing (§3.9, §3.12).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct ServerConfig {
|
||||
pub bind: String,
|
||||
/// Base URL rating links are built from.
|
||||
pub public_url: String,
|
||||
/// HMAC key for rating tokens; supply via `DAILY_EPUB_SERVER__HMAC_SECRET`.
|
||||
pub hmac_secret: Option<String>,
|
||||
/// Optional Basic auth for `/opds/xtc.xml` and `/files/xtc/`.
|
||||
pub basic_auth_user: Option<String>,
|
||||
pub basic_auth_pass: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for ServerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bind: "127.0.0.1:3499".into(),
|
||||
public_url: "https://daily.hallada.net".into(),
|
||||
hmac_secret: None,
|
||||
basic_auth_user: None,
|
||||
basic_auth_pass: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Build the figment layer stack. `path` is required to exist when explicit.
|
||||
fn figment(path: Option<&Path>, require_file: bool) -> Result<Figment, ConfigError> {
|
||||
let mut fig = Figment::from(Serialized::defaults(Config::default()));
|
||||
if let Some(p) = path {
|
||||
if require_file && !p.exists() {
|
||||
return Err(ConfigError::Missing(p.to_path_buf()));
|
||||
}
|
||||
if p.exists() {
|
||||
fig = fig.merge(Toml::file(p));
|
||||
}
|
||||
}
|
||||
Ok(fig.merge(Env::prefixed(ENV_PREFIX).split(ENV_SPLIT)))
|
||||
}
|
||||
|
||||
/// Load config for the CLI: explicit `--config` path, else `./config.toml`
|
||||
/// when it exists, then `DAILY_EPUB_*` env overrides (§3.14).
|
||||
pub fn load(explicit: Option<&Path>) -> Result<Self, ConfigError> {
|
||||
let (path, require) = match explicit {
|
||||
Some(p) => (Some(p.to_path_buf()), true),
|
||||
None => (Some(PathBuf::from(DEFAULT_CONFIG_FILE)), false),
|
||||
};
|
||||
let mut config: Config = Self::figment(path.as_deref(), require)?.extract()?;
|
||||
// §1 tells the operator to set `DAILY_EPUB_SECRET`; §3.14 calls the key
|
||||
// `server.hmac_secret`. Accept both, with the explicit key winning.
|
||||
if config.server.hmac_secret.is_none() {
|
||||
config.server.hmac_secret = std::env::var(ENV_SECRET_ALIAS)
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty());
|
||||
}
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// Cheap sanity checks so misconfiguration fails at startup, not mid-run.
|
||||
pub fn validate(&self) -> Result<(), ConfigError> {
|
||||
if self.lookback_hours == 0 {
|
||||
return Err(ConfigError::Invalid("lookback_hours must be > 0".into()));
|
||||
}
|
||||
if self.target_article_count == 0 {
|
||||
return Err(ConfigError::Invalid(
|
||||
"target_article_count must be > 0".into(),
|
||||
));
|
||||
}
|
||||
if self.prefilter_keep < self.target_article_count {
|
||||
return Err(ConfigError::Invalid(
|
||||
"prefilter_keep must be >= target_article_count".into(),
|
||||
));
|
||||
}
|
||||
if self.curation.sections.is_empty() {
|
||||
return Err(ConfigError::Invalid(
|
||||
"curation.sections must not be empty".into(),
|
||||
));
|
||||
}
|
||||
self.tz()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve [`Config::timezone`] into a `jiff` time zone (notes §2).
|
||||
pub fn tz(&self) -> Result<jiff::tz::TimeZone, ConfigError> {
|
||||
jiff::tz::TimeZone::get(&self.timezone)
|
||||
.map_err(|e| ConfigError::Invalid(format!("unknown timezone {}: {e}", self.timezone)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use figment::Jail;
|
||||
|
||||
#[test]
|
||||
fn defaults_match_the_spec() {
|
||||
let c = Config::default();
|
||||
assert_eq!(c.timezone, "America/New_York");
|
||||
assert_eq!(c.lookback_hours, 26);
|
||||
assert_eq!(c.target_article_count, 20);
|
||||
assert_eq!(c.prefilter_keep, 120);
|
||||
assert_eq!(c.retention_days, 21);
|
||||
assert_eq!(c.max_daily_usd, 2.0);
|
||||
assert!(c.world_briefing);
|
||||
assert_eq!(c.deepseek.model, "deepseek-v4-flash");
|
||||
assert_eq!(c.xtc.format, XtcFormat::Xtch);
|
||||
assert_eq!(c.curation.sections.len(), 8);
|
||||
c.validate().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
// `Jail::expect_with` dictates the closure's `figment::Error` return type.
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn toml_then_env_layering() {
|
||||
Jail::expect_with(|jail| {
|
||||
jail.create_file(
|
||||
"config.toml",
|
||||
r#"
|
||||
lookback_hours = 30
|
||||
world_briefing = false
|
||||
|
||||
[miniflux]
|
||||
base_url = "http://127.0.0.1:9999"
|
||||
|
||||
[curation]
|
||||
sections = ["Top Stories", "Niche Corner"]
|
||||
|
||||
[xtc]
|
||||
command = "node"
|
||||
args = ["/opt/epub-to-xtc-converter/cli/index.js", "convert"]
|
||||
format = "xtc"
|
||||
"#,
|
||||
)?;
|
||||
jail.set_env("DAILY_EPUB_MINIFLUX__API_KEY", "secret-token");
|
||||
jail.set_env("DAILY_EPUB_TARGET_ARTICLE_COUNT", "12");
|
||||
jail.set_env("DAILY_EPUB_SERVER__HMAC_SECRET", "hunter2");
|
||||
|
||||
let c = Config::load(None).map_err(|e| figment::Error::from(e.to_string()))?;
|
||||
// from file
|
||||
assert_eq!(c.lookback_hours, 30);
|
||||
assert!(!c.world_briefing);
|
||||
assert_eq!(c.miniflux.base_url, "http://127.0.0.1:9999");
|
||||
assert_eq!(c.curation.sections, ["Top Stories", "Niche Corner"]);
|
||||
assert_eq!(c.xtc.format, XtcFormat::Xtc);
|
||||
assert_eq!(c.xtc.args.len(), 2);
|
||||
// from env
|
||||
assert_eq!(c.miniflux.api_key.as_deref(), Some("secret-token"));
|
||||
assert_eq!(c.target_article_count, 12);
|
||||
assert_eq!(c.server.hmac_secret.as_deref(), Some("hunter2"));
|
||||
// untouched default
|
||||
assert_eq!(c.retention_days, 21);
|
||||
assert_eq!(c.timezone, "America/New_York");
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_missing_path_is_an_error() {
|
||||
assert!(matches!(
|
||||
Config::load(Some(Path::new("/nonexistent/daily-epub.toml"))),
|
||||
Err(ConfigError::Missing(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shipped_example_config_parses() {
|
||||
let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("config.example.toml");
|
||||
let c = Config::load(Some(&example)).expect("config.example.toml must parse");
|
||||
assert_eq!(c.xtc.command, "node");
|
||||
assert_eq!(c.xtc.format, XtcFormat::Xtch);
|
||||
assert_eq!(c.server.bind, "127.0.0.1:3499");
|
||||
assert_eq!(c.deepseek.base_url, "https://api.deepseek.com/v1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_rejects_nonsense() {
|
||||
assert!(
|
||||
Config {
|
||||
prefilter_keep: 5,
|
||||
..Config::default()
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Config {
|
||||
timezone: "Mars/Olympus_Mons".into(),
|
||||
..Config::default()
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Config {
|
||||
lookback_hours: 0,
|
||||
..Config::default()
|
||||
}
|
||||
.validate()
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,634 @@
|
||||
//! Stage C — summaries, section intros and the front page (spec §3.6).
|
||||
//!
|
||||
//! Voice: warm, literate, a little playful; never fabricates facts that are not
|
||||
//! present in the summaries.
|
||||
//!
|
||||
//! Everything here is best-effort. If the cost ceiling trips mid-way (§3.6) or a
|
||||
//! call fails, the affected article silently falls back to its own opening words
|
||||
//! and the run continues — an issue with plain excerpts is far better than no
|
||||
//! issue at all.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::llm::{LlmClient, LlmError};
|
||||
use super::{escape_html, html_to_text, text_to_paragraphs, truncate_tokens, truncate_words};
|
||||
use crate::types::{ArticleId, Editorial, Lineup, Pick};
|
||||
|
||||
/// Article text is truncated to roughly this many tokens per summary call (§3.6).
|
||||
pub const SUMMARY_INPUT_TOKEN_BUDGET: usize = 5000;
|
||||
/// Target length of the "From the Editor" front page, in words (§3.6).
|
||||
pub const FRONT_PAGE_WORDS: (usize, usize) = (250, 400);
|
||||
/// Words of body text used when a summary has to fall back to the excerpt.
|
||||
pub const FALLBACK_SUMMARY_WORDS: usize = 45;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prompts (reusable instructions here; per-call material in the user message)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-article summary instructions (§3.6 stage C).
|
||||
pub const SUMMARY_INSTRUCTIONS: &str = "\
|
||||
TASK: write the newspaper abstract for one article in today's issue.
|
||||
|
||||
Two or three sentences, 40–70 words, present tense, third person. It runs under \
|
||||
the headline in the \"In This Issue\" page, so the reader decides from it alone \
|
||||
whether to open the piece.
|
||||
|
||||
DO
|
||||
- Say what the article actually argues, reports or builds — the specific claim, \
|
||||
number, method or story, not the topic.
|
||||
- Add the one detail that makes it worth his time: the surprising result, the \
|
||||
scale, the person involved, the unusual method.
|
||||
- Match the piece's register: a technical post-mortem gets a technical abstract, \
|
||||
an essay gets an essayistic one.
|
||||
- Stay strictly inside the supplied text.
|
||||
|
||||
DO NOT
|
||||
- Tease (\"you won't believe what happens next\"), moralize, or address the \
|
||||
reader as \"you\".
|
||||
- Open with \"This article…\", \"The author…\", \"In this post…\", or repeat the \
|
||||
headline's words.
|
||||
- Invent facts, names, numbers or conclusions that are not in the text. If the \
|
||||
text is a truncated excerpt, summarize only what is there and say it is an \
|
||||
excerpt.
|
||||
- Recommend, rate or editorialize — that is the front page's job.
|
||||
|
||||
Return JSON exactly: {\"summary\": \"<two or three sentences>\"}";
|
||||
|
||||
/// Front-page + section-intro instructions (§3.6 stage C).
|
||||
pub const FRONT_PAGE_INSTRUCTIONS: &str = "\
|
||||
TASK: write the front page of today's issue of The Daily EPUB.
|
||||
|
||||
You are given the whole lineup: sections, headlines, sources and the abstract \
|
||||
written for each article. Everything you write must come from those abstracts — \
|
||||
you have not read the articles themselves, and inventing a fact would be worse \
|
||||
than saying less.
|
||||
|
||||
Produce two things.
|
||||
|
||||
1. \"from_the_editor\" — 250 to 400 words of prose addressed to the paper's one \
|
||||
reader. Find the two or three threads that actually run through today's lineup \
|
||||
(a shared question, an argument between two pieces, an accidental theme) and use \
|
||||
them to guide the read: what to start with over coffee, what to save for the \
|
||||
commute, what rewards patience. Name the lead story and say why it leads. It is \
|
||||
fine — good, even — to note when a day is quiet or lopsided. Voice: warm, \
|
||||
literate, lightly playful, never breathless; a real editor writing to someone \
|
||||
whose taste he knows. No bullet lists, no headings, no emoji, 2–4 paragraphs \
|
||||
separated by a blank line.
|
||||
|
||||
2. \"section_intros\" — for EACH section name given below, two or three \
|
||||
sentences (35–60 words) introducing what is in it today. Concrete, specific to \
|
||||
these articles, no filler like \"a variety of interesting stories\". Use the \
|
||||
section names exactly as spelled in the lineup.
|
||||
|
||||
Return JSON exactly:
|
||||
{\"from_the_editor\": \"<paragraphs separated by \\n\\n>\", \
|
||||
\"section_intros\": {\"<section name>\": \"<2-3 sentences>\"}}";
|
||||
|
||||
/// The single front-page call's JSON response (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct FrontPageResponse {
|
||||
/// "From the Editor", 250–400 words.
|
||||
pub from_the_editor: String,
|
||||
/// Section name → 2–3 sentence intro.
|
||||
#[serde(default)]
|
||||
pub section_intros: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
/// The per-article summary call's JSON response.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
struct SummaryResponse {
|
||||
#[serde(default)]
|
||||
summary: String,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-article summaries
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One 2–3 sentence newspaper abstract: what it argues, why it's worth reading (§3.6).
|
||||
pub async fn summarize_article(
|
||||
llm: &LlmClient,
|
||||
title: &str,
|
||||
body_html: &str,
|
||||
temperature: f32,
|
||||
) -> Result<String, LlmError> {
|
||||
llm.meter.check_budget()?;
|
||||
let body = truncate_tokens(&html_to_text(body_html), SUMMARY_INPUT_TOKEN_BUDGET);
|
||||
let mut prompt = String::with_capacity(body.len() + SUMMARY_INSTRUCTIONS.len() + 256);
|
||||
prompt.push_str(SUMMARY_INSTRUCTIONS);
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nHEADLINE: {}\n\nARTICLE TEXT{}:\n{}\n",
|
||||
title.trim(),
|
||||
if body.ends_with('…') {
|
||||
" (truncated for length)"
|
||||
} else {
|
||||
""
|
||||
},
|
||||
if body.is_empty() {
|
||||
"(no body text was extracted; summarize from the headline alone and say the \
|
||||
full text was unavailable)"
|
||||
} else {
|
||||
&body
|
||||
}
|
||||
);
|
||||
let response: SummaryResponse = llm.complete_json(&prompt, temperature).await?;
|
||||
let summary = response.summary.trim().to_string();
|
||||
if summary.is_empty() {
|
||||
return Err(LlmError::EmptyResponse);
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Summarize every pick, returning `article_id → summary` (§3.6).
|
||||
///
|
||||
/// Stops early and returns what it has when the cost guardrail trips (§3.6).
|
||||
pub async fn summarize_all(
|
||||
llm: &LlmClient,
|
||||
lineup: &Lineup,
|
||||
temperature: f32,
|
||||
) -> BTreeMap<ArticleId, String> {
|
||||
let mut out = BTreeMap::new();
|
||||
for (n, pick) in lineup.picks.iter().enumerate() {
|
||||
if llm.meter.budget_exceeded() {
|
||||
tracing::error!(
|
||||
summarized = out.len(),
|
||||
remaining = lineup.picks.len() - n,
|
||||
spent_usd = llm.meter.cost_usd(),
|
||||
"COST CEILING HIT during stage C — the remaining articles fall back to \
|
||||
feed excerpts as summaries"
|
||||
);
|
||||
break;
|
||||
}
|
||||
match summarize_article(
|
||||
llm,
|
||||
&pick.article.title,
|
||||
&pick.article.content_html,
|
||||
temperature,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(summary) => {
|
||||
out.insert(pick.article.id, summary);
|
||||
}
|
||||
Err(LlmError::BudgetExceeded { spent, limit }) => {
|
||||
tracing::error!(spent, limit, "COST CEILING HIT during stage C");
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
article_id = pick.article.id,
|
||||
title = %pick.article.title,
|
||||
error = %e,
|
||||
"summary failed; falling back to the article's own opening"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!(
|
||||
summarized = out.len(),
|
||||
picks = lineup.picks.len(),
|
||||
"stage C summaries complete"
|
||||
);
|
||||
out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Front page
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The single front-page + section-intro call (§3.6).
|
||||
pub async fn front_page(
|
||||
llm: &LlmClient,
|
||||
lineup: &Lineup,
|
||||
summaries: &BTreeMap<ArticleId, String>,
|
||||
temperature: f32,
|
||||
) -> Result<FrontPageResponse, LlmError> {
|
||||
llm.meter.check_budget()?;
|
||||
let prompt = build_front_page_prompt(lineup, summaries);
|
||||
tracing::debug!(
|
||||
approx_tokens = super::approx_tokens(&prompt),
|
||||
"stage C front-page request"
|
||||
);
|
||||
let mut response: FrontPageResponse = llm.complete_json(&prompt, temperature).await?;
|
||||
response.from_the_editor = response.from_the_editor.trim().to_string();
|
||||
if response.from_the_editor.is_empty() {
|
||||
return Err(LlmError::EmptyResponse);
|
||||
}
|
||||
// Keep only intros for sections that actually exist in the issue.
|
||||
response
|
||||
.section_intros
|
||||
.retain(|name, text| lineup.section_order.contains(name) && !text.trim().is_empty());
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Render the front-page user prompt: the whole lineup with its abstracts (§3.6).
|
||||
pub fn build_front_page_prompt(lineup: &Lineup, summaries: &BTreeMap<ArticleId, String>) -> String {
|
||||
let mut prompt = String::with_capacity(4096);
|
||||
prompt.push_str(FRONT_PAGE_INSTRUCTIONS);
|
||||
let minutes: i64 = lineup
|
||||
.picks
|
||||
.iter()
|
||||
.map(|p| p.article.reading_minutes())
|
||||
.sum();
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nISSUE: {} · {} articles across {} sections · about {} minutes of reading\n\
|
||||
SECTIONS, in order: {}\n\nLINEUP\n",
|
||||
lineup.date,
|
||||
lineup.picks.len(),
|
||||
lineup.section_order.len(),
|
||||
minutes,
|
||||
lineup.section_order.join(" | ")
|
||||
);
|
||||
for section in &lineup.section_order {
|
||||
let _ = write!(prompt, "\n## {section}\n");
|
||||
for pick in lineup.section_picks(section) {
|
||||
let _ = write!(prompt, "{}", render_pick(pick, summaries));
|
||||
}
|
||||
}
|
||||
prompt
|
||||
}
|
||||
|
||||
fn render_pick(pick: &Pick, summaries: &BTreeMap<ArticleId, String>) -> String {
|
||||
let a = &pick.article;
|
||||
let mut block = String::with_capacity(400);
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"\n- {}{}",
|
||||
a.title.trim(),
|
||||
if pick.is_lead { " [LEAD STORY]" } else { "" }
|
||||
);
|
||||
let _ = writeln!(
|
||||
block,
|
||||
" source: {} · {} words (~{} min){}",
|
||||
if a.feed_title.is_empty() {
|
||||
"unknown"
|
||||
} else {
|
||||
a.feed_title.trim()
|
||||
},
|
||||
a.word_count,
|
||||
a.reading_minutes(),
|
||||
social_note(pick)
|
||||
);
|
||||
let abstract_text = summaries
|
||||
.get(&a.id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| excerpt_summary(pick));
|
||||
let _ = writeln!(block, " abstract: {abstract_text}");
|
||||
block
|
||||
}
|
||||
|
||||
fn social_note(pick: &Pick) -> String {
|
||||
if pick.article.social.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let parts: Vec<String> = pick
|
||||
.article
|
||||
.social
|
||||
.iter()
|
||||
.map(|s| {
|
||||
format!(
|
||||
"{} {} pts/{} comments",
|
||||
s.source.display_name(),
|
||||
s.score,
|
||||
s.num_comments
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
format!(" · {}", parts.join(", "))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fallbacks (§3.6, notes §6)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The article's own opening words, used when no LLM summary exists (§3.6).
|
||||
pub fn excerpt_summary(pick: &Pick) -> String {
|
||||
let text = truncate_words(
|
||||
&html_to_text(&pick.article.content_html),
|
||||
FALLBACK_SUMMARY_WORDS,
|
||||
);
|
||||
if text.is_empty() {
|
||||
format!(
|
||||
"From {}. (No preview text was available; open the article to read it.)",
|
||||
if pick.article.feed_title.is_empty() {
|
||||
"an unknown feed"
|
||||
} else {
|
||||
pick.article.feed_title.trim()
|
||||
}
|
||||
)
|
||||
} else {
|
||||
text
|
||||
}
|
||||
}
|
||||
|
||||
/// A plain, factual front page used when the model is unavailable (§3.6, notes §6).
|
||||
pub fn fallback_front_page_html(lineup: &Lineup) -> String {
|
||||
let minutes: i64 = lineup
|
||||
.picks
|
||||
.iter()
|
||||
.map(|p| p.article.reading_minutes())
|
||||
.sum();
|
||||
let mut text = format!(
|
||||
"Today's issue collects {} articles across {} sections — about {} minutes of \
|
||||
reading. Editorial notes are unavailable for this issue, so the lineup speaks \
|
||||
for itself.",
|
||||
lineup.picks.len(),
|
||||
lineup.section_order.len(),
|
||||
minutes
|
||||
);
|
||||
if let Some(lead) = lineup.lead() {
|
||||
let _ = write!(
|
||||
text,
|
||||
"\n\nLeading today: “{}” ({}).",
|
||||
lead.article.title.trim(),
|
||||
if lead.article.feed_title.is_empty() {
|
||||
"source unknown"
|
||||
} else {
|
||||
lead.article.feed_title.trim()
|
||||
}
|
||||
);
|
||||
}
|
||||
if !lineup.section_order.is_empty() {
|
||||
let _ = write!(
|
||||
text,
|
||||
"\n\nIn this issue: {}.",
|
||||
lineup.section_order.join(", ")
|
||||
);
|
||||
}
|
||||
text_to_paragraphs(&text)
|
||||
}
|
||||
|
||||
/// `--skip-llm` / budget-exceeded fallback: feed excerpts stand in for summaries
|
||||
/// and the front page is a plain stats line (§3.6, notes §6).
|
||||
pub fn fallback_editorial(lineup: &Lineup) -> Editorial {
|
||||
Editorial {
|
||||
front_page_html: fallback_front_page_html(lineup),
|
||||
section_intros: BTreeMap::new(),
|
||||
summaries: lineup
|
||||
.picks
|
||||
.iter()
|
||||
.map(|pick| (pick.article.id, excerpt_summary(pick)))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Stage C end to end: summaries, then one front-page call, with excerpts filling
|
||||
/// every gap (§3.6).
|
||||
pub async fn run(llm: &LlmClient, lineup: &Lineup, temperature: f32) -> Editorial {
|
||||
if lineup.picks.is_empty() {
|
||||
return fallback_editorial(lineup);
|
||||
}
|
||||
|
||||
let mut summaries = summarize_all(llm, lineup, temperature).await;
|
||||
let missing: Vec<&Pick> = lineup
|
||||
.picks
|
||||
.iter()
|
||||
.filter(|p| !summaries.contains_key(&p.article.id))
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
tracing::warn!(
|
||||
count = missing.len(),
|
||||
"using feed excerpts as summaries for articles the model did not cover"
|
||||
);
|
||||
for pick in missing {
|
||||
summaries.insert(pick.article.id, excerpt_summary(pick));
|
||||
}
|
||||
}
|
||||
|
||||
let (front_page_html, section_intros) =
|
||||
match front_page(llm, lineup, &summaries, temperature).await {
|
||||
Ok(response) => (
|
||||
text_to_paragraphs(&response.from_the_editor),
|
||||
response.section_intros,
|
||||
),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e,
|
||||
"front-page generation failed; using the plain front page");
|
||||
(fallback_front_page_html(lineup), BTreeMap::new())
|
||||
}
|
||||
};
|
||||
|
||||
Editorial {
|
||||
front_page_html,
|
||||
section_intros,
|
||||
summaries,
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape-and-wrap helper for callers rendering a summary straight into XHTML.
|
||||
pub fn summary_to_html(summary: &str) -> String {
|
||||
format!("<p>{}</p>", escape_html(summary.trim()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DeepseekConfig;
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::curate::prefilter::tests::article;
|
||||
use crate::types::TokenUsage;
|
||||
use std::sync::Arc;
|
||||
|
||||
const FRONT_PAGE_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_front_page.json"
|
||||
));
|
||||
|
||||
fn pick(id: i64, title: &str, section: &str, is_lead: bool) -> Pick {
|
||||
let mut a = article(id, title, 900);
|
||||
a.content_html = format!("<p>{title} opens with a specific, concrete claim.</p>");
|
||||
Pick {
|
||||
article: a,
|
||||
section: section.into(),
|
||||
position: 1,
|
||||
is_lead,
|
||||
summary: None,
|
||||
llm: None,
|
||||
discussion: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lineup() -> Lineup {
|
||||
Lineup {
|
||||
date: "2026-08-15".parse().expect("date"),
|
||||
picks: vec![
|
||||
pick(1, "Migrating 40TB off Postgres", "Top Stories", true),
|
||||
pick(2, "The MBTA slow-zone dataset", "Boston & Local", false),
|
||||
],
|
||||
section_order: vec!["Top Stories".into(), "Boston & Local".into()],
|
||||
}
|
||||
}
|
||||
|
||||
fn client(backend: Arc<MockBackend>, limit: f64) -> LlmClient {
|
||||
LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), limit),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn summary_prompt_carries_headline_and_truncated_body() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"summary": "A team moves 40TB of relational data off Postgres and documents every rollback."}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let body = format!("<p>{}</p>", "word ".repeat(20_000));
|
||||
let summary = summarize_article(&llm, "Migrating 40TB", &body, 0.8)
|
||||
.await
|
||||
.expect("summary");
|
||||
assert!(summary.starts_with("A team moves 40TB"));
|
||||
|
||||
let prompt = &backend.prompts()[0].user;
|
||||
assert!(prompt.starts_with(SUMMARY_INSTRUCTIONS));
|
||||
assert!(prompt.contains("HEADLINE: Migrating 40TB"));
|
||||
assert!(prompt.contains("(truncated for length)"));
|
||||
// ~5k tokens ≈ 20k characters of body, not the full 100k.
|
||||
assert!(prompt.len() < 26_000, "prompt was {} bytes", prompt.len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn front_page_parses_and_filters_unknown_sections() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(FRONT_PAGE_FIXTURE, TokenUsage::default());
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let lineup = lineup();
|
||||
let summaries = BTreeMap::from([
|
||||
(1, "A migration story with numbers.".to_string()),
|
||||
(2, "Transit data, charted.".to_string()),
|
||||
]);
|
||||
|
||||
let response = front_page(&llm, &lineup, &summaries, 0.8)
|
||||
.await
|
||||
.expect("front page");
|
||||
assert!(response.from_the_editor.split_whitespace().count() > 40);
|
||||
assert_eq!(response.section_intros.len(), 2);
|
||||
assert!(response.section_intros.contains_key("Top Stories"));
|
||||
assert!(
|
||||
!response.section_intros.contains_key("Niche Corner"),
|
||||
"intros for absent sections are dropped"
|
||||
);
|
||||
|
||||
let prompt = &backend.prompts()[0].user;
|
||||
assert!(prompt.starts_with(FRONT_PAGE_INSTRUCTIONS));
|
||||
assert!(prompt.contains("## Top Stories"));
|
||||
assert!(prompt.contains("[LEAD STORY]"));
|
||||
assert!(prompt.contains("abstract: A migration story with numbers."));
|
||||
assert!(prompt.contains("2026-08-15"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_stage_c_produces_summaries_intros_and_front_page() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(r#"{"summary": "First abstract."}"#, TokenUsage::default());
|
||||
backend.push(r#"{"summary": "Second abstract."}"#, TokenUsage::default());
|
||||
backend.push(FRONT_PAGE_FIXTURE, TokenUsage::default());
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
let editorial = run(&llm, &lineup(), 0.8).await;
|
||||
assert_eq!(
|
||||
backend.calls(),
|
||||
3,
|
||||
"one call per article plus the front page"
|
||||
);
|
||||
assert_eq!(editorial.summaries.len(), 2);
|
||||
assert_eq!(editorial.summaries[&1], "First abstract.");
|
||||
assert!(editorial.front_page_html.starts_with("<p>"));
|
||||
assert!(editorial.front_page_html.contains("</p>"));
|
||||
assert!(!editorial.front_page_html.contains("<script"));
|
||||
assert_eq!(editorial.section_intros.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn budget_exhaustion_degrades_to_excerpts() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
// The first summary alone blows a $0.05 ceiling.
|
||||
backend.push(
|
||||
r#"{"summary": "The one summary we could afford."}"#,
|
||||
TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 0,
|
||||
output_tokens: 0,
|
||||
},
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 0.05);
|
||||
|
||||
let editorial = run(&llm, &lineup(), 0.8).await;
|
||||
assert_eq!(backend.calls(), 1, "no further calls after the ceiling");
|
||||
assert!(llm.meter.budget_exceeded());
|
||||
assert_eq!(
|
||||
editorial.summaries.len(),
|
||||
2,
|
||||
"every pick still has a summary"
|
||||
);
|
||||
assert_eq!(editorial.summaries[&1], "The one summary we could afford.");
|
||||
assert!(
|
||||
editorial.summaries[&2].contains("opens with a specific"),
|
||||
"second summary fell back to the excerpt: {}",
|
||||
editorial.summaries[&2]
|
||||
);
|
||||
// The front page degraded to the plain version.
|
||||
assert!(editorial.front_page_html.contains("2 articles"));
|
||||
assert!(editorial.section_intros.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_summary_call_is_not_fatal() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push_error("400 bad request");
|
||||
backend.push(r#"{"summary": "Second abstract."}"#, TokenUsage::default());
|
||||
backend.push_error("500 front page exploded");
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
let editorial = run(&llm, &lineup(), 0.8).await;
|
||||
assert_eq!(editorial.summaries.len(), 2);
|
||||
assert!(editorial.summaries[&1].contains("opens with a specific"));
|
||||
assert_eq!(editorial.summaries[&2], "Second abstract.");
|
||||
assert!(editorial.front_page_html.contains("Leading today"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_editorial_covers_every_pick() {
|
||||
let lineup = lineup();
|
||||
let editorial = fallback_editorial(&lineup);
|
||||
assert_eq!(editorial.summaries.len(), lineup.picks.len());
|
||||
assert!(editorial.section_intros.is_empty());
|
||||
assert!(editorial.front_page_html.contains("2 articles"));
|
||||
assert!(
|
||||
editorial
|
||||
.front_page_html
|
||||
.contains("Top Stories, Boston & Local")
|
||||
);
|
||||
assert!(editorial.front_page_html.starts_with("<p>"));
|
||||
|
||||
// An empty lineup is still a valid editorial.
|
||||
let empty = Lineup {
|
||||
date: "2026-08-15".parse().expect("date"),
|
||||
picks: vec![],
|
||||
section_order: vec![],
|
||||
};
|
||||
let editorial = fallback_editorial(&empty);
|
||||
assert!(editorial.summaries.is_empty());
|
||||
assert!(editorial.front_page_html.contains("0 articles"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excerpt_summary_handles_empty_bodies() {
|
||||
let mut p = pick(9, "No body here", "Top Stories", false);
|
||||
p.article.content_html = String::new();
|
||||
assert!(excerpt_summary(&p).contains("No preview text"));
|
||||
assert_eq!(summary_to_html("a <b> c"), "<p>a <b> c</p>");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,708 @@
|
||||
//! DeepSeek client and token/cost accounting (spec §3.6).
|
||||
//!
|
||||
//! The OpenAI-compatible chat-completions endpoint at `https://api.deepseek.com/v1`.
|
||||
//! DeepSeek prefix-caches automatically, so the (identical, long) taste-profile
|
||||
//! system prompt must come first in every request: cached input is $0.0028/M vs
|
||||
//! $0.14/M.
|
||||
//!
|
||||
//! **Why not `async-openai`** (spec §2 crate table): the published crate exposes
|
||||
//! neither `Client` nor `types::chat` under any feature combination we could get
|
||||
//! to build here, and it would drag in a second HTTP stack besides the shared
|
||||
//! `reqwest` client (notes §4). [`DeepseekBackend`] therefore speaks the same
|
||||
//! OpenAI-compatible wire protocol directly — about 80 lines, no new dependency,
|
||||
//! and the request/response shapes are pinned by this module's tests. The
|
||||
//! dependency was dropped from `Cargo.toml`; swapping a vendor SDK back in later
|
||||
//! is a single [`ChatBackend`] impl and nothing else moves.
|
||||
//!
|
||||
//! Every call in the project goes through [`LlmClient`], which
|
||||
//!
|
||||
//! 1. always sends [`LlmClient::system_prompt`] as the **first** message, byte for
|
||||
//! byte identical across requests (that is what makes the prefix cache hit),
|
||||
//! 2. folds the response's token usage into a shared [`UsageMeter`], and
|
||||
//! 3. refuses further work once `max_daily_usd` has been spent (§3.6 guardrail).
|
||||
//!
|
||||
//! The network is reached through a [`ChatBackend`] so tests can inject canned
|
||||
//! responses ([`MockBackend`]) without touching the wire (notes §6).
|
||||
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::config::DeepseekConfig;
|
||||
use crate::http::RetryPolicy;
|
||||
use crate::types::TokenUsage;
|
||||
|
||||
/// `response_format` value used for every structured call (§3.6).
|
||||
pub const JSON_OBJECT: &str = "json_object";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LlmError {
|
||||
#[error("deepseek api key is not configured (set DAILY_EPUB_DEEPSEEK__API_KEY)")]
|
||||
MissingApiKey,
|
||||
#[error("deepseek request failed: {0}")]
|
||||
Api(String),
|
||||
/// A 5xx/429/network failure: worth retrying (crate table "retry").
|
||||
#[error("deepseek request failed (transient): {0}")]
|
||||
Transient(String),
|
||||
#[error("deepseek returned an empty completion")]
|
||||
EmptyResponse,
|
||||
#[error("deepseek returned unparseable JSON: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
/// The `max_daily_usd` ceiling was reached: callers must skip remaining
|
||||
/// editorial calls and fall back to feed excerpts, loudly (§3.6).
|
||||
#[error("daily cost ceiling of ${limit:.2} reached (spent ${spent:.4})")]
|
||||
BudgetExceeded { spent: f64, limit: f64 },
|
||||
}
|
||||
|
||||
impl LlmError {
|
||||
/// True for failures the [`RetryPolicy`] should retry.
|
||||
pub fn is_transient(&self) -> bool {
|
||||
matches!(self, LlmError::Transient(_))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Usage metering (§3.6 cost guardrail, notes §5)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Shared token/cost accumulator enforcing `max_daily_usd` (notes §5).
|
||||
///
|
||||
/// Cloning shares the counters: one meter per run, cloned into every stage.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UsageMeter {
|
||||
inner: Arc<Mutex<TokenUsage>>,
|
||||
/// Sticky: once the ceiling is crossed the run stays degraded (§3.6).
|
||||
exceeded: Arc<AtomicBool>,
|
||||
limit_usd: f64,
|
||||
price_input: f64,
|
||||
price_cached: f64,
|
||||
price_output: f64,
|
||||
}
|
||||
|
||||
impl UsageMeter {
|
||||
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(Mutex::new(TokenUsage::default())),
|
||||
exceeded: Arc::new(AtomicBool::new(false)),
|
||||
limit_usd,
|
||||
price_input: cfg.price_input_per_mtok,
|
||||
price_cached: cfg.price_cached_input_per_mtok,
|
||||
price_output: cfg.price_output_per_mtok,
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed the meter with spend already recorded for the day (§3.6): the
|
||||
/// guardrail is a *daily* ceiling, not a per-run one.
|
||||
pub fn preload_cost(&self, spent_usd: f64) {
|
||||
if spent_usd > 0.0 && self.limit_usd > 0.0 && spent_usd >= self.limit_usd {
|
||||
self.trip("prior spend for today already exceeds the ceiling");
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold one response's usage in and return the running total.
|
||||
pub fn record(&self, usage: TokenUsage) -> TokenUsage {
|
||||
let total = match self.inner.lock() {
|
||||
Ok(mut guard) => {
|
||||
guard.add(usage);
|
||||
*guard
|
||||
}
|
||||
// A poisoned mutex must not abort a run: accounting is advisory.
|
||||
Err(poisoned) => {
|
||||
let mut guard = poisoned.into_inner();
|
||||
guard.add(usage);
|
||||
*guard
|
||||
}
|
||||
};
|
||||
let cost = self.cost_of(total);
|
||||
tracing::debug!(
|
||||
input = usage.input_tokens,
|
||||
cached = usage.cached_tokens,
|
||||
output = usage.output_tokens,
|
||||
total_cost_usd = cost,
|
||||
"recorded llm usage"
|
||||
);
|
||||
if self.limit_usd > 0.0 && cost > self.limit_usd && !self.exceeded.load(Ordering::SeqCst) {
|
||||
self.trip("token spend crossed the ceiling");
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
fn trip(&self, why: &str) {
|
||||
self.exceeded.store(true, Ordering::SeqCst);
|
||||
tracing::error!(
|
||||
spent_usd = self.cost_usd(),
|
||||
limit_usd = self.limit_usd,
|
||||
"LLM budget exceeded ({why}): remaining editorial calls will be skipped \
|
||||
and feed excerpts used instead"
|
||||
);
|
||||
}
|
||||
|
||||
pub fn total(&self) -> TokenUsage {
|
||||
match self.inner.lock() {
|
||||
Ok(guard) => *guard,
|
||||
Err(poisoned) => *poisoned.into_inner(),
|
||||
}
|
||||
}
|
||||
|
||||
fn cost_of(&self, usage: TokenUsage) -> f64 {
|
||||
usage.cost_usd(self.price_input, self.price_cached, self.price_output)
|
||||
}
|
||||
|
||||
pub fn cost_usd(&self) -> f64 {
|
||||
self.cost_of(self.total())
|
||||
}
|
||||
|
||||
pub fn limit_usd(&self) -> f64 {
|
||||
self.limit_usd
|
||||
}
|
||||
|
||||
/// True once the ceiling has been crossed — editorial stages check this and
|
||||
/// silently degrade to excerpts (§3.6).
|
||||
pub fn budget_exceeded(&self) -> bool {
|
||||
self.exceeded.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// `Err(BudgetExceeded)` once the run has spent more than `max_daily_usd` (§3.6).
|
||||
pub fn check_budget(&self) -> Result<(), LlmError> {
|
||||
if self.budget_exceeded() {
|
||||
return Err(LlmError::BudgetExceeded {
|
||||
spent: self.cost_usd(),
|
||||
limit: self.limit_usd,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Backend abstraction (notes §6: no network in tests)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One chat completion request. The system prompt is an [`Arc`] so that the
|
||||
/// identical bytes are reused for every call (DeepSeek prefix caching, §3.6).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChatRequest {
|
||||
pub model: String,
|
||||
pub system: Arc<String>,
|
||||
pub user: String,
|
||||
pub temperature: f32,
|
||||
/// Ask for `response_format: {"type": "json_object"}` (§3.6).
|
||||
pub json: bool,
|
||||
}
|
||||
|
||||
/// One chat completion response, reduced to what the pipeline needs.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ChatCompletion {
|
||||
pub content: String,
|
||||
pub usage: TokenUsage,
|
||||
}
|
||||
|
||||
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
/// The seam between [`LlmClient`] and the network (notes §6).
|
||||
pub trait ChatBackend: std::fmt::Debug + Send + Sync {
|
||||
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>>;
|
||||
}
|
||||
|
||||
/// LLM calls are slow; the shared 10s HTTP timeout would kill them (notes §4).
|
||||
const LLM_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180);
|
||||
|
||||
/// The real thing: the OpenAI-compatible endpoint at `deepseek.base_url` (§3.6).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DeepseekBackend {
|
||||
http: reqwest::Client,
|
||||
endpoint: String,
|
||||
api_key: String,
|
||||
}
|
||||
|
||||
impl DeepseekBackend {
|
||||
pub fn new(cfg: &DeepseekConfig) -> Result<Self, LlmError> {
|
||||
let api_key = cfg
|
||||
.api_key
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|k| !k.is_empty())
|
||||
.ok_or(LlmError::MissingApiKey)?
|
||||
.to_string();
|
||||
let http = crate::http::build_client(LLM_TIMEOUT)
|
||||
.map_err(|e| LlmError::Api(format!("building the deepseek http client: {e}")))?;
|
||||
Ok(Self {
|
||||
http,
|
||||
endpoint: format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')),
|
||||
api_key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for DeepseekBackend {
|
||||
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>> {
|
||||
Box::pin(async move {
|
||||
let mut body = json!({
|
||||
"model": req.model,
|
||||
"messages": [
|
||||
// FIRST and byte-identical across every request: prefix cache (§3.6).
|
||||
{"role": "system", "content": req.system.as_str()},
|
||||
{"role": "user", "content": req.user},
|
||||
],
|
||||
"temperature": req.temperature,
|
||||
"stream": false,
|
||||
});
|
||||
if req.json
|
||||
&& let Some(obj) = body.as_object_mut()
|
||||
{
|
||||
obj.insert("response_format".into(), json!({"type": JSON_OBJECT}));
|
||||
}
|
||||
|
||||
let response = self
|
||||
.http
|
||||
.post(&self.endpoint)
|
||||
.bearer_auth(&self.api_key)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(classify_reqwest_error)?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
let detail = response.text().await.unwrap_or_default();
|
||||
let detail = detail.chars().take(500).collect::<String>();
|
||||
let msg = format!("{status}: {detail}");
|
||||
return Err(if status.is_server_error() || status.as_u16() == 429 {
|
||||
LlmError::Transient(msg)
|
||||
} else {
|
||||
LlmError::Api(msg)
|
||||
});
|
||||
}
|
||||
|
||||
let parsed: ApiResponse = response.json().await.map_err(|e| {
|
||||
LlmError::Api(format!("decoding the deepseek chat completion: {e}"))
|
||||
})?;
|
||||
let content = parsed
|
||||
.choices
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|c| c.message.content)
|
||||
.filter(|c| !c.trim().is_empty())
|
||||
.ok_or(LlmError::EmptyResponse)?;
|
||||
let usage = parsed.usage.map(usage_from_api).unwrap_or_default();
|
||||
Ok(ChatCompletion { content, usage })
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The slice of the chat-completions response we consume.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ApiResponse {
|
||||
#[serde(default)]
|
||||
choices: Vec<ApiChoice>,
|
||||
#[serde(default)]
|
||||
usage: Option<ApiUsage>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ApiChoice {
|
||||
message: ApiMessage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ApiMessage {
|
||||
#[serde(default)]
|
||||
content: Option<String>,
|
||||
}
|
||||
|
||||
/// DeepSeek reports cache hits both OpenAI-style (`prompt_tokens_details`) and
|
||||
/// natively (`prompt_cache_hit_tokens`); we accept either (§3.6 pricing).
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct ApiUsage {
|
||||
#[serde(default)]
|
||||
prompt_tokens: i64,
|
||||
#[serde(default)]
|
||||
completion_tokens: i64,
|
||||
#[serde(default)]
|
||||
prompt_cache_hit_tokens: Option<i64>,
|
||||
#[serde(default)]
|
||||
prompt_tokens_details: Option<ApiPromptTokensDetails>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct ApiPromptTokensDetails {
|
||||
#[serde(default)]
|
||||
cached_tokens: Option<i64>,
|
||||
}
|
||||
|
||||
/// Split `prompt_tokens` into cache-miss and cache-hit halves (§3.6 pricing).
|
||||
fn usage_from_api(u: ApiUsage) -> TokenUsage {
|
||||
let cached = u
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|d| d.cached_tokens)
|
||||
.or(u.prompt_cache_hit_tokens)
|
||||
.unwrap_or(0)
|
||||
.max(0);
|
||||
let prompt = u.prompt_tokens.max(0);
|
||||
let cached = cached.min(prompt);
|
||||
TokenUsage {
|
||||
input_tokens: prompt - cached,
|
||||
cached_tokens: cached,
|
||||
output_tokens: u.completion_tokens.max(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_reqwest_error(err: reqwest::Error) -> LlmError {
|
||||
if crate::http::is_retryable(&err) {
|
||||
LlmError::Transient(err.to_string())
|
||||
} else {
|
||||
LlmError::Api(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Every LLM call in the project goes through this client (notes §5).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LlmClient {
|
||||
/// The taste profile, sent as the first (cacheable) system message (§3.6).
|
||||
pub system_prompt: Arc<String>,
|
||||
pub model: String,
|
||||
pub meter: UsageMeter,
|
||||
backend: Arc<dyn ChatBackend>,
|
||||
retry: RetryPolicy,
|
||||
}
|
||||
|
||||
impl LlmClient {
|
||||
/// Build against the configured base URL; fails without an API key.
|
||||
pub fn new(
|
||||
cfg: &DeepseekConfig,
|
||||
system_prompt: String,
|
||||
meter: UsageMeter,
|
||||
) -> Result<Self, LlmError> {
|
||||
let backend = DeepseekBackend::new(cfg)?;
|
||||
tracing::debug!(
|
||||
base_url = %cfg.base_url,
|
||||
model = %cfg.model,
|
||||
system_prompt_chars = system_prompt.len(),
|
||||
"deepseek client ready"
|
||||
);
|
||||
Ok(Self::with_backend(
|
||||
&cfg.model,
|
||||
system_prompt,
|
||||
meter,
|
||||
Arc::new(backend),
|
||||
))
|
||||
}
|
||||
|
||||
/// Construct around an arbitrary backend — the seam used by tests (notes §6).
|
||||
pub fn with_backend(
|
||||
model: &str,
|
||||
system_prompt: String,
|
||||
meter: UsageMeter,
|
||||
backend: Arc<dyn ChatBackend>,
|
||||
) -> Self {
|
||||
Self {
|
||||
system_prompt: Arc::new(system_prompt),
|
||||
model: model.to_string(),
|
||||
meter,
|
||||
backend,
|
||||
retry: RetryPolicy::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw completion: budget check → retry loop → usage accounting.
|
||||
pub async fn complete(
|
||||
&self,
|
||||
user_prompt: &str,
|
||||
temperature: f32,
|
||||
json: bool,
|
||||
) -> Result<String, LlmError> {
|
||||
self.meter.check_budget()?;
|
||||
let req = ChatRequest {
|
||||
model: self.model.clone(),
|
||||
system: Arc::clone(&self.system_prompt),
|
||||
user: user_prompt.to_string(),
|
||||
temperature,
|
||||
json,
|
||||
};
|
||||
let completion = self
|
||||
.retry
|
||||
.run("deepseek chat completion", LlmError::is_transient, || {
|
||||
self.backend.complete(req.clone())
|
||||
})
|
||||
.await?;
|
||||
self.meter.record(completion.usage);
|
||||
Ok(completion.content)
|
||||
}
|
||||
|
||||
/// One chat completion returning parsed JSON of type `T`, with the system
|
||||
/// prompt first and `response_format: json_object` (§3.6).
|
||||
pub async fn complete_json<T: serde::de::DeserializeOwned>(
|
||||
&self,
|
||||
user_prompt: &str,
|
||||
temperature: f32,
|
||||
) -> Result<T, LlmError> {
|
||||
let raw = self.complete(user_prompt, temperature, true).await?;
|
||||
let cleaned = strip_code_fence(&raw);
|
||||
match serde_json::from_str::<T>(cleaned) {
|
||||
Ok(v) => Ok(v),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
preview = %cleaned.chars().take(400).collect::<String>(),
|
||||
"deepseek returned malformed JSON"
|
||||
);
|
||||
Err(LlmError::Json(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One plain-text completion (used for the front page / intros) (§3.6).
|
||||
pub async fn complete_text(
|
||||
&self,
|
||||
user_prompt: &str,
|
||||
temperature: f32,
|
||||
) -> Result<String, LlmError> {
|
||||
self.complete(user_prompt, temperature, false).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Models occasionally wrap JSON in ```` ```json ```` fences despite `json_object`.
|
||||
pub fn strip_code_fence(raw: &str) -> &str {
|
||||
let trimmed = raw.trim();
|
||||
let Some(rest) = trimmed.strip_prefix("```") else {
|
||||
return trimmed;
|
||||
};
|
||||
let rest = rest.strip_prefix("json").unwrap_or(rest);
|
||||
rest.trim_start_matches(['\n', '\r'])
|
||||
.trim_end()
|
||||
.trim_end_matches("```")
|
||||
.trim()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test backend
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Canned-response backend for tests: pops scripted replies in order (notes §6).
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MockBackend {
|
||||
scripted: Mutex<std::collections::VecDeque<Result<ChatCompletion, String>>>,
|
||||
/// Every prompt the code under test sent, in order.
|
||||
pub seen: Mutex<Vec<ChatRequest>>,
|
||||
}
|
||||
|
||||
impl MockBackend {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Queue a successful reply carrying `usage` tokens.
|
||||
pub fn push(&self, content: impl Into<String>, usage: TokenUsage) {
|
||||
if let Ok(mut q) = self.scripted.lock() {
|
||||
q.push_back(Ok(ChatCompletion {
|
||||
content: content.into(),
|
||||
usage,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue a permanent (non-retryable) failure.
|
||||
pub fn push_error(&self, message: impl Into<String>) {
|
||||
if let Ok(mut q) = self.scripted.lock() {
|
||||
q.push_back(Err(message.into()));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calls(&self) -> usize {
|
||||
self.seen.lock().map(|s| s.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn prompts(&self) -> Vec<ChatRequest> {
|
||||
self.seen.lock().map(|s| s.clone()).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatBackend for MockBackend {
|
||||
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>> {
|
||||
Box::pin(async move {
|
||||
let next = self.scripted.lock().ok().and_then(|mut q| q.pop_front());
|
||||
if let Ok(mut seen) = self.seen.lock() {
|
||||
seen.push(req);
|
||||
}
|
||||
match next {
|
||||
Some(Ok(c)) => Ok(c),
|
||||
Some(Err(msg)) => Err(LlmError::Api(msg)),
|
||||
None => Err(LlmError::Api("mock backend ran out of responses".into())),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cfg() -> DeepseekConfig {
|
||||
DeepseekConfig::default()
|
||||
}
|
||||
|
||||
pub(crate) fn tokens(input: i64, cached: i64, output: i64) -> TokenUsage {
|
||||
TokenUsage {
|
||||
input_tokens: input,
|
||||
cached_tokens: cached,
|
||||
output_tokens: output,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meter_accumulates_and_prices() {
|
||||
let meter = UsageMeter::new(&cfg(), 2.0);
|
||||
meter.record(tokens(1_000_000, 0, 0));
|
||||
meter.record(tokens(0, 1_000_000, 1_000_000));
|
||||
let total = meter.total();
|
||||
assert_eq!(total.input_tokens, 1_000_000);
|
||||
assert_eq!(total.cached_tokens, 1_000_000);
|
||||
assert_eq!(total.output_tokens, 1_000_000);
|
||||
assert!((meter.cost_usd() - 0.4228).abs() < 1e-9);
|
||||
assert!(!meter.budget_exceeded());
|
||||
assert!(meter.check_budget().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meter_trips_the_budget_flag_and_stays_tripped() {
|
||||
// Ceiling of $0.10; 1M cache-miss input tokens costs $0.14.
|
||||
let meter = UsageMeter::new(&cfg(), 0.10);
|
||||
meter.record(tokens(1_000_000, 0, 0));
|
||||
assert!(meter.budget_exceeded());
|
||||
assert!(matches!(
|
||||
meter.check_budget(),
|
||||
Err(LlmError::BudgetExceeded { .. })
|
||||
));
|
||||
// Cloned meters share the flag.
|
||||
assert!(meter.clone().budget_exceeded());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preloaded_daily_spend_trips_the_flag() {
|
||||
let meter = UsageMeter::new(&cfg(), 1.0);
|
||||
meter.preload_cost(0.5);
|
||||
assert!(!meter.budget_exceeded());
|
||||
meter.preload_cost(1.5);
|
||||
assert!(meter.budget_exceeded());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_split_uses_prompt_token_details() {
|
||||
let u: ApiUsage = serde_json::from_str(
|
||||
r#"{"prompt_tokens": 1000, "completion_tokens": 120, "total_tokens": 1120,
|
||||
"prompt_tokens_details": {"cached_tokens": 800}}"#,
|
||||
)
|
||||
.expect("fixture usage");
|
||||
assert_eq!(usage_from_api(u), tokens(200, 800, 120));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_falls_back_to_deepseek_native_cache_fields() {
|
||||
let u: ApiUsage = serde_json::from_str(
|
||||
r#"{"prompt_tokens": 500, "completion_tokens": 40,
|
||||
"prompt_cache_hit_tokens": 448, "prompt_cache_miss_tokens": 52}"#,
|
||||
)
|
||||
.expect("fixture usage");
|
||||
assert_eq!(usage_from_api(u), tokens(52, 448, 40));
|
||||
// Missing usage is not an error, just zero.
|
||||
let empty: ApiUsage = serde_json::from_str("{}").expect("empty usage");
|
||||
assert_eq!(usage_from_api(empty), TokenUsage::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_fences_are_stripped() {
|
||||
assert_eq!(strip_code_fence("{\"a\":1}"), "{\"a\":1}");
|
||||
assert_eq!(strip_code_fence("```json\n{\"a\":1}\n```"), "{\"a\":1}");
|
||||
assert_eq!(strip_code_fence("```\n{\"a\":1}\n```"), "{\"a\":1}");
|
||||
}
|
||||
|
||||
fn client(backend: Arc<MockBackend>, limit: f64) -> LlmClient {
|
||||
LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM PROMPT".into(),
|
||||
UsageMeter::new(&cfg(), limit),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn json_completion_records_usage_and_sends_system_prompt_first() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(r#"{"value": 42}"#, tokens(10, 90, 5));
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Out {
|
||||
value: i64,
|
||||
}
|
||||
let out: Out = llm
|
||||
.complete_json("score these", 0.3)
|
||||
.await
|
||||
.expect("mock completion");
|
||||
assert_eq!(out.value, 42);
|
||||
assert_eq!(llm.meter.total(), tokens(10, 90, 5));
|
||||
|
||||
let prompts = backend.prompts();
|
||||
assert_eq!(prompts.len(), 1);
|
||||
assert_eq!(prompts[0].system.as_str(), "SYSTEM PROMPT");
|
||||
assert!(prompts[0].json);
|
||||
assert_eq!(prompts[0].user, "score these");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identical_system_prompt_bytes_across_calls() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push("{}", TokenUsage::default());
|
||||
backend.push("{}", TokenUsage::default());
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let _: serde_json::Value = llm.complete_json("a", 0.3).await.expect("first");
|
||||
let _: serde_json::Value = llm.complete_json("b", 0.3).await.expect("second");
|
||||
let prompts = backend.prompts();
|
||||
assert_eq!(prompts[0].system.as_bytes(), prompts[1].system.as_bytes());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn calls_are_refused_once_the_budget_is_gone() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push("{}", tokens(1_000_000, 0, 0));
|
||||
let llm = client(Arc::clone(&backend), 0.01);
|
||||
let _: serde_json::Value = llm.complete_json("first", 0.3).await.expect("first call");
|
||||
let err = llm
|
||||
.complete_text("second", 0.3)
|
||||
.await
|
||||
.expect_err("budget must be enforced");
|
||||
assert!(matches!(err, LlmError::BudgetExceeded { .. }));
|
||||
// The refused call never reached the backend.
|
||||
assert_eq!(backend.calls(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_json_surfaces_as_json_error() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push("not json at all", TokenUsage::default());
|
||||
let llm = client(backend, 2.0);
|
||||
let out: Result<serde_json::Value, _> = llm.complete_json("x", 0.3).await;
|
||||
assert!(matches!(out, Err(LlmError::Json(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_api_key_is_reported() {
|
||||
let cfg = DeepseekConfig {
|
||||
api_key: Some(" ".into()),
|
||||
..DeepseekConfig::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
DeepseekBackend::new(&cfg),
|
||||
Err(LlmError::MissingApiKey)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
//! Curation pipeline: pre-filter → LLM scoring → selection → editorial (spec §3.5, §3.6).
|
||||
//!
|
||||
//! ```text
|
||||
//! ~400 articles ─prefilter─▶ ~120 candidates ─stage A─▶ scored ─stage B─▶ lineup ─stage C─▶ editorial
|
||||
//! ```
|
||||
//!
|
||||
//! [`Curator`] is the thin orchestration layer the `generate` pipeline calls; the
|
||||
//! interesting logic lives in the stage modules. Every stage is safe to run with
|
||||
//! `llm == None` (`--skip-llm`): the prefilter order stands in for selection and
|
||||
//! feed excerpts stand in for summaries (notes §6).
|
||||
|
||||
pub mod editorial;
|
||||
pub mod llm;
|
||||
pub mod prefilter;
|
||||
pub mod profile;
|
||||
pub mod score;
|
||||
pub mod select;
|
||||
|
||||
use jiff::civil::Date;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::Db;
|
||||
use crate::types::{Article, Editorial, Lineup, ScoredArticle};
|
||||
|
||||
/// Runs the three curation stages against one day's articles (§3.5, §3.6).
|
||||
pub struct Curator {
|
||||
pub config: Config,
|
||||
pub db: Db,
|
||||
pub llm: Option<llm::LlmClient>,
|
||||
}
|
||||
|
||||
impl Curator {
|
||||
/// `llm == None` corresponds to `--skip-llm`: prefilter order is used for
|
||||
/// selection and feed excerpts stand in for summaries (notes §6).
|
||||
pub fn new(config: Config, db: Db, llm: Option<llm::LlmClient>) -> Self {
|
||||
Self { config, db, llm }
|
||||
}
|
||||
|
||||
/// Heuristic pre-filter: 300–500 articles → `prefilter_keep` (§3.5).
|
||||
///
|
||||
/// Also persists each candidate's `prefilter_score` for the day so that a
|
||||
/// re-run of the same date is idempotent (notes §12).
|
||||
pub async fn prefilter(
|
||||
&self,
|
||||
articles: Vec<Article>,
|
||||
date: Date,
|
||||
) -> anyhow::Result<Vec<ScoredArticle>> {
|
||||
let span = tracing::info_span!("prefilter", articles = articles.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let ctx = prefilter::PrefilterContext::load(&self.db, date).await?;
|
||||
let candidates = prefilter::run(articles, &ctx, &self.config);
|
||||
for candidate in &candidates {
|
||||
if candidate.article.id == 0 {
|
||||
continue; // not persisted yet (dry run over synthetic articles)
|
||||
}
|
||||
if let Err(e) = self
|
||||
.db
|
||||
.upsert_score(
|
||||
candidate.article.id,
|
||||
date,
|
||||
Some(candidate.prefilter_score),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(article_id = candidate.article.id, error = %e,
|
||||
"could not persist the prefilter score");
|
||||
}
|
||||
}
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
/// Stage A: batched LLM scoring of the surviving candidates (§3.6).
|
||||
///
|
||||
/// A no-op under `--skip-llm`. Scores are persisted per `(article, date)`.
|
||||
pub async fn score(&self, candidates: &mut [ScoredArticle], date: Date) -> anyhow::Result<()> {
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
tracing::info!("--skip-llm: stage A scoring skipped");
|
||||
return Ok(());
|
||||
};
|
||||
let span = tracing::info_span!("llm_score", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let scored = score::score_all(
|
||||
llm,
|
||||
candidates,
|
||||
self.config.deepseek.score_batch_size,
|
||||
&self.config.curation.sections,
|
||||
self.config.deepseek.score_temperature,
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(scored, total = candidates.len(), "stage A complete");
|
||||
|
||||
for candidate in candidates.iter() {
|
||||
if candidate.article.id == 0 {
|
||||
continue;
|
||||
}
|
||||
if let Some(llm_score) = candidate.llm.as_ref()
|
||||
&& let Err(e) = self
|
||||
.db
|
||||
.upsert_score(candidate.article.id, date, None, Some(llm_score))
|
||||
.await
|
||||
{
|
||||
tracing::warn!(article_id = candidate.article.id, error = %e,
|
||||
"could not persist the llm score");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stage B: single-call lineup selection into sections (§3.6).
|
||||
pub async fn select(
|
||||
&self,
|
||||
candidates: Vec<ScoredArticle>,
|
||||
date: Date,
|
||||
) -> anyhow::Result<Lineup> {
|
||||
let sections = &self.config.curation.sections;
|
||||
let target = self.config.target_article_count;
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
tracing::info!("--skip-llm: selecting by prefilter order");
|
||||
return Ok(select::select_without_llm(
|
||||
candidates, sections, target, date,
|
||||
));
|
||||
};
|
||||
let span = tracing::info_span!("llm_select", candidates = candidates.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
match select::select(llm, candidates.clone(), sections, target, date).await {
|
||||
Ok(lineup) => Ok(lineup),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e,
|
||||
"stage B selection failed; falling back to prefilter order");
|
||||
Ok(select::select_without_llm(
|
||||
candidates, sections, target, date,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage C: per-article summaries, section intros and the front page (§3.6).
|
||||
///
|
||||
/// Never fails the run: a budget trip or an API error degrades to excerpts.
|
||||
pub async fn editorial(&self, lineup: &Lineup) -> anyhow::Result<Editorial> {
|
||||
let Some(llm) = self.llm.as_ref() else {
|
||||
tracing::info!("--skip-llm: using feed excerpts as summaries");
|
||||
return Ok(editorial::fallback_editorial(lineup));
|
||||
};
|
||||
let span = tracing::info_span!("llm_editorial", picks = lineup.picks.len());
|
||||
let _guard = span.enter();
|
||||
Ok(editorial::run(llm, lineup, self.config.deepseek.editorial_temperature).await)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Small text helpers shared by the prompt builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Crude token estimate: DeepSeek averages ~4 characters per token for English
|
||||
/// prose. Only used to size prompt budgets (§3.6 stage C).
|
||||
pub fn approx_tokens(text: &str) -> usize {
|
||||
text.len().div_ceil(4)
|
||||
}
|
||||
|
||||
/// Strip markup and collapse whitespace, so article bodies can go into prompts
|
||||
/// as plain text (cheaper and less confusing for the model than raw HTML).
|
||||
pub fn html_to_text(html: &str) -> String {
|
||||
/// Does `tail` open the named element, i.e. `<name` or `</name`?
|
||||
fn opens(tail: &str, name: &str) -> bool {
|
||||
let bytes = tail.as_bytes();
|
||||
bytes.len() > name.len() && bytes[1..=name.len()].eq_ignore_ascii_case(name.as_bytes())
|
||||
}
|
||||
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut rest = html;
|
||||
while let Some(ch) = rest.chars().next() {
|
||||
if ch != '<' {
|
||||
out.push(ch);
|
||||
rest = &rest[ch.len_utf8()..];
|
||||
continue;
|
||||
}
|
||||
// Drop <script>/<style> bodies wholesale rather than reading them aloud.
|
||||
for (name, close) in [("script", "</script"), ("style", "</style")] {
|
||||
if opens(rest, name) {
|
||||
rest = match rest[1..].find(close) {
|
||||
Some(idx) => &rest[1 + idx + close.len()..],
|
||||
None => "",
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
// A tag becomes a word boundary.
|
||||
rest = match rest.find('>') {
|
||||
Some(idx) => &rest[idx + 1..],
|
||||
None => "",
|
||||
};
|
||||
out.push(' ');
|
||||
}
|
||||
|
||||
let decoded = out
|
||||
.replace(" ", " ")
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
.replace("’", "'")
|
||||
.replace("—", "—");
|
||||
decoded.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// First `max_words` words of `text`, with an ellipsis when truncated.
|
||||
pub fn truncate_words(text: &str, max_words: usize) -> String {
|
||||
let mut words = text.split_whitespace();
|
||||
let head: Vec<&str> = words.by_ref().take(max_words).collect();
|
||||
let mut out = head.join(" ");
|
||||
if words.next().is_some() {
|
||||
out.push('…');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Truncate to roughly `max_tokens` tokens on a word boundary (§3.6 stage C).
|
||||
pub fn truncate_tokens(text: &str, max_tokens: usize) -> String {
|
||||
let max_chars = max_tokens.saturating_mul(4);
|
||||
if text.len() <= max_chars {
|
||||
return text.to_string();
|
||||
}
|
||||
let mut cut = max_chars.min(text.len());
|
||||
while cut > 0 && !text.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
let slice = &text[..cut];
|
||||
let slice = slice
|
||||
.rsplit_once(' ')
|
||||
.map(|(head, _)| head)
|
||||
.unwrap_or(slice);
|
||||
format!("{slice}…")
|
||||
}
|
||||
|
||||
/// Minimal XHTML escaping for text we drop into generated markup (§3.10).
|
||||
pub fn escape_html(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
for ch in text.chars() {
|
||||
match ch {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
'"' => out.push_str("""),
|
||||
'\'' => out.push_str("'"),
|
||||
c => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Render plain text (possibly with blank-line paragraphs) as XHTML paragraphs.
|
||||
pub fn text_to_paragraphs(text: &str) -> String {
|
||||
text.split("\n\n")
|
||||
.map(str::trim)
|
||||
.filter(|p| !p.is_empty())
|
||||
.map(|p| format!("<p>{}</p>", escape_html(&p.replace('\n', " "))))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn html_becomes_readable_text() {
|
||||
let html = "<h1>Title</h1><p>First & best.</p><script>alert('x')</script>\
|
||||
<p>Second<br/>line</p><style>p{color:red}</style>";
|
||||
assert_eq!(html_to_text(html), "Title First & best. Second line");
|
||||
assert_eq!(html_to_text(""), "");
|
||||
assert_eq!(html_to_text("no markup at all"), "no markup at all");
|
||||
// Unicode survives byte-wise walking.
|
||||
assert_eq!(html_to_text("<p>café — naïve</p>"), "café — naïve");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_and_token_truncation() {
|
||||
assert_eq!(truncate_words("one two three", 5), "one two three");
|
||||
assert_eq!(truncate_words("one two three", 2), "one two…");
|
||||
let long = "word ".repeat(1000);
|
||||
// 10 tokens ≈ 40 characters, cut back to a word boundary, plus the ellipsis.
|
||||
let cut = truncate_tokens(&long, 10);
|
||||
assert!(cut.len() <= 43, "{}", cut.len());
|
||||
assert!(cut.split_whitespace().count() <= 10);
|
||||
assert!(cut.ends_with('…'));
|
||||
assert_eq!(truncate_tokens("short", 10), "short");
|
||||
assert!(approx_tokens("abcd") <= 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escaping_and_paragraphs() {
|
||||
assert_eq!(escape_html("a<b>&'\""), "a<b>&'"");
|
||||
assert_eq!(
|
||||
text_to_paragraphs("One\nline.\n\nTwo <b>."),
|
||||
"<p>One line.</p>\n<p>Two <b>.</p>"
|
||||
);
|
||||
assert_eq!(text_to_paragraphs(" "), "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,665 @@
|
||||
//! Heuristic pre-filter: 300–500 articles → ~120 candidates (spec §3.5).
|
||||
//!
|
||||
//! Pure Rust and free: this is what keeps LLM cost flat as feed volume grows.
|
||||
//!
|
||||
//! The 0–100 score is a sum of bounded components so that no single signal can
|
||||
//! dominate, and every component is monotonic in its input:
|
||||
//!
|
||||
//! | component | range | source |
|
||||
//! |---|---|---|
|
||||
//! | long-form word count | 0 … +35 | §3.5 "0 pts <300 words, max at ~2500+" |
|
||||
//! | social proof | 0 … +25 | §3.4 composite, log-scaled again |
|
||||
//! | came via Scour | +8 | §3.5 (already matched a stated interest) |
|
||||
//! | came via HN frontpage | +8 | §3.5 |
|
||||
//! | carried by several feeds | 0 … +8 | §3.2 (multi-source *is* social proof) |
|
||||
//! | feed prior | −12 … +12 | §3.9 beta-smoothed upvote rate, neutral at 0.5 |
|
||||
//! | excerpt only | −20 | §3.5 (penalized, never banned — §7) |
|
||||
//! | roundup/release-notes title | −15 | §3.5 |
|
||||
//! | blocked domain | excluded | §3.5 |
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::config::{Config, CurationConfig};
|
||||
use crate::types::{Article, ArticleId, FeedId, FeedPrior, ScoredArticle, SourceKind};
|
||||
|
||||
/// Title patterns that mark low-effort posts: link roundups, release notes,
|
||||
/// sponsor posts (§3.5).
|
||||
pub const PENALTY_TITLE_PATTERNS: &[&str] = &[
|
||||
"link roundup",
|
||||
"links for",
|
||||
"weekly digest",
|
||||
"release notes",
|
||||
"changelog",
|
||||
"sponsored",
|
||||
"this week in",
|
||||
"linkdump",
|
||||
"link dump",
|
||||
"weekly roundup",
|
||||
"roundup:",
|
||||
"in case you missed it",
|
||||
"what we're reading",
|
||||
"sponsor post",
|
||||
"now available",
|
||||
"is now generally available",
|
||||
"release candidate",
|
||||
"patch notes",
|
||||
"job board",
|
||||
"who's hiring",
|
||||
"newsletter #",
|
||||
"digest #",
|
||||
];
|
||||
|
||||
/// Word count at which the long-form bonus saturates (§3.5).
|
||||
pub const LONGFORM_SATURATION_WORDS: i64 = 2500;
|
||||
/// Below this word count the long-form bonus is zero (§3.5).
|
||||
pub const LONGFORM_FLOOR_WORDS: i64 = 300;
|
||||
/// Articles the LLM scored below this within the last week are not re-scored (§3.5).
|
||||
pub const STALE_LOW_SCORE: f64 = 3.0;
|
||||
/// Lookback for the "don't re-score churn" rule (§3.5).
|
||||
pub const STALE_LOOKBACK_DAYS: i64 = 7;
|
||||
|
||||
/// Maximum contribution of each scoring component (§3.5).
|
||||
pub const MAX_LONGFORM_POINTS: f64 = 35.0;
|
||||
pub const MAX_SOCIAL_POINTS: f64 = 25.0;
|
||||
pub const SCOUR_BONUS: f64 = 8.0;
|
||||
pub const HN_FRONTPAGE_BONUS: f64 = 8.0;
|
||||
pub const MAX_MULTI_SOURCE_POINTS: f64 = 8.0;
|
||||
pub const MAX_FEED_PRIOR_POINTS: f64 = 12.0;
|
||||
pub const EXCERPT_ONLY_PENALTY: f64 = 20.0;
|
||||
pub const ROUNDUP_TITLE_PENALTY: f64 = 15.0;
|
||||
|
||||
/// `composite_social_score` value that earns the full social bonus. Empirically
|
||||
/// ~6.0 is a 1,000-point HN story with 500 comments (§3.4 formula).
|
||||
const SOCIAL_SATURATION: f64 = 6.0;
|
||||
|
||||
/// Everything the pre-filter needs beyond the articles themselves (§3.5, §3.9).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PrefilterContext {
|
||||
/// Per-feed Bayesian upvote rate from ratings history (§3.9).
|
||||
pub feed_priors: HashMap<FeedId, FeedPrior>,
|
||||
/// Article ids already published in a previous issue (§3.5).
|
||||
pub already_published: Vec<ArticleId>,
|
||||
/// Article ids the LLM scored < [`STALE_LOW_SCORE`] recently (§3.5).
|
||||
pub recently_rejected: Vec<ArticleId>,
|
||||
}
|
||||
|
||||
impl PrefilterContext {
|
||||
/// Load the history/priors context from SQLite (§3.5 dedup-vs-history, §3.9).
|
||||
///
|
||||
/// `today` anchors the [`STALE_LOOKBACK_DAYS`] window.
|
||||
pub async fn load(
|
||||
db: &crate::db::Db,
|
||||
today: jiff::civil::Date,
|
||||
) -> Result<Self, crate::db::DbError> {
|
||||
let since = today
|
||||
.checked_sub(jiff::Span::new().days(STALE_LOOKBACK_DAYS))
|
||||
.unwrap_or(today);
|
||||
let feed_priors = db
|
||||
.feed_priors()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|p| (p.feed_id, p))
|
||||
.collect();
|
||||
let already_published = db.previously_published_ids().await?;
|
||||
let recently_rejected = db.recently_low_scored_ids(STALE_LOW_SCORE, since).await?;
|
||||
tracing::debug!(
|
||||
priors = ?feed_priors_len(&feed_priors),
|
||||
published = already_published.len(),
|
||||
rejected = recently_rejected.len(),
|
||||
"loaded prefilter context"
|
||||
);
|
||||
Ok(Self {
|
||||
feed_priors,
|
||||
already_published,
|
||||
recently_rejected,
|
||||
})
|
||||
}
|
||||
|
||||
fn prior_for(&self, article: &Article) -> f64 {
|
||||
// The cluster's feeds are all candidates; take the most favourable one,
|
||||
// since a story carried by a well-rated feed is a better bet.
|
||||
let mut best = self.feed_priors.get(&article.feed_id).map(FeedPrior::rate);
|
||||
for source in &article.sources {
|
||||
if let Some(p) = self.feed_priors.get(&source.feed_id) {
|
||||
let rate = p.rate();
|
||||
best = Some(best.map_or(rate, |b: f64| b.max(rate)));
|
||||
}
|
||||
}
|
||||
best.unwrap_or(0.5)
|
||||
}
|
||||
}
|
||||
|
||||
fn feed_priors_len(m: &HashMap<FeedId, FeedPrior>) -> usize {
|
||||
m.len()
|
||||
}
|
||||
|
||||
/// True when the article's feed is in `curation.always_include_feeds` (§3.5).
|
||||
///
|
||||
/// Entries are matched either as a Miniflux feed id (any feed in the cluster) or
|
||||
/// as a case-insensitive substring of the article/site URL.
|
||||
///
|
||||
/// Auto-includes are still LLM-scored (for section + summary) but can't be dropped.
|
||||
pub fn is_auto_include(article: &Article, cfg: &CurationConfig) -> bool {
|
||||
if cfg.always_include_feeds.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let url = article.url.to_lowercase();
|
||||
let canonical = article.canonical_url.to_lowercase();
|
||||
cfg.always_include_feeds.iter().any(|raw| {
|
||||
let needle = raw.trim();
|
||||
if needle.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if let Ok(id) = needle.parse::<FeedId>()
|
||||
&& (article.feed_id == id || article.sources.iter().any(|s| s.feed_id == id))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let needle = needle.to_lowercase();
|
||||
// Bare host or full site URL: compare against both URLs we hold.
|
||||
let needle = needle
|
||||
.trim_start_matches("https://")
|
||||
.trim_start_matches("http://")
|
||||
.trim_end_matches('/');
|
||||
!needle.is_empty() && (url.contains(needle) || canonical.contains(needle))
|
||||
})
|
||||
}
|
||||
|
||||
/// True when the article's host matches `curation.blocked_domains` (§3.5).
|
||||
pub fn is_blocked(article: &Article, cfg: &CurationConfig) -> bool {
|
||||
if cfg.blocked_domains.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let host = host_of(&article.canonical_url)
|
||||
.or_else(|| host_of(&article.url))
|
||||
.unwrap_or_default();
|
||||
if host.is_empty() {
|
||||
return false;
|
||||
}
|
||||
cfg.blocked_domains.iter().any(|raw| {
|
||||
let blocked = raw.trim().trim_start_matches('.').to_lowercase();
|
||||
!blocked.is_empty() && (host == blocked || host.ends_with(&format!(".{blocked}")))
|
||||
})
|
||||
}
|
||||
|
||||
/// Lowercased host of a URL, `www.` stripped.
|
||||
fn host_of(url: &str) -> Option<String> {
|
||||
let rest = url
|
||||
.split_once("://")
|
||||
.map(|(_, rest)| rest)
|
||||
.unwrap_or(url)
|
||||
.split(['/', '?', '#'])
|
||||
.next()?;
|
||||
let host = rest.rsplit_once('@').map(|(_, h)| h).unwrap_or(rest);
|
||||
let host = host.split_once(':').map(|(h, _)| h).unwrap_or(host);
|
||||
let host = host.trim().to_lowercase();
|
||||
if host.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(host.trim_start_matches("www.").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// True when the title reads like a link roundup / release note / sponsor post (§3.5).
|
||||
pub fn looks_like_roundup(title: &str) -> bool {
|
||||
let lower = title.to_lowercase();
|
||||
PENALTY_TITLE_PATTERNS
|
||||
.iter()
|
||||
.any(|pattern| lower.contains(pattern))
|
||||
}
|
||||
|
||||
/// Long-form bonus: zero below [`LONGFORM_FLOOR_WORDS`], saturating at
|
||||
/// [`LONGFORM_SATURATION_WORDS`], with a concave curve so that the jump from a
|
||||
/// 400-word note to a 1,200-word piece matters more than 2,000 → 2,500 (§3.5).
|
||||
pub fn longform_points(word_count: i64) -> f64 {
|
||||
let span = (LONGFORM_SATURATION_WORDS - LONGFORM_FLOOR_WORDS) as f64;
|
||||
let over = (word_count - LONGFORM_FLOOR_WORDS).max(0) as f64;
|
||||
MAX_LONGFORM_POINTS * (over / span).min(1.0).powf(0.65)
|
||||
}
|
||||
|
||||
/// Social proof, log-scaled a second time so that a viral story cannot swamp the
|
||||
/// long-form preference (§3.4, §3.5).
|
||||
pub fn social_points(social_score: f64) -> f64 {
|
||||
if social_score <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
MAX_SOCIAL_POINTS * (social_score / SOCIAL_SATURATION).min(1.0).sqrt()
|
||||
}
|
||||
|
||||
/// Score one article 0–100 from word count, social proof, source signals, feed
|
||||
/// prior, and the excerpt/roundup/blocklist penalties (§3.5).
|
||||
pub fn score_article(article: &Article, ctx: &PrefilterContext, cfg: &Config) -> f64 {
|
||||
if is_blocked(article, &cfg.curation) {
|
||||
return 0.0;
|
||||
}
|
||||
let mut score = longform_points(article.word_count);
|
||||
score += social_points(article.social_score());
|
||||
|
||||
if article.came_via(SourceKind::Scour) {
|
||||
score += SCOUR_BONUS;
|
||||
}
|
||||
if article.came_via(SourceKind::HnFrontpage) {
|
||||
score += HN_FRONTPAGE_BONUS;
|
||||
}
|
||||
|
||||
let extra_feeds = article.sources.len().saturating_sub(1) as f64;
|
||||
score += (extra_feeds * 4.0).min(MAX_MULTI_SOURCE_POINTS);
|
||||
|
||||
// Beta-smoothed upvote rate, neutral (0.5) contributing nothing (§3.9).
|
||||
score += (ctx.prior_for(article) - 0.5) * 2.0 * MAX_FEED_PRIOR_POINTS;
|
||||
|
||||
if article.excerpt_only {
|
||||
score -= EXCERPT_ONLY_PENALTY;
|
||||
}
|
||||
if looks_like_roundup(&article.title) {
|
||||
score -= ROUNDUP_TITLE_PENALTY;
|
||||
}
|
||||
|
||||
score.clamp(0.0, 100.0)
|
||||
}
|
||||
|
||||
/// Apply [`score_article`] to everything, drop history duplicates, then keep the
|
||||
/// top `prefilter_keep` plus every auto-include (§3.5).
|
||||
pub fn run(articles: Vec<Article>, ctx: &PrefilterContext, cfg: &Config) -> Vec<ScoredArticle> {
|
||||
let published: HashSet<ArticleId> = ctx.already_published.iter().copied().collect();
|
||||
let rejected: HashSet<ArticleId> = ctx.recently_rejected.iter().copied().collect();
|
||||
|
||||
let total = articles.len();
|
||||
let (mut dropped_history, mut dropped_blocked) = (0usize, 0usize);
|
||||
let mut scored: Vec<ScoredArticle> = Vec::with_capacity(total);
|
||||
|
||||
for article in articles {
|
||||
let auto_include = is_auto_include(&article, &cfg.curation);
|
||||
|
||||
// Never print the same story twice, not even from an always-include feed.
|
||||
if published.contains(&article.id) {
|
||||
dropped_history += 1;
|
||||
continue;
|
||||
}
|
||||
// "Don't re-score churn" (§3.5) — but an always-include feed still gets in.
|
||||
if !auto_include && rejected.contains(&article.id) {
|
||||
dropped_history += 1;
|
||||
continue;
|
||||
}
|
||||
if !auto_include && is_blocked(&article, &cfg.curation) {
|
||||
dropped_blocked += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let prefilter_score = score_article(&article, ctx, cfg);
|
||||
let social_score = article.social_score();
|
||||
let feed_prior = ctx.prior_for(&article);
|
||||
scored.push(ScoredArticle {
|
||||
article,
|
||||
prefilter_score,
|
||||
social_score,
|
||||
feed_prior,
|
||||
llm: None,
|
||||
auto_include,
|
||||
});
|
||||
}
|
||||
|
||||
// Descending by score; ties broken by word count then id so the order is
|
||||
// deterministic across runs (notes §12).
|
||||
sort_by_prefilter(&mut scored);
|
||||
|
||||
let keep = cfg.prefilter_keep.max(cfg.target_article_count);
|
||||
let kept: Vec<ScoredArticle> = if scored.len() <= keep {
|
||||
scored
|
||||
} else {
|
||||
let (head, tail) = scored.split_at(keep);
|
||||
let mut kept = head.to_vec();
|
||||
// Auto-includes below the cut are pulled back in — they can't be dropped.
|
||||
kept.extend(tail.iter().filter(|s| s.auto_include).cloned());
|
||||
sort_by_prefilter(&mut kept);
|
||||
kept
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
input = total,
|
||||
kept = kept.len(),
|
||||
auto_includes = kept.iter().filter(|s| s.auto_include).count(),
|
||||
dropped_history,
|
||||
dropped_blocked,
|
||||
"pre-filter complete"
|
||||
);
|
||||
kept
|
||||
}
|
||||
|
||||
/// Deterministic ranking: score desc, then longer, then lowest id (notes §12).
|
||||
pub fn sort_by_prefilter(scored: &mut [ScoredArticle]) {
|
||||
scored.sort_by(|a, b| {
|
||||
b.prefilter_score
|
||||
.partial_cmp(&a.prefilter_score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| b.article.word_count.cmp(&a.article.word_count))
|
||||
.then_with(|| a.article.id.cmp(&b.article.id))
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::types::{ExtractMethod, SocialRef, SocialSource, SourceRef};
|
||||
use jiff::Timestamp;
|
||||
|
||||
pub(crate) fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z"
|
||||
.parse()
|
||||
.expect("static timestamp parses")
|
||||
}
|
||||
|
||||
/// A plain 800-word article from feed 7 with no social proof.
|
||||
pub(crate) fn article(id: ArticleId, title: &str, word_count: i64) -> Article {
|
||||
Article {
|
||||
id,
|
||||
canonical_url: format!("https://example.com/{id}"),
|
||||
title: title.into(),
|
||||
best_entry_id: 1000 + id,
|
||||
content_html: format!("<p>{}</p>", "word ".repeat(word_count.max(0) as usize)),
|
||||
word_count,
|
||||
excerpt_only: false,
|
||||
image_count: 0,
|
||||
sources: vec![SourceRef {
|
||||
entry_id: 1000 + id,
|
||||
feed_id: 7,
|
||||
feed_title: "Some Blog".into(),
|
||||
category: Some("Tech".into()),
|
||||
kind: SourceKind::Feed,
|
||||
}],
|
||||
first_seen: ts(),
|
||||
url: format!("https://example.com/{id}"),
|
||||
author: Some("A. Writer".into()),
|
||||
feed_id: 7,
|
||||
feed_title: "Some Blog".into(),
|
||||
category: Some("Tech".into()),
|
||||
published_at: Some(ts()),
|
||||
comments_url: None,
|
||||
image_urls: vec![],
|
||||
social: vec![],
|
||||
extract_method: ExtractMethod::Readability,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_social(mut a: Article, points: i64, comments: i64) -> Article {
|
||||
a.social = vec![SocialRef {
|
||||
article_id: a.id,
|
||||
source: SocialSource::Hn,
|
||||
item_id: Some("1".into()),
|
||||
score: points,
|
||||
num_comments: comments,
|
||||
item_url: Some("https://news.ycombinator.com/item?id=1".into()),
|
||||
fetched_at: ts(),
|
||||
}];
|
||||
a
|
||||
}
|
||||
|
||||
pub(crate) fn via(mut a: Article, kind: SourceKind, feed_id: FeedId) -> Article {
|
||||
a.sources.push(SourceRef {
|
||||
entry_id: a.best_entry_id,
|
||||
feed_id,
|
||||
feed_title: format!("{kind:?} feed"),
|
||||
category: None,
|
||||
kind,
|
||||
});
|
||||
a
|
||||
}
|
||||
|
||||
fn cfg() -> Config {
|
||||
Config {
|
||||
prefilter_keep: 3,
|
||||
target_article_count: 2,
|
||||
..Config::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longform_curve_is_monotonic_and_bounded() {
|
||||
assert_eq!(longform_points(0), 0.0);
|
||||
assert_eq!(longform_points(LONGFORM_FLOOR_WORDS), 0.0);
|
||||
let mut prev = -1.0;
|
||||
for wc in [0, 100, 299, 300, 500, 900, 1500, 2200, 2500, 9000] {
|
||||
let pts = longform_points(wc);
|
||||
assert!(pts >= prev, "not monotonic at {wc}");
|
||||
assert!(pts <= MAX_LONGFORM_POINTS);
|
||||
prev = pts;
|
||||
}
|
||||
assert!((longform_points(2500) - MAX_LONGFORM_POINTS).abs() < 1e-9);
|
||||
assert!((longform_points(50_000) - MAX_LONGFORM_POINTS).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn social_curve_is_monotonic_and_bounded() {
|
||||
let mut prev = -1.0;
|
||||
for s in [0.0, 0.5, 1.0, 2.0, 4.0, 6.0, 20.0] {
|
||||
let pts = social_points(s);
|
||||
assert!(pts >= prev);
|
||||
assert!(pts <= MAX_SOCIAL_POINTS);
|
||||
prev = pts;
|
||||
}
|
||||
assert_eq!(social_points(0.0), 0.0);
|
||||
assert!((social_points(6.0) - MAX_SOCIAL_POINTS).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_rises_with_length_and_social_proof() {
|
||||
let (ctx, cfg) = (PrefilterContext::default(), cfg());
|
||||
let short = score_article(&article(1, "A thought", 200), &ctx, &cfg);
|
||||
let medium = score_article(&article(2, "An essay", 1200), &ctx, &cfg);
|
||||
let long = score_article(&article(3, "A treatise", 3000), &ctx, &cfg);
|
||||
assert!(short < medium, "{short} !< {medium}");
|
||||
assert!(medium < long, "{medium} !< {long}");
|
||||
|
||||
let quiet = score_article(&article(4, "An essay", 1200), &ctx, &cfg);
|
||||
let loud = score_article(
|
||||
&with_social(article(5, "An essay", 1200), 400, 250),
|
||||
&ctx,
|
||||
&cfg,
|
||||
);
|
||||
assert!(loud > quiet);
|
||||
assert!(loud <= 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_bonuses_and_penalties_apply() {
|
||||
let (ctx, cfg) = (PrefilterContext::default(), cfg());
|
||||
// Long enough that the penalties do not run into the 0 floor.
|
||||
let plain = score_article(&article(1, "Deep dive", 3000), &ctx, &cfg);
|
||||
assert!(plain > EXCERPT_ONLY_PENALTY);
|
||||
|
||||
let scoured = score_article(
|
||||
&via(article(2, "Deep dive", 3000), SourceKind::Scour, 42),
|
||||
&ctx,
|
||||
&cfg,
|
||||
);
|
||||
// Scour bonus + one extra feed in the cluster.
|
||||
assert!(scoured > plain + SCOUR_BONUS - 0.001);
|
||||
|
||||
let mut excerpt = article(3, "Deep dive", 3000);
|
||||
excerpt.excerpt_only = true;
|
||||
assert!(
|
||||
(score_article(&excerpt, &ctx, &cfg) - (plain - EXCERPT_ONLY_PENALTY)).abs() < 1e-9
|
||||
);
|
||||
|
||||
let roundup = article(4, "This Week in Rust #612", 3000);
|
||||
assert!(looks_like_roundup(&roundup.title));
|
||||
assert!(
|
||||
(score_article(&roundup, &ctx, &cfg) - (plain - ROUNDUP_TITLE_PENALTY)).abs() < 1e-9
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_prior_moves_the_score_both_ways() {
|
||||
let cfg = cfg();
|
||||
let mut liked = PrefilterContext::default();
|
||||
liked.feed_priors.insert(
|
||||
7,
|
||||
FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 18,
|
||||
downvotes: 0,
|
||||
included: 18,
|
||||
},
|
||||
);
|
||||
let mut disliked = PrefilterContext::default();
|
||||
disliked.feed_priors.insert(
|
||||
7,
|
||||
FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 0,
|
||||
downvotes: 18,
|
||||
included: 18,
|
||||
},
|
||||
);
|
||||
let a = article(1, "Deep dive", 1200);
|
||||
let neutral = score_article(&a, &PrefilterContext::default(), &cfg);
|
||||
assert!(score_article(&a, &liked, &cfg) > neutral);
|
||||
assert!(score_article(&a, &disliked, &cfg) < neutral);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocked_domains_and_auto_includes_match_urls_and_ids() {
|
||||
let mut cfg = cfg();
|
||||
cfg.curation.blocked_domains = vec!["spam.example".into()];
|
||||
cfg.curation.always_include_feeds = vec!["99".into(), "tyler.blog".into()];
|
||||
|
||||
let mut blocked = article(1, "Buy now", 1200);
|
||||
blocked.canonical_url = "https://news.spam.example/post".into();
|
||||
blocked.url.clone_from(&blocked.canonical_url);
|
||||
assert!(is_blocked(&blocked, &cfg.curation));
|
||||
assert_eq!(
|
||||
score_article(&blocked, &PrefilterContext::default(), &cfg),
|
||||
0.0
|
||||
);
|
||||
|
||||
let mut by_url = article(2, "A rare post", 900);
|
||||
by_url.url = "https://tyler.blog/2026/rare".into();
|
||||
assert!(is_auto_include(&by_url, &cfg.curation));
|
||||
|
||||
let mut by_id = article(3, "Another rare post", 900);
|
||||
by_id.feed_id = 99;
|
||||
assert!(is_auto_include(&by_id, &cfg.curation));
|
||||
|
||||
assert!(!is_auto_include(&article(4, "Normal", 900), &cfg.curation));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_top_n_plus_auto_includes_and_drops_history() {
|
||||
let mut cfg = cfg();
|
||||
cfg.prefilter_keep = 2;
|
||||
cfg.curation.always_include_feeds = vec!["99".into()];
|
||||
|
||||
let mut auto = article(5, "A short personal note", 120);
|
||||
auto.feed_id = 99;
|
||||
|
||||
let articles = vec![
|
||||
article(1, "Long treatise", 4000),
|
||||
article(2, "Medium essay", 1500),
|
||||
article(3, "Shorter piece", 700),
|
||||
article(4, "Already printed", 5000),
|
||||
auto,
|
||||
article(6, "Rejected yesterday", 3000),
|
||||
];
|
||||
let ctx = PrefilterContext {
|
||||
already_published: vec![4],
|
||||
recently_rejected: vec![6],
|
||||
..PrefilterContext::default()
|
||||
};
|
||||
|
||||
let kept = run(articles, &ctx, &cfg);
|
||||
let ids: Vec<ArticleId> = kept.iter().map(|s| s.article.id).collect();
|
||||
assert!(!ids.contains(&4), "previously published must be dropped");
|
||||
assert!(!ids.contains(&6), "recently rejected must be dropped");
|
||||
assert!(ids.contains(&5), "auto-include survives below the cut");
|
||||
assert!(ids.contains(&1) && ids.contains(&2));
|
||||
assert!(!ids.contains(&3), "cut at prefilter_keep");
|
||||
assert_eq!(kept.len(), 3); // 2 kept + 1 auto-include
|
||||
|
||||
// Sorted by score, descending.
|
||||
for pair in kept.windows(2) {
|
||||
assert!(pair[0].prefilter_score >= pair[1].prefilter_score);
|
||||
}
|
||||
assert!(
|
||||
kept.iter()
|
||||
.find(|s| s.article.id == 5)
|
||||
.is_some_and(|s| s.auto_include)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_include_survives_the_recently_rejected_list_but_not_republication() {
|
||||
let mut cfg = cfg();
|
||||
cfg.curation.always_include_feeds = vec!["99".into()];
|
||||
let mut a = article(1, "Personal note", 200);
|
||||
a.feed_id = 99;
|
||||
let mut b = article(2, "Personal note two", 200);
|
||||
b.feed_id = 99;
|
||||
|
||||
let ctx = PrefilterContext {
|
||||
recently_rejected: vec![1],
|
||||
already_published: vec![2],
|
||||
..PrefilterContext::default()
|
||||
};
|
||||
let kept = run(vec![a, b], &ctx, &cfg);
|
||||
let ids: Vec<ArticleId> = kept.iter().map(|s| s.article.id).collect();
|
||||
assert_eq!(ids, vec![1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_loads_history_from_sqlite() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db = crate::db::Db::open_and_migrate(&dir.path().join("t.db"))
|
||||
.await
|
||||
.expect("db");
|
||||
let date: jiff::civil::Date = "2026-08-15".parse().expect("date");
|
||||
|
||||
db.upsert_feed_prior(&FeedPrior {
|
||||
feed_id: 7,
|
||||
upvotes: 4,
|
||||
downvotes: 1,
|
||||
included: 5,
|
||||
})
|
||||
.await
|
||||
.expect("prior");
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, first_seen) VALUES
|
||||
(42, 'https://example.com/42', 'Printed', '2026-08-14T00:00:00Z'),
|
||||
(43, 'https://example.com/43', 'Rejected', '2026-08-14T00:00:00Z'),
|
||||
(44, 'https://example.com/44', 'Ancient', '2020-01-01T00:00:00Z')",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("articles");
|
||||
db.upsert_issue(
|
||||
"2026-08-14".parse().expect("date"),
|
||||
1,
|
||||
ts(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("issue");
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position, is_lead)
|
||||
VALUES ('2026-08-14', 42, 'Top Stories', 1, 0)",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("issue article");
|
||||
sqlx::query(
|
||||
"INSERT INTO scores (article_id, run_date, llm_score) VALUES (43, '2026-08-14', 1.5),
|
||||
(44, '2020-01-01', 1.0)",
|
||||
)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("scores");
|
||||
|
||||
let ctx = PrefilterContext::load(&db, date).await.expect("context");
|
||||
assert_eq!(ctx.already_published, vec![42]);
|
||||
assert_eq!(ctx.recently_rejected, vec![43], "old rejects age out");
|
||||
assert!((ctx.feed_priors[&7].rate() - 5.0 / 7.0).abs() < 1e-12);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,593 @@
|
||||
//! Stage A — batched LLM scoring (spec §3.6).
|
||||
//!
|
||||
//! Batches of `deepseek.score_batch_size` articles per request. Per article we
|
||||
//! send title, source feed, author, word count, social stats, sources list and a
|
||||
//! ~200-word excerpt; the model returns one JSON object per article.
|
||||
//!
|
||||
//! Parsing is deliberately forgiving: one malformed item must not cost us the
|
||||
//! other eleven, and a failed batch must not fail the run.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::llm::{LlmClient, LlmError, strip_code_fence};
|
||||
use super::{html_to_text, truncate_words};
|
||||
use crate::types::{ArticleId, LlmScore, ScoredArticle, SourceKind};
|
||||
|
||||
/// Words of article text sent per candidate in stage A (§3.6).
|
||||
pub const EXCERPT_WORDS: usize = 200;
|
||||
|
||||
/// One element of the stage-A JSON response (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreItem {
|
||||
pub id: ArticleId,
|
||||
/// 0–10.
|
||||
pub score: f64,
|
||||
pub category: String,
|
||||
/// ≤ 20 words.
|
||||
#[serde(default)]
|
||||
pub rationale: String,
|
||||
#[serde(default)]
|
||||
pub is_paywalled_guess: bool,
|
||||
}
|
||||
|
||||
impl From<ScoreItem> for LlmScore {
|
||||
fn from(i: ScoreItem) -> Self {
|
||||
LlmScore {
|
||||
score: i.score,
|
||||
category: i.category,
|
||||
rationale: i.rationale,
|
||||
is_paywalled_guess: i.is_paywalled_guess,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Envelope the model is asked to return (`{"articles": [...]}`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoreResponse {
|
||||
#[serde(default)]
|
||||
pub articles: Vec<ScoreItem>,
|
||||
}
|
||||
|
||||
/// The invariant instruction block for stage A. Everything article-specific goes
|
||||
/// in the per-batch tail so this prefix stays cacheable (§3.6).
|
||||
pub const SCORE_INSTRUCTIONS: &str = "\
|
||||
TASK: score a batch of candidate articles for today's issue of The Daily EPUB.
|
||||
|
||||
Judge each article against the reader profile in your system prompt — not against \
|
||||
a general audience, and not against what is objectively newsworthy.
|
||||
|
||||
Return one object per input article with these fields:
|
||||
\"id\" integer, copied exactly from the input
|
||||
\"score\" number 0-10, the rubric below
|
||||
\"category\" one short label from the palette below
|
||||
\"rationale\" at most 20 words, concrete, no hedging, no restating the title
|
||||
\"is_paywalled_guess\" true when the text looks truncated, teaser-like or paywalled
|
||||
|
||||
SCORING RUBRIC — calibrate hard; a normal day averages about 4, and a 9 should \
|
||||
appear a couple of times a week, not a couple of times a day:
|
||||
9-10 Exceptional. Original reporting, a deep technical dive, or an essay he \
|
||||
will still be thinking about next week. Evident effort and a real point of view.
|
||||
7-8 Strong. A well-made long-form piece squarely in his interests, or an \
|
||||
outstanding piece outside them.
|
||||
5-6 Worth a slot on a thin day. Solid, useful, a little thin or a little \
|
||||
familiar.
|
||||
3-4 Marginal. Competent news-of-the-day, short posts, incremental updates, \
|
||||
good writing about an over-covered story.
|
||||
1-2 Weak. Announcements, changelogs and release notes, link roundups, \
|
||||
listicles, rewrites of a story available at the source, thin AI-industry churn.
|
||||
0 Unusable. Press releases, sponsored content, engagement bait, spam, \
|
||||
pure crypto promotion, or an entry with no readable body.
|
||||
|
||||
CALIBRATION NOTES
|
||||
- Length alone is not quality; padding scores worse than a tight short piece. But \
|
||||
between two equally good pieces, prefer the one with more substance.
|
||||
- Social proof is evidence, not a verdict: hundreds of HN points mean a critical \
|
||||
audience read it; a quiet post from a good blog can still outrank it.
|
||||
- \"came via scour\" means the story already matched one of his standing \
|
||||
interests. \"came via hn_frontpage\" means it cleared HN's front page.
|
||||
- Boston/New England local stories and ultra-niche community news get a genuine \
|
||||
lift — this paper wants them.
|
||||
- Wire-service world/US news should score low here: the World Briefing section \
|
||||
covers that separately.
|
||||
- Excerpt-only or paywalled text is a real cost to the reader; score it lower \
|
||||
unless the piece is clearly excellent.
|
||||
|
||||
Return JSON exactly in this shape, with one entry per input article and nothing \
|
||||
else:
|
||||
{\"articles\": [{\"id\": 123, \"score\": 7.5, \"category\": \"Tech & Engineering\", \
|
||||
\"rationale\": \"first-hand account of migrating 40TB off Postgres\", \
|
||||
\"is_paywalled_guess\": false}]}";
|
||||
|
||||
/// Render the user prompt for one batch (§3.6).
|
||||
pub fn build_batch_prompt(batch: &[ScoredArticle], sections: &[String]) -> String {
|
||||
let mut prompt = String::with_capacity(4096 + batch.len() * 1500);
|
||||
prompt.push_str(SCORE_INSTRUCTIONS);
|
||||
let _ = write!(
|
||||
prompt,
|
||||
"\n\nCATEGORY PALETTE (use one of these exact strings): {}\n\nARTICLES ({} in this batch)\n",
|
||||
sections.join(" | "),
|
||||
batch.len()
|
||||
);
|
||||
for candidate in batch {
|
||||
prompt.push('\n');
|
||||
prompt.push_str(&render_candidate(candidate));
|
||||
}
|
||||
prompt
|
||||
}
|
||||
|
||||
/// One article's block in the stage-A prompt (§3.6).
|
||||
fn render_candidate(candidate: &ScoredArticle) -> String {
|
||||
let a = &candidate.article;
|
||||
let mut block = String::with_capacity(1500);
|
||||
let _ = writeln!(block, "--- id: {}", a.id);
|
||||
let _ = writeln!(block, "title: {}", a.title.trim());
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"feed: {}{}",
|
||||
if a.feed_title.is_empty() {
|
||||
"unknown"
|
||||
} else {
|
||||
a.feed_title.trim()
|
||||
},
|
||||
a.category
|
||||
.as_deref()
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(|c| format!(" (category: {c})"))
|
||||
.unwrap_or_default()
|
||||
);
|
||||
if let Some(author) = a.author.as_deref().filter(|s| !s.trim().is_empty()) {
|
||||
let _ = writeln!(block, "author: {}", author.trim());
|
||||
}
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"length: {} words (~{} min read){}",
|
||||
a.word_count,
|
||||
a.reading_minutes(),
|
||||
if a.excerpt_only {
|
||||
" [EXCERPT ONLY — full text unavailable]"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
let _ = writeln!(block, "social: {}", social_line(candidate));
|
||||
let _ = writeln!(block, "came via: {}", sources_line(candidate));
|
||||
let excerpt = truncate_words(&html_to_text(&a.content_html), EXCERPT_WORDS);
|
||||
let _ = writeln!(
|
||||
block,
|
||||
"excerpt: {}",
|
||||
if excerpt.is_empty() {
|
||||
"(no body text extracted)"
|
||||
} else {
|
||||
&excerpt
|
||||
}
|
||||
);
|
||||
block
|
||||
}
|
||||
|
||||
fn social_line(candidate: &ScoredArticle) -> String {
|
||||
if candidate.article.social.is_empty() {
|
||||
return "none found".into();
|
||||
}
|
||||
let mut parts: Vec<String> = candidate
|
||||
.article
|
||||
.social
|
||||
.iter()
|
||||
.map(|s| {
|
||||
format!(
|
||||
"{} {} points / {} comments",
|
||||
s.source.display_name(),
|
||||
s.score,
|
||||
s.num_comments
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
parts.push(format!("composite {:.2}", candidate.social_score));
|
||||
parts.join("; ")
|
||||
}
|
||||
|
||||
fn sources_line(candidate: &ScoredArticle) -> String {
|
||||
let mut kinds: Vec<&str> = candidate
|
||||
.article
|
||||
.sources
|
||||
.iter()
|
||||
.map(|s| match s.kind {
|
||||
SourceKind::Scour => "scour",
|
||||
SourceKind::HnFrontpage => "hn_frontpage",
|
||||
SourceKind::Lobsters => "lobsters",
|
||||
SourceKind::Reddit => "reddit",
|
||||
SourceKind::Feed => "feed",
|
||||
})
|
||||
.collect();
|
||||
kinds.sort_unstable();
|
||||
kinds.dedup();
|
||||
if candidate.auto_include {
|
||||
kinds.push("always-include feed (cannot be dropped)");
|
||||
}
|
||||
if kinds.is_empty() {
|
||||
"feed".into()
|
||||
} else {
|
||||
kinds.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Response parsing (§3.6: tolerate anything the model does to us)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Keys the model might wrap the array in, in preference order.
|
||||
const ARRAY_KEYS: &[&str] = &["articles", "scores", "results", "items", "data"];
|
||||
|
||||
/// Parse a stage-A response leniently: missing optional fields default, scores
|
||||
/// are clamped to 0–10, and malformed items are skipped with a warning (§3.6).
|
||||
pub fn parse_score_response(raw: &str) -> Vec<ScoreItem> {
|
||||
let cleaned = strip_code_fence(raw);
|
||||
let value: Value = match serde_json::from_str(cleaned) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "stage A response was not JSON at all");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
|
||||
let array = match &value {
|
||||
Value::Array(items) => Some(items),
|
||||
Value::Object(map) => ARRAY_KEYS
|
||||
.iter()
|
||||
.find_map(|k| map.get(*k).and_then(Value::as_array))
|
||||
// Some models return {"1234": {...}} or a single bare object.
|
||||
.or_else(|| map.values().find_map(Value::as_array)),
|
||||
_ => None,
|
||||
};
|
||||
let Some(array) = array else {
|
||||
tracing::warn!("stage A response contained no array of scores");
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut out = Vec::with_capacity(array.len());
|
||||
let mut skipped = 0usize;
|
||||
for item in array {
|
||||
match parse_item(item) {
|
||||
Some(parsed) => out.push(parsed),
|
||||
None => {
|
||||
skipped += 1;
|
||||
tracing::warn!(item = %truncate_debug(item), "skipping malformed stage A item");
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, kept = out.len(), "stage A items were dropped");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn parse_item(item: &Value) -> Option<ScoreItem> {
|
||||
let obj = item.as_object()?;
|
||||
let id = obj.get("id").and_then(as_i64_lenient)?;
|
||||
let score = obj
|
||||
.get("score")
|
||||
.and_then(as_f64_lenient)
|
||||
.or_else(|| obj.get("rating").and_then(as_f64_lenient))?;
|
||||
Some(ScoreItem {
|
||||
id,
|
||||
score: score.clamp(0.0, 10.0),
|
||||
category: obj
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
rationale: obj
|
||||
.get("rationale")
|
||||
.or_else(|| obj.get("reason"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
is_paywalled_guess: obj
|
||||
.get("is_paywalled_guess")
|
||||
.or_else(|| obj.get("paywalled"))
|
||||
.and_then(as_bool_lenient)
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn as_i64_lenient(v: &Value) -> Option<i64> {
|
||||
v.as_i64()
|
||||
.or_else(|| v.as_f64().map(|f| f as i64))
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
}
|
||||
|
||||
fn as_f64_lenient(v: &Value) -> Option<f64> {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
|
||||
.filter(|f| f.is_finite())
|
||||
}
|
||||
|
||||
fn as_bool_lenient(v: &Value) -> Option<bool> {
|
||||
v.as_bool().or_else(|| match v.as_str()?.trim() {
|
||||
"true" | "yes" => Some(true),
|
||||
"false" | "no" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn truncate_debug(v: &Value) -> String {
|
||||
v.to_string().chars().take(160).collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Score every candidate, filling in [`ScoredArticle::llm`] (§3.6).
|
||||
///
|
||||
/// Batches that fail are logged and left unscored rather than aborting the run.
|
||||
/// Returns how many candidates came back with a score.
|
||||
pub async fn score_all(
|
||||
llm: &LlmClient,
|
||||
candidates: &mut [ScoredArticle],
|
||||
batch_size: usize,
|
||||
sections: &[String],
|
||||
temperature: f32,
|
||||
) -> Result<usize, LlmError> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let batch_size = batch_size.max(1);
|
||||
let batches = candidates.len().div_ceil(batch_size);
|
||||
let mut scores: HashMap<ArticleId, LlmScore> = HashMap::with_capacity(candidates.len());
|
||||
|
||||
for (n, batch) in candidates.chunks(batch_size).enumerate() {
|
||||
if let Err(e) = llm.meter.check_budget() {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
unscored = candidates.len() - scores.len(),
|
||||
"COST CEILING HIT during stage A scoring — remaining batches skipped; \
|
||||
the lineup will fall back to heuristic ranking for them"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let prompt = build_batch_prompt(batch, sections);
|
||||
tracing::debug!(
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
articles = batch.len(),
|
||||
approx_tokens = super::approx_tokens(&prompt),
|
||||
"stage A request"
|
||||
);
|
||||
match llm.complete(&prompt, temperature, true).await {
|
||||
Ok(raw) => {
|
||||
let items = parse_score_response(&raw);
|
||||
if items.is_empty() {
|
||||
tracing::warn!(
|
||||
batch = n + 1,
|
||||
of = batches,
|
||||
"stage A batch returned no scores"
|
||||
);
|
||||
}
|
||||
for item in items {
|
||||
scores.insert(item.id, item.into());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(batch = n + 1, of = batches, error = %e,
|
||||
"stage A batch failed; its articles stay unscored");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut applied = 0usize;
|
||||
for candidate in candidates.iter_mut() {
|
||||
if let Some(score) = scores.remove(&candidate.article.id) {
|
||||
candidate.llm = Some(score);
|
||||
applied += 1;
|
||||
}
|
||||
}
|
||||
if !scores.is_empty() {
|
||||
tracing::warn!(
|
||||
unknown_ids = scores.len(),
|
||||
"stage A returned scores for ids that were not in the batch"
|
||||
);
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DeepseekConfig;
|
||||
use crate::curate::llm::{MockBackend, UsageMeter};
|
||||
use crate::curate::prefilter::tests::{article, via, with_social};
|
||||
use crate::types::TokenUsage;
|
||||
use std::sync::Arc;
|
||||
|
||||
const BATCH_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch.json"
|
||||
));
|
||||
const MESSY_FIXTURE: &str = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/deepseek_score_batch_messy.json"
|
||||
));
|
||||
|
||||
fn sections() -> Vec<String> {
|
||||
crate::config::CurationConfig::default().sections
|
||||
}
|
||||
|
||||
fn candidate(id: i64, title: &str, words: i64) -> ScoredArticle {
|
||||
ScoredArticle {
|
||||
article: article(id, title, words),
|
||||
prefilter_score: 50.0,
|
||||
social_score: 0.0,
|
||||
feed_prior: 0.5,
|
||||
llm: None,
|
||||
auto_include: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_prompt_carries_every_documented_signal() {
|
||||
let mut c = candidate(12, "Migrating 40TB off Postgres", 3200);
|
||||
c.article = via(
|
||||
with_social(c.article, 342, 210),
|
||||
SourceKind::HnFrontpage,
|
||||
9001,
|
||||
);
|
||||
c.social_score = c.article.social_score();
|
||||
c.auto_include = true;
|
||||
let prompt = build_batch_prompt(&[c], §ions());
|
||||
|
||||
assert!(prompt.starts_with(SCORE_INSTRUCTIONS));
|
||||
assert!(prompt.contains("--- id: 12"));
|
||||
assert!(prompt.contains("title: Migrating 40TB off Postgres"));
|
||||
assert!(prompt.contains("feed: Some Blog (category: Tech)"));
|
||||
assert!(prompt.contains("author: A. Writer"));
|
||||
assert!(prompt.contains("length: 3200 words"));
|
||||
assert!(prompt.contains("HN 342 points / 210 comments"));
|
||||
assert!(prompt.contains("hn_frontpage"));
|
||||
assert!(prompt.contains("always-include feed"));
|
||||
assert!(prompt.contains("excerpt: word word"));
|
||||
assert!(prompt.contains("Tech & Engineering"));
|
||||
// The excerpt is capped.
|
||||
let excerpt_line = prompt
|
||||
.lines()
|
||||
.find(|l| l.starts_with("excerpt:"))
|
||||
.expect("excerpt line");
|
||||
assert!(excerpt_line.split_whitespace().count() <= EXCERPT_WORDS + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_realistic_deepseek_batch() {
|
||||
let items = parse_score_response(BATCH_FIXTURE);
|
||||
assert_eq!(items.len(), 4);
|
||||
assert_eq!(items[0].id, 101);
|
||||
assert!((items[0].score - 8.5).abs() < 1e-9);
|
||||
assert_eq!(items[0].category, "Tech & Engineering");
|
||||
assert!(items[0].rationale.split_whitespace().count() <= 20);
|
||||
assert!(!items[0].is_paywalled_guess);
|
||||
assert!(items[3].is_paywalled_guess);
|
||||
let score: LlmScore = items[0].clone().into();
|
||||
assert_eq!(score.category, "Tech & Engineering");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_survives_everything_a_model_might_do() {
|
||||
let items = parse_score_response(MESSY_FIXTURE);
|
||||
let ids: Vec<ArticleId> = items.iter().map(|i| i.id).collect();
|
||||
// 201 fine; 202 string score clamped; 203 missing rationale/category;
|
||||
// 204 out-of-range clamped; the two malformed entries are dropped.
|
||||
assert_eq!(ids, vec![201, 202, 203, 204]);
|
||||
assert!((items[1].score - 6.0).abs() < 1e-9);
|
||||
assert_eq!(items[2].rationale, "");
|
||||
assert_eq!(items[2].category, "");
|
||||
assert!(
|
||||
(items[3].score - 10.0).abs() < 1e-9,
|
||||
"clamped to the 0-10 range"
|
||||
);
|
||||
assert!(items.iter().all(|i| (0.0..=10.0).contains(&i.score)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsing_tolerates_fences_arrays_and_junk() {
|
||||
assert_eq!(
|
||||
parse_score_response("```json\n{\"articles\":[{\"id\":1,\"score\":5}]}\n```").len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(parse_score_response("[{\"id\": 2, \"score\": 3}]").len(), 1);
|
||||
assert_eq!(
|
||||
parse_score_response("{\"results\":[{\"id\":3,\"score\":\"4.5\"}]}")[0].score,
|
||||
4.5
|
||||
);
|
||||
assert!(parse_score_response("I'm sorry, I can't do that").is_empty());
|
||||
assert!(parse_score_response("{\"articles\": {}}").is_empty());
|
||||
}
|
||||
|
||||
fn client(backend: Arc<MockBackend>, limit_usd: f64) -> LlmClient {
|
||||
LlmClient::with_backend(
|
||||
"deepseek-v4-flash",
|
||||
"SYSTEM".into(),
|
||||
UsageMeter::new(&DeepseekConfig::default(), limit_usd),
|
||||
backend,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scores_are_applied_batch_by_batch() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":8,"category":"Tech & Engineering","rationale":"good"},
|
||||
{"id":2,"score":2,"category":"Niche Corner","rationale":"thin"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":3,"score":6.5,"category":"Culture & Essays","rationale":"solid"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
|
||||
let mut candidates = vec![
|
||||
candidate(1, "One", 1000),
|
||||
candidate(2, "Two", 1000),
|
||||
candidate(3, "Three", 1000),
|
||||
];
|
||||
let scored = score_all(&llm, &mut candidates, 2, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 3);
|
||||
assert_eq!(backend.calls(), 2, "batched by score_batch_size");
|
||||
assert_eq!(candidates[0].llm.as_ref().map(|l| l.score), Some(8.0));
|
||||
assert_eq!(candidates[2].llm.as_ref().map(|l| l.score), Some(6.5));
|
||||
// combined_score now reflects the LLM verdict.
|
||||
assert!(candidates[0].combined_score() > candidates[1].combined_score());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_batch_does_not_sink_the_run() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
backend.push_error("500 upstream exploded");
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":7,"category":"Top Stories","rationale":"ok"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 2.0);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring must not abort");
|
||||
assert_eq!(scored, 1);
|
||||
assert!(candidates[0].llm.is_none());
|
||||
assert!(candidates[1].llm.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoring_stops_when_the_budget_is_gone() {
|
||||
let backend = Arc::new(MockBackend::new());
|
||||
// First batch alone blows a $0.05 ceiling ($0.14 per 1M input tokens).
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":1,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 0,
|
||||
output_tokens: 0,
|
||||
},
|
||||
);
|
||||
backend.push(
|
||||
r#"{"articles":[{"id":2,"score":9,"category":"Top Stories","rationale":"great"}]}"#,
|
||||
TokenUsage::default(),
|
||||
);
|
||||
let llm = client(Arc::clone(&backend), 0.05);
|
||||
let mut candidates = vec![candidate(1, "One", 900), candidate(2, "Two", 900)];
|
||||
let scored = score_all(&llm, &mut candidates, 1, §ions(), 0.3)
|
||||
.await
|
||||
.expect("scoring");
|
||||
assert_eq!(scored, 1, "only the first batch ran");
|
||||
assert_eq!(backend.calls(), 1);
|
||||
assert!(llm.meter.budget_exceeded());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+655
@@ -0,0 +1,655 @@
|
||||
//! Normalization and duplicate clustering (spec §3.2).
|
||||
//!
|
||||
//! Canonicalizes URLs, clusters entries that tell the same story (HN frontpage feed
|
||||
//! + Scour feed + the blog's own feed), and drops obvious non-articles.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use url::Url;
|
||||
|
||||
use crate::types::{Article, Entry, ExtractMethod, SourceKind, SourceRef};
|
||||
|
||||
/// Query parameters stripped during canonicalization (§3.2).
|
||||
pub const TRACKING_PARAMS: &[&str] = &["ref", "fbclid", "gclid", "s", "si", "mc_cid", "mc_eid"];
|
||||
|
||||
/// URL hosts that are never articles (§3.2).
|
||||
pub const NON_ARTICLE_HOSTS: &[&str] = &[
|
||||
"youtube.com",
|
||||
"www.youtube.com",
|
||||
"youtu.be",
|
||||
"vimeo.com",
|
||||
"open.spotify.com",
|
||||
"podcasts.apple.com",
|
||||
];
|
||||
|
||||
/// Path suffixes that mark an audio/video enclosure rather than an article (§3.2).
|
||||
const MEDIA_EXTENSIONS: &[&str] = &[
|
||||
".mp3", ".m4a", ".m4v", ".mp4", ".ogg", ".oga", ".opus", ".wav", ".flac", ".aac", ".mov",
|
||||
".webm", ".mkv",
|
||||
];
|
||||
|
||||
/// How many redirector hops [`canonical_url`] will follow before giving up (§3.2).
|
||||
const MAX_REDIRECT_DEPTH: u8 = 3;
|
||||
|
||||
/// Minimum length of a [`normalized_title`] before it may merge two clusters.
|
||||
/// Short titles ("News", "Weekly") collide far too easily (§3.2 secondary pass).
|
||||
const MIN_TITLE_KEY_LEN: usize = 12;
|
||||
|
||||
/// Canonicalize a URL: lowercase host, drop the fragment, strip tracking params
|
||||
/// (`utm_*` and [`TRACKING_PARAMS`]), trim the trailing slash, and resolve known
|
||||
/// redirectors such as Google News links to their target (§3.2).
|
||||
///
|
||||
/// Returns `None` when the input is not a parseable absolute http(s) URL.
|
||||
pub fn canonical_url(raw: &str) -> Option<String> {
|
||||
canonicalize(raw, 0)
|
||||
}
|
||||
|
||||
fn canonicalize(raw: &str, depth: u8) -> Option<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut url = Url::parse(trimmed).ok()?;
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return None;
|
||||
}
|
||||
url.host_str()?;
|
||||
|
||||
// Known redirectors (Google News et al) carry the real article in a param.
|
||||
if depth < MAX_REDIRECT_DEPTH
|
||||
&& let Some(target) = redirect_target(&url)
|
||||
&& let Some(resolved) = canonicalize(&target, depth + 1)
|
||||
{
|
||||
return Some(resolved);
|
||||
}
|
||||
|
||||
url.set_fragment(None);
|
||||
|
||||
if let Some(host) = url.host_str() {
|
||||
let lower = host.to_ascii_lowercase();
|
||||
if lower != host {
|
||||
url.set_host(Some(&lower)).ok()?;
|
||||
}
|
||||
}
|
||||
|
||||
let kept: Vec<(String, String)> = url
|
||||
.query_pairs()
|
||||
.filter(|(k, _)| !is_tracking_param(k))
|
||||
.map(|(k, v)| (k.into_owned(), v.into_owned()))
|
||||
.collect();
|
||||
if kept.is_empty() {
|
||||
url.set_query(None);
|
||||
} else {
|
||||
let mut pairs = url.query_pairs_mut();
|
||||
pairs.clear();
|
||||
for (k, v) in &kept {
|
||||
pairs.append_pair(k, v);
|
||||
}
|
||||
drop(pairs);
|
||||
}
|
||||
|
||||
let path = url.path().to_string();
|
||||
if path.len() > 1 && path.ends_with('/') {
|
||||
url.set_path(path.trim_end_matches('/'));
|
||||
}
|
||||
|
||||
let mut out = url.to_string();
|
||||
if url.query().is_none() && out.ends_with('/') {
|
||||
out.pop();
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn is_tracking_param(key: &str) -> bool {
|
||||
let key = key.to_ascii_lowercase();
|
||||
key.starts_with("utm_") || TRACKING_PARAMS.contains(&key.as_str())
|
||||
}
|
||||
|
||||
/// The real destination behind a known redirector, if any (§3.2).
|
||||
fn redirect_target(url: &Url) -> Option<String> {
|
||||
let host = url.host_str()?.to_ascii_lowercase();
|
||||
let is_google_news = host == "news.google.com" || host.ends_with(".news.google.com");
|
||||
let is_google_redirect = host == "news.url.google.com"
|
||||
|| ((host == "www.google.com" || host == "google.com") && url.path() == "/url");
|
||||
if !(is_google_news || is_google_redirect) {
|
||||
return None;
|
||||
}
|
||||
url.query_pairs()
|
||||
.find(|(k, _)| k == "url" || k == "q")
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.filter(|v| v.starts_with("http"))
|
||||
}
|
||||
|
||||
/// Title normalized for the fuzzy second dedupe pass: lowercased, alphanumeric only (§3.2).
|
||||
pub fn normalized_title(title: &str) -> String {
|
||||
title
|
||||
.chars()
|
||||
.filter(|c| c.is_alphanumeric())
|
||||
.flat_map(|c| c.to_lowercase())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// True for entries that are not articles at all: media-enclosure-only items,
|
||||
/// [`NON_ARTICLE_HOSTS`], empty titles (§3.2).
|
||||
pub fn is_non_article(entry: &Entry) -> bool {
|
||||
if entry.title.trim().is_empty() {
|
||||
return true;
|
||||
}
|
||||
let Some(url) = Url::parse(entry.url.trim()).ok().filter(|u| {
|
||||
matches!(u.scheme(), "http" | "https") && u.host_str().is_some_and(|h| !h.is_empty())
|
||||
}) else {
|
||||
return true;
|
||||
};
|
||||
let host = url.host_str().unwrap_or_default().to_ascii_lowercase();
|
||||
if NON_ARTICLE_HOSTS
|
||||
.iter()
|
||||
.any(|blocked| host == *blocked || host.ends_with(&format!(".{blocked}")))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let path = url.path().to_ascii_lowercase();
|
||||
if MEDIA_EXTENSIONS.iter().any(|ext| path.ends_with(ext)) {
|
||||
return true;
|
||||
}
|
||||
is_enclosure_only(&entry.raw_content)
|
||||
}
|
||||
|
||||
/// Content that is nothing but an embedded player has no text worth reading (§3.2).
|
||||
fn is_enclosure_only(raw_content: &str) -> bool {
|
||||
let lower = raw_content.to_ascii_lowercase();
|
||||
let embeds = lower.contains("<audio")
|
||||
|| lower.contains("<video")
|
||||
|| lower.contains("<embed")
|
||||
|| lower.contains("<iframe");
|
||||
embeds && crate::extract::word_count(raw_content) < 25
|
||||
}
|
||||
|
||||
/// Classify which kind of feed an entry arrived through, for the sources list (§3.2, §3.5).
|
||||
pub fn classify_source(entry: &Entry) -> SourceKind {
|
||||
classify_source_with_feed(entry, None)
|
||||
}
|
||||
|
||||
/// [`classify_source`] with the feed's own URL/site URL when the caller has it.
|
||||
///
|
||||
/// The `entries` table does not store the feed URL, so the entry-only form falls
|
||||
/// back to the feed title plus the entry/comments URLs (§3.2).
|
||||
pub fn classify_source_with_feed(entry: &Entry, feed_url_or_site: Option<&str>) -> SourceKind {
|
||||
let mut haystack = String::new();
|
||||
if let Some(feed) = feed_url_or_site {
|
||||
haystack.push_str(&feed.to_ascii_lowercase());
|
||||
haystack.push(' ');
|
||||
}
|
||||
if let Some(title) = &entry.feed_title {
|
||||
haystack.push_str(&title.to_ascii_lowercase());
|
||||
haystack.push(' ');
|
||||
}
|
||||
if let Some(category) = &entry.category {
|
||||
haystack.push_str(&category.to_ascii_lowercase());
|
||||
haystack.push(' ');
|
||||
}
|
||||
let comments = entry
|
||||
.comments_url
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase();
|
||||
let url = entry.url.to_ascii_lowercase();
|
||||
|
||||
if haystack.contains("scour.ing") || haystack.contains("scour") {
|
||||
return SourceKind::Scour;
|
||||
}
|
||||
if haystack.contains("hnrss")
|
||||
|| haystack.contains("news.ycombinator")
|
||||
|| haystack.contains("hacker news")
|
||||
|| comments.contains("news.ycombinator.com")
|
||||
{
|
||||
return SourceKind::HnFrontpage;
|
||||
}
|
||||
if haystack.contains("lobste.rs")
|
||||
|| haystack.contains("lobsters")
|
||||
|| comments.contains("lobste.rs")
|
||||
|| url.contains("lobste.rs/s/")
|
||||
{
|
||||
return SourceKind::Lobsters;
|
||||
}
|
||||
if haystack.contains("reddit.com")
|
||||
|| haystack.contains("reddit")
|
||||
|| comments.contains("reddit.com")
|
||||
|| url.contains("reddit.com/r/")
|
||||
{
|
||||
return SourceKind::Reddit;
|
||||
}
|
||||
SourceKind::Feed
|
||||
}
|
||||
|
||||
/// `feed_id → feed URL (or site URL)`, as built by [`crate::miniflux::feed_urls`].
|
||||
///
|
||||
/// Passing it into [`cluster_with_feeds`] is what makes "came via Scour" exact:
|
||||
/// a Scour interest feed is only recognizable from its `feed_url`, and the
|
||||
/// `entries` table does not store one (§3.2).
|
||||
pub type FeedUrls = HashMap<crate::types::FeedId, String>;
|
||||
|
||||
/// Build a [`SourceRef`] describing how `entry` reached us.
|
||||
pub fn source_ref(entry: &Entry) -> SourceRef {
|
||||
source_ref_with_feeds(entry, &FeedUrls::new())
|
||||
}
|
||||
|
||||
/// [`source_ref`] with the run's `feed_id → feed url` map for exact classification.
|
||||
pub fn source_ref_with_feeds(entry: &Entry, feed_urls: &FeedUrls) -> SourceRef {
|
||||
SourceRef {
|
||||
entry_id: entry.id,
|
||||
feed_id: entry.feed_id,
|
||||
feed_title: entry
|
||||
.feed_title
|
||||
.clone()
|
||||
.filter(|t| !t.trim().is_empty())
|
||||
.unwrap_or_else(|| format!("feed {}", entry.feed_id)),
|
||||
category: entry.category.clone(),
|
||||
kind: classify_source_with_feed(entry, feed_urls.get(&entry.feed_id).map(String::as_str)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of the dedupe stage.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DedupeStats {
|
||||
pub entries_in: usize,
|
||||
pub dropped_non_article: usize,
|
||||
pub clusters: usize,
|
||||
/// Clusters that merged more than one entry.
|
||||
pub merged: usize,
|
||||
}
|
||||
|
||||
/// Cluster `entries` into [`Article`]s: primary key is the canonical URL, secondary
|
||||
/// pass matches [`normalized_title`] within the window. Each cluster keeps the
|
||||
/// richest content, the union of source refs and the earliest `first_seen` (§3.2).
|
||||
///
|
||||
/// The returned articles have `id == 0` (not yet persisted) and carry the richest
|
||||
/// *raw* Miniflux content in `content_html`; [`crate::extract`] replaces it with the
|
||||
/// sanitized body and the real `word_count`.
|
||||
pub fn cluster(entries: Vec<Entry>) -> (Vec<Article>, DedupeStats) {
|
||||
cluster_with_feeds(entries, &FeedUrls::new())
|
||||
}
|
||||
|
||||
/// [`cluster`] with the run's `feed_id → feed url` map, so `SourceKind::Scour`
|
||||
/// (and the other feed-shaped kinds) are detected from the feed URL rather than
|
||||
/// guessed from the feed title (§3.2).
|
||||
pub fn cluster_with_feeds(
|
||||
entries: Vec<Entry>,
|
||||
feed_urls: &FeedUrls,
|
||||
) -> (Vec<Article>, DedupeStats) {
|
||||
let span = tracing::info_span!("dedupe", entries = entries.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let mut stats = DedupeStats {
|
||||
entries_in: entries.len(),
|
||||
..DedupeStats::default()
|
||||
};
|
||||
|
||||
let mut clusters: Vec<Vec<(Entry, String)>> = Vec::new();
|
||||
let mut by_url: HashMap<String, usize> = HashMap::new();
|
||||
let mut by_title: HashMap<String, usize> = HashMap::new();
|
||||
|
||||
for entry in entries {
|
||||
if is_non_article(&entry) {
|
||||
stats.dropped_non_article += 1;
|
||||
continue;
|
||||
}
|
||||
let Some(canon) = canonical_url(&entry.url) else {
|
||||
stats.dropped_non_article += 1;
|
||||
continue;
|
||||
};
|
||||
let title_key = normalized_title(&entry.title);
|
||||
let title_key = (title_key.len() >= MIN_TITLE_KEY_LEN).then_some(title_key);
|
||||
|
||||
let index = match by_url.get(&canon) {
|
||||
Some(&i) => i,
|
||||
None => match title_key.as_ref().and_then(|k| by_title.get(k)) {
|
||||
Some(&i) => i,
|
||||
None => {
|
||||
clusters.push(Vec::new());
|
||||
clusters.len() - 1
|
||||
}
|
||||
},
|
||||
};
|
||||
by_url.entry(canon.clone()).or_insert(index);
|
||||
if let Some(key) = title_key {
|
||||
by_title.entry(key).or_insert(index);
|
||||
}
|
||||
clusters[index].push((entry, canon));
|
||||
}
|
||||
|
||||
let mut articles: Vec<Article> = Vec::with_capacity(clusters.len());
|
||||
for members in clusters {
|
||||
if members.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if members.len() > 1 {
|
||||
stats.merged += 1;
|
||||
}
|
||||
articles.push(build_article(members, feed_urls));
|
||||
}
|
||||
stats.clusters = articles.len();
|
||||
|
||||
tracing::info!(
|
||||
clusters = stats.clusters,
|
||||
merged = stats.merged,
|
||||
dropped = stats.dropped_non_article,
|
||||
"clustered entries into articles"
|
||||
);
|
||||
(articles, stats)
|
||||
}
|
||||
|
||||
/// Merge one cluster's entries into a single [`Article`], keeping the richest body.
|
||||
fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article {
|
||||
// Richest content wins; ties break on the lowest entry id so runs are stable.
|
||||
let best = members
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(_, (entry, _))| (crate::extract::word_count(&entry.raw_content), -(entry.id)))
|
||||
.map(|(i, _)| i)
|
||||
.unwrap_or(0);
|
||||
let (best_entry, canonical) = &members[best];
|
||||
|
||||
let first_seen = members
|
||||
.iter()
|
||||
.map(|(e, _)| e.published_at.unwrap_or(e.fetched_at))
|
||||
.min()
|
||||
.unwrap_or_else(Timestamp::now);
|
||||
let published_at = members.iter().filter_map(|(e, _)| e.published_at).min();
|
||||
|
||||
let mut sources: Vec<SourceRef> = members
|
||||
.iter()
|
||||
.map(|(e, _)| source_ref_with_feeds(e, feed_urls))
|
||||
.collect();
|
||||
sources.sort_by_key(|s| s.entry_id);
|
||||
sources.dedup_by_key(|s| s.entry_id);
|
||||
|
||||
// Prefer a comments URL that actually points at a discussion we can look up.
|
||||
let comments_url = members
|
||||
.iter()
|
||||
.filter_map(|(e, _)| e.comments_url.clone())
|
||||
.filter(|c| !c.trim().is_empty())
|
||||
.max_by_key(|c| {
|
||||
let lower = c.to_ascii_lowercase();
|
||||
if lower.contains("news.ycombinator.com") {
|
||||
2
|
||||
} else if lower.contains("lobste.rs") {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
});
|
||||
|
||||
let author = members
|
||||
.iter()
|
||||
.filter_map(|(e, _)| e.author.clone())
|
||||
.find(|a| !a.trim().is_empty());
|
||||
|
||||
let word_count = crate::extract::word_count(&best_entry.raw_content);
|
||||
|
||||
Article {
|
||||
id: 0,
|
||||
canonical_url: canonical.clone(),
|
||||
title: best_entry.title.trim().to_string(),
|
||||
best_entry_id: best_entry.id,
|
||||
content_html: best_entry.raw_content.clone(),
|
||||
word_count,
|
||||
excerpt_only: false,
|
||||
image_count: 0,
|
||||
sources,
|
||||
first_seen,
|
||||
url: best_entry.url.clone(),
|
||||
author,
|
||||
feed_id: best_entry.feed_id,
|
||||
feed_title: best_entry
|
||||
.feed_title
|
||||
.clone()
|
||||
.unwrap_or_else(|| format!("feed {}", best_entry.feed_id)),
|
||||
category: best_entry.category.clone(),
|
||||
published_at,
|
||||
comments_url,
|
||||
image_urls: Vec::new(),
|
||||
social: Vec::new(),
|
||||
extract_method: ExtractMethod::Miniflux,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
fn entry(id: i64, url: &str, title: &str) -> Entry {
|
||||
Entry {
|
||||
id,
|
||||
feed_id: id * 10,
|
||||
feed_title: Some(format!("Feed {id}")),
|
||||
category: Some("Tech".into()),
|
||||
title: title.into(),
|
||||
url: url.into(),
|
||||
canonical_url: None,
|
||||
author: None,
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
raw_content: "<p>hello world</p>".into(),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalization_table() {
|
||||
let cases: &[(&str, Option<&str>)] = &[
|
||||
// host case + fragment
|
||||
(
|
||||
"https://Example.COM/Posts/One#section",
|
||||
Some("https://example.com/Posts/One"),
|
||||
),
|
||||
// trailing slash
|
||||
("https://example.com/a/b/", Some("https://example.com/a/b")),
|
||||
// bare root loses its slash
|
||||
("https://example.com/", Some("https://example.com")),
|
||||
("http://example.com", Some("http://example.com")),
|
||||
// utm_* and friends
|
||||
(
|
||||
"https://example.com/p?utm_source=rss&utm_medium=feed&utm_campaign=x",
|
||||
Some("https://example.com/p"),
|
||||
),
|
||||
(
|
||||
"https://example.com/p?ref=hn&fbclid=abc&gclid=def&s=1&si=2",
|
||||
Some("https://example.com/p"),
|
||||
),
|
||||
// meaningful params survive
|
||||
(
|
||||
"https://example.com/p?id=7&utm_source=rss",
|
||||
Some("https://example.com/p?id=7"),
|
||||
),
|
||||
// mixed: everything at once
|
||||
(
|
||||
"HTTPS://WWW.Example.com/Path/?utm_source=a&page=2#frag",
|
||||
Some("https://www.example.com/Path?page=2"),
|
||||
),
|
||||
// google news redirector resolves to the target
|
||||
(
|
||||
"https://news.google.com/rss/articles/CBMi?oc=5&url=https%3A%2F%2Fexample.com%2Freal%2F",
|
||||
Some("https://example.com/real"),
|
||||
),
|
||||
(
|
||||
"https://www.google.com/url?q=https://example.com/real&sa=D",
|
||||
Some("https://example.com/real"),
|
||||
),
|
||||
// non-http schemes and junk
|
||||
("mailto:tyler@hallada.net", None),
|
||||
("ftp://example.com/file", None),
|
||||
("not a url", None),
|
||||
("", None),
|
||||
(" ", None),
|
||||
];
|
||||
for (input, expected) in cases {
|
||||
assert_eq!(
|
||||
canonical_url(input).as_deref(),
|
||||
*expected,
|
||||
"canonicalizing {input:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalization_is_idempotent() {
|
||||
let once = canonical_url("https://Example.com/A/?utm_source=x#y").unwrap();
|
||||
assert_eq!(canonical_url(&once).as_deref(), Some(once.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn title_normalization() {
|
||||
assert_eq!(
|
||||
normalized_title("Rust 1.90: What's *New*?"),
|
||||
"rust190whatsnew"
|
||||
);
|
||||
assert_eq!(
|
||||
normalized_title(" The Quick — Brown Fox "),
|
||||
"thequickbrownfox"
|
||||
);
|
||||
// Same story, different feed punctuation, same key.
|
||||
assert_eq!(
|
||||
normalized_title("Show HN: My Tiny Database"),
|
||||
normalized_title("Show HN – My Tiny Database!")
|
||||
);
|
||||
assert_eq!(normalized_title("!!!"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_articles_are_rejected() {
|
||||
let mut youtube = entry(1, "https://www.youtube.com/watch?v=abc", "A video");
|
||||
assert!(is_non_article(&youtube));
|
||||
youtube.url = "https://m.youtube.com/watch?v=abc".into();
|
||||
assert!(is_non_article(&youtube));
|
||||
|
||||
assert!(is_non_article(&entry(
|
||||
2,
|
||||
"https://open.spotify.com/episode/x",
|
||||
"An episode"
|
||||
)));
|
||||
assert!(is_non_article(&entry(3, "https://example.com/p", " ")));
|
||||
assert!(is_non_article(&entry(4, "javascript:void(0)", "Bad url")));
|
||||
assert!(is_non_article(&entry(
|
||||
5,
|
||||
"https://cdn.example.com/ep/12.mp3",
|
||||
"Episode 12"
|
||||
)));
|
||||
|
||||
let mut enclosure = entry(6, "https://example.com/pod/12", "Episode 12");
|
||||
enclosure.raw_content = "<audio src=\"https://x/1.mp3\"></audio>".into();
|
||||
assert!(is_non_article(&enclosure));
|
||||
|
||||
// An article that merely embeds a video is still an article.
|
||||
let mut with_video = entry(7, "https://example.com/post", "A real post");
|
||||
with_video.raw_content =
|
||||
format!("<iframe src=\"x\"></iframe><p>{}</p>", "word ".repeat(60));
|
||||
assert!(!is_non_article(&with_video));
|
||||
|
||||
assert!(!is_non_article(&entry(8, "https://example.com/p", "Fine")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_kinds_come_from_feed_metadata() {
|
||||
let mut e = entry(1, "https://example.com/p", "T");
|
||||
e.feed_title = Some("Scour: Rust".into());
|
||||
assert_eq!(classify_source(&e), SourceKind::Scour);
|
||||
|
||||
e.feed_title = Some("Hacker News: Front Page".into());
|
||||
assert_eq!(classify_source(&e), SourceKind::HnFrontpage);
|
||||
|
||||
e.feed_title = Some("Some Blog".into());
|
||||
e.comments_url = Some("https://news.ycombinator.com/item?id=1".into());
|
||||
assert_eq!(classify_source(&e), SourceKind::HnFrontpage);
|
||||
|
||||
e.comments_url = Some("https://lobste.rs/s/abcdef/thing".into());
|
||||
assert_eq!(classify_source(&e), SourceKind::Lobsters);
|
||||
|
||||
e.comments_url = None;
|
||||
e.feed_title = Some("r/rust".into());
|
||||
e.url = "https://www.reddit.com/r/rust/comments/x/y/".into();
|
||||
assert_eq!(classify_source(&e), SourceKind::Reddit);
|
||||
|
||||
e.feed_title = Some("Tyler's Blog".into());
|
||||
e.category = Some("Blogroll".into());
|
||||
e.url = "https://hallada.net/post".into();
|
||||
assert_eq!(classify_source(&e), SourceKind::Feed);
|
||||
|
||||
// The feed URL wins when the caller has it.
|
||||
assert_eq!(
|
||||
classify_source_with_feed(&e, Some("https://scour.ing/feed/rust")),
|
||||
SourceKind::Scour
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clustering_merges_by_url_then_title() {
|
||||
let long_body = format!("<p>{}</p>", "word ".repeat(400));
|
||||
|
||||
// Same story from three feeds: two share a URL (modulo tracking params),
|
||||
// the third differs only in punctuation of the title.
|
||||
let mut hn = entry(
|
||||
1,
|
||||
"https://blog.dev/post?utm_source=hn",
|
||||
"A Deep Dive Into B-Trees",
|
||||
);
|
||||
hn.feed_title = Some("Hacker News".into());
|
||||
hn.comments_url = Some("https://news.ycombinator.com/item?id=42".into());
|
||||
|
||||
let mut scour = entry(2, "https://blog.dev/post/", "A Deep Dive Into B-Trees");
|
||||
scour.feed_title = Some("Scour: Databases".into());
|
||||
scour.raw_content = long_body.clone();
|
||||
|
||||
let mut own = entry(3, "https://blog.dev/post-alt", "A Deep Dive into B-Trees!");
|
||||
own.feed_title = Some("Blog.dev".into());
|
||||
own.published_at = Some(ts("2026-08-15T02:00:00Z"));
|
||||
|
||||
let other = entry(4, "https://other.dev/x", "Something Else Entirely Here");
|
||||
|
||||
let (articles, stats) = cluster(vec![hn, scour, own, other]);
|
||||
assert_eq!(stats.entries_in, 4);
|
||||
assert_eq!(stats.clusters, 2);
|
||||
assert_eq!(stats.merged, 1);
|
||||
assert_eq!(stats.dropped_non_article, 0);
|
||||
|
||||
let merged = &articles[0];
|
||||
assert_eq!(merged.canonical_url, "https://blog.dev/post");
|
||||
assert_eq!(merged.sources.len(), 3);
|
||||
// Richest content won.
|
||||
assert_eq!(merged.best_entry_id, 2);
|
||||
assert!(merged.word_count > 300);
|
||||
// Union of source kinds, used as a curation signal.
|
||||
assert!(merged.came_via(SourceKind::Scour));
|
||||
assert!(merged.came_via(SourceKind::HnFrontpage));
|
||||
assert!(merged.came_via(SourceKind::Feed));
|
||||
// Earliest publication time and the HN comments link survive the merge.
|
||||
assert_eq!(merged.first_seen, ts("2026-08-15T02:00:00Z"));
|
||||
assert_eq!(
|
||||
merged.comments_url.as_deref(),
|
||||
Some("https://news.ycombinator.com/item?id=42")
|
||||
);
|
||||
assert_eq!(merged.id, 0);
|
||||
|
||||
assert_eq!(articles[1].canonical_url, "https://other.dev/x");
|
||||
assert_eq!(articles[1].sources.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clustering_drops_non_articles_and_keeps_short_titles_apart() {
|
||||
let mut a = entry(1, "https://a.dev/1", "News");
|
||||
a.raw_content = "<p>one</p>".into();
|
||||
let mut b = entry(2, "https://b.dev/2", "News");
|
||||
b.raw_content = "<p>two</p>".into();
|
||||
let video = entry(3, "https://youtu.be/xyz", "A video");
|
||||
|
||||
let (articles, stats) = cluster(vec![a, b, video]);
|
||||
assert_eq!(stats.dropped_non_article, 1);
|
||||
// "news" is below MIN_TITLE_KEY_LEN, so the two stay separate.
|
||||
assert_eq!(articles.len(), 2);
|
||||
assert_eq!(stats.merged, 0);
|
||||
}
|
||||
}
|
||||
+1440
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,715 @@
|
||||
//! Image download and re-encoding (spec §3.10 "Images").
|
||||
//!
|
||||
//! Failed downloads degrade to a `[image: alt text]` placeholder paragraph — the
|
||||
//! run never fails because of an image (notes §3).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use image::{DynamicImage, GenericImageView, ImageFormat};
|
||||
|
||||
use crate::types::{Edition, ImageAsset, Pick};
|
||||
|
||||
/// Per-image download timeout (§3.10).
|
||||
pub const DOWNLOAD_TIMEOUT_SECS: u64 = 10;
|
||||
/// Per-image size cap (§3.10).
|
||||
pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
|
||||
/// Concurrent downloads (§3.10).
|
||||
pub const CONCURRENCY: usize = 8;
|
||||
/// Whole-issue asset budget (§3.10).
|
||||
pub const ISSUE_ASSET_BUDGET_BYTES: usize = 25 * 1024 * 1024;
|
||||
/// Images smaller than this in either dimension are decorative — skipped (§3.10).
|
||||
pub const MIN_DIMENSION_PX: u32 = 24;
|
||||
/// Images referenced per article are already capped at 12 by extraction (§3.3).
|
||||
pub const MAX_IMAGES_PER_ARTICLE: usize = 12;
|
||||
|
||||
/// HTML void elements: XHTML requires them self-closed (§3.10 "valid XHTML").
|
||||
pub const VOID_ELEMENTS: &[&str] = &[
|
||||
"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source",
|
||||
"track", "wbr",
|
||||
];
|
||||
|
||||
/// Per-edition re-encoding parameters (§3.10).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ImageProfile {
|
||||
pub max_width: u32,
|
||||
pub max_height: u32,
|
||||
pub jpeg_quality: u8,
|
||||
pub grayscale: bool,
|
||||
}
|
||||
|
||||
impl ImageProfile {
|
||||
/// Standard edition: max width 1200px, JPEG q80, color (§3.10).
|
||||
pub const STANDARD: ImageProfile = ImageProfile {
|
||||
max_width: 1200,
|
||||
max_height: 4000,
|
||||
jpeg_quality: 80,
|
||||
grayscale: false,
|
||||
};
|
||||
|
||||
/// X4 edition: grayscale Luma8, fit within 480×800, JPEG q70 (§3.10).
|
||||
pub const X4: ImageProfile = ImageProfile {
|
||||
max_width: 480,
|
||||
max_height: 800,
|
||||
jpeg_quality: 70,
|
||||
grayscale: true,
|
||||
};
|
||||
|
||||
pub fn for_edition(edition: Edition) -> Self {
|
||||
match edition {
|
||||
Edition::Standard => Self::STANDARD,
|
||||
Edition::X4 => Self::X4,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One `<img>` found in article markup, with the caption of its `<figure>` if any.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ImgRef {
|
||||
pub src: String,
|
||||
pub alt: String,
|
||||
pub caption: Option<String>,
|
||||
}
|
||||
|
||||
/// Collect `<img>` references (src, alt, enclosing figcaption) from article markup.
|
||||
pub fn extract_img_refs(html: &str) -> Vec<ImgRef> {
|
||||
let doc = scraper::Html::parse_fragment(html);
|
||||
let Ok(img_sel) = scraper::Selector::parse("img") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let cap_sel = scraper::Selector::parse("figcaption").ok();
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
for el in doc.select(&img_sel) {
|
||||
let Some(src) = el.value().attr("src") else {
|
||||
continue;
|
||||
};
|
||||
let src = src.trim();
|
||||
if src.is_empty() || src.starts_with("data:") {
|
||||
continue;
|
||||
}
|
||||
if seen.iter().any(|s| s == src) {
|
||||
continue;
|
||||
}
|
||||
seen.push(src.to_string());
|
||||
let alt = el
|
||||
.value()
|
||||
.attr("alt")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
// Walk up to an enclosing <figure> and take its caption, if any.
|
||||
let mut caption = None;
|
||||
if let Some(cap_sel) = &cap_sel {
|
||||
let mut cursor = el.parent();
|
||||
while let Some(node) = cursor {
|
||||
if let Some(elem) = scraper::ElementRef::wrap(node) {
|
||||
if elem.value().name() == "figure" {
|
||||
caption = elem.select(cap_sel).next().map(|c| {
|
||||
c.text()
|
||||
.collect::<String>()
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
});
|
||||
break;
|
||||
}
|
||||
cursor = elem.parent();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(ImgRef {
|
||||
src: src.to_string(),
|
||||
alt,
|
||||
caption: caption.filter(|c| !c.is_empty()),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Download one image, honoring the timeout and size cap (§3.10).
|
||||
pub async fn download(http: &reqwest::Client, url: &str) -> Option<Vec<u8>> {
|
||||
let resp = http
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(DOWNLOAD_TIMEOUT_SECS))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::debug!(url, "image download failed: {e}"))
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
tracing::debug!(url, status = %resp.status(), "image download rejected");
|
||||
return None;
|
||||
}
|
||||
if let Some(len) = resp.content_length()
|
||||
&& len as usize > MAX_IMAGE_BYTES
|
||||
{
|
||||
tracing::debug!(url, len, "image exceeds the size cap");
|
||||
return None;
|
||||
}
|
||||
let mut resp = resp;
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
loop {
|
||||
match resp.chunk().await {
|
||||
Ok(Some(chunk)) => {
|
||||
if buf.len() + chunk.len() > MAX_IMAGE_BYTES {
|
||||
tracing::debug!(url, "image exceeds the size cap mid-stream");
|
||||
return None;
|
||||
}
|
||||
buf.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
tracing::debug!(url, "image download interrupted: {e}");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf.is_empty() { None } else { Some(buf) }
|
||||
}
|
||||
|
||||
/// Decode, resize/grayscale, flatten transparency to white and re-encode (§3.10).
|
||||
///
|
||||
/// Line art with transparency is kept as PNG after flattening; everything else
|
||||
/// becomes JPEG. Returns `None` for undecodable sources (SVG/WebP without support).
|
||||
pub fn reencode(bytes: &[u8], profile: ImageProfile) -> Option<(Vec<u8>, &'static str)> {
|
||||
let format = image::guess_format(bytes).ok();
|
||||
let decoded = image::load_from_memory(bytes)
|
||||
.map_err(|e| tracing::debug!("undecodable image: {e}"))
|
||||
.ok()?;
|
||||
|
||||
let (w, h) = decoded.dimensions();
|
||||
if w < MIN_DIMENSION_PX || h < MIN_DIMENSION_PX {
|
||||
tracing::debug!(w, h, "skipping decorative image");
|
||||
return None;
|
||||
}
|
||||
|
||||
let has_alpha = decoded.color().has_alpha();
|
||||
let flattened = if has_alpha {
|
||||
flatten_to_white(&decoded)
|
||||
} else {
|
||||
decoded
|
||||
};
|
||||
|
||||
let resized = if w > profile.max_width || h > profile.max_height {
|
||||
flattened.resize(
|
||||
profile.max_width,
|
||||
profile.max_height,
|
||||
image::imageops::FilterType::Lanczos3,
|
||||
)
|
||||
} else {
|
||||
flattened
|
||||
};
|
||||
|
||||
// Keep line art (PNG source, few distinct tones) lossless; everything else
|
||||
// becomes JPEG, which is far smaller for photographs (§3.10).
|
||||
let keep_png = format == Some(ImageFormat::Png) && is_line_art(&resized);
|
||||
|
||||
let mut out = Cursor::new(Vec::new());
|
||||
// NB: encode the concrete buffer, not the `DynamicImage` — the latter always
|
||||
// reports RGBA pixels, which would silently re-colorize a grayscale image.
|
||||
if profile.grayscale {
|
||||
let gray = resized.to_luma8();
|
||||
if keep_png {
|
||||
DynamicImage::ImageLuma8(gray)
|
||||
.write_to(&mut out, ImageFormat::Png)
|
||||
.ok()?;
|
||||
return Some((out.into_inner(), "image/png"));
|
||||
}
|
||||
let mut enc =
|
||||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
|
||||
enc.encode_image(&gray).ok()?;
|
||||
return Some((out.into_inner(), "image/jpeg"));
|
||||
}
|
||||
|
||||
let rgb = resized.to_rgb8();
|
||||
if keep_png {
|
||||
DynamicImage::ImageRgb8(rgb)
|
||||
.write_to(&mut out, ImageFormat::Png)
|
||||
.ok()?;
|
||||
return Some((out.into_inner(), "image/png"));
|
||||
}
|
||||
let mut enc =
|
||||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, profile.jpeg_quality);
|
||||
enc.encode_image(&rgb).ok()?;
|
||||
Some((out.into_inner(), "image/jpeg"))
|
||||
}
|
||||
|
||||
/// Composite over an opaque white page — e-ink has no transparency (§3.10).
|
||||
fn flatten_to_white(img: &DynamicImage) -> DynamicImage {
|
||||
let rgba = img.to_rgba8();
|
||||
let mut rgb = image::RgbImage::new(rgba.width(), rgba.height());
|
||||
for (x, y, px) in rgba.enumerate_pixels() {
|
||||
let a = f32::from(px[3]) / 255.0;
|
||||
let blend = |c: u8| {
|
||||
((f32::from(c) * a) + 255.0 * (1.0 - a))
|
||||
.round()
|
||||
.clamp(0.0, 255.0) as u8
|
||||
};
|
||||
rgb.put_pixel(x, y, image::Rgb([blend(px[0]), blend(px[1]), blend(px[2])]));
|
||||
}
|
||||
DynamicImage::ImageRgb8(rgb)
|
||||
}
|
||||
|
||||
/// Cheap line-art test: few distinct colors (diagrams, logos, screenshots of text).
|
||||
fn is_line_art(img: &DynamicImage) -> bool {
|
||||
const SAMPLE_LIMIT: usize = 20_000;
|
||||
const DISTINCT_LIMIT: usize = 64;
|
||||
let rgb = img.to_rgb8();
|
||||
let mut distinct: Vec<[u8; 3]> = Vec::with_capacity(DISTINCT_LIMIT + 1);
|
||||
for (i, px) in rgb.pixels().enumerate() {
|
||||
if i >= SAMPLE_LIMIT {
|
||||
break;
|
||||
}
|
||||
let c = [px[0], px[1], px[2]];
|
||||
if !distinct.contains(&c) {
|
||||
distinct.push(c);
|
||||
if distinct.len() > DISTINCT_LIMIT {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Everything needed to fetch one image, in deterministic issue order.
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingImage {
|
||||
id: String,
|
||||
url: String,
|
||||
alt: String,
|
||||
caption: Option<String>,
|
||||
}
|
||||
|
||||
fn pending_for_pick(pick: &Pick) -> Vec<PendingImage> {
|
||||
let entry_id = pick.article.best_entry_id;
|
||||
let mut refs = extract_img_refs(&pick.article.content_html);
|
||||
if refs.is_empty() {
|
||||
refs = pick
|
||||
.article
|
||||
.image_urls
|
||||
.iter()
|
||||
.map(|u| ImgRef {
|
||||
src: u.clone(),
|
||||
alt: String::new(),
|
||||
caption: None,
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
refs.into_iter()
|
||||
.filter(|r| r.src.starts_with("http://") || r.src.starts_with("https://"))
|
||||
.take(MAX_IMAGES_PER_ARTICLE)
|
||||
.enumerate()
|
||||
.map(|(i, r)| PendingImage {
|
||||
id: format!("img-{entry_id}-{i}"),
|
||||
url: r.src,
|
||||
alt: r.alt,
|
||||
caption: r.caption,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Download and re-encode every image referenced by the lineup for one edition,
|
||||
/// respecting [`ISSUE_ASSET_BUDGET_BYTES`] (§3.10).
|
||||
pub async fn collect_for_issue(
|
||||
http: &reqwest::Client,
|
||||
picks: &[Pick],
|
||||
edition: Edition,
|
||||
) -> Vec<ImageAsset> {
|
||||
let profile = ImageProfile::for_edition(edition);
|
||||
let pending: Vec<PendingImage> = picks.iter().flat_map(pending_for_pick).collect();
|
||||
if pending.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
tracing::info!(count = pending.len(), ?edition, "downloading issue images");
|
||||
|
||||
let results: Vec<Option<(PendingImage, Vec<u8>, &'static str)>> =
|
||||
futures::stream::iter(pending.into_iter().map(|p| {
|
||||
let http = http.clone();
|
||||
async move {
|
||||
let raw = download(&http, &p.url).await?;
|
||||
let (bytes, mime) = tokio::task::spawn_blocking(move || reencode(&raw, profile))
|
||||
.await
|
||||
.ok()
|
||||
.flatten()?;
|
||||
Some((p, bytes, mime))
|
||||
}
|
||||
}))
|
||||
.buffered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut assets = Vec::new();
|
||||
let mut budget_used = 0usize;
|
||||
let mut skipped = 0usize;
|
||||
for result in results.into_iter().flatten() {
|
||||
let (pending, bytes, mime) = result;
|
||||
if budget_used + bytes.len() > ISSUE_ASSET_BUDGET_BYTES {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
budget_used += bytes.len();
|
||||
let ext = if mime == "image/png" { "png" } else { "jpg" };
|
||||
assets.push(ImageAsset {
|
||||
href: format!("images/{}.{ext}", pending.id),
|
||||
id: pending.id,
|
||||
mime: mime.to_string(),
|
||||
data: bytes,
|
||||
alt: pending.alt,
|
||||
caption: pending.caption,
|
||||
source_url: pending.url,
|
||||
});
|
||||
}
|
||||
if skipped > 0 {
|
||||
tracing::warn!(skipped, budget_used, "issue image budget exhausted");
|
||||
}
|
||||
tracing::info!(
|
||||
embedded = assets.len(),
|
||||
bytes = budget_used,
|
||||
"issue images ready"
|
||||
);
|
||||
assets
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Markup rewriting
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// End index (exclusive) of the tag starting at `start` (`html[start] == '<'`),
|
||||
/// respecting quoted attribute values and comments.
|
||||
pub(crate) fn tag_end(html: &str, start: usize) -> Option<usize> {
|
||||
let rest = &html[start..];
|
||||
if rest.starts_with("<!--") {
|
||||
return rest.find("-->").map(|i| start + i + 3);
|
||||
}
|
||||
let mut quote: Option<char> = None;
|
||||
for (i, c) in rest.char_indices().skip(1) {
|
||||
match (quote, c) {
|
||||
(Some(q), c) if c == q => quote = None,
|
||||
(Some(_), _) => {}
|
||||
(None, '"') | (None, '\'') => quote = Some(c),
|
||||
(None, '>') => return Some(start + i + c.len_utf8()),
|
||||
(None, _) => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Lowercased element name of a tag body such as `img src="…"`.
|
||||
pub(crate) fn tag_name(inner: &str) -> String {
|
||||
inner
|
||||
.trim_start_matches('/')
|
||||
.split(|c: char| c.is_whitespace() || c == '/' || c == '>')
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
/// Parse `name="value"` pairs out of a tag body.
|
||||
fn parse_attrs(inner: &str) -> Vec<(String, String)> {
|
||||
let mut attrs = Vec::new();
|
||||
let bytes: Vec<char> = inner.chars().collect();
|
||||
let mut i = 0;
|
||||
// Skip the element name.
|
||||
while i < bytes.len() && !bytes[i].is_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
while i < bytes.len() {
|
||||
while i < bytes.len() && (bytes[i].is_whitespace() || bytes[i] == '/') {
|
||||
i += 1;
|
||||
}
|
||||
let name_start = i;
|
||||
while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '=' && bytes[i] != '/' {
|
||||
i += 1;
|
||||
}
|
||||
if i == name_start {
|
||||
break;
|
||||
}
|
||||
let name: String = bytes[name_start..i]
|
||||
.iter()
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
while i < bytes.len() && bytes[i].is_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
let mut value = String::new();
|
||||
if i < bytes.len() && bytes[i] == '=' {
|
||||
i += 1;
|
||||
while i < bytes.len() && bytes[i].is_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
if i < bytes.len() && (bytes[i] == '"' || bytes[i] == '\'') {
|
||||
let quote = bytes[i];
|
||||
i += 1;
|
||||
while i < bytes.len() && bytes[i] != quote {
|
||||
value.push(bytes[i]);
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
} else {
|
||||
while i < bytes.len() && !bytes[i].is_whitespace() && bytes[i] != '>' {
|
||||
value.push(bytes[i]);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
attrs.push((name, value));
|
||||
}
|
||||
attrs
|
||||
}
|
||||
|
||||
/// Escape a string for use inside a double-quoted XML attribute.
|
||||
fn attr_escape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
'"' => out.push_str("""),
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Escape a string for XML text content.
|
||||
pub fn text_escape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Rewrite `<img src>` to the embedded hrefs, replacing misses with the
|
||||
/// `[image: alt]` placeholder paragraph (§3.10).
|
||||
pub fn rewrite_img_srcs(html: &str, assets: &[ImageAsset]) -> String {
|
||||
let by_url: HashMap<&str, &ImageAsset> =
|
||||
assets.iter().map(|a| (a.source_url.as_str(), a)).collect();
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
let inner = raw
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>')
|
||||
.trim_end_matches('/');
|
||||
if tag_name(inner) == "img" {
|
||||
let attrs = parse_attrs(inner);
|
||||
let src = attrs
|
||||
.iter()
|
||||
.find(|(k, _)| k == "src")
|
||||
.map(|(_, v)| v.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
let alt = attrs
|
||||
.iter()
|
||||
.find(|(k, _)| k == "alt")
|
||||
.map(|(_, v)| v.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
match by_url.get(src.as_str()) {
|
||||
Some(asset) => {
|
||||
let alt = if alt.is_empty() { &asset.alt } else { &alt };
|
||||
out.push_str(&format!(
|
||||
"<img src=\"{}\" alt=\"{}\"/>",
|
||||
attr_escape(&asset.href),
|
||||
attr_escape(alt)
|
||||
));
|
||||
}
|
||||
None => {
|
||||
let label = if alt.is_empty() {
|
||||
"image unavailable"
|
||||
} else {
|
||||
&alt
|
||||
};
|
||||
out.push_str(&format!(
|
||||
"<p class=\"image-placeholder\">[image: {}]</p>",
|
||||
text_escape(label)
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.push_str(raw);
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Self-close HTML void elements and normalize ` ` so the markup parses as
|
||||
/// XML — EPUB3 content documents are XHTML (§3.10).
|
||||
pub fn to_xhtml(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
cursor = html.len();
|
||||
break;
|
||||
};
|
||||
let raw = &html[start..end];
|
||||
let inner = raw.trim_start_matches('<').trim_end_matches('>');
|
||||
let name = tag_name(inner);
|
||||
if VOID_ELEMENTS.contains(&name.as_str()) && !inner.trim_end().ends_with('/') {
|
||||
out.push('<');
|
||||
out.push_str(inner.trim_end());
|
||||
out.push_str("/>");
|
||||
} else {
|
||||
out.push_str(raw);
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
// html5ever (via ammonia) emits ` `, which is undefined in XML.
|
||||
out.replace(" ", " ")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn asset(url: &str, href: &str) -> ImageAsset {
|
||||
ImageAsset {
|
||||
id: "img-1-0".into(),
|
||||
href: href.into(),
|
||||
mime: "image/jpeg".into(),
|
||||
data: vec![1, 2, 3],
|
||||
alt: "fallback alt".into(),
|
||||
caption: None,
|
||||
source_url: url.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_img_refs_with_captions() {
|
||||
let html = r#"<p>hi</p>
|
||||
<figure><img src="https://e.g/a.png" alt="A diagram"/>
|
||||
<figcaption>Figure 1: the thing</figcaption></figure>
|
||||
<img src="https://e.g/b.jpg"/>
|
||||
<img src="data:image/png;base64,zz"/>
|
||||
<img src="https://e.g/a.png" alt="dupe"/>"#;
|
||||
let refs = extract_img_refs(html);
|
||||
assert_eq!(refs.len(), 2);
|
||||
assert_eq!(refs[0].src, "https://e.g/a.png");
|
||||
assert_eq!(refs[0].alt, "A diagram");
|
||||
assert_eq!(refs[0].caption.as_deref(), Some("Figure 1: the thing"));
|
||||
assert_eq!(refs[1].src, "https://e.g/b.jpg");
|
||||
assert!(refs[1].caption.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_hits_and_placeholders_misses() {
|
||||
let assets = vec![asset("https://e.g/a.png", "images/img-1-0.jpg")];
|
||||
let html = r#"<p>x</p><img src="https://e.g/a.png" alt="Alt & more"><img src="https://e.g/gone.png" alt="Missing">"#;
|
||||
let out = rewrite_img_srcs(html, &assets);
|
||||
assert!(out.contains(r#"<img src="images/img-1-0.jpg" alt="Alt &amp; more"/>"#));
|
||||
assert!(out.contains(r#"<p class="image-placeholder">[image: Missing]</p>"#));
|
||||
assert!(!out.contains("gone.png"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placeholder_falls_back_when_alt_is_missing() {
|
||||
let out = rewrite_img_srcs(r#"<img src="https://e.g/x.png">"#, &[]);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"<p class="image-placeholder">[image: image unavailable]</p>"#
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_xhtml_self_closes_voids_and_entities() {
|
||||
let html = "<p>a<br>b<hr>c d<img src=\"x.png\" alt=\"y\"></p><p>e<br/></p>";
|
||||
let out = to_xhtml(html);
|
||||
assert!(out.contains("<br/>"));
|
||||
assert!(out.contains("<hr/>"));
|
||||
assert!(out.contains("<img src=\"x.png\" alt=\"y\"/>"));
|
||||
assert!(out.contains(" "));
|
||||
assert!(!out.contains(" "));
|
||||
assert!(!out.contains("<br/ >"));
|
||||
// Already-closed voids are left alone (no double slash).
|
||||
assert_eq!(out.matches("<br/>").count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag_scanner_ignores_angle_brackets_in_attributes() {
|
||||
let html = r#"<a title="a > b">x</a>"#;
|
||||
assert_eq!(to_xhtml(html), html);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reencode_resizes_grayscales_and_encodes() {
|
||||
let mut img = image::RgbaImage::new(200, 100);
|
||||
for (x, y, px) in img.enumerate_pixels_mut() {
|
||||
*px = image::Rgba([(x % 256) as u8, (y % 256) as u8, 128, 255]);
|
||||
}
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(img)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let raw = png.into_inner();
|
||||
|
||||
let (std_bytes, std_mime) = reencode(&raw, ImageProfile::STANDARD).unwrap();
|
||||
assert_eq!(std_mime, "image/jpeg");
|
||||
let decoded = image::load_from_memory(&std_bytes).unwrap();
|
||||
assert_eq!(decoded.dimensions(), (200, 100), "no upscaling");
|
||||
|
||||
let (x4_bytes, _) = reencode(&raw, ImageProfile::X4).unwrap();
|
||||
let x4 = image::load_from_memory(&x4_bytes).unwrap();
|
||||
assert!(x4.width() <= 480 && x4.height() <= 800);
|
||||
assert_eq!(x4.color(), image::ColorType::L8, "X4 is grayscale Luma8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reencode_skips_decorative_images_and_junk() {
|
||||
let tiny = image::RgbaImage::new(8, 8);
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(tiny)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
assert!(reencode(&png.into_inner(), ImageProfile::STANDARD).is_none());
|
||||
assert!(reencode(b"<svg>not an image</svg>", ImageProfile::STANDARD).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_art_png_stays_png_and_is_flattened() {
|
||||
let mut img = image::RgbaImage::new(120, 60);
|
||||
for (x, _y, px) in img.enumerate_pixels_mut() {
|
||||
*px = if x % 12 == 0 {
|
||||
image::Rgba([0, 0, 0, 255])
|
||||
} else {
|
||||
image::Rgba([255, 255, 255, 0])
|
||||
};
|
||||
}
|
||||
let mut png = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(img)
|
||||
.write_to(&mut png, ImageFormat::Png)
|
||||
.unwrap();
|
||||
let (bytes, mime) = reencode(&png.into_inner(), ImageProfile::STANDARD).unwrap();
|
||||
assert_eq!(mime, "image/png");
|
||||
let decoded = image::load_from_memory(&bytes).unwrap();
|
||||
assert!(!decoded.color().has_alpha(), "transparency is flattened");
|
||||
// Transparent pixels became white.
|
||||
assert_eq!(
|
||||
decoded.to_rgb8().get_pixel(1, 1),
|
||||
&image::Rgb([255, 255, 255])
|
||||
);
|
||||
}
|
||||
}
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
//! EPUB assembly (spec §3.10).
|
||||
//!
|
||||
//! Two editions per issue: `Standard` and `X4`. Both are fully offline (every
|
||||
//! asset embedded), EPUB3 with a nav TOC + NCX fallback, chapter ids
|
||||
//! `art-{entry_id}` so rating links stay stable across regenerations.
|
||||
|
||||
pub mod build;
|
||||
pub mod images;
|
||||
pub mod x4;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::types::{Artifact, Edition, ImageAsset, Issue};
|
||||
|
||||
/// Chapter order inside an issue (§3.10).
|
||||
pub const CHAPTER_ORDER: &[&str] = &[
|
||||
"cover",
|
||||
"from-the-editor",
|
||||
"in-this-issue",
|
||||
"sections",
|
||||
"world-briefing",
|
||||
"colophon",
|
||||
];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum EpubError {
|
||||
#[error("epub build failed: {0}")]
|
||||
Build(String),
|
||||
#[error("template rendering failed: {0}")]
|
||||
Template(#[from] askama::Error),
|
||||
#[error("io error writing {path}: {source}")]
|
||||
Io {
|
||||
path: PathBuf,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// Output filename: `The Daily EPUB - 2026-08-15.epub` / `… (X4).epub` (§3.11).
|
||||
pub fn output_filename(issue: &Issue, edition: Edition) -> String {
|
||||
format!(
|
||||
"The Daily EPUB - {}{}.epub",
|
||||
issue.meta.date,
|
||||
edition.file_suffix()
|
||||
)
|
||||
}
|
||||
|
||||
/// Build one edition into `out_dir`, returning the written artifact (§3.10).
|
||||
///
|
||||
/// Downloads and re-encodes the issue's images first; everything else is offline.
|
||||
pub async fn build_edition(
|
||||
issue: &Issue,
|
||||
edition: Edition,
|
||||
cfg: &Config,
|
||||
out_dir: &Path,
|
||||
) -> Result<Artifact, EpubError> {
|
||||
let http = crate::http::build_client(crate::http::DEFAULT_TIMEOUT)
|
||||
.map_err(|e| EpubError::Build(format!("http client: {e}")))?;
|
||||
let assets = images::collect_for_issue(&http, &issue.lineup.picks, edition).await;
|
||||
build_edition_with_images(issue, edition, cfg, out_dir, &assets)
|
||||
}
|
||||
|
||||
/// The offline half of [`build_edition`]: render, zip and write (§3.10).
|
||||
pub fn build_edition_with_images(
|
||||
issue: &Issue,
|
||||
edition: Edition,
|
||||
cfg: &Config,
|
||||
out_dir: &Path,
|
||||
assets: &[ImageAsset],
|
||||
) -> Result<Artifact, EpubError> {
|
||||
let span = tracing::info_span!("epub", %issue.meta.date, ?edition);
|
||||
let _guard = span.enter();
|
||||
|
||||
let chapters = build::render_all(
|
||||
issue,
|
||||
edition,
|
||||
assets,
|
||||
&cfg.server.public_url,
|
||||
cfg.server.hmac_secret.as_deref(),
|
||||
)?;
|
||||
let cover = build::render_cover(issue, edition)?;
|
||||
let bytes = build::assemble(issue, edition, &chapters, assets, &cover)?;
|
||||
|
||||
std::fs::create_dir_all(out_dir).map_err(|source| EpubError::Io {
|
||||
path: out_dir.to_path_buf(),
|
||||
source,
|
||||
})?;
|
||||
let path = out_dir.join(output_filename(issue, edition));
|
||||
// Write + rename so a reader (or BookOrbit's watcher) never sees a partial file.
|
||||
let tmp = path.with_extension("epub.part");
|
||||
std::fs::write(&tmp, &bytes).map_err(|source| EpubError::Io {
|
||||
path: tmp.clone(),
|
||||
source,
|
||||
})?;
|
||||
std::fs::rename(&tmp, &path).map_err(|source| EpubError::Io {
|
||||
path: path.clone(),
|
||||
source,
|
||||
})?;
|
||||
|
||||
tracing::info!(
|
||||
path = %path.display(),
|
||||
bytes = bytes.len(),
|
||||
chapters = chapters.len(),
|
||||
images = assets.len(),
|
||||
"wrote edition"
|
||||
);
|
||||
Ok(Artifact {
|
||||
edition,
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build both editions, returning the artifacts and how many images were
|
||||
/// embedded across them (the run report records the count, §3.10, §3.13).
|
||||
pub async fn build_all(
|
||||
issue: &Issue,
|
||||
cfg: &Config,
|
||||
out_dir: &Path,
|
||||
) -> Result<(Vec<Artifact>, usize), EpubError> {
|
||||
let http = crate::http::build_client(crate::http::DEFAULT_TIMEOUT)
|
||||
.map_err(|e| EpubError::Build(format!("http client: {e}")))?;
|
||||
let mut artifacts = Vec::with_capacity(2);
|
||||
let mut embedded = 0;
|
||||
for edition in [Edition::Standard, Edition::X4] {
|
||||
// Downloaded per edition: the two editions need different resolutions
|
||||
// and colour profiles (§3.10 images).
|
||||
let assets = images::collect_for_issue(&http, &issue.lineup.picks, edition).await;
|
||||
embedded += assets.len();
|
||||
artifacts.push(build_edition_with_images(
|
||||
issue, edition, cfg, out_dir, &assets,
|
||||
)?);
|
||||
}
|
||||
Ok((artifacts, embedded))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::epub::build::fixtures;
|
||||
|
||||
/// Local file headers store entry names verbatim, so a byte search over the
|
||||
/// archive is enough to assert its contents without a zip reader.
|
||||
fn contains_entry(zip: &[u8], name: &str) -> bool {
|
||||
zip.windows(name.len()).any(|w| w == name.as_bytes())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_filenames_follow_the_spec() {
|
||||
let issue = fixtures::issue();
|
||||
assert_eq!(
|
||||
output_filename(&issue, Edition::Standard),
|
||||
"The Daily EPUB - 2026-08-15.epub"
|
||||
);
|
||||
assert_eq!(
|
||||
output_filename(&issue, Edition::X4),
|
||||
"The Daily EPUB - 2026-08-15 (X4).epub"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_a_complete_epub_for_both_editions() {
|
||||
let issue = fixtures::issue();
|
||||
let cfg = Config::default();
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
|
||||
for edition in [Edition::Standard, Edition::X4] {
|
||||
let artifact =
|
||||
build_edition_with_images(&issue, edition, &cfg, dir.path(), &[]).expect("build");
|
||||
assert_eq!(artifact.edition, edition);
|
||||
assert!(artifact.path.exists());
|
||||
assert!(artifact.bytes > 1000);
|
||||
|
||||
let zip = std::fs::read(&artifact.path).expect("read epub");
|
||||
assert_eq!(&zip[0..4], b"PK\x03\x04", "is a zip");
|
||||
assert_eq!(&zip[30..38], b"mimetype", "mimetype is the first entry");
|
||||
assert_eq!(&zip[38..58], b"application/epub+zip");
|
||||
for entry in [
|
||||
"META-INF/container.xml",
|
||||
"OEBPS/content.opf",
|
||||
"OEBPS/toc.ncx",
|
||||
"OEBPS/nav.xhtml",
|
||||
"OEBPS/stylesheet.css",
|
||||
"OEBPS/cover.png",
|
||||
"OEBPS/cover.xhtml",
|
||||
"OEBPS/front.xhtml",
|
||||
"OEBPS/in-this-issue.xhtml",
|
||||
"OEBPS/art-1001.xhtml",
|
||||
"OEBPS/disc-1001.xhtml",
|
||||
"OEBPS/art-1002.xhtml",
|
||||
"OEBPS/world.xhtml",
|
||||
"OEBPS/colophon.xhtml",
|
||||
] {
|
||||
assert!(
|
||||
contains_entry(&zip, entry),
|
||||
"missing {entry} in {edition:?}"
|
||||
);
|
||||
}
|
||||
// No leftover temp file.
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join("The Daily EPUB - 2026-08-15.epub.part")
|
||||
.exists()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
# EPUB templates
|
||||
|
||||
Askama templates and stylesheets for the two editions (spec §3.10, implementation
|
||||
notes §11). `src/epub/build.rs` owns the structs they bind to; `askama.toml` at
|
||||
the crate root points askama here (`dirs = ["src/epub/templates"]`).
|
||||
|
||||
| File | Template struct | Purpose |
|
||||
|---|---|---|
|
||||
| `base.xhtml` | — | Shared XHTML skeleton (`{% block body_class %}`, `{% block content %}`) |
|
||||
| `cover_page.xhtml` | `CoverPage` | Page that displays the rasterized cover image |
|
||||
| `front_page.xhtml` | `FrontPage` | "From the Editor" + issue stats line |
|
||||
| `in_this_issue.xhtml` | `InThisIssue` | Introduction chapter: per-section linked index |
|
||||
| `section.xhtml` | `SectionPage` | Section title page + LLM intro |
|
||||
| `chapter.xhtml` | `ArticleChapter` | Article: header, body, rating/read-online footer |
|
||||
| `discussion.xhtml` | `DiscussionChapter` | Comment chapter (§3.7); body from `comments::render_xhtml` |
|
||||
| `world_briefing.xhtml` | `WorldBriefingChapter` | Wikipedia Current Events (§3.8), body from `world::render_xhtml` |
|
||||
| `colophon.xhtml` | `ColophonChapter` | Back matter: models, cost, counts |
|
||||
| `cover.svg` | `CoverSvg` | Typographic cover, rasterized with resvg + tiny-skia |
|
||||
| `style.css` | — | Standard-edition stylesheet, embedded as `stylesheet.css` |
|
||||
| `style-x4.css` | — | X4 stylesheet: no floats/flex/grid, no fonts, hyphenation on |
|
||||
|
||||
Conventions:
|
||||
|
||||
- Every content template `{% extends "base.xhtml" %}` and provides `title`.
|
||||
- Templates declare `escape = "html"` — `.xhtml`/`.svg` are not in askama's
|
||||
default escaper extension list. Escaping emits numeric character references
|
||||
(`&`), which are valid XML; only pre-sanitized markup uses `|safe`.
|
||||
- Markup that reaches `|safe` has gone through `ammonia` **and**
|
||||
`epub::images::to_xhtml` (void elements self-closed, ` ` → ` `) so
|
||||
the output parses as XML, as EPUB3 content documents must.
|
||||
- Entities other than the five XML built-ins are written as numeric references
|
||||
in the templates themselves (`·`, `👍`, …).
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>{{ title }}</title>
|
||||
<link rel="stylesheet" type="text/css" href="stylesheet.css"/>
|
||||
</head>
|
||||
<body>
|
||||
<div class="chapter {% block body_class %}text{% endblock %}">
|
||||
{% block content %}{% endblock %}
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,34 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}article{% endblock %}
|
||||
{% block content %}
|
||||
<div class="article-header">
|
||||
<h1 class="article-title">{{ article_title }}</h1>
|
||||
{% if let Some(line) = byline %}
|
||||
<p class="byline">{{ line }}</p>
|
||||
{% endif %}
|
||||
<p class="meta">{{ meta_line }}</p>
|
||||
{% if let Some(line) = social_line %}
|
||||
<p class="social">{{ line }}</p>
|
||||
{% endif %}
|
||||
{% if let Some(text) = summary %}
|
||||
<p class="summary">{{ text }}</p>
|
||||
{% endif %}
|
||||
{% if excerpt_only %}
|
||||
<p class="notice">(excerpt only — read online)</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
<hr class="rule"/>
|
||||
<div class="article-body">
|
||||
{{ body_html|safe }}
|
||||
</div>
|
||||
<hr class="rule"/>
|
||||
<div class="article-footer">
|
||||
{% if let Some(links) = rating %}
|
||||
<p class="rating">Was this a good pick? <a href="{{ links.up_url }}">[ 👍 Yes ]</a> · <a href="{{ links.down_url }}">[ 👎 No ]</a></p>
|
||||
{% endif %}
|
||||
<p class="read-online"><a href="{{ read_online_url }}">Read online ↗</a></p>
|
||||
{% if let Some(href) = discussion_href %}
|
||||
<p class="see-discussion"><a href="{{ href }}">💬 Read the discussion</a></p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,25 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}colophon{% endblock %}
|
||||
{% block content %}
|
||||
<h1>Colophon</h1>
|
||||
<p>
|
||||
<em>The Daily EPUB</em> is assembled every morning from a personal Miniflux
|
||||
feed reader: entries are deduplicated, read in full, weighed against social
|
||||
proof, then scored, sectioned and introduced by a language model.
|
||||
</p>
|
||||
<dl class="colophon-facts">
|
||||
<dt class="fact-key">Issue</dt><dd class="fact-value">No. {{ issue_number }} · {{ display_date }}</dd>
|
||||
<dt class="fact-key">Generated</dt><dd class="fact-value">{{ generated_at }}</dd>
|
||||
<dt class="fact-key">Curation model</dt><dd class="fact-value">{{ model }}</dd>
|
||||
<dt class="fact-key">Entries considered</dt><dd class="fact-value">{{ entries_fetched }} from {{ feeds_seen }} feeds</dd>
|
||||
<dt class="fact-key">Candidates scored</dt><dd class="fact-value">{{ candidates }}</dd>
|
||||
<dt class="fact-key">Articles selected</dt><dd class="fact-value">{{ article_count }} across {{ section_count }} sections</dd>
|
||||
<dt class="fact-key">Words</dt><dd class="fact-value">{{ total_words }} · {{ reading_line }}</dd>
|
||||
<dt class="fact-key">Token cost</dt><dd class="fact-value">{{ cost_usd }}</dd>
|
||||
<dt class="fact-key">Generator</dt><dd class="fact-value">{{ generator_version }}</dd>
|
||||
</dl>
|
||||
<p class="attribution">
|
||||
Article text belongs to its authors and publications; excerpts and links are
|
||||
provided for personal reading. Comment excerpts belong to their posters.
|
||||
</p>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,25 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="{{ width }}" height="{{ height }}" viewBox="0 0 {{ width }} {{ height }}">
|
||||
<rect x="0" y="0" width="{{ width }}" height="{{ height }}" fill="#ffffff"/>
|
||||
<rect x="{{ margin }}" y="{{ margin }}" width="{{ inner_width }}" height="{{ inner_height }}"
|
||||
fill="none" stroke="#111111" stroke-width="{{ border }}"/>
|
||||
<text x="{{ center_x }}" y="{{ masthead_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ masthead_size }}">The Daily EPUB</text>
|
||||
<line x1="{{ rule_x1 }}" y1="{{ rule_y }}" x2="{{ rule_x2 }}" y2="{{ rule_y }}" stroke="#111111" stroke-width="{{ border }}"/>
|
||||
<text x="{{ center_x }}" y="{{ weekday_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ weekday_size }}">{{ weekday }}</text>
|
||||
<text x="{{ center_x }}" y="{{ date_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ date_size }}">{{ long_date }}</text>
|
||||
<line x1="{{ rule_x1 }}" y1="{{ rule2_y }}" x2="{{ rule_x2 }}" y2="{{ rule2_y }}" stroke="#111111" stroke-width="{{ hairline }}"/>
|
||||
<text x="{{ center_x }}" y="{{ issue_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ issue_size }}">No. {{ issue_number }}</text>
|
||||
<text x="{{ center_x }}" y="{{ stats_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ stats_size }}">{{ stats_line }}</text>
|
||||
{% if !edition_tag.is_empty() %}
|
||||
<rect x="{{ badge_x }}" y="{{ badge_y }}" width="{{ badge_width }}" height="{{ badge_height }}"
|
||||
rx="{{ badge_radius }}" fill="#111111"/>
|
||||
<text x="{{ center_x }}" y="{{ badge_text_y }}" text-anchor="middle" fill="#ffffff"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ badge_size }}">{{ edition_tag }}</text>
|
||||
{% endif %}
|
||||
<text x="{{ center_x }}" y="{{ footer_y }}" text-anchor="middle" fill="#111111"
|
||||
font-family="Georgia, 'Times New Roman', Times, serif" font-size="{{ footer_size }}">{{ footer }}</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
@@ -0,0 +1,5 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}cover-page{% endblock %}
|
||||
{% block content %}
|
||||
<div class="cover-image"><img src="cover.png" alt="{{ alt }}"/></div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,8 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}discussion{% endblock %}
|
||||
{% block content %}
|
||||
<h1 class="discussion-title">{{ heading }}</h1>
|
||||
<p class="discussion-note">Selected threads, truncated for reading on e-ink.</p>
|
||||
{{ body_html|safe }}
|
||||
<p class="back-link"><a href="{{ article_href }}">↩ Back to the article</a></p>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,12 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}front-page{% endblock %}
|
||||
{% block content %}
|
||||
<h1 class="masthead">The Daily EPUB</h1>
|
||||
<p class="dateline">{{ display_date }} · No. {{ issue_number }}</p>
|
||||
<hr class="rule"/>
|
||||
<h2 class="kicker">From the Editor</h2>
|
||||
<div class="editorial">
|
||||
{{ body_html|safe }}
|
||||
</div>
|
||||
<p class="stats">{{ stats_line }}</p>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,20 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}in-this-issue{% endblock %}
|
||||
{% block content %}
|
||||
<h1>In This Issue</h1>
|
||||
<p class="stats">{{ stats_line }}</p>
|
||||
{% for section in sections %}
|
||||
<h2 class="index-section">{{ section.name }}</h2>
|
||||
<ul class="index-list">
|
||||
{% for entry in section.entries %}
|
||||
<li class="index-entry">
|
||||
<p class="index-title"><a href="{{ entry.href }}">{{ entry.title }}</a></p>
|
||||
<p class="index-meta">{{ entry.source }} · {{ entry.reading_minutes }} min read</p>
|
||||
{% if !entry.summary.is_empty() %}
|
||||
<p class="index-summary">{{ entry.summary }}</p>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endfor %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,9 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}section-page{% endblock %}
|
||||
{% block content %}
|
||||
<h1 class="section-title">{{ name }}</h1>
|
||||
<hr class="rule"/>
|
||||
{% if let Some(text) = intro %}
|
||||
<p class="section-intro">{{ text }}</p>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,188 @@
|
||||
/* Xteink X4 stylesheet (spec §3.10 "X4 edition").
|
||||
No floats, no flex, no grid, no embedded fonts, larger base font,
|
||||
generous line-height, hyphenation on. 480x800, 2-bit grayscale.
|
||||
|
||||
Selectors are `tag`, `.class` and `tag.class` only — the X4 firmware's CSS
|
||||
engine does not support descendant combinators, so a rule like
|
||||
`.comment-body p` is silently dropped on the device. Where a tag rule needs
|
||||
an exception, the `tag.class` override follows it immediately so the result
|
||||
is right whether the engine resolves by specificity or by source order. */
|
||||
|
||||
@page {
|
||||
margin: 0.4em;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: serif;
|
||||
font-size: 1.2em;
|
||||
line-height: 1.7;
|
||||
margin: 0 0.5em;
|
||||
text-align: left;
|
||||
hyphens: auto;
|
||||
-webkit-hyphens: auto;
|
||||
adobe-hyphenate: auto;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
.chapter {
|
||||
page-break-before: always;
|
||||
break-before: page;
|
||||
}
|
||||
|
||||
h1, h2, h3, h4 {
|
||||
font-weight: bold;
|
||||
line-height: 1.3;
|
||||
page-break-after: avoid;
|
||||
break-after: avoid;
|
||||
margin: 0.6em 0 0.35em 0;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
h1 { font-size: 1.35em; }
|
||||
h2 { font-size: 1.15em; }
|
||||
h3, h4 { font-size: 1em; }
|
||||
|
||||
p {
|
||||
margin: 0 0 0.6em 0;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #000000;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
hr.rule {
|
||||
border: 0;
|
||||
border-top: 1px solid #000000;
|
||||
margin: 0.7em 0;
|
||||
height: 0;
|
||||
}
|
||||
|
||||
img {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
}
|
||||
|
||||
.cover-page {
|
||||
text-align: center;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.masthead {
|
||||
font-size: 1.6em;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.dateline,
|
||||
.stats,
|
||||
.meta,
|
||||
.social,
|
||||
.index-meta,
|
||||
.discussion-note,
|
||||
.attribution,
|
||||
.comment-meta {
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.dateline,
|
||||
.stats {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.section-page {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
font-size: 1.5em;
|
||||
margin-top: 1.5em;
|
||||
}
|
||||
|
||||
.section-intro,
|
||||
.summary,
|
||||
.byline {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
ul, ol {
|
||||
margin: 0 0 0.5em 1em;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
ul.index-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.index-entry {
|
||||
margin: 0 0 0.8em 0;
|
||||
}
|
||||
|
||||
.index-title,
|
||||
.index-meta,
|
||||
.index-summary {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
figure {
|
||||
margin: 0.6em 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
figcaption,
|
||||
.image-caption,
|
||||
.image-placeholder {
|
||||
font-size: 0.85em;
|
||||
font-style: italic;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
pre, code {
|
||||
font-family: monospace;
|
||||
font-size: 0.85em;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
td, th {
|
||||
border: 1px solid #666666;
|
||||
padding: 0.15em 0.3em;
|
||||
}
|
||||
|
||||
blockquote {
|
||||
margin: 0.5em 0 0.5em 0.3em;
|
||||
padding-left: 0.5em;
|
||||
border-left: 2px solid #666666;
|
||||
}
|
||||
|
||||
blockquote.comment {
|
||||
margin: 0.4em 0;
|
||||
padding-left: 0.5em;
|
||||
border-left: 2px solid #666666;
|
||||
}
|
||||
|
||||
blockquote.reply {
|
||||
border-left: 1px solid #999999;
|
||||
}
|
||||
|
||||
p.comment-line {
|
||||
margin: 0 0 0.35em 0;
|
||||
}
|
||||
|
||||
dt.fact-key {
|
||||
font-weight: bold;
|
||||
margin-top: 0.35em;
|
||||
}
|
||||
|
||||
dd.fact-value {
|
||||
margin: 0 0 0 0.8em;
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
/* Standard-edition stylesheet (spec §3.10 "CSS").
|
||||
Serif body, grayscale only, page-break-before on chapters,
|
||||
blockquote-indent comment styling. Tuned for e-ink readers. */
|
||||
|
||||
@page {
|
||||
margin: 1em;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: Georgia, "Times New Roman", Times, serif;
|
||||
font-size: 1em;
|
||||
line-height: 1.5;
|
||||
margin: 0 1em;
|
||||
text-align: left;
|
||||
widows: 2;
|
||||
orphans: 2;
|
||||
}
|
||||
|
||||
.chapter {
|
||||
page-break-before: always;
|
||||
break-before: page;
|
||||
}
|
||||
|
||||
h1, h2, h3, h4 {
|
||||
font-weight: normal;
|
||||
line-height: 1.25;
|
||||
page-break-after: avoid;
|
||||
break-after: avoid;
|
||||
margin: 0.8em 0 0.4em 0;
|
||||
}
|
||||
|
||||
h1 { font-size: 1.5em; }
|
||||
h2 { font-size: 1.25em; }
|
||||
h3 { font-size: 1.1em; }
|
||||
h4 { font-size: 1em; font-style: italic; }
|
||||
|
||||
p {
|
||||
margin: 0 0 0.7em 0;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #000000;
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
hr.rule {
|
||||
border: 0;
|
||||
border-top: 1px solid #000000;
|
||||
margin: 0.9em 0;
|
||||
height: 0;
|
||||
}
|
||||
|
||||
/* --- cover ------------------------------------------------------------- */
|
||||
|
||||
.cover-page {
|
||||
text-align: center;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.cover-image img {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
}
|
||||
|
||||
/* --- front page -------------------------------------------------------- */
|
||||
|
||||
.masthead {
|
||||
font-size: 2.1em;
|
||||
text-align: center;
|
||||
letter-spacing: 0.02em;
|
||||
margin-bottom: 0.1em;
|
||||
}
|
||||
|
||||
.dateline {
|
||||
text-align: center;
|
||||
font-size: 0.9em;
|
||||
font-variant: small-caps;
|
||||
margin-bottom: 0.6em;
|
||||
}
|
||||
|
||||
.kicker {
|
||||
font-variant: small-caps;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.stats {
|
||||
font-size: 0.85em;
|
||||
font-style: italic;
|
||||
text-align: center;
|
||||
margin-top: 1em;
|
||||
}
|
||||
|
||||
/* --- in this issue ----------------------------------------------------- */
|
||||
|
||||
.index-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.index-entry {
|
||||
margin: 0 0 0.9em 0;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid;
|
||||
}
|
||||
|
||||
.index-title {
|
||||
margin: 0;
|
||||
font-size: 1.05em;
|
||||
}
|
||||
|
||||
.index-meta {
|
||||
margin: 0;
|
||||
font-size: 0.8em;
|
||||
font-variant: small-caps;
|
||||
}
|
||||
|
||||
.index-summary {
|
||||
margin: 0.2em 0 0 0;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
/* --- sections and articles --------------------------------------------- */
|
||||
|
||||
.section-page {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
font-size: 2em;
|
||||
font-variant: small-caps;
|
||||
margin-top: 2.5em;
|
||||
}
|
||||
|
||||
.section-intro {
|
||||
font-style: italic;
|
||||
margin: 0 1.5em;
|
||||
}
|
||||
|
||||
.article-title {
|
||||
font-size: 1.6em;
|
||||
margin-bottom: 0.2em;
|
||||
}
|
||||
|
||||
.byline {
|
||||
margin: 0;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.meta, .social {
|
||||
margin: 0;
|
||||
font-size: 0.8em;
|
||||
font-variant: small-caps;
|
||||
}
|
||||
|
||||
.summary {
|
||||
margin: 0.5em 0 0 0;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.notice {
|
||||
font-size: 0.85em;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.article-body img {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
}
|
||||
|
||||
.article-body figure {
|
||||
margin: 0.8em 0;
|
||||
text-align: center;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid;
|
||||
}
|
||||
|
||||
.article-body figcaption,
|
||||
.image-caption {
|
||||
font-size: 0.8em;
|
||||
font-style: italic;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.image-placeholder {
|
||||
font-size: 0.85em;
|
||||
font-style: italic;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.article-body blockquote {
|
||||
margin: 0.6em 0 0.6em 1em;
|
||||
padding-left: 0.6em;
|
||||
border-left: 2px solid #999999;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.article-body pre,
|
||||
.article-body code {
|
||||
font-family: "DejaVu Sans Mono", "Courier New", monospace;
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.article-body pre {
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
border-left: 2px solid #cccccc;
|
||||
padding-left: 0.5em;
|
||||
}
|
||||
|
||||
.article-body table {
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.article-body td,
|
||||
.article-body th {
|
||||
border: 1px solid #999999;
|
||||
padding: 0.2em 0.4em;
|
||||
}
|
||||
|
||||
.article-footer {
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.rating a {
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
/* --- discussion chapters (§3.7) ---------------------------------------- */
|
||||
|
||||
.discussion-note {
|
||||
font-size: 0.8em;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.discussion-source {
|
||||
font-variant: small-caps;
|
||||
font-size: 1.1em;
|
||||
margin-top: 1em;
|
||||
}
|
||||
|
||||
blockquote.comment {
|
||||
border-left: 2px solid #888888;
|
||||
margin: 0.5em 0 0.5em 0;
|
||||
padding-left: 0.7em;
|
||||
page-break-inside: avoid;
|
||||
break-inside: avoid;
|
||||
}
|
||||
|
||||
blockquote.comment blockquote.comment {
|
||||
border-left: 1px solid #aaaaaa;
|
||||
margin-left: 0.2em;
|
||||
}
|
||||
|
||||
.comment-meta {
|
||||
font-size: 0.78em;
|
||||
font-variant: small-caps;
|
||||
margin: 0 0 0.15em 0;
|
||||
}
|
||||
|
||||
.comment-body p {
|
||||
margin: 0 0 0.4em 0;
|
||||
}
|
||||
|
||||
/* --- world briefing and colophon --------------------------------------- */
|
||||
|
||||
.world-body ul {
|
||||
margin: 0 0 0.6em 1.1em;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.world-body li {
|
||||
margin-bottom: 0.25em;
|
||||
}
|
||||
|
||||
.attribution {
|
||||
font-size: 0.8em;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.colophon-facts dt {
|
||||
font-variant: small-caps;
|
||||
margin-top: 0.4em;
|
||||
}
|
||||
|
||||
.colophon-facts dd {
|
||||
margin: 0 0 0 1em;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{% extends "base.xhtml" %}
|
||||
{% block body_class %}world-briefing{% endblock %}
|
||||
{% block content %}
|
||||
<h1>World Briefing</h1>
|
||||
<p class="dateline">{{ display_date }}</p>
|
||||
<hr class="rule"/>
|
||||
<div class="world-body">
|
||||
{{ body_html|safe }}
|
||||
</div>
|
||||
{% endblock %}
|
||||
+644
@@ -0,0 +1,644 @@
|
||||
//! Xteink X4 edition transforms and the XTC converter invocation
|
||||
//! (spec §3.10 X4 edition, §3.11).
|
||||
//!
|
||||
//! The converter has no global npm bin: it is run as
|
||||
//! `node <repo>/cli/index.js convert <in.epub> -o <out.xtch> -f xtch [-c settings.json]`
|
||||
//! (implementation notes, verified facts). A missing or failing converter is
|
||||
//! non-fatal — XTC is a bonus artifact.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::config::XtcConfig;
|
||||
|
||||
use super::images::tag_end;
|
||||
|
||||
/// Native X4 screen size, used for the cover and image fitting (§3.10).
|
||||
pub const X4_SCREEN: (u32, u32) = (480, 800);
|
||||
|
||||
/// Attributes that let a document lay itself out — dropped for the X4 (§3.10).
|
||||
pub const DROPPED_ATTRIBUTES: &[&str] = &[
|
||||
"style", "align", "width", "height", "srcset", "sizes", "loading", "hspace", "vspace", "border",
|
||||
];
|
||||
|
||||
/// Longest unbroken run of non-whitespace the X4 firmware will lay out; past
|
||||
/// this it stops wrapping and the line runs off the 480px screen (§3.10).
|
||||
///
|
||||
/// Real text never gets near 200 characters — this is for minified source in a
|
||||
/// code block and for bare URLs pasted into comment threads.
|
||||
pub const MAX_WORD_CHARS: usize = 200;
|
||||
|
||||
/// U+00AD, invisible unless the renderer actually needs to break there.
|
||||
const SOFT_HYPHEN: char = '\u{00ad}';
|
||||
|
||||
/// Elements whose content is code, not prose, and must be copied through
|
||||
/// untouched — a soft hyphen inside a stylesheet would corrupt it.
|
||||
const RAW_TEXT_ELEMENTS: &[&str] = &["script", "style"];
|
||||
|
||||
/// Declarations the X4 renderer cannot honor (§3.10).
|
||||
const DROPPED_PROPERTIES: &[&str] = &[
|
||||
"float",
|
||||
"clear",
|
||||
"position",
|
||||
"z-index",
|
||||
"box-shadow",
|
||||
"text-shadow",
|
||||
"transform",
|
||||
"columns",
|
||||
"column-count",
|
||||
"column-gap",
|
||||
"letter-spacing",
|
||||
];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum XtcError {
|
||||
#[error("could not run `{command}`: {source}")]
|
||||
Spawn {
|
||||
command: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("converter exited with status {status}: {stderr}")]
|
||||
Failed { status: i32, stderr: String },
|
||||
#[error("converter produced no output at {0}")]
|
||||
NoOutput(PathBuf),
|
||||
}
|
||||
|
||||
/// Simplify CSS for the X4: no floats/flex/grid, no embedded fonts, larger base
|
||||
/// font, generous line-height, hyphenation on (§3.10).
|
||||
pub fn simplify_css(css: &str) -> String {
|
||||
let mut out = String::with_capacity(css.len());
|
||||
let mut rest = css;
|
||||
while let Some(open) = rest.find('{') {
|
||||
let selector = &rest[..open];
|
||||
let Some(close) = rest[open..].find('}') else {
|
||||
break;
|
||||
};
|
||||
let body = &rest[open + 1..open + close];
|
||||
rest = &rest[open + close + 1..];
|
||||
|
||||
// `@font-face` (and any other embedded-font rule) is dropped wholesale.
|
||||
if selector.to_ascii_lowercase().contains("@font-face") {
|
||||
continue;
|
||||
}
|
||||
let kept: Vec<&str> = body
|
||||
.split(';')
|
||||
.filter(|decl| !decl.trim().is_empty())
|
||||
.filter(|decl| !is_dropped_declaration(decl))
|
||||
.collect();
|
||||
if kept.is_empty() {
|
||||
continue;
|
||||
}
|
||||
out.push_str(selector.trim_start_matches('\n'));
|
||||
out.push('{');
|
||||
for decl in kept {
|
||||
out.push_str(decl);
|
||||
out.push(';');
|
||||
}
|
||||
out.push('}');
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn is_dropped_declaration(decl: &str) -> bool {
|
||||
let Some((property, value)) = decl.split_once(':') else {
|
||||
return true;
|
||||
};
|
||||
let property = property.trim().to_ascii_lowercase();
|
||||
let value = value.trim().to_ascii_lowercase();
|
||||
if DROPPED_PROPERTIES.contains(&property.as_str()) {
|
||||
return true;
|
||||
}
|
||||
if property == "display" && (value.contains("flex") || value.contains("grid")) {
|
||||
return true;
|
||||
}
|
||||
if property.starts_with("flex") || property.starts_with("grid") {
|
||||
return true;
|
||||
}
|
||||
if property == "font-family" && value.contains("url(") {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Strip layout constructs the X4 renderer handles poorly from chapter markup,
|
||||
/// then soft-hyphenate anything too long for it to wrap (§3.10).
|
||||
pub fn simplify_xhtml(xhtml: &str) -> String {
|
||||
break_long_words(&strip_attributes(xhtml, DROPPED_ATTRIBUTES))
|
||||
}
|
||||
|
||||
/// Insert soft hyphens into words longer than [`MAX_WORD_CHARS`], in text
|
||||
/// content only (§3.10).
|
||||
fn break_long_words(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
soften_text(&html[cursor..start], &mut out);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
out.push_str(tag);
|
||||
cursor = end;
|
||||
// `<style>`/`<script>` bodies are not prose: copy to the closing tag verbatim.
|
||||
if let Some(name) = raw_text_name(tag)
|
||||
&& let Some(close) = find_close_tag(html, cursor, name)
|
||||
{
|
||||
out.push_str(&html[cursor..close]);
|
||||
cursor = close;
|
||||
}
|
||||
}
|
||||
soften_text(&html[cursor..], &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// The element name when `tag` opens a raw-text element, else `None`.
|
||||
fn raw_text_name(tag: &str) -> Option<&'static str> {
|
||||
let rest = tag.strip_prefix('<')?;
|
||||
if rest.starts_with('/') {
|
||||
return None;
|
||||
}
|
||||
RAW_TEXT_ELEMENTS.iter().copied().find(|name| {
|
||||
rest.len() >= name.len()
|
||||
&& rest[..name.len()].eq_ignore_ascii_case(name)
|
||||
// Only `<style>` and `<style type=…>`, never `<styled-thing>`.
|
||||
&& rest[name.len()..]
|
||||
.starts_with([' ', '\t', '\n', '\r', '>', '/'])
|
||||
})
|
||||
}
|
||||
|
||||
/// Byte offset of `</name` at or after `from`, else `None`.
|
||||
fn find_close_tag(html: &str, from: usize, name: &str) -> Option<usize> {
|
||||
let needle = format!("</{name}");
|
||||
let hay = html.get(from..)?.to_ascii_lowercase();
|
||||
hay.find(&needle).map(|i| from + i)
|
||||
}
|
||||
|
||||
/// Copy `text` into `out`, soft-hyphenating any over-long word.
|
||||
fn soften_text(text: &str, out: &mut String) {
|
||||
// Byte length bounds character count, so a short run holds no long word.
|
||||
if text.len() <= MAX_WORD_CHARS {
|
||||
out.push_str(text);
|
||||
return;
|
||||
}
|
||||
let mut word_start = 0usize;
|
||||
for (i, c) in text.char_indices() {
|
||||
if c.is_whitespace() {
|
||||
push_soft_hyphenated(&text[word_start..i], out);
|
||||
out.push(c);
|
||||
word_start = i + c.len_utf8();
|
||||
}
|
||||
}
|
||||
push_soft_hyphenated(&text[word_start..], out);
|
||||
}
|
||||
|
||||
fn push_soft_hyphenated(word: &str, out: &mut String) {
|
||||
if word.len() <= MAX_WORD_CHARS {
|
||||
out.push_str(word);
|
||||
return;
|
||||
}
|
||||
let mut units = 0usize;
|
||||
let mut rest = word;
|
||||
while !rest.is_empty() {
|
||||
if units == MAX_WORD_CHARS {
|
||||
out.push(SOFT_HYPHEN);
|
||||
units = 0;
|
||||
}
|
||||
let take =
|
||||
entity_len(rest).unwrap_or_else(|| rest.chars().next().map_or(1, char::len_utf8));
|
||||
out.push_str(&rest[..take]);
|
||||
rest = &rest[take..];
|
||||
units += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Byte length of the `&…;` reference starting `s`, if there is one.
|
||||
///
|
||||
/// A character reference is one unit: splitting `&` down the middle would
|
||||
/// turn it into literal text and break the XHTML.
|
||||
fn entity_len(s: &str) -> Option<usize> {
|
||||
/// `≈` is 13 bytes; nothing we emit is longer.
|
||||
const MAX_ENTITY_BYTES: usize = 16;
|
||||
let bytes = s.as_bytes();
|
||||
if bytes.first() != Some(&b'&') {
|
||||
return None;
|
||||
}
|
||||
bytes
|
||||
.iter()
|
||||
.take(MAX_ENTITY_BYTES)
|
||||
.position(|&b| b == b';')
|
||||
.map(|p| p + 1)
|
||||
}
|
||||
|
||||
/// Remove the named attributes from every tag, leaving the rest verbatim.
|
||||
fn strip_attributes(html: &str, drop: &[&str]) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
out.push_str(&filter_tag(&html[start..end], drop));
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// `<p style="x" class="y">` → `<p class="y">`.
|
||||
fn filter_tag(tag: &str, drop: &[&str]) -> String {
|
||||
if tag.starts_with("<!") || tag.starts_with("<?") || tag.starts_with("</") {
|
||||
return tag.to_string();
|
||||
}
|
||||
let bytes = tag.as_bytes();
|
||||
let mut out = String::with_capacity(tag.len());
|
||||
let mut i = 1; // past '<'
|
||||
while i < bytes.len() && !bytes[i].is_ascii_whitespace() && bytes[i] != b'>' {
|
||||
i += 1;
|
||||
}
|
||||
out.push_str(&tag[..i]);
|
||||
|
||||
while i < bytes.len() {
|
||||
let ws_start = i;
|
||||
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
if i >= bytes.len() || bytes[i] == b'>' || bytes[i] == b'/' {
|
||||
out.push_str(&tag[ws_start..]);
|
||||
return out;
|
||||
}
|
||||
let name_start = i;
|
||||
while i < bytes.len()
|
||||
&& !bytes[i].is_ascii_whitespace()
|
||||
&& bytes[i] != b'='
|
||||
&& bytes[i] != b'>'
|
||||
&& bytes[i] != b'/'
|
||||
{
|
||||
i += 1;
|
||||
}
|
||||
let name = tag[name_start..i].to_ascii_lowercase();
|
||||
let mut after_name = i;
|
||||
while after_name < bytes.len() && bytes[after_name].is_ascii_whitespace() {
|
||||
after_name += 1;
|
||||
}
|
||||
if after_name < bytes.len() && bytes[after_name] == b'=' {
|
||||
i = after_name + 1;
|
||||
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
||||
i += 1;
|
||||
}
|
||||
if i < bytes.len() && (bytes[i] == b'"' || bytes[i] == b'\'') {
|
||||
let quote = bytes[i];
|
||||
i += 1;
|
||||
while i < bytes.len() && bytes[i] != quote {
|
||||
i += 1;
|
||||
}
|
||||
i = (i + 1).min(bytes.len());
|
||||
} else {
|
||||
while i < bytes.len() && !bytes[i].is_ascii_whitespace() && bytes[i] != b'>' {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !drop.contains(&name.as_str()) {
|
||||
out.push_str(&tag[ws_start..i]);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Full argv for the converter: `command` + `args` + `<input> -o <output> -f <format>`
|
||||
/// (+ `-c <settings>` when configured) (§3.11).
|
||||
pub fn build_command(cfg: &XtcConfig, input: &Path, output: &Path) -> (String, Vec<String>) {
|
||||
let mut args = cfg.args.clone();
|
||||
args.push(input.display().to_string());
|
||||
args.push("-o".to_string());
|
||||
args.push(output.display().to_string());
|
||||
args.push("-f".to_string());
|
||||
args.push(cfg.format.as_str().to_string());
|
||||
if let Some(settings) = &cfg.settings {
|
||||
args.push("-c".to_string());
|
||||
args.push(settings.display().to_string());
|
||||
}
|
||||
(cfg.command.clone(), args)
|
||||
}
|
||||
|
||||
/// Output path for an input EPUB: `{out_dir}/{stem}.{xtc|xtch}` (§3.11).
|
||||
pub fn output_path(cfg: &XtcConfig, input: &Path, out_dir: &Path) -> PathBuf {
|
||||
let stem = input
|
||||
.file_stem()
|
||||
.map(|s| s.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "issue".to_string());
|
||||
out_dir.join(format!("{stem}.{}", cfg.format.extension()))
|
||||
}
|
||||
|
||||
/// Convert the X4 EPUB to `.xtc`/`.xtch` via `tokio::process::Command` (§3.11).
|
||||
///
|
||||
/// Callers treat every error as a warning and continue — XTC is a bonus
|
||||
/// artifact, the X4 can always fall back to the X4 EPUB from BookOrbit.
|
||||
pub async fn convert(cfg: &XtcConfig, input: &Path, out_dir: &Path) -> Result<PathBuf, XtcError> {
|
||||
if cfg.settings.is_none() {
|
||||
// The converter refuses to start without `font.path`, which can only be
|
||||
// supplied through the settings JSON: `-c` is mandatory in practice even
|
||||
// though the flag is optional.
|
||||
tracing::warn!(
|
||||
"xtc.settings is unset; epub-to-xtc-converter requires a settings \
|
||||
file with a font.path and will refuse to run without one"
|
||||
);
|
||||
}
|
||||
let output = output_path(cfg, input, out_dir);
|
||||
if let Some(parent) = output.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| XtcError::Spawn {
|
||||
command: parent.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
let (command, args) = build_command(cfg, input, &output);
|
||||
tracing::info!(command, ?args, "running the xtc converter");
|
||||
|
||||
let result = tokio::process::Command::new(&command)
|
||||
.args(&args)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| XtcError::Spawn {
|
||||
command: command.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
if !result.status.success() {
|
||||
return Err(XtcError::Failed {
|
||||
status: result.status.code().unwrap_or(-1),
|
||||
stderr: String::from_utf8_lossy(&result.stderr)
|
||||
.lines()
|
||||
.take(5)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" | "),
|
||||
});
|
||||
}
|
||||
if !output.exists() {
|
||||
return Err(XtcError::NoOutput(output));
|
||||
}
|
||||
tracing::info!(path = %output.display(), "xtc conversion complete");
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::XtcFormat;
|
||||
|
||||
fn cfg() -> XtcConfig {
|
||||
XtcConfig {
|
||||
enabled: true,
|
||||
command: "node".into(),
|
||||
args: vec![
|
||||
"/opt/epub-to-xtc-converter/cli/index.js".into(),
|
||||
"convert".into(),
|
||||
],
|
||||
format: XtcFormat::Xtch,
|
||||
settings: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_the_documented_converter_argv() {
|
||||
let (command, args) = build_command(
|
||||
&cfg(),
|
||||
Path::new("/out/The Daily EPUB - 2026-08-15 (X4).epub"),
|
||||
Path::new("/xtc/The Daily EPUB - 2026-08-15 (X4).xtch"),
|
||||
);
|
||||
assert_eq!(command, "node");
|
||||
assert_eq!(
|
||||
args,
|
||||
vec![
|
||||
"/opt/epub-to-xtc-converter/cli/index.js",
|
||||
"convert",
|
||||
"/out/The Daily EPUB - 2026-08-15 (X4).epub",
|
||||
"-o",
|
||||
"/xtc/The Daily EPUB - 2026-08-15 (X4).xtch",
|
||||
"-f",
|
||||
"xtch",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_file_is_passed_with_dash_c() {
|
||||
let mut cfg = cfg();
|
||||
cfg.settings = Some(PathBuf::from("/etc/xtc.json"));
|
||||
cfg.format = XtcFormat::Xtc;
|
||||
let (_, args) = build_command(&cfg, Path::new("in.epub"), Path::new("out.xtc"));
|
||||
assert_eq!(args[args.len() - 4..], ["-f", "xtc", "-c", "/etc/xtc.json"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_path_follows_the_format_extension() {
|
||||
let out = output_path(&cfg(), Path::new("/out/Issue (X4).epub"), Path::new("/xtc"));
|
||||
assert_eq!(out, PathBuf::from("/xtc/Issue (X4).xtch"));
|
||||
}
|
||||
|
||||
/// The pipeline turns every one of these into a report warning, so the error
|
||||
/// has to say which of them happened (§3.11).
|
||||
#[tokio::test]
|
||||
async fn converter_failures_are_distinguishable() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let mut missing = cfg();
|
||||
missing.command = "definitely-not-a-real-binary-9f3b".into();
|
||||
missing.args.clear();
|
||||
match convert(&missing, Path::new("in.epub"), dir.path()).await {
|
||||
Err(XtcError::Spawn { command, .. }) => {
|
||||
assert_eq!(command, "definitely-not-a-real-binary-9f3b")
|
||||
}
|
||||
other => panic!("expected a spawn failure, got {other:?}"),
|
||||
}
|
||||
|
||||
let mut failing = cfg();
|
||||
failing.command = "false".into();
|
||||
failing.args.clear();
|
||||
match convert(&failing, Path::new("in.epub"), dir.path()).await {
|
||||
Err(XtcError::Failed { status, .. }) => assert_ne!(status, 0),
|
||||
other => panic!("expected a nonzero exit, got {other:?}"),
|
||||
}
|
||||
|
||||
// Exit 0 but nothing written is its own error, not a silent success.
|
||||
let mut silent = cfg();
|
||||
silent.command = "true".into();
|
||||
silent.args.clear();
|
||||
match convert(&silent, Path::new("in.epub"), dir.path()).await {
|
||||
Err(XtcError::NoOutput(path)) => assert_eq!(path, dir.path().join("in.xtch")),
|
||||
other => panic!("expected NoOutput, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn css_simplification_drops_layout_and_fonts() {
|
||||
let css = r#"
|
||||
@font-face { font-family: "Serif"; src: url(serif.woff2); }
|
||||
.a { float: left; color: #000; }
|
||||
.b { display: flex; flex-direction: row; }
|
||||
.c { position: absolute; margin: 1em; }
|
||||
.d { float: right; }
|
||||
"#;
|
||||
let out = simplify_css(css);
|
||||
assert!(!out.contains("@font-face"));
|
||||
assert!(!out.contains("float"));
|
||||
assert!(!out.contains("flex"));
|
||||
assert!(!out.contains("position"));
|
||||
assert!(out.contains("color: #000"));
|
||||
assert!(out.contains("margin: 1em"));
|
||||
// A rule left with no declarations disappears entirely.
|
||||
assert!(!out.contains(".d"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xhtml_simplification_drops_layout_attributes_only() {
|
||||
let input = r#"<p class="meta" style="float:left" align="center">a & b</p><img src="x.jpg" alt="An x" width="900"/>"#;
|
||||
let out = simplify_xhtml(input);
|
||||
assert_eq!(
|
||||
out,
|
||||
r#"<p class="meta">a & b</p><img src="x.jpg" alt="An x"/>"#
|
||||
);
|
||||
}
|
||||
|
||||
/// The shipped X4 stylesheet must already satisfy the X4 rules, so the
|
||||
/// simplifier is a no-op over it (§3.10).
|
||||
#[test]
|
||||
fn the_shipped_x4_stylesheet_is_already_simplified() {
|
||||
let css = super::super::build::stylesheet(crate::types::Edition::X4);
|
||||
let simplified = simplify_css(css);
|
||||
assert_eq!(
|
||||
css.matches(';').count(),
|
||||
simplified.matches(';').count(),
|
||||
"the simplifier dropped a declaration from style-x4.css"
|
||||
);
|
||||
// Declarations only — the file's header comment mentions what it avoids.
|
||||
for banned in [
|
||||
"float:",
|
||||
"clear:",
|
||||
"display: flex",
|
||||
"display: grid",
|
||||
"position:",
|
||||
"@font-face",
|
||||
] {
|
||||
assert!(!css.contains(banned), "style-x4.css must not use {banned}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The firmware stops wrapping past 200 characters and the line runs off
|
||||
/// the screen, so long tokens get soft hyphens (§3.10).
|
||||
#[test]
|
||||
fn over_long_words_are_soft_hyphenated() {
|
||||
let long = "a".repeat(450);
|
||||
let out = simplify_xhtml(&format!("<p>short {long} tail</p>"));
|
||||
assert_eq!(out.matches(SOFT_HYPHEN).count(), 2);
|
||||
// Only the long token is touched; the rest of the line is byte-identical.
|
||||
assert!(out.starts_with("<p>short "));
|
||||
assert!(out.ends_with(" tail</p>"));
|
||||
assert!(!out.contains(&format!("short{SOFT_HYPHEN}")));
|
||||
// Removing the hyphens gets the original word back — nothing was lost.
|
||||
assert!(out.replace(SOFT_HYPHEN, "").contains(&long));
|
||||
// Every run between hyphens is within the limit.
|
||||
for run in out.replace(['<', '>'], " ").split_whitespace() {
|
||||
for piece in run.split(SOFT_HYPHEN) {
|
||||
assert!(piece.chars().count() <= MAX_WORD_CHARS, "{}", piece.len());
|
||||
}
|
||||
}
|
||||
// Words at the limit are left alone.
|
||||
let exact = "b".repeat(MAX_WORD_CHARS);
|
||||
assert_eq!(
|
||||
simplify_xhtml(&format!("<p>{exact} {exact}</p>")),
|
||||
format!("<p>{exact} {exact}</p>")
|
||||
);
|
||||
}
|
||||
|
||||
/// A soft hyphen dropped into `&` would turn it into literal text and
|
||||
/// break the XHTML, so character references are indivisible (§3.10).
|
||||
#[test]
|
||||
fn entities_and_markup_survive_word_breaking() {
|
||||
// 120 entities: the raw string is far past the limit, but it is only
|
||||
// 120 units, so no break is due — and the entities stay intact.
|
||||
let entities = "&".repeat(120);
|
||||
let out = simplify_xhtml(&format!("<p>{entities}</p>"));
|
||||
assert!(!out.contains(SOFT_HYPHEN));
|
||||
assert_eq!(out.matches("&").count(), 120);
|
||||
|
||||
// Past the limit the breaks land between entities, never inside one.
|
||||
let out = simplify_xhtml(&format!("<p>{}</p>", "&".repeat(260)));
|
||||
assert_eq!(out.matches("&").count(), 260);
|
||||
assert_eq!(out.matches(SOFT_HYPHEN).count(), 1);
|
||||
assert!(!out.contains(&format!("&{SOFT_HYPHEN}")));
|
||||
assert!(!out.contains(&format!("&{SOFT_HYPHEN}")));
|
||||
|
||||
// Attribute values are not text content and must not be rewritten.
|
||||
let href = "https://example.com/".to_string() + &"z".repeat(300);
|
||||
let out = simplify_xhtml(&format!("<p><a href=\"{href}\">link</a></p>"));
|
||||
assert!(out.contains(&format!("href=\"{href}\"")), "{out}");
|
||||
assert!(!out.contains(SOFT_HYPHEN));
|
||||
}
|
||||
|
||||
/// Stylesheets and scripts are code: a soft hyphen inside one corrupts it.
|
||||
#[test]
|
||||
fn raw_text_elements_are_copied_through_verbatim() {
|
||||
let css = format!("p{{content:\"{}\"}}", "x".repeat(400));
|
||||
let out = simplify_xhtml(&format!("<style type=\"text/css\">{css}</style>"));
|
||||
assert!(out.contains(&css), "{out}");
|
||||
assert!(!out.contains(SOFT_HYPHEN));
|
||||
|
||||
// A tag that merely starts with the same letters is ordinary prose.
|
||||
let long = "y".repeat(400);
|
||||
let out = simplify_xhtml(&format!("<styled-note>{long}</styled-note>"));
|
||||
assert_eq!(out.matches(SOFT_HYPHEN).count(), 1);
|
||||
}
|
||||
|
||||
/// `/* … */` runs, which are prose and may contain anything.
|
||||
fn strip_css_comments(css: &str) -> String {
|
||||
let mut out = String::with_capacity(css.len());
|
||||
let mut rest = css;
|
||||
while let Some(open) = rest.find("/*") {
|
||||
out.push_str(&rest[..open]);
|
||||
match rest[open + 2..].find("*/") {
|
||||
Some(close) => rest = &rest[open + 4 + close..],
|
||||
None => return out,
|
||||
}
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
/// The X4's CSS engine understands `tag`, `.class` and `tag.class` only —
|
||||
/// a descendant combinator silently drops the whole rule (§3.10).
|
||||
#[test]
|
||||
fn the_x4_stylesheet_uses_no_descendant_selectors() {
|
||||
let css = strip_css_comments(super::super::build::stylesheet(crate::types::Edition::X4));
|
||||
for (i, _) in css.match_indices('{') {
|
||||
let selector_list = css[..i].rsplit('}').next().unwrap_or_default().trim();
|
||||
for selector in selector_list.split(',') {
|
||||
let selector = selector.trim();
|
||||
if selector.is_empty() || selector.starts_with('@') {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
!selector.contains(char::is_whitespace),
|
||||
"descendant selector {selector:?} will not match on the X4"
|
||||
);
|
||||
for combinator in ['>', '+', '~'] {
|
||||
assert!(
|
||||
!selector.contains(combinator),
|
||||
"combinator {combinator:?} in {selector:?} is unsupported on the X4"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simplification_leaves_prologue_and_text_untouched() {
|
||||
let input =
|
||||
"<?xml version=\"1.0\"?>\n<!DOCTYPE html>\n<html><body><p>2 < 3</p></body></html>";
|
||||
assert_eq!(simplify_xhtml(input), input);
|
||||
}
|
||||
}
|
||||
+713
@@ -0,0 +1,713 @@
|
||||
//! Content extraction, sanitization and word counting (spec §3.3).
|
||||
//!
|
||||
//! Priority order per article: Miniflux content if it looks like full text →
|
||||
//! fetch + `dom_smoothie` readability → feed excerpt with a "(excerpt only)" note.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use scraper::{Html, Node, Selector};
|
||||
use url::Url;
|
||||
|
||||
use crate::types::{Article, ExtractMethod, Extracted};
|
||||
|
||||
/// Word count at or above which Miniflux content is treated as full text (§3.3).
|
||||
pub const FULL_TEXT_MIN_WORDS: i64 = 250;
|
||||
/// Maximum bytes downloaded when fetching an article page (§3.3).
|
||||
pub const MAX_FETCH_BYTES: usize = 3 * 1024 * 1024;
|
||||
/// Maximum images collected per article (§3.3).
|
||||
pub const MAX_IMAGES_PER_ARTICLE: usize = 12;
|
||||
/// Note appended to bodies we could only excerpt (§3.3).
|
||||
pub const EXCERPT_NOTE: &str = "(excerpt only — read online)";
|
||||
|
||||
/// Article pages are fetched with a desktop UA, not our bot UA (§3.3).
|
||||
pub const DESKTOP_UA: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";
|
||||
/// Per-fetch timeout for article pages (§3.3).
|
||||
pub const FETCH_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// Parallel article fetches during the extraction stage.
|
||||
pub const CONCURRENCY: usize = 8;
|
||||
/// Below this many words, a page on a [`DEFAULT_PAYWALL_DOMAINS`] host is a stub (§3.3).
|
||||
pub const PAYWALL_MAX_WORDS: i64 = 400;
|
||||
/// Any page this short is an excerpt regardless of host (§3.3).
|
||||
pub const EXCERPT_MAX_WORDS: i64 = 120;
|
||||
|
||||
/// Hosts that routinely serve a teaser instead of the article (§3.3).
|
||||
///
|
||||
/// The built-in list; `curation.paywall_domains` from the config file is merged
|
||||
/// on top of it by [`Extractor::new`] / [`Extractor::offline`] (§3.3).
|
||||
pub const DEFAULT_PAYWALL_DOMAINS: &[&str] = &[
|
||||
"nytimes.com",
|
||||
"wsj.com",
|
||||
"ft.com",
|
||||
"economist.com",
|
||||
"bloomberg.com",
|
||||
"washingtonpost.com",
|
||||
"newyorker.com",
|
||||
"theatlantic.com",
|
||||
"wired.com",
|
||||
"businessinsider.com",
|
||||
"barrons.com",
|
||||
"forbes.com",
|
||||
"latimes.com",
|
||||
"bostonglobe.com",
|
||||
"theinformation.com",
|
||||
"hbr.org",
|
||||
"nature.com",
|
||||
"science.org",
|
||||
"sciencedirect.com",
|
||||
"seekingalpha.com",
|
||||
"statnews.com",
|
||||
"thetimes.co.uk",
|
||||
"telegraph.co.uk",
|
||||
"medium.com",
|
||||
"towardsdatascience.com",
|
||||
];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ExtractError {
|
||||
#[error("fetch failed: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("response exceeded {MAX_FETCH_BYTES} bytes")]
|
||||
TooLarge,
|
||||
#[error("readability found no main content")]
|
||||
NoContent,
|
||||
#[error("server returned {0}")]
|
||||
Status(u16),
|
||||
#[error("response was {0}, not html")]
|
||||
NotHtml(String),
|
||||
#[error("fetching is disabled on this extractor")]
|
||||
FetchDisabled,
|
||||
}
|
||||
|
||||
/// Counters for the extraction stage, folded into the run report.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ExtractStats {
|
||||
pub from_miniflux: usize,
|
||||
pub from_readability: usize,
|
||||
pub excerpt_only: usize,
|
||||
pub fetch_failures: usize,
|
||||
}
|
||||
|
||||
/// The extraction stage for one article (§3.3).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Extractor {
|
||||
/// `None` disables the network path entirely (tests, `--dry-run` reruns).
|
||||
http: Option<reqwest::Client>,
|
||||
/// Hosts known to paywall, used by the [`looks_paywalled`] heuristic.
|
||||
paywall_domains: Vec<String>,
|
||||
}
|
||||
|
||||
impl Extractor {
|
||||
pub fn new(http: reqwest::Client, paywall_domains: Vec<String>) -> Self {
|
||||
Self {
|
||||
http: Some(http),
|
||||
paywall_domains: merge_paywall_domains(paywall_domains),
|
||||
}
|
||||
}
|
||||
|
||||
/// An extractor that never touches the network: the Miniflux/excerpt paths only.
|
||||
///
|
||||
/// This is what tests use, and it keeps the fetch step injectable (§6 testing).
|
||||
pub fn offline(paywall_domains: Vec<String>) -> Self {
|
||||
Self {
|
||||
http: None,
|
||||
paywall_domains: merge_paywall_domains(paywall_domains),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn can_fetch(&self) -> bool {
|
||||
self.http.is_some()
|
||||
}
|
||||
|
||||
/// Run the full priority order for one article and return its body (§3.3).
|
||||
///
|
||||
/// Never fails the run: on fetch/readability failure it degrades to the feed
|
||||
/// excerpt (notes §3).
|
||||
pub async fn extract(&self, article: &Article) -> Extracted {
|
||||
let span = tracing::debug_span!("extract", entry = article.best_entry_id);
|
||||
let _guard = span.enter();
|
||||
|
||||
// 1. Miniflux content, when it already looks like full text.
|
||||
let feed_html = sanitize_with_base(&article.content_html, &article.url);
|
||||
let feed_words = word_count(&feed_html);
|
||||
if feed_words >= FULL_TEXT_MIN_WORDS {
|
||||
return self.finish(article, feed_html, feed_words, ExtractMethod::Miniflux);
|
||||
}
|
||||
|
||||
// 2. Fetch the page and run readability over it.
|
||||
if self.can_fetch() {
|
||||
match self.fetch_readable(&article.url).await {
|
||||
Ok(html) => {
|
||||
let clean = sanitize_with_base(&html, &article.url);
|
||||
let words = word_count(&clean);
|
||||
if words > feed_words && words > 0 {
|
||||
return self.finish(article, clean, words, ExtractMethod::Readability);
|
||||
}
|
||||
tracing::debug!(words, feed_words, "readability was not an improvement");
|
||||
}
|
||||
Err(e) => tracing::debug!(url = %article.url, "extraction fetch failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Excerpt fallback. Re-extraction must not stack up notes (notes §12).
|
||||
let body = if feed_html.trim().is_empty() {
|
||||
format!("<p>{EXCERPT_NOTE}</p>")
|
||||
} else if feed_html.contains(EXCERPT_NOTE) {
|
||||
feed_html
|
||||
} else {
|
||||
format!("{feed_html}<p>{EXCERPT_NOTE}</p>")
|
||||
};
|
||||
let words = word_count(&body);
|
||||
let mut out = self.finish(article, body, words, ExtractMethod::Excerpt);
|
||||
out.excerpt_only = true;
|
||||
out
|
||||
}
|
||||
|
||||
/// Assemble the [`Extracted`] value once a body has been chosen.
|
||||
fn finish(
|
||||
&self,
|
||||
article: &Article,
|
||||
content_html: String,
|
||||
words: i64,
|
||||
method: ExtractMethod,
|
||||
) -> Extracted {
|
||||
let image_urls = collect_image_urls(&content_html, &article.url);
|
||||
let excerpt_only = method == ExtractMethod::Excerpt
|
||||
|| looks_paywalled(&article.url, words, &self.paywall_domains);
|
||||
Extracted {
|
||||
content_html,
|
||||
word_count: words,
|
||||
excerpt_only,
|
||||
image_urls,
|
||||
method,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract every article in place, up to [`CONCURRENCY`] fetches at a time (§3.3).
|
||||
pub async fn extract_all(&self, articles: &mut [Article]) -> ExtractStats {
|
||||
let span = tracing::info_span!("extract_all", articles = articles.len());
|
||||
let _guard = span.enter();
|
||||
|
||||
let inputs: Vec<Article> = articles.to_vec();
|
||||
let results: Vec<(usize, Extracted)> = futures::stream::iter(inputs.iter().enumerate())
|
||||
.map(|(i, article)| async move { (i, self.extract(article).await) })
|
||||
.buffer_unordered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut stats = ExtractStats::default();
|
||||
for (i, extracted) in results {
|
||||
match extracted.method {
|
||||
ExtractMethod::Miniflux => stats.from_miniflux += 1,
|
||||
ExtractMethod::Readability => stats.from_readability += 1,
|
||||
ExtractMethod::Excerpt => stats.fetch_failures += 1,
|
||||
}
|
||||
if extracted.excerpt_only {
|
||||
stats.excerpt_only += 1;
|
||||
}
|
||||
apply(&mut articles[i], extracted);
|
||||
}
|
||||
tracing::info!(
|
||||
miniflux = stats.from_miniflux,
|
||||
readability = stats.from_readability,
|
||||
excerpt_only = stats.excerpt_only,
|
||||
"extraction complete"
|
||||
);
|
||||
stats
|
||||
}
|
||||
|
||||
/// Fetch `url` (10s timeout, desktop UA, [`MAX_FETCH_BYTES`] cap) and run
|
||||
/// `dom_smoothie` readability over it (§3.3).
|
||||
pub async fn fetch_readable(&self, url: &str) -> Result<String, ExtractError> {
|
||||
let Some(http) = &self.http else {
|
||||
return Err(ExtractError::FetchDisabled);
|
||||
};
|
||||
let mut response = http
|
||||
.get(url)
|
||||
.header(reqwest::header::USER_AGENT, DESKTOP_UA)
|
||||
.header(
|
||||
reqwest::header::ACCEPT,
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
)
|
||||
.timeout(FETCH_TIMEOUT)
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(ExtractError::Status(response.status().as_u16()));
|
||||
}
|
||||
if let Some(ct) = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
{
|
||||
let ct = ct.to_ascii_lowercase();
|
||||
if !(ct.contains("html") || ct.contains("xml") || ct.contains("text/plain")) {
|
||||
return Err(ExtractError::NotHtml(ct));
|
||||
}
|
||||
}
|
||||
|
||||
let mut body: Vec<u8> = Vec::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
if body.len() + chunk.len() > MAX_FETCH_BYTES {
|
||||
return Err(ExtractError::TooLarge);
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
let html = String::from_utf8_lossy(&body).into_owned();
|
||||
readability(&html, url)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run `dom_smoothie` over a fetched page and return its main-content HTML (§3.3).
|
||||
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
||||
let config = dom_smoothie::Config {
|
||||
max_elements_to_parse: 60_000,
|
||||
..Default::default()
|
||||
};
|
||||
let mut readability = dom_smoothie::Readability::new(html, Some(url), Some(config))
|
||||
.map_err(|_| ExtractError::NoContent)?;
|
||||
let parsed = readability.parse().map_err(|_| ExtractError::NoContent)?;
|
||||
let content = parsed.content.to_string();
|
||||
if content.trim().is_empty() {
|
||||
return Err(ExtractError::NoContent);
|
||||
}
|
||||
Ok(content)
|
||||
}
|
||||
|
||||
/// Copy an [`Extracted`] onto its [`Article`].
|
||||
pub fn apply(article: &mut Article, extracted: Extracted) {
|
||||
article.image_count = extracted.image_urls.len() as i64;
|
||||
article.image_urls = extracted.image_urls;
|
||||
article.content_html = extracted.content_html;
|
||||
article.word_count = extracted.word_count;
|
||||
article.excerpt_only = extracted.excerpt_only;
|
||||
article.extract_method = extracted.method;
|
||||
}
|
||||
|
||||
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
|
||||
let mut domains: Vec<String> = DEFAULT_PAYWALL_DOMAINS
|
||||
.iter()
|
||||
.map(|d| (*d).to_string())
|
||||
.collect();
|
||||
for d in configured {
|
||||
let d = d.trim().to_ascii_lowercase();
|
||||
if !d.is_empty() && !domains.contains(&d) {
|
||||
domains.push(d);
|
||||
}
|
||||
}
|
||||
domains
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sanitization (§3.3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Tags the EPUB templates accept (§3.3).
|
||||
pub const ALLOWED_TAGS: &[&str] = &[
|
||||
"p",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"blockquote",
|
||||
"pre",
|
||||
"code",
|
||||
"em",
|
||||
"strong",
|
||||
"a",
|
||||
"img",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tr",
|
||||
"th",
|
||||
"td",
|
||||
"caption",
|
||||
"hr",
|
||||
"br",
|
||||
];
|
||||
|
||||
fn builder() -> ammonia::Builder<'static> {
|
||||
let mut tag_attributes: std::collections::HashMap<&str, HashSet<&str>> =
|
||||
std::collections::HashMap::new();
|
||||
tag_attributes.insert("a", ["href", "title"].into_iter().collect());
|
||||
tag_attributes.insert("img", ["src", "alt", "title"].into_iter().collect());
|
||||
tag_attributes.insert("th", ["colspan", "rowspan", "scope"].into_iter().collect());
|
||||
tag_attributes.insert("td", ["colspan", "rowspan"].into_iter().collect());
|
||||
|
||||
let mut b = ammonia::Builder::default();
|
||||
b.tags(ALLOWED_TAGS.iter().copied().collect())
|
||||
.tag_attributes(tag_attributes)
|
||||
.generic_attributes(HashSet::new())
|
||||
.link_rel(None)
|
||||
.strip_comments(true);
|
||||
b
|
||||
}
|
||||
|
||||
/// Sanitize to the safe XHTML subset the EPUB templates allow (§3.3).
|
||||
///
|
||||
/// Allowed: `p`, `h1`–`h4`, `ul`/`ol`/`li`, `blockquote`, `pre`, `code`, `em`,
|
||||
/// `strong`, `a`, `img`, `figure`, `figcaption`, table basics, `hr`, `br`.
|
||||
pub fn sanitize(html: &str) -> String {
|
||||
builder().clean(html).to_string()
|
||||
}
|
||||
|
||||
/// [`sanitize`], additionally rewriting relative `href`/`src` against `base_url`
|
||||
/// so the EPUB (which has no base URL) still resolves them (§3.3).
|
||||
pub fn sanitize_with_base(html: &str, base_url: &str) -> String {
|
||||
match Url::parse(base_url) {
|
||||
Ok(base) => builder()
|
||||
.url_relative(ammonia::UrlRelative::RewriteWithBase(base))
|
||||
.clean(html)
|
||||
.to_string(),
|
||||
Err(_) => sanitize(html),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Text measurement (§3.3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Visible text of an HTML fragment, entities decoded, `script`/`style` skipped.
|
||||
pub fn html_to_text(html: &str) -> String {
|
||||
let document = Html::parse_fragment(html);
|
||||
let mut out = String::with_capacity(html.len() / 2);
|
||||
for node in document.tree.nodes() {
|
||||
let Node::Text(text) = node.value() else {
|
||||
continue;
|
||||
};
|
||||
let hidden = node.ancestors().any(|a| match a.value() {
|
||||
Node::Element(e) => matches!(e.name(), "script" | "style" | "noscript"),
|
||||
_ => false,
|
||||
});
|
||||
if hidden {
|
||||
continue;
|
||||
}
|
||||
out.push_str(text);
|
||||
out.push(' ');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Count words in rendered text (tags stripped) (§3.3).
|
||||
pub fn word_count(html: &str) -> i64 {
|
||||
html_to_text(html)
|
||||
.split_whitespace()
|
||||
.filter(|w| w.chars().any(char::is_alphanumeric))
|
||||
.count() as i64
|
||||
}
|
||||
|
||||
/// Absolute image URLs referenced by `html`, resolved against `base_url`,
|
||||
/// capped at [`MAX_IMAGES_PER_ARTICLE`] (§3.3).
|
||||
pub fn collect_image_urls(html: &str, base_url: &str) -> Vec<String> {
|
||||
let Ok(selector) = Selector::parse("img") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let base = Url::parse(base_url).ok();
|
||||
let document = Html::parse_fragment(html);
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for element in document.select(&selector) {
|
||||
let raw = element
|
||||
.value()
|
||||
.attr("src")
|
||||
.or_else(|| element.value().attr("data-src"))
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty());
|
||||
let Some(raw) = raw else { continue };
|
||||
let resolved = match Url::parse(raw) {
|
||||
Ok(u) => Some(u),
|
||||
Err(_) => base.as_ref().and_then(|b| b.join(raw).ok()),
|
||||
};
|
||||
let Some(url) = resolved.filter(|u| matches!(u.scheme(), "http" | "https")) else {
|
||||
continue;
|
||||
};
|
||||
let url = url.to_string();
|
||||
if seen.insert(url.clone()) {
|
||||
out.push(url);
|
||||
}
|
||||
if out.len() >= MAX_IMAGES_PER_ARTICLE {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Heuristic paywall detection: very short text on a known paywall domain (§3.3).
|
||||
///
|
||||
/// Two rules: anything under [`EXCERPT_MAX_WORDS`] is a stub whatever the host,
|
||||
/// and anything under [`PAYWALL_MAX_WORDS`] on a `paywall_domains` host is a teaser.
|
||||
pub fn looks_paywalled(url: &str, word_count: i64, paywall_domains: &[String]) -> bool {
|
||||
if word_count <= 0 {
|
||||
return true;
|
||||
}
|
||||
if word_count < EXCERPT_MAX_WORDS {
|
||||
return true;
|
||||
}
|
||||
if word_count >= PAYWALL_MAX_WORDS {
|
||||
return false;
|
||||
}
|
||||
let Some(host) = Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|u| u.host_str().map(|h| h.to_ascii_lowercase()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
paywall_domains
|
||||
.iter()
|
||||
.any(|d| host == *d || host.ends_with(&format!(".{d}")))
|
||||
}
|
||||
|
||||
/// Shared-ownership helper for callers that want one extractor across tasks.
|
||||
pub fn shared(extractor: Extractor) -> Arc<Extractor> {
|
||||
Arc::new(extractor)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::SourceKind;
|
||||
use jiff::Timestamp;
|
||||
|
||||
fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z".parse().unwrap()
|
||||
}
|
||||
|
||||
fn article(url: &str, content: &str) -> Article {
|
||||
Article {
|
||||
id: 0,
|
||||
canonical_url: url.into(),
|
||||
title: "T".into(),
|
||||
best_entry_id: 1,
|
||||
content_html: content.into(),
|
||||
word_count: 0,
|
||||
excerpt_only: false,
|
||||
image_count: 0,
|
||||
sources: vec![crate::types::SourceRef {
|
||||
entry_id: 1,
|
||||
feed_id: 1,
|
||||
feed_title: "Feed".into(),
|
||||
category: None,
|
||||
kind: SourceKind::Feed,
|
||||
}],
|
||||
first_seen: ts(),
|
||||
url: url.into(),
|
||||
author: None,
|
||||
feed_id: 1,
|
||||
feed_title: "Feed".into(),
|
||||
category: None,
|
||||
published_at: None,
|
||||
comments_url: None,
|
||||
image_urls: vec![],
|
||||
social: vec![],
|
||||
extract_method: ExtractMethod::Miniflux,
|
||||
}
|
||||
}
|
||||
|
||||
fn long_body(words: usize) -> String {
|
||||
format!("<p>{}</p>", "lorem ".repeat(words))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_keeps_the_allowlist_and_drops_everything_else() {
|
||||
let dirty = r#"
|
||||
<h1>Title</h1><h5>too deep</h5>
|
||||
<p class="x" onclick="evil()">Hello <em>there</em> <strong>you</strong></p>
|
||||
<script>alert(1)</script><style>p{color:red}</style>
|
||||
<div><span>unwrapped</span></div>
|
||||
<ul><li>one</li></ul><ol><li>two</li></ol>
|
||||
<blockquote>quote</blockquote><pre><code>fn main() {}</code></pre>
|
||||
<table><thead><tr><th>h</th></tr></thead><tbody><tr><td>c</td></tr></tbody></table>
|
||||
<figure><img src="https://x.dev/a.png" alt="a" width="10"><figcaption>cap</figcaption></figure>
|
||||
<a href="https://x.dev" target="_blank" rel="nofollow">link</a>
|
||||
<a href="javascript:alert(1)">bad</a>
|
||||
<iframe src="https://evil.dev"></iframe><hr><br>
|
||||
<!-- comment -->
|
||||
"#;
|
||||
let clean = sanitize(dirty);
|
||||
|
||||
for keep in [
|
||||
"<h1>",
|
||||
"<p>",
|
||||
"<em>",
|
||||
"<strong>",
|
||||
"<ul>",
|
||||
"<li>",
|
||||
"<ol>",
|
||||
"<blockquote>",
|
||||
"<pre>",
|
||||
"<code>",
|
||||
"<table>",
|
||||
"<th>",
|
||||
"<td>",
|
||||
"<figure>",
|
||||
"<figcaption>",
|
||||
"<hr",
|
||||
"<br",
|
||||
] {
|
||||
assert!(clean.contains(keep), "expected {keep} in {clean}");
|
||||
}
|
||||
assert!(clean.contains(r#"src="https://x.dev/a.png""#));
|
||||
assert!(clean.contains(r#"alt="a""#));
|
||||
assert!(clean.contains(r#"href="https://x.dev""#));
|
||||
|
||||
for drop in [
|
||||
"<h5",
|
||||
"<script",
|
||||
"<style",
|
||||
"alert(1)",
|
||||
"<div",
|
||||
"<span",
|
||||
"<iframe",
|
||||
"onclick",
|
||||
"class=",
|
||||
"width=",
|
||||
"javascript:",
|
||||
"<!--",
|
||||
] {
|
||||
assert!(!clean.contains(drop), "did not expect {drop} in {clean}");
|
||||
}
|
||||
// Text inside stripped containers survives; the tags do not.
|
||||
assert!(clean.contains("unwrapped"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_with_base_absolutizes_urls() {
|
||||
let html = r#"<p><a href="/next">n</a><img src="img/a.png" alt="a"></p>"#;
|
||||
let clean = sanitize_with_base(html, "https://blog.dev/posts/one");
|
||||
assert!(clean.contains(r#"href="https://blog.dev/next""#));
|
||||
assert!(clean.contains(r#"src="https://blog.dev/posts/img/a.png""#));
|
||||
// A bad base degrades to plain sanitization rather than failing.
|
||||
assert!(sanitize_with_base(html, "not a url").contains("/next"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_count_ignores_markup_and_script() {
|
||||
assert_eq!(word_count("<p>one two three</p>"), 3);
|
||||
assert_eq!(word_count("<p>a</p><script>b c d e</script>"), 1);
|
||||
assert_eq!(word_count("<p>& — ok</p>"), 1);
|
||||
assert_eq!(word_count(""), 0);
|
||||
assert_eq!(word_count("<p></p>"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_collection_resolves_and_caps() {
|
||||
let mut html = String::from(r#"<img src="/a.png"><img src="https://cdn.dev/b.png">"#);
|
||||
html.push_str(r#"<img data-src="c.png"><img src="/a.png"><img src="data:image/png;x">"#);
|
||||
for i in 0..20 {
|
||||
html.push_str(&format!(r#"<img src="/n{i}.png">"#));
|
||||
}
|
||||
let urls = collect_image_urls(&html, "https://blog.dev/posts/one");
|
||||
assert_eq!(urls.len(), MAX_IMAGES_PER_ARTICLE);
|
||||
assert_eq!(urls[0], "https://blog.dev/a.png");
|
||||
assert_eq!(urls[1], "https://cdn.dev/b.png");
|
||||
assert_eq!(urls[2], "https://blog.dev/posts/c.png");
|
||||
// Duplicates and data: URIs never appear.
|
||||
assert_eq!(urls.iter().filter(|u| u.ends_with("/a.png")).count(), 1);
|
||||
assert!(!urls.iter().any(|u| u.starts_with("data:")));
|
||||
assert!(collect_image_urls("<p>none</p>", "https://blog.dev").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paywall_heuristic() {
|
||||
let domains = merge_paywall_domains(vec!["paywalled.dev".into()]);
|
||||
// Known paywall host with a stub body.
|
||||
assert!(looks_paywalled("https://www.nytimes.com/x", 200, &domains));
|
||||
assert!(looks_paywalled("https://paywalled.dev/x", 200, &domains));
|
||||
// Same host, full article.
|
||||
assert!(!looks_paywalled(
|
||||
"https://www.nytimes.com/x",
|
||||
1500,
|
||||
&domains
|
||||
));
|
||||
// Unknown host with a normal-length body.
|
||||
assert!(!looks_paywalled("https://blog.dev/x", 200, &domains));
|
||||
// Anything this short is an excerpt no matter the host.
|
||||
assert!(looks_paywalled("https://blog.dev/x", 40, &domains));
|
||||
assert!(looks_paywalled("https://blog.dev/x", 0, &domains));
|
||||
// Unparseable URLs never claim a paywall on their own.
|
||||
assert!(!looks_paywalled("nonsense", 900, &domains));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn miniflux_content_wins_when_it_is_full_text() {
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
let article = article("https://blog.dev/p", &long_body(600));
|
||||
let out = extractor.extract(&article).await;
|
||||
assert_eq!(out.method, ExtractMethod::Miniflux);
|
||||
assert!(out.word_count >= FULL_TEXT_MIN_WORDS);
|
||||
assert!(!out.excerpt_only);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_content_falls_back_to_the_excerpt_note() {
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
let stub = article("https://blog.dev/p", "<p>Just a teaser.</p>");
|
||||
let out = extractor.extract(&stub).await;
|
||||
assert_eq!(out.method, ExtractMethod::Excerpt);
|
||||
assert!(out.excerpt_only);
|
||||
assert!(out.content_html.contains(EXCERPT_NOTE));
|
||||
assert!(out.content_html.contains("Just a teaser."));
|
||||
|
||||
// Empty feed content still yields a body, never a panic.
|
||||
let empty = article("https://blog.dev/p", "");
|
||||
let out = extractor.extract(&empty).await;
|
||||
assert_eq!(out.method, ExtractMethod::Excerpt);
|
||||
assert!(out.content_html.contains(EXCERPT_NOTE));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn extract_all_applies_results_and_counts() {
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
let mut articles = vec![
|
||||
article("https://blog.dev/full", &long_body(600)),
|
||||
article("https://blog.dev/stub", "<p>teaser</p>"),
|
||||
];
|
||||
articles[0]
|
||||
.content_html
|
||||
.push_str(r#"<p><img src="/pic.png" alt="p"></p>"#);
|
||||
|
||||
let stats = extractor.extract_all(&mut articles).await;
|
||||
assert_eq!(stats.from_miniflux, 1);
|
||||
assert_eq!(stats.excerpt_only, 1);
|
||||
assert_eq!(articles[0].extract_method, ExtractMethod::Miniflux);
|
||||
assert_eq!(articles[0].image_count, 1);
|
||||
assert_eq!(articles[0].image_urls, ["https://blog.dev/pic.png"]);
|
||||
assert!(articles[0].word_count >= 600);
|
||||
assert_eq!(articles[1].extract_method, ExtractMethod::Excerpt);
|
||||
assert!(articles[1].excerpt_only);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn offline_extractor_never_fetches() {
|
||||
let extractor = Extractor::offline(vec![]);
|
||||
assert!(!extractor.can_fetch());
|
||||
assert!(matches!(
|
||||
extractor.fetch_readable("https://blog.dev/p").await,
|
||||
Err(ExtractError::FetchDisabled)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readability_pulls_the_main_content_out_of_a_page() {
|
||||
let paragraph = "Readability keeps the body copy and throws away the chrome. ".repeat(20);
|
||||
let html = format!(
|
||||
"<html><head><title>A Post</title></head><body>\
|
||||
<nav><a href=\"/\">home</a></nav>\
|
||||
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
||||
<footer>© 2026</footer></body></html>"
|
||||
);
|
||||
let content = readability(&html, "https://blog.dev/p").expect("main content");
|
||||
assert!(content.contains("Readability keeps the body copy"));
|
||||
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
||||
assert!(word_count(&clean) > 200);
|
||||
assert!(!clean.contains("<nav"));
|
||||
}
|
||||
}
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
//! Shared HTTP client and retry policy (implementation notes §4, spec §3 "retry").
|
||||
//!
|
||||
//! One [`reqwest::Client`] is built at startup and cloned into every stage that
|
||||
//! talks to the network (Miniflux, social, extraction, images, world briefing).
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// Descriptive UA required by Reddit and polite everywhere else (§3.4, notes §4).
|
||||
pub const USER_AGENT: &str = "the-daily-epub/1.0 (personal rss digest; contact tyler@hallada.net)";
|
||||
|
||||
/// Default per-request timeout (§3.3: 10s).
|
||||
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Build the process-wide HTTP client: rustls, gzip, no cookie jar (notes §4).
|
||||
pub fn build_client(timeout: Duration) -> reqwest::Result<reqwest::Client> {
|
||||
reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.timeout(timeout)
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.gzip(true)
|
||||
// No cookie jar: the `cookies` feature is deliberately off (notes §4).
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Jittered exponential backoff, max 3 attempts (crate table "retry").
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct RetryPolicy {
|
||||
pub max_attempts: u32,
|
||||
pub base_delay: Duration,
|
||||
pub max_delay: Duration,
|
||||
}
|
||||
|
||||
impl Default for RetryPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_attempts: 3,
|
||||
base_delay: Duration::from_millis(500),
|
||||
max_delay: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryPolicy {
|
||||
/// Delay before attempt `attempt` (1-based), with ±25% jitter.
|
||||
pub fn delay_for(&self, attempt: u32) -> Duration {
|
||||
let exp = self
|
||||
.base_delay
|
||||
.saturating_mul(2u32.saturating_pow(attempt.saturating_sub(1)));
|
||||
let capped = exp.min(self.max_delay);
|
||||
let jitter = rand::random_range(0.75f64..1.25f64);
|
||||
Duration::from_secs_f64(capped.as_secs_f64() * jitter).min(self.max_delay)
|
||||
}
|
||||
|
||||
/// Run `op` until it succeeds or returns a non-retryable error.
|
||||
///
|
||||
/// `op` is retried while it yields an error for which `retryable` is true —
|
||||
/// network failures and 5xx responses (§3.1).
|
||||
pub async fn run<T, E, F, Fut>(
|
||||
&self,
|
||||
what: &str,
|
||||
retryable: impl Fn(&E) -> bool,
|
||||
mut op: F,
|
||||
) -> Result<T, E>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, E>>,
|
||||
E: std::fmt::Display,
|
||||
{
|
||||
let mut attempt = 1;
|
||||
loop {
|
||||
match op().await {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(e) if attempt < self.max_attempts && retryable(&e) => {
|
||||
let delay = self.delay_for(attempt);
|
||||
tracing::warn!(
|
||||
attempt,
|
||||
max = self.max_attempts,
|
||||
delay_ms = delay.as_millis() as u64,
|
||||
"{what} failed, retrying: {e}"
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
attempt += 1;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True for network-level failures and 5xx/429 responses (§3.1, §3.4).
|
||||
pub fn is_retryable(err: &reqwest::Error) -> bool {
|
||||
if err.is_timeout() || err.is_connect() || err.is_request() {
|
||||
return true;
|
||||
}
|
||||
match err.status() {
|
||||
Some(s) => s.is_server_error() || s == reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn client_builds() {
|
||||
build_client(DEFAULT_TIMEOUT).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_grows_and_is_capped() {
|
||||
let p = RetryPolicy::default();
|
||||
for attempt in 1..=3 {
|
||||
let d = p.delay_for(attempt);
|
||||
assert!(d <= p.max_delay);
|
||||
assert!(d >= Duration::from_millis(300));
|
||||
}
|
||||
// Second attempt doubles the base delay before jitter (1000ms ± 25%).
|
||||
assert!(p.delay_for(2) >= Duration::from_millis(750));
|
||||
// Overflow-safe and still capped for absurd attempt numbers.
|
||||
let huge = p.delay_for(30);
|
||||
assert!(huge <= p.max_delay && huge >= Duration::from_secs(7));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retries_until_success_then_stops() {
|
||||
let policy = RetryPolicy {
|
||||
max_attempts: 3,
|
||||
base_delay: Duration::from_millis(1),
|
||||
max_delay: Duration::from_millis(2),
|
||||
};
|
||||
let mut calls = 0;
|
||||
let out: Result<u8, String> = policy
|
||||
.run(
|
||||
"test",
|
||||
|_| true,
|
||||
|| {
|
||||
calls += 1;
|
||||
let n = calls;
|
||||
async move {
|
||||
if n < 3 {
|
||||
Err("boom".to_string())
|
||||
} else {
|
||||
Ok(7u8)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(out, Ok(7));
|
||||
assert_eq!(calls, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gives_up_after_max_attempts() {
|
||||
let policy = RetryPolicy {
|
||||
max_attempts: 3,
|
||||
base_delay: Duration::from_millis(1),
|
||||
max_delay: Duration::from_millis(2),
|
||||
};
|
||||
let mut calls = 0;
|
||||
let out: Result<u8, String> = policy
|
||||
.run(
|
||||
"test",
|
||||
|_| true,
|
||||
|| {
|
||||
calls += 1;
|
||||
async { Err("boom".to_string()) }
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(out.is_err());
|
||||
assert_eq!(calls, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_retryable_errors_fail_fast() {
|
||||
let policy = RetryPolicy::default();
|
||||
let mut calls = 0;
|
||||
let out: Result<u8, String> = policy
|
||||
.run(
|
||||
"test",
|
||||
|_| false,
|
||||
|| {
|
||||
calls += 1;
|
||||
async { Err("nope".to_string()) }
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(out.is_err());
|
||||
assert_eq!(calls, 1);
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
//! `daily-epub` — a personalized daily newspaper as an EPUB (spec §1, §2).
|
||||
//!
|
||||
//! The crate ships both a library and a thin `daily-epub` binary. Everything the
|
||||
//! pipeline does lives here so that integration tests can drive the stages
|
||||
//! directly (see `tests/e2e_pipeline.rs`) instead of shelling out to the binary.
|
||||
//!
|
||||
//! Pipeline order (spec §2), all of it wired in [`crate::pipeline`]:
|
||||
//!
|
||||
//! ```text
|
||||
//! Miniflux ingest → dedupe → extraction → persist → social enrichment
|
||||
//! → pre-filter → LLM scoring → selection → comments → world briefing
|
||||
//! → editorial → EPUB build (standard + X4) → XTC → publish → report
|
||||
//! ```
|
||||
|
||||
pub mod auth;
|
||||
pub mod comments;
|
||||
pub mod config;
|
||||
pub mod curate;
|
||||
pub mod db;
|
||||
pub mod dedupe;
|
||||
pub mod epub;
|
||||
pub mod extract;
|
||||
pub mod http;
|
||||
pub mod miniflux;
|
||||
pub mod pipeline;
|
||||
pub mod publish;
|
||||
pub mod report;
|
||||
pub mod server;
|
||||
pub mod social;
|
||||
pub mod types;
|
||||
pub mod world;
|
||||
|
||||
/// `CARGO_PKG_VERSION`, printed in the colophon and the OPDS generator tag.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
+331
@@ -0,0 +1,331 @@
|
||||
//! `daily-epub` — CLI entry point (spec §2).
|
||||
//!
|
||||
//! Everything of substance lives in the library (`src/lib.rs`); this binary only
|
||||
//! parses flags, loads config, opens the database and dispatches.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use daily_epub::config::Config;
|
||||
use daily_epub::db::Db;
|
||||
use daily_epub::pipeline::{self, GenerateOptions, GenerateOutcome};
|
||||
use daily_epub::report::RunReport;
|
||||
use daily_epub::{curate, http, server, social};
|
||||
|
||||
/// A personalized daily newspaper, delivered as an EPUB.
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "daily-epub", version, about, long_about = None)]
|
||||
struct Cli {
|
||||
/// Config file path (defaults to ./config.toml when present).
|
||||
#[arg(long, short, global = true, value_name = "FILE")]
|
||||
config: Option<PathBuf>,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
/// Build (and publish) one issue.
|
||||
Generate(GenerateArgs),
|
||||
/// Run the rating endpoints, XTC OPDS feed and static files.
|
||||
Serve,
|
||||
/// Taste-profile maintenance.
|
||||
#[command(subcommand)]
|
||||
Profile(ProfileCommand),
|
||||
/// Re-poll social scores for recent entries.
|
||||
BackfillSocial(BackfillSocialArgs),
|
||||
/// Database maintenance.
|
||||
#[command(subcommand)]
|
||||
Db(DbCommand),
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct GenerateArgs {
|
||||
/// Issue date in the configured timezone (defaults to today).
|
||||
#[arg(long, value_name = "YYYY-MM-DD")]
|
||||
date: Option<String>,
|
||||
/// Build everything but publish nothing: no BookOrbit copy, no issue record.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
/// Write artifacts here instead of `out_dir`.
|
||||
#[arg(long, value_name = "DIR")]
|
||||
out: Option<PathBuf>,
|
||||
/// Cap the lineup size (overrides `target_article_count`).
|
||||
#[arg(long, value_name = "N")]
|
||||
max_articles: Option<usize>,
|
||||
/// Skip every LLM call: prefilter order selects, excerpts stand in for summaries.
|
||||
#[arg(long)]
|
||||
skip_llm: bool,
|
||||
}
|
||||
|
||||
impl From<&GenerateArgs> for GenerateOptions {
|
||||
fn from(args: &GenerateArgs) -> Self {
|
||||
Self {
|
||||
date: args.date.clone(),
|
||||
dry_run: args.dry_run,
|
||||
out: args.out.clone(),
|
||||
max_articles: args.max_articles,
|
||||
skip_llm: args.skip_llm,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum ProfileCommand {
|
||||
/// Regenerate the taste profile from ratings history.
|
||||
Rebuild,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct BackfillSocialArgs {
|
||||
/// How many days back to re-poll.
|
||||
#[arg(long, default_value_t = 7)]
|
||||
days: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum DbCommand {
|
||||
/// Run pending sqlx migrations.
|
||||
Migrate,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let cli = Cli::parse();
|
||||
let config = Config::load(cli.config.as_deref()).context("loading configuration")?;
|
||||
tracing::debug!(?config.database_path, "configuration loaded");
|
||||
|
||||
match cli.command {
|
||||
Command::Generate(args) => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
let outcome = pipeline::generate(&config, &db, &GenerateOptions::from(&args)).await?;
|
||||
print_outcome(&outcome);
|
||||
}
|
||||
Command::Serve => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
server::serve(config, db).await?;
|
||||
}
|
||||
Command::Profile(ProfileCommand::Rebuild) => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
cmd_profile_rebuild(&config, &db).await?;
|
||||
}
|
||||
Command::BackfillSocial(args) => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
cmd_backfill_social(&db, args.days).await?;
|
||||
}
|
||||
Command::Db(DbCommand::Migrate) => {
|
||||
let db = Db::open(&config.database_path).await?;
|
||||
db.migrate().await?;
|
||||
println!("migrations up to date: {}", config.database_path.display());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `RUST_LOG`-driven tracing, defaulting to `info` (crate table "logging").
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info,sqlx=warn,hyper=warn,reqwest=warn"));
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(filter)
|
||||
.with_target(false)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Human-readable end-of-run output; the machine-readable form lives in `runs`
|
||||
/// and in the issue's `report_json` (§3.13).
|
||||
fn print_outcome(outcome: &GenerateOutcome) {
|
||||
print_report(&outcome.report);
|
||||
if let Some(issue) = &outcome.issue {
|
||||
print_lineup(issue);
|
||||
}
|
||||
for artifact in &outcome.artifacts {
|
||||
println!(
|
||||
"built: {} ({:.1} MiB)",
|
||||
artifact.path.display(),
|
||||
artifact.bytes as f64 / (1024.0 * 1024.0)
|
||||
);
|
||||
}
|
||||
if let Some(xtc) = &outcome.xtc {
|
||||
println!("xtc: {}", xtc.display());
|
||||
}
|
||||
match &outcome.published {
|
||||
Some(published) => {
|
||||
for artifact in &published.epubs {
|
||||
println!("published: {}", artifact.path.display());
|
||||
}
|
||||
if let Some(xtc) = &published.xtc {
|
||||
println!("published: {}", xtc.display());
|
||||
}
|
||||
if let Some(opds) = &published.opds {
|
||||
println!("opds: {}", opds.display());
|
||||
}
|
||||
if published.pruned > 0 {
|
||||
println!("pruned: {} expired files", published.pruned);
|
||||
}
|
||||
}
|
||||
None => println!("dry run: nothing was published"),
|
||||
}
|
||||
}
|
||||
|
||||
fn print_report(report: &RunReport) {
|
||||
println!("{}", report.summary_line());
|
||||
if let (Some(start), Some(end)) = (report.window_start, report.window_end) {
|
||||
println!("window: {start} → {end}");
|
||||
}
|
||||
println!(
|
||||
"entries: {} from {} feeds → {} articles ({} merged, {} dropped)",
|
||||
report.counts.entries_fetched,
|
||||
report.counts.feeds_seen,
|
||||
report.counts.articles,
|
||||
report.counts.duplicates_merged,
|
||||
report.counts.entries_dropped,
|
||||
);
|
||||
println!(
|
||||
"tokens: {} input · {} cached · {} output = ${:.4}",
|
||||
report.usage.input_tokens,
|
||||
report.usage.cached_tokens,
|
||||
report.usage.output_tokens,
|
||||
report.cost_usd,
|
||||
);
|
||||
for warning in &report.warnings {
|
||||
println!("warning: {warning}");
|
||||
}
|
||||
if let Some(err) = &report.error {
|
||||
println!("error: {err}");
|
||||
}
|
||||
tracing::debug!("{}", report.to_json());
|
||||
}
|
||||
|
||||
/// The day's lineup, section by section — the `--dry-run` deliverable (§4 M3).
|
||||
fn print_lineup(issue: &daily_epub::types::Issue) {
|
||||
println!(
|
||||
"\nThe Daily EPUB No. {} — {} · {}",
|
||||
issue.meta.issue_number,
|
||||
issue.meta.display_date,
|
||||
issue.meta.stats_line()
|
||||
);
|
||||
for section in &issue.lineup.section_order {
|
||||
println!("\n {section}");
|
||||
for pick in issue.lineup.section_picks(section) {
|
||||
let lead = if pick.is_lead { "★ " } else { " " };
|
||||
println!(
|
||||
" {lead}{} — {} ({} min)",
|
||||
pick.article.title,
|
||||
pick.article.feed_title,
|
||||
pick.article.reading_minutes()
|
||||
);
|
||||
}
|
||||
}
|
||||
if issue.world_briefing.is_some() {
|
||||
println!("\n {}", daily_epub::types::WORLD_BRIEFING_SECTION);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Other subcommands
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn cmd_profile_rebuild(config: &Config, db: &Db) -> Result<()> {
|
||||
let meter = curate::llm::UsageMeter::new(&config.deepseek, config.max_daily_usd);
|
||||
let profile = curate::profile::load_or_build(db, &config.interests_opml).await?;
|
||||
let llm = curate::llm::LlmClient::new(&config.deepseek, profile.text, meter)?;
|
||||
let rebuilt = curate::profile::rebuild(db, &llm, &config.interests_opml).await?;
|
||||
let feeds = curate::profile::rebuild_feed_priors(db).await?;
|
||||
println!(
|
||||
"taste profile rebuilt (version {}, {} chars); {feeds} feed priors refreshed",
|
||||
rebuilt.version,
|
||||
rebuilt.text.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cmd_backfill_social(db: &Db, days: u32) -> Result<()> {
|
||||
let http = http::build_client(http::DEFAULT_TIMEOUT)?;
|
||||
let enricher = social::SocialEnricher::new(http, db.clone());
|
||||
let updated = enricher.backfill(days).await?;
|
||||
println!("refreshed social scores for {updated} articles");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::CommandFactory;
|
||||
|
||||
#[test]
|
||||
fn cli_definition_is_valid() {
|
||||
Cli::command().debug_assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_every_subcommand_from_the_spec() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"daily-epub",
|
||||
"generate",
|
||||
"--date",
|
||||
"2026-08-15",
|
||||
"--dry-run",
|
||||
"--out",
|
||||
"./out",
|
||||
"--max-articles",
|
||||
"6",
|
||||
"--skip-llm",
|
||||
])
|
||||
.unwrap();
|
||||
match cli.command {
|
||||
Command::Generate(a) => {
|
||||
assert_eq!(a.date.as_deref(), Some("2026-08-15"));
|
||||
assert!(a.dry_run);
|
||||
assert_eq!(a.out, Some(PathBuf::from("./out")));
|
||||
assert_eq!(a.max_articles, Some(6));
|
||||
assert!(a.skip_llm);
|
||||
|
||||
let opts = GenerateOptions::from(&a);
|
||||
assert_eq!(opts.date.as_deref(), Some("2026-08-15"));
|
||||
assert!(opts.dry_run && opts.skip_llm);
|
||||
assert_eq!(opts.max_articles, Some(6));
|
||||
}
|
||||
other => panic!("expected generate, got {other:?}"),
|
||||
}
|
||||
|
||||
assert!(matches!(
|
||||
Cli::try_parse_from(["daily-epub", "serve"])
|
||||
.unwrap()
|
||||
.command,
|
||||
Command::Serve
|
||||
));
|
||||
assert!(matches!(
|
||||
Cli::try_parse_from(["daily-epub", "profile", "rebuild"])
|
||||
.unwrap()
|
||||
.command,
|
||||
Command::Profile(ProfileCommand::Rebuild)
|
||||
));
|
||||
assert!(matches!(
|
||||
Cli::try_parse_from(["daily-epub", "backfill-social", "--days", "14"])
|
||||
.unwrap()
|
||||
.command,
|
||||
Command::BackfillSocial(BackfillSocialArgs { days: 14 })
|
||||
));
|
||||
assert!(matches!(
|
||||
Cli::try_parse_from(["daily-epub", "db", "migrate"])
|
||||
.unwrap()
|
||||
.command,
|
||||
Command::Db(DbCommand::Migrate)
|
||||
));
|
||||
|
||||
let cli = Cli::try_parse_from(["daily-epub", "--config", "/tmp/x.toml", "serve"]).unwrap();
|
||||
assert_eq!(cli.config, Some(PathBuf::from("/tmp/x.toml")));
|
||||
}
|
||||
}
|
||||
+504
@@ -0,0 +1,504 @@
|
||||
//! Miniflux API client (spec §3.1).
|
||||
//!
|
||||
//! Reads only: entries are fetched with `published_after` inside the lookback
|
||||
//! window **regardless of read/unread status**, and read state is never mutated
|
||||
//! so normal reader usage is undisturbed.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::config::MinifluxConfig;
|
||||
use crate::http::{RetryPolicy, is_retryable};
|
||||
use crate::types::{Entry, FeedId};
|
||||
|
||||
/// Miniflux caps `limit` at 250 (§3.1).
|
||||
pub const MAX_PAGE_LIMIT: u32 = 250;
|
||||
/// Safety valve so a misconfigured window cannot page forever.
|
||||
const MAX_PAGES: u32 = 200;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MinifluxError {
|
||||
#[error("miniflux api key is not configured (set DAILY_EPUB_MINIFLUX__API_KEY)")]
|
||||
MissingApiKey,
|
||||
#[error("miniflux request failed: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("miniflux returned {status} for {path}: {body}")]
|
||||
Status {
|
||||
status: u16,
|
||||
path: String,
|
||||
body: String,
|
||||
},
|
||||
#[error("could not parse miniflux response for {path}: {source}")]
|
||||
Decode {
|
||||
path: String,
|
||||
#[source]
|
||||
source: serde_json::Error,
|
||||
},
|
||||
}
|
||||
|
||||
type Result<T> = std::result::Result<T, MinifluxError>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire types (only the fields §3.1 lists as used)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A category as embedded in a feed object.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MinifluxCategory {
|
||||
#[serde(default)]
|
||||
pub id: i64,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
}
|
||||
|
||||
/// `GET /v1/feeds` element — used to build the `feed_id → metadata` map (§3.1).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MinifluxFeed {
|
||||
pub id: FeedId,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub site_url: String,
|
||||
#[serde(default)]
|
||||
pub feed_url: String,
|
||||
#[serde(default)]
|
||||
pub category: Option<MinifluxCategory>,
|
||||
}
|
||||
|
||||
/// `GET /v1/entries` element.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MinifluxEntry {
|
||||
pub id: i64,
|
||||
pub feed_id: FeedId,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub comments_url: String,
|
||||
#[serde(default)]
|
||||
pub author: String,
|
||||
/// RFC3339 with offset, e.g. `2026-08-15T04:00:00-04:00`.
|
||||
#[serde(default)]
|
||||
pub published_at: String,
|
||||
/// Miniflux's stored content: full text when "fetch original content" is on.
|
||||
#[serde(default)]
|
||||
pub content: String,
|
||||
#[serde(default)]
|
||||
pub status: String,
|
||||
#[serde(default)]
|
||||
pub starred: bool,
|
||||
#[serde(default)]
|
||||
pub reading_time: i64,
|
||||
/// Present when Miniflux inlines the feed object on the entry.
|
||||
#[serde(default)]
|
||||
pub feed: Option<MinifluxFeed>,
|
||||
}
|
||||
|
||||
/// Envelope returned by `GET /v1/entries`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EntriesResponse {
|
||||
#[serde(default)]
|
||||
pub total: i64,
|
||||
#[serde(default)]
|
||||
pub entries: Vec<MinifluxEntry>,
|
||||
}
|
||||
|
||||
/// Feed metadata joined onto every entry we persist (§3.1).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct FeedMeta {
|
||||
pub id: FeedId,
|
||||
pub title: String,
|
||||
pub site_url: String,
|
||||
/// The subscribed feed URL — the only reliable "came via Scour" tell (§3.2).
|
||||
pub feed_url: String,
|
||||
pub category: Option<String>,
|
||||
}
|
||||
|
||||
impl From<&MinifluxFeed> for FeedMeta {
|
||||
fn from(f: &MinifluxFeed) -> Self {
|
||||
Self {
|
||||
id: f.id,
|
||||
title: f.title.clone(),
|
||||
site_url: f.site_url.clone(),
|
||||
feed_url: f.feed_url.clone(),
|
||||
category: f.category.as_ref().map(|c| c.title.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `feed_id → "{feed_url} {site_url}"`, the haystack
|
||||
/// [`crate::dedupe::classify_source_with_feed`] matches against (§3.2).
|
||||
pub fn feed_urls(feeds: &HashMap<FeedId, FeedMeta>) -> crate::dedupe::FeedUrls {
|
||||
feeds
|
||||
.iter()
|
||||
.map(|(id, meta)| {
|
||||
(
|
||||
*id,
|
||||
format!("{} {}", meta.feed_url, meta.site_url)
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl MinifluxEntry {
|
||||
/// Parse `published_at`, tolerating the empty/zero values Miniflux can emit.
|
||||
pub fn published_timestamp(&self) -> Option<Timestamp> {
|
||||
if self.published_at.is_empty() {
|
||||
return None;
|
||||
}
|
||||
self.published_at.parse::<Timestamp>().ok()
|
||||
}
|
||||
|
||||
fn opt(s: &str) -> Option<String> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to the persisted [`Entry`] shape (§3.13).
|
||||
///
|
||||
/// `canonical_url` is left `None`: the dedupe stage (§3.2) fills it in.
|
||||
pub fn into_entry(self, feeds: &HashMap<FeedId, FeedMeta>, fetched_at: Timestamp) -> Entry {
|
||||
let meta = feeds.get(&self.feed_id);
|
||||
let inline = self.feed.as_ref();
|
||||
let feed_title = meta
|
||||
.map(|m| m.title.clone())
|
||||
.or_else(|| inline.map(|f| f.title.clone()))
|
||||
.filter(|t| !t.is_empty());
|
||||
let category = meta
|
||||
.and_then(|m| m.category.clone())
|
||||
.or_else(|| {
|
||||
inline
|
||||
.and_then(|f| f.category.as_ref())
|
||||
.map(|c| c.title.clone())
|
||||
})
|
||||
.filter(|c| !c.is_empty());
|
||||
Entry {
|
||||
id: self.id,
|
||||
feed_id: self.feed_id,
|
||||
feed_title,
|
||||
category,
|
||||
published_at: self.published_timestamp(),
|
||||
title: self.title.trim().to_string(),
|
||||
url: self.url.trim().to_string(),
|
||||
canonical_url: None,
|
||||
author: Self::opt(&self.author),
|
||||
comments_url: Self::opt(&self.comments_url),
|
||||
raw_content: self.content,
|
||||
fetched_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Read-only Miniflux client (§3.1).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MinifluxClient {
|
||||
http: reqwest::Client,
|
||||
base_url: String,
|
||||
api_key: String,
|
||||
page_limit: u32,
|
||||
retry: RetryPolicy,
|
||||
}
|
||||
|
||||
impl MinifluxClient {
|
||||
/// Build a client from `[miniflux]` config; fails if the API key is absent.
|
||||
pub fn new(cfg: &MinifluxConfig, http: reqwest::Client) -> Result<Self> {
|
||||
let api_key = cfg
|
||||
.api_key
|
||||
.clone()
|
||||
.filter(|k| !k.trim().is_empty())
|
||||
.ok_or(MinifluxError::MissingApiKey)?;
|
||||
Ok(Self {
|
||||
http,
|
||||
base_url: cfg.base_url.trim_end_matches('/').to_string(),
|
||||
api_key,
|
||||
page_limit: cfg.page_limit.clamp(1, MAX_PAGE_LIMIT),
|
||||
retry: RetryPolicy::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_retry_policy(mut self, retry: RetryPolicy) -> Self {
|
||||
self.retry = retry;
|
||||
self
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}/v1{}", self.base_url, path)
|
||||
}
|
||||
|
||||
/// GET `path` with the auth header, retrying network/5xx failures (§3.1).
|
||||
async fn get_json<T: serde::de::DeserializeOwned>(
|
||||
&self,
|
||||
path: &str,
|
||||
query: &[(&str, String)],
|
||||
) -> Result<T> {
|
||||
let url = self.url(path);
|
||||
let body = self
|
||||
.retry
|
||||
.run(
|
||||
&format!("GET {path}"),
|
||||
|e: &MinifluxError| match e {
|
||||
MinifluxError::Http(e) => is_retryable(e),
|
||||
MinifluxError::Status { status, .. } => *status >= 500 || *status == 429,
|
||||
_ => false,
|
||||
},
|
||||
|| async {
|
||||
let resp = self
|
||||
.http
|
||||
.get(&url)
|
||||
.header("X-Auth-Token", &self.api_key)
|
||||
.header("Accept", "application/json")
|
||||
.query(query)
|
||||
.send()
|
||||
.await?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await?;
|
||||
if !status.is_success() {
|
||||
return Err(MinifluxError::Status {
|
||||
status: status.as_u16(),
|
||||
path: path.to_string(),
|
||||
body: text.chars().take(300).collect(),
|
||||
});
|
||||
}
|
||||
Ok(text)
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
serde_json::from_str(&body).map_err(|source| MinifluxError::Decode {
|
||||
path: path.to_string(),
|
||||
source,
|
||||
})
|
||||
}
|
||||
|
||||
/// `GET /v1/feeds` — once per run (§3.1).
|
||||
pub async fn feeds(&self) -> Result<Vec<MinifluxFeed>> {
|
||||
self.get_json("/feeds", &[]).await
|
||||
}
|
||||
|
||||
/// `feed_id → {title, site_url, category.title}` (§3.1).
|
||||
pub async fn feed_map(&self) -> Result<HashMap<FeedId, FeedMeta>> {
|
||||
Ok(self
|
||||
.feeds()
|
||||
.await?
|
||||
.iter()
|
||||
.map(|f| (f.id, FeedMeta::from(f)))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// One page of `GET /v1/entries`, ordered by `published_at` desc (§3.1).
|
||||
pub async fn entries_page(
|
||||
&self,
|
||||
published_after: Timestamp,
|
||||
offset: u32,
|
||||
) -> Result<EntriesResponse> {
|
||||
self.get_json(
|
||||
"/entries",
|
||||
&[
|
||||
("order", "published_at".to_string()),
|
||||
("direction", "desc".to_string()),
|
||||
("published_after", published_after.as_second().to_string()),
|
||||
("limit", self.page_limit.to_string()),
|
||||
("offset", offset.to_string()),
|
||||
],
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Page through every entry published after `published_after`, read or not (§3.1).
|
||||
pub async fn entries_since(&self, published_after: Timestamp) -> Result<Vec<MinifluxEntry>> {
|
||||
let mut all: Vec<MinifluxEntry> = Vec::new();
|
||||
let mut offset = 0u32;
|
||||
for page in 0..MAX_PAGES {
|
||||
let resp = self.entries_page(published_after, offset).await?;
|
||||
let got = resp.entries.len();
|
||||
tracing::debug!(
|
||||
page,
|
||||
offset,
|
||||
got,
|
||||
total = resp.total,
|
||||
"miniflux entries page"
|
||||
);
|
||||
all.extend(resp.entries);
|
||||
if got < self.page_limit as usize || all.len() as i64 >= resp.total {
|
||||
break;
|
||||
}
|
||||
offset += self.page_limit;
|
||||
}
|
||||
Ok(all)
|
||||
}
|
||||
|
||||
/// Full ingest: fetch the feed map, page the window, and map to [`Entry`] rows.
|
||||
///
|
||||
/// Entries published after `until` (the run's "now") are dropped so a
|
||||
/// re-run for a past date does not pull in newer stories.
|
||||
pub async fn ingest_window(
|
||||
&self,
|
||||
since: Timestamp,
|
||||
until: Timestamp,
|
||||
fetched_at: Timestamp,
|
||||
) -> Result<(Vec<Entry>, HashMap<FeedId, FeedMeta>)> {
|
||||
let feeds = self.feed_map().await?;
|
||||
tracing::info!(feeds = feeds.len(), "loaded miniflux feed metadata");
|
||||
let raw = self.entries_since(since).await?;
|
||||
tracing::info!(entries = raw.len(), "fetched miniflux entries");
|
||||
let entries = raw
|
||||
.into_iter()
|
||||
.filter(|e| match e.published_timestamp() {
|
||||
Some(ts) => ts <= until,
|
||||
// Keep entries with unparseable dates; dedupe will judge them.
|
||||
None => true,
|
||||
})
|
||||
.map(|e| e.into_entry(&feeds, fetched_at))
|
||||
.collect();
|
||||
Ok((entries, feeds))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const ENTRIES_JSON: &str = r#"{
|
||||
"total": 2,
|
||||
"entries": [
|
||||
{
|
||||
"id": 30011,
|
||||
"user_id": 1,
|
||||
"feed_id": 42,
|
||||
"status": "unread",
|
||||
"hash": "abc",
|
||||
"title": " Writing a Kernel in Rust ",
|
||||
"url": "https://example.com/kernel-rust",
|
||||
"comments_url": "https://news.ycombinator.com/item?id=44551122",
|
||||
"published_at": "2026-08-15T04:12:00-04:00",
|
||||
"created_at": "2026-08-15T08:13:00Z",
|
||||
"author": "Jane Dev",
|
||||
"content": "<p>A long post.</p>",
|
||||
"starred": false,
|
||||
"reading_time": 14,
|
||||
"enclosures": null,
|
||||
"feed": {
|
||||
"id": 42,
|
||||
"title": "Inline Feed Title",
|
||||
"site_url": "https://example.com",
|
||||
"feed_url": "https://example.com/feed.xml",
|
||||
"category": {"id": 3, "title": "Inline Category"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 30012,
|
||||
"feed_id": 99,
|
||||
"status": "read",
|
||||
"title": "No feed object here",
|
||||
"url": "https://other.example/post",
|
||||
"comments_url": "",
|
||||
"published_at": "",
|
||||
"author": "",
|
||||
"content": "",
|
||||
"starred": true,
|
||||
"reading_time": 0
|
||||
}
|
||||
]
|
||||
}"#;
|
||||
|
||||
const FEEDS_JSON: &str = r#"[
|
||||
{"id": 42, "title": "Lobsters", "site_url": "https://lobste.rs",
|
||||
"feed_url": "https://lobste.rs/rss", "category": {"id": 1, "title": "Tech"}},
|
||||
{"id": 99, "title": "Scour: Rust", "site_url": "https://scour.ing",
|
||||
"feed_url": "https://scour.ing/feed", "category": null}
|
||||
]"#;
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserializes_entries_response() {
|
||||
let resp: EntriesResponse = serde_json::from_str(ENTRIES_JSON).unwrap();
|
||||
assert_eq!(resp.total, 2);
|
||||
assert_eq!(resp.entries.len(), 2);
|
||||
let first = &resp.entries[0];
|
||||
assert_eq!(first.id, 30011);
|
||||
assert_eq!(first.feed_id, 42);
|
||||
assert_eq!(
|
||||
first.comments_url,
|
||||
"https://news.ycombinator.com/item?id=44551122"
|
||||
);
|
||||
assert_eq!(
|
||||
first.published_timestamp(),
|
||||
Some(ts("2026-08-15T08:12:00Z"))
|
||||
);
|
||||
assert_eq!(first.feed.as_ref().unwrap().title, "Inline Feed Title");
|
||||
assert!(resp.entries[1].published_timestamp().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserializes_feeds_and_builds_map() {
|
||||
let feeds: Vec<MinifluxFeed> = serde_json::from_str(FEEDS_JSON).unwrap();
|
||||
let map: HashMap<FeedId, FeedMeta> =
|
||||
feeds.iter().map(|f| (f.id, FeedMeta::from(f))).collect();
|
||||
assert_eq!(map[&42].title, "Lobsters");
|
||||
assert_eq!(map[&42].category.as_deref(), Some("Tech"));
|
||||
assert_eq!(map[&99].category, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_entries_onto_feed_metadata() {
|
||||
let feeds: Vec<MinifluxFeed> = serde_json::from_str(FEEDS_JSON).unwrap();
|
||||
let map: HashMap<FeedId, FeedMeta> =
|
||||
feeds.iter().map(|f| (f.id, FeedMeta::from(f))).collect();
|
||||
let resp: EntriesResponse = serde_json::from_str(ENTRIES_JSON).unwrap();
|
||||
let fetched = ts("2026-08-15T09:30:00Z");
|
||||
let entries: Vec<Entry> = resp
|
||||
.entries
|
||||
.into_iter()
|
||||
.map(|e| e.into_entry(&map, fetched))
|
||||
.collect();
|
||||
|
||||
// /v1/feeds metadata wins over the inline feed object.
|
||||
assert_eq!(entries[0].feed_title.as_deref(), Some("Lobsters"));
|
||||
assert_eq!(entries[0].category.as_deref(), Some("Tech"));
|
||||
assert_eq!(entries[0].title, "Writing a Kernel in Rust");
|
||||
assert_eq!(entries[0].author.as_deref(), Some("Jane Dev"));
|
||||
assert!(entries[0].comments_url.is_some());
|
||||
assert_eq!(entries[0].canonical_url, None);
|
||||
assert_eq!(entries[0].fetched_at, fetched);
|
||||
|
||||
// Empty strings become None, not "".
|
||||
assert_eq!(entries[1].author, None);
|
||||
assert_eq!(entries[1].comments_url, None);
|
||||
assert_eq!(entries[1].feed_title.as_deref(), Some("Scour: Rust"));
|
||||
assert_eq!(entries[1].category, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requires_an_api_key() {
|
||||
let http = crate::http::build_client(crate::http::DEFAULT_TIMEOUT).unwrap();
|
||||
let mut cfg = MinifluxConfig::default();
|
||||
assert!(matches!(
|
||||
MinifluxClient::new(&cfg, http.clone()),
|
||||
Err(MinifluxError::MissingApiKey)
|
||||
));
|
||||
cfg.api_key = Some(" ".into());
|
||||
assert!(MinifluxClient::new(&cfg, http.clone()).is_err());
|
||||
cfg.api_key = Some("token".into());
|
||||
cfg.base_url = "http://127.0.0.1:8082/".into();
|
||||
cfg.page_limit = 5000;
|
||||
let c = MinifluxClient::new(&cfg, http).unwrap();
|
||||
assert_eq!(c.base_url, "http://127.0.0.1:8082");
|
||||
assert_eq!(c.page_limit, MAX_PAGE_LIMIT);
|
||||
assert_eq!(c.url("/entries"), "http://127.0.0.1:8082/v1/entries");
|
||||
}
|
||||
}
|
||||
+707
@@ -0,0 +1,707 @@
|
||||
//! The `generate` pipeline, wired end to end (spec §2, §3.6 wiring).
|
||||
//!
|
||||
//! ```text
|
||||
//! Miniflux ingest ─▶ dedupe ─▶ extraction ─▶ persist ─▶ social enrichment
|
||||
//! ─▶ pre-filter ─▶ LLM scoring ─▶ selection ─▶ comments ─▶ world briefing
|
||||
//! ─▶ editorial ─▶ EPUB (standard + X4) ─▶ XTC ─▶ publish ─▶ report
|
||||
//! ```
|
||||
//!
|
||||
//! Failure policy (notes §3):
|
||||
//!
|
||||
//! * **Fatal** — Miniflux ingest, SQLite writes, EPUB assembly, publishing. Without
|
||||
//! any one of them there is no issue, so the run fails loudly and the `runs` row
|
||||
//! records why.
|
||||
//! * **Best effort** — social enrichment, comments, the world briefing, images and
|
||||
//! the XTC conversion. They log, add a warning to the report (status `degraded`)
|
||||
//! and the run continues.
|
||||
//! * **Degrading** — every DeepSeek stage. A missing key, a dead API or a tripped
|
||||
//! `max_daily_usd` guardrail turns the run into the `--skip-llm` shape
|
||||
//! (prefilter order selects, feed excerpts stand in for summaries) rather than
|
||||
//! losing the day's issue.
|
||||
//!
|
||||
//! The run is idempotent per date (notes §12): entries, articles, scores and the
|
||||
//! issue itself are upserted, `issue_articles` is replaced wholesale, and the
|
||||
//! published filenames are derived from the date.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use jiff::civil::Date;
|
||||
use jiff::{Timestamp, Zoned};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::curate::llm::{LlmClient, UsageMeter};
|
||||
use crate::curate::{Curator, editorial, profile};
|
||||
use crate::db::Db;
|
||||
use crate::extract::Extractor;
|
||||
use crate::miniflux::MinifluxClient;
|
||||
use crate::publish::Published;
|
||||
use crate::report::{RunReport, RunStatus};
|
||||
use crate::types::{
|
||||
Article, Artifact, Colophon, Edition, Issue, IssueMeta, Lineup, reading_minutes,
|
||||
};
|
||||
use crate::{comments, dedupe, epub, http, miniflux, publish, social, world};
|
||||
|
||||
/// One `generate` invocation's inputs — the CLI flags, already parsed (§2).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct GenerateOptions {
|
||||
/// `--date YYYY-MM-DD`; `None` means today in the configured timezone.
|
||||
pub date: Option<String>,
|
||||
/// `--dry-run`: build everything, publish nothing, record no issue.
|
||||
pub dry_run: bool,
|
||||
/// `--out DIR`, overriding `out_dir`.
|
||||
pub out: Option<PathBuf>,
|
||||
/// `--max-articles N`, overriding `target_article_count`.
|
||||
pub max_articles: Option<usize>,
|
||||
/// `--skip-llm`: no DeepSeek call at all.
|
||||
pub skip_llm: bool,
|
||||
}
|
||||
|
||||
/// What one run produced, for the caller to print (§3.13).
|
||||
#[derive(Debug)]
|
||||
pub struct GenerateOutcome {
|
||||
pub report: RunReport,
|
||||
/// `None` only when the run failed before assembly.
|
||||
pub issue: Option<Issue>,
|
||||
/// The EPUBs as written into the output directory.
|
||||
pub artifacts: Vec<Artifact>,
|
||||
/// The converted XTC artifact, when the converter ran (§3.11).
|
||||
pub xtc: Option<PathBuf>,
|
||||
/// `None` under `--dry-run`.
|
||||
pub published: Option<Published>,
|
||||
}
|
||||
|
||||
/// Resolve `--date` (or today) in the configured timezone (notes §2).
|
||||
pub fn resolve_date(config: &Config, raw: Option<&str>) -> Result<Date> {
|
||||
let tz = config.tz()?;
|
||||
match raw {
|
||||
Some(s) => s
|
||||
.parse::<Date>()
|
||||
.with_context(|| format!("invalid --date {s:?}, expected YYYY-MM-DD")),
|
||||
None => Ok(Zoned::now().with_time_zone(tz).date()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The ingest window `[end - lookback_hours, end]` where `end` is the end of the
|
||||
/// issue's day in the configured timezone, clamped to now (§3.1).
|
||||
pub fn ingest_window(config: &Config, date: Date) -> Result<(Timestamp, Timestamp)> {
|
||||
let tz = config.tz()?;
|
||||
let now = Timestamp::now();
|
||||
let end_of_day = date
|
||||
.to_zoned(tz)
|
||||
.context("resolving issue date in the configured timezone")?
|
||||
.tomorrow()
|
||||
.context("computing the end of the issue day")?
|
||||
.timestamp();
|
||||
let end = end_of_day.min(now);
|
||||
let start = end - jiff::Span::new().hours(i64::from(config.lookback_hours));
|
||||
Ok((start, end))
|
||||
}
|
||||
|
||||
/// "Friday, August 15, 2026" — the cover/front-page dateline (§3.10).
|
||||
pub fn display_date(date: Date) -> String {
|
||||
const WEEKDAYS: [&str; 7] = [
|
||||
"Monday",
|
||||
"Tuesday",
|
||||
"Wednesday",
|
||||
"Thursday",
|
||||
"Friday",
|
||||
"Saturday",
|
||||
"Sunday",
|
||||
];
|
||||
const MONTHS: [&str; 12] = [
|
||||
"January",
|
||||
"February",
|
||||
"March",
|
||||
"April",
|
||||
"May",
|
||||
"June",
|
||||
"July",
|
||||
"August",
|
||||
"September",
|
||||
"October",
|
||||
"November",
|
||||
"December",
|
||||
];
|
||||
let weekday = WEEKDAYS[(date.weekday().to_monday_zero_offset() as usize).min(6)];
|
||||
let month = MONTHS[(date.month() as usize).clamp(1, 12) - 1];
|
||||
format!("{weekday}, {month} {}, {}", date.day(), date.year())
|
||||
}
|
||||
|
||||
/// Materialize the [`Issue`] the EPUB builder consumes (§3.10).
|
||||
///
|
||||
/// Pure: every count is derived from the lineup, so the same inputs always give
|
||||
/// the same cover and stats line (notes §12).
|
||||
pub fn build_issue(
|
||||
date: Date,
|
||||
issue_number: i64,
|
||||
generated_at: Timestamp,
|
||||
lineup: Lineup,
|
||||
editorial: crate::types::Editorial,
|
||||
world_briefing: Option<crate::types::WorldBriefing>,
|
||||
colophon: Colophon,
|
||||
) -> Issue {
|
||||
let total_words = lineup.total_words();
|
||||
let section_count = lineup.section_order.len() as i64
|
||||
+ i64::from(world_briefing.is_some() && !lineup.section_order.is_empty());
|
||||
let meta = IssueMeta {
|
||||
date,
|
||||
issue_number,
|
||||
generated_at,
|
||||
display_date: display_date(date),
|
||||
article_count: lineup.picks.len() as i64,
|
||||
section_count,
|
||||
total_words,
|
||||
reading_minutes: reading_minutes(total_words),
|
||||
};
|
||||
Issue {
|
||||
meta,
|
||||
lineup,
|
||||
editorial,
|
||||
world_briefing,
|
||||
colophon,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy stage-C summaries onto their picks so `issue_articles` and the EPUB agree.
|
||||
pub fn apply_summaries(lineup: &mut Lineup, editorial: &crate::types::Editorial) {
|
||||
for pick in &mut lineup.picks {
|
||||
if pick.summary.is_none()
|
||||
&& let Some(summary) = editorial.summaries.get(&pick.article.id)
|
||||
{
|
||||
pick.summary = Some(summary.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Driver
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run one issue end to end, recording a `runs` row either way (§2, §3.13).
|
||||
pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Result<GenerateOutcome> {
|
||||
let date = resolve_date(config, opts.date.as_deref())?;
|
||||
let started_at = Timestamp::now();
|
||||
let (window_start, window_end) = ingest_window(config, date)?;
|
||||
let out_dir = opts.out.clone().unwrap_or_else(|| config.out_dir.clone());
|
||||
let target = opts.max_articles.unwrap_or(config.target_article_count);
|
||||
|
||||
let span = tracing::info_span!("generate", %date, dry_run = opts.dry_run);
|
||||
let _guard = span.enter();
|
||||
tracing::info!(
|
||||
%window_start,
|
||||
%window_end,
|
||||
lookback_hours = config.lookback_hours,
|
||||
target,
|
||||
skip_llm = opts.skip_llm,
|
||||
out = %out_dir.display(),
|
||||
"starting run"
|
||||
);
|
||||
|
||||
let run_id = db.start_run(date, started_at).await?;
|
||||
let mut report = RunReport::new(date, started_at);
|
||||
report.window_start = Some(window_start);
|
||||
report.window_end = Some(window_end);
|
||||
if opts.dry_run {
|
||||
report.status = RunStatus::DryRun;
|
||||
}
|
||||
|
||||
let ctx = StageContext {
|
||||
config,
|
||||
db,
|
||||
date,
|
||||
target,
|
||||
out_dir,
|
||||
dry_run: opts.dry_run,
|
||||
skip_llm: opts.skip_llm,
|
||||
};
|
||||
let stages = match run_stages(&ctx, window_start, window_end, &mut report).await {
|
||||
Ok(stages) => {
|
||||
report.finish(
|
||||
Timestamp::now(),
|
||||
config.deepseek.price_input_per_mtok,
|
||||
config.deepseek.price_cached_input_per_mtok,
|
||||
config.deepseek.price_output_per_mtok,
|
||||
);
|
||||
stages
|
||||
}
|
||||
Err(e) => {
|
||||
report.fail(Timestamp::now(), format!("{e:#}"));
|
||||
db.finish_run(run_id, &report).await?;
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
db.finish_run(run_id, &report).await?;
|
||||
// The issue row is written before the report is costed, so stamp the finished
|
||||
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
||||
if !opts.dry_run
|
||||
&& let Some(issue) = stages.issue.as_ref()
|
||||
&& let Err(e) = db
|
||||
.upsert_issue(
|
||||
date,
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&report.to_json()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(error = %e, "could not attach the run report to the issue");
|
||||
}
|
||||
Ok(GenerateOutcome {
|
||||
report,
|
||||
issue: stages.issue,
|
||||
artifacts: stages.artifacts,
|
||||
xtc: stages.xtc,
|
||||
published: stages.published,
|
||||
})
|
||||
}
|
||||
|
||||
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
||||
#[derive(Debug)]
|
||||
struct StageOutput {
|
||||
issue: Option<Issue>,
|
||||
artifacts: Vec<Artifact>,
|
||||
xtc: Option<PathBuf>,
|
||||
published: Option<Published>,
|
||||
}
|
||||
|
||||
/// Everything the stages need that does not change between them.
|
||||
struct StageContext<'a> {
|
||||
config: &'a Config,
|
||||
db: &'a Db,
|
||||
date: Date,
|
||||
target: usize,
|
||||
out_dir: PathBuf,
|
||||
dry_run: bool,
|
||||
skip_llm: bool,
|
||||
}
|
||||
|
||||
async fn run_stages(
|
||||
ctx: &StageContext<'_>,
|
||||
window_start: Timestamp,
|
||||
window_end: Timestamp,
|
||||
report: &mut RunReport,
|
||||
) -> Result<StageOutput> {
|
||||
let (config, db, date) = (ctx.config, ctx.db, ctx.date);
|
||||
let http = http::build_client(http::DEFAULT_TIMEOUT).context("building http client")?;
|
||||
|
||||
// --- Stage 1: Miniflux ingest (§3.1) — fatal on failure ---
|
||||
let stage = Timestamp::now();
|
||||
let client = MinifluxClient::new(&config.miniflux, http.clone())
|
||||
.context("constructing the miniflux client")?;
|
||||
let (entries, feeds) = client
|
||||
.ingest_window(window_start, window_end, Timestamp::now())
|
||||
.await
|
||||
.context("ingesting entries from miniflux")?;
|
||||
|
||||
report.counts.entries_fetched = entries.len() as i64;
|
||||
report.counts.feeds_seen = entries
|
||||
.iter()
|
||||
.map(|e| e.feed_id)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len() as i64;
|
||||
let mut per_feed: BTreeMap<String, i64> = BTreeMap::new();
|
||||
for entry in &entries {
|
||||
let name = entry
|
||||
.feed_title
|
||||
.clone()
|
||||
.or_else(|| feeds.get(&entry.feed_id).map(|f| f.title.clone()))
|
||||
.unwrap_or_else(|| format!("feed {}", entry.feed_id));
|
||||
*per_feed.entry(name).or_insert(0) += 1;
|
||||
}
|
||||
report.per_feed_counts = per_feed;
|
||||
|
||||
// Entries are persisted even on a dry run: `articles.best_entry_id` is a real
|
||||
// foreign key, and the social cache keys off the article ids. Only the
|
||||
// watermark (an ingest bookmark) is left alone.
|
||||
let written = db
|
||||
.upsert_entries(&entries)
|
||||
.await
|
||||
.context("persisting entries")?;
|
||||
if ctx.dry_run {
|
||||
tracing::info!(written, "dry run: persisted entries, watermark left alone");
|
||||
} else {
|
||||
db.set_watermark(window_end).await?;
|
||||
tracing::info!(written, "persisted entries and advanced the watermark");
|
||||
}
|
||||
report.timings.record("ingest", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 2: normalize + dedupe (§3.2) ---
|
||||
let stage = Timestamp::now();
|
||||
let feed_urls = miniflux::feed_urls(&feeds);
|
||||
let (mut articles, dedupe_stats) = dedupe::cluster_with_feeds(entries, &feed_urls);
|
||||
report.counts.entries_dropped = dedupe_stats.dropped_non_article as i64;
|
||||
report.counts.articles = dedupe_stats.clusters as i64;
|
||||
report.counts.duplicates_merged = dedupe_stats.merged as i64;
|
||||
report.timings.record("dedupe", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 3: content extraction (§3.3) — before persisting, because it
|
||||
// replaces the raw Miniflux body that `dedupe` left on the cluster ---
|
||||
let stage = Timestamp::now();
|
||||
let extractor = Extractor::new(http.clone(), config.curation.paywall_domains.clone());
|
||||
let extract_stats = extractor.extract_all(&mut articles).await;
|
||||
report.counts.extracted = (extract_stats.from_miniflux + extract_stats.from_readability) as i64;
|
||||
report.counts.excerpt_only = extract_stats.excerpt_only as i64;
|
||||
if extract_stats.fetch_failures > 0 {
|
||||
report.warn(format!(
|
||||
"{} articles fell back to a feed excerpt",
|
||||
extract_stats.fetch_failures
|
||||
));
|
||||
}
|
||||
report.timings.record("extract", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 4: persist the clusters, minting real article ids (§3.13) ---
|
||||
let stage = Timestamp::now();
|
||||
persist_articles(db, &mut articles).await?;
|
||||
report.timings.record("persist", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 5: social enrichment (§3.4) — best effort, needs real ids ---
|
||||
let stage = Timestamp::now();
|
||||
let enricher = social::SocialEnricher::new(http.clone(), db.clone());
|
||||
report.counts.social_hits = enricher.enrich_all(&mut articles).await as i64;
|
||||
report.timings.record("social", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 6: feed priors, then the heuristic pre-filter (§3.5, §3.9) ---
|
||||
let stage = Timestamp::now();
|
||||
if let Err(e) = profile::rebuild_feed_priors(db).await {
|
||||
report.warn(format!("could not rebuild feed priors: {e:#}"));
|
||||
}
|
||||
|
||||
let meter = UsageMeter::new(&config.deepseek, config.max_daily_usd);
|
||||
// `max_daily_usd` is a ceiling for the *day*, not for one invocation, so a
|
||||
// re-run inherits what earlier runs for this date already spent (§3.6).
|
||||
match db.spend_for_date(date).await {
|
||||
Ok(spent) if spent > 0.0 => {
|
||||
tracing::info!(spent, "preloading today's recorded DeepSeek spend");
|
||||
meter.preload_cost(spent);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::warn!(error = %e, "could not read today's spend; starting from zero"),
|
||||
}
|
||||
|
||||
let llm = build_llm(ctx, &meter, report).await;
|
||||
let llm_available = llm.is_some();
|
||||
let mut curator_config = config.clone();
|
||||
curator_config.target_article_count = ctx.target;
|
||||
let curator = Curator::new(curator_config, db.clone(), llm);
|
||||
|
||||
let mut candidates = curator
|
||||
.prefilter(articles, date)
|
||||
.await
|
||||
.context("running the heuristic pre-filter")?;
|
||||
report.counts.candidates = candidates.len() as i64;
|
||||
report.timings.record("prefilter", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 7: LLM scoring, then selection (§3.6 A + B) ---
|
||||
let stage = Timestamp::now();
|
||||
if llm_available && let Err(e) = curator.score(&mut candidates, date).await {
|
||||
// A dead API or a tripped budget must not cost us the issue: selection
|
||||
// degrades to prefilter order exactly as `--skip-llm` does.
|
||||
report.warn(format!("LLM scoring failed; ranking heuristically: {e:#}"));
|
||||
}
|
||||
report.counts.llm_scored = candidates.iter().filter(|c| c.llm.is_some()).count() as i64;
|
||||
|
||||
let mut lineup = curator
|
||||
.select(candidates, date)
|
||||
.await
|
||||
.context("selecting the lineup")?;
|
||||
report.counts.selected = lineup.picks.len() as i64;
|
||||
if lineup.picks.is_empty() {
|
||||
report.warn("the lineup is empty — check the lookback window and pre-filter");
|
||||
}
|
||||
report.timings.record("curate", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 8: comment chapters for the selected articles (§3.7) ---
|
||||
let stage = Timestamp::now();
|
||||
report.counts.discussions = comments::fetch_all(&http, &mut lineup.picks).await as i64;
|
||||
report.timings.record("comments", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 9: world briefing (§3.8) — non-fatal by construction ---
|
||||
let stage = Timestamp::now();
|
||||
let world_briefing = world::fetch_optional(&http, date, config.world_briefing).await;
|
||||
if config.world_briefing && world_briefing.is_none() {
|
||||
report.warn("the world briefing was unavailable; the section is omitted");
|
||||
}
|
||||
report.timings.record("world", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 10: editorial (§3.6 C) ---
|
||||
let stage = Timestamp::now();
|
||||
let editorial = match curator.editorial(&lineup).await {
|
||||
Ok(editorial) => editorial,
|
||||
Err(e) => {
|
||||
report.warn(format!(
|
||||
"editorial generation failed; using excerpts: {e:#}"
|
||||
));
|
||||
editorial::fallback_editorial(&lineup)
|
||||
}
|
||||
};
|
||||
apply_summaries(&mut lineup, &editorial);
|
||||
report.timings.record("editorial", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 11: assemble the issue (§3.10) ---
|
||||
let issue_number = db
|
||||
.next_issue_number(date)
|
||||
.await
|
||||
.context("computing the issue number")?;
|
||||
let colophon = Colophon {
|
||||
model: if llm_available {
|
||||
config.deepseek.model.clone()
|
||||
} else {
|
||||
"none (--skip-llm)".into()
|
||||
},
|
||||
entries_fetched: report.counts.entries_fetched,
|
||||
feeds_seen: report.counts.feeds_seen,
|
||||
candidates: report.counts.candidates,
|
||||
cost_usd: meter.cost_usd(),
|
||||
generator_version: format!("daily-epub {}", crate::VERSION),
|
||||
};
|
||||
report.usage = meter.total();
|
||||
let issue = build_issue(
|
||||
date,
|
||||
issue_number,
|
||||
Timestamp::now(),
|
||||
lineup,
|
||||
editorial,
|
||||
world_briefing,
|
||||
colophon,
|
||||
);
|
||||
|
||||
// --- Stage 12: build both EPUB editions (§3.10) — fatal on failure ---
|
||||
let stage = Timestamp::now();
|
||||
let (artifacts, images) = epub::build_all(&issue, config, &ctx.out_dir)
|
||||
.await
|
||||
.context("building the EPUB editions")?;
|
||||
report.counts.images_embedded = images as i64;
|
||||
report.timings.record("epub", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 13: XTC conversion (§3.11) — best effort ---
|
||||
let stage = Timestamp::now();
|
||||
let x4 = artifacts.iter().find(|a| a.edition == Edition::X4);
|
||||
let xtc = match x4 {
|
||||
Some(artifact) if config.xtc.enabled => {
|
||||
match epub::x4::convert(&config.xtc, &artifact.path, &ctx.out_dir).await {
|
||||
Ok(path) => Some(path),
|
||||
Err(e) => {
|
||||
// Carry the converter's own message into the report: "did not
|
||||
// produce a file" alone cannot distinguish a missing Node from
|
||||
// a missing settings file from a genuine conversion failure.
|
||||
tracing::warn!("xtc conversion skipped: {e}");
|
||||
report.warn(format!("the XTC conversion did not produce a file: {e}"));
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
report.timings.record("xtc", elapsed_ms(stage));
|
||||
|
||||
// --- Stage 14: publish + record the issue (§3.11) ---
|
||||
let stage = Timestamp::now();
|
||||
let published = if ctx.dry_run {
|
||||
tracing::info!(
|
||||
out = %ctx.out_dir.display(),
|
||||
"dry run: skipping BookOrbit/XTC publishing and the issue record"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
let published = publish::publish_issue(db, config, &issue, &artifacts, xtc.as_deref())
|
||||
.await
|
||||
.context("publishing the issue")?;
|
||||
record_issue(db, &issue, &published)
|
||||
.await
|
||||
.context("recording the issue")?;
|
||||
Some(published)
|
||||
};
|
||||
report.timings.record("publish", elapsed_ms(stage));
|
||||
|
||||
Ok(StageOutput {
|
||||
issue: Some(issue),
|
||||
artifacts,
|
||||
xtc,
|
||||
published,
|
||||
})
|
||||
}
|
||||
|
||||
/// Insert/refresh the `articles` rows and stamp the returned ids back on (§3.13).
|
||||
async fn persist_articles(db: &Db, articles: &mut [Article]) -> Result<()> {
|
||||
for article in articles.iter_mut() {
|
||||
let id = db
|
||||
.upsert_article(article)
|
||||
.await
|
||||
.with_context(|| format!("persisting article {}", article.canonical_url))?;
|
||||
article.id = id;
|
||||
for social_ref in &mut article.social {
|
||||
social_ref.article_id = id;
|
||||
}
|
||||
}
|
||||
tracing::info!(articles = articles.len(), "persisted article clusters");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write the `issues` row and replace `issue_articles` for the date (notes §12).
|
||||
async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<()> {
|
||||
let path_for = |edition: Edition| {
|
||||
published
|
||||
.epubs
|
||||
.iter()
|
||||
.find(|a| a.edition == edition)
|
||||
.map(|a| a.path.display().to_string())
|
||||
};
|
||||
let epub_path = path_for(Edition::Standard);
|
||||
let x4_path = path_for(Edition::X4);
|
||||
let xtc_path = published.xtc.as_ref().map(|p| p.display().to_string());
|
||||
|
||||
db.upsert_issue(
|
||||
issue.meta.date,
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
epub_path.as_deref(),
|
||||
x4_path.as_deref(),
|
||||
xtc_path.as_deref(),
|
||||
Some(&issue.editorial.front_page_html),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
db.replace_issue_articles(issue.meta.date, &issue.lineup.picks)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the DeepSeek client, running the weekly profile rebuild when it is due.
|
||||
///
|
||||
/// Returns `None` for `--skip-llm` and for every configuration/API problem: the
|
||||
/// caller then curates heuristically instead of failing the run (§3.6).
|
||||
async fn build_llm(
|
||||
ctx: &StageContext<'_>,
|
||||
meter: &UsageMeter,
|
||||
report: &mut RunReport,
|
||||
) -> Option<LlmClient> {
|
||||
if ctx.skip_llm {
|
||||
tracing::info!("--skip-llm: no DeepSeek call will be made");
|
||||
return None;
|
||||
}
|
||||
let profile = match profile::load_or_build(ctx.db, &ctx.config.interests_opml).await {
|
||||
Ok(profile) => profile,
|
||||
Err(e) => {
|
||||
report.warn(format!(
|
||||
"could not build the taste profile; curating heuristically: {e:#}"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let client = match LlmClient::new(&ctx.config.deepseek, profile.text, meter.clone()) {
|
||||
Ok(client) => client,
|
||||
Err(e) => {
|
||||
report.warn(format!(
|
||||
"DeepSeek is unavailable; curating heuristically: {e}"
|
||||
));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Weekly rewrite of the "learned adjustments" section (§3.6c). It changes the
|
||||
// system prompt, so the client is rebuilt around the new profile.
|
||||
match profile::weekly_rebuild_if_due(ctx.db, &client, &ctx.config.interests_opml).await {
|
||||
Ok(Some(rebuilt)) => {
|
||||
tracing::info!(version = rebuilt.version, "taste profile rebuilt");
|
||||
match LlmClient::new(&ctx.config.deepseek, rebuilt.text, meter.clone()) {
|
||||
Ok(refreshed) => Some(refreshed),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "keeping the previous profile client");
|
||||
Some(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => Some(client),
|
||||
Err(e) => {
|
||||
report.warn(format!("weekly profile rebuild failed: {e:#}"));
|
||||
Some(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn elapsed_ms(since: Timestamp) -> i64 {
|
||||
(Timestamp::now().as_millisecond() - since.as_millisecond()).max(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn display_date_matches_the_masthead_format() {
|
||||
let date: Date = "2026-08-15".parse().unwrap();
|
||||
assert_eq!(display_date(date), "Saturday, August 15, 2026");
|
||||
let date: Date = "2026-01-01".parse().unwrap();
|
||||
assert_eq!(display_date(date), "Thursday, January 1, 2026");
|
||||
let date: Date = "2026-12-31".parse().unwrap();
|
||||
assert_eq!(display_date(date), "Thursday, December 31, 2026");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingest_window_spans_the_lookback() {
|
||||
let config = Config::default();
|
||||
let date: Date = "2020-01-15".parse().unwrap();
|
||||
let (start, end) = ingest_window(&config, date).unwrap();
|
||||
assert!(start < end);
|
||||
let hours = (end.as_second() - start.as_second()) / 3600;
|
||||
assert_eq!(hours, i64::from(config.lookback_hours));
|
||||
// 2020-01-16T00:00 America/New_York == 2020-01-16T05:00Z
|
||||
assert_eq!(end.to_string(), "2020-01-16T05:00:00Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_date_parses_and_defaults() {
|
||||
let config = Config::default();
|
||||
assert_eq!(
|
||||
resolve_date(&config, Some("2026-08-15"))
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
"2026-08-15"
|
||||
);
|
||||
assert!(resolve_date(&config, Some("nope")).is_err());
|
||||
assert!(resolve_date(&config, None).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_meta_is_derived_from_the_lineup() {
|
||||
let lineup = crate::epub::build::fixtures::issue().lineup;
|
||||
let words = lineup.total_words();
|
||||
let sections = lineup.section_order.len() as i64;
|
||||
let issue = build_issue(
|
||||
"2026-08-15".parse().unwrap(),
|
||||
7,
|
||||
"2026-08-15T09:30:00Z".parse().unwrap(),
|
||||
lineup,
|
||||
crate::types::Editorial::default(),
|
||||
None,
|
||||
Colophon::default(),
|
||||
);
|
||||
assert_eq!(issue.meta.issue_number, 7);
|
||||
assert_eq!(issue.meta.display_date, "Saturday, August 15, 2026");
|
||||
assert_eq!(issue.meta.article_count, issue.lineup.picks.len() as i64);
|
||||
assert_eq!(issue.meta.section_count, sections);
|
||||
assert_eq!(issue.meta.total_words, words);
|
||||
assert_eq!(issue.meta.reading_minutes, reading_minutes(words));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summaries_land_on_their_picks() {
|
||||
let mut lineup = crate::epub::build::fixtures::issue().lineup;
|
||||
for pick in &mut lineup.picks {
|
||||
pick.summary = None;
|
||||
}
|
||||
let mut editorial = crate::types::Editorial::default();
|
||||
let first = lineup.picks[0].article.id;
|
||||
editorial.summaries.insert(first, "An abstract.".into());
|
||||
apply_summaries(&mut lineup, &editorial);
|
||||
assert_eq!(lineup.picks[0].summary.as_deref(), Some("An abstract."));
|
||||
assert!(lineup.picks[1..].iter().all(|p| p.summary.is_none()));
|
||||
}
|
||||
}
|
||||
+875
@@ -0,0 +1,875 @@
|
||||
//! Publishing: BookOrbit watched folder, XTC delivery, OPDS feed, retention
|
||||
//! (spec §3.11).
|
||||
//!
|
||||
//! Everything here is deliberately dumb about *how* artifacts were produced: the
|
||||
//! EPUB/XTC stages hand over finished files, this module only copies, indexes and
|
||||
//! prunes them. Copies are atomic (temp file in the destination directory, then
|
||||
//! `rename`) so BookOrbit's watcher and CrossPoint's OPDS client never observe a
|
||||
//! half-written book.
|
||||
//!
|
||||
//! [`crate::pipeline`] ends a non-dry run with one call —
|
||||
//! `publish_issue(db, config, &issue, &artifacts, xtc_path.as_deref())`, where
|
||||
//! `artifacts` are the `epub::build_all` outputs and `xtc_path` is
|
||||
//! `epub::x4::convert`'s output (`None` when the converter is disabled or
|
||||
//! failed) — and feeds the returned [`Published`] paths into
|
||||
//! `db.upsert_issue(..., epub_path, x4_path, xtc_path, ...)`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use jiff::civil::Date;
|
||||
use jiff::{Timestamp, Zoned};
|
||||
use sqlx::Row;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::Db;
|
||||
use crate::types::{Artifact, Edition, Issue};
|
||||
|
||||
/// Filename of the generated static OPDS feed (§3.11).
|
||||
pub const XTC_OPDS_FILENAME: &str = "xtc.xml";
|
||||
/// Number of issues listed in the XTC OPDS feed (§3.11).
|
||||
pub const XTC_FEED_ENTRIES: usize = 14;
|
||||
/// Every published file starts with this (the retention sweep keys off it).
|
||||
pub const FILE_PREFIX: &str = "The Daily EPUB - ";
|
||||
/// Extensions the retention sweep is allowed to delete (§3.11).
|
||||
pub const PRUNABLE_EXTENSIONS: [&str; 3] = ["epub", "xtc", "xtch"];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum PublishError {
|
||||
#[error("io error at {path}: {source}")]
|
||||
Io {
|
||||
path: PathBuf,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("publish directory does not exist: {0}")]
|
||||
MissingDir(PathBuf),
|
||||
}
|
||||
|
||||
impl PublishError {
|
||||
fn at(path: impl Into<PathBuf>) -> impl FnOnce(std::io::Error) -> PublishError {
|
||||
let path = path.into();
|
||||
move |source| PublishError::Io { path, source }
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything one `generate` run published (§3.11).
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct Published {
|
||||
/// The EPUB artifacts, rewritten to point at their published locations.
|
||||
pub epubs: Vec<Artifact>,
|
||||
/// The XTC artifact's published location, when the converter produced one.
|
||||
pub xtc: Option<PathBuf>,
|
||||
/// The regenerated OPDS feed.
|
||||
pub opds: Option<PathBuf>,
|
||||
/// How many expired files the retention sweep removed.
|
||||
pub pruned: usize,
|
||||
}
|
||||
|
||||
/// Canonical published filename: `The Daily EPUB - 2026-08-15 (X4).epub` (§3.11).
|
||||
pub fn issue_filename(date: Date, edition: Edition, extension: &str) -> String {
|
||||
format!("{FILE_PREFIX}{date}{}.{extension}", edition.file_suffix())
|
||||
}
|
||||
|
||||
/// Parse the issue date back out of a published filename, `None` when the name
|
||||
/// is not one of ours (the retention sweep must never touch foreign files).
|
||||
pub fn date_from_filename(name: &str) -> Option<Date> {
|
||||
let rest = name.strip_prefix(FILE_PREFIX)?;
|
||||
let extension = Path::new(name).extension()?.to_str()?.to_ascii_lowercase();
|
||||
if !PRUNABLE_EXTENSIONS.contains(&extension.as_str()) {
|
||||
return None;
|
||||
}
|
||||
rest.get(..10)?.parse::<Date>().ok()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Copying
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Atomic copy: write to a temp file in the destination dir, then rename (§3.11).
|
||||
///
|
||||
/// The temp file is created beside the destination so the rename stays within one
|
||||
/// filesystem; a failed copy leaves the previous version of `dest` intact.
|
||||
pub async fn atomic_copy(src: &Path, dest: &Path) -> Result<(), PublishError> {
|
||||
let dir = dest.parent().unwrap_or_else(|| Path::new("."));
|
||||
ensure_dir(dir).await?;
|
||||
let tmp = dir.join(format!(
|
||||
".{}.{}.{}.tmp",
|
||||
dest.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("daily-epub"),
|
||||
std::process::id(),
|
||||
Timestamp::now().as_nanosecond()
|
||||
));
|
||||
|
||||
let bytes = tokio::fs::read(src).await.map_err(PublishError::at(src))?;
|
||||
let write = async {
|
||||
let mut file = tokio::fs::File::create(&tmp).await?;
|
||||
file.write_all(&bytes).await?;
|
||||
file.sync_all().await?;
|
||||
Ok::<(), std::io::Error>(())
|
||||
};
|
||||
if let Err(source) = write.await {
|
||||
let _ = tokio::fs::remove_file(&tmp).await;
|
||||
return Err(PublishError::Io { path: tmp, source });
|
||||
}
|
||||
if let Err(source) = tokio::fs::rename(&tmp, dest).await {
|
||||
let _ = tokio::fs::remove_file(&tmp).await;
|
||||
return Err(PublishError::Io {
|
||||
path: dest.to_path_buf(),
|
||||
source,
|
||||
});
|
||||
}
|
||||
tracing::debug!(
|
||||
src = %src.display(),
|
||||
dest = %dest.display(),
|
||||
bytes = bytes.len(),
|
||||
"published atomically"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_dir(dir: &Path) -> Result<(), PublishError> {
|
||||
tokio::fs::create_dir_all(dir)
|
||||
.await
|
||||
.map_err(PublishError::at(dir))
|
||||
}
|
||||
|
||||
/// Copy both EPUB editions into the BookOrbit watched folder (§3.11).
|
||||
///
|
||||
/// Returns the published paths in the same order as `artifacts`.
|
||||
pub async fn publish_epubs(
|
||||
artifacts: &[Artifact],
|
||||
issue: &Issue,
|
||||
cfg: &Config,
|
||||
) -> Result<Vec<PathBuf>, PublishError> {
|
||||
let dir = &cfg.publish.bookorbit_dir;
|
||||
ensure_dir(dir).await?;
|
||||
let mut published = Vec::with_capacity(artifacts.len());
|
||||
for artifact in artifacts {
|
||||
let extension = artifact
|
||||
.path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("epub");
|
||||
let dest = dir.join(issue_filename(issue.meta.date, artifact.edition, extension));
|
||||
atomic_copy(&artifact.path, &dest).await?;
|
||||
tracing::info!(
|
||||
edition = ?artifact.edition,
|
||||
dest = %dest.display(),
|
||||
bytes = artifact.bytes,
|
||||
"published edition to the BookOrbit library"
|
||||
);
|
||||
published.push(dest);
|
||||
}
|
||||
Ok(published)
|
||||
}
|
||||
|
||||
/// Copy the `.xtc`/`.xtch` artifact into `publish.xtc_dir` (§3.11).
|
||||
pub async fn publish_xtc(xtc: &Path, cfg: &Config) -> Result<PathBuf, PublishError> {
|
||||
let dir = &cfg.publish.xtc_dir;
|
||||
ensure_dir(dir).await?;
|
||||
let name = xtc
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("daily-epub.xtch");
|
||||
let dest = dir.join(name);
|
||||
atomic_copy(xtc, &dest).await?;
|
||||
tracing::info!(dest = %dest.display(), "published XTC artifact");
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Publish everything one run produced, refresh the OPDS feed and prune (§3.11).
|
||||
///
|
||||
/// `xtc` is `None` when the converter is disabled or failed — that is not an
|
||||
/// error, the X4 falls back to the EPUB edition from BookOrbit.
|
||||
pub async fn publish_issue(
|
||||
db: &Db,
|
||||
cfg: &Config,
|
||||
issue: &Issue,
|
||||
artifacts: &[Artifact],
|
||||
xtc: Option<&Path>,
|
||||
) -> Result<Published, PublishError> {
|
||||
let span = tracing::info_span!("publish", date = %issue.meta.date);
|
||||
let _guard = span.enter();
|
||||
|
||||
let paths = publish_epubs(artifacts, issue, cfg).await?;
|
||||
let epubs = artifacts
|
||||
.iter()
|
||||
.zip(paths)
|
||||
.map(|(artifact, path)| Artifact {
|
||||
path,
|
||||
..artifact.clone()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let xtc = match xtc {
|
||||
Some(src) => Some(publish_xtc(src, cfg).await?),
|
||||
None => None,
|
||||
};
|
||||
let opds = Some(write_xtc_opds(db, cfg).await?);
|
||||
let pruned = prune(cfg, issue.meta.date).await?;
|
||||
|
||||
Ok(Published {
|
||||
epubs,
|
||||
xtc,
|
||||
opds,
|
||||
pruned,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OPDS 1.2 acquisition feed (§3.11)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One published XTC file, as listed in the feed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct XtcFile {
|
||||
name: String,
|
||||
date: Option<Date>,
|
||||
modified: Timestamp,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
/// Regenerate the static OPDS 1.2 acquisition feed for the XTC directory:
|
||||
/// newest first, last [`XTC_FEED_ENTRIES`], entries typed
|
||||
/// `application/octet-stream` (§3.11).
|
||||
pub async fn write_xtc_opds(db: &Db, cfg: &Config) -> Result<PathBuf, PublishError> {
|
||||
let dir = &cfg.publish.xtc_dir;
|
||||
ensure_dir(dir).await?;
|
||||
let files = scan_xtc_dir(dir).await?;
|
||||
let numbers = issue_numbers(db, &files).await;
|
||||
let feed = render_opds(&files, &numbers, &cfg.server.public_url, Timestamp::now());
|
||||
|
||||
let dest = dir.join(XTC_OPDS_FILENAME);
|
||||
write_atomic(&dest, feed.as_bytes()).await?;
|
||||
tracing::info!(entries = files.len(), dest = %dest.display(), "wrote the XTC OPDS feed");
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// XTC artifacts in `dir`, newest first, capped at [`XTC_FEED_ENTRIES`].
|
||||
async fn scan_xtc_dir(dir: &Path) -> Result<Vec<XtcFile>, PublishError> {
|
||||
let mut entries = tokio::fs::read_dir(dir)
|
||||
.await
|
||||
.map_err(PublishError::at(dir))?;
|
||||
let mut files = Vec::new();
|
||||
while let Some(entry) = entries.next_entry().await.map_err(PublishError::at(dir))? {
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
let extension = Path::new(&name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
if !matches!(extension.as_str(), "xtc" | "xtch") {
|
||||
continue;
|
||||
}
|
||||
let meta = match entry.metadata().await {
|
||||
Ok(meta) if meta.is_file() => meta,
|
||||
Ok(_) => continue,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, name, "skipping unreadable XTC file");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let modified = meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|m| Timestamp::try_from(m).ok())
|
||||
.unwrap_or_else(Timestamp::now);
|
||||
files.push(XtcFile {
|
||||
date: date_from_filename(&name),
|
||||
name,
|
||||
modified,
|
||||
bytes: meta.len(),
|
||||
});
|
||||
}
|
||||
// Newest first: by issue date when the filename carries one, else by mtime.
|
||||
files.sort_by(|a, b| {
|
||||
b.date
|
||||
.cmp(&a.date)
|
||||
.then(b.modified.cmp(&a.modified))
|
||||
.then(a.name.cmp(&b.name))
|
||||
});
|
||||
files.truncate(XTC_FEED_ENTRIES);
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Issue numbers for the dated files, best-effort (the feed is still valid
|
||||
/// without them). Uses the `db` escape hatch — no bespoke helper in `db.rs`.
|
||||
async fn issue_numbers(db: &Db, files: &[XtcFile]) -> BTreeMap<Date, i64> {
|
||||
let mut numbers = BTreeMap::new();
|
||||
for date in files.iter().filter_map(|f| f.date) {
|
||||
let row = sqlx::query("SELECT issue_number FROM issues WHERE date = ?")
|
||||
.bind(date.to_string())
|
||||
.fetch_optional(db.pool())
|
||||
.await;
|
||||
match row {
|
||||
Ok(Some(row)) => {
|
||||
numbers.insert(date, row.get::<i64, _>("issue_number"));
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => tracing::warn!(error = %e, %date, "issue number lookup failed"),
|
||||
}
|
||||
}
|
||||
numbers
|
||||
}
|
||||
|
||||
/// Render the Atom/OPDS document (§3.11).
|
||||
fn render_opds(
|
||||
files: &[XtcFile],
|
||||
numbers: &BTreeMap<Date, i64>,
|
||||
public_url: &str,
|
||||
now: Timestamp,
|
||||
) -> String {
|
||||
let base = public_url.trim_end_matches('/');
|
||||
let self_href = format!("{base}/opds/xtc.xml");
|
||||
let updated = files.first().map(|f| f.modified).unwrap_or(now);
|
||||
|
||||
let mut out = String::with_capacity(1024 + files.len() * 512);
|
||||
out.push_str("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
|
||||
out.push_str(
|
||||
"<feed xmlns=\"http://www.w3.org/2005/Atom\" \
|
||||
xmlns:dc=\"http://purl.org/dc/terms/\" \
|
||||
xmlns:opds=\"http://opds-spec.org/2010/catalog\">\n",
|
||||
);
|
||||
out.push_str(" <id>urn:daily-epub:xtc</id>\n");
|
||||
out.push_str(" <title>The Daily EPUB — XTC editions</title>\n");
|
||||
out.push_str(&format!(" <updated>{}</updated>\n", rfc3339(updated)));
|
||||
out.push_str(" <author><name>The Daily EPUB</name></author>\n");
|
||||
out.push_str(&format!(
|
||||
" <link rel=\"self\" href=\"{}\" type=\"{}\"/>\n",
|
||||
xml_escape(&self_href),
|
||||
crate::server::OPDS_CONTENT_TYPE
|
||||
));
|
||||
out.push_str(&format!(
|
||||
" <link rel=\"start\" href=\"{}\" type=\"{}\"/>\n",
|
||||
xml_escape(&self_href),
|
||||
crate::server::OPDS_CONTENT_TYPE
|
||||
));
|
||||
|
||||
for file in files {
|
||||
let title = match file.date {
|
||||
Some(date) => format!("The Daily EPUB — {date}"),
|
||||
None => file.name.clone(),
|
||||
};
|
||||
let summary = match file.date.and_then(|d| numbers.get(&d)) {
|
||||
Some(n) => format!("Issue #{n} · {}", human_bytes(file.bytes)),
|
||||
None => human_bytes(file.bytes),
|
||||
};
|
||||
let href = format!("{base}/files/xtc/{}", percent_encode(&file.name));
|
||||
out.push_str(" <entry>\n");
|
||||
out.push_str(&format!(" <title>{}</title>\n", xml_escape(&title)));
|
||||
out.push_str(&format!(
|
||||
" <id>urn:daily-epub:xtc:{}</id>\n",
|
||||
xml_escape(&percent_encode(&file.name))
|
||||
));
|
||||
out.push_str(&format!(
|
||||
" <updated>{}</updated>\n",
|
||||
rfc3339(file.modified)
|
||||
));
|
||||
if let Some(date) = file.date {
|
||||
out.push_str(&format!(" <dc:issued>{date}</dc:issued>\n"));
|
||||
}
|
||||
out.push_str(" <author><name>The Daily EPUB</name></author>\n");
|
||||
out.push_str(&format!(
|
||||
" <summary>{}</summary>\n",
|
||||
xml_escape(&summary)
|
||||
));
|
||||
out.push_str(&format!(
|
||||
" <link rel=\"http://opds-spec.org/acquisition\" href=\"{}\" \
|
||||
type=\"application/octet-stream\" length=\"{}\"/>\n",
|
||||
xml_escape(&href),
|
||||
file.bytes
|
||||
));
|
||||
out.push_str(" </entry>\n");
|
||||
}
|
||||
out.push_str("</feed>\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// Atom wants `1996-12-19T16:39:57-08:00`; jiff's `Timestamp` prints `…Z`.
|
||||
fn rfc3339(ts: Timestamp) -> String {
|
||||
ts.to_string()
|
||||
}
|
||||
|
||||
fn human_bytes(bytes: u64) -> String {
|
||||
if bytes >= 1_048_576 {
|
||||
format!("{:.1} MB", bytes as f64 / 1_048_576.0)
|
||||
} else {
|
||||
format!("{:.0} KB", (bytes as f64 / 1024.0).ceil())
|
||||
}
|
||||
}
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
'"' => out.push_str("""),
|
||||
'\'' => out.push_str("'"),
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Percent-encode one URL path segment (filenames contain spaces and parens).
|
||||
fn percent_encode(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for byte in s.as_bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
|
||||
out.push(*byte as char)
|
||||
}
|
||||
_ => out.push_str(&format!("%{byte:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
async fn write_atomic(dest: &Path, bytes: &[u8]) -> Result<(), PublishError> {
|
||||
let dir = dest.parent().unwrap_or_else(|| Path::new("."));
|
||||
let tmp = dir.join(format!(
|
||||
".{}.{}.tmp",
|
||||
dest.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("daily-epub"),
|
||||
std::process::id()
|
||||
));
|
||||
tokio::fs::write(&tmp, bytes)
|
||||
.await
|
||||
.map_err(PublishError::at(&tmp))?;
|
||||
tokio::fs::rename(&tmp, dest)
|
||||
.await
|
||||
.map_err(PublishError::at(dest))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Retention (§3.11)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Delete issue files older than `retention_days` from both publish dirs.
|
||||
/// SQLite history is kept forever — it's the training data (§3.11).
|
||||
///
|
||||
/// Only files named `The Daily EPUB - YYYY-MM-DD*.{epub,xtc,xtch}` are ever
|
||||
/// considered; anything else in those directories (including `xtc.xml` and other
|
||||
/// people's books) is left strictly alone.
|
||||
pub async fn prune(cfg: &Config, today: Date) -> Result<usize, PublishError> {
|
||||
let cutoff = today
|
||||
.checked_sub(jiff::Span::new().days(i64::from(cfg.retention_days)))
|
||||
.unwrap_or(today);
|
||||
let mut removed = 0;
|
||||
for dir in [&cfg.publish.bookorbit_dir, &cfg.publish.xtc_dir] {
|
||||
removed += prune_dir(dir, cutoff).await?;
|
||||
}
|
||||
if removed > 0 {
|
||||
tracing::info!(removed, %cutoff, "retention sweep removed expired issues");
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn prune_dir(dir: &Path, cutoff: Date) -> Result<usize, PublishError> {
|
||||
if !dir.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut entries = tokio::fs::read_dir(dir)
|
||||
.await
|
||||
.map_err(PublishError::at(dir))?;
|
||||
let mut removed = 0;
|
||||
while let Some(entry) = entries.next_entry().await.map_err(PublishError::at(dir))? {
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
let Some(date) = date_from_filename(&name) else {
|
||||
continue;
|
||||
};
|
||||
if date >= cutoff {
|
||||
continue;
|
||||
}
|
||||
if !entry.metadata().await.map(|m| m.is_file()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path();
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => {
|
||||
tracing::info!(path = %path.display(), %date, "pruned expired issue file");
|
||||
removed += 1;
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, path = %path.display(), "could not prune file"),
|
||||
}
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// Today in the configured timezone — the reference point for [`prune`].
|
||||
pub fn today_in_tz(cfg: &Config) -> Date {
|
||||
match cfg.tz() {
|
||||
Ok(tz) => Zoned::now().with_time_zone(tz).date(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "falling back to UTC for the retention cutoff");
|
||||
Timestamp::now().to_zoned(jiff::tz::TimeZone::UTC).date()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn cfg(dir: &Path) -> Config {
|
||||
let mut cfg = Config::default();
|
||||
cfg.publish.bookorbit_dir = dir.join("bookorbit");
|
||||
cfg.publish.xtc_dir = dir.join("xtc");
|
||||
cfg.server.public_url = "https://daily.hallada.net".into();
|
||||
cfg
|
||||
}
|
||||
|
||||
fn date(s: &str) -> Date {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filenames_match_the_spec() {
|
||||
assert_eq!(
|
||||
issue_filename(date("2026-08-15"), Edition::Standard, "epub"),
|
||||
"The Daily EPUB - 2026-08-15.epub"
|
||||
);
|
||||
assert_eq!(
|
||||
issue_filename(date("2026-08-15"), Edition::X4, "epub"),
|
||||
"The Daily EPUB - 2026-08-15 (X4).epub"
|
||||
);
|
||||
assert_eq!(
|
||||
issue_filename(date("2026-08-15"), Edition::X4, "xtch"),
|
||||
"The Daily EPUB - 2026-08-15 (X4).xtch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_our_filenames_are_recognized() {
|
||||
assert_eq!(
|
||||
date_from_filename("The Daily EPUB - 2026-08-15.epub"),
|
||||
Some(date("2026-08-15"))
|
||||
);
|
||||
assert_eq!(
|
||||
date_from_filename("The Daily EPUB - 2026-08-15 (X4).xtch"),
|
||||
Some(date("2026-08-15"))
|
||||
);
|
||||
for foreign in [
|
||||
"xtc.xml",
|
||||
"Moby Dick.epub",
|
||||
"The Daily EPUB - notadate.epub",
|
||||
"The Daily EPUB - 2026-08-15.txt",
|
||||
"the daily epub - 2026-08-15.epub",
|
||||
"metadata.db",
|
||||
] {
|
||||
assert_eq!(date_from_filename(foreign), None, "{foreign}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atomic_copy_creates_the_final_name_and_leaves_no_temp_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let src = dir.path().join("build.epub");
|
||||
tokio::fs::write(&src, b"EPUB BYTES").await.unwrap();
|
||||
let dest = dir
|
||||
.path()
|
||||
.join("out")
|
||||
.join("The Daily EPUB - 2026-08-15.epub");
|
||||
|
||||
atomic_copy(&src, &dest).await.unwrap();
|
||||
assert_eq!(tokio::fs::read(&dest).await.unwrap(), b"EPUB BYTES");
|
||||
|
||||
// Overwriting an existing issue works and stays atomic.
|
||||
tokio::fs::write(&src, b"REGENERATED").await.unwrap();
|
||||
atomic_copy(&src, &dest).await.unwrap();
|
||||
assert_eq!(tokio::fs::read(&dest).await.unwrap(), b"REGENERATED");
|
||||
|
||||
let leftovers: Vec<String> = std::fs::read_dir(dir.path().join("out"))
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
|
||||
.filter(|n| n.ends_with(".tmp"))
|
||||
.collect();
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"temp files left behind: {leftovers:?}"
|
||||
);
|
||||
|
||||
assert!(
|
||||
atomic_copy(Path::new("/nonexistent/x.epub"), &dest)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publish_epubs_uses_canonical_names() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = cfg(dir.path());
|
||||
let std_src = dir.path().join("a.epub");
|
||||
let x4_src = dir.path().join("b.epub");
|
||||
tokio::fs::write(&std_src, b"standard").await.unwrap();
|
||||
tokio::fs::write(&x4_src, b"x4").await.unwrap();
|
||||
|
||||
let artifacts = vec![
|
||||
Artifact {
|
||||
edition: Edition::Standard,
|
||||
path: std_src,
|
||||
bytes: 8,
|
||||
},
|
||||
Artifact {
|
||||
edition: Edition::X4,
|
||||
path: x4_src,
|
||||
bytes: 2,
|
||||
},
|
||||
];
|
||||
let issue = fake_issue(date("2026-08-15"));
|
||||
let paths = publish_epubs(&artifacts, &issue, &cfg).await.unwrap();
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
cfg.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-08-15.epub"),
|
||||
cfg.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-08-15 (X4).epub"),
|
||||
]
|
||||
);
|
||||
assert_eq!(tokio::fs::read(&paths[0]).await.unwrap(), b"standard");
|
||||
|
||||
let xtc_src = dir.path().join("c.xtch");
|
||||
tokio::fs::write(&xtc_src, b"xtch").await.unwrap();
|
||||
let published = publish_xtc(&xtc_src, &cfg).await.unwrap();
|
||||
assert_eq!(published, cfg.publish.xtc_dir.join("c.xtch"));
|
||||
}
|
||||
|
||||
fn fake_issue(date: Date) -> Issue {
|
||||
use crate::types::{Colophon, Editorial, IssueMeta, Lineup};
|
||||
Issue {
|
||||
meta: IssueMeta {
|
||||
date,
|
||||
issue_number: 12,
|
||||
generated_at: ts("2026-08-15T05:30:00Z"),
|
||||
display_date: "Saturday, August 15, 2026".into(),
|
||||
article_count: 3,
|
||||
section_count: 1,
|
||||
total_words: 900,
|
||||
reading_minutes: 5,
|
||||
},
|
||||
lineup: Lineup {
|
||||
date,
|
||||
picks: vec![],
|
||||
section_order: vec![],
|
||||
},
|
||||
editorial: Editorial::default(),
|
||||
world_briefing: None,
|
||||
colophon: Colophon::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opds_feed_is_newest_first_with_acquisition_links() {
|
||||
let files = vec![
|
||||
XtcFile {
|
||||
name: "The Daily EPUB - 2026-08-15 (X4).xtch".into(),
|
||||
date: Some(date("2026-08-15")),
|
||||
modified: ts("2026-08-15T05:40:00Z"),
|
||||
bytes: 2_500_000,
|
||||
},
|
||||
XtcFile {
|
||||
name: "The Daily EPUB - 2026-08-14 (X4).xtch".into(),
|
||||
date: Some(date("2026-08-14")),
|
||||
modified: ts("2026-08-14T05:40:00Z"),
|
||||
bytes: 4096,
|
||||
},
|
||||
];
|
||||
let mut numbers = BTreeMap::new();
|
||||
numbers.insert(date("2026-08-15"), 12);
|
||||
let feed = render_opds(
|
||||
&files,
|
||||
&numbers,
|
||||
"https://daily.hallada.net/",
|
||||
ts("2026-08-15T06:00:00Z"),
|
||||
);
|
||||
|
||||
assert!(feed.starts_with("<?xml version=\"1.0\" encoding=\"utf-8\"?>"));
|
||||
assert!(feed.contains("<feed xmlns=\"http://www.w3.org/2005/Atom\""));
|
||||
assert!(feed.contains("<id>urn:daily-epub:xtc</id>"));
|
||||
assert!(feed.contains("<updated>2026-08-15T05:40:00Z</updated>"));
|
||||
assert_eq!(feed.matches("<entry>").count(), 2);
|
||||
assert_eq!(feed.matches("</entry>").count(), 2);
|
||||
// Newest first.
|
||||
let i15 = feed.find("The Daily EPUB — 2026-08-15").unwrap();
|
||||
let i14 = feed.find("The Daily EPUB — 2026-08-14").unwrap();
|
||||
assert!(i15 < i14);
|
||||
// Acquisition link, encoded filename, absolute public URL, size.
|
||||
assert!(feed.contains(
|
||||
"<link rel=\"http://opds-spec.org/acquisition\" \
|
||||
href=\"https://daily.hallada.net/files/xtc/The%20Daily%20EPUB%20-%202026-08-15%20%28X4%29.xtch\" \
|
||||
type=\"application/octet-stream\" length=\"2500000\"/>"
|
||||
));
|
||||
assert!(feed.contains("Issue #12 · 2.4 MB"));
|
||||
assert!(feed.trim_end().ends_with("</feed>"));
|
||||
assert!(!feed.contains("&<"), "unescaped markup leaked in");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_xtc_opds_lists_only_xtc_files_capped_at_fourteen() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = cfg(dir.path());
|
||||
std::fs::create_dir_all(&cfg.publish.xtc_dir).unwrap();
|
||||
for day in 1..=20 {
|
||||
let name = format!("The Daily EPUB - 2026-08-{day:02} (X4).xtch");
|
||||
std::fs::write(cfg.publish.xtc_dir.join(name), b"x").unwrap();
|
||||
}
|
||||
// Non-XTC neighbours must be ignored.
|
||||
std::fs::write(cfg.publish.xtc_dir.join("README.txt"), b"x").unwrap();
|
||||
std::fs::write(cfg.publish.xtc_dir.join("cover.epub"), b"x").unwrap();
|
||||
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.upsert_issue(
|
||||
date("2026-08-20"),
|
||||
20,
|
||||
ts("2026-08-20T05:30:00Z"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let path = write_xtc_opds(&db, &cfg).await.unwrap();
|
||||
assert_eq!(path, cfg.publish.xtc_dir.join(XTC_OPDS_FILENAME));
|
||||
let feed = std::fs::read_to_string(&path).unwrap();
|
||||
assert_eq!(feed.matches("<entry>").count(), XTC_FEED_ENTRIES);
|
||||
assert!(feed.contains("2026-08-20"));
|
||||
assert!(!feed.contains("2026-08-06"), "older than the last 14");
|
||||
assert!(!feed.contains("README"));
|
||||
assert!(!feed.contains("cover.epub"));
|
||||
assert!(feed.contains("Issue #20"));
|
||||
|
||||
// Regenerating replaces the file in place.
|
||||
write_xtc_opds(&db, &cfg).await.unwrap();
|
||||
let leftovers: Vec<String> = std::fs::read_dir(&cfg.publish.xtc_dir)
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
|
||||
.filter(|n| n.ends_with(".tmp"))
|
||||
.collect();
|
||||
assert!(leftovers.is_empty(), "{leftovers:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prune_only_deletes_old_matching_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut conf = cfg(dir.path());
|
||||
conf.retention_days = 21;
|
||||
std::fs::create_dir_all(&conf.publish.bookorbit_dir).unwrap();
|
||||
std::fs::create_dir_all(&conf.publish.xtc_dir).unwrap();
|
||||
|
||||
let keep_epub = conf
|
||||
.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-08-14.epub");
|
||||
let old_epub = conf
|
||||
.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-07-01.epub");
|
||||
let old_x4 = conf
|
||||
.publish
|
||||
.bookorbit_dir
|
||||
.join("The Daily EPUB - 2026-07-01 (X4).epub");
|
||||
let foreign = conf.publish.bookorbit_dir.join("Moby Dick.epub");
|
||||
let old_xtc = conf
|
||||
.publish
|
||||
.xtc_dir
|
||||
.join("The Daily EPUB - 2026-07-01 (X4).xtch");
|
||||
let feed = conf.publish.xtc_dir.join(XTC_OPDS_FILENAME);
|
||||
for path in [&keep_epub, &old_epub, &old_x4, &foreign, &old_xtc, &feed] {
|
||||
std::fs::write(path, b"x").unwrap();
|
||||
}
|
||||
|
||||
let removed = prune(&conf, date("2026-08-15")).await.unwrap();
|
||||
assert_eq!(removed, 3);
|
||||
assert!(keep_epub.exists());
|
||||
assert!(foreign.exists(), "never touch other people's books");
|
||||
assert!(feed.exists(), "the OPDS feed is not an issue file");
|
||||
assert!(!old_epub.exists());
|
||||
assert!(!old_x4.exists());
|
||||
assert!(!old_xtc.exists());
|
||||
|
||||
// Idempotent, and tolerant of missing directories.
|
||||
assert_eq!(prune(&conf, date("2026-08-15")).await.unwrap(), 0);
|
||||
let missing_dirs = cfg(&dir.path().join("nowhere"));
|
||||
assert_eq!(prune(&missing_dirs, date("2026-08-15")).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publish_issue_does_the_whole_dance() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cfg = cfg(dir.path());
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let src = dir.path().join("std.epub");
|
||||
let x4_src = dir.path().join("x4.epub");
|
||||
let xtc_src = dir.path().join("out.xtch");
|
||||
for (path, body) in [
|
||||
(&src, "standard"),
|
||||
(&x4_src, "x4"),
|
||||
(&xtc_src, "xtch-bytes"),
|
||||
] {
|
||||
std::fs::write(path, body).unwrap();
|
||||
}
|
||||
let artifacts = vec![
|
||||
Artifact {
|
||||
edition: Edition::Standard,
|
||||
path: src,
|
||||
bytes: 8,
|
||||
},
|
||||
Artifact {
|
||||
edition: Edition::X4,
|
||||
path: x4_src,
|
||||
bytes: 2,
|
||||
},
|
||||
];
|
||||
let issue = fake_issue(date("2026-08-15"));
|
||||
|
||||
let published = publish_issue(&db, &cfg, &issue, &artifacts, Some(&xtc_src))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(published.epubs.len(), 2);
|
||||
assert!(published.epubs.iter().all(|a| a.path.exists()));
|
||||
assert_eq!(published.epubs[1].edition, Edition::X4);
|
||||
assert!(published.xtc.as_ref().is_some_and(|p| p.exists()));
|
||||
assert!(published.opds.as_ref().is_some_and(|p| p.exists()));
|
||||
assert_eq!(published.pruned, 0);
|
||||
|
||||
let feed = std::fs::read_to_string(cfg.publish.xtc_dir.join(XTC_OPDS_FILENAME)).unwrap();
|
||||
assert!(feed.contains("out.xtch"));
|
||||
|
||||
// No XTC artifact is fine — the feed is still regenerated.
|
||||
let published = publish_issue(&db, &cfg, &issue, &artifacts, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(published.xtc.is_none());
|
||||
assert!(published.opds.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn helpers_escape_and_encode() {
|
||||
assert_eq!(xml_escape("a & b < c"), "a & b < c");
|
||||
assert_eq!(percent_encode("a b(c).xtch"), "a%20b%28c%29.xtch");
|
||||
assert_eq!(human_bytes(2_500_000), "2.4 MB");
|
||||
assert_eq!(human_bytes(4096), "4 KB");
|
||||
}
|
||||
}
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
//! Run report — counts, token usage, cost, timings, status (spec §3.13 `runs`, §3.12
|
||||
//! `/issues.json`).
|
||||
//!
|
||||
//! `generate` builds one of these, prints it at the end of the run and stores the
|
||||
//! serialized form in `runs` / `issues.report_json`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::types::TokenUsage;
|
||||
|
||||
/// Terminal state of a run, stored in `runs.status`.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RunStatus {
|
||||
#[default]
|
||||
Running,
|
||||
/// Everything completed.
|
||||
Ok,
|
||||
/// The issue was produced but a best-effort stage failed (social, XTC,
|
||||
/// world briefing, images) or the cost guardrail tripped (§3.6).
|
||||
Degraded,
|
||||
/// No issue was produced.
|
||||
Failed,
|
||||
/// `--dry-run`: nothing was published.
|
||||
DryRun,
|
||||
}
|
||||
|
||||
impl RunStatus {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
RunStatus::Running => "running",
|
||||
RunStatus::Ok => "ok",
|
||||
RunStatus::Degraded => "degraded",
|
||||
RunStatus::Failed => "failed",
|
||||
RunStatus::DryRun => "dry_run",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for RunStatus {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-stage article counts as the pipeline narrows the day's feed volume (§2).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct StageCounts {
|
||||
/// Entries returned by Miniflux inside the lookback window (§3.1).
|
||||
pub entries_fetched: i64,
|
||||
/// Distinct feeds those entries came from.
|
||||
pub feeds_seen: i64,
|
||||
/// Entries dropped as non-articles (video/audio/empty title) (§3.2).
|
||||
pub entries_dropped: i64,
|
||||
/// Deduped article clusters (§3.2).
|
||||
pub articles: i64,
|
||||
/// Clusters that merged ≥ 2 entries.
|
||||
pub duplicates_merged: i64,
|
||||
/// Articles whose full text was fetched + extracted (§3.3).
|
||||
pub extracted: i64,
|
||||
/// Articles left with only an excerpt (§3.3).
|
||||
pub excerpt_only: i64,
|
||||
/// Social lookups that returned a hit (§3.4).
|
||||
pub social_hits: i64,
|
||||
/// Articles surviving the heuristic pre-filter (§3.5).
|
||||
pub candidates: i64,
|
||||
/// Articles scored by the LLM (§3.6 stage A).
|
||||
pub llm_scored: i64,
|
||||
/// Articles in the final lineup (§3.6 stage B).
|
||||
pub selected: i64,
|
||||
/// Discussion chapters rendered (§3.7).
|
||||
pub discussions: i64,
|
||||
/// Images embedded across both editions (§3.10).
|
||||
pub images_embedded: i64,
|
||||
}
|
||||
|
||||
/// Wall-clock milliseconds per pipeline stage.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct StageTimings(pub BTreeMap<String, i64>);
|
||||
|
||||
impl StageTimings {
|
||||
pub fn record(&mut self, stage: &str, millis: i64) {
|
||||
*self.0.entry(stage.to_string()).or_insert(0) += millis;
|
||||
}
|
||||
|
||||
pub fn total_ms(&self) -> i64 {
|
||||
self.0.values().sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// The full summary of one `generate` invocation (§3.13).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RunReport {
|
||||
pub date: Date,
|
||||
pub started_at: Timestamp,
|
||||
pub finished_at: Option<Timestamp>,
|
||||
pub status: RunStatus,
|
||||
pub counts: StageCounts,
|
||||
pub usage: TokenUsage,
|
||||
pub cost_usd: f64,
|
||||
pub timings: StageTimings,
|
||||
/// Ingest window actually used, RFC3339 (§3.1).
|
||||
pub window_start: Option<Timestamp>,
|
||||
pub window_end: Option<Timestamp>,
|
||||
/// Entry counts per feed title, for spotting noisy feeds (M1 verification).
|
||||
pub per_feed_counts: BTreeMap<String, i64>,
|
||||
/// Non-fatal problems from best-effort stages (notes §3).
|
||||
pub warnings: Vec<String>,
|
||||
/// Fatal error message when `status == Failed`.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl RunReport {
|
||||
pub fn new(date: Date, started_at: Timestamp) -> Self {
|
||||
Self {
|
||||
date,
|
||||
started_at,
|
||||
finished_at: None,
|
||||
status: RunStatus::Running,
|
||||
counts: StageCounts::default(),
|
||||
usage: TokenUsage::default(),
|
||||
cost_usd: 0.0,
|
||||
timings: StageTimings::default(),
|
||||
window_start: None,
|
||||
window_end: None,
|
||||
per_feed_counts: BTreeMap::new(),
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn warn(&mut self, msg: impl Into<String>) {
|
||||
let msg = msg.into();
|
||||
tracing::warn!(target: "daily_epub::report", "{msg}");
|
||||
self.warnings.push(msg);
|
||||
}
|
||||
|
||||
pub fn fail(&mut self, finished_at: Timestamp, err: impl fmt::Display) {
|
||||
self.finished_at = Some(finished_at);
|
||||
self.status = RunStatus::Failed;
|
||||
self.error = Some(err.to_string());
|
||||
}
|
||||
|
||||
/// Stamp the end time, compute cost from [`TokenUsage`] and settle the status.
|
||||
pub fn finish(
|
||||
&mut self,
|
||||
finished_at: Timestamp,
|
||||
price_input: f64,
|
||||
price_cached: f64,
|
||||
price_output: f64,
|
||||
) {
|
||||
self.finished_at = Some(finished_at);
|
||||
self.cost_usd = self.usage.cost_usd(price_input, price_cached, price_output);
|
||||
if self.status == RunStatus::Running {
|
||||
self.status = if self.warnings.is_empty() {
|
||||
RunStatus::Ok
|
||||
} else {
|
||||
RunStatus::Degraded
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Total wall-clock duration in seconds, when finished.
|
||||
pub fn duration_secs(&self) -> Option<i64> {
|
||||
self.finished_at
|
||||
.map(|end| (end.as_second() - self.started_at.as_second()).max(0))
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| "{}".into())
|
||||
}
|
||||
|
||||
pub fn to_json_pretty(&self) -> String {
|
||||
serde_json::to_string_pretty(self).unwrap_or_else(|_| "{}".into())
|
||||
}
|
||||
|
||||
/// Compact human-readable summary printed at the end of `generate`.
|
||||
pub fn summary_line(&self) -> String {
|
||||
format!(
|
||||
"{} [{}] {} entries → {} articles → {} candidates → {} selected · ${:.4} · {}s",
|
||||
self.date,
|
||||
self.status,
|
||||
self.counts.entries_fetched,
|
||||
self.counts.articles,
|
||||
self.counts.candidates,
|
||||
self.counts.selected,
|
||||
self.cost_usd,
|
||||
self.duration_secs().unwrap_or(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// Feeds ordered by entry count, descending — the M1 dry-run breakdown.
|
||||
pub fn top_feeds(&self, limit: usize) -> Vec<(&str, i64)> {
|
||||
let mut v: Vec<(&str, i64)> = self
|
||||
.per_feed_counts
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str(), *v))
|
||||
.collect();
|
||||
v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
|
||||
v.truncate(limit);
|
||||
v
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_computes_cost_and_status() {
|
||||
let mut r = RunReport::new("2026-08-15".parse().unwrap(), ts("2026-08-15T05:30:00Z"));
|
||||
r.usage.add(TokenUsage {
|
||||
input_tokens: 1_000_000,
|
||||
cached_tokens: 1_000_000,
|
||||
output_tokens: 1_000_000,
|
||||
});
|
||||
r.finish(ts("2026-08-15T05:36:00Z"), 0.14, 0.0028, 0.28);
|
||||
assert_eq!(r.status, RunStatus::Ok);
|
||||
assert!((r.cost_usd - 0.4228).abs() < 1e-9);
|
||||
assert_eq!(r.duration_secs(), Some(360));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warnings_degrade_the_run() {
|
||||
let mut r = RunReport::new("2026-08-15".parse().unwrap(), ts("2026-08-15T05:30:00Z"));
|
||||
r.warn("xtc converter missing");
|
||||
r.finish(ts("2026-08-15T05:31:00Z"), 0.14, 0.0028, 0.28);
|
||||
assert_eq!(r.status, RunStatus::Degraded);
|
||||
assert_eq!(r.warnings.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serializes_round_trip() {
|
||||
let mut r = RunReport::new("2026-08-15".parse().unwrap(), ts("2026-08-15T05:30:00Z"));
|
||||
r.counts.entries_fetched = 412;
|
||||
r.per_feed_counts.insert("Hacker News".into(), 30);
|
||||
r.per_feed_counts.insert("Lobsters".into(), 12);
|
||||
r.timings.record("ingest", 1500);
|
||||
r.finish(ts("2026-08-15T05:31:00Z"), 0.14, 0.0028, 0.28);
|
||||
let json = r.to_json();
|
||||
let back: RunReport = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, r);
|
||||
assert_eq!(back.top_feeds(1), vec![("Hacker News", 30)]);
|
||||
assert_eq!(back.timings.total_ms(), 1500);
|
||||
assert!(back.summary_line().contains("412 entries"));
|
||||
}
|
||||
}
|
||||
+973
@@ -0,0 +1,973 @@
|
||||
//! axum server: rating endpoints, XTC OPDS, static files (spec §3.9, §3.12).
|
||||
//!
|
||||
//! Rating links must work from an e-reader's built-in browser, so every rating
|
||||
//! endpoint is a `GET` and the response is a tiny e-ink-sized HTML page.
|
||||
//!
|
||||
//! Routes (§3.12):
|
||||
//! | route | behaviour |
|
||||
//! |---|---|
|
||||
//! | `GET /r/{date}/{article_id}/{vote}?t=` | verify HMAC, upsert rating, rebuild feed priors |
|
||||
//! | `GET /opds/xtc.xml` | static OPDS 1.2 acquisition feed from `publish.xtc_dir` |
|
||||
//! | `GET /files/xtc/{name}` | XTC artifact download (no path traversal) |
|
||||
//! | `GET /healthz` | liveness |
|
||||
//! | `GET /issues.json` | the last 30 run reports, newest first |
|
||||
//!
|
||||
//! `/opds/*` and `/files/*` sit behind optional Basic auth (`server.basic_auth_*`).
|
||||
//!
|
||||
//! The EPUB article footer (§3.10) mints its 👍/👎 links with the very same
|
||||
//! [`rating_url`] this module verifies with — both re-export [`crate::auth`],
|
||||
//! which pins the shared test vector (`secret = "test-secret"`, `2026-08-15`,
|
||||
//! article `42`, `up` → `3b314cf7e6d8f50f`). An issue generated while
|
||||
//! `server.hmac_secret` is unset carries links this server rejects with 403.
|
||||
|
||||
use std::path::{Path as FsPath, PathBuf};
|
||||
|
||||
use axum::Router;
|
||||
use axum::body::Body;
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::get;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
use serde::Deserialize;
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::Db;
|
||||
use crate::types::{ArticleId, Rating, Vote};
|
||||
|
||||
/// Characters of the hex HMAC kept in rating links (§3.9).
|
||||
pub const TOKEN_LEN: usize = crate::auth::TOKEN_LEN;
|
||||
/// How many issues `GET /issues.json` returns (§3.12).
|
||||
pub const ISSUES_JSON_LIMIT: i64 = 30;
|
||||
/// Basic auth realm advertised for the OPDS routes (§3.11).
|
||||
pub const AUTH_REALM: &str = "The Daily EPUB";
|
||||
/// Content type of an OPDS 1.2 acquisition feed (§3.11).
|
||||
pub const OPDS_CONTENT_TYPE: &str = "application/atom+xml;profile=opds-catalog;kind=acquisition";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ServerError {
|
||||
#[error("server.hmac_secret is not configured (set DAILY_EPUB_SERVER__HMAC_SECRET)")]
|
||||
MissingSecret,
|
||||
#[error("could not bind {addr}: {source}")]
|
||||
Bind {
|
||||
addr: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("io error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
/// Shared axum state.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppState {
|
||||
pub db: Db,
|
||||
pub config: Config,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rating tokens (§3.9)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// The formula lives in [`crate::auth`] so the EPUB writer and this verifier can
|
||||
// never drift apart; these re-exports keep the historical call sites intact.
|
||||
pub use crate::auth::{constant_time_eq, rating_token, rating_url, verify_token};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Router (§3.12)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Build the router: `/r/{date}/{article_id}/{vote}`, `/opds/xtc.xml`,
|
||||
/// `/files/xtc/{name}`, `/healthz`, `/issues.json`, with `tower-http` tracing (§3.12).
|
||||
pub fn router(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.route("/r/{date}/{article_id}/{vote}", get(handle_rating))
|
||||
.route("/opds/xtc.xml", get(handle_opds))
|
||||
// OPDS browsers are typed into by hand on a 6" e-ink keyboard: serve the
|
||||
// same feed from the catalog root so a URL without the filename works.
|
||||
.route("/opds", get(handle_opds))
|
||||
.route("/opds/", get(handle_opds))
|
||||
.route("/files/xtc/{name}", get(handle_xtc_file))
|
||||
.route("/healthz", get(handle_healthz))
|
||||
.route("/issues.json", get(handle_issues_json))
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// `daily-epub serve` — bind, serve, graceful shutdown on SIGTERM (§3.12).
|
||||
pub async fn serve(config: Config, db: Db) -> Result<(), ServerError> {
|
||||
if config.server.hmac_secret.is_none() {
|
||||
// Not fatal for the OPDS routes, but every rating link would 500.
|
||||
tracing::warn!("server.hmac_secret is unset — rating links will be rejected");
|
||||
}
|
||||
let addr = config.server.bind.clone();
|
||||
let listener = tokio::net::TcpListener::bind(&addr)
|
||||
.await
|
||||
.map_err(|source| ServerError::Bind {
|
||||
addr: addr.clone(),
|
||||
source,
|
||||
})?;
|
||||
let local = listener.local_addr().map(|a| a.to_string()).unwrap_or(addr);
|
||||
tracing::info!(bind = %local, public_url = %config.server.public_url, "serving");
|
||||
|
||||
let app = router(AppState { db, config });
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
tracing::info!("server stopped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve on SIGTERM (systemd stop) or ctrl-c (§3.12, §3.15).
|
||||
async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
if let Err(e) = tokio::signal::ctrl_c().await {
|
||||
tracing::error!(error = %e, "failed to install the ctrl-c handler");
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
};
|
||||
let terminate = async {
|
||||
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
|
||||
Ok(mut sig) => {
|
||||
sig.recv().await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "failed to install the SIGTERM handler");
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
_ = ctrl_c => tracing::info!("ctrl-c received, shutting down"),
|
||||
_ = terminate => tracing::info!("SIGTERM received, shutting down"),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TokenQuery {
|
||||
#[serde(default)]
|
||||
t: String,
|
||||
}
|
||||
|
||||
async fn handle_healthz() -> Response {
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
|
||||
"ok",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `GET /issues.json` — the last [`ISSUES_JSON_LIMIT`] run reports, newest first (§3.12).
|
||||
async fn handle_issues_json(State(state): State<AppState>) -> Response {
|
||||
let rows = match state.db.recent_reports(ISSUES_JSON_LIMIT).await {
|
||||
Ok(rows) => rows,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "issues.json query failed");
|
||||
return (StatusCode::INTERNAL_SERVER_ERROR, "database error").into_response();
|
||||
}
|
||||
};
|
||||
let issues: Vec<serde_json::Value> = rows
|
||||
.into_iter()
|
||||
.map(|(date, report)| {
|
||||
let report = report
|
||||
.as_deref()
|
||||
.and_then(|r| serde_json::from_str::<serde_json::Value>(r).ok())
|
||||
.unwrap_or(serde_json::Value::Null);
|
||||
serde_json::json!({ "date": date.to_string(), "report": report })
|
||||
})
|
||||
.collect();
|
||||
match serde_json::to_string_pretty(&issues) {
|
||||
Ok(body) => (
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/json")],
|
||||
body,
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "serializing issues.json failed");
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "serialization error").into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /r/{date}/{article_id}/{vote}?t=TOKEN` (§3.9).
|
||||
async fn handle_rating(
|
||||
State(state): State<AppState>,
|
||||
Path((date, article_id, vote)): Path<(String, String, String)>,
|
||||
Query(query): Query<TokenQuery>,
|
||||
) -> Response {
|
||||
let Ok(date) = date.parse::<Date>() else {
|
||||
tracing::warn!(%date, "rating link with a malformed date");
|
||||
return page(StatusCode::BAD_REQUEST, "Bad link — invalid date.", None);
|
||||
};
|
||||
let Ok(article_id) = article_id.parse::<ArticleId>() else {
|
||||
return page(StatusCode::BAD_REQUEST, "Bad link — invalid article.", None);
|
||||
};
|
||||
let Some(vote) = Vote::parse(&vote) else {
|
||||
return page(StatusCode::BAD_REQUEST, "Bad link — invalid vote.", None);
|
||||
};
|
||||
|
||||
let Some(secret) = state.config.server.hmac_secret.as_deref() else {
|
||||
tracing::error!("rating request but server.hmac_secret is unset");
|
||||
return page(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"Server misconfigured.",
|
||||
None,
|
||||
);
|
||||
};
|
||||
if !verify_token(secret, date, article_id, vote, &query.t) {
|
||||
tracing::warn!(%date, article_id, vote = vote.as_str(), "rejected rating token");
|
||||
return page(StatusCode::FORBIDDEN, "Invalid link.", None);
|
||||
}
|
||||
|
||||
let article = match state.db.get_article(article_id).await {
|
||||
Ok(Some(a)) => a,
|
||||
Ok(None) => {
|
||||
tracing::warn!(article_id, "rating for an unknown article");
|
||||
return page(StatusCode::NOT_FOUND, "Unknown article.", None);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, article_id, "loading the rated article failed");
|
||||
return page(StatusCode::INTERNAL_SERVER_ERROR, "Database error.", None);
|
||||
}
|
||||
};
|
||||
|
||||
let rating = Rating {
|
||||
issue_date: date,
|
||||
article_id,
|
||||
vote,
|
||||
rated_at: Timestamp::now(),
|
||||
};
|
||||
let changed = match state.db.upsert_rating(&rating).await {
|
||||
Ok(changed) => changed,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, article_id, "recording the rating failed");
|
||||
return page(StatusCode::INTERNAL_SERVER_ERROR, "Database error.", None);
|
||||
}
|
||||
};
|
||||
if changed && let Err(e) = crate::curate::profile::rebuild_feed_priors(&state.db).await {
|
||||
// The vote is stored; a stale prior only affects the next run's ranking.
|
||||
tracing::error!(error = %e, "refreshing feed priors failed");
|
||||
}
|
||||
tracing::info!(
|
||||
%date,
|
||||
article_id,
|
||||
feed_id = article.feed_id,
|
||||
vote = vote.as_str(),
|
||||
changed,
|
||||
title = %article.title,
|
||||
"recorded rating"
|
||||
);
|
||||
|
||||
let glyph = match vote {
|
||||
Vote::Up => "👍",
|
||||
Vote::Down => "👎",
|
||||
};
|
||||
let message = if changed {
|
||||
format!("Recorded {glyph} — thanks!")
|
||||
} else {
|
||||
format!("Already recorded {glyph} — thanks!")
|
||||
};
|
||||
page(
|
||||
StatusCode::OK,
|
||||
&message,
|
||||
Some(&format!("{date} · article {article_id}")),
|
||||
)
|
||||
}
|
||||
|
||||
/// `GET /opds/xtc.xml` — the static feed written by [`crate::publish`] (§3.11).
|
||||
async fn handle_opds(State(state): State<AppState>, headers: HeaderMap) -> Response {
|
||||
if let Some(challenge) = check_basic_auth(&state.config, &headers) {
|
||||
return challenge;
|
||||
}
|
||||
let path = state
|
||||
.config
|
||||
.publish
|
||||
.xtc_dir
|
||||
.join(crate::publish::XTC_OPDS_FILENAME);
|
||||
match tokio::fs::read(&path).await {
|
||||
Ok(bytes) => (
|
||||
StatusCode::OK,
|
||||
[
|
||||
(header::CONTENT_TYPE, OPDS_CONTENT_TYPE),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
bytes,
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, path = %path.display(), "no XTC OPDS feed yet");
|
||||
(StatusCode::NOT_FOUND, "no feed yet").into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /files/xtc/{name}` — download one XTC artifact (§3.11).
|
||||
async fn handle_xtc_file(
|
||||
State(state): State<AppState>,
|
||||
Path(name): Path<String>,
|
||||
headers: HeaderMap,
|
||||
) -> Response {
|
||||
if let Some(challenge) = check_basic_auth(&state.config, &headers) {
|
||||
return challenge;
|
||||
}
|
||||
let Some(path) = safe_join(&state.config.publish.xtc_dir, &name) else {
|
||||
tracing::warn!(name, "rejected an unsafe XTC file name");
|
||||
return (StatusCode::BAD_REQUEST, "bad file name").into_response();
|
||||
};
|
||||
// An XTCH issue is a pre-rendered page bitmap per page — ~100 MB for a full
|
||||
// day. Stream it rather than buffering the whole file per request (§3.11).
|
||||
let (file, len) = match tokio::fs::File::open(&path).await {
|
||||
Ok(file) => {
|
||||
let len = file.metadata().await.map(|m| m.len()).ok();
|
||||
(file, len)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, path = %path.display(), "XTC file not found");
|
||||
return (StatusCode::NOT_FOUND, "not found").into_response();
|
||||
}
|
||||
};
|
||||
let content_type = if name.ends_with(".xml") {
|
||||
OPDS_CONTENT_TYPE
|
||||
} else {
|
||||
"application/octet-stream"
|
||||
};
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type));
|
||||
if let Ok(value) = HeaderValue::from_str(&format!(
|
||||
"attachment; filename=\"{}\"",
|
||||
name.replace('"', "")
|
||||
)) {
|
||||
headers.insert(header::CONTENT_DISPOSITION, value);
|
||||
}
|
||||
// CrossPoint shows a progress bar only when it knows the size up front.
|
||||
if let Some(len) = len
|
||||
&& let Ok(value) = HeaderValue::from_str(&len.to_string())
|
||||
{
|
||||
headers.insert(header::CONTENT_LENGTH, value);
|
||||
}
|
||||
let body = Body::from_stream(tokio_util::io::ReaderStream::new(file));
|
||||
(StatusCode::OK, headers, body).into_response()
|
||||
}
|
||||
|
||||
/// Resolve `name` inside `dir`, rejecting anything that could escape it (§3.12).
|
||||
///
|
||||
/// The name must be a single, plain file name: no separators, no `..`, no
|
||||
/// absolute paths, no hidden files, and — belt and braces — the joined path must
|
||||
/// still live inside `dir` once resolved.
|
||||
pub fn safe_join(dir: &FsPath, name: &str) -> Option<PathBuf> {
|
||||
if name.is_empty()
|
||||
|| name.len() > 255
|
||||
|| name.starts_with('.')
|
||||
|| name.contains('/')
|
||||
|| name.contains('\\')
|
||||
|| name.contains('\0')
|
||||
|| name.contains("..")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut components = FsPath::new(name).components();
|
||||
let only = match (components.next(), components.next()) {
|
||||
(Some(std::path::Component::Normal(c)), None) => c.to_owned(),
|
||||
_ => return None,
|
||||
};
|
||||
let candidate = dir.join(only);
|
||||
// When both sides resolve, require containment (defends against symlinked names).
|
||||
match (candidate.canonicalize(), dir.canonicalize()) {
|
||||
(Ok(resolved), Ok(root)) if !resolved.starts_with(&root) => None,
|
||||
_ => Some(candidate),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Basic auth (§3.11 optional OPDS credentials)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// `Some(challenge_response)` when the request must be rejected, `None` when it
|
||||
/// may proceed (including when no credentials are configured).
|
||||
fn check_basic_auth(config: &Config, headers: &HeaderMap) -> Option<Response> {
|
||||
let (Some(user), Some(pass)) = (
|
||||
config.server.basic_auth_user.as_deref(),
|
||||
config.server.basic_auth_pass.as_deref(),
|
||||
) else {
|
||||
return None;
|
||||
};
|
||||
let expected = BASE64.encode(format!("{user}:{pass}"));
|
||||
let supplied = headers
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Basic "))
|
||||
.map(str::trim)
|
||||
.unwrap_or_default();
|
||||
if constant_time_eq(expected.as_bytes(), supplied.as_bytes()) {
|
||||
return None;
|
||||
}
|
||||
tracing::warn!("rejected an unauthenticated OPDS request");
|
||||
let challenge =
|
||||
HeaderValue::from_str(&format!("Basic realm=\"{AUTH_REALM}\", charset=\"UTF-8\""))
|
||||
.unwrap_or(HeaderValue::from_static("Basic"));
|
||||
Some(
|
||||
(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
[(header::WWW_AUTHENTICATE, challenge)],
|
||||
"authentication required",
|
||||
)
|
||||
.into_response(),
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tiny e-ink pages (§3.9)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A self-contained response page — no external CSS, well under 1 KB, legible on
|
||||
/// a 6" e-ink browser (§3.9).
|
||||
fn page(status: StatusCode, message: &str, note: Option<&str>) -> Response {
|
||||
let body = page_html(message, note);
|
||||
(
|
||||
status,
|
||||
[
|
||||
(header::CONTENT_TYPE, "text/html; charset=utf-8"),
|
||||
(header::CACHE_CONTROL, "no-store"),
|
||||
],
|
||||
body,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// The page markup itself: no stylesheet, no script, no images (§3.9).
|
||||
fn page_html(message: &str, note: Option<&str>) -> String {
|
||||
let note = note
|
||||
.map(|n| format!("<p><small>{}</small></p>", escape(n)))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"<!doctype html><html lang=\"en\"><meta charset=\"utf-8\">\
|
||||
<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\
|
||||
<title>The Daily EPUB</title>\
|
||||
<style>body{{margin:3em auto;max-width:16em;padding:0 1em;text-align:center;\
|
||||
font:1.3em/1.5 Georgia,serif}}small{{font-size:.65em}}</style>\
|
||||
<p>{}</p>{}",
|
||||
escape(message),
|
||||
note
|
||||
)
|
||||
}
|
||||
|
||||
fn escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{Article, ExtractMethod, SourceKind, SourceRef};
|
||||
|
||||
fn date() -> Date {
|
||||
"2026-08-15".parse().unwrap()
|
||||
}
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
/// The shared fixture vector: the EPUB footer builder must produce the same
|
||||
/// token for these inputs (§3.9).
|
||||
const VECTOR_SECRET: &str = "test-secret";
|
||||
const VECTOR_TOKEN_UP: &str = "3b314cf7e6d8f50f";
|
||||
|
||||
#[test]
|
||||
fn token_matches_the_shared_test_vector() {
|
||||
assert_eq!(
|
||||
rating_token(VECTOR_SECRET, date(), 42, Vote::Up),
|
||||
VECTOR_TOKEN_UP
|
||||
);
|
||||
assert_eq!(rating_token(VECTOR_SECRET, date(), 42, Vote::Up).len(), 16);
|
||||
// Down differs from up, and both verify.
|
||||
let down = rating_token(VECTOR_SECRET, date(), 42, Vote::Down);
|
||||
assert_ne!(down, VECTOR_TOKEN_UP);
|
||||
assert!(verify_token(
|
||||
VECTOR_SECRET,
|
||||
date(),
|
||||
42,
|
||||
Vote::Up,
|
||||
VECTOR_TOKEN_UP
|
||||
));
|
||||
assert!(verify_token(VECTOR_SECRET, date(), 42, Vote::Down, &down));
|
||||
}
|
||||
|
||||
/// The links the EPUB footer embeds must verify here — this is the whole
|
||||
/// feedback loop in one assertion (§3.9).
|
||||
#[test]
|
||||
fn epub_footer_links_verify_against_this_server() {
|
||||
for (id, vote) in [(42, Vote::Up), (1234, Vote::Down)] {
|
||||
let from_epub = crate::epub::build::rating_url(
|
||||
"https://daily.hallada.net",
|
||||
VECTOR_SECRET,
|
||||
date(),
|
||||
id,
|
||||
vote,
|
||||
);
|
||||
assert_eq!(
|
||||
from_epub,
|
||||
rating_url("https://daily.hallada.net", VECTOR_SECRET, date(), id, vote)
|
||||
);
|
||||
let token = from_epub.rsplit("?t=").next().unwrap_or_default();
|
||||
assert!(
|
||||
verify_token(VECTOR_SECRET, date(), id, vote, token),
|
||||
"{from_epub}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_verification_rejects_tampering() {
|
||||
let t = rating_token(VECTOR_SECRET, date(), 42, Vote::Up);
|
||||
assert!(!verify_token(VECTOR_SECRET, date(), 42, Vote::Down, &t));
|
||||
assert!(!verify_token(VECTOR_SECRET, date(), 43, Vote::Up, &t));
|
||||
assert!(!verify_token("other-secret", date(), 42, Vote::Up, &t));
|
||||
assert!(!verify_token(
|
||||
VECTOR_SECRET,
|
||||
"2026-08-16".parse().unwrap(),
|
||||
42,
|
||||
Vote::Up,
|
||||
&t
|
||||
));
|
||||
assert!(!verify_token(VECTOR_SECRET, date(), 42, Vote::Up, ""));
|
||||
assert!(!verify_token(
|
||||
VECTOR_SECRET,
|
||||
date(),
|
||||
42,
|
||||
Vote::Up,
|
||||
&format!("{t}00")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rating_url_is_the_link_the_epub_embeds() {
|
||||
assert_eq!(
|
||||
rating_url(
|
||||
"https://daily.hallada.net/",
|
||||
VECTOR_SECRET,
|
||||
date(),
|
||||
42,
|
||||
Vote::Up
|
||||
),
|
||||
format!("https://daily.hallada.net/r/2026-08-15/42/up?t={VECTOR_TOKEN_UP}")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_time_eq_behaves_like_eq() {
|
||||
assert!(constant_time_eq(b"abc", b"abc"));
|
||||
assert!(!constant_time_eq(b"abc", b"abd"));
|
||||
assert!(!constant_time_eq(b"abc", b"abcd"));
|
||||
assert!(!constant_time_eq(b"", b"a"));
|
||||
assert!(constant_time_eq(b"", b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn safe_join_rejects_traversal() {
|
||||
let dir = FsPath::new("/var/lib/daily-epub/xtc");
|
||||
assert_eq!(
|
||||
safe_join(dir, "The Daily EPUB - 2026-08-15 (X4).xtch"),
|
||||
Some(dir.join("The Daily EPUB - 2026-08-15 (X4).xtch"))
|
||||
);
|
||||
for bad in [
|
||||
"",
|
||||
"..",
|
||||
"../secret",
|
||||
"..%2Fsecret",
|
||||
"a/../../secret",
|
||||
"sub/dir.xtch",
|
||||
"/etc/passwd",
|
||||
".hidden",
|
||||
"back\\slash",
|
||||
] {
|
||||
assert!(safe_join(dir, bad).is_none(), "should reject {bad:?}");
|
||||
}
|
||||
// Percent-decoding happens before us: a decoded traversal is rejected too.
|
||||
assert!(safe_join(dir, "../../etc/passwd").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_confirmation_page_is_tiny_and_self_contained() {
|
||||
let html = page_html("Recorded 👍 — thanks!", Some("2026-08-15 · article 42"));
|
||||
assert!(html.len() < 1024, "page is {} bytes", html.len());
|
||||
assert!(!html.contains("<link"), "no external stylesheet");
|
||||
assert!(!html.contains("<script"), "no script");
|
||||
assert!(html.contains("Recorded 👍"));
|
||||
assert_eq!(
|
||||
page(StatusCode::FORBIDDEN, "Invalid link.", None).status(),
|
||||
StatusCode::FORBIDDEN
|
||||
);
|
||||
assert!(page_html("<b>x</b>", None).contains("<b>"));
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// End-to-end over a real listener (the crate has no lib target, so the
|
||||
// HTTP-level tests live here rather than in `tests/`).
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
struct TestServer {
|
||||
base: String,
|
||||
db: Db,
|
||||
_dir: tempfile::TempDir,
|
||||
handle: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl TestServer {
|
||||
async fn start(with_auth: bool) -> TestServer {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let xtc_dir = dir.path().join("xtc");
|
||||
std::fs::create_dir_all(&xtc_dir).unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut config = Config::default();
|
||||
config.server.hmac_secret = Some(VECTOR_SECRET.into());
|
||||
config.publish.xtc_dir = xtc_dir;
|
||||
if with_auth {
|
||||
config.server.basic_auth_user = Some("opds".into());
|
||||
config.server.basic_auth_pass = Some("hunter2".into());
|
||||
}
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let app = router(AppState {
|
||||
db: db.clone(),
|
||||
config,
|
||||
});
|
||||
let handle = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
TestServer {
|
||||
base: format!("http://{addr}"),
|
||||
db,
|
||||
_dir: dir,
|
||||
handle,
|
||||
}
|
||||
}
|
||||
|
||||
fn xtc_dir(&self) -> PathBuf {
|
||||
self._dir.path().join("xtc")
|
||||
}
|
||||
|
||||
async fn seed_article(&self) -> ArticleId {
|
||||
let entry = crate::types::Entry {
|
||||
id: 1,
|
||||
feed_id: 7,
|
||||
feed_title: Some("Hacker News".into()),
|
||||
category: None,
|
||||
title: "Story".into(),
|
||||
url: "https://example.com/1".into(),
|
||||
canonical_url: Some("https://example.com/1".into()),
|
||||
author: None,
|
||||
published_at: Some(ts("2026-08-15T04:00:00Z")),
|
||||
comments_url: None,
|
||||
raw_content: "<p>hi</p>".into(),
|
||||
fetched_at: ts("2026-08-15T05:30:00Z"),
|
||||
};
|
||||
self.db.upsert_entry(&entry).await.unwrap();
|
||||
let article = Article {
|
||||
id: 0,
|
||||
canonical_url: "https://example.com/1".into(),
|
||||
title: "Story".into(),
|
||||
best_entry_id: 1,
|
||||
content_html: "<p>hi</p>".into(),
|
||||
word_count: 500,
|
||||
excerpt_only: false,
|
||||
image_count: 0,
|
||||
sources: vec![SourceRef {
|
||||
entry_id: 1,
|
||||
feed_id: 7,
|
||||
feed_title: "Hacker News".into(),
|
||||
category: None,
|
||||
kind: SourceKind::HnFrontpage,
|
||||
}],
|
||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||
url: "https://example.com/1".into(),
|
||||
author: None,
|
||||
feed_id: 7,
|
||||
feed_title: "Hacker News".into(),
|
||||
category: None,
|
||||
published_at: None,
|
||||
comments_url: None,
|
||||
image_urls: vec![],
|
||||
social: vec![],
|
||||
extract_method: ExtractMethod::Miniflux,
|
||||
};
|
||||
self.db.upsert_article(&article).await.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestServer {
|
||||
fn drop(&mut self) {
|
||||
self.handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
fn client() -> reqwest::Client {
|
||||
reqwest::Client::builder().build().unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthz_and_issues_json() {
|
||||
let server = TestServer::start(false).await;
|
||||
let res = client()
|
||||
.get(format!("{}/healthz", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
assert_eq!(res.text().await.unwrap(), "ok");
|
||||
|
||||
let res = client()
|
||||
.get(format!("{}/issues.json", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
let body: serde_json::Value = res.json().await.unwrap();
|
||||
assert_eq!(body.as_array().map(Vec::len), Some(0));
|
||||
|
||||
// Newest first, report JSON inlined.
|
||||
for (day, n) in [("2026-08-13", 1), ("2026-08-15", 3), ("2026-08-14", 2)] {
|
||||
server
|
||||
.db
|
||||
.upsert_issue(
|
||||
day.parse().unwrap(),
|
||||
n,
|
||||
ts("2026-08-15T05:30:00Z"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&format!("{{\"selected\":{n}}}")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let body: serde_json::Value = client()
|
||||
.get(format!("{}/issues.json", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let dates: Vec<&str> = body
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v["date"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(dates, ["2026-08-15", "2026-08-14", "2026-08-13"]);
|
||||
assert_eq!(body[0]["report"]["selected"], 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rating_happy_path_is_idempotent_and_updates_priors() {
|
||||
let server = TestServer::start(false).await;
|
||||
let id = server.seed_article().await;
|
||||
let url = rating_url(&server.base, VECTOR_SECRET, date(), id, Vote::Up);
|
||||
|
||||
let res = client().get(&url).send().await.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
let body = res.text().await.unwrap();
|
||||
assert!(body.contains("Recorded"), "{body}");
|
||||
assert!(!body.contains("Already"), "{body}");
|
||||
assert!(
|
||||
body.len() < 1024,
|
||||
"confirmation page is {} bytes",
|
||||
body.len()
|
||||
);
|
||||
|
||||
// Same tap again: still 200, but reported as already recorded.
|
||||
let body = client()
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.text()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(body.contains("Already recorded"), "{body}");
|
||||
|
||||
let ratings = server.db.ratings_with_feed().await.unwrap();
|
||||
assert_eq!(ratings, vec![(7, Vote::Up)]);
|
||||
let priors = server.db.feed_priors().await.unwrap();
|
||||
assert_eq!(priors.len(), 1);
|
||||
assert_eq!(
|
||||
(priors[0].feed_id, priors[0].upvotes, priors[0].downvotes),
|
||||
(7, 1, 0)
|
||||
);
|
||||
|
||||
// Flipping the vote rewrites the prior rather than double-counting.
|
||||
let down = rating_url(&server.base, VECTOR_SECRET, date(), id, Vote::Down);
|
||||
assert_eq!(client().get(&down).send().await.unwrap().status(), 200);
|
||||
let priors = server.db.feed_priors().await.unwrap();
|
||||
assert_eq!((priors[0].upvotes, priors[0].downvotes), (0, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rating_rejects_bad_tokens_dates_and_unknown_articles() {
|
||||
let server = TestServer::start(false).await;
|
||||
let id = server.seed_article().await;
|
||||
|
||||
let bad = format!("{}/r/2026-08-15/{id}/up?t=deadbeefdeadbeef", server.base);
|
||||
assert_eq!(client().get(&bad).send().await.unwrap().status(), 403);
|
||||
let missing = format!("{}/r/2026-08-15/{id}/up", server.base);
|
||||
assert_eq!(client().get(&missing).send().await.unwrap().status(), 403);
|
||||
|
||||
// A valid token for an article that does not exist.
|
||||
let unknown = rating_url(&server.base, VECTOR_SECRET, date(), 9999, Vote::Up);
|
||||
assert_eq!(client().get(&unknown).send().await.unwrap().status(), 404);
|
||||
|
||||
// Malformed date / vote.
|
||||
let token = rating_token(VECTOR_SECRET, date(), id, Vote::Up);
|
||||
let bad_date = format!("{}/r/not-a-date/{id}/up?t={token}", server.base);
|
||||
assert_eq!(client().get(&bad_date).send().await.unwrap().status(), 400);
|
||||
let bad_vote = format!("{}/r/2026-08-15/{id}/sideways?t={token}", server.base);
|
||||
assert_eq!(client().get(&bad_vote).send().await.unwrap().status(), 400);
|
||||
|
||||
assert!(server.db.ratings_with_feed().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opds_and_files_are_served_behind_basic_auth() {
|
||||
let server = TestServer::start(true).await;
|
||||
std::fs::write(
|
||||
server.xtc_dir().join(crate::publish::XTC_OPDS_FILENAME),
|
||||
"<feed/>",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
server
|
||||
.xtc_dir()
|
||||
.join("The Daily EPUB - 2026-08-15 (X4).xtch"),
|
||||
b"XTCH",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let res = client()
|
||||
.get(format!("{}/opds/xtc.xml", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 401);
|
||||
assert!(
|
||||
res.headers()
|
||||
.get("www-authenticate")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.starts_with("Basic realm=")
|
||||
);
|
||||
|
||||
let res = client()
|
||||
.get(format!("{}/opds/xtc.xml", server.base))
|
||||
.basic_auth("opds", Some("wrong"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 401);
|
||||
|
||||
let res = client()
|
||||
.get(format!("{}/opds/xtc.xml", server.base))
|
||||
.basic_auth("opds", Some("hunter2"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
assert!(
|
||||
res.headers()[header::CONTENT_TYPE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("application/atom+xml")
|
||||
);
|
||||
assert_eq!(res.text().await.unwrap(), "<feed/>");
|
||||
|
||||
// The catalog root serves the same feed, behind the same auth.
|
||||
for alias in ["/opds", "/opds/"] {
|
||||
let res = client()
|
||||
.get(format!("{}{alias}", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 401, "{alias}");
|
||||
let res = client()
|
||||
.get(format!("{}{alias}", server.base))
|
||||
.basic_auth("opds", Some("hunter2"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 200, "{alias}");
|
||||
assert_eq!(res.text().await.unwrap(), "<feed/>", "{alias}");
|
||||
}
|
||||
|
||||
let res = client()
|
||||
.get(format!(
|
||||
"{}/files/xtc/The%20Daily%20EPUB%20-%202026-08-15%20(X4).xtch",
|
||||
server.base
|
||||
))
|
||||
.basic_auth("opds", Some("hunter2"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
// CrossPoint needs the size up front to show download progress.
|
||||
assert_eq!(res.content_length(), Some(4));
|
||||
assert_eq!(res.bytes().await.unwrap().as_ref(), b"XTCH");
|
||||
|
||||
// Ratings are not behind auth (the token is the credential).
|
||||
assert_eq!(
|
||||
client()
|
||||
.get(format!("{}/healthz", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn file_route_rejects_path_traversal() {
|
||||
let server = TestServer::start(false).await;
|
||||
std::fs::write(server._dir.path().join("secret"), b"top secret").unwrap();
|
||||
|
||||
// Encoded traversal survives URL normalization and reaches the handler.
|
||||
let res = client()
|
||||
.get(format!("{}/files/xtc/..%2Fsecret", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 400);
|
||||
let res = client()
|
||||
.get(format!(
|
||||
"{}/files/xtc/%2e%2e%2f%2e%2e%2fsecret",
|
||||
server.base
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 400);
|
||||
// A plain `..` segment is not even a match for the single-segment route.
|
||||
let res = client()
|
||||
.get(format!("{}/files/xtc/nope.xtch", server.base))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(res.status(), 404);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
//! HackerNews via the Algolia API (spec §3.4, §3.7).
|
||||
|
||||
use jiff::Timestamp;
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
|
||||
use super::SocialError;
|
||||
use crate::types::{ArticleId, Comment, CommentThread, SocialRef, SocialSource};
|
||||
|
||||
/// Algolia search endpoint (free, generous limits) (§3.4).
|
||||
pub const SEARCH_URL: &str = "https://hn.algolia.com/api/v1/search";
|
||||
/// Algolia item-tree endpoint used for comment chapters (§3.7).
|
||||
pub const ITEM_URL: &str = "https://hn.algolia.com/api/v1/items";
|
||||
/// Canonical HN item page prefix.
|
||||
pub const ITEM_PAGE: &str = "https://news.ycombinator.com/item?id=";
|
||||
|
||||
/// Hits requested per URL search — enough to spot the canonical submission.
|
||||
const HITS_PER_PAGE: &str = "10";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One Algolia search hit (only the fields §3.4 uses).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct Hit {
|
||||
#[serde(rename = "objectID")]
|
||||
pub object_id: String,
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub url: Option<String>,
|
||||
#[serde(default)]
|
||||
pub points: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub num_comments: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct SearchResponse {
|
||||
#[serde(default)]
|
||||
hits: Vec<Hit>,
|
||||
}
|
||||
|
||||
/// A node of the Algolia item tree (`/items/{id}`) (§3.7).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct Item {
|
||||
#[serde(default)]
|
||||
id: Option<i64>,
|
||||
#[serde(default, rename = "type")]
|
||||
kind: Option<String>,
|
||||
#[serde(default)]
|
||||
author: Option<String>,
|
||||
// The story's own `title` is deliberately not deserialized here: the comment
|
||||
// renderer takes the article title from the `Pick`, not from Algolia (§3.7).
|
||||
#[serde(default)]
|
||||
text: Option<String>,
|
||||
#[serde(default)]
|
||||
points: Option<i64>,
|
||||
#[serde(default)]
|
||||
children: Vec<Item>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure parsing (unit-tested against `tests/fixtures/hn_*.json`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract the story id from a `news.ycombinator.com/item?id=N` comments URL (§3.4).
|
||||
pub fn story_id_from_comments_url(comments_url: &str) -> Option<String> {
|
||||
let url = Url::parse(comments_url.trim()).ok()?;
|
||||
let host = url.host_str()?.to_ascii_lowercase();
|
||||
if host != "news.ycombinator.com" && !host.ends_with(".ycombinator.com") {
|
||||
return None;
|
||||
}
|
||||
if !url.path().starts_with("/item") {
|
||||
return None;
|
||||
}
|
||||
url.query_pairs()
|
||||
.find(|(k, _)| k == "id")
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.filter(|id| !id.is_empty() && id.chars().all(|c| c.is_ascii_digit()))
|
||||
}
|
||||
|
||||
fn parse_hits(body: &str) -> Result<Vec<Hit>, SocialError> {
|
||||
serde_json::from_str::<SearchResponse>(body)
|
||||
.map(|r| r.hits)
|
||||
.map_err(|e| SocialError::Unexpected {
|
||||
platform: "hn",
|
||||
detail: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Turn a search response into the best [`SocialRef`] for `canonical_url` (§3.4).
|
||||
///
|
||||
/// Prefers a hit whose own URL canonicalizes to `canonical_url`; otherwise the
|
||||
/// highest-scoring hit wins (Algolia sorts by relevance, not points).
|
||||
pub fn parse_search_response(
|
||||
body: &str,
|
||||
canonical_url: &str,
|
||||
article_id: ArticleId,
|
||||
fetched_at: Timestamp,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let hits = parse_hits(body)?;
|
||||
Ok(best_hit(&hits, Some(canonical_url)).map(|hit| social_ref(hit, article_id, fetched_at)))
|
||||
}
|
||||
|
||||
/// Turn a `search?tags=story_{id}` response into a [`SocialRef`] (§3.4).
|
||||
pub fn parse_story_response(
|
||||
body: &str,
|
||||
article_id: ArticleId,
|
||||
fetched_at: Timestamp,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let hits = parse_hits(body)?;
|
||||
Ok(best_hit(&hits, None).map(|hit| social_ref(hit, article_id, fetched_at)))
|
||||
}
|
||||
|
||||
fn best_hit<'a>(hits: &'a [Hit], canonical_url: Option<&str>) -> Option<&'a Hit> {
|
||||
let exact = canonical_url.and_then(|want| {
|
||||
hits.iter()
|
||||
.filter(|h| {
|
||||
h.url
|
||||
.as_deref()
|
||||
.and_then(crate::dedupe::canonical_url)
|
||||
.is_some_and(|c| c == want)
|
||||
})
|
||||
.max_by_key(|h| h.points.unwrap_or(0))
|
||||
});
|
||||
exact.or_else(|| hits.iter().max_by_key(|h| h.points.unwrap_or(0)))
|
||||
}
|
||||
|
||||
fn social_ref(hit: &Hit, article_id: ArticleId, fetched_at: Timestamp) -> SocialRef {
|
||||
SocialRef {
|
||||
article_id,
|
||||
source: SocialSource::Hn,
|
||||
item_id: Some(hit.object_id.clone()),
|
||||
score: hit.points.unwrap_or(0),
|
||||
num_comments: hit.num_comments.unwrap_or(0),
|
||||
item_url: Some(format!("{ITEM_PAGE}{}", hit.object_id)),
|
||||
fetched_at,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `/items/{id}` into a [`CommentThread`] (§3.7).
|
||||
///
|
||||
/// The tree is returned in full; [`crate::comments::truncate`] applies the §3.7
|
||||
/// display limits.
|
||||
pub fn parse_item_response(body: &str) -> Result<CommentThread, SocialError> {
|
||||
let item: Item = serde_json::from_str(body).map_err(|e| SocialError::Unexpected {
|
||||
platform: "hn",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
let object_id = item.id.map(|i| i.to_string()).unwrap_or_default();
|
||||
let comments = map_children(&item.children, 0);
|
||||
let total = count_comments(&item.children);
|
||||
Ok(CommentThread {
|
||||
source: SocialSource::Hn,
|
||||
item_url: format!("{ITEM_PAGE}{object_id}"),
|
||||
total_comments: total,
|
||||
comments,
|
||||
})
|
||||
}
|
||||
|
||||
fn map_children(children: &[Item], depth: usize) -> Vec<Comment> {
|
||||
children
|
||||
.iter()
|
||||
.filter(|c| c.kind.as_deref() != Some("story"))
|
||||
.filter_map(|child| {
|
||||
let text = child.text.as_deref().unwrap_or("").trim();
|
||||
let kids = map_children(&child.children, depth + 1);
|
||||
if text.is_empty() && kids.is_empty() {
|
||||
return None; // deleted comment with no surviving replies
|
||||
}
|
||||
Some(Comment {
|
||||
author: child.author.clone().unwrap_or_else(|| "[deleted]".into()),
|
||||
points: child.points,
|
||||
text_html: crate::extract::sanitize(text),
|
||||
depth,
|
||||
children: kids,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn count_comments(children: &[Item]) -> i64 {
|
||||
children
|
||||
.iter()
|
||||
.map(|c| 1 + count_comments(&c.children))
|
||||
.sum()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Network
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn get_text(
|
||||
http: &reqwest::Client,
|
||||
url: &str,
|
||||
query: &[(&str, &str)],
|
||||
) -> Result<String, SocialError> {
|
||||
let response = http.get(url).query(query).send().await?;
|
||||
if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
return Err(SocialError::RateLimited("hn"));
|
||||
}
|
||||
Ok(response.error_for_status()?.text().await?)
|
||||
}
|
||||
|
||||
/// `GET /search?query=<url>&restrictSearchableAttributes=url` → best hit (§3.4).
|
||||
///
|
||||
/// Returns `None` when HN has no submission for this URL.
|
||||
pub async fn search_by_url(
|
||||
http: &reqwest::Client,
|
||||
canonical_url: &str,
|
||||
article_id: ArticleId,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let body = get_text(
|
||||
http,
|
||||
SEARCH_URL,
|
||||
&[
|
||||
("query", canonical_url),
|
||||
("restrictSearchableAttributes", "url"),
|
||||
("tags", "story"),
|
||||
("hitsPerPage", HITS_PER_PAGE),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
parse_search_response(&body, canonical_url, article_id, Timestamp::now())
|
||||
}
|
||||
|
||||
/// `GET /search?tags=story_{id}` → points/comment count for a known story (§3.4).
|
||||
///
|
||||
/// The `/items/{id}` endpoint carries the whole comment tree; the search endpoint
|
||||
/// answers the same question with a fraction of the bytes.
|
||||
pub async fn fetch_story(
|
||||
http: &reqwest::Client,
|
||||
object_id: &str,
|
||||
article_id: ArticleId,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let tag = format!("story_{object_id}");
|
||||
let body = get_text(
|
||||
http,
|
||||
SEARCH_URL,
|
||||
&[("tags", tag.as_str()), ("hitsPerPage", "1")],
|
||||
)
|
||||
.await?;
|
||||
parse_story_response(&body, article_id, Timestamp::now())
|
||||
}
|
||||
|
||||
/// Full comment tree for a story (§3.7).
|
||||
pub async fn fetch_comments(
|
||||
http: &reqwest::Client,
|
||||
object_id: &str,
|
||||
) -> Result<CommentThread, SocialError> {
|
||||
let url = format!("{ITEM_URL}/{object_id}");
|
||||
let body = get_text(http, &url, &[]).await?;
|
||||
parse_item_response(&body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SEARCH: &str = include_str!("../../tests/fixtures/m2_hn_search_by_url.json");
|
||||
const EMPTY: &str = include_str!("../../tests/fixtures/m2_hn_search_empty.json");
|
||||
const ITEM: &str = include_str!("../../tests/fixtures/m2_hn_item.json");
|
||||
|
||||
fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z".parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_url_yields_the_story_id() {
|
||||
assert_eq!(
|
||||
story_id_from_comments_url("https://news.ycombinator.com/item?id=41234567").as_deref(),
|
||||
Some("41234567")
|
||||
);
|
||||
assert_eq!(
|
||||
story_id_from_comments_url("http://news.ycombinator.com/item?id=1&foo=bar").as_deref(),
|
||||
Some("1")
|
||||
);
|
||||
assert_eq!(story_id_from_comments_url("https://lobste.rs/s/abc"), None);
|
||||
assert_eq!(
|
||||
story_id_from_comments_url("https://news.ycombinator.com/newest"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
story_id_from_comments_url("https://news.ycombinator.com/item?id=abc"),
|
||||
None
|
||||
);
|
||||
assert_eq!(story_id_from_comments_url(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_picks_the_matching_submission() {
|
||||
let got = parse_search_response(SEARCH, "https://blog.dev/post", 7, ts())
|
||||
.unwrap()
|
||||
.expect("a hit");
|
||||
assert_eq!(got.article_id, 7);
|
||||
assert_eq!(got.source, SocialSource::Hn);
|
||||
assert_eq!(got.item_id.as_deref(), Some("41234567"));
|
||||
assert_eq!(got.score, 342);
|
||||
assert_eq!(got.num_comments, 210);
|
||||
assert_eq!(
|
||||
got.item_url.as_deref(),
|
||||
Some("https://news.ycombinator.com/item?id=41234567")
|
||||
);
|
||||
assert_eq!(got.fetched_at, ts());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_response_falls_back_to_the_top_hit() {
|
||||
// No hit canonicalizes to this URL, so the highest-scoring one wins.
|
||||
let got = parse_search_response(SEARCH, "https://elsewhere.dev/x", 7, ts())
|
||||
.unwrap()
|
||||
.expect("a hit");
|
||||
assert_eq!(got.item_id.as_deref(), Some("41234567"));
|
||||
assert_eq!(got.score, 342);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_search_response_is_not_an_error() {
|
||||
assert!(
|
||||
parse_search_response(EMPTY, "https://blog.dev/never-submitted", 1, ts())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(parse_story_response(EMPTY, 1, ts()).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_json_is_reported_not_panicked() {
|
||||
let err = parse_search_response("{not json", "https://x.dev", 1, ts()).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
SocialError::Unexpected { platform: "hn", .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_response_becomes_a_comment_tree() {
|
||||
let thread = parse_item_response(ITEM).unwrap();
|
||||
assert_eq!(thread.source, SocialSource::Hn);
|
||||
assert_eq!(
|
||||
thread.item_url,
|
||||
"https://news.ycombinator.com/item?id=41234567"
|
||||
);
|
||||
// 5 comment nodes in the fixture (one of them deleted).
|
||||
assert_eq!(thread.total_comments, 5);
|
||||
// The deleted, childless comment is dropped from the render tree.
|
||||
assert_eq!(thread.comments.len(), 2);
|
||||
|
||||
let first = &thread.comments[0];
|
||||
assert_eq!(first.author, "dbnerd");
|
||||
assert_eq!(first.points, Some(88));
|
||||
assert_eq!(first.depth, 0);
|
||||
assert!(first.text_html.contains("page splits"));
|
||||
// <i> is not in the allowlist, its text survives.
|
||||
assert!(!first.text_html.contains("<i>"));
|
||||
assert!(first.text_html.contains("Bookmarked."));
|
||||
|
||||
let reply = &first.children[0];
|
||||
assert_eq!(reply.author, "tylerh");
|
||||
assert_eq!(reply.depth, 1);
|
||||
assert_eq!(reply.children[0].depth, 2);
|
||||
assert_eq!(thread.comments[1].author, "skeptic");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
//! Lobsters via `/s/{id}.json` (spec §3.4, §3.7).
|
||||
//!
|
||||
//! Lobsters has no public URL-search API, so linkage only works when the entry
|
||||
//! arrived through a lobste.rs feed or its `comments_url` points at a story (§7).
|
||||
|
||||
use jiff::Timestamp;
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
|
||||
use super::SocialError;
|
||||
use crate::types::{ArticleId, Comment, CommentThread, SocialRef, SocialSource};
|
||||
|
||||
pub const STORY_URL_PREFIX: &str = "https://lobste.rs/s/";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct Story {
|
||||
#[serde(default)]
|
||||
short_id: Option<String>,
|
||||
#[serde(default)]
|
||||
short_id_url: Option<String>,
|
||||
#[serde(default)]
|
||||
comments_url: Option<String>,
|
||||
#[serde(default)]
|
||||
score: Option<i64>,
|
||||
#[serde(default)]
|
||||
comment_count: Option<i64>,
|
||||
#[serde(default)]
|
||||
comments: Vec<RawComment>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct RawComment {
|
||||
#[serde(default)]
|
||||
short_id: Option<String>,
|
||||
#[serde(default)]
|
||||
parent_comment: Option<String>,
|
||||
#[serde(default)]
|
||||
comment: Option<String>,
|
||||
#[serde(default)]
|
||||
score: Option<i64>,
|
||||
#[serde(default)]
|
||||
is_deleted: bool,
|
||||
/// String in the current API; older responses nested it in an object.
|
||||
#[serde(default)]
|
||||
commenting_user: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl RawComment {
|
||||
fn author(&self) -> String {
|
||||
match &self.commenting_user {
|
||||
Some(serde_json::Value::String(s)) => s.clone(),
|
||||
Some(serde_json::Value::Object(o)) => o
|
||||
.get("username")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("[unknown]")
|
||||
.to_string(),
|
||||
_ => "[unknown]".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure parsing (unit-tested against `tests/fixtures/m2_lobsters_story.json`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract the story id from a `lobste.rs/s/<id>` URL (§3.4).
|
||||
pub fn story_id_from_url(url: &str) -> Option<String> {
|
||||
let parsed = Url::parse(url.trim()).ok()?;
|
||||
let host = parsed.host_str()?.to_ascii_lowercase();
|
||||
if host != "lobste.rs" && !host.ends_with(".lobste.rs") {
|
||||
return None;
|
||||
}
|
||||
let mut segments = parsed.path_segments()?;
|
||||
if segments.next()? != "s" {
|
||||
return None;
|
||||
}
|
||||
segments
|
||||
.next()
|
||||
.map(str::to_string)
|
||||
.filter(|id| !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric()))
|
||||
}
|
||||
|
||||
fn parse_story(body: &str) -> Result<Story, SocialError> {
|
||||
serde_json::from_str(body).map_err(|e| SocialError::Unexpected {
|
||||
platform: "lobsters",
|
||||
detail: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Turn a `/s/{id}.json` body into a [`SocialRef`] (§3.4).
|
||||
pub fn parse_story_response(
|
||||
body: &str,
|
||||
story_id: &str,
|
||||
article_id: ArticleId,
|
||||
fetched_at: Timestamp,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let story = parse_story(body)?;
|
||||
let id = story
|
||||
.short_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| story_id.to_string());
|
||||
Ok(Some(SocialRef {
|
||||
article_id,
|
||||
source: SocialSource::Lobsters,
|
||||
item_id: Some(id.clone()),
|
||||
score: story.score.unwrap_or(0),
|
||||
num_comments: story.comment_count.unwrap_or(story.comments.len() as i64),
|
||||
item_url: Some(
|
||||
story
|
||||
.short_id_url
|
||||
.or(story.comments_url)
|
||||
.unwrap_or_else(|| format!("{STORY_URL_PREFIX}{id}")),
|
||||
),
|
||||
fetched_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Turn the same body's flat `comments` array into a nested tree (§3.7).
|
||||
pub fn parse_comments_response(body: &str, story_id: &str) -> Result<CommentThread, SocialError> {
|
||||
let story = parse_story(body)?;
|
||||
let id = story
|
||||
.short_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| story_id.to_string());
|
||||
let item_url = story
|
||||
.short_id_url
|
||||
.clone()
|
||||
.or_else(|| story.comments_url.clone())
|
||||
.unwrap_or_else(|| format!("{STORY_URL_PREFIX}{id}"));
|
||||
let total = story.comment_count.unwrap_or(story.comments.len() as i64);
|
||||
Ok(CommentThread {
|
||||
source: SocialSource::Lobsters,
|
||||
item_url,
|
||||
total_comments: total,
|
||||
comments: build_tree(&story.comments),
|
||||
})
|
||||
}
|
||||
|
||||
/// Lobsters returns a flat list ordered depth-first with `parent_comment` links.
|
||||
fn build_tree(raw: &[RawComment]) -> Vec<Comment> {
|
||||
let mut roots: Vec<Comment> = Vec::new();
|
||||
// Path of `short_id`s from the root to the comment most recently inserted.
|
||||
let mut path: Vec<String> = Vec::new();
|
||||
|
||||
for item in raw {
|
||||
if item.is_deleted {
|
||||
continue;
|
||||
}
|
||||
let text = item.comment.as_deref().unwrap_or("").trim();
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let comment = Comment {
|
||||
author: item.author(),
|
||||
points: item.score,
|
||||
text_html: crate::extract::sanitize(text),
|
||||
depth: 0,
|
||||
children: Vec::new(),
|
||||
};
|
||||
let short_id = item.short_id.clone().unwrap_or_default();
|
||||
match item.parent_comment.as_deref() {
|
||||
Some(parent) => {
|
||||
while path.last().is_some_and(|p| p != parent) {
|
||||
path.pop();
|
||||
}
|
||||
if path.is_empty() {
|
||||
// Parent was dropped (deleted); promote to a root thread.
|
||||
roots.push(comment);
|
||||
path = vec![short_id];
|
||||
continue;
|
||||
}
|
||||
let depth = path.len();
|
||||
if let Some(node) = descend(&mut roots, &path) {
|
||||
let mut child = comment;
|
||||
child.depth = depth;
|
||||
node.children.push(child);
|
||||
path.push(short_id);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
roots.push(comment);
|
||||
path = vec![short_id];
|
||||
}
|
||||
}
|
||||
}
|
||||
roots
|
||||
}
|
||||
|
||||
/// Walk `roots` along the ids in `path`, returning the last node on it.
|
||||
fn descend<'a>(roots: &'a mut [Comment], path: &[String]) -> Option<&'a mut Comment> {
|
||||
let mut node = roots.last_mut()?;
|
||||
for _ in 1..path.len() {
|
||||
node = node.children.last_mut()?;
|
||||
}
|
||||
Some(node)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Network
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn get_text(http: &reqwest::Client, story_id: &str) -> Result<String, SocialError> {
|
||||
let url = format!("{STORY_URL_PREFIX}{story_id}.json");
|
||||
let response = http.get(&url).send().await?;
|
||||
if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
return Err(SocialError::RateLimited("lobsters"));
|
||||
}
|
||||
Ok(response.error_for_status()?.text().await?)
|
||||
}
|
||||
|
||||
/// `GET https://lobste.rs/s/{id}.json` → score + comment count (§3.4).
|
||||
pub async fn fetch_story(
|
||||
http: &reqwest::Client,
|
||||
story_id: &str,
|
||||
article_id: ArticleId,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let body = get_text(http, story_id).await?;
|
||||
parse_story_response(&body, story_id, article_id, Timestamp::now())
|
||||
}
|
||||
|
||||
/// The same endpoint's `comments` array, as a tree (§3.7).
|
||||
pub async fn fetch_comments(
|
||||
http: &reqwest::Client,
|
||||
story_id: &str,
|
||||
) -> Result<CommentThread, SocialError> {
|
||||
let body = get_text(http, story_id).await?;
|
||||
parse_comments_response(&body, story_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const STORY: &str = include_str!("../../tests/fixtures/m2_lobsters_story.json");
|
||||
|
||||
fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z".parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn story_ids_come_out_of_lobsters_urls() {
|
||||
assert_eq!(
|
||||
story_id_from_url("https://lobste.rs/s/abcdef/a_deep_dive").as_deref(),
|
||||
Some("abcdef")
|
||||
);
|
||||
assert_eq!(
|
||||
story_id_from_url("https://lobste.rs/s/abcdef").as_deref(),
|
||||
Some("abcdef")
|
||||
);
|
||||
assert_eq!(story_id_from_url("https://lobste.rs/"), None);
|
||||
assert_eq!(story_id_from_url("https://lobste.rs/s/"), None);
|
||||
assert_eq!(
|
||||
story_id_from_url("https://news.ycombinator.com/item?id=1"),
|
||||
None
|
||||
);
|
||||
assert_eq!(story_id_from_url("garbage"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn story_response_gives_score_and_comment_count() {
|
||||
let got = parse_story_response(STORY, "abcdef", 9, ts())
|
||||
.unwrap()
|
||||
.expect("a story");
|
||||
assert_eq!(got.article_id, 9);
|
||||
assert_eq!(got.source, SocialSource::Lobsters);
|
||||
assert_eq!(got.item_id.as_deref(), Some("abcdef"));
|
||||
assert_eq!(got.score, 78);
|
||||
assert_eq!(got.num_comments, 4);
|
||||
assert_eq!(got.item_url.as_deref(), Some("https://lobste.rs/s/abcdef"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_comments_become_a_tree() {
|
||||
let thread = parse_comments_response(STORY, "abcdef").unwrap();
|
||||
assert_eq!(thread.source, SocialSource::Lobsters);
|
||||
assert_eq!(thread.total_comments, 4);
|
||||
// Two top-level threads; the deleted reply is dropped.
|
||||
assert_eq!(thread.comments.len(), 2);
|
||||
|
||||
let first = &thread.comments[0];
|
||||
assert_eq!(first.author, "bob");
|
||||
assert_eq!(first.points, Some(21));
|
||||
assert_eq!(first.depth, 0);
|
||||
assert_eq!(first.children.len(), 1);
|
||||
assert_eq!(first.children[0].author, "carol");
|
||||
assert_eq!(first.children[0].depth, 1);
|
||||
assert!(first.children[0].text_html.contains("tight too"));
|
||||
|
||||
let second = &thread.comments[1];
|
||||
assert_eq!(second.author, "dave");
|
||||
assert!(second.children.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_json_is_reported_not_panicked() {
|
||||
assert!(matches!(
|
||||
parse_story_response("nope", "abcdef", 1, ts()).unwrap_err(),
|
||||
SocialError::Unexpected {
|
||||
platform: "lobsters",
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
//! Social-proof enrichment (spec §3.4).
|
||||
//!
|
||||
//! For every deduped article, look up HackerNews (Algolia), Lobsters and Reddit
|
||||
//! in parallel behind a semaphore, caching results in the `social` table. Every
|
||||
//! lookup is best-effort: failures never fail the run (notes §3).
|
||||
|
||||
pub mod hn;
|
||||
pub mod lobsters;
|
||||
pub mod reddit;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use jiff::Timestamp;
|
||||
use tokio::sync::{Mutex, Semaphore};
|
||||
|
||||
use crate::db::Db;
|
||||
use crate::types::{Article, ArticleId, SocialRef, SocialSource, SourceKind};
|
||||
|
||||
/// Concurrent social lookups (§3.4).
|
||||
pub const CONCURRENCY: usize = 8;
|
||||
/// Reddit pacing: roughly one request per second (§3.4).
|
||||
pub const REDDIT_MIN_INTERVAL_MS: u64 = 1000;
|
||||
/// Cached rows younger than this are reused instead of re-fetched (§3.4).
|
||||
pub const CACHE_TTL_HOURS: i64 = 24;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum SocialError {
|
||||
#[error("http error: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("unexpected response from {platform}: {detail}")]
|
||||
Unexpected {
|
||||
platform: &'static str,
|
||||
detail: String,
|
||||
},
|
||||
#[error("rate limited by {0}")]
|
||||
RateLimited(&'static str),
|
||||
}
|
||||
|
||||
/// Serializes a platform's requests to at most one per `interval` (§3.4 Reddit).
|
||||
#[derive(Debug, Clone)]
|
||||
struct Pacer {
|
||||
last: Arc<Mutex<Option<tokio::time::Instant>>>,
|
||||
interval: Duration,
|
||||
}
|
||||
|
||||
impl Pacer {
|
||||
fn new(interval: Duration) -> Self {
|
||||
Self {
|
||||
last: Arc::new(Mutex::new(None)),
|
||||
interval,
|
||||
}
|
||||
}
|
||||
|
||||
/// Block until the caller may issue the next request.
|
||||
async fn tick(&self) {
|
||||
let mut last = self.last.lock().await;
|
||||
if let Some(previous) = *last {
|
||||
let elapsed = previous.elapsed();
|
||||
if elapsed < self.interval {
|
||||
tokio::time::sleep(self.interval - elapsed).await;
|
||||
}
|
||||
}
|
||||
*last = Some(tokio::time::Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifiers one article needs for its three lookups (§3.4).
|
||||
#[derive(Debug, Clone)]
|
||||
struct Lookup {
|
||||
article_id: ArticleId,
|
||||
canonical_url: String,
|
||||
/// HN story id from `comments_url`, when the feed handed us one.
|
||||
hn_story_id: Option<String>,
|
||||
/// Lobsters story id — only available for lobste.rs-originated entries (§7).
|
||||
lobsters_story_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Lookup {
|
||||
fn for_article(article: &Article) -> Self {
|
||||
let comments_url = article.comments_url.as_deref().unwrap_or("");
|
||||
let hn_story_id = hn::story_id_from_comments_url(comments_url);
|
||||
|
||||
// Lobsters linkage requires a lobste.rs origin: either the feed itself or
|
||||
// a comments URL pointing at a story (§3.4, §7).
|
||||
let via_lobsters = article.came_via(SourceKind::Lobsters);
|
||||
let lobsters_story_id = lobsters::story_id_from_url(comments_url).or_else(|| {
|
||||
via_lobsters
|
||||
.then(|| lobsters::story_id_from_url(&article.url))
|
||||
.flatten()
|
||||
});
|
||||
|
||||
Self {
|
||||
article_id: article.id,
|
||||
canonical_url: article.canonical_url.clone(),
|
||||
hn_story_id,
|
||||
lobsters_story_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Orchestrates the per-platform clients and the `social` cache (§3.4).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SocialEnricher {
|
||||
http: reqwest::Client,
|
||||
db: Db,
|
||||
reddit_pacer: Pacer,
|
||||
/// Set once Reddit 429s: the rest of the run skips Reddit entirely (§3.4).
|
||||
reddit_blocked: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl SocialEnricher {
|
||||
pub fn new(http: reqwest::Client, db: Db) -> Self {
|
||||
Self {
|
||||
http,
|
||||
db,
|
||||
reddit_pacer: Pacer::new(Duration::from_millis(REDDIT_MIN_INTERVAL_MS)),
|
||||
reddit_blocked: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Enrich every article in place, writing hits to the `social` table (§3.4).
|
||||
///
|
||||
/// Returns the number of articles for which at least one platform had a hit.
|
||||
pub async fn enrich_all(&self, articles: &mut [Article]) -> usize {
|
||||
self.run(articles, false).await
|
||||
}
|
||||
|
||||
/// [`Self::enrich_all`] ignoring the cache — used by `backfill-social` (§2).
|
||||
pub async fn refresh_all(&self, articles: &mut [Article]) -> usize {
|
||||
self.run(articles, true).await
|
||||
}
|
||||
|
||||
async fn run(&self, articles: &mut [Article], force: bool) -> usize {
|
||||
let span = tracing::info_span!("social", articles = articles.len(), force);
|
||||
let _guard = span.enter();
|
||||
|
||||
let lookups: Vec<Lookup> = articles.iter().map(Lookup::for_article).collect();
|
||||
let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
|
||||
let results: Vec<(usize, Vec<SocialRef>)> =
|
||||
futures::stream::iter(lookups.iter().enumerate())
|
||||
.map(|(i, lookup)| {
|
||||
let semaphore = Arc::clone(&semaphore);
|
||||
async move {
|
||||
// A closed semaphore is impossible here; treat it as "no limit".
|
||||
let _permit = semaphore.acquire().await.ok();
|
||||
(i, self.lookup(lookup, force).await)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(CONCURRENCY)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut hits = 0;
|
||||
for (i, refs) in results {
|
||||
if !refs.is_empty() {
|
||||
hits += 1;
|
||||
}
|
||||
articles[i].social = refs;
|
||||
}
|
||||
tracing::info!(hits, "social enrichment complete");
|
||||
hits
|
||||
}
|
||||
|
||||
/// Look up every platform for one article, honoring the cache TTL (§3.4).
|
||||
pub async fn enrich_one(&self, article: &Article) -> Vec<SocialRef> {
|
||||
self.lookup(&Lookup::for_article(article), false).await
|
||||
}
|
||||
|
||||
async fn lookup(&self, target: &Lookup, force: bool) -> Vec<SocialRef> {
|
||||
let mut refs: Vec<SocialRef> = if force || target.article_id == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
cached_refs(&self.db, target.article_id).await
|
||||
};
|
||||
let cached = refs.len();
|
||||
|
||||
if !refs.iter().any(|r| r.source == SocialSource::Hn)
|
||||
&& let Some(found) = self.lookup_hn(target).await
|
||||
{
|
||||
refs.push(found);
|
||||
}
|
||||
if !refs.iter().any(|r| r.source == SocialSource::Lobsters)
|
||||
&& let Some(found) = self.lookup_lobsters(target).await
|
||||
{
|
||||
refs.push(found);
|
||||
}
|
||||
if !refs.iter().any(|r| r.source == SocialSource::Reddit)
|
||||
&& let Some(found) = self.lookup_reddit(target).await
|
||||
{
|
||||
refs.push(found);
|
||||
}
|
||||
|
||||
// Persist only what we just fetched; cached rows are already stored.
|
||||
if target.article_id != 0 {
|
||||
for r in refs.iter().skip(cached) {
|
||||
if let Err(e) = self.db.upsert_social(r).await {
|
||||
tracing::warn!(
|
||||
article = target.article_id,
|
||||
"storing social ref failed: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
refs.sort_by_key(|r| r.source);
|
||||
refs
|
||||
}
|
||||
|
||||
async fn lookup_hn(&self, target: &Lookup) -> Option<SocialRef> {
|
||||
let result = match &target.hn_story_id {
|
||||
Some(id) => hn::fetch_story(&self.http, id, target.article_id).await,
|
||||
None => hn::search_by_url(&self.http, &target.canonical_url, target.article_id).await,
|
||||
};
|
||||
best_effort("hn", &target.canonical_url, result)
|
||||
}
|
||||
|
||||
async fn lookup_lobsters(&self, target: &Lookup) -> Option<SocialRef> {
|
||||
let id = target.lobsters_story_id.as_deref()?;
|
||||
let result = lobsters::fetch_story(&self.http, id, target.article_id).await;
|
||||
best_effort("lobsters", &target.canonical_url, result)
|
||||
}
|
||||
|
||||
async fn lookup_reddit(&self, target: &Lookup) -> Option<SocialRef> {
|
||||
if self.reddit_blocked.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
self.reddit_pacer.tick().await;
|
||||
let result =
|
||||
reddit::lookup_by_url(&self.http, &target.canonical_url, target.article_id).await;
|
||||
if matches!(result, Err(SocialError::RateLimited(_))) {
|
||||
// Back off for the rest of the run: social data is best-effort (§3.4).
|
||||
self.reddit_blocked.store(true, Ordering::Relaxed);
|
||||
tracing::warn!("reddit rate-limited us; skipping reddit for the rest of this run");
|
||||
return None;
|
||||
}
|
||||
best_effort("reddit", &target.canonical_url, result)
|
||||
}
|
||||
|
||||
/// Re-poll recent articles' social scores (`daily-epub backfill-social`, §2).
|
||||
pub async fn backfill(&self, days: u32) -> anyhow::Result<usize> {
|
||||
let span = tracing::info_span!("backfill_social", days);
|
||||
let _guard = span.enter();
|
||||
|
||||
let cutoff = Timestamp::now() - jiff::Span::new().hours(24 * i64::from(days.max(1)));
|
||||
let rows = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT id FROM articles WHERE first_seen >= ? ORDER BY first_seen DESC",
|
||||
)
|
||||
.bind(crate::db::fmt_ts(cutoff))
|
||||
.fetch_all(self.db.pool())
|
||||
.await?;
|
||||
|
||||
let mut articles: Vec<Article> = Vec::with_capacity(rows.len());
|
||||
for id in rows {
|
||||
match self.db.get_article(id).await {
|
||||
Ok(Some(article)) => articles.push(article),
|
||||
Ok(None) => {}
|
||||
Err(e) => tracing::warn!(article = id, "loading article failed: {e}"),
|
||||
}
|
||||
}
|
||||
tracing::info!(articles = articles.len(), "re-polling social scores");
|
||||
Ok(self.refresh_all(&mut articles).await)
|
||||
}
|
||||
}
|
||||
|
||||
/// Log-and-drop wrapper: no social lookup may ever fail the run (notes §3).
|
||||
fn best_effort(
|
||||
platform: &'static str,
|
||||
url: &str,
|
||||
result: Result<Option<SocialRef>, SocialError>,
|
||||
) -> Option<SocialRef> {
|
||||
match result {
|
||||
Ok(found) => found,
|
||||
Err(e) => {
|
||||
tracing::debug!(platform, url, "social lookup failed: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load cached refs for an article, ignoring rows older than [`CACHE_TTL_HOURS`].
|
||||
pub async fn cached_refs(db: &Db, article_id: ArticleId) -> Vec<SocialRef> {
|
||||
match db.social_for_article(article_id).await {
|
||||
Ok(refs) => {
|
||||
let now = Timestamp::now();
|
||||
refs.into_iter().filter(|r| is_fresh(r, now)).collect()
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(article = article_id, "reading social cache failed: {e}");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True while a cached row is younger than [`CACHE_TTL_HOURS`] (§3.4).
|
||||
pub fn is_fresh(social_ref: &SocialRef, now: Timestamp) -> bool {
|
||||
// A negative age (clock skew, a row written moments ago) is fresh too.
|
||||
now.as_second() - social_ref.fetched_at.as_second() < CACHE_TTL_HOURS * 3600
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{ExtractMethod, SourceRef};
|
||||
|
||||
fn ts(s: &str) -> Timestamp {
|
||||
s.parse().unwrap()
|
||||
}
|
||||
|
||||
fn article(url: &str) -> Article {
|
||||
Article {
|
||||
id: 1,
|
||||
canonical_url: url.into(),
|
||||
title: "T".into(),
|
||||
best_entry_id: 1,
|
||||
content_html: "<p>x</p>".into(),
|
||||
word_count: 1,
|
||||
excerpt_only: false,
|
||||
image_count: 0,
|
||||
sources: vec![SourceRef {
|
||||
entry_id: 1,
|
||||
feed_id: 1,
|
||||
feed_title: "Feed".into(),
|
||||
category: None,
|
||||
kind: SourceKind::Feed,
|
||||
}],
|
||||
first_seen: ts("2026-08-15T05:00:00Z"),
|
||||
url: url.into(),
|
||||
author: None,
|
||||
feed_id: 1,
|
||||
feed_title: "Feed".into(),
|
||||
category: None,
|
||||
published_at: None,
|
||||
comments_url: None,
|
||||
image_urls: vec![],
|
||||
social: vec![],
|
||||
extract_method: ExtractMethod::Miniflux,
|
||||
}
|
||||
}
|
||||
|
||||
fn social(source: SocialSource, fetched_at: Timestamp) -> SocialRef {
|
||||
SocialRef {
|
||||
article_id: 1,
|
||||
source,
|
||||
item_id: Some("1".into()),
|
||||
score: 10,
|
||||
num_comments: 5,
|
||||
item_url: None,
|
||||
fetched_at,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_freshness_window() {
|
||||
let now = ts("2026-08-15T12:00:00Z");
|
||||
assert!(is_fresh(&social(SocialSource::Hn, now), now));
|
||||
assert!(is_fresh(
|
||||
&social(SocialSource::Hn, ts("2026-08-14T13:00:00Z")),
|
||||
now
|
||||
));
|
||||
assert!(!is_fresh(
|
||||
&social(SocialSource::Hn, ts("2026-08-14T11:00:00Z")),
|
||||
now
|
||||
));
|
||||
// Clock skew (a row from the future) counts as fresh, never as ancient.
|
||||
assert!(is_fresh(
|
||||
&social(SocialSource::Hn, ts("2026-08-15T13:00:00Z")),
|
||||
now
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_targets_come_from_comments_urls_and_sources() {
|
||||
let mut a = article("https://blog.dev/post");
|
||||
a.comments_url = Some("https://news.ycombinator.com/item?id=41234567".into());
|
||||
let l = Lookup::for_article(&a);
|
||||
assert_eq!(l.hn_story_id.as_deref(), Some("41234567"));
|
||||
assert_eq!(l.lobsters_story_id, None);
|
||||
assert_eq!(l.canonical_url, "https://blog.dev/post");
|
||||
|
||||
a.comments_url = Some("https://lobste.rs/s/abcdef/a_deep_dive".into());
|
||||
let l = Lookup::for_article(&a);
|
||||
assert_eq!(l.hn_story_id, None);
|
||||
assert_eq!(l.lobsters_story_id.as_deref(), Some("abcdef"));
|
||||
|
||||
// No comments URL and no lobsters origin: no lobsters lookup at all (§7).
|
||||
a.comments_url = None;
|
||||
assert_eq!(Lookup::for_article(&a).lobsters_story_id, None);
|
||||
|
||||
// Lobsters-origin entry whose URL is the story itself.
|
||||
a.url = "https://lobste.rs/s/zzzzzz/title".into();
|
||||
a.sources[0].kind = SourceKind::Lobsters;
|
||||
assert_eq!(
|
||||
Lookup::for_article(&a).lobsters_story_id.as_deref(),
|
||||
Some("zzzzzz")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pacer_serializes_requests() {
|
||||
let pacer = Pacer::new(Duration::from_millis(30));
|
||||
let started = std::time::Instant::now();
|
||||
pacer.tick().await;
|
||||
pacer.tick().await;
|
||||
assert!(started.elapsed() >= Duration::from_millis(30));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cache_reads_skip_stale_rows() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("t.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
db.upsert_entry(&crate::types::Entry {
|
||||
id: 1,
|
||||
feed_id: 1,
|
||||
feed_title: Some("Feed".into()),
|
||||
category: None,
|
||||
title: "T".into(),
|
||||
url: "https://blog.dev/post".into(),
|
||||
canonical_url: Some("https://blog.dev/post".into()),
|
||||
author: None,
|
||||
published_at: None,
|
||||
comments_url: None,
|
||||
raw_content: "<p>x</p>".into(),
|
||||
fetched_at: Timestamp::now(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let id = db
|
||||
.upsert_article(&article("https://blog.dev/post"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
db.upsert_social(&SocialRef {
|
||||
article_id: id,
|
||||
fetched_at: Timestamp::now(),
|
||||
..social(SocialSource::Hn, Timestamp::now())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
db.upsert_social(&SocialRef {
|
||||
article_id: id,
|
||||
fetched_at: Timestamp::now() - jiff::Span::new().hours(48),
|
||||
..social(SocialSource::Reddit, Timestamp::now())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let fresh = cached_refs(&db, id).await;
|
||||
assert_eq!(fresh.len(), 1);
|
||||
assert_eq!(fresh[0].source, SocialSource::Hn);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
//! Reddit via the public JSON endpoints (spec §3.4, §3.7).
|
||||
//!
|
||||
//! Requires the descriptive User-Agent from [`crate::http::USER_AGENT`], ~1 req/s
|
||||
//! pacing, and must degrade gracefully on 429 (social data is best-effort).
|
||||
|
||||
use jiff::Timestamp;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::SocialError;
|
||||
use crate::types::{ArticleId, Comment, CommentThread, SocialRef, SocialSource};
|
||||
|
||||
/// `GET /api/info.json?url=…` — finds submissions of a given URL (§3.4).
|
||||
pub const INFO_URL: &str = "https://www.reddit.com/api/info.json";
|
||||
pub const BASE_URL: &str = "https://www.reddit.com";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct Listing {
|
||||
#[serde(default)]
|
||||
data: ListingData,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
struct ListingData {
|
||||
#[serde(default)]
|
||||
children: Vec<Child>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct Child {
|
||||
#[serde(default)]
|
||||
kind: String,
|
||||
#[serde(default)]
|
||||
data: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A `t3` submission (only the fields §3.4 uses).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct Post {
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
#[serde(default)]
|
||||
pub id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub score: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub num_comments: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub permalink: Option<String>,
|
||||
#[serde(default)]
|
||||
pub subreddit: Option<String>,
|
||||
}
|
||||
|
||||
impl Post {
|
||||
/// Absolute link a human can open (§3.4).
|
||||
pub fn item_url(&self) -> Option<String> {
|
||||
self.permalink.as_ref().map(|p| format!("{BASE_URL}{p}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
struct RawComment {
|
||||
#[serde(default)]
|
||||
author: Option<String>,
|
||||
#[serde(default)]
|
||||
score: Option<i64>,
|
||||
#[serde(default)]
|
||||
body: Option<String>,
|
||||
#[serde(default)]
|
||||
stickied: bool,
|
||||
#[serde(default)]
|
||||
replies: serde_json::Value,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure parsing (unit-tested against `tests/fixtures/reddit_*.json`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Every `t3` submission in an `api/info.json` response.
|
||||
pub fn parse_posts(body: &str) -> Result<Vec<Post>, SocialError> {
|
||||
let listing: Listing = serde_json::from_str(body).map_err(|e| SocialError::Unexpected {
|
||||
platform: "reddit",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
Ok(listing
|
||||
.data
|
||||
.children
|
||||
.into_iter()
|
||||
.filter(|c| c.kind == "t3")
|
||||
.filter_map(|c| serde_json::from_value::<Post>(c.data).ok())
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Best (highest score) submission in an `api/info.json` response (§3.4).
|
||||
pub fn parse_info_response(
|
||||
body: &str,
|
||||
article_id: ArticleId,
|
||||
fetched_at: Timestamp,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let posts = parse_posts(body)?;
|
||||
let Some(best) = posts.into_iter().max_by_key(|p| p.score.unwrap_or(0)) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(SocialRef {
|
||||
article_id,
|
||||
source: SocialSource::Reddit,
|
||||
item_id: best.name.clone().or_else(|| best.id.clone()),
|
||||
score: best.score.unwrap_or(0),
|
||||
num_comments: best.num_comments.unwrap_or(0),
|
||||
item_url: best.item_url(),
|
||||
fetched_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Parse a `{permalink}.json` body — `[post listing, comment listing]` (§3.7).
|
||||
pub fn parse_comments_response(body: &str, permalink: &str) -> Result<CommentThread, SocialError> {
|
||||
let listings: Vec<Listing> =
|
||||
serde_json::from_str(body).map_err(|e| SocialError::Unexpected {
|
||||
platform: "reddit",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
let total = listings
|
||||
.first()
|
||||
.and_then(|l| l.data.children.first())
|
||||
.and_then(|c| serde_json::from_value::<Post>(c.data.clone()).ok())
|
||||
.and_then(|p| p.num_comments)
|
||||
.unwrap_or(0);
|
||||
let comments = listings
|
||||
.get(1)
|
||||
.map(|l| map_children(&l.data.children, 0))
|
||||
.unwrap_or_default();
|
||||
Ok(CommentThread {
|
||||
source: SocialSource::Reddit,
|
||||
item_url: format!("{BASE_URL}{permalink}"),
|
||||
total_comments: total,
|
||||
comments,
|
||||
})
|
||||
}
|
||||
|
||||
fn map_children(children: &[Child], depth: usize) -> Vec<Comment> {
|
||||
children
|
||||
.iter()
|
||||
.filter(|c| c.kind == "t1")
|
||||
.filter_map(|c| serde_json::from_value::<RawComment>(c.data.clone()).ok())
|
||||
.filter(|c| !c.stickied)
|
||||
.filter_map(|raw| {
|
||||
let body = raw.body.as_deref().unwrap_or("").trim().to_string();
|
||||
if body.is_empty() || body == "[removed]" || body == "[deleted]" {
|
||||
return None;
|
||||
}
|
||||
let kids = match serde_json::from_value::<Listing>(raw.replies.clone()) {
|
||||
Ok(listing) => map_children(&listing.data.children, depth + 1),
|
||||
// `replies` is `""` when a comment has none.
|
||||
Err(_) => Vec::new(),
|
||||
};
|
||||
Some(Comment {
|
||||
author: raw.author.clone().unwrap_or_else(|| "[deleted]".into()),
|
||||
points: raw.score,
|
||||
text_html: crate::extract::sanitize(&markdown_to_html(&body)),
|
||||
depth,
|
||||
children: kids,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Reddit comment bodies are markdown; the EPUB only needs paragraphs (§3.7).
|
||||
fn markdown_to_html(body: &str) -> String {
|
||||
body.split("\n\n")
|
||||
.map(str::trim)
|
||||
.filter(|p| !p.is_empty())
|
||||
.map(|p| format!("<p>{}</p>", escape_text(p)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn escape_text(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Network
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn get_text(
|
||||
http: &reqwest::Client,
|
||||
url: &str,
|
||||
query: &[(&str, &str)],
|
||||
) -> Result<String, SocialError> {
|
||||
let response = http
|
||||
.get(url)
|
||||
.header(reqwest::header::USER_AGENT, crate::http::USER_AGENT)
|
||||
.query(query)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.as_u16() == 403 {
|
||||
return Err(SocialError::RateLimited("reddit"));
|
||||
}
|
||||
Ok(response.error_for_status()?.text().await?)
|
||||
}
|
||||
|
||||
/// Best (highest score) Reddit post for `canonical_url` (§3.4).
|
||||
///
|
||||
/// Returns `None` when no submission exists or the API rate-limited us.
|
||||
pub async fn lookup_by_url(
|
||||
http: &reqwest::Client,
|
||||
canonical_url: &str,
|
||||
article_id: ArticleId,
|
||||
) -> Result<Option<SocialRef>, SocialError> {
|
||||
let body = get_text(http, INFO_URL, &[("url", canonical_url), ("raw_json", "1")]).await?;
|
||||
parse_info_response(&body, article_id, Timestamp::now())
|
||||
}
|
||||
|
||||
/// `GET {permalink}.json?limit=100&depth=3&sort=top` → comment tree (§3.7).
|
||||
pub async fn fetch_comments(
|
||||
http: &reqwest::Client,
|
||||
permalink: &str,
|
||||
) -> Result<CommentThread, SocialError> {
|
||||
let path = permalink.trim_end_matches('/');
|
||||
let url = format!("{BASE_URL}{path}.json");
|
||||
let body = get_text(
|
||||
http,
|
||||
&url,
|
||||
&[
|
||||
("limit", "100"),
|
||||
("depth", "3"),
|
||||
("sort", "top"),
|
||||
("raw_json", "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
parse_comments_response(&body, permalink)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const INFO: &str = include_str!("../../tests/fixtures/m2_reddit_info.json");
|
||||
const INFO_EMPTY: &str = include_str!("../../tests/fixtures/m2_reddit_info_empty.json");
|
||||
const COMMENTS: &str = include_str!("../../tests/fixtures/m2_reddit_comments.json");
|
||||
|
||||
fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z".parse().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_response_picks_the_highest_scoring_post() {
|
||||
let got = parse_info_response(INFO, 11, ts())
|
||||
.unwrap()
|
||||
.expect("a post");
|
||||
assert_eq!(got.article_id, 11);
|
||||
assert_eq!(got.source, SocialSource::Reddit);
|
||||
assert_eq!(got.item_id.as_deref(), Some("t3_1abcd2"));
|
||||
assert_eq!(got.score, 845);
|
||||
assert_eq!(got.num_comments, 231);
|
||||
assert_eq!(
|
||||
got.item_url.as_deref(),
|
||||
Some("https://www.reddit.com/r/programming/comments/1abcd2/a_deep_dive_into_btrees/")
|
||||
);
|
||||
assert_eq!(parse_posts(INFO).unwrap().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_info_response_is_not_an_error() {
|
||||
assert!(parse_info_response(INFO_EMPTY, 1, ts()).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_json_is_reported_not_panicked() {
|
||||
assert!(matches!(
|
||||
parse_info_response("<html>rate limited</html>", 1, ts()).unwrap_err(),
|
||||
SocialError::Unexpected {
|
||||
platform: "reddit",
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_listing_becomes_a_tree() {
|
||||
let thread =
|
||||
parse_comments_response(COMMENTS, "/r/programming/comments/1abcd2/x/").unwrap();
|
||||
assert_eq!(thread.source, SocialSource::Reddit);
|
||||
assert_eq!(thread.total_comments, 231);
|
||||
assert_eq!(
|
||||
thread.item_url,
|
||||
"https://www.reddit.com/r/programming/comments/1abcd2/x/"
|
||||
);
|
||||
// `more` stubs, the stickied automod post and the removed comment are dropped.
|
||||
assert_eq!(thread.comments.len(), 1);
|
||||
|
||||
let top = &thread.comments[0];
|
||||
assert_eq!(top.author, "index_nerd");
|
||||
assert_eq!(top.points, Some(412));
|
||||
assert_eq!(top.depth, 0);
|
||||
assert!(
|
||||
top.text_html
|
||||
.starts_with("<p>Fan-out is the whole ballgame")
|
||||
);
|
||||
assert_eq!(top.children.len(), 1);
|
||||
assert_eq!(top.children[0].author, "pagecache");
|
||||
assert_eq!(top.children[0].depth, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_bodies_are_escaped_into_paragraphs() {
|
||||
let html = markdown_to_html("first & <b>bold</b>\n\nsecond");
|
||||
assert_eq!(
|
||||
html,
|
||||
"<p>first & <b>bold</b></p><p>second</p>"
|
||||
);
|
||||
assert!(!crate::extract::sanitize(&html).contains("<b>"));
|
||||
}
|
||||
}
|
||||
+702
@@ -0,0 +1,702 @@
|
||||
//! Shared domain types — the contract between pipeline stages (spec §2, §3.13).
|
||||
//!
|
||||
//! Every stage module (`dedupe`, `extract`, `social`, `curate`, `comments`, `epub`,
|
||||
//! `publish`, `server`, `world`) codes against the types defined here so that the
|
||||
//! stages can be implemented independently. Keep this module free of I/O.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use jiff::civil::Date;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Miniflux entry id (also our `entries.id`).
|
||||
pub type EntryId = i64;
|
||||
/// Row id of a deduped article cluster (`articles.id`).
|
||||
pub type ArticleId = i64;
|
||||
/// Miniflux feed id.
|
||||
pub type FeedId = i64;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ingest (§3.1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A raw Miniflux entry as persisted in the `entries` table (§3.1, §3.13).
|
||||
///
|
||||
/// `canonical_url` is `None` at ingest time; the dedupe stage
|
||||
/// ([`crate::dedupe::canonical_url`], §3.2) fills it in.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Entry {
|
||||
pub id: EntryId,
|
||||
pub feed_id: FeedId,
|
||||
pub feed_title: Option<String>,
|
||||
pub category: Option<String>,
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
pub canonical_url: Option<String>,
|
||||
pub author: Option<String>,
|
||||
pub published_at: Option<Timestamp>,
|
||||
pub comments_url: Option<String>,
|
||||
pub raw_content: String,
|
||||
pub fetched_at: Timestamp,
|
||||
}
|
||||
|
||||
/// Where an article reached us from — a curation signal in its own right (§3.2, §3.5).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SourceKind {
|
||||
/// Arrived via a Scour interest feed — it already matched a stated interest.
|
||||
Scour,
|
||||
/// Arrived via the Hacker News frontpage feed (hnrss et al).
|
||||
HnFrontpage,
|
||||
/// Arrived via a lobste.rs feed.
|
||||
Lobsters,
|
||||
/// Arrived via a Reddit feed.
|
||||
Reddit,
|
||||
/// A plain blog/publication feed.
|
||||
Feed,
|
||||
}
|
||||
|
||||
/// One feed that carried this story; an article cluster keeps the union of them (§3.2).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SourceRef {
|
||||
pub entry_id: EntryId,
|
||||
pub feed_id: FeedId,
|
||||
pub feed_title: String,
|
||||
pub category: Option<String>,
|
||||
pub kind: SourceKind,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dedupe + extraction (§3.2, §3.3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// How an article's body text was obtained (§3.3).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ExtractMethod {
|
||||
/// Miniflux's stored content already looked like full text.
|
||||
Miniflux,
|
||||
/// Fetched the article URL and ran readability over it.
|
||||
Readability,
|
||||
/// Only a feed summary/excerpt was available.
|
||||
Excerpt,
|
||||
}
|
||||
|
||||
/// Result of the content-extraction stage for one article (§3.3).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Extracted {
|
||||
/// Sanitized XHTML-safe body markup.
|
||||
pub content_html: String,
|
||||
pub word_count: i64,
|
||||
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
|
||||
pub excerpt_only: bool,
|
||||
/// Absolute image URLs referenced by the body, capped at 12 (§3.3).
|
||||
pub image_urls: Vec<String>,
|
||||
pub method: ExtractMethod,
|
||||
}
|
||||
|
||||
/// A deduped story cluster: the unit everything downstream operates on (§3.2).
|
||||
///
|
||||
/// Persisted fields map to the `articles` table; the remaining fields are
|
||||
/// denormalized from the best entry / `social` table for convenience and are
|
||||
/// re-hydrated by [`crate::db`] when an article is loaded.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Article {
|
||||
/// Zero until the row has been inserted.
|
||||
pub id: ArticleId,
|
||||
pub canonical_url: String,
|
||||
pub title: String,
|
||||
/// The entry whose content we kept (the richest one).
|
||||
pub best_entry_id: EntryId,
|
||||
pub content_html: String,
|
||||
pub word_count: i64,
|
||||
pub excerpt_only: bool,
|
||||
pub image_count: i64,
|
||||
/// Union of the feeds that carried this story (`articles.sources_json`).
|
||||
pub sources: Vec<SourceRef>,
|
||||
pub first_seen: Timestamp,
|
||||
|
||||
// --- denormalized, not stored on `articles` ---
|
||||
pub url: String,
|
||||
pub author: Option<String>,
|
||||
pub feed_id: FeedId,
|
||||
pub feed_title: String,
|
||||
pub category: Option<String>,
|
||||
pub published_at: Option<Timestamp>,
|
||||
pub comments_url: Option<String>,
|
||||
pub image_urls: Vec<String>,
|
||||
pub social: Vec<SocialRef>,
|
||||
pub extract_method: ExtractMethod,
|
||||
}
|
||||
|
||||
impl Article {
|
||||
/// Estimated reading time at 220 wpm, minimum one minute (§3.10).
|
||||
pub fn reading_minutes(&self) -> i64 {
|
||||
reading_minutes(self.word_count)
|
||||
}
|
||||
|
||||
/// Composite social proof across all sources (§3.4).
|
||||
pub fn social_score(&self) -> f64 {
|
||||
composite_social_score(&self.social)
|
||||
}
|
||||
|
||||
/// True when this story arrived via a feed of the given kind (§3.5).
|
||||
pub fn came_via(&self, kind: SourceKind) -> bool {
|
||||
self.sources.iter().any(|s| s.kind == kind)
|
||||
}
|
||||
|
||||
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
|
||||
pub fn chapter_id(&self) -> String {
|
||||
format!("art-{}", self.best_entry_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Estimated reading time at 220 wpm, minimum one minute.
|
||||
pub fn reading_minutes(word_count: i64) -> i64 {
|
||||
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Social proof (§3.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Social platforms we look up. `X` is reserved: no free API today (§3.4, §7).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SocialSource {
|
||||
Hn,
|
||||
Lobsters,
|
||||
Reddit,
|
||||
X,
|
||||
}
|
||||
|
||||
impl SocialSource {
|
||||
/// Value stored in `social.source` (matches the CHECK constraint).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
SocialSource::Hn => "hn",
|
||||
SocialSource::Lobsters => "lobsters",
|
||||
SocialSource::Reddit => "reddit",
|
||||
SocialSource::X => "x",
|
||||
}
|
||||
}
|
||||
|
||||
/// Human-readable label used in chapter titles and stat lines (§3.7, §3.10).
|
||||
pub fn display_name(self) -> &'static str {
|
||||
match self {
|
||||
SocialSource::Hn => "HN",
|
||||
SocialSource::Lobsters => "Lobsters",
|
||||
SocialSource::Reddit => "Reddit",
|
||||
SocialSource::X => "X",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"hn" => Some(SocialSource::Hn),
|
||||
"lobsters" => Some(SocialSource::Lobsters),
|
||||
"reddit" => Some(SocialSource::Reddit),
|
||||
"x" => Some(SocialSource::X),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SocialSource {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// A cached social-proof lookup for one article on one platform (`social` table).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SocialRef {
|
||||
pub article_id: ArticleId,
|
||||
pub source: SocialSource,
|
||||
/// Platform item id: HN `objectID`, lobsters story id, reddit fullname.
|
||||
pub item_id: Option<String>,
|
||||
pub score: i64,
|
||||
pub num_comments: i64,
|
||||
/// Link a human can open (HN item page, lobsters story, reddit permalink).
|
||||
pub item_url: Option<String>,
|
||||
pub fetched_at: Timestamp,
|
||||
}
|
||||
|
||||
/// `log10(1+hn) + 0.7*log10(1+reddit) + log10(1+lobsters) + 0.5*log10(1+comments)` (§3.4).
|
||||
///
|
||||
/// Lives here rather than in `social/` because both the pre-filter and the EPUB
|
||||
/// stat line need it.
|
||||
pub fn composite_social_score(refs: &[SocialRef]) -> f64 {
|
||||
let mut score = 0.0;
|
||||
let mut comments = 0i64;
|
||||
for r in refs {
|
||||
let points = (r.score.max(0)) as f64;
|
||||
let weight = match r.source {
|
||||
SocialSource::Hn | SocialSource::Lobsters => 1.0,
|
||||
SocialSource::Reddit => 0.7,
|
||||
SocialSource::X => 0.0,
|
||||
};
|
||||
score += weight * (1.0 + points).log10();
|
||||
comments += r.num_comments.max(0);
|
||||
}
|
||||
score + 0.5 * (1.0 + comments as f64).log10()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Curation (§3.5, §3.6)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// DeepSeek stage-A output for one article (§3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct LlmScore {
|
||||
/// 0–10.
|
||||
pub score: f64,
|
||||
pub category: String,
|
||||
/// ≤ 20 words.
|
||||
pub rationale: String,
|
||||
#[serde(default)]
|
||||
pub is_paywalled_guess: bool,
|
||||
}
|
||||
|
||||
/// An article carrying every ranking signal computed so far (§3.5, §3.6).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScoredArticle {
|
||||
pub article: Article,
|
||||
/// Heuristic pre-filter score, 0–100 (§3.5).
|
||||
pub prefilter_score: f64,
|
||||
/// Cached [`composite_social_score`] for the article.
|
||||
pub social_score: f64,
|
||||
/// Beta-smoothed per-feed upvote rate applied by the pre-filter (§3.9).
|
||||
pub feed_prior: f64,
|
||||
/// `None` until stage A has run (or when `--skip-llm`).
|
||||
pub llm: Option<LlmScore>,
|
||||
/// From `curation.always_include_feeds`: may be scored but never dropped (§3.5).
|
||||
pub auto_include: bool,
|
||||
}
|
||||
|
||||
impl ScoredArticle {
|
||||
/// Ranking key for stage B: LLM score weighted with social proof and priors (§3.6).
|
||||
pub fn combined_score(&self) -> f64 {
|
||||
let llm = self.llm.as_ref().map(|l| l.score).unwrap_or(0.0);
|
||||
llm * 10.0 + self.social_score * 4.0 + self.feed_prior * 10.0 + self.prefilter_score * 0.1
|
||||
}
|
||||
}
|
||||
|
||||
/// One selected article with its section placement (§3.6 stage B).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Pick {
|
||||
pub article: Article,
|
||||
/// One of `curation.sections` (or the reserved `World Briefing`).
|
||||
pub section: String,
|
||||
/// Order within the section, ascending.
|
||||
pub position: i64,
|
||||
pub is_lead: bool,
|
||||
/// Newspaper-abstract summary from stage C; `None` until editorial runs.
|
||||
pub summary: Option<String>,
|
||||
pub llm: Option<LlmScore>,
|
||||
/// Rendered comment chapter, when the article had social refs (§3.7).
|
||||
pub discussion: Option<Discussion>,
|
||||
}
|
||||
|
||||
/// The day's final lineup: 15–25 picks grouped into sections (§3.6 stage B).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Lineup {
|
||||
pub date: Date,
|
||||
/// Sorted by (section order, position).
|
||||
pub picks: Vec<Pick>,
|
||||
/// Section names in issue order; empty sections are omitted (§3.6).
|
||||
pub section_order: Vec<String>,
|
||||
}
|
||||
|
||||
impl Lineup {
|
||||
/// Picks belonging to `section`, in position order.
|
||||
pub fn section_picks(&self, section: &str) -> Vec<&Pick> {
|
||||
let mut v: Vec<&Pick> = self.picks.iter().filter(|p| p.section == section).collect();
|
||||
v.sort_by_key(|p| p.position);
|
||||
v
|
||||
}
|
||||
|
||||
pub fn lead(&self) -> Option<&Pick> {
|
||||
self.picks.iter().find(|p| p.is_lead)
|
||||
}
|
||||
|
||||
pub fn total_words(&self) -> i64 {
|
||||
self.picks.iter().map(|p| p.article.word_count).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage-C editorial output (§3.6).
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Editorial {
|
||||
/// "From the Editor", 250–400 words, already sanitized XHTML.
|
||||
pub front_page_html: String,
|
||||
/// Section name → 2–3 sentence intro.
|
||||
pub section_intros: BTreeMap<String, String>,
|
||||
/// Article id → 2–3 sentence newspaper abstract.
|
||||
pub summaries: BTreeMap<ArticleId, String>,
|
||||
}
|
||||
|
||||
/// The taste profile that forms the DeepSeek system prompt (§3.6, `kv`).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TasteProfile {
|
||||
/// Full ~600-word prompt document.
|
||||
pub text: String,
|
||||
pub version: i64,
|
||||
pub built_at: Timestamp,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Comments (§3.7)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One comment node in a discussion tree (§3.7).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Comment {
|
||||
pub author: String,
|
||||
pub points: Option<i64>,
|
||||
/// Sanitized comment body, ellipsized to 1,200 chars.
|
||||
pub text_html: String,
|
||||
/// 0 for top-level; rendering stops at depth 3.
|
||||
pub depth: usize,
|
||||
pub children: Vec<Comment>,
|
||||
}
|
||||
|
||||
/// The comment tree fetched from one platform for one article (§3.7).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CommentThread {
|
||||
pub source: SocialSource,
|
||||
pub item_url: String,
|
||||
pub total_comments: i64,
|
||||
/// Top ~8 top-level threads by score.
|
||||
pub comments: Vec<Comment>,
|
||||
}
|
||||
|
||||
/// A rendered discussion chapter: one per article, HN → Lobsters → Reddit (§3.7).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Discussion {
|
||||
pub article_id: ArticleId,
|
||||
/// Chapter id, `disc-{entry_id}` (implementation notes §12).
|
||||
pub chapter_id: String,
|
||||
pub threads: Vec<CommentThread>,
|
||||
}
|
||||
|
||||
impl Discussion {
|
||||
pub fn total_comments(&self) -> i64 {
|
||||
self.threads.iter().map(|t| t.total_comments).sum()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// World briefing (§3.8)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Wikipedia Current Events portal digest for one day (§3.8).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct WorldBriefing {
|
||||
pub date: Date,
|
||||
/// Portal URL the content came from (also used for CC BY-SA attribution).
|
||||
pub source_url: String,
|
||||
/// Sanitized `<ul>`-style markup of the day's events.
|
||||
pub body_html: String,
|
||||
}
|
||||
|
||||
/// Reserved section name for [`WorldBriefing`] — never offered to the LLM (§3.6).
|
||||
pub const WORLD_BRIEFING_SECTION: &str = "World Briefing";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Issue assembly (§3.10)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Which of the two editions is being built (§3.10).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Edition {
|
||||
/// 1200px images, full CSS.
|
||||
Standard,
|
||||
/// Grayscale, 480×800, simplified CSS — input for the XTC converter.
|
||||
X4,
|
||||
}
|
||||
|
||||
impl Edition {
|
||||
/// Filename suffix: `""` / `" (X4)"` (§3.11).
|
||||
pub fn file_suffix(self) -> &'static str {
|
||||
match self {
|
||||
Edition::Standard => "",
|
||||
Edition::X4 => " (X4)",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Issue-level metadata rendered on the cover, front page and OPF (§3.10).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct IssueMeta {
|
||||
pub date: Date,
|
||||
/// Days since the first issue; EPUB3 `group-position`.
|
||||
pub issue_number: i64,
|
||||
pub generated_at: Timestamp,
|
||||
/// "Friday, August 15, 2026".
|
||||
pub display_date: String,
|
||||
pub article_count: i64,
|
||||
pub section_count: i64,
|
||||
pub total_words: i64,
|
||||
pub reading_minutes: i64,
|
||||
}
|
||||
|
||||
impl IssueMeta {
|
||||
/// The issue's name, without an edition tag: "The Daily EPUB — 2026-08-15".
|
||||
pub fn title(&self) -> String {
|
||||
format!("The Daily EPUB — {}", self.date)
|
||||
}
|
||||
|
||||
/// `dc:title` for one edition: [`title`](Self::title) plus the edition tag
|
||||
/// (§3.10).
|
||||
///
|
||||
/// Both editions land in the same BookOrbit library, and BookOrbit — like
|
||||
/// every OPDS client — lists books by `dc:title`. Carrying the distinction
|
||||
/// only in the filename makes them indistinguishable everywhere except the
|
||||
/// per-book file listing, so the title and the filename share one suffix.
|
||||
pub fn title_for(&self, edition: Edition) -> String {
|
||||
format!("{}{}", self.title(), edition.file_suffix())
|
||||
}
|
||||
|
||||
/// "22 articles · ~1h 45m read · 6 sections" (§3.10).
|
||||
pub fn stats_line(&self) -> String {
|
||||
let (h, m) = (self.reading_minutes / 60, self.reading_minutes % 60);
|
||||
let time = if h > 0 {
|
||||
format!("~{h}h {m}m read")
|
||||
} else {
|
||||
format!("~{m}m read")
|
||||
};
|
||||
format!(
|
||||
"{} articles · {} · {} sections",
|
||||
self.article_count, time, self.section_count
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything the EPUB builder needs; fully materialized before rendering (§3.10).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Issue {
|
||||
pub meta: IssueMeta,
|
||||
pub lineup: Lineup,
|
||||
pub editorial: Editorial,
|
||||
pub world_briefing: Option<WorldBriefing>,
|
||||
/// Colophon facts: models used, token cost, feed counts (§3.10).
|
||||
pub colophon: Colophon,
|
||||
}
|
||||
|
||||
/// Back-matter facts printed in the colophon chapter (§3.10).
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Colophon {
|
||||
pub model: String,
|
||||
pub entries_fetched: i64,
|
||||
pub feeds_seen: i64,
|
||||
pub candidates: i64,
|
||||
pub cost_usd: f64,
|
||||
pub generator_version: String,
|
||||
}
|
||||
|
||||
/// A downloaded, re-encoded image embedded in an edition (§3.10 images).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ImageAsset {
|
||||
/// Manifest id, unique within the issue.
|
||||
pub id: String,
|
||||
/// Path inside the EPUB, e.g. `images/art-1234-0.jpg`.
|
||||
pub href: String,
|
||||
pub mime: String,
|
||||
pub data: Vec<u8>,
|
||||
pub alt: String,
|
||||
pub caption: Option<String>,
|
||||
/// The original remote URL, used to rewrite `<img src>`.
|
||||
pub source_url: String,
|
||||
}
|
||||
|
||||
/// A file produced by the build/publish stages (§3.11).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Artifact {
|
||||
pub edition: Edition,
|
||||
pub path: PathBuf,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Feedback (§3.9)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// 👍 / 👎 stored as `+1` / `-1` in `ratings.vote` (§3.9).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Vote {
|
||||
Up,
|
||||
Down,
|
||||
}
|
||||
|
||||
impl Vote {
|
||||
pub fn as_i64(self) -> i64 {
|
||||
match self {
|
||||
Vote::Up => 1,
|
||||
Vote::Down => -1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Path segment used in rating links: `up` / `down` (§3.9).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Vote::Up => "up",
|
||||
Vote::Down => "down",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"up" => Some(Vote::Up),
|
||||
"down" => Some(Vote::Down),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A recorded reader vote (`ratings` table, §3.9).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Rating {
|
||||
pub issue_date: Date,
|
||||
pub article_id: ArticleId,
|
||||
pub vote: Vote,
|
||||
pub rated_at: Timestamp,
|
||||
}
|
||||
|
||||
/// Beta-smoothed per-feed upvote rate used by the pre-filter (`feed_priors`, §3.9).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct FeedPrior {
|
||||
pub feed_id: FeedId,
|
||||
pub upvotes: i64,
|
||||
pub downvotes: i64,
|
||||
pub included: i64,
|
||||
}
|
||||
|
||||
impl FeedPrior {
|
||||
/// `(up + 1) / (up + down + 2)` — 0.5 with no evidence (§3.9).
|
||||
pub fn rate(&self) -> f64 {
|
||||
(self.upvotes + 1) as f64 / (self.upvotes + self.downvotes + 2) as f64
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM accounting (§3.6 cost guardrail)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Token counters accumulated across every DeepSeek call in a run (§3.6).
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct TokenUsage {
|
||||
/// Cache-miss input tokens (billed at the full input rate).
|
||||
pub input_tokens: i64,
|
||||
/// Prefix-cache hits (billed at the cached rate).
|
||||
pub cached_tokens: i64,
|
||||
pub output_tokens: i64,
|
||||
}
|
||||
|
||||
impl TokenUsage {
|
||||
pub fn add(&mut self, other: TokenUsage) {
|
||||
self.input_tokens += other.input_tokens;
|
||||
self.cached_tokens += other.cached_tokens;
|
||||
self.output_tokens += other.output_tokens;
|
||||
}
|
||||
|
||||
/// USD cost given the per-1M-token prices from `[deepseek]` config (§3.6).
|
||||
pub fn cost_usd(&self, price_input: f64, price_cached: f64, price_output: f64) -> f64 {
|
||||
(self.input_tokens as f64 * price_input
|
||||
+ self.cached_tokens as f64 * price_cached
|
||||
+ self.output_tokens as f64 * price_output)
|
||||
/ 1_000_000.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ts() -> Timestamp {
|
||||
"2026-08-15T05:30:00Z".parse().unwrap()
|
||||
}
|
||||
|
||||
fn social(source: SocialSource, score: i64, comments: i64) -> SocialRef {
|
||||
SocialRef {
|
||||
article_id: 1,
|
||||
source,
|
||||
item_id: Some("1".into()),
|
||||
score,
|
||||
num_comments: comments,
|
||||
item_url: None,
|
||||
fetched_at: ts(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn composite_social_score_matches_spec_formula() {
|
||||
let refs = vec![
|
||||
social(SocialSource::Hn, 342, 210),
|
||||
social(SocialSource::Reddit, 99, 40),
|
||||
];
|
||||
let expected = (343f64).log10() + 0.7 * (100f64).log10() + 0.5 * (251f64).log10();
|
||||
assert!((composite_social_score(&refs) - expected).abs() < 1e-9);
|
||||
assert_eq!(composite_social_score(&[]), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_prior_is_beta_smoothed() {
|
||||
assert_eq!(FeedPrior::default().rate(), 0.5);
|
||||
let p = FeedPrior {
|
||||
feed_id: 1,
|
||||
upvotes: 3,
|
||||
downvotes: 1,
|
||||
included: 4,
|
||||
};
|
||||
assert!((p.rate() - 4.0 / 6.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_line_and_reading_time() {
|
||||
assert_eq!(reading_minutes(0), 1);
|
||||
assert_eq!(reading_minutes(440), 2);
|
||||
let meta = IssueMeta {
|
||||
date: "2026-08-15".parse().unwrap(),
|
||||
issue_number: 1,
|
||||
generated_at: ts(),
|
||||
display_date: "Friday, August 15, 2026".into(),
|
||||
article_count: 22,
|
||||
section_count: 6,
|
||||
total_words: 23_000,
|
||||
reading_minutes: 105,
|
||||
};
|
||||
assert_eq!(meta.stats_line(), "22 articles · ~1h 45m read · 6 sections");
|
||||
assert_eq!(meta.title(), "The Daily EPUB — 2026-08-15");
|
||||
assert_eq!(
|
||||
meta.title_for(Edition::Standard),
|
||||
"The Daily EPUB — 2026-08-15"
|
||||
);
|
||||
assert_eq!(
|
||||
meta.title_for(Edition::X4),
|
||||
"The Daily EPUB — 2026-08-15 (X4)"
|
||||
);
|
||||
// Title and filename carry the same tag, so a book found in the library
|
||||
// maps back to a file without guessing.
|
||||
assert!(
|
||||
meta.title_for(Edition::X4)
|
||||
.ends_with(Edition::X4.file_suffix())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vote_and_social_source_round_trip() {
|
||||
assert_eq!(Vote::parse("up"), Some(Vote::Up));
|
||||
assert_eq!(Vote::Down.as_i64(), -1);
|
||||
assert_eq!(
|
||||
SocialSource::parse("lobsters"),
|
||||
Some(SocialSource::Lobsters)
|
||||
);
|
||||
assert_eq!(SocialSource::Hn.as_str(), "hn");
|
||||
}
|
||||
}
|
||||
+395
@@ -0,0 +1,395 @@
|
||||
//! World Briefing from the Wikipedia Current Events portal (spec §3.8).
|
||||
//!
|
||||
//! Failure is non-fatal: the section is simply omitted.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use jiff::civil::Date;
|
||||
use scraper::node::Node;
|
||||
|
||||
use crate::epub::images::{text_escape, to_xhtml};
|
||||
use crate::types::WorldBriefing;
|
||||
|
||||
/// `ego_tree::NodeRef<'_, Node>` without depending on `ego_tree` directly.
|
||||
type NodeRef<'a> = <scraper::ElementRef<'a> as std::ops::Deref>::Target;
|
||||
|
||||
/// Portal page pattern: `Portal:Current_events/{YYYY}_{Month}_{D}` (§3.8).
|
||||
pub const PORTAL_BASE: &str = "https://en.wikipedia.org/wiki/Portal:Current_events/";
|
||||
/// MediaWiki REST HTML endpoint used to fetch the rendered page (§3.8).
|
||||
pub const REST_HTML_BASE: &str = "https://en.wikipedia.org/api/rest_v1/page/html/";
|
||||
/// Attribution line required by the portal's licence (§3.8).
|
||||
pub const ATTRIBUTION: &str = "Source: Wikipedia Current Events Portal, CC BY-SA 4.0.";
|
||||
/// How many days back [`fetch_with_fallback`] will look for a populated page.
|
||||
///
|
||||
/// The portal page for a day is created as an empty stub a day ahead and filled
|
||||
/// in over the course of that day, so the 05:30 run finds nothing under the
|
||||
/// issue's own date. Walking back one or two days lands on a complete page —
|
||||
/// which is also the news the reader has not seen yet at breakfast.
|
||||
pub const MAX_LOOKBACK_DAYS: i8 = 3;
|
||||
|
||||
const MONTHS: [&str; 12] = [
|
||||
"January",
|
||||
"February",
|
||||
"March",
|
||||
"April",
|
||||
"May",
|
||||
"June",
|
||||
"July",
|
||||
"August",
|
||||
"September",
|
||||
"October",
|
||||
"November",
|
||||
"December",
|
||||
];
|
||||
|
||||
/// Containers the day's events live in, most specific first (§3.8).
|
||||
const CONTENT_SELECTORS: &[&str] = &[
|
||||
"div.current-events-content",
|
||||
"div.description",
|
||||
"div.current-events-main",
|
||||
"section",
|
||||
"body",
|
||||
];
|
||||
|
||||
/// Elements whose entire subtree is dropped: citations, edit links, chrome.
|
||||
const DROP_ELEMENTS: &[&str] = &[
|
||||
"script", "style", "sup", "table", "figure", "img", "link", "meta", "noscript", "input",
|
||||
"button", "h1", "h2", "h3", "h4", "h5", "h6",
|
||||
];
|
||||
|
||||
/// Class fragments marking wiki chrome rather than content.
|
||||
const DROP_CLASSES: &[&str] = &[
|
||||
"mw-editsection",
|
||||
"reference",
|
||||
"navbox",
|
||||
"metadata",
|
||||
"noprint",
|
||||
"current-events-navbar",
|
||||
"current-events-heading",
|
||||
"hatnote",
|
||||
"mw-jump-link",
|
||||
];
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum WorldError {
|
||||
#[error("http error: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("no events found for {0}")]
|
||||
Empty(Date),
|
||||
}
|
||||
|
||||
/// Build the portal page title for a date, e.g. `2026_August_15` (§3.8).
|
||||
pub fn portal_title(date: Date) -> String {
|
||||
let month = MONTHS
|
||||
.get((date.month() as usize).saturating_sub(1))
|
||||
.copied()
|
||||
.unwrap_or("January");
|
||||
format!("{}_{}_{}", date.year(), month, date.day())
|
||||
}
|
||||
|
||||
/// Human-readable portal URL, used for the CC BY-SA attribution link (§3.8).
|
||||
pub fn portal_url(date: Date) -> String {
|
||||
format!("{PORTAL_BASE}{}", portal_title(date))
|
||||
}
|
||||
|
||||
/// MediaWiki REST HTML URL for the day's portal page (§3.8).
|
||||
pub fn rest_html_url(date: Date) -> String {
|
||||
format!(
|
||||
"{REST_HTML_BASE}Portal%3ACurrent_events%2F{}",
|
||||
portal_title(date)
|
||||
)
|
||||
}
|
||||
|
||||
/// Fetch the day's portal page, strip citations/edit links, flatten internal
|
||||
/// links to plain text and return a compact briefing (§3.8).
|
||||
pub async fn fetch(http: &reqwest::Client, date: Date) -> Result<WorldBriefing, WorldError> {
|
||||
let url = rest_html_url(date);
|
||||
tracing::debug!(%url, "fetching the world briefing");
|
||||
let html = http
|
||||
.get(&url)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
.await?;
|
||||
let body_html = extract_events(&html).ok_or(WorldError::Empty(date))?;
|
||||
Ok(WorldBriefing {
|
||||
date,
|
||||
source_url: portal_url(date),
|
||||
body_html,
|
||||
})
|
||||
}
|
||||
|
||||
/// The days [`fetch_with_fallback`] tries, newest first: `date`, then each
|
||||
/// earlier day up to `max_days_back` (§3.8).
|
||||
pub fn candidate_days(date: Date, max_days_back: i8) -> Vec<Date> {
|
||||
(0..=max_days_back.max(0))
|
||||
.map_while(|back| date.checked_sub(jiff::Span::new().days(back)).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Fetch the newest populated portal page at or before `date`, looking back at
|
||||
/// most [`MAX_LOOKBACK_DAYS`] days (§3.8).
|
||||
///
|
||||
/// The issue's own day is almost always still an empty stub at 05:30, so this is
|
||||
/// the entry point the pipeline uses; the returned briefing carries the date it
|
||||
/// actually covers in [`WorldBriefing::date`].
|
||||
pub async fn fetch_with_fallback(
|
||||
http: &reqwest::Client,
|
||||
date: Date,
|
||||
max_days_back: i8,
|
||||
) -> Result<WorldBriefing, WorldError> {
|
||||
let mut last = WorldError::Empty(date);
|
||||
for day in candidate_days(date, max_days_back) {
|
||||
match fetch(http, day).await {
|
||||
Ok(briefing) => {
|
||||
if day != date {
|
||||
tracing::info!(
|
||||
%date,
|
||||
covering = %day,
|
||||
"the issue day's portal page was not populated yet; using an earlier day"
|
||||
);
|
||||
}
|
||||
return Ok(briefing);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(%day, "world briefing not available for this day: {e}");
|
||||
last = e;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(last)
|
||||
}
|
||||
|
||||
/// Best-effort wrapper used by the pipeline: never fails the run (§3.8).
|
||||
pub async fn fetch_optional(
|
||||
http: &reqwest::Client,
|
||||
date: Date,
|
||||
enabled: bool,
|
||||
) -> Option<WorldBriefing> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
match fetch_with_fallback(http, date, MAX_LOOKBACK_DAYS).await {
|
||||
Ok(b) => Some(b),
|
||||
Err(e) => {
|
||||
tracing::warn!(%date, "world briefing unavailable: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the day's bulleted events from a rendered portal page (§3.8).
|
||||
///
|
||||
/// Citations, edit links and navigation are dropped; internal links become plain
|
||||
/// text; the result is a sanitized `<p>`/`<ul>` fragment.
|
||||
pub fn extract_events(html: &str) -> Option<String> {
|
||||
let doc = scraper::Html::parse_document(html);
|
||||
for selector in CONTENT_SELECTORS {
|
||||
let Ok(sel) = scraper::Selector::parse(selector) else {
|
||||
continue;
|
||||
};
|
||||
for container in doc.select(&sel) {
|
||||
let mut out = String::new();
|
||||
walk_children(*container, &mut out);
|
||||
let cleaned = sanitize(&out);
|
||||
if !cleaned.is_empty() && cleaned.contains("<li>") {
|
||||
return Some(to_xhtml(&cleaned));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn sanitize(fragment: &str) -> String {
|
||||
let tags: HashSet<&str> = ["p", "ul", "ol", "li", "strong", "em", "br"]
|
||||
.into_iter()
|
||||
.collect();
|
||||
ammonia::Builder::new()
|
||||
.tags(tags)
|
||||
.clean(fragment)
|
||||
.to_string()
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn is_dropped(el: &scraper::node::Element) -> bool {
|
||||
if DROP_ELEMENTS.contains(&el.name()) {
|
||||
return true;
|
||||
}
|
||||
if let Some(class) = el.attr("class")
|
||||
&& DROP_CLASSES
|
||||
.iter()
|
||||
.any(|dropped| class.split_whitespace().any(|c| c == *dropped))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if el.attr("role") == Some("navigation") {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn walk_children(node: NodeRef<'_>, out: &mut String) {
|
||||
for child in node.children() {
|
||||
walk(child, out);
|
||||
}
|
||||
}
|
||||
|
||||
fn walk(node: NodeRef<'_>, out: &mut String) {
|
||||
match node.value() {
|
||||
Node::Text(text) => out.push_str(&text_escape(text)),
|
||||
Node::Element(el) => {
|
||||
if is_dropped(el) {
|
||||
return;
|
||||
}
|
||||
match el.name() {
|
||||
"ul" | "ol" | "li" | "p" => {
|
||||
let name = el.name();
|
||||
out.push('<');
|
||||
out.push_str(name);
|
||||
out.push('>');
|
||||
walk_children(node, out);
|
||||
out.push_str("</");
|
||||
out.push_str(name);
|
||||
out.push('>');
|
||||
}
|
||||
"b" | "strong" => {
|
||||
out.push_str("<strong>");
|
||||
walk_children(node, out);
|
||||
out.push_str("</strong>");
|
||||
}
|
||||
"i" | "em" => {
|
||||
out.push_str("<em>");
|
||||
walk_children(node, out);
|
||||
out.push_str("</em>");
|
||||
}
|
||||
"dt" => {
|
||||
out.push_str("<p><strong>");
|
||||
walk_children(node, out);
|
||||
out.push_str("</strong></p>");
|
||||
}
|
||||
"br" => out.push(' '),
|
||||
// `a`, `span`, `div`, `dl`, `dd`, `section` … are transparent:
|
||||
// internal links keep their text only (§3.8).
|
||||
_ => walk_children(node, out),
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Render the briefing to sanitized XHTML with the CC BY-SA attribution (§3.8).
|
||||
pub fn render_xhtml(briefing: &WorldBriefing) -> String {
|
||||
format!(
|
||||
"{}\n <p class=\"attribution\">{} <a href=\"{}\">{}</a></p>\n",
|
||||
briefing.body_html,
|
||||
text_escape(ATTRIBUTION),
|
||||
text_escape(&briefing.source_url),
|
||||
text_escape(&briefing.source_url)
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fixture() -> String {
|
||||
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/wikipedia_current_events.html");
|
||||
std::fs::read_to_string(path).expect("fixture must exist")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn portal_titles_and_urls_match_the_spec() {
|
||||
let date: Date = "2026-08-15".parse().unwrap();
|
||||
assert_eq!(portal_title(date), "2026_August_15");
|
||||
assert_eq!(
|
||||
portal_url(date),
|
||||
"https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15"
|
||||
);
|
||||
assert_eq!(
|
||||
rest_html_url(date),
|
||||
"https://en.wikipedia.org/api/rest_v1/page/html/Portal%3ACurrent_events%2F2026_August_15"
|
||||
);
|
||||
let single_digit: Date = "2026-01-05".parse().unwrap();
|
||||
assert_eq!(portal_title(single_digit), "2026_January_5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_events_and_strips_wiki_chrome() {
|
||||
let body = extract_events(&fixture()).expect("events");
|
||||
assert!(body.contains("<ul>"));
|
||||
assert!(body.contains("<strong>Armed conflicts and attacks</strong>"));
|
||||
assert!(body.contains("Heavy rain floods the Charles River basin"));
|
||||
// Internal links are flattened to plain text.
|
||||
assert!(!body.contains("<a"));
|
||||
assert!(body.contains("Boston"));
|
||||
// Citations, edit links and navboxes are gone.
|
||||
assert!(!body.contains("[1]"));
|
||||
assert!(!body.contains("edit"));
|
||||
assert!(!body.contains("Ongoing events"));
|
||||
// Nested sub-bullets survive.
|
||||
assert!(body.contains("A second-level detail"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_events_yield_none() {
|
||||
assert!(extract_events("<html><body><p>Nothing here</p></body></html>").is_none());
|
||||
assert!(extract_events("").is_none());
|
||||
}
|
||||
|
||||
/// Wikipedia creates each day's portal page as an empty stub a day ahead and
|
||||
/// fills it in over that day, so the 05:30 run sees this, not news (§3.8).
|
||||
/// Its only `<li>`s are the edit/history/watch navbar, which must not count
|
||||
/// as content — otherwise the fallback never triggers.
|
||||
#[test]
|
||||
fn an_unpopulated_stub_page_yields_no_events() {
|
||||
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/wikipedia_current_events_empty_stub.html");
|
||||
let stub = std::fs::read_to_string(path).expect("fixture must exist");
|
||||
assert!(stub.contains("current-events-navbar"), "fixture sanity");
|
||||
assert!(extract_events(&stub).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_fallback_walks_backwards_from_the_issue_date() {
|
||||
let date: Date = "2026-08-15".parse().unwrap();
|
||||
assert_eq!(
|
||||
candidate_days(date, 3),
|
||||
[
|
||||
"2026-08-15".parse().unwrap(),
|
||||
"2026-08-14".parse().unwrap(),
|
||||
"2026-08-13".parse().unwrap(),
|
||||
"2026-08-12".parse().unwrap(),
|
||||
]
|
||||
);
|
||||
// Never forwards, and never fewer than the issue's own day.
|
||||
assert_eq!(candidate_days(date, 0), [date]);
|
||||
assert_eq!(candidate_days(date, -1), [date]);
|
||||
// Month and year boundaries.
|
||||
assert_eq!(
|
||||
candidate_days("2026-01-01".parse().unwrap(), 2),
|
||||
[
|
||||
"2026-01-01".parse().unwrap(),
|
||||
"2025-12-31".parse().unwrap(),
|
||||
"2025-12-30".parse().unwrap(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendering_appends_the_attribution() {
|
||||
let briefing = WorldBriefing {
|
||||
date: "2026-08-15".parse().unwrap(),
|
||||
source_url: portal_url("2026-08-15".parse().unwrap()),
|
||||
body_html: "<ul><li>Something happened</li></ul>".into(),
|
||||
};
|
||||
let xhtml = render_xhtml(&briefing);
|
||||
assert!(xhtml.contains("CC BY-SA 4.0"));
|
||||
assert!(xhtml.contains(
|
||||
"<a href=\"https://en.wikipedia.org/wiki/Portal:Current_events/2026_August_15\">"
|
||||
));
|
||||
assert!(xhtml.contains("Something happened"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user