//! Comment chapters: fetch trees for selected articles and render them (spec §3.7).
//!
//! Heuristic only, no LLM. Rendered as nested border-left indentation that reads
//! well on e-ink (no color). Every fetch is best-effort: a platform that errors
//! out is simply left out of the chapter (notes §3).
use std::collections::HashSet;
use futures::StreamExt;
use serde_json::Value;
use crate::epub::images::{text_escape, to_xhtml};
use crate::types::{Comment, CommentThread, Discussion, Pick, SocialSource};
/// Top-level threads kept per source (§3.7).
pub const MAX_TOP_LEVEL: usize = 8;
/// Maximum nesting depth rendered: depths 0, 1 and 2 (§3.7).
pub const MAX_DEPTH: usize = 3;
/// Maximum children rendered per node (§3.7).
pub const MAX_CHILDREN: usize = 4;
/// Per-comment character cap before ellipsizing (§3.7).
pub const MAX_COMMENT_CHARS: usize = 1200;
/// Whole-chapter word cap (§3.7).
pub const MAX_CHAPTER_WORDS: usize = 4000;
/// Concurrent discussion fetches.
pub const CONCURRENCY: usize = 4;
/// Platform order inside a discussion chapter: HN → Lobsters → Reddit (§3.7).
pub const SOURCE_ORDER: &[SocialSource] = &[
SocialSource::Hn,
SocialSource::Lobsters,
SocialSource::Reddit,
];
// ---------------------------------------------------------------------------
// Fetching
// ---------------------------------------------------------------------------
/// `https://hn.algolia.com/api/v1/items/{objectID}` (§3.7).
pub fn hn_items_url(item_id: &str) -> String {
format!("https://hn.algolia.com/api/v1/items/{item_id}")
}
/// `https://lobste.rs/s/{id}.json` (§3.7).
pub fn lobsters_url(item_id: &str) -> String {
format!("https://lobste.rs/s/{item_id}.json")
}
/// `https://www.reddit.com{permalink}.json?limit=100&depth=3&sort=top` (§3.7).
pub fn reddit_url(permalink_or_url: &str) -> String {
let base = permalink_or_url.trim_end_matches('/');
let base = if base.starts_with("http://") || base.starts_with("https://") {
base.to_string()
} else if base.starts_with('/') {
format!("https://www.reddit.com{base}")
} else {
format!("https://www.reddit.com/{base}")
};
let base = base.trim_end_matches(".json").to_string();
format!("{base}.json?limit=100&depth=3&sort=top")
}
async fn get_json(http: &reqwest::Client, url: &str) -> Option {
let resp = http
.get(url)
.send()
.await
.map_err(|e| tracing::debug!(url, "comment fetch failed: {e}"))
.ok()?;
if !resp.status().is_success() {
tracing::debug!(url, status = %resp.status(), "comment fetch rejected");
return None;
}
resp.json::()
.await
.map_err(|e| tracing::debug!(url, "comment payload was not json: {e}"))
.ok()
}
/// Fetch and assemble the discussion for one selected article, ordered
/// HN → Lobsters → Reddit (§3.7). Best-effort: returns `None` on failure.
pub async fn fetch_discussion(http: &reqwest::Client, pick: &Pick) -> Option {
let mut threads: Vec = Vec::new();
for source in SOURCE_ORDER {
let Some(social) = pick.article.social.iter().find(|s| s.source == *source) else {
continue;
};
let thread = match source {
SocialSource::Hn => match social.item_id.as_deref() {
Some(id) => get_json(http, &hn_items_url(id))
.await
.and_then(|v| parse_hn(&v)),
None => None,
},
SocialSource::Lobsters => match social.item_id.as_deref() {
Some(id) => get_json(http, &lobsters_url(id))
.await
.and_then(|v| parse_lobsters(&v)),
None => None,
},
SocialSource::Reddit => {
let target = social
.item_url
.as_deref()
.or(social.item_id.as_deref())
.map(reddit_url);
match target {
Some(url) => get_json(http, &url)
.await
.and_then(|v| parse_reddit(&v, "")),
None => None,
}
}
SocialSource::X => None,
};
match thread {
Some(mut t) => {
t.comments = truncate(t.comments);
if !t.comments.is_empty() {
threads.push(t);
}
}
None => tracing::debug!(
source = %source,
article = pick.article.id,
"no comment tree for this source"
),
}
}
if threads.is_empty() {
return None;
}
enforce_chapter_budget(&mut threads);
Some(Discussion {
article_id: pick.article.id,
chapter_id: format!("disc-{}", pick.article.best_entry_id),
threads,
})
}
/// Fetch discussions for every pick in parallel, filling [`Pick::discussion`] (§3.7).
pub async fn fetch_all(http: &reqwest::Client, picks: &mut [Pick]) -> usize {
let fetched: Vec
"));
// Short comments are left untouched.
assert_eq!(ellipsize_html("
short
", 100), "
short
");
}
fn tree_words(comments: &[Comment]) -> usize {
comments
.iter()
.map(|c| word_count(&c.text_html) + tree_words(&c.children))
.sum()
}
fn tree_len(comments: &[Comment]) -> usize {
comments.iter().map(|c| 1 + tree_len(&c.children)).sum()
}
#[test]
fn chapter_word_budget_is_enforced() {
// Every comment ellipsizes to ~240 words, so a full 8×4×4 tree is far
// over the 4,000-word chapter budget.
let long = "word ".repeat(400);
let roots: Vec = (0..MAX_TOP_LEVEL)
.map(|i| {
let mut root = leaf(&format!("u{i}"), 100 - i as i64, &long);
for j in 0..MAX_CHILDREN {
let mut child = leaf(&format!("c{i}{j}"), 10, &long);
child.children.push(leaf("grandchild", 1, &long));
root.children.push(child);
}
root
})
.collect();
let full = tree_len(&roots);
let out = truncate(roots);
let total = tree_words(&out);
assert!(total <= MAX_CHAPTER_WORDS, "{total} words is over budget");
assert!(!out.is_empty());
assert!(
tree_len(&out) < full,
"comments past the budget are dropped"
);
}
#[test]
fn renders_nested_blockquotes_and_a_title() {
let mut root = leaf("alice", 42, "top level");
root.children.push(leaf("bob", 3, "reply"));
let discussion = Discussion {
article_id: 7,
chapter_id: "disc-1001".into(),
threads: vec![CommentThread {
source: SocialSource::Hn,
item_url: "https://news.ycombinator.com/item?id=1".into(),
total_comments: 210,
comments: vec![root],
}],
};
let xhtml = render_xhtml(&discussion, "A Title");
assert!(xhtml.contains("HN \u{00b7} 210 comments"));
assert!(xhtml.contains("alice \u{00b7} 42 points"));
// Top-level comments and replies are distinguishable by class alone, so
// the X4 needs no descendant selector to indent them (§3.10).
assert_eq!(xhtml.matches("
").count(), 1);
assert_eq!(
xhtml
.matches("
")
.count(),
1
);
assert_eq!(
xhtml.matches("
").count(),
2,
"every blockquote is closed"
);
// Comment paragraphs carry their own class for the same reason.
assert!(
xhtml.contains("
top level
"),
"{xhtml}"
);
assert!(!xhtml.contains("
"), "an unclassed paragraph survived");
assert_eq!(
chapter_title("A Title", &discussion),
"\u{1f4ac} Discussion: A Title (210 comments on HN)"
);
}
}