From 7ce8c65939d03104f67adf54e82e919111334b72 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Mon, 7 Sep 2026 18:34:21 +0000 Subject: [PATCH] Take the author from the article page, not the aggregator submitter Readability already finds a byline (JSON-LD author, meta author tags, byline markup); keep it on the article and let it replace an aggregator entry's author, which for HN and friends is the submitter. A direct feed's own author is still trusted over the page, and an aggregator name is only used at all when no direct feed carried the story. The author is stored on articles so it survives independently of the best entry. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QVPagF6jfDv78CC5Jv2wp4 --- migrations/0011_article_author.sql | 3 + src/db.rs | 25 ++++-- src/dedupe.rs | 41 ++++++++++ src/extract.rs | 118 +++++++++++++++++++++++++++-- src/imports.rs | 5 +- src/types.rs | 4 +- 6 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 migrations/0011_article_author.sql diff --git a/migrations/0011_article_author.sql b/migrations/0011_article_author.sql new file mode 100644 index 0000000..079cd92 --- /dev/null +++ b/migrations/0011_article_author.sql @@ -0,0 +1,3 @@ +-- Persist page-extracted authors independently of the best Miniflux entry. + +ALTER TABLE articles ADD COLUMN author TEXT; diff --git a/src/db.rs b/src/db.rs index f7d38de..5254fac 100644 --- a/src/db.rs +++ b/src/db.rs @@ -291,16 +291,17 @@ impl Db { /// Upsert a deduped cluster by canonical URL; returns its `articles.id`. /// - /// The denormalized fields on [`Article`] are not stored here — they come - /// from the joined `entries` row when loading. + /// Most denormalized fields on [`Article`] come from the joined `entries` + /// row when loading; the extracted author is stored on `articles`. pub async fn upsert_article(&self, article: &Article) -> Result { let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into()); let row = sqlx::query( - "INSERT INTO articles (canonical_url, title, best_entry_id, content_html, word_count, - excerpt_only, image_count, sources_json, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + "INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html, + word_count, excerpt_only, image_count, sources_json, first_seen) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(canonical_url) DO UPDATE SET title = excluded.title, + author = excluded.author, best_entry_id = excluded.best_entry_id, content_html = excluded.content_html, word_count = excluded.word_count, @@ -311,6 +312,7 @@ impl Db { ) .bind(&article.canonical_url) .bind(&article.title) + .bind(&article.author) .bind((article.best_entry_id != 0).then_some(article.best_entry_id)) .bind(&article.content_html) .bind(article.word_count) @@ -860,7 +862,7 @@ macro_rules! article_select { a.word_count AS word_count, a.excerpt_only AS excerpt_only, a.image_count AS image_count, a.sources_json AS sources_json, a.first_seen AS first_seen, - e.url AS entry_url, e.author AS author, e.feed_id AS feed_id, + e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id, e.feed_title AS feed_title, e.category AS category, e.published_at AS published_at, e.comments_url AS comments_url FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id @@ -1163,7 +1165,7 @@ mod tests { }], first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), - author: None, + author: Some("Page Writer".into()), feed_id: 7, feed_title: "Hacker News".into(), category: None, @@ -1194,11 +1196,20 @@ mod tests { assert_eq!(loaded.social.len(), 1); assert_eq!(loaded.social[0].score, 342); assert_eq!(loaded.feed_title, "Hacker News"); + assert_eq!(loaded.author.as_deref(), Some("Page Writer")); // The batched lookup agrees with the single one and skips unknown ids. let batch = db.get_articles(&[id, 9_999]).await.unwrap(); assert_eq!(batch.len(), 1); assert_eq!(batch[&id], loaded); assert!(db.get_articles(&[]).await.unwrap().is_empty()); + + sqlx::query("UPDATE articles SET author = NULL WHERE id = ?") + .bind(id) + .execute(db.pool()) + .await + .unwrap(); + let fallback = db.get_article(id).await.unwrap().unwrap(); + assert_eq!(fallback.author.as_deref(), Some("someone")); assert_eq!( db.article_id_for_url("https://example.com/1") .await diff --git a/src/dedupe.rs b/src/dedupe.rs index 8517708..c92e2b0 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -380,8 +380,17 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article } }); + // An aggregator entry's author is usually the submitter, so it only counts + // when no direct feed carried the story; extraction may still replace it + // with the page's own byline. + let is_direct = |e: &Entry| { + classify_source_with_feed(e, feed_urls.get(&e.feed_id).map(String::as_str)) + == SourceKind::Feed + }; + let has_direct = members.iter().any(|(e, _)| is_direct(e)); let author = members .iter() + .filter(|(e, _)| !has_direct || is_direct(e)) .filter_map(|(e, _)| e.author.clone()) .find(|a| !a.trim().is_empty()); @@ -638,6 +647,38 @@ mod tests { assert_eq!(articles[1].sources.len(), 1); } + #[test] + fn direct_feed_author_beats_aggregator_submitter() { + let mut aggregator = entry(1, "https://blog.dev/post", "A Distinct Article Title"); + aggregator.author = Some("HN Submitter".into()); + aggregator.raw_content = format!("

{}

", "word ".repeat(50)); + + let mut direct = entry(2, "https://blog.dev/post", "A Distinct Article Title"); + direct.author = Some("Real Writer".into()); + + let feed_urls = FeedUrls::from([ + (aggregator.feed_id, "https://hnrss.org/frontpage".into()), + (direct.feed_id, "https://blog.dev/feed.xml".into()), + ]); + let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls); + + assert_eq!(articles.len(), 1); + assert_eq!(articles[0].best_entry_id, 1); + assert_eq!(articles[0].author.as_deref(), Some("Real Writer")); + + // With a direct feed present, a submitter name is not used as a fallback. + let mut aggregator = entry(3, "https://blog.dev/other", "Another Distinct Title"); + aggregator.author = Some("HN Submitter".into()); + let mut direct = entry(4, "https://blog.dev/other", "Another Distinct Title"); + direct.author = None; + let feed_urls = FeedUrls::from([ + (aggregator.feed_id, "https://hnrss.org/frontpage".into()), + (direct.feed_id, "https://blog.dev/feed.xml".into()), + ]); + let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls); + assert_eq!(articles[0].author, None); + } + #[test] fn clustering_drops_non_articles_and_keeps_short_titles_apart() { let mut a = entry(1, "https://a.dev/1", "News"); diff --git a/src/extract.rs b/src/extract.rs index 085254b..f997e9b 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -18,7 +18,7 @@ use url::Url; use crate::html::word_count; use crate::images::normalize::{normalize_img_tags, prepare_for_readability}; use crate::images::refs::collect_image_urls; -use crate::types::{Article, ExtractMethod, Extracted}; +use crate::types::{Article, ExtractMethod, Extracted, SourceKind}; /// Word count at or above which Miniflux content is treated as full text (§3.3). pub const FULL_TEXT_MIN_WORDS: i64 = 250; @@ -197,6 +197,7 @@ impl Extractor { || looks_paywalled(&article.url, words, &self.paywall_domains); Extracted { content_html, + author: None, word_count: words, excerpt_only, image_urls, @@ -279,10 +280,11 @@ impl Extractor { body.extend_from_slice(&chunk); } let html = String::from_utf8_lossy(&body).into_owned(); - let (title, html) = readable_page(&html, &final_url)?; + let (title, html, author) = readable_page(&html, &final_url)?; Ok(Page { title, html, + author, final_url, }) } @@ -294,6 +296,7 @@ impl Extractor { let image_urls = collect_image_urls(&clean, &page.final_url); Extracted { content_html: clean, + author: page.author.clone(), word_count: words, excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains), image_urls, @@ -309,6 +312,8 @@ pub struct Page { pub title: String, /// Readability's main-content markup. pub html: String, + /// Readability's normalized byline for the page. + pub author: Option, /// Where the fetch ended up, after any redirects — the base for relative URLs. pub final_url: String, } @@ -321,10 +326,10 @@ pub struct Page { /// their image with them) and its lazy-image heuristic overwrites a perfectly /// good `src` with whatever other attribute happens to contain `.jpg`. pub fn readability(html: &str, url: &str) -> Result { - readable_page(html, url).map(|(_, content)| content) + readable_page(html, url).map(|(_, content, _)| content) } -fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError> { +fn readable_page(html: &str, url: &str) -> Result<(String, String, Option), ExtractError> { let html = prepare_for_readability(html); let config = dom_smoothie::Config { max_elements_to_parse: 60_000, @@ -337,7 +342,20 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError if content.trim().is_empty() { return Err(ExtractError::NoContent); } - Ok((parsed.title.trim().to_string(), content)) + Ok(( + parsed.title.trim().to_string(), + content, + normalize_author(parsed.byline), + )) +} + +fn normalize_author(author: Option) -> Option { + let author = author?; + if author.chars().any(|c| matches!(c, '\n' | '\r')) { + return None; + } + let author = author.split_whitespace().collect::>().join(" "); + (!author.is_empty() && author.chars().count() <= 100).then_some(author) } /// Copy an [`Extracted`] onto its [`Article`]. @@ -348,6 +366,11 @@ pub fn apply(article: &mut Article, extracted: Extracted) { article.word_count = extracted.word_count; article.excerpt_only = extracted.excerpt_only; article.extract_method = extracted.method; + if let Some(author) = extracted.author + && (article.author.is_none() || !article.came_via(SourceKind::Feed)) + { + article.author = Some(author); + } } fn merge_paywall_domains(configured: Vec) -> Vec { @@ -722,11 +745,94 @@ mod tests {

A Post

{paragraph}

{paragraph}

\
© 2026
" ); - let (title, content) = readable_page(&html, "https://blog.dev/p").expect("main content"); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); assert_eq!(title, "A Post"); + assert_eq!(author, None); assert!(content.contains("Readability keeps the body copy")); let clean = sanitize_with_base(&content, "https://blog.dev/p"); assert!(word_count(&clean) > 200); assert!(!clean.contains("A Post\ + \ +

A Post

{}

", + "Substantial body copy for readability. ".repeat(40) + ); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); + let page = Page { + title, + html: content, + author, + final_url: "https://blog.dev/p".into(), + }; + let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); + assert_eq!(extracted.author.as_deref(), Some("Jane Dev")); + } + + #[test] + fn readable_page_plumbs_json_ld_author_through_extracted() { + let html = format!( + r#"A Post + +

A Post

{}

"#, + "Substantial body copy for readability. ".repeat(40) + ); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); + let page = Page { + title, + html: content, + author, + final_url: "https://blog.dev/p".into(), + }; + let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); + assert_eq!(extracted.author.as_deref(), Some("Alex Writer")); + } + + #[test] + fn apply_uses_page_author_with_feed_precedence() { + let extracted = |author: &str| Extracted { + content_html: "

body

".into(), + author: Some(author.into()), + word_count: 1, + excerpt_only: false, + image_urls: vec![], + method: ExtractMethod::Readability, + }; + + let mut aggregator = article("https://blog.dev/aggregator", ""); + aggregator.sources[0].kind = SourceKind::HnFrontpage; + aggregator.author = Some("Submitter".into()); + apply(&mut aggregator, extracted("Page Writer")); + assert_eq!(aggregator.author.as_deref(), Some("Page Writer")); + + let mut direct = article("https://blog.dev/direct", ""); + direct.author = Some("Feed Writer".into()); + apply(&mut direct, extracted("Page Writer")); + assert_eq!(direct.author.as_deref(), Some("Feed Writer")); + + let mut missing = article("https://blog.dev/missing", ""); + apply(&mut missing, extracted("Page Writer")); + assert_eq!(missing.author.as_deref(), Some("Page Writer")); + } + + #[test] + fn implausible_page_authors_are_dropped() { + assert_eq!( + normalize_author(Some(" Jane Dev ".into())).as_deref(), + Some("Jane Dev") + ); + assert_eq!(normalize_author(Some("Jane\nDev".into())), None); + assert_eq!(normalize_author(Some("x".repeat(101))), None); + assert_eq!(normalize_author(Some(" ".into())), None); + } } diff --git a/src/imports.rs b/src/imports.rs index d742017..18130e2 100644 --- a/src/imports.rs +++ b/src/imports.rs @@ -165,7 +165,7 @@ fn imported_article( sources: Vec::new(), first_seen, url: page.final_url, - author: None, + author: extracted.author, feed_id: 0, feed_title: "Imported".into(), category: None, @@ -482,6 +482,7 @@ mod tests { let page = Page { title: "A historical essay".into(), html: "

Useful old writing.

".into(), + author: Some("Essay Writer".into()), final_url: "https://example.com/essays/old".into(), }; let extracted = extractor.finish_readable("https://example.com/old", &page); @@ -494,6 +495,7 @@ mod tests { assert_eq!(article.title, "A historical essay"); assert_eq!(article.best_entry_id, 0); assert_eq!(article.feed_title, "Imported"); + assert_eq!(article.author.as_deref(), Some("Essay Writer")); assert!(article.sources.is_empty()); assert!(!article.content_html.contains("script")); assert_eq!(article.image_urls, ["https://example.com/chart.png"]); @@ -502,6 +504,7 @@ mod tests { assert_eq!(loaded.title, "A historical essay"); assert_eq!(loaded.best_entry_id, 0); assert_eq!(loaded.feed_title, "Imported"); + assert_eq!(loaded.author.as_deref(), Some("Essay Writer")); } #[tokio::test] diff --git a/src/types.rs b/src/types.rs index 5d0df8d..6ebcbc0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -90,6 +90,8 @@ pub enum ExtractMethod { pub struct Extracted { /// Sanitized XHTML-safe body markup. pub content_html: String, + /// Author discovered in the fetched page, if any. + pub author: Option, pub word_count: i64, /// True when we only have an excerpt/paywall stub — penalized in pre-filter. pub excerpt_only: bool, @@ -119,7 +121,7 @@ pub struct Article { pub sources: Vec, pub first_seen: Timestamp, - // --- denormalized, not stored on `articles` --- + // --- joined / derived fields --- pub url: String, pub author: Option, pub feed_id: FeedId,