diff --git a/migrations/0011_article_author.sql b/migrations/0011_article_author.sql new file mode 100644 index 0000000..079cd92 --- /dev/null +++ b/migrations/0011_article_author.sql @@ -0,0 +1,3 @@ +-- Persist page-extracted authors independently of the best Miniflux entry. + +ALTER TABLE articles ADD COLUMN author TEXT; diff --git a/src/db.rs b/src/db.rs index f7d38de..5254fac 100644 --- a/src/db.rs +++ b/src/db.rs @@ -291,16 +291,17 @@ impl Db { /// Upsert a deduped cluster by canonical URL; returns its `articles.id`. /// - /// The denormalized fields on [`Article`] are not stored here — they come - /// from the joined `entries` row when loading. + /// Most denormalized fields on [`Article`] come from the joined `entries` + /// row when loading; the extracted author is stored on `articles`. pub async fn upsert_article(&self, article: &Article) -> Result { let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into()); let row = sqlx::query( - "INSERT INTO articles (canonical_url, title, best_entry_id, content_html, word_count, - excerpt_only, image_count, sources_json, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + "INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html, + word_count, excerpt_only, image_count, sources_json, first_seen) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(canonical_url) DO UPDATE SET title = excluded.title, + author = excluded.author, best_entry_id = excluded.best_entry_id, content_html = excluded.content_html, word_count = excluded.word_count, @@ -311,6 +312,7 @@ impl Db { ) .bind(&article.canonical_url) .bind(&article.title) + .bind(&article.author) .bind((article.best_entry_id != 0).then_some(article.best_entry_id)) .bind(&article.content_html) .bind(article.word_count) @@ -860,7 +862,7 @@ macro_rules! article_select { a.word_count AS word_count, a.excerpt_only AS excerpt_only, a.image_count AS image_count, a.sources_json AS sources_json, a.first_seen AS first_seen, - e.url AS entry_url, e.author AS author, e.feed_id AS feed_id, + e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id, e.feed_title AS feed_title, e.category AS category, e.published_at AS published_at, e.comments_url AS comments_url FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id @@ -1163,7 +1165,7 @@ mod tests { }], first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), - author: None, + author: Some("Page Writer".into()), feed_id: 7, feed_title: "Hacker News".into(), category: None, @@ -1194,11 +1196,20 @@ mod tests { assert_eq!(loaded.social.len(), 1); assert_eq!(loaded.social[0].score, 342); assert_eq!(loaded.feed_title, "Hacker News"); + assert_eq!(loaded.author.as_deref(), Some("Page Writer")); // The batched lookup agrees with the single one and skips unknown ids. let batch = db.get_articles(&[id, 9_999]).await.unwrap(); assert_eq!(batch.len(), 1); assert_eq!(batch[&id], loaded); assert!(db.get_articles(&[]).await.unwrap().is_empty()); + + sqlx::query("UPDATE articles SET author = NULL WHERE id = ?") + .bind(id) + .execute(db.pool()) + .await + .unwrap(); + let fallback = db.get_article(id).await.unwrap().unwrap(); + assert_eq!(fallback.author.as_deref(), Some("someone")); assert_eq!( db.article_id_for_url("https://example.com/1") .await diff --git a/src/dedupe.rs b/src/dedupe.rs index 8517708..c92e2b0 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -380,8 +380,17 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article } }); + // An aggregator entry's author is usually the submitter, so it only counts + // when no direct feed carried the story; extraction may still replace it + // with the page's own byline. + let is_direct = |e: &Entry| { + classify_source_with_feed(e, feed_urls.get(&e.feed_id).map(String::as_str)) + == SourceKind::Feed + }; + let has_direct = members.iter().any(|(e, _)| is_direct(e)); let author = members .iter() + .filter(|(e, _)| !has_direct || is_direct(e)) .filter_map(|(e, _)| e.author.clone()) .find(|a| !a.trim().is_empty()); @@ -638,6 +647,38 @@ mod tests { assert_eq!(articles[1].sources.len(), 1); } + #[test] + fn direct_feed_author_beats_aggregator_submitter() { + let mut aggregator = entry(1, "https://blog.dev/post", "A Distinct Article Title"); + aggregator.author = Some("HN Submitter".into()); + aggregator.raw_content = format!("

{}

", "word ".repeat(50)); + + let mut direct = entry(2, "https://blog.dev/post", "A Distinct Article Title"); + direct.author = Some("Real Writer".into()); + + let feed_urls = FeedUrls::from([ + (aggregator.feed_id, "https://hnrss.org/frontpage".into()), + (direct.feed_id, "https://blog.dev/feed.xml".into()), + ]); + let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls); + + assert_eq!(articles.len(), 1); + assert_eq!(articles[0].best_entry_id, 1); + assert_eq!(articles[0].author.as_deref(), Some("Real Writer")); + + // With a direct feed present, a submitter name is not used as a fallback. + let mut aggregator = entry(3, "https://blog.dev/other", "Another Distinct Title"); + aggregator.author = Some("HN Submitter".into()); + let mut direct = entry(4, "https://blog.dev/other", "Another Distinct Title"); + direct.author = None; + let feed_urls = FeedUrls::from([ + (aggregator.feed_id, "https://hnrss.org/frontpage".into()), + (direct.feed_id, "https://blog.dev/feed.xml".into()), + ]); + let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls); + assert_eq!(articles[0].author, None); + } + #[test] fn clustering_drops_non_articles_and_keeps_short_titles_apart() { let mut a = entry(1, "https://a.dev/1", "News"); diff --git a/src/extract.rs b/src/extract.rs index 085254b..f997e9b 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -18,7 +18,7 @@ use url::Url; use crate::html::word_count; use crate::images::normalize::{normalize_img_tags, prepare_for_readability}; use crate::images::refs::collect_image_urls; -use crate::types::{Article, ExtractMethod, Extracted}; +use crate::types::{Article, ExtractMethod, Extracted, SourceKind}; /// Word count at or above which Miniflux content is treated as full text (§3.3). pub const FULL_TEXT_MIN_WORDS: i64 = 250; @@ -197,6 +197,7 @@ impl Extractor { || looks_paywalled(&article.url, words, &self.paywall_domains); Extracted { content_html, + author: None, word_count: words, excerpt_only, image_urls, @@ -279,10 +280,11 @@ impl Extractor { body.extend_from_slice(&chunk); } let html = String::from_utf8_lossy(&body).into_owned(); - let (title, html) = readable_page(&html, &final_url)?; + let (title, html, author) = readable_page(&html, &final_url)?; Ok(Page { title, html, + author, final_url, }) } @@ -294,6 +296,7 @@ impl Extractor { let image_urls = collect_image_urls(&clean, &page.final_url); Extracted { content_html: clean, + author: page.author.clone(), word_count: words, excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains), image_urls, @@ -309,6 +312,8 @@ pub struct Page { pub title: String, /// Readability's main-content markup. pub html: String, + /// Readability's normalized byline for the page. + pub author: Option, /// Where the fetch ended up, after any redirects — the base for relative URLs. pub final_url: String, } @@ -321,10 +326,10 @@ pub struct Page { /// their image with them) and its lazy-image heuristic overwrites a perfectly /// good `src` with whatever other attribute happens to contain `.jpg`. pub fn readability(html: &str, url: &str) -> Result { - readable_page(html, url).map(|(_, content)| content) + readable_page(html, url).map(|(_, content, _)| content) } -fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError> { +fn readable_page(html: &str, url: &str) -> Result<(String, String, Option), ExtractError> { let html = prepare_for_readability(html); let config = dom_smoothie::Config { max_elements_to_parse: 60_000, @@ -337,7 +342,20 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError if content.trim().is_empty() { return Err(ExtractError::NoContent); } - Ok((parsed.title.trim().to_string(), content)) + Ok(( + parsed.title.trim().to_string(), + content, + normalize_author(parsed.byline), + )) +} + +fn normalize_author(author: Option) -> Option { + let author = author?; + if author.chars().any(|c| matches!(c, '\n' | '\r')) { + return None; + } + let author = author.split_whitespace().collect::>().join(" "); + (!author.is_empty() && author.chars().count() <= 100).then_some(author) } /// Copy an [`Extracted`] onto its [`Article`]. @@ -348,6 +366,11 @@ pub fn apply(article: &mut Article, extracted: Extracted) { article.word_count = extracted.word_count; article.excerpt_only = extracted.excerpt_only; article.extract_method = extracted.method; + if let Some(author) = extracted.author + && (article.author.is_none() || !article.came_via(SourceKind::Feed)) + { + article.author = Some(author); + } } fn merge_paywall_domains(configured: Vec) -> Vec { @@ -722,11 +745,94 @@ mod tests {

A Post

{paragraph}

{paragraph}

\
© 2026
" ); - let (title, content) = readable_page(&html, "https://blog.dev/p").expect("main content"); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); assert_eq!(title, "A Post"); + assert_eq!(author, None); assert!(content.contains("Readability keeps the body copy")); let clean = sanitize_with_base(&content, "https://blog.dev/p"); assert!(word_count(&clean) > 200); assert!(!clean.contains("A Post\ + \ +

A Post

{}

", + "Substantial body copy for readability. ".repeat(40) + ); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); + let page = Page { + title, + html: content, + author, + final_url: "https://blog.dev/p".into(), + }; + let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); + assert_eq!(extracted.author.as_deref(), Some("Jane Dev")); + } + + #[test] + fn readable_page_plumbs_json_ld_author_through_extracted() { + let html = format!( + r#"A Post + +

A Post

{}

"#, + "Substantial body copy for readability. ".repeat(40) + ); + let (title, content, author) = + readable_page(&html, "https://blog.dev/p").expect("main content"); + let page = Page { + title, + html: content, + author, + final_url: "https://blog.dev/p".into(), + }; + let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); + assert_eq!(extracted.author.as_deref(), Some("Alex Writer")); + } + + #[test] + fn apply_uses_page_author_with_feed_precedence() { + let extracted = |author: &str| Extracted { + content_html: "

body

".into(), + author: Some(author.into()), + word_count: 1, + excerpt_only: false, + image_urls: vec![], + method: ExtractMethod::Readability, + }; + + let mut aggregator = article("https://blog.dev/aggregator", ""); + aggregator.sources[0].kind = SourceKind::HnFrontpage; + aggregator.author = Some("Submitter".into()); + apply(&mut aggregator, extracted("Page Writer")); + assert_eq!(aggregator.author.as_deref(), Some("Page Writer")); + + let mut direct = article("https://blog.dev/direct", ""); + direct.author = Some("Feed Writer".into()); + apply(&mut direct, extracted("Page Writer")); + assert_eq!(direct.author.as_deref(), Some("Feed Writer")); + + let mut missing = article("https://blog.dev/missing", ""); + apply(&mut missing, extracted("Page Writer")); + assert_eq!(missing.author.as_deref(), Some("Page Writer")); + } + + #[test] + fn implausible_page_authors_are_dropped() { + assert_eq!( + normalize_author(Some(" Jane Dev ".into())).as_deref(), + Some("Jane Dev") + ); + assert_eq!(normalize_author(Some("Jane\nDev".into())), None); + assert_eq!(normalize_author(Some("x".repeat(101))), None); + assert_eq!(normalize_author(Some(" ".into())), None); + } } diff --git a/src/imports.rs b/src/imports.rs index d742017..18130e2 100644 --- a/src/imports.rs +++ b/src/imports.rs @@ -165,7 +165,7 @@ fn imported_article( sources: Vec::new(), first_seen, url: page.final_url, - author: None, + author: extracted.author, feed_id: 0, feed_title: "Imported".into(), category: None, @@ -482,6 +482,7 @@ mod tests { let page = Page { title: "A historical essay".into(), html: "

Useful old writing.

".into(), + author: Some("Essay Writer".into()), final_url: "https://example.com/essays/old".into(), }; let extracted = extractor.finish_readable("https://example.com/old", &page); @@ -494,6 +495,7 @@ mod tests { assert_eq!(article.title, "A historical essay"); assert_eq!(article.best_entry_id, 0); assert_eq!(article.feed_title, "Imported"); + assert_eq!(article.author.as_deref(), Some("Essay Writer")); assert!(article.sources.is_empty()); assert!(!article.content_html.contains("script")); assert_eq!(article.image_urls, ["https://example.com/chart.png"]); @@ -502,6 +504,7 @@ mod tests { assert_eq!(loaded.title, "A historical essay"); assert_eq!(loaded.best_entry_id, 0); assert_eq!(loaded.feed_title, "Imported"); + assert_eq!(loaded.author.as_deref(), Some("Essay Writer")); } #[tokio::test] diff --git a/src/types.rs b/src/types.rs index 5d0df8d..6ebcbc0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -90,6 +90,8 @@ pub enum ExtractMethod { pub struct Extracted { /// Sanitized XHTML-safe body markup. pub content_html: String, + /// Author discovered in the fetched page, if any. + pub author: Option, pub word_count: i64, /// True when we only have an excerpt/paywall stub — penalized in pre-filter. pub excerpt_only: bool, @@ -119,7 +121,7 @@ pub struct Article { pub sources: Vec, pub first_seen: Timestamp, - // --- denormalized, not stored on `articles` --- + // --- joined / derived fields --- pub url: String, pub author: Option, pub feed_id: FeedId,