Take the author from the article page, not the aggregator submitter
Readability already finds a byline (JSON-LD author, meta author tags, byline markup); keep it on the article and let it replace an aggregator entry's author, which for HN and friends is the submitter. A direct feed's own author is still trusted over the page, and an aggregator name is only used at all when no direct feed carried the story. The author is stored on articles so it survives independently of the best entry. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QVPagF6jfDv78CC5Jv2wp4
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,3 @@
|
||||
-- Persist page-extracted authors independently of the best Miniflux entry.
|
||||
|
||||
ALTER TABLE articles ADD COLUMN author TEXT;
|
||||
@@ -291,16 +291,17 @@ impl Db {
|
||||
|
||||
/// Upsert a deduped cluster by canonical URL; returns its `articles.id`.
|
||||
///
|
||||
/// The denormalized fields on [`Article`] are not stored here — they come
|
||||
/// from the joined `entries` row when loading.
|
||||
/// Most denormalized fields on [`Article`] come from the joined `entries`
|
||||
/// row when loading; the extracted author is stored on `articles`.
|
||||
pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> {
|
||||
let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into());
|
||||
let row = sqlx::query(
|
||||
"INSERT INTO articles (canonical_url, title, best_entry_id, content_html, word_count,
|
||||
excerpt_only, image_count, sources_json, first_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html,
|
||||
word_count, excerpt_only, image_count, sources_json, first_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(canonical_url) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
author = excluded.author,
|
||||
best_entry_id = excluded.best_entry_id,
|
||||
content_html = excluded.content_html,
|
||||
word_count = excluded.word_count,
|
||||
@@ -311,6 +312,7 @@ impl Db {
|
||||
)
|
||||
.bind(&article.canonical_url)
|
||||
.bind(&article.title)
|
||||
.bind(&article.author)
|
||||
.bind((article.best_entry_id != 0).then_some(article.best_entry_id))
|
||||
.bind(&article.content_html)
|
||||
.bind(article.word_count)
|
||||
@@ -860,7 +862,7 @@ macro_rules! article_select {
|
||||
a.word_count AS word_count, a.excerpt_only AS excerpt_only,
|
||||
a.image_count AS image_count, a.sources_json AS sources_json,
|
||||
a.first_seen AS first_seen,
|
||||
e.url AS entry_url, e.author AS author, e.feed_id AS feed_id,
|
||||
e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id,
|
||||
e.feed_title AS feed_title, e.category AS category,
|
||||
e.published_at AS published_at, e.comments_url AS comments_url
|
||||
FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||
@@ -1163,7 +1165,7 @@ mod tests {
|
||||
}],
|
||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||
url: "https://example.com/1".into(),
|
||||
author: None,
|
||||
author: Some("Page Writer".into()),
|
||||
feed_id: 7,
|
||||
feed_title: "Hacker News".into(),
|
||||
category: None,
|
||||
@@ -1194,11 +1196,20 @@ mod tests {
|
||||
assert_eq!(loaded.social.len(), 1);
|
||||
assert_eq!(loaded.social[0].score, 342);
|
||||
assert_eq!(loaded.feed_title, "Hacker News");
|
||||
assert_eq!(loaded.author.as_deref(), Some("Page Writer"));
|
||||
// The batched lookup agrees with the single one and skips unknown ids.
|
||||
let batch = db.get_articles(&[id, 9_999]).await.unwrap();
|
||||
assert_eq!(batch.len(), 1);
|
||||
assert_eq!(batch[&id], loaded);
|
||||
assert!(db.get_articles(&[]).await.unwrap().is_empty());
|
||||
|
||||
sqlx::query("UPDATE articles SET author = NULL WHERE id = ?")
|
||||
.bind(id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let fallback = db.get_article(id).await.unwrap().unwrap();
|
||||
assert_eq!(fallback.author.as_deref(), Some("someone"));
|
||||
assert_eq!(
|
||||
db.article_id_for_url("https://example.com/1")
|
||||
.await
|
||||
|
||||
@@ -380,8 +380,17 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
|
||||
}
|
||||
});
|
||||
|
||||
// An aggregator entry's author is usually the submitter, so it only counts
|
||||
// when no direct feed carried the story; extraction may still replace it
|
||||
// with the page's own byline.
|
||||
let is_direct = |e: &Entry| {
|
||||
classify_source_with_feed(e, feed_urls.get(&e.feed_id).map(String::as_str))
|
||||
== SourceKind::Feed
|
||||
};
|
||||
let has_direct = members.iter().any(|(e, _)| is_direct(e));
|
||||
let author = members
|
||||
.iter()
|
||||
.filter(|(e, _)| !has_direct || is_direct(e))
|
||||
.filter_map(|(e, _)| e.author.clone())
|
||||
.find(|a| !a.trim().is_empty());
|
||||
|
||||
@@ -638,6 +647,38 @@ mod tests {
|
||||
assert_eq!(articles[1].sources.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_feed_author_beats_aggregator_submitter() {
|
||||
let mut aggregator = entry(1, "https://blog.dev/post", "A Distinct Article Title");
|
||||
aggregator.author = Some("HN Submitter".into());
|
||||
aggregator.raw_content = format!("<p>{}</p>", "word ".repeat(50));
|
||||
|
||||
let mut direct = entry(2, "https://blog.dev/post", "A Distinct Article Title");
|
||||
direct.author = Some("Real Writer".into());
|
||||
|
||||
let feed_urls = FeedUrls::from([
|
||||
(aggregator.feed_id, "https://hnrss.org/frontpage".into()),
|
||||
(direct.feed_id, "https://blog.dev/feed.xml".into()),
|
||||
]);
|
||||
let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls);
|
||||
|
||||
assert_eq!(articles.len(), 1);
|
||||
assert_eq!(articles[0].best_entry_id, 1);
|
||||
assert_eq!(articles[0].author.as_deref(), Some("Real Writer"));
|
||||
|
||||
// With a direct feed present, a submitter name is not used as a fallback.
|
||||
let mut aggregator = entry(3, "https://blog.dev/other", "Another Distinct Title");
|
||||
aggregator.author = Some("HN Submitter".into());
|
||||
let mut direct = entry(4, "https://blog.dev/other", "Another Distinct Title");
|
||||
direct.author = None;
|
||||
let feed_urls = FeedUrls::from([
|
||||
(aggregator.feed_id, "https://hnrss.org/frontpage".into()),
|
||||
(direct.feed_id, "https://blog.dev/feed.xml".into()),
|
||||
]);
|
||||
let (articles, _) = cluster_with_feeds(vec![aggregator, direct], &feed_urls);
|
||||
assert_eq!(articles[0].author, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clustering_drops_non_articles_and_keeps_short_titles_apart() {
|
||||
let mut a = entry(1, "https://a.dev/1", "News");
|
||||
|
||||
+112
-6
@@ -18,7 +18,7 @@ use url::Url;
|
||||
use crate::html::word_count;
|
||||
use crate::images::normalize::{normalize_img_tags, prepare_for_readability};
|
||||
use crate::images::refs::collect_image_urls;
|
||||
use crate::types::{Article, ExtractMethod, Extracted};
|
||||
use crate::types::{Article, ExtractMethod, Extracted, SourceKind};
|
||||
|
||||
/// Word count at or above which Miniflux content is treated as full text (§3.3).
|
||||
pub const FULL_TEXT_MIN_WORDS: i64 = 250;
|
||||
@@ -197,6 +197,7 @@ impl Extractor {
|
||||
|| looks_paywalled(&article.url, words, &self.paywall_domains);
|
||||
Extracted {
|
||||
content_html,
|
||||
author: None,
|
||||
word_count: words,
|
||||
excerpt_only,
|
||||
image_urls,
|
||||
@@ -279,10 +280,11 @@ impl Extractor {
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
let html = String::from_utf8_lossy(&body).into_owned();
|
||||
let (title, html) = readable_page(&html, &final_url)?;
|
||||
let (title, html, author) = readable_page(&html, &final_url)?;
|
||||
Ok(Page {
|
||||
title,
|
||||
html,
|
||||
author,
|
||||
final_url,
|
||||
})
|
||||
}
|
||||
@@ -294,6 +296,7 @@ impl Extractor {
|
||||
let image_urls = collect_image_urls(&clean, &page.final_url);
|
||||
Extracted {
|
||||
content_html: clean,
|
||||
author: page.author.clone(),
|
||||
word_count: words,
|
||||
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
|
||||
image_urls,
|
||||
@@ -309,6 +312,8 @@ pub struct Page {
|
||||
pub title: String,
|
||||
/// Readability's main-content markup.
|
||||
pub html: String,
|
||||
/// Readability's normalized byline for the page.
|
||||
pub author: Option<String>,
|
||||
/// Where the fetch ended up, after any redirects — the base for relative URLs.
|
||||
pub final_url: String,
|
||||
}
|
||||
@@ -321,10 +326,10 @@ pub struct Page {
|
||||
/// their image with them) and its lazy-image heuristic overwrites a perfectly
|
||||
/// good `src` with whatever other attribute happens to contain `.jpg`.
|
||||
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
||||
readable_page(html, url).map(|(_, content)| content)
|
||||
readable_page(html, url).map(|(_, content, _)| content)
|
||||
}
|
||||
|
||||
fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError> {
|
||||
fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String>), ExtractError> {
|
||||
let html = prepare_for_readability(html);
|
||||
let config = dom_smoothie::Config {
|
||||
max_elements_to_parse: 60_000,
|
||||
@@ -337,7 +342,20 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String), ExtractError
|
||||
if content.trim().is_empty() {
|
||||
return Err(ExtractError::NoContent);
|
||||
}
|
||||
Ok((parsed.title.trim().to_string(), content))
|
||||
Ok((
|
||||
parsed.title.trim().to_string(),
|
||||
content,
|
||||
normalize_author(parsed.byline),
|
||||
))
|
||||
}
|
||||
|
||||
fn normalize_author(author: Option<String>) -> Option<String> {
|
||||
let author = author?;
|
||||
if author.chars().any(|c| matches!(c, '\n' | '\r')) {
|
||||
return None;
|
||||
}
|
||||
let author = author.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
(!author.is_empty() && author.chars().count() <= 100).then_some(author)
|
||||
}
|
||||
|
||||
/// Copy an [`Extracted`] onto its [`Article`].
|
||||
@@ -348,6 +366,11 @@ pub fn apply(article: &mut Article, extracted: Extracted) {
|
||||
article.word_count = extracted.word_count;
|
||||
article.excerpt_only = extracted.excerpt_only;
|
||||
article.extract_method = extracted.method;
|
||||
if let Some(author) = extracted.author
|
||||
&& (article.author.is_none() || !article.came_via(SourceKind::Feed))
|
||||
{
|
||||
article.author = Some(author);
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
|
||||
@@ -722,11 +745,94 @@ mod tests {
|
||||
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
||||
<footer>© 2026</footer></body></html>"
|
||||
);
|
||||
let (title, content) = readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||
let (title, content, author) =
|
||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||
assert_eq!(title, "A Post");
|
||||
assert_eq!(author, None);
|
||||
assert!(content.contains("Readability keeps the body copy"));
|
||||
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
||||
assert!(word_count(&clean) > 200);
|
||||
assert!(!clean.contains("<nav"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readable_page_plumbs_meta_author_through_extracted() {
|
||||
let html = format!(
|
||||
"<html><head><title>A Post</title>\
|
||||
<meta name=\"author\" content=\" Jane Dev \"></head>\
|
||||
<body><article><h1>A Post</h1><p>{}</p></article></body></html>",
|
||||
"Substantial body copy for readability. ".repeat(40)
|
||||
);
|
||||
let (title, content, author) =
|
||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||
let page = Page {
|
||||
title,
|
||||
html: content,
|
||||
author,
|
||||
final_url: "https://blog.dev/p".into(),
|
||||
};
|
||||
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
||||
assert_eq!(extracted.author.as_deref(), Some("Jane Dev"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readable_page_plumbs_json_ld_author_through_extracted() {
|
||||
let html = format!(
|
||||
r#"<html><head><title>A Post</title>
|
||||
<script type="application/ld+json">{{
|
||||
"@context":"https://schema.org", "@type":"Article",
|
||||
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}}
|
||||
}}</script></head>
|
||||
<body><article><h1>A Post</h1><p>{}</p></article></body></html>"#,
|
||||
"Substantial body copy for readability. ".repeat(40)
|
||||
);
|
||||
let (title, content, author) =
|
||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||
let page = Page {
|
||||
title,
|
||||
html: content,
|
||||
author,
|
||||
final_url: "https://blog.dev/p".into(),
|
||||
};
|
||||
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
||||
assert_eq!(extracted.author.as_deref(), Some("Alex Writer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_uses_page_author_with_feed_precedence() {
|
||||
let extracted = |author: &str| Extracted {
|
||||
content_html: "<p>body</p>".into(),
|
||||
author: Some(author.into()),
|
||||
word_count: 1,
|
||||
excerpt_only: false,
|
||||
image_urls: vec![],
|
||||
method: ExtractMethod::Readability,
|
||||
};
|
||||
|
||||
let mut aggregator = article("https://blog.dev/aggregator", "");
|
||||
aggregator.sources[0].kind = SourceKind::HnFrontpage;
|
||||
aggregator.author = Some("Submitter".into());
|
||||
apply(&mut aggregator, extracted("Page Writer"));
|
||||
assert_eq!(aggregator.author.as_deref(), Some("Page Writer"));
|
||||
|
||||
let mut direct = article("https://blog.dev/direct", "");
|
||||
direct.author = Some("Feed Writer".into());
|
||||
apply(&mut direct, extracted("Page Writer"));
|
||||
assert_eq!(direct.author.as_deref(), Some("Feed Writer"));
|
||||
|
||||
let mut missing = article("https://blog.dev/missing", "");
|
||||
apply(&mut missing, extracted("Page Writer"));
|
||||
assert_eq!(missing.author.as_deref(), Some("Page Writer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn implausible_page_authors_are_dropped() {
|
||||
assert_eq!(
|
||||
normalize_author(Some(" Jane Dev ".into())).as_deref(),
|
||||
Some("Jane Dev")
|
||||
);
|
||||
assert_eq!(normalize_author(Some("Jane\nDev".into())), None);
|
||||
assert_eq!(normalize_author(Some("x".repeat(101))), None);
|
||||
assert_eq!(normalize_author(Some(" ".into())), None);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -165,7 +165,7 @@ fn imported_article(
|
||||
sources: Vec::new(),
|
||||
first_seen,
|
||||
url: page.final_url,
|
||||
author: None,
|
||||
author: extracted.author,
|
||||
feed_id: 0,
|
||||
feed_title: "Imported".into(),
|
||||
category: None,
|
||||
@@ -482,6 +482,7 @@ mod tests {
|
||||
let page = Page {
|
||||
title: "A historical essay".into(),
|
||||
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
|
||||
author: Some("Essay Writer".into()),
|
||||
final_url: "https://example.com/essays/old".into(),
|
||||
};
|
||||
let extracted = extractor.finish_readable("https://example.com/old", &page);
|
||||
@@ -494,6 +495,7 @@ mod tests {
|
||||
assert_eq!(article.title, "A historical essay");
|
||||
assert_eq!(article.best_entry_id, 0);
|
||||
assert_eq!(article.feed_title, "Imported");
|
||||
assert_eq!(article.author.as_deref(), Some("Essay Writer"));
|
||||
assert!(article.sources.is_empty());
|
||||
assert!(!article.content_html.contains("script"));
|
||||
assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
|
||||
@@ -502,6 +504,7 @@ mod tests {
|
||||
assert_eq!(loaded.title, "A historical essay");
|
||||
assert_eq!(loaded.best_entry_id, 0);
|
||||
assert_eq!(loaded.feed_title, "Imported");
|
||||
assert_eq!(loaded.author.as_deref(), Some("Essay Writer"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+3
-1
@@ -90,6 +90,8 @@ pub enum ExtractMethod {
|
||||
pub struct Extracted {
|
||||
/// Sanitized XHTML-safe body markup.
|
||||
pub content_html: String,
|
||||
/// Author discovered in the fetched page, if any.
|
||||
pub author: Option<String>,
|
||||
pub word_count: i64,
|
||||
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
|
||||
pub excerpt_only: bool,
|
||||
@@ -119,7 +121,7 @@ pub struct Article {
|
||||
pub sources: Vec<SourceRef>,
|
||||
pub first_seen: Timestamp,
|
||||
|
||||
// --- denormalized, not stored on `articles` ---
|
||||
// --- joined / derived fields ---
|
||||
pub url: String,
|
||||
pub author: Option<String>,
|
||||
pub feed_id: FeedId,
|
||||
|
||||
Reference in New Issue
Block a user