Keep the publication an article's page declares

Readability already reads og:site_name and JSON-LD publisher.name; carry
that through Page and Extracted and store it in articles.publication, the
same way the page byline is kept. `publication_label` is what the readers
will see after the feed name: the site name, else the domain, and nothing
when it would only repeat the feed's own title.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YWmCpUojfHXhSZ2129Z7Nv
This commit is contained in:
2026-09-09 05:48:46 +00:00
co-authored by Claude Fable 5.1
parent c8df55e91a
commit f71a4c2799
16 changed files with 161 additions and 37 deletions
+1
View File
@@ -358,6 +358,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
first_seen: "2026-08-15T05:30:00Z".parse().unwrap(), first_seen: "2026-08-15T05:30:00Z".parse().unwrap(),
url: target.url.clone(), url: target.url.clone(),
author: None, author: None,
publication: None,
feed_id: 1, feed_id: 1,
feed_title: host(&target.url), feed_title: host(&target.url),
category: None, category: None,
+3
View File
@@ -0,0 +1,3 @@
-- Persist page-extracted publication names independently of the best Miniflux entry.
ALTER TABLE articles ADD COLUMN publication TEXT;
+1
View File
@@ -1089,6 +1089,7 @@ mod tests {
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
url: format!("https://example.com/{id}"), url: format!("https://example.com/{id}"),
author: Some("Secret Author".into()), author: Some("Secret Author".into()),
publication: None,
feed_id: 9, feed_id: 9,
feed_title: "Secret Feed".into(), feed_title: "Secret Feed".into(),
category: None, category: None,
+1
View File
@@ -154,6 +154,7 @@ pub(crate) mod tests {
first_seen: ts(), first_seen: ts(),
url: format!("https://example.com/{id}"), url: format!("https://example.com/{id}"),
author: Some("A. Writer".into()), author: Some("A. Writer".into()),
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Some Blog".into(), feed_title: "Some Blog".into(),
category: Some("Tech".into()), category: Some("Tech".into()),
+1
View File
@@ -763,6 +763,7 @@ mod tests {
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
url: format!("https://example.com/{id}"), url: format!("https://example.com/{id}"),
author: None, author: None,
publication: None,
feed_id: feeds.first().copied().unwrap_or(0), feed_id: feeds.first().copied().unwrap_or(0),
feed_title: String::new(), feed_title: String::new(),
category: None, category: None,
+15 -5
View File
@@ -292,16 +292,18 @@ impl Db {
/// Upsert a deduped cluster by canonical URL; returns its `articles.id`. /// Upsert a deduped cluster by canonical URL; returns its `articles.id`.
/// ///
/// Most denormalized fields on [`Article`] come from the joined `entries` /// Most denormalized fields on [`Article`] come from the joined `entries`
/// row when loading; the extracted author is stored on `articles`. /// row when loading; extracted page metadata is stored on `articles`.
pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> { pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> {
let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into()); let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into());
let row = sqlx::query( let row = sqlx::query(
"INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html, "INSERT INTO articles (canonical_url, title, author, publication, best_entry_id,
word_count, excerpt_only, image_count, sources_json, first_seen) content_html, word_count, excerpt_only, image_count,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) sources_json, first_seen)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(canonical_url) DO UPDATE SET ON CONFLICT(canonical_url) DO UPDATE SET
title = excluded.title, title = excluded.title,
author = excluded.author, author = excluded.author,
publication = excluded.publication,
best_entry_id = excluded.best_entry_id, best_entry_id = excluded.best_entry_id,
content_html = excluded.content_html, content_html = excluded.content_html,
word_count = excluded.word_count, word_count = excluded.word_count,
@@ -313,6 +315,7 @@ impl Db {
.bind(&article.canonical_url) .bind(&article.canonical_url)
.bind(&article.title) .bind(&article.title)
.bind(&article.author) .bind(&article.author)
.bind(&article.publication)
.bind((article.best_entry_id != 0).then_some(article.best_entry_id)) .bind((article.best_entry_id != 0).then_some(article.best_entry_id))
.bind(&article.content_html) .bind(&article.content_html)
.bind(article.word_count) .bind(article.word_count)
@@ -889,7 +892,8 @@ macro_rules! article_select {
a.word_count AS word_count, a.excerpt_only AS excerpt_only, a.word_count AS word_count, a.excerpt_only AS excerpt_only,
a.image_count AS image_count, a.sources_json AS sources_json, a.image_count AS image_count, a.sources_json AS sources_json,
a.first_seen AS first_seen, a.first_seen AS first_seen,
e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id, e.url AS entry_url, COALESCE(a.author, e.author) AS author,
a.publication AS publication, e.feed_id AS feed_id,
e.feed_title AS feed_title, e.category AS category, e.feed_title AS feed_title, e.category AS category,
e.published_at AS published_at, e.comments_url AS comments_url e.published_at AS published_at, e.comments_url AS comments_url
FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id
@@ -967,6 +971,7 @@ fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
.get::<Option<String>, _>("entry_url") .get::<Option<String>, _>("entry_url")
.unwrap_or_else(|| row.get("canonical_url")), .unwrap_or_else(|| row.get("canonical_url")),
author: row.get("author"), author: row.get("author"),
publication: row.get("publication"),
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0), feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
feed_title: row feed_title: row
.get::<Option<String>, _>("feed_title") .get::<Option<String>, _>("feed_title")
@@ -1193,6 +1198,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"), first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(), url: "https://example.com/1".into(),
author: Some("Page Writer".into()), author: Some("Page Writer".into()),
publication: Some("Example Gazette".into()),
feed_id: 7, feed_id: 7,
feed_title: "Hacker News".into(), feed_title: "Hacker News".into(),
category: None, category: None,
@@ -1224,6 +1230,7 @@ mod tests {
assert_eq!(loaded.social[0].score, 342); assert_eq!(loaded.social[0].score, 342);
assert_eq!(loaded.feed_title, "Hacker News"); assert_eq!(loaded.feed_title, "Hacker News");
assert_eq!(loaded.author.as_deref(), Some("Page Writer")); assert_eq!(loaded.author.as_deref(), Some("Page Writer"));
assert_eq!(loaded.publication.as_deref(), Some("Example Gazette"));
// The batched lookup agrees with the single one and skips unknown ids. // The batched lookup agrees with the single one and skips unknown ids.
let batch = db.get_articles(&[id, 9_999]).await.unwrap(); let batch = db.get_articles(&[id, 9_999]).await.unwrap();
assert_eq!(batch.len(), 1); assert_eq!(batch.len(), 1);
@@ -1261,6 +1268,7 @@ mod tests {
first_seen: ts("2026-09-06T00:00:00Z"), first_seen: ts("2026-09-06T00:00:00Z"),
url: "https://example.com/imported".into(), url: "https://example.com/imported".into(),
author: None, author: None,
publication: None,
feed_id: 0, feed_id: 0,
feed_title: "Imported".into(), feed_title: "Imported".into(),
category: None, category: None,
@@ -1479,6 +1487,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"), first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(), url: "https://example.com/1".into(),
author: None, author: None,
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Hacker News".into(), feed_title: "Hacker News".into(),
category: None, category: None,
@@ -1596,6 +1605,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"), first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(), url: "https://example.com/1".into(),
author: Some("Page Writer".into()), author: Some("Page Writer".into()),
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Hacker News".into(), feed_title: "Hacker News".into(),
category: None, category: None,
+1
View File
@@ -409,6 +409,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
first_seen, first_seen,
url: best_entry.url.clone(), url: best_entry.url.clone(),
author, author,
publication: None,
feed_id: best_entry.feed_id, feed_id: best_entry.feed_id,
feed_title: best_entry feed_title: best_entry
.feed_title .feed_title
+1
View File
@@ -1049,6 +1049,7 @@ mod tests {
first_seen: ts("2026-09-07T00:00:00Z"), first_seen: ts("2026-09-07T00:00:00Z"),
url: url.into(), url: url.into(),
author: None, author: None,
publication: None,
feed_id: 1, feed_id: 1,
feed_title: "Feed".into(), feed_title: "Feed".into(),
category: None, category: None,
+1
View File
@@ -35,6 +35,7 @@ pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article {
first_seen: timestamp(), first_seen: timestamp(),
url: format!("https://example.com/{entry_id}"), url: format!("https://example.com/{entry_id}"),
author: Some("A. Writer".into()), author: Some("A. Writer".into()),
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Example Feed".into(), feed_title: "Example Feed".into(),
category: Some("Tech".into()), category: Some("Tech".into()),
+44 -22
View File
@@ -198,6 +198,7 @@ impl Extractor {
Extracted { Extracted {
content_html, content_html,
author: None, author: None,
publication: None,
word_count: words, word_count: words,
excerpt_only, excerpt_only,
image_urls, image_urls,
@@ -280,11 +281,12 @@ impl Extractor {
body.extend_from_slice(&chunk); body.extend_from_slice(&chunk);
} }
let html = String::from_utf8_lossy(&body).into_owned(); let html = String::from_utf8_lossy(&body).into_owned();
let (title, html, author) = readable_page(&html, &final_url)?; let (title, html, author, site_name) = readable_page(&html, &final_url)?;
Ok(Page { Ok(Page {
title, title,
html, html,
author, author,
site_name,
final_url, final_url,
}) })
} }
@@ -297,6 +299,7 @@ impl Extractor {
Extracted { Extracted {
content_html: clean, content_html: clean,
author: page.author.clone(), author: page.author.clone(),
publication: page.site_name.clone(),
word_count: words, word_count: words,
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains), excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
image_urls, image_urls,
@@ -314,6 +317,8 @@ pub struct Page {
pub html: String, pub html: String,
/// Readability's normalized byline for the page. /// Readability's normalized byline for the page.
pub author: Option<String>, pub author: Option<String>,
/// Readability's normalized site name for the page.
pub site_name: Option<String>,
/// Where the fetch ended up, after any redirects — the base for relative URLs. /// Where the fetch ended up, after any redirects — the base for relative URLs.
pub final_url: String, pub final_url: String,
} }
@@ -326,10 +331,13 @@ pub struct Page {
/// their image with them) and its lazy-image heuristic overwrites a perfectly /// their image with them) and its lazy-image heuristic overwrites a perfectly
/// good `src` with whatever other attribute happens to contain `.jpg`. /// good `src` with whatever other attribute happens to contain `.jpg`.
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> { pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
readable_page(html, url).map(|(_, content, _)| content) readable_page(html, url).map(|(_, content, _, _)| content)
} }
fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String>), ExtractError> { fn readable_page(
html: &str,
url: &str,
) -> Result<(String, String, Option<String>, Option<String>), ExtractError> {
let html = prepare_for_readability(html); let html = prepare_for_readability(html);
let config = dom_smoothie::Config { let config = dom_smoothie::Config {
max_elements_to_parse: 60_000, max_elements_to_parse: 60_000,
@@ -345,17 +353,18 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String
Ok(( Ok((
parsed.title.trim().to_string(), parsed.title.trim().to_string(),
content, content,
normalize_author(parsed.byline), normalize_meta_text(parsed.byline),
normalize_meta_text(parsed.site_name),
)) ))
} }
fn normalize_author(author: Option<String>) -> Option<String> { fn normalize_meta_text(text: Option<String>) -> Option<String> {
let author = author?; let text = text?;
if author.chars().any(|c| matches!(c, '\n' | '\r')) { if text.chars().any(|c| matches!(c, '\n' | '\r')) {
return None; return None;
} }
let author = author.split_whitespace().collect::<Vec<_>>().join(" "); let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
(!author.is_empty() && author.chars().count() <= 100).then_some(author) (!text.is_empty() && text.chars().count() <= 100).then_some(text)
} }
/// Copy an [`Extracted`] onto its [`Article`]. /// Copy an [`Extracted`] onto its [`Article`].
@@ -371,6 +380,9 @@ pub fn apply(article: &mut Article, extracted: Extracted) {
{ {
article.author = Some(author); article.author = Some(author);
} }
if let Some(publication) = extracted.publication {
article.publication = Some(publication);
}
} }
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> { fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
@@ -518,6 +530,7 @@ mod tests {
first_seen: ts(), first_seen: ts(),
url: url.into(), url: url.into(),
author: None, author: None,
publication: None,
feed_id: 1, feed_id: 1,
feed_title: "Feed".into(), feed_title: "Feed".into(),
category: None, category: None,
@@ -745,10 +758,11 @@ mod tests {
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\ <article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
<footer>© 2026</footer></body></html>" <footer>© 2026</footer></body></html>"
); );
let (title, content, author) = let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content"); readable_page(&html, "https://blog.dev/p").expect("main content");
assert_eq!(title, "A Post"); assert_eq!(title, "A Post");
assert_eq!(author, None); assert_eq!(author, None);
assert_eq!(site_name, None);
assert!(content.contains("Readability keeps the body copy")); assert!(content.contains("Readability keeps the body copy"));
let clean = sanitize_with_base(&content, "https://blog.dev/p"); let clean = sanitize_with_base(&content, "https://blog.dev/p");
assert!(word_count(&clean) > 200); assert!(word_count(&clean) > 200);
@@ -756,53 +770,60 @@ mod tests {
} }
#[test] #[test]
fn readable_page_plumbs_meta_author_through_extracted() { fn readable_page_plumbs_meta_author_and_og_site_name_through_extracted() {
let html = format!( let html = format!(
"<html><head><title>A Post</title>\ "<html><head><title>A Post</title>\
<meta name=\"author\" content=\" Jane Dev \"></head>\ <meta name=\"author\" content=\" Jane Dev \">\
<meta property=\"og:site_name\" content=\" Dev Journal \"></head>\
<body><article><h1>A Post</h1><p>{}</p></article></body></html>", <body><article><h1>A Post</h1><p>{}</p></article></body></html>",
"Substantial body copy for readability. ".repeat(40) "Substantial body copy for readability. ".repeat(40)
); );
let (title, content, author) = let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content"); readable_page(&html, "https://blog.dev/p").expect("main content");
let page = Page { let page = Page {
title, title,
html: content, html: content,
author, author,
site_name,
final_url: "https://blog.dev/p".into(), final_url: "https://blog.dev/p".into(),
}; };
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
assert_eq!(extracted.author.as_deref(), Some("Jane Dev")); assert_eq!(extracted.author.as_deref(), Some("Jane Dev"));
assert_eq!(extracted.publication.as_deref(), Some("Dev Journal"));
} }
#[test] #[test]
fn readable_page_plumbs_json_ld_author_through_extracted() { fn readable_page_plumbs_json_ld_author_and_publisher_through_extracted() {
let html = format!( let html = format!(
r#"<html><head><title>A Post</title> r#"<html><head><title>A Post</title>
<script type="application/ld+json">{{ <script type="application/ld+json">{{
"@context":"https://schema.org", "@type":"Article", "@context":"https://schema.org", "@type":"Article",
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}} "headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}},
"publisher":{{"@type":"Organization","name":"Example Gazette"}}
}}</script></head> }}</script></head>
<body><article><h1>A Post</h1><p>{}</p></article></body></html>"#, <body><article><h1>A Post</h1><p>{}</p></article></body></html>"#,
"Substantial body copy for readability. ".repeat(40) "Substantial body copy for readability. ".repeat(40)
); );
let (title, content, author) = let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content"); readable_page(&html, "https://blog.dev/p").expect("main content");
let page = Page { let page = Page {
title, title,
html: content, html: content,
author, author,
site_name,
final_url: "https://blog.dev/p".into(), final_url: "https://blog.dev/p".into(),
}; };
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
assert_eq!(extracted.author.as_deref(), Some("Alex Writer")); assert_eq!(extracted.author.as_deref(), Some("Alex Writer"));
assert_eq!(extracted.publication.as_deref(), Some("Example Gazette"));
} }
#[test] #[test]
fn apply_uses_page_author_with_feed_precedence() { fn apply_uses_page_metadata() {
let extracted = |author: &str| Extracted { let extracted = |author: &str| Extracted {
content_html: "<p>body</p>".into(), content_html: "<p>body</p>".into(),
author: Some(author.into()), author: Some(author.into()),
publication: Some("Example Gazette".into()),
word_count: 1, word_count: 1,
excerpt_only: false, excerpt_only: false,
image_urls: vec![], image_urls: vec![],
@@ -814,6 +835,7 @@ mod tests {
aggregator.author = Some("Submitter".into()); aggregator.author = Some("Submitter".into());
apply(&mut aggregator, extracted("Page Writer")); apply(&mut aggregator, extracted("Page Writer"));
assert_eq!(aggregator.author.as_deref(), Some("Page Writer")); assert_eq!(aggregator.author.as_deref(), Some("Page Writer"));
assert_eq!(aggregator.publication.as_deref(), Some("Example Gazette"));
let mut direct = article("https://blog.dev/direct", ""); let mut direct = article("https://blog.dev/direct", "");
direct.author = Some("Feed Writer".into()); direct.author = Some("Feed Writer".into());
@@ -826,13 +848,13 @@ mod tests {
} }
#[test] #[test]
fn implausible_page_authors_are_dropped() { fn implausible_page_metadata_is_dropped() {
assert_eq!( assert_eq!(
normalize_author(Some(" Jane Dev ".into())).as_deref(), normalize_meta_text(Some(" Jane Dev ".into())).as_deref(),
Some("Jane Dev") Some("Jane Dev")
); );
assert_eq!(normalize_author(Some("Jane\nDev".into())), None); assert_eq!(normalize_meta_text(Some("Jane\nDev".into())), None);
assert_eq!(normalize_author(Some("x".repeat(101))), None); assert_eq!(normalize_meta_text(Some("x".repeat(101))), None);
assert_eq!(normalize_author(Some(" ".into())), None); assert_eq!(normalize_meta_text(Some(" ".into())), None);
} }
} }
+5
View File
@@ -167,6 +167,7 @@ fn imported_article(
first_seen, first_seen,
url: page.final_url, url: page.final_url,
author: extracted.author, author: extracted.author,
publication: extracted.publication,
feed_id: 0, feed_id: 0,
feed_title: "Imported".into(), feed_title: "Imported".into(),
category: None, category: None,
@@ -379,6 +380,7 @@ mod tests {
first_seen: now, first_seen: now,
url: url.into(), url: url.into(),
author: None, author: None,
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Feed".into(), feed_title: "Feed".into(),
category: None, category: None,
@@ -484,6 +486,7 @@ mod tests {
title: "A historical essay".into(), title: "A historical essay".into(),
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(), html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
author: Some("Essay Writer".into()), author: Some("Essay Writer".into()),
site_name: Some("Example Review".into()),
final_url: "https://example.com/essays/old".into(), final_url: "https://example.com/essays/old".into(),
}; };
let extracted = extractor.finish_readable("https://example.com/old", &page); let extracted = extractor.finish_readable("https://example.com/old", &page);
@@ -497,6 +500,7 @@ mod tests {
assert_eq!(article.best_entry_id, 0); assert_eq!(article.best_entry_id, 0);
assert_eq!(article.feed_title, "Imported"); assert_eq!(article.feed_title, "Imported");
assert_eq!(article.author.as_deref(), Some("Essay Writer")); assert_eq!(article.author.as_deref(), Some("Essay Writer"));
assert_eq!(article.publication.as_deref(), Some("Example Review"));
assert!(article.sources.is_empty()); assert!(article.sources.is_empty());
assert!(!article.content_html.contains("script")); assert!(!article.content_html.contains("script"));
assert_eq!(article.image_urls, ["https://example.com/chart.png"]); assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
@@ -506,6 +510,7 @@ mod tests {
assert_eq!(loaded.best_entry_id, 0); assert_eq!(loaded.best_entry_id, 0);
assert_eq!(loaded.feed_title, "Imported"); assert_eq!(loaded.feed_title, "Imported");
assert_eq!(loaded.author.as_deref(), Some("Essay Writer")); assert_eq!(loaded.author.as_deref(), Some("Essay Writer"));
assert_eq!(loaded.publication.as_deref(), Some("Example Review"));
} }
#[tokio::test] #[tokio::test]
+1
View File
@@ -1530,6 +1530,7 @@ mod tests {
first_seen: now(), first_seen: now(),
url, url,
author: None, author: None,
publication: None,
feed_id: 100 + entry_id, feed_id: 100 + entry_id,
feed_title: format!("Feed {entry_id}"), feed_title: format!("Feed {entry_id}"),
category: None, category: None,
+1
View File
@@ -972,6 +972,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"), first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(), url: "https://example.com/1".into(),
author: None, author: None,
publication: None,
feed_id: 7, feed_id: 7,
feed_title: "Hacker News".into(), feed_title: "Hacker News".into(),
category: None, category: None,
+1
View File
@@ -328,6 +328,7 @@ mod tests {
first_seen: ts("2026-08-15T05:00:00Z"), first_seen: ts("2026-08-15T05:00:00Z"),
url: url.into(), url: url.into(),
author: None, author: None,
publication: None,
feed_id: 1, feed_id: 1,
feed_title: "Feed".into(), feed_title: "Feed".into(),
category: None, category: None,
+82
View File
@@ -92,6 +92,8 @@ pub struct Extracted {
pub content_html: String, pub content_html: String,
/// Author discovered in the fetched page, if any. /// Author discovered in the fetched page, if any.
pub author: Option<String>, pub author: Option<String>,
/// Site name discovered in the fetched page, if any.
pub publication: Option<String>,
pub word_count: i64, pub word_count: i64,
/// True when we only have an excerpt/paywall stub — penalized in pre-filter. /// True when we only have an excerpt/paywall stub — penalized in pre-filter.
pub excerpt_only: bool, pub excerpt_only: bool,
@@ -124,6 +126,8 @@ pub struct Article {
// --- joined / derived fields --- // --- joined / derived fields ---
pub url: String, pub url: String,
pub author: Option<String>, pub author: Option<String>,
/// Page-extracted site name persisted on `articles`.
pub publication: Option<String>,
pub feed_id: FeedId, pub feed_id: FeedId,
pub feed_title: String, pub feed_title: String,
pub category: Option<String>, pub category: Option<String>,
@@ -150,6 +154,19 @@ impl Article {
self.sources.iter().any(|s| s.kind == kind) self.sources.iter().any(|s| s.kind == kind)
} }
/// What to show after the feed name: the page's site name, else the domain;
/// nothing when it would just repeat the feed name.
pub fn publication_label(&self) -> Option<String> {
let publication = self
.publication
.clone()
.or_else(|| domain(&self.canonical_url))?;
(!publication
.trim()
.eq_ignore_ascii_case(self.feed_title.trim()))
.then_some(publication)
}
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12). /// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
pub fn chapter_id(&self) -> String { pub fn chapter_id(&self) -> String {
format!("art-{}", self.best_entry_id) format!("art-{}", self.best_entry_id)
@@ -161,6 +178,14 @@ pub fn reading_minutes(word_count: i64) -> i64 {
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64 (word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
} }
/// URL host without a leading `www.`.
pub fn domain(url: &str) -> Option<String> {
url::Url::parse(url)
.ok()?
.host_str()
.map(|host| host.strip_prefix("www.").unwrap_or(host).to_string())
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Social proof (§3.4) // Social proof (§3.4)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -847,6 +872,63 @@ mod tests {
} }
} }
fn publication_article() -> Article {
Article {
id: 1,
canonical_url: "https://www.example.com/post".into(),
title: "A post".into(),
best_entry_id: 1,
content_html: String::new(),
word_count: 1,
excerpt_only: false,
image_count: 0,
sources: Vec::new(),
first_seen: ts(),
url: "https://www.example.com/post".into(),
author: None,
publication: None,
feed_id: 1,
feed_title: "A Feed".into(),
category: None,
published_at: None,
comments_url: None,
image_urls: Vec::new(),
social: Vec::new(),
extract_method: ExtractMethod::Readability,
}
}
#[test]
fn publication_label_prefers_the_stored_publication() {
let mut article = publication_article();
article.publication = Some("Example Journal".into());
assert_eq!(
article.publication_label().as_deref(),
Some("Example Journal")
);
}
#[test]
fn publication_label_falls_back_to_the_domain_without_www() {
let article = publication_article();
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
}
#[test]
fn publication_label_omits_a_publication_matching_the_feed() {
let mut article = publication_article();
article.feed_title = " example JOURNAL ".into();
article.publication = Some(" Example Journal ".into());
assert_eq!(article.publication_label(), None);
}
#[test]
fn publication_label_omits_a_domain_matching_the_feed() {
let mut article = publication_article();
article.feed_title = " EXAMPLE.com ".into();
assert_eq!(article.publication_label(), None);
}
#[test] #[test]
fn composite_social_score_matches_spec_formula() { fn composite_social_score_matches_spec_formula() {
let refs = vec![ let refs = vec![
+2 -10
View File
@@ -6,7 +6,7 @@ use axum_login::tower_sessions::Session;
use jiff::civil::Date; use jiff::civil::Date;
use crate::server::AppState; use crate::server::AppState;
use crate::types::{Issue, SocialSource}; use crate::types::{Issue, SocialSource, domain};
use crate::web::issue::{self, Download}; use crate::web::issue::{self, Download};
use crate::web::session::{AuthSession, Viewer}; use crate::web::session::{AuthSession, Viewer};
use crate::web::{Html, Page, WebError}; use crate::web::{Html, Page, WebError};
@@ -116,7 +116,7 @@ impl From<&Issue> for PublicIssue {
url: article.canonical_url.clone(), url: article.canonical_url.clone(),
author: article.author.clone(), author: article.author.clone(),
source: article.feed_title.clone(), source: article.feed_title.clone(),
domain: domain(&article.canonical_url), domain: domain(&article.canonical_url).unwrap_or_default(),
reading_minutes: article.reading_minutes(), reading_minutes: article.reading_minutes(),
word_count: article.word_count, word_count: article.word_count,
summary: pick summary: pick
@@ -169,14 +169,6 @@ impl PublicIssue {
} }
} }
fn domain(raw: &str) -> String {
url::Url::parse(raw)
.ok()
.and_then(|url| url.host_str().map(str::to_string))
.map(|host| host.strip_prefix("www.").unwrap_or(&host).to_string())
.unwrap_or_default()
}
#[derive(Template)] #[derive(Template)]
#[template(path = "issue_public.html")] #[template(path = "issue_public.html")]
struct IssuePublicTemplate { struct IssuePublicTemplate {