Keep the publication an article's page declares
Readability already reads og:site_name and JSON-LD publisher.name; carry that through Page and Extracted and store it in articles.publication, the same way the page byline is kept. `publication_label` is what the readers will see after the feed name: the site name, else the domain, and nothing when it would only repeat the feed's own title. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01YWmCpUojfHXhSZ2129Z7Nv
This commit is contained in:
@@ -358,6 +358,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
|
|||||||
first_seen: "2026-08-15T05:30:00Z".parse().unwrap(),
|
first_seen: "2026-08-15T05:30:00Z".parse().unwrap(),
|
||||||
url: target.url.clone(),
|
url: target.url.clone(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 1,
|
feed_id: 1,
|
||||||
feed_title: host(&target.url),
|
feed_title: host(&target.url),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- Persist page-extracted publication names independently of the best Miniflux entry.
|
||||||
|
|
||||||
|
ALTER TABLE articles ADD COLUMN publication TEXT;
|
||||||
@@ -1089,6 +1089,7 @@ mod tests {
|
|||||||
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
|
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
|
||||||
url: format!("https://example.com/{id}"),
|
url: format!("https://example.com/{id}"),
|
||||||
author: Some("Secret Author".into()),
|
author: Some("Secret Author".into()),
|
||||||
|
publication: None,
|
||||||
feed_id: 9,
|
feed_id: 9,
|
||||||
feed_title: "Secret Feed".into(),
|
feed_title: "Secret Feed".into(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -154,6 +154,7 @@ pub(crate) mod tests {
|
|||||||
first_seen: ts(),
|
first_seen: ts(),
|
||||||
url: format!("https://example.com/{id}"),
|
url: format!("https://example.com/{id}"),
|
||||||
author: Some("A. Writer".into()),
|
author: Some("A. Writer".into()),
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Some Blog".into(),
|
feed_title: "Some Blog".into(),
|
||||||
category: Some("Tech".into()),
|
category: Some("Tech".into()),
|
||||||
|
|||||||
@@ -763,6 +763,7 @@ mod tests {
|
|||||||
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
|
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
|
||||||
url: format!("https://example.com/{id}"),
|
url: format!("https://example.com/{id}"),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: feeds.first().copied().unwrap_or(0),
|
feed_id: feeds.first().copied().unwrap_or(0),
|
||||||
feed_title: String::new(),
|
feed_title: String::new(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -292,16 +292,18 @@ impl Db {
|
|||||||
/// Upsert a deduped cluster by canonical URL; returns its `articles.id`.
|
/// Upsert a deduped cluster by canonical URL; returns its `articles.id`.
|
||||||
///
|
///
|
||||||
/// Most denormalized fields on [`Article`] come from the joined `entries`
|
/// Most denormalized fields on [`Article`] come from the joined `entries`
|
||||||
/// row when loading; the extracted author is stored on `articles`.
|
/// row when loading; extracted page metadata is stored on `articles`.
|
||||||
pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> {
|
pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> {
|
||||||
let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into());
|
let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into());
|
||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
"INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html,
|
"INSERT INTO articles (canonical_url, title, author, publication, best_entry_id,
|
||||||
word_count, excerpt_only, image_count, sources_json, first_seen)
|
content_html, word_count, excerpt_only, image_count,
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
sources_json, first_seen)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(canonical_url) DO UPDATE SET
|
ON CONFLICT(canonical_url) DO UPDATE SET
|
||||||
title = excluded.title,
|
title = excluded.title,
|
||||||
author = excluded.author,
|
author = excluded.author,
|
||||||
|
publication = excluded.publication,
|
||||||
best_entry_id = excluded.best_entry_id,
|
best_entry_id = excluded.best_entry_id,
|
||||||
content_html = excluded.content_html,
|
content_html = excluded.content_html,
|
||||||
word_count = excluded.word_count,
|
word_count = excluded.word_count,
|
||||||
@@ -313,6 +315,7 @@ impl Db {
|
|||||||
.bind(&article.canonical_url)
|
.bind(&article.canonical_url)
|
||||||
.bind(&article.title)
|
.bind(&article.title)
|
||||||
.bind(&article.author)
|
.bind(&article.author)
|
||||||
|
.bind(&article.publication)
|
||||||
.bind((article.best_entry_id != 0).then_some(article.best_entry_id))
|
.bind((article.best_entry_id != 0).then_some(article.best_entry_id))
|
||||||
.bind(&article.content_html)
|
.bind(&article.content_html)
|
||||||
.bind(article.word_count)
|
.bind(article.word_count)
|
||||||
@@ -889,7 +892,8 @@ macro_rules! article_select {
|
|||||||
a.word_count AS word_count, a.excerpt_only AS excerpt_only,
|
a.word_count AS word_count, a.excerpt_only AS excerpt_only,
|
||||||
a.image_count AS image_count, a.sources_json AS sources_json,
|
a.image_count AS image_count, a.sources_json AS sources_json,
|
||||||
a.first_seen AS first_seen,
|
a.first_seen AS first_seen,
|
||||||
e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id,
|
e.url AS entry_url, COALESCE(a.author, e.author) AS author,
|
||||||
|
a.publication AS publication, e.feed_id AS feed_id,
|
||||||
e.feed_title AS feed_title, e.category AS category,
|
e.feed_title AS feed_title, e.category AS category,
|
||||||
e.published_at AS published_at, e.comments_url AS comments_url
|
e.published_at AS published_at, e.comments_url AS comments_url
|
||||||
FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id
|
FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id
|
||||||
@@ -967,6 +971,7 @@ fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
|
|||||||
.get::<Option<String>, _>("entry_url")
|
.get::<Option<String>, _>("entry_url")
|
||||||
.unwrap_or_else(|| row.get("canonical_url")),
|
.unwrap_or_else(|| row.get("canonical_url")),
|
||||||
author: row.get("author"),
|
author: row.get("author"),
|
||||||
|
publication: row.get("publication"),
|
||||||
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
|
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
|
||||||
feed_title: row
|
feed_title: row
|
||||||
.get::<Option<String>, _>("feed_title")
|
.get::<Option<String>, _>("feed_title")
|
||||||
@@ -1193,6 +1198,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||||
url: "https://example.com/1".into(),
|
url: "https://example.com/1".into(),
|
||||||
author: Some("Page Writer".into()),
|
author: Some("Page Writer".into()),
|
||||||
|
publication: Some("Example Gazette".into()),
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Hacker News".into(),
|
feed_title: "Hacker News".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -1224,6 +1230,7 @@ mod tests {
|
|||||||
assert_eq!(loaded.social[0].score, 342);
|
assert_eq!(loaded.social[0].score, 342);
|
||||||
assert_eq!(loaded.feed_title, "Hacker News");
|
assert_eq!(loaded.feed_title, "Hacker News");
|
||||||
assert_eq!(loaded.author.as_deref(), Some("Page Writer"));
|
assert_eq!(loaded.author.as_deref(), Some("Page Writer"));
|
||||||
|
assert_eq!(loaded.publication.as_deref(), Some("Example Gazette"));
|
||||||
// The batched lookup agrees with the single one and skips unknown ids.
|
// The batched lookup agrees with the single one and skips unknown ids.
|
||||||
let batch = db.get_articles(&[id, 9_999]).await.unwrap();
|
let batch = db.get_articles(&[id, 9_999]).await.unwrap();
|
||||||
assert_eq!(batch.len(), 1);
|
assert_eq!(batch.len(), 1);
|
||||||
@@ -1261,6 +1268,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-09-06T00:00:00Z"),
|
first_seen: ts("2026-09-06T00:00:00Z"),
|
||||||
url: "https://example.com/imported".into(),
|
url: "https://example.com/imported".into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 0,
|
feed_id: 0,
|
||||||
feed_title: "Imported".into(),
|
feed_title: "Imported".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -1479,6 +1487,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||||
url: "https://example.com/1".into(),
|
url: "https://example.com/1".into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Hacker News".into(),
|
feed_title: "Hacker News".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -1596,6 +1605,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||||
url: "https://example.com/1".into(),
|
url: "https://example.com/1".into(),
|
||||||
author: Some("Page Writer".into()),
|
author: Some("Page Writer".into()),
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Hacker News".into(),
|
feed_title: "Hacker News".into(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -409,6 +409,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
|
|||||||
first_seen,
|
first_seen,
|
||||||
url: best_entry.url.clone(),
|
url: best_entry.url.clone(),
|
||||||
author,
|
author,
|
||||||
|
publication: None,
|
||||||
feed_id: best_entry.feed_id,
|
feed_id: best_entry.feed_id,
|
||||||
feed_title: best_entry
|
feed_title: best_entry
|
||||||
.feed_title
|
.feed_title
|
||||||
|
|||||||
@@ -1049,6 +1049,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-09-07T00:00:00Z"),
|
first_seen: ts("2026-09-07T00:00:00Z"),
|
||||||
url: url.into(),
|
url: url.into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 1,
|
feed_id: 1,
|
||||||
feed_title: "Feed".into(),
|
feed_title: "Feed".into(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article {
|
|||||||
first_seen: timestamp(),
|
first_seen: timestamp(),
|
||||||
url: format!("https://example.com/{entry_id}"),
|
url: format!("https://example.com/{entry_id}"),
|
||||||
author: Some("A. Writer".into()),
|
author: Some("A. Writer".into()),
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Example Feed".into(),
|
feed_title: "Example Feed".into(),
|
||||||
category: Some("Tech".into()),
|
category: Some("Tech".into()),
|
||||||
|
|||||||
+44
-22
@@ -198,6 +198,7 @@ impl Extractor {
|
|||||||
Extracted {
|
Extracted {
|
||||||
content_html,
|
content_html,
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
word_count: words,
|
word_count: words,
|
||||||
excerpt_only,
|
excerpt_only,
|
||||||
image_urls,
|
image_urls,
|
||||||
@@ -280,11 +281,12 @@ impl Extractor {
|
|||||||
body.extend_from_slice(&chunk);
|
body.extend_from_slice(&chunk);
|
||||||
}
|
}
|
||||||
let html = String::from_utf8_lossy(&body).into_owned();
|
let html = String::from_utf8_lossy(&body).into_owned();
|
||||||
let (title, html, author) = readable_page(&html, &final_url)?;
|
let (title, html, author, site_name) = readable_page(&html, &final_url)?;
|
||||||
Ok(Page {
|
Ok(Page {
|
||||||
title,
|
title,
|
||||||
html,
|
html,
|
||||||
author,
|
author,
|
||||||
|
site_name,
|
||||||
final_url,
|
final_url,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -297,6 +299,7 @@ impl Extractor {
|
|||||||
Extracted {
|
Extracted {
|
||||||
content_html: clean,
|
content_html: clean,
|
||||||
author: page.author.clone(),
|
author: page.author.clone(),
|
||||||
|
publication: page.site_name.clone(),
|
||||||
word_count: words,
|
word_count: words,
|
||||||
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
|
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
|
||||||
image_urls,
|
image_urls,
|
||||||
@@ -314,6 +317,8 @@ pub struct Page {
|
|||||||
pub html: String,
|
pub html: String,
|
||||||
/// Readability's normalized byline for the page.
|
/// Readability's normalized byline for the page.
|
||||||
pub author: Option<String>,
|
pub author: Option<String>,
|
||||||
|
/// Readability's normalized site name for the page.
|
||||||
|
pub site_name: Option<String>,
|
||||||
/// Where the fetch ended up, after any redirects — the base for relative URLs.
|
/// Where the fetch ended up, after any redirects — the base for relative URLs.
|
||||||
pub final_url: String,
|
pub final_url: String,
|
||||||
}
|
}
|
||||||
@@ -326,10 +331,13 @@ pub struct Page {
|
|||||||
/// their image with them) and its lazy-image heuristic overwrites a perfectly
|
/// their image with them) and its lazy-image heuristic overwrites a perfectly
|
||||||
/// good `src` with whatever other attribute happens to contain `.jpg`.
|
/// good `src` with whatever other attribute happens to contain `.jpg`.
|
||||||
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
|
||||||
readable_page(html, url).map(|(_, content, _)| content)
|
readable_page(html, url).map(|(_, content, _, _)| content)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String>), ExtractError> {
|
fn readable_page(
|
||||||
|
html: &str,
|
||||||
|
url: &str,
|
||||||
|
) -> Result<(String, String, Option<String>, Option<String>), ExtractError> {
|
||||||
let html = prepare_for_readability(html);
|
let html = prepare_for_readability(html);
|
||||||
let config = dom_smoothie::Config {
|
let config = dom_smoothie::Config {
|
||||||
max_elements_to_parse: 60_000,
|
max_elements_to_parse: 60_000,
|
||||||
@@ -345,17 +353,18 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String
|
|||||||
Ok((
|
Ok((
|
||||||
parsed.title.trim().to_string(),
|
parsed.title.trim().to_string(),
|
||||||
content,
|
content,
|
||||||
normalize_author(parsed.byline),
|
normalize_meta_text(parsed.byline),
|
||||||
|
normalize_meta_text(parsed.site_name),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_author(author: Option<String>) -> Option<String> {
|
fn normalize_meta_text(text: Option<String>) -> Option<String> {
|
||||||
let author = author?;
|
let text = text?;
|
||||||
if author.chars().any(|c| matches!(c, '\n' | '\r')) {
|
if text.chars().any(|c| matches!(c, '\n' | '\r')) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let author = author.split_whitespace().collect::<Vec<_>>().join(" ");
|
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||||
(!author.is_empty() && author.chars().count() <= 100).then_some(author)
|
(!text.is_empty() && text.chars().count() <= 100).then_some(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Copy an [`Extracted`] onto its [`Article`].
|
/// Copy an [`Extracted`] onto its [`Article`].
|
||||||
@@ -371,6 +380,9 @@ pub fn apply(article: &mut Article, extracted: Extracted) {
|
|||||||
{
|
{
|
||||||
article.author = Some(author);
|
article.author = Some(author);
|
||||||
}
|
}
|
||||||
|
if let Some(publication) = extracted.publication {
|
||||||
|
article.publication = Some(publication);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
|
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
|
||||||
@@ -518,6 +530,7 @@ mod tests {
|
|||||||
first_seen: ts(),
|
first_seen: ts(),
|
||||||
url: url.into(),
|
url: url.into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 1,
|
feed_id: 1,
|
||||||
feed_title: "Feed".into(),
|
feed_title: "Feed".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -745,10 +758,11 @@ mod tests {
|
|||||||
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
|
||||||
<footer>© 2026</footer></body></html>"
|
<footer>© 2026</footer></body></html>"
|
||||||
);
|
);
|
||||||
let (title, content, author) =
|
let (title, content, author, site_name) =
|
||||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||||
assert_eq!(title, "A Post");
|
assert_eq!(title, "A Post");
|
||||||
assert_eq!(author, None);
|
assert_eq!(author, None);
|
||||||
|
assert_eq!(site_name, None);
|
||||||
assert!(content.contains("Readability keeps the body copy"));
|
assert!(content.contains("Readability keeps the body copy"));
|
||||||
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
let clean = sanitize_with_base(&content, "https://blog.dev/p");
|
||||||
assert!(word_count(&clean) > 200);
|
assert!(word_count(&clean) > 200);
|
||||||
@@ -756,53 +770,60 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn readable_page_plumbs_meta_author_through_extracted() {
|
fn readable_page_plumbs_meta_author_and_og_site_name_through_extracted() {
|
||||||
let html = format!(
|
let html = format!(
|
||||||
"<html><head><title>A Post</title>\
|
"<html><head><title>A Post</title>\
|
||||||
<meta name=\"author\" content=\" Jane Dev \"></head>\
|
<meta name=\"author\" content=\" Jane Dev \">\
|
||||||
|
<meta property=\"og:site_name\" content=\" Dev Journal \"></head>\
|
||||||
<body><article><h1>A Post</h1><p>{}</p></article></body></html>",
|
<body><article><h1>A Post</h1><p>{}</p></article></body></html>",
|
||||||
"Substantial body copy for readability. ".repeat(40)
|
"Substantial body copy for readability. ".repeat(40)
|
||||||
);
|
);
|
||||||
let (title, content, author) =
|
let (title, content, author, site_name) =
|
||||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||||
let page = Page {
|
let page = Page {
|
||||||
title,
|
title,
|
||||||
html: content,
|
html: content,
|
||||||
author,
|
author,
|
||||||
|
site_name,
|
||||||
final_url: "https://blog.dev/p".into(),
|
final_url: "https://blog.dev/p".into(),
|
||||||
};
|
};
|
||||||
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
||||||
assert_eq!(extracted.author.as_deref(), Some("Jane Dev"));
|
assert_eq!(extracted.author.as_deref(), Some("Jane Dev"));
|
||||||
|
assert_eq!(extracted.publication.as_deref(), Some("Dev Journal"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn readable_page_plumbs_json_ld_author_through_extracted() {
|
fn readable_page_plumbs_json_ld_author_and_publisher_through_extracted() {
|
||||||
let html = format!(
|
let html = format!(
|
||||||
r#"<html><head><title>A Post</title>
|
r#"<html><head><title>A Post</title>
|
||||||
<script type="application/ld+json">{{
|
<script type="application/ld+json">{{
|
||||||
"@context":"https://schema.org", "@type":"Article",
|
"@context":"https://schema.org", "@type":"Article",
|
||||||
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}}
|
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}},
|
||||||
|
"publisher":{{"@type":"Organization","name":"Example Gazette"}}
|
||||||
}}</script></head>
|
}}</script></head>
|
||||||
<body><article><h1>A Post</h1><p>{}</p></article></body></html>"#,
|
<body><article><h1>A Post</h1><p>{}</p></article></body></html>"#,
|
||||||
"Substantial body copy for readability. ".repeat(40)
|
"Substantial body copy for readability. ".repeat(40)
|
||||||
);
|
);
|
||||||
let (title, content, author) =
|
let (title, content, author, site_name) =
|
||||||
readable_page(&html, "https://blog.dev/p").expect("main content");
|
readable_page(&html, "https://blog.dev/p").expect("main content");
|
||||||
let page = Page {
|
let page = Page {
|
||||||
title,
|
title,
|
||||||
html: content,
|
html: content,
|
||||||
author,
|
author,
|
||||||
|
site_name,
|
||||||
final_url: "https://blog.dev/p".into(),
|
final_url: "https://blog.dev/p".into(),
|
||||||
};
|
};
|
||||||
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
|
||||||
assert_eq!(extracted.author.as_deref(), Some("Alex Writer"));
|
assert_eq!(extracted.author.as_deref(), Some("Alex Writer"));
|
||||||
|
assert_eq!(extracted.publication.as_deref(), Some("Example Gazette"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn apply_uses_page_author_with_feed_precedence() {
|
fn apply_uses_page_metadata() {
|
||||||
let extracted = |author: &str| Extracted {
|
let extracted = |author: &str| Extracted {
|
||||||
content_html: "<p>body</p>".into(),
|
content_html: "<p>body</p>".into(),
|
||||||
author: Some(author.into()),
|
author: Some(author.into()),
|
||||||
|
publication: Some("Example Gazette".into()),
|
||||||
word_count: 1,
|
word_count: 1,
|
||||||
excerpt_only: false,
|
excerpt_only: false,
|
||||||
image_urls: vec![],
|
image_urls: vec![],
|
||||||
@@ -814,6 +835,7 @@ mod tests {
|
|||||||
aggregator.author = Some("Submitter".into());
|
aggregator.author = Some("Submitter".into());
|
||||||
apply(&mut aggregator, extracted("Page Writer"));
|
apply(&mut aggregator, extracted("Page Writer"));
|
||||||
assert_eq!(aggregator.author.as_deref(), Some("Page Writer"));
|
assert_eq!(aggregator.author.as_deref(), Some("Page Writer"));
|
||||||
|
assert_eq!(aggregator.publication.as_deref(), Some("Example Gazette"));
|
||||||
|
|
||||||
let mut direct = article("https://blog.dev/direct", "");
|
let mut direct = article("https://blog.dev/direct", "");
|
||||||
direct.author = Some("Feed Writer".into());
|
direct.author = Some("Feed Writer".into());
|
||||||
@@ -826,13 +848,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn implausible_page_authors_are_dropped() {
|
fn implausible_page_metadata_is_dropped() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
normalize_author(Some(" Jane Dev ".into())).as_deref(),
|
normalize_meta_text(Some(" Jane Dev ".into())).as_deref(),
|
||||||
Some("Jane Dev")
|
Some("Jane Dev")
|
||||||
);
|
);
|
||||||
assert_eq!(normalize_author(Some("Jane\nDev".into())), None);
|
assert_eq!(normalize_meta_text(Some("Jane\nDev".into())), None);
|
||||||
assert_eq!(normalize_author(Some("x".repeat(101))), None);
|
assert_eq!(normalize_meta_text(Some("x".repeat(101))), None);
|
||||||
assert_eq!(normalize_author(Some(" ".into())), None);
|
assert_eq!(normalize_meta_text(Some(" ".into())), None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ fn imported_article(
|
|||||||
first_seen,
|
first_seen,
|
||||||
url: page.final_url,
|
url: page.final_url,
|
||||||
author: extracted.author,
|
author: extracted.author,
|
||||||
|
publication: extracted.publication,
|
||||||
feed_id: 0,
|
feed_id: 0,
|
||||||
feed_title: "Imported".into(),
|
feed_title: "Imported".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -379,6 +380,7 @@ mod tests {
|
|||||||
first_seen: now,
|
first_seen: now,
|
||||||
url: url.into(),
|
url: url.into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Feed".into(),
|
feed_title: "Feed".into(),
|
||||||
category: None,
|
category: None,
|
||||||
@@ -484,6 +486,7 @@ mod tests {
|
|||||||
title: "A historical essay".into(),
|
title: "A historical essay".into(),
|
||||||
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
|
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
|
||||||
author: Some("Essay Writer".into()),
|
author: Some("Essay Writer".into()),
|
||||||
|
site_name: Some("Example Review".into()),
|
||||||
final_url: "https://example.com/essays/old".into(),
|
final_url: "https://example.com/essays/old".into(),
|
||||||
};
|
};
|
||||||
let extracted = extractor.finish_readable("https://example.com/old", &page);
|
let extracted = extractor.finish_readable("https://example.com/old", &page);
|
||||||
@@ -497,6 +500,7 @@ mod tests {
|
|||||||
assert_eq!(article.best_entry_id, 0);
|
assert_eq!(article.best_entry_id, 0);
|
||||||
assert_eq!(article.feed_title, "Imported");
|
assert_eq!(article.feed_title, "Imported");
|
||||||
assert_eq!(article.author.as_deref(), Some("Essay Writer"));
|
assert_eq!(article.author.as_deref(), Some("Essay Writer"));
|
||||||
|
assert_eq!(article.publication.as_deref(), Some("Example Review"));
|
||||||
assert!(article.sources.is_empty());
|
assert!(article.sources.is_empty());
|
||||||
assert!(!article.content_html.contains("script"));
|
assert!(!article.content_html.contains("script"));
|
||||||
assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
|
assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
|
||||||
@@ -506,6 +510,7 @@ mod tests {
|
|||||||
assert_eq!(loaded.best_entry_id, 0);
|
assert_eq!(loaded.best_entry_id, 0);
|
||||||
assert_eq!(loaded.feed_title, "Imported");
|
assert_eq!(loaded.feed_title, "Imported");
|
||||||
assert_eq!(loaded.author.as_deref(), Some("Essay Writer"));
|
assert_eq!(loaded.author.as_deref(), Some("Essay Writer"));
|
||||||
|
assert_eq!(loaded.publication.as_deref(), Some("Example Review"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -1530,6 +1530,7 @@ mod tests {
|
|||||||
first_seen: now(),
|
first_seen: now(),
|
||||||
url,
|
url,
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 100 + entry_id,
|
feed_id: 100 + entry_id,
|
||||||
feed_title: format!("Feed {entry_id}"),
|
feed_title: format!("Feed {entry_id}"),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -972,6 +972,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-08-15T05:30:00Z"),
|
first_seen: ts("2026-08-15T05:30:00Z"),
|
||||||
url: "https://example.com/1".into(),
|
url: "https://example.com/1".into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 7,
|
feed_id: 7,
|
||||||
feed_title: "Hacker News".into(),
|
feed_title: "Hacker News".into(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -328,6 +328,7 @@ mod tests {
|
|||||||
first_seen: ts("2026-08-15T05:00:00Z"),
|
first_seen: ts("2026-08-15T05:00:00Z"),
|
||||||
url: url.into(),
|
url: url.into(),
|
||||||
author: None,
|
author: None,
|
||||||
|
publication: None,
|
||||||
feed_id: 1,
|
feed_id: 1,
|
||||||
feed_title: "Feed".into(),
|
feed_title: "Feed".into(),
|
||||||
category: None,
|
category: None,
|
||||||
|
|||||||
@@ -92,6 +92,8 @@ pub struct Extracted {
|
|||||||
pub content_html: String,
|
pub content_html: String,
|
||||||
/// Author discovered in the fetched page, if any.
|
/// Author discovered in the fetched page, if any.
|
||||||
pub author: Option<String>,
|
pub author: Option<String>,
|
||||||
|
/// Site name discovered in the fetched page, if any.
|
||||||
|
pub publication: Option<String>,
|
||||||
pub word_count: i64,
|
pub word_count: i64,
|
||||||
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
|
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
|
||||||
pub excerpt_only: bool,
|
pub excerpt_only: bool,
|
||||||
@@ -124,6 +126,8 @@ pub struct Article {
|
|||||||
// --- joined / derived fields ---
|
// --- joined / derived fields ---
|
||||||
pub url: String,
|
pub url: String,
|
||||||
pub author: Option<String>,
|
pub author: Option<String>,
|
||||||
|
/// Page-extracted site name persisted on `articles`.
|
||||||
|
pub publication: Option<String>,
|
||||||
pub feed_id: FeedId,
|
pub feed_id: FeedId,
|
||||||
pub feed_title: String,
|
pub feed_title: String,
|
||||||
pub category: Option<String>,
|
pub category: Option<String>,
|
||||||
@@ -150,6 +154,19 @@ impl Article {
|
|||||||
self.sources.iter().any(|s| s.kind == kind)
|
self.sources.iter().any(|s| s.kind == kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What to show after the feed name: the page's site name, else the domain;
|
||||||
|
/// nothing when it would just repeat the feed name.
|
||||||
|
pub fn publication_label(&self) -> Option<String> {
|
||||||
|
let publication = self
|
||||||
|
.publication
|
||||||
|
.clone()
|
||||||
|
.or_else(|| domain(&self.canonical_url))?;
|
||||||
|
(!publication
|
||||||
|
.trim()
|
||||||
|
.eq_ignore_ascii_case(self.feed_title.trim()))
|
||||||
|
.then_some(publication)
|
||||||
|
}
|
||||||
|
|
||||||
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
|
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
|
||||||
pub fn chapter_id(&self) -> String {
|
pub fn chapter_id(&self) -> String {
|
||||||
format!("art-{}", self.best_entry_id)
|
format!("art-{}", self.best_entry_id)
|
||||||
@@ -161,6 +178,14 @@ pub fn reading_minutes(word_count: i64) -> i64 {
|
|||||||
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
|
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// URL host without a leading `www.`.
|
||||||
|
pub fn domain(url: &str) -> Option<String> {
|
||||||
|
url::Url::parse(url)
|
||||||
|
.ok()?
|
||||||
|
.host_str()
|
||||||
|
.map(|host| host.strip_prefix("www.").unwrap_or(host).to_string())
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Social proof (§3.4)
|
// Social proof (§3.4)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -847,6 +872,63 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn publication_article() -> Article {
|
||||||
|
Article {
|
||||||
|
id: 1,
|
||||||
|
canonical_url: "https://www.example.com/post".into(),
|
||||||
|
title: "A post".into(),
|
||||||
|
best_entry_id: 1,
|
||||||
|
content_html: String::new(),
|
||||||
|
word_count: 1,
|
||||||
|
excerpt_only: false,
|
||||||
|
image_count: 0,
|
||||||
|
sources: Vec::new(),
|
||||||
|
first_seen: ts(),
|
||||||
|
url: "https://www.example.com/post".into(),
|
||||||
|
author: None,
|
||||||
|
publication: None,
|
||||||
|
feed_id: 1,
|
||||||
|
feed_title: "A Feed".into(),
|
||||||
|
category: None,
|
||||||
|
published_at: None,
|
||||||
|
comments_url: None,
|
||||||
|
image_urls: Vec::new(),
|
||||||
|
social: Vec::new(),
|
||||||
|
extract_method: ExtractMethod::Readability,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_label_prefers_the_stored_publication() {
|
||||||
|
let mut article = publication_article();
|
||||||
|
article.publication = Some("Example Journal".into());
|
||||||
|
assert_eq!(
|
||||||
|
article.publication_label().as_deref(),
|
||||||
|
Some("Example Journal")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_label_falls_back_to_the_domain_without_www() {
|
||||||
|
let article = publication_article();
|
||||||
|
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_label_omits_a_publication_matching_the_feed() {
|
||||||
|
let mut article = publication_article();
|
||||||
|
article.feed_title = " example JOURNAL ".into();
|
||||||
|
article.publication = Some(" Example Journal ".into());
|
||||||
|
assert_eq!(article.publication_label(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn publication_label_omits_a_domain_matching_the_feed() {
|
||||||
|
let mut article = publication_article();
|
||||||
|
article.feed_title = " EXAMPLE.com ".into();
|
||||||
|
assert_eq!(article.publication_label(), None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn composite_social_score_matches_spec_formula() {
|
fn composite_social_score_matches_spec_formula() {
|
||||||
let refs = vec![
|
let refs = vec![
|
||||||
|
|||||||
+2
-10
@@ -6,7 +6,7 @@ use axum_login::tower_sessions::Session;
|
|||||||
use jiff::civil::Date;
|
use jiff::civil::Date;
|
||||||
|
|
||||||
use crate::server::AppState;
|
use crate::server::AppState;
|
||||||
use crate::types::{Issue, SocialSource};
|
use crate::types::{Issue, SocialSource, domain};
|
||||||
use crate::web::issue::{self, Download};
|
use crate::web::issue::{self, Download};
|
||||||
use crate::web::session::{AuthSession, Viewer};
|
use crate::web::session::{AuthSession, Viewer};
|
||||||
use crate::web::{Html, Page, WebError};
|
use crate::web::{Html, Page, WebError};
|
||||||
@@ -116,7 +116,7 @@ impl From<&Issue> for PublicIssue {
|
|||||||
url: article.canonical_url.clone(),
|
url: article.canonical_url.clone(),
|
||||||
author: article.author.clone(),
|
author: article.author.clone(),
|
||||||
source: article.feed_title.clone(),
|
source: article.feed_title.clone(),
|
||||||
domain: domain(&article.canonical_url),
|
domain: domain(&article.canonical_url).unwrap_or_default(),
|
||||||
reading_minutes: article.reading_minutes(),
|
reading_minutes: article.reading_minutes(),
|
||||||
word_count: article.word_count,
|
word_count: article.word_count,
|
||||||
summary: pick
|
summary: pick
|
||||||
@@ -169,14 +169,6 @@ impl PublicIssue {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn domain(raw: &str) -> String {
|
|
||||||
url::Url::parse(raw)
|
|
||||||
.ok()
|
|
||||||
.and_then(|url| url.host_str().map(str::to_string))
|
|
||||||
.map(|host| host.strip_prefix("www.").unwrap_or(&host).to_string())
|
|
||||||
.unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Template)]
|
#[derive(Template)]
|
||||||
#[template(path = "issue_public.html")]
|
#[template(path = "issue_public.html")]
|
||||||
struct IssuePublicTemplate {
|
struct IssuePublicTemplate {
|
||||||
|
|||||||
Reference in New Issue
Block a user