diff --git a/examples/image_audit.rs b/examples/image_audit.rs index 8aa11cf..088d1a8 100644 --- a/examples/image_audit.rs +++ b/examples/image_audit.rs @@ -358,6 +358,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick { first_seen: "2026-08-15T05:30:00Z".parse().unwrap(), url: target.url.clone(), author: None, + publication: None, feed_id: 1, feed_title: host(&target.url), category: None, diff --git a/migrations/0012_article_publication.sql b/migrations/0012_article_publication.sql new file mode 100644 index 0000000..251e11d --- /dev/null +++ b/migrations/0012_article_publication.sql @@ -0,0 +1,3 @@ +-- Persist page-extracted publication names independently of the best Miniflux entry. + +ALTER TABLE articles ADD COLUMN publication TEXT; diff --git a/src/curate/embedding.rs b/src/curate/embedding.rs index 5de1be7..405a85f 100644 --- a/src/curate/embedding.rs +++ b/src/curate/embedding.rs @@ -1089,6 +1089,7 @@ mod tests { first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), url: format!("https://example.com/{id}"), author: Some("Secret Author".into()), + publication: None, feed_id: 9, feed_title: "Secret Feed".into(), category: None, diff --git a/src/curate/prefilter.rs b/src/curate/prefilter.rs index c967e43..a848e74 100644 --- a/src/curate/prefilter.rs +++ b/src/curate/prefilter.rs @@ -154,6 +154,7 @@ pub(crate) mod tests { first_seen: ts(), url: format!("https://example.com/{id}"), author: Some("A. Writer".into()), + publication: None, feed_id: 7, feed_title: "Some Blog".into(), category: Some("Tech".into()), diff --git a/src/curate/signals.rs b/src/curate/signals.rs index c17b22d..aefd9cf 100644 --- a/src/curate/signals.rs +++ b/src/curate/signals.rs @@ -763,6 +763,7 @@ mod tests { first_seen: "2026-08-15T00:00:00Z".parse().unwrap(), url: format!("https://example.com/{id}"), author: None, + publication: None, feed_id: feeds.first().copied().unwrap_or(0), feed_title: String::new(), category: None, diff --git a/src/db.rs b/src/db.rs index 39408a2..5dae60f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -292,16 +292,18 @@ impl Db { /// Upsert a deduped cluster by canonical URL; returns its `articles.id`. /// /// Most denormalized fields on [`Article`] come from the joined `entries` - /// row when loading; the extracted author is stored on `articles`. + /// row when loading; extracted page metadata is stored on `articles`. pub async fn upsert_article(&self, article: &Article) -> Result { let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into()); let row = sqlx::query( - "INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html, - word_count, excerpt_only, image_count, sources_json, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "INSERT INTO articles (canonical_url, title, author, publication, best_entry_id, + content_html, word_count, excerpt_only, image_count, + sources_json, first_seen) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(canonical_url) DO UPDATE SET title = excluded.title, author = excluded.author, + publication = excluded.publication, best_entry_id = excluded.best_entry_id, content_html = excluded.content_html, word_count = excluded.word_count, @@ -313,6 +315,7 @@ impl Db { .bind(&article.canonical_url) .bind(&article.title) .bind(&article.author) + .bind(&article.publication) .bind((article.best_entry_id != 0).then_some(article.best_entry_id)) .bind(&article.content_html) .bind(article.word_count) @@ -889,7 +892,8 @@ macro_rules! article_select { a.word_count AS word_count, a.excerpt_only AS excerpt_only, a.image_count AS image_count, a.sources_json AS sources_json, a.first_seen AS first_seen, - e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id, + e.url AS entry_url, COALESCE(a.author, e.author) AS author, + a.publication AS publication, e.feed_id AS feed_id, e.feed_title AS feed_title, e.category AS category, e.published_at AS published_at, e.comments_url AS comments_url FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id @@ -967,6 +971,7 @@ fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result
{ .get::, _>("entry_url") .unwrap_or_else(|| row.get("canonical_url")), author: row.get("author"), + publication: row.get("publication"), feed_id: row.get::, _>("feed_id").unwrap_or(0), feed_title: row .get::, _>("feed_title") @@ -1193,6 +1198,7 @@ mod tests { first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), author: Some("Page Writer".into()), + publication: Some("Example Gazette".into()), feed_id: 7, feed_title: "Hacker News".into(), category: None, @@ -1224,6 +1230,7 @@ mod tests { assert_eq!(loaded.social[0].score, 342); assert_eq!(loaded.feed_title, "Hacker News"); assert_eq!(loaded.author.as_deref(), Some("Page Writer")); + assert_eq!(loaded.publication.as_deref(), Some("Example Gazette")); // The batched lookup agrees with the single one and skips unknown ids. let batch = db.get_articles(&[id, 9_999]).await.unwrap(); assert_eq!(batch.len(), 1); @@ -1261,6 +1268,7 @@ mod tests { first_seen: ts("2026-09-06T00:00:00Z"), url: "https://example.com/imported".into(), author: None, + publication: None, feed_id: 0, feed_title: "Imported".into(), category: None, @@ -1479,6 +1487,7 @@ mod tests { first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), author: None, + publication: None, feed_id: 7, feed_title: "Hacker News".into(), category: None, @@ -1596,6 +1605,7 @@ mod tests { first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), author: Some("Page Writer".into()), + publication: None, feed_id: 7, feed_title: "Hacker News".into(), category: None, diff --git a/src/dedupe.rs b/src/dedupe.rs index c92e2b0..31c845f 100644 --- a/src/dedupe.rs +++ b/src/dedupe.rs @@ -409,6 +409,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article first_seen, url: best_entry.url.clone(), author, + publication: None, feed_id: best_entry.feed_id, feed_title: best_entry .feed_title diff --git a/src/discovery.rs b/src/discovery.rs index ca1be69..2e4dfde 100644 --- a/src/discovery.rs +++ b/src/discovery.rs @@ -1049,6 +1049,7 @@ mod tests { first_seen: ts("2026-09-07T00:00:00Z"), url: url.into(), author: None, + publication: None, feed_id: 1, feed_title: "Feed".into(), category: None, diff --git a/src/epub/fixtures.rs b/src/epub/fixtures.rs index 36f3cbc..217679a 100644 --- a/src/epub/fixtures.rs +++ b/src/epub/fixtures.rs @@ -35,6 +35,7 @@ pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article { first_seen: timestamp(), url: format!("https://example.com/{entry_id}"), author: Some("A. Writer".into()), + publication: None, feed_id: 7, feed_title: "Example Feed".into(), category: Some("Tech".into()), diff --git a/src/extract.rs b/src/extract.rs index f997e9b..30edc42 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -198,6 +198,7 @@ impl Extractor { Extracted { content_html, author: None, + publication: None, word_count: words, excerpt_only, image_urls, @@ -280,11 +281,12 @@ impl Extractor { body.extend_from_slice(&chunk); } let html = String::from_utf8_lossy(&body).into_owned(); - let (title, html, author) = readable_page(&html, &final_url)?; + let (title, html, author, site_name) = readable_page(&html, &final_url)?; Ok(Page { title, html, author, + site_name, final_url, }) } @@ -297,6 +299,7 @@ impl Extractor { Extracted { content_html: clean, author: page.author.clone(), + publication: page.site_name.clone(), word_count: words, excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains), image_urls, @@ -314,6 +317,8 @@ pub struct Page { pub html: String, /// Readability's normalized byline for the page. pub author: Option, + /// Readability's normalized site name for the page. + pub site_name: Option, /// Where the fetch ended up, after any redirects — the base for relative URLs. pub final_url: String, } @@ -326,10 +331,13 @@ pub struct Page { /// their image with them) and its lazy-image heuristic overwrites a perfectly /// good `src` with whatever other attribute happens to contain `.jpg`. pub fn readability(html: &str, url: &str) -> Result { - readable_page(html, url).map(|(_, content, _)| content) + readable_page(html, url).map(|(_, content, _, _)| content) } -fn readable_page(html: &str, url: &str) -> Result<(String, String, Option), ExtractError> { +fn readable_page( + html: &str, + url: &str, +) -> Result<(String, String, Option, Option), ExtractError> { let html = prepare_for_readability(html); let config = dom_smoothie::Config { max_elements_to_parse: 60_000, @@ -345,17 +353,18 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String, Option) -> Option { - let author = author?; - if author.chars().any(|c| matches!(c, '\n' | '\r')) { +fn normalize_meta_text(text: Option) -> Option { + let text = text?; + if text.chars().any(|c| matches!(c, '\n' | '\r')) { return None; } - let author = author.split_whitespace().collect::>().join(" "); - (!author.is_empty() && author.chars().count() <= 100).then_some(author) + let text = text.split_whitespace().collect::>().join(" "); + (!text.is_empty() && text.chars().count() <= 100).then_some(text) } /// Copy an [`Extracted`] onto its [`Article`]. @@ -371,6 +380,9 @@ pub fn apply(article: &mut Article, extracted: Extracted) { { article.author = Some(author); } + if let Some(publication) = extracted.publication { + article.publication = Some(publication); + } } fn merge_paywall_domains(configured: Vec) -> Vec { @@ -518,6 +530,7 @@ mod tests { first_seen: ts(), url: url.into(), author: None, + publication: None, feed_id: 1, feed_title: "Feed".into(), category: None, @@ -745,10 +758,11 @@ mod tests {

A Post

{paragraph}

{paragraph}

\
© 2026
" ); - let (title, content, author) = + let (title, content, author, site_name) = readable_page(&html, "https://blog.dev/p").expect("main content"); assert_eq!(title, "A Post"); assert_eq!(author, None); + assert_eq!(site_name, None); assert!(content.contains("Readability keeps the body copy")); let clean = sanitize_with_base(&content, "https://blog.dev/p"); assert!(word_count(&clean) > 200); @@ -756,53 +770,60 @@ mod tests { } #[test] - fn readable_page_plumbs_meta_author_through_extracted() { + fn readable_page_plumbs_meta_author_and_og_site_name_through_extracted() { let html = format!( "A Post\ - \ + \ + \

A Post

{}

", "Substantial body copy for readability. ".repeat(40) ); - let (title, content, author) = + let (title, content, author, site_name) = readable_page(&html, "https://blog.dev/p").expect("main content"); let page = Page { title, html: content, author, + site_name, final_url: "https://blog.dev/p".into(), }; let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); assert_eq!(extracted.author.as_deref(), Some("Jane Dev")); + assert_eq!(extracted.publication.as_deref(), Some("Dev Journal")); } #[test] - fn readable_page_plumbs_json_ld_author_through_extracted() { + fn readable_page_plumbs_json_ld_author_and_publisher_through_extracted() { let html = format!( r#"A Post

A Post

{}

"#, "Substantial body copy for readability. ".repeat(40) ); - let (title, content, author) = + let (title, content, author, site_name) = readable_page(&html, "https://blog.dev/p").expect("main content"); let page = Page { title, html: content, author, + site_name, final_url: "https://blog.dev/p".into(), }; let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page); assert_eq!(extracted.author.as_deref(), Some("Alex Writer")); + assert_eq!(extracted.publication.as_deref(), Some("Example Gazette")); } #[test] - fn apply_uses_page_author_with_feed_precedence() { + fn apply_uses_page_metadata() { let extracted = |author: &str| Extracted { content_html: "

body

".into(), author: Some(author.into()), + publication: Some("Example Gazette".into()), word_count: 1, excerpt_only: false, image_urls: vec![], @@ -814,6 +835,7 @@ mod tests { aggregator.author = Some("Submitter".into()); apply(&mut aggregator, extracted("Page Writer")); assert_eq!(aggregator.author.as_deref(), Some("Page Writer")); + assert_eq!(aggregator.publication.as_deref(), Some("Example Gazette")); let mut direct = article("https://blog.dev/direct", ""); direct.author = Some("Feed Writer".into()); @@ -826,13 +848,13 @@ mod tests { } #[test] - fn implausible_page_authors_are_dropped() { + fn implausible_page_metadata_is_dropped() { assert_eq!( - normalize_author(Some(" Jane Dev ".into())).as_deref(), + normalize_meta_text(Some(" Jane Dev ".into())).as_deref(), Some("Jane Dev") ); - assert_eq!(normalize_author(Some("Jane\nDev".into())), None); - assert_eq!(normalize_author(Some("x".repeat(101))), None); - assert_eq!(normalize_author(Some(" ".into())), None); + assert_eq!(normalize_meta_text(Some("Jane\nDev".into())), None); + assert_eq!(normalize_meta_text(Some("x".repeat(101))), None); + assert_eq!(normalize_meta_text(Some(" ".into())), None); } } diff --git a/src/imports.rs b/src/imports.rs index 2811016..b0cd0b2 100644 --- a/src/imports.rs +++ b/src/imports.rs @@ -167,6 +167,7 @@ fn imported_article( first_seen, url: page.final_url, author: extracted.author, + publication: extracted.publication, feed_id: 0, feed_title: "Imported".into(), category: None, @@ -379,6 +380,7 @@ mod tests { first_seen: now, url: url.into(), author: None, + publication: None, feed_id: 7, feed_title: "Feed".into(), category: None, @@ -484,6 +486,7 @@ mod tests { title: "A historical essay".into(), html: "

Useful old writing.

".into(), author: Some("Essay Writer".into()), + site_name: Some("Example Review".into()), final_url: "https://example.com/essays/old".into(), }; let extracted = extractor.finish_readable("https://example.com/old", &page); @@ -497,6 +500,7 @@ mod tests { assert_eq!(article.best_entry_id, 0); assert_eq!(article.feed_title, "Imported"); assert_eq!(article.author.as_deref(), Some("Essay Writer")); + assert_eq!(article.publication.as_deref(), Some("Example Review")); assert!(article.sources.is_empty()); assert!(!article.content_html.contains("script")); assert_eq!(article.image_urls, ["https://example.com/chart.png"]); @@ -506,6 +510,7 @@ mod tests { assert_eq!(loaded.best_entry_id, 0); assert_eq!(loaded.feed_title, "Imported"); assert_eq!(loaded.author.as_deref(), Some("Essay Writer")); + assert_eq!(loaded.publication.as_deref(), Some("Example Review")); } #[tokio::test] diff --git a/src/pipeline.rs b/src/pipeline.rs index cb456b5..f5b3b8e 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -1530,6 +1530,7 @@ mod tests { first_seen: now(), url, author: None, + publication: None, feed_id: 100 + entry_id, feed_title: format!("Feed {entry_id}"), category: None, diff --git a/src/server.rs b/src/server.rs index 71541f3..4d7ba05 100644 --- a/src/server.rs +++ b/src/server.rs @@ -972,6 +972,7 @@ mod tests { first_seen: ts("2026-08-15T05:30:00Z"), url: "https://example.com/1".into(), author: None, + publication: None, feed_id: 7, feed_title: "Hacker News".into(), category: None, diff --git a/src/social/mod.rs b/src/social/mod.rs index d663533..0df6be5 100644 --- a/src/social/mod.rs +++ b/src/social/mod.rs @@ -328,6 +328,7 @@ mod tests { first_seen: ts("2026-08-15T05:00:00Z"), url: url.into(), author: None, + publication: None, feed_id: 1, feed_title: "Feed".into(), category: None, diff --git a/src/types.rs b/src/types.rs index dfae363..65d88e0 100644 --- a/src/types.rs +++ b/src/types.rs @@ -92,6 +92,8 @@ pub struct Extracted { pub content_html: String, /// Author discovered in the fetched page, if any. pub author: Option, + /// Site name discovered in the fetched page, if any. + pub publication: Option, pub word_count: i64, /// True when we only have an excerpt/paywall stub — penalized in pre-filter. pub excerpt_only: bool, @@ -124,6 +126,8 @@ pub struct Article { // --- joined / derived fields --- pub url: String, pub author: Option, + /// Page-extracted site name persisted on `articles`. + pub publication: Option, pub feed_id: FeedId, pub feed_title: String, pub category: Option, @@ -150,6 +154,19 @@ impl Article { self.sources.iter().any(|s| s.kind == kind) } + /// What to show after the feed name: the page's site name, else the domain; + /// nothing when it would just repeat the feed name. + pub fn publication_label(&self) -> Option { + let publication = self + .publication + .clone() + .or_else(|| domain(&self.canonical_url))?; + (!publication + .trim() + .eq_ignore_ascii_case(self.feed_title.trim())) + .then_some(publication) + } + /// Stable EPUB chapter id used by TOC and rating links (implementation notes §12). pub fn chapter_id(&self) -> String { format!("art-{}", self.best_entry_id) @@ -161,6 +178,14 @@ pub fn reading_minutes(word_count: i64) -> i64 { (word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64 } +/// URL host without a leading `www.`. +pub fn domain(url: &str) -> Option { + url::Url::parse(url) + .ok()? + .host_str() + .map(|host| host.strip_prefix("www.").unwrap_or(host).to_string()) +} + // --------------------------------------------------------------------------- // Social proof (§3.4) // --------------------------------------------------------------------------- @@ -847,6 +872,63 @@ mod tests { } } + fn publication_article() -> Article { + Article { + id: 1, + canonical_url: "https://www.example.com/post".into(), + title: "A post".into(), + best_entry_id: 1, + content_html: String::new(), + word_count: 1, + excerpt_only: false, + image_count: 0, + sources: Vec::new(), + first_seen: ts(), + url: "https://www.example.com/post".into(), + author: None, + publication: None, + feed_id: 1, + feed_title: "A Feed".into(), + category: None, + published_at: None, + comments_url: None, + image_urls: Vec::new(), + social: Vec::new(), + extract_method: ExtractMethod::Readability, + } + } + + #[test] + fn publication_label_prefers_the_stored_publication() { + let mut article = publication_article(); + article.publication = Some("Example Journal".into()); + assert_eq!( + article.publication_label().as_deref(), + Some("Example Journal") + ); + } + + #[test] + fn publication_label_falls_back_to_the_domain_without_www() { + let article = publication_article(); + assert_eq!(article.publication_label().as_deref(), Some("example.com")); + } + + #[test] + fn publication_label_omits_a_publication_matching_the_feed() { + let mut article = publication_article(); + article.feed_title = " example JOURNAL ".into(); + article.publication = Some(" Example Journal ".into()); + assert_eq!(article.publication_label(), None); + } + + #[test] + fn publication_label_omits_a_domain_matching_the_feed() { + let mut article = publication_article(); + article.feed_title = " EXAMPLE.com ".into(); + assert_eq!(article.publication_label(), None); + } + #[test] fn composite_social_score_matches_spec_formula() { let refs = vec![ diff --git a/src/web/public.rs b/src/web/public.rs index 367c159..8233394 100644 --- a/src/web/public.rs +++ b/src/web/public.rs @@ -6,7 +6,7 @@ use axum_login::tower_sessions::Session; use jiff::civil::Date; use crate::server::AppState; -use crate::types::{Issue, SocialSource}; +use crate::types::{Issue, SocialSource, domain}; use crate::web::issue::{self, Download}; use crate::web::session::{AuthSession, Viewer}; use crate::web::{Html, Page, WebError}; @@ -116,7 +116,7 @@ impl From<&Issue> for PublicIssue { url: article.canonical_url.clone(), author: article.author.clone(), source: article.feed_title.clone(), - domain: domain(&article.canonical_url), + domain: domain(&article.canonical_url).unwrap_or_default(), reading_minutes: article.reading_minutes(), word_count: article.word_count, summary: pick @@ -169,14 +169,6 @@ impl PublicIssue { } } -fn domain(raw: &str) -> String { - url::Url::parse(raw) - .ok() - .and_then(|url| url.host_str().map(str::to_string)) - .map(|host| host.strip_prefix("www.").unwrap_or(&host).to_string()) - .unwrap_or_default() -} - #[derive(Template)] #[template(path = "issue_public.html")] struct IssuePublicTemplate {