thallada/the-daily-epub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep the publication an article's page declares

Browse Source 
Readability already reads og:site_name and JSON-LD publisher.name; carry
that through Page and Extracted and store it in articles.publication, the
same way the page byline is kept. `publication_label` is what the readers
will see after the feed name: the site name, else the domain, and nothing
when it would only repeat the feed's own title.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YWmCpUojfHXhSZ2129Z7Nv
This commit is contained in:
thallada
2026-09-09 05:48:46 +00:00
co-authored by Claude Fable 5.1
parent c8df55e91a
commit f71a4c2799
16 changed files with 161 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/image_audit.rs
+1
View File
@@ -358,6 +358,7 @@ fn pick_for(target: &Target, content_html: String) -> Pick {
first_seen: "2026-08-15T05:30:00Z".parse().unwrap(),
url: target.url.clone(),
author: None,
publication: None,
feed_id: 1,
feed_title: host(&target.url),
category: None,
migrations/0012_article_publication.sql
+3
View File
@@ -0,0 +1,3 @@
-- Persist page-extracted publication names independently of the best Miniflux entry.
ALTER TABLE articles ADD COLUMN publication TEXT;
src/curate/embedding.rs
+1
View File
@@ -1089,6 +1089,7 @@ mod tests {
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
url: format!("https://example.com/{id}"),
author: Some("Secret Author".into()),
publication: None,
feed_id: 9,
feed_title: "Secret Feed".into(),
category: None,
src/curate/prefilter.rs
+1
View File
@@ -154,6 +154,7 @@ pub(crate) mod tests {
first_seen: ts(),
url: format!("https://example.com/{id}"),
author: Some("A. Writer".into()),
publication: None,
feed_id: 7,
feed_title: "Some Blog".into(),
category: Some("Tech".into()),
src/curate/signals.rs
+1
View File
@@ -763,6 +763,7 @@ mod tests {
first_seen: "2026-08-15T00:00:00Z".parse().unwrap(),
url: format!("https://example.com/{id}"),
author: None,
publication: None,
feed_id: feeds.first().copied().unwrap_or(0),
feed_title: String::new(),
category: None,
src/db.rs
+15 -5
View File
@@ -292,16 +292,18 @@ impl Db {
/// Upsert a deduped cluster by canonical URL; returns its `articles.id`.
///
/// Most denormalized fields on [`Article`] come from the joined `entries`
/// row when loading; the extracted author is stored on `articles`.
/// row when loading; extracted page metadata is stored on `articles`.
pub async fn upsert_article(&self, article: &Article) -> Result<ArticleId> {
let sources = serde_json::to_string(&article.sources).unwrap_or_else(|_| "[]".into());
let row = sqlx::query(
"INSERT INTO articles (canonical_url, title, author, best_entry_id, content_html,
word_count, excerpt_only, image_count, sources_json, first_seen)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"INSERT INTO articles (canonical_url, title, author, publication, best_entry_id,
content_html, word_count, excerpt_only, image_count,
sources_json, first_seen)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(canonical_url) DO UPDATE SET
title = excluded.title,
author = excluded.author,
publication = excluded.publication,
best_entry_id = excluded.best_entry_id,
content_html = excluded.content_html,
word_count = excluded.word_count,
@@ -313,6 +315,7 @@ impl Db {
.bind(&article.canonical_url)
.bind(&article.title)
.bind(&article.author)
.bind(&article.publication)
.bind((article.best_entry_id != 0).then_some(article.best_entry_id))
.bind(&article.content_html)
.bind(article.word_count)
@@ -889,7 +892,8 @@ macro_rules! article_select {
a.word_count AS word_count, a.excerpt_only AS excerpt_only,
a.image_count AS image_count, a.sources_json AS sources_json,
a.first_seen AS first_seen,
e.url AS entry_url, COALESCE(a.author, e.author) AS author, e.feed_id AS feed_id,
e.url AS entry_url, COALESCE(a.author, e.author) AS author,
a.publication AS publication, e.feed_id AS feed_id,
e.feed_title AS feed_title, e.category AS category,
e.published_at AS published_at, e.comments_url AS comments_url
FROM articles a LEFT JOIN entries e ON e.id = a.best_entry_id
@@ -967,6 +971,7 @@ fn article_from_row(row: &sqlx::sqlite::SqliteRow) -> Result<Article> {
.get::<Option<String>, _>("entry_url")
.unwrap_or_else(|| row.get("canonical_url")),
author: row.get("author"),
publication: row.get("publication"),
feed_id: row.get::<Option<i64>, _>("feed_id").unwrap_or(0),
feed_title: row
.get::<Option<String>, _>("feed_title")
@@ -1193,6 +1198,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(),
author: Some("Page Writer".into()),
publication: Some("Example Gazette".into()),
feed_id: 7,
feed_title: "Hacker News".into(),
category: None,
@@ -1224,6 +1230,7 @@ mod tests {
assert_eq!(loaded.social[0].score, 342);
assert_eq!(loaded.feed_title, "Hacker News");
assert_eq!(loaded.author.as_deref(), Some("Page Writer"));
assert_eq!(loaded.publication.as_deref(), Some("Example Gazette"));
// The batched lookup agrees with the single one and skips unknown ids.
let batch = db.get_articles(&[id, 9_999]).await.unwrap();
assert_eq!(batch.len(), 1);
@@ -1261,6 +1268,7 @@ mod tests {
first_seen: ts("2026-09-06T00:00:00Z"),
url: "https://example.com/imported".into(),
author: None,
publication: None,
feed_id: 0,
feed_title: "Imported".into(),
category: None,
@@ -1479,6 +1487,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(),
author: None,
publication: None,
feed_id: 7,
feed_title: "Hacker News".into(),
category: None,
@@ -1596,6 +1605,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(),
author: Some("Page Writer".into()),
publication: None,
feed_id: 7,
feed_title: "Hacker News".into(),
category: None,
src/dedupe.rs
+1
View File
@@ -409,6 +409,7 @@ fn build_article(members: Vec<(Entry, String)>, feed_urls: &FeedUrls) -> Article
first_seen,
url: best_entry.url.clone(),
author,
publication: None,
feed_id: best_entry.feed_id,
feed_title: best_entry
.feed_title
src/discovery.rs
+1
View File
@@ -1049,6 +1049,7 @@ mod tests {
first_seen: ts("2026-09-07T00:00:00Z"),
url: url.into(),
author: None,
publication: None,
feed_id: 1,
feed_title: "Feed".into(),
category: None,
src/epub/fixtures.rs
+1
View File
@@ -35,6 +35,7 @@ pub fn article(id: ArticleId, entry_id: EntryId, title: &str) -> Article {
first_seen: timestamp(),
url: format!("https://example.com/{entry_id}"),
author: Some("A. Writer".into()),
publication: None,
feed_id: 7,
feed_title: "Example Feed".into(),
category: Some("Tech".into()),
src/extract.rs
+44 -22
View File
@@ -198,6 +198,7 @@ impl Extractor {
Extracted {
content_html,
author: None,
publication: None,
word_count: words,
excerpt_only,
image_urls,
@@ -280,11 +281,12 @@ impl Extractor {
body.extend_from_slice(&chunk);
}
let html = String::from_utf8_lossy(&body).into_owned();
let (title, html, author) = readable_page(&html, &final_url)?;
let (title, html, author, site_name) = readable_page(&html, &final_url)?;
Ok(Page {
title,
html,
author,
site_name,
final_url,
})
}
@@ -297,6 +299,7 @@ impl Extractor {
Extracted {
content_html: clean,
author: page.author.clone(),
publication: page.site_name.clone(),
word_count: words,
excerpt_only: looks_paywalled(requested_url, words, &self.paywall_domains),
image_urls,
@@ -314,6 +317,8 @@ pub struct Page {
pub html: String,
/// Readability's normalized byline for the page.
pub author: Option<String>,
/// Readability's normalized site name for the page.
pub site_name: Option<String>,
/// Where the fetch ended up, after any redirects — the base for relative URLs.
pub final_url: String,
}
@@ -326,10 +331,13 @@ pub struct Page {
/// their image with them) and its lazy-image heuristic overwrites a perfectly
/// good `src` with whatever other attribute happens to contain `.jpg`.
pub fn readability(html: &str, url: &str) -> Result<String, ExtractError> {
readable_page(html, url).map(|(_, content, _)| content)
readable_page(html, url).map(|(_, content, _, _)| content)
}
fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String>), ExtractError> {
fn readable_page(
html: &str,
url: &str,
) -> Result<(String, String, Option<String>, Option<String>), ExtractError> {
let html = prepare_for_readability(html);
let config = dom_smoothie::Config {
max_elements_to_parse: 60_000,
@@ -345,17 +353,18 @@ fn readable_page(html: &str, url: &str) -> Result<(String, String, Option<String
Ok((
parsed.title.trim().to_string(),
content,
normalize_author(parsed.byline),
normalize_meta_text(parsed.byline),
normalize_meta_text(parsed.site_name),
))
}
fn normalize_author(author: Option<String>) -> Option<String> {
let author = author?;
if author.chars().any(|c| matches!(c, '\n' | '\r')) {
fn normalize_meta_text(text: Option<String>) -> Option<String> {
let text = text?;
if text.chars().any(|c| matches!(c, '\n' | '\r')) {
return None;
}
let author = author.split_whitespace().collect::<Vec<_>>().join(" ");
(!author.is_empty() && author.chars().count() <= 100).then_some(author)
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
(!text.is_empty() && text.chars().count() <= 100).then_some(text)
}
/// Copy an [`Extracted`] onto its [`Article`].
@@ -371,6 +380,9 @@ pub fn apply(article: &mut Article, extracted: Extracted) {
{
article.author = Some(author);
}
if let Some(publication) = extracted.publication {
article.publication = Some(publication);
}
}
fn merge_paywall_domains(configured: Vec<String>) -> Vec<String> {
@@ -518,6 +530,7 @@ mod tests {
first_seen: ts(),
url: url.into(),
author: None,
publication: None,
feed_id: 1,
feed_title: "Feed".into(),
category: None,
@@ -745,10 +758,11 @@ mod tests {
<article><h1>A Post</h1><p>{paragraph}</p><p>{paragraph}</p></article>\
<footer>© 2026</footer></body></html>"
);
let (title, content, author) =
let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content");
assert_eq!(title, "A Post");
assert_eq!(author, None);
assert_eq!(site_name, None);
assert!(content.contains("Readability keeps the body copy"));
let clean = sanitize_with_base(&content, "https://blog.dev/p");
assert!(word_count(&clean) > 200);
@@ -756,53 +770,60 @@ mod tests {
}
#[test]
fn readable_page_plumbs_meta_author_through_extracted() {
fn readable_page_plumbs_meta_author_and_og_site_name_through_extracted() {
let html = format!(
"<html><head><title>A Post</title>\
<meta name=\"author\" content=\" Jane Dev \"></head>\
<meta name=\"author\" content=\" Jane Dev \">\
<meta property=\"og:site_name\" content=\" Dev Journal \"></head>\
<body><article><h1>A Post</h1><p>{}</p></article></body></html>",
"Substantial body copy for readability. ".repeat(40)
);
let (title, content, author) =
let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content");
let page = Page {
title,
html: content,
author,
site_name,
final_url: "https://blog.dev/p".into(),
};
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
assert_eq!(extracted.author.as_deref(), Some("Jane Dev"));
assert_eq!(extracted.publication.as_deref(), Some("Dev Journal"));
}
#[test]
fn readable_page_plumbs_json_ld_author_through_extracted() {
fn readable_page_plumbs_json_ld_author_and_publisher_through_extracted() {
let html = format!(
r#"<html><head><title>A Post</title>
<script type="application/ld+json">{{
"@context":"https://schema.org", "@type":"Article",
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}}
"headline":"A Post", "author":{{"@type":"Person","name":"Alex Writer"}},
"publisher":{{"@type":"Organization","name":"Example Gazette"}}
}}</script></head>
<body><article><h1>A Post</h1><p>{}</p></article></body></html>"#,
"Substantial body copy for readability. ".repeat(40)
);
let (title, content, author) =
let (title, content, author, site_name) =
readable_page(&html, "https://blog.dev/p").expect("main content");
let page = Page {
title,
html: content,
author,
site_name,
final_url: "https://blog.dev/p".into(),
};
let extracted = Extractor::offline(vec![]).finish_readable(&page.final_url, &page);
assert_eq!(extracted.author.as_deref(), Some("Alex Writer"));
assert_eq!(extracted.publication.as_deref(), Some("Example Gazette"));
}
#[test]
fn apply_uses_page_author_with_feed_precedence() {
fn apply_uses_page_metadata() {
let extracted = |author: &str| Extracted {
content_html: "<p>body</p>".into(),
author: Some(author.into()),
publication: Some("Example Gazette".into()),
word_count: 1,
excerpt_only: false,
image_urls: vec![],
@@ -814,6 +835,7 @@ mod tests {
aggregator.author = Some("Submitter".into());
apply(&mut aggregator, extracted("Page Writer"));
assert_eq!(aggregator.author.as_deref(), Some("Page Writer"));
assert_eq!(aggregator.publication.as_deref(), Some("Example Gazette"));
let mut direct = article("https://blog.dev/direct", "");
direct.author = Some("Feed Writer".into());
@@ -826,13 +848,13 @@ mod tests {
}
#[test]
fn implausible_page_authors_are_dropped() {
fn implausible_page_metadata_is_dropped() {
assert_eq!(
normalize_author(Some(" Jane Dev ".into())).as_deref(),
normalize_meta_text(Some(" Jane Dev ".into())).as_deref(),
Some("Jane Dev")
);
assert_eq!(normalize_author(Some("Jane\nDev".into())), None);
assert_eq!(normalize_author(Some("x".repeat(101))), None);
assert_eq!(normalize_author(Some(" ".into())), None);
assert_eq!(normalize_meta_text(Some("Jane\nDev".into())), None);
assert_eq!(normalize_meta_text(Some("x".repeat(101))), None);
assert_eq!(normalize_meta_text(Some(" ".into())), None);
}
}
src/imports.rs
+5
View File
@@ -167,6 +167,7 @@ fn imported_article(
first_seen,
url: page.final_url,
author: extracted.author,
publication: extracted.publication,
feed_id: 0,
feed_title: "Imported".into(),
category: None,
@@ -379,6 +380,7 @@ mod tests {
first_seen: now,
url: url.into(),
author: None,
publication: None,
feed_id: 7,
feed_title: "Feed".into(),
category: None,
@@ -484,6 +486,7 @@ mod tests {
title: "A historical essay".into(),
html: "<article><p>Useful old writing.</p><script>bad()</script><img src=\"/chart.png\"></article>".into(),
author: Some("Essay Writer".into()),
site_name: Some("Example Review".into()),
final_url: "https://example.com/essays/old".into(),
};
let extracted = extractor.finish_readable("https://example.com/old", &page);
@@ -497,6 +500,7 @@ mod tests {
assert_eq!(article.best_entry_id, 0);
assert_eq!(article.feed_title, "Imported");
assert_eq!(article.author.as_deref(), Some("Essay Writer"));
assert_eq!(article.publication.as_deref(), Some("Example Review"));
assert!(article.sources.is_empty());
assert!(!article.content_html.contains("script"));
assert_eq!(article.image_urls, ["https://example.com/chart.png"]);
@@ -506,6 +510,7 @@ mod tests {
assert_eq!(loaded.best_entry_id, 0);
assert_eq!(loaded.feed_title, "Imported");
assert_eq!(loaded.author.as_deref(), Some("Essay Writer"));
assert_eq!(loaded.publication.as_deref(), Some("Example Review"));
}
#[tokio::test]
src/pipeline.rs
+1
View File
@@ -1530,6 +1530,7 @@ mod tests {
first_seen: now(),
url,
author: None,
publication: None,
feed_id: 100 + entry_id,
feed_title: format!("Feed {entry_id}"),
category: None,
src/server.rs
+1
View File
@@ -972,6 +972,7 @@ mod tests {
first_seen: ts("2026-08-15T05:30:00Z"),
url: "https://example.com/1".into(),
author: None,
publication: None,
feed_id: 7,
feed_title: "Hacker News".into(),
category: None,
src/social/mod.rs
+1
View File
@@ -328,6 +328,7 @@ mod tests {
first_seen: ts("2026-08-15T05:00:00Z"),
url: url.into(),
author: None,
publication: None,
feed_id: 1,
feed_title: "Feed".into(),
category: None,
src/types.rs
+82
View File
@@ -92,6 +92,8 @@ pub struct Extracted {
pub content_html: String,
/// Author discovered in the fetched page, if any.
pub author: Option<String>,
/// Site name discovered in the fetched page, if any.
pub publication: Option<String>,
pub word_count: i64,
/// True when we only have an excerpt/paywall stub — penalized in pre-filter.
pub excerpt_only: bool,
@@ -124,6 +126,8 @@ pub struct Article {
// --- joined / derived fields ---
pub url: String,
pub author: Option<String>,
/// Page-extracted site name persisted on `articles`.
pub publication: Option<String>,
pub feed_id: FeedId,
pub feed_title: String,
pub category: Option<String>,
@@ -150,6 +154,19 @@ impl Article {
self.sources.iter().any(|s| s.kind == kind)
}
/// What to show after the feed name: the page's site name, else the domain;
/// nothing when it would just repeat the feed name.
pub fn publication_label(&self) -> Option<String> {
let publication = self
.publication
.clone()
.or_else(|| domain(&self.canonical_url))?;
(!publication
.trim()
.eq_ignore_ascii_case(self.feed_title.trim()))
.then_some(publication)
}
/// Stable EPUB chapter id used by TOC and rating links (implementation notes §12).
pub fn chapter_id(&self) -> String {
format!("art-{}", self.best_entry_id)
@@ -161,6 +178,14 @@ pub fn reading_minutes(word_count: i64) -> i64 {
(word_count.max(0) as f64 / 220.0).ceil().max(1.0) as i64
}
/// URL host without a leading `www.`.
pub fn domain(url: &str) -> Option<String> {
url::Url::parse(url)
.ok()?
.host_str()
.map(|host| host.strip_prefix("www.").unwrap_or(host).to_string())
}
// ---------------------------------------------------------------------------
// Social proof (§3.4)
// ---------------------------------------------------------------------------
@@ -847,6 +872,63 @@ mod tests {
}
}
fn publication_article() -> Article {
Article {
id: 1,
canonical_url: "https://www.example.com/post".into(),
title: "A post".into(),
best_entry_id: 1,
content_html: String::new(),
word_count: 1,
excerpt_only: false,
image_count: 0,
sources: Vec::new(),
first_seen: ts(),
url: "https://www.example.com/post".into(),
author: None,
publication: None,
feed_id: 1,
feed_title: "A Feed".into(),
category: None,
published_at: None,
comments_url: None,
image_urls: Vec::new(),
social: Vec::new(),
extract_method: ExtractMethod::Readability,
}
}
#[test]
fn publication_label_prefers_the_stored_publication() {
let mut article = publication_article();
article.publication = Some("Example Journal".into());
assert_eq!(
article.publication_label().as_deref(),
Some("Example Journal")
);
}
#[test]
fn publication_label_falls_back_to_the_domain_without_www() {
let article = publication_article();
assert_eq!(article.publication_label().as_deref(), Some("example.com"));
}
#[test]
fn publication_label_omits_a_publication_matching_the_feed() {
let mut article = publication_article();
article.feed_title = " example JOURNAL ".into();
article.publication = Some(" Example Journal ".into());
assert_eq!(article.publication_label(), None);
}
#[test]
fn publication_label_omits_a_domain_matching_the_feed() {
let mut article = publication_article();
article.feed_title = " EXAMPLE.com ".into();
assert_eq!(article.publication_label(), None);
}
#[test]
fn composite_social_score_matches_spec_formula() {
let refs = vec![
src/web/public.rs
+2 -10
View File
@@ -6,7 +6,7 @@ use axum_login::tower_sessions::Session;
use jiff::civil::Date;
use crate::server::AppState;
use crate::types::{Issue, SocialSource};
use crate::types::{Issue, SocialSource, domain};
use crate::web::issue::{self, Download};
use crate::web::session::{AuthSession, Viewer};
use crate::web::{Html, Page, WebError};
@@ -116,7 +116,7 @@ impl From<&Issue> for PublicIssue {
url: article.canonical_url.clone(),
author: article.author.clone(),
source: article.feed_title.clone(),
domain: domain(&article.canonical_url),
domain: domain(&article.canonical_url).unwrap_or_default(),
reading_minutes: article.reading_minutes(),
word_count: article.word_count,
summary: pick
@@ -169,14 +169,6 @@ impl PublicIssue {
}
}
fn domain(raw: &str) -> String {
url::Url::parse(raw)
.ok()
.and_then(|url| url.host_str().map(str::to_string))
.map(|host| host.strip_prefix("www.").unwrap_or(&host).to_string())
.unwrap_or_default()
}
#[derive(Template)]
#[template(path = "issue_public.html")]
struct IssuePublicTemplate {