diff --git a/src/related.rs b/src/related.rs index 7e1b931..75da6fc 100644 --- a/src/related.rs +++ b/src/related.rs @@ -24,9 +24,13 @@ const PER_ARTICLE: usize = 5; const MIN_COSINE: f64 = 0.55; /// Embedding rows decoded per statement, so memory stays flat as the archive grows. const SCAN_PAGE: i64 = 1000; -/// Candidates kept per pick between pages: enough headroom for the title -/// de-duplication at the end to still fill [`PER_ARTICLE`]. +/// Candidates kept per pick between pages: enough headroom for the +/// same-story filter at the end to still fill [`PER_ARTICLE`]. const KEEP: usize = PER_ARTICLE * 4; +/// Share of title words two articles from one site must have in common to be +/// one story re-published under a new URL ("…after AI accusations" and +/// "…after AI allegations"). +const SAME_SITE_TITLE_OVERLAP: f64 = 0.6; /// What [`refresh`] wrote. #[derive(Debug, Default, PartialEq)] @@ -39,8 +43,8 @@ pub struct Refreshed { /// empty) in one pass over the embedding cache. /// /// Excluded: the pick itself, the rest of its issue, articles that only have a -/// feed excerpt (their page would be near-empty), and articles whose title -/// matches the pick's or a closer link's (syndicated copies of one story). +/// feed excerpt (their page would be near-empty), and articles that are the +/// pick's or a closer link's story again (see [`Story::same_as`]). pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result { let dates = if dates.is_empty() { sqlx::query_scalar::<_, String>("SELECT date FROM issues ORDER BY date") @@ -53,11 +57,11 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result = Vec::with_capacity(dates.len()); for date in &dates { let rows = sqlx::query( - "SELECT ia.article_id, a.title FROM issue_articles ia + "SELECT ia.article_id, a.title, a.canonical_url FROM issue_articles ia JOIN articles a ON a.id = ia.article_id WHERE ia.issue_date = ? ORDER BY ia.section, ia.position", ) @@ -71,7 +75,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result, _>("title").as_deref()), + story: Story::from_row(row), vector: Vec::new(), best: Vec::new(), }) @@ -99,7 +103,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result ? @@ -125,7 +129,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result, _>("title").as_deref()); + let story = Story::from_row(row); for group in &mut groups { if group.members.contains(&candidate) { continue; @@ -133,7 +137,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result= MIN_COSINE { - pick.best.push((candidate, cosine, title.clone())); + pick.best.push((candidate, cosine, story.clone())); } } } @@ -161,16 +165,18 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result = HashSet::new(); - if !pick.title_key.is_empty() { - seen.insert(pick.title_key.as_str()); + let mut seen = vec![&pick.story]; + let mut chosen = Vec::new(); + for (id, cosine, story) in &pick.best { + if chosen.len() == PER_ARTICLE { + break; + } + if !seen.iter().any(|earlier| earlier.same_as(story)) { + seen.push(story); + chosen.push((id, cosine)); + } } - let chosen = pick - .best - .iter() - .filter(|(_, _, title)| title.is_empty() || seen.insert(title)) - .take(PER_ARTICLE); - for (position, (related_id, cosine, _)) in chosen.enumerate() { + for (position, (related_id, cosine)) in chosen.into_iter().enumerate() { sqlx::query( "INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine) VALUES (?, ?, ?, ?, ?)", @@ -229,20 +235,60 @@ struct Group { struct Target { article_id: ArticleId, - title_key: String, + story: Story, vector: Vec, - /// (candidate, cosine, title key), closest first after each scan page. - best: Vec<(ArticleId, f64, String)>, + /// (candidate, cosine, story), closest first after each scan page. + best: Vec<(ArticleId, f64, Story)>, } -/// Lowercased letters and digits only, so "Foo: Bar" and "foo — bar" collide. -fn title_key(title: Option<&str>) -> String { - title - .unwrap_or_default() - .chars() - .filter(|c| c.is_alphanumeric()) - .flat_map(char::to_lowercase) - .collect() +/// What decides whether two articles are one story told twice. The ingest +/// dedupe only merges within one run, so a story re-published on another day +/// under a new URL reaches the archive as a second article. +#[derive(Clone, Debug)] +struct Story { + /// Canonical URL host without `www.`. + site: String, + /// Lowercased title words. + words: HashSet, +} + +impl Story { + fn from_row(row: &sqlx::sqlite::SqliteRow) -> Self { + // Re-canonicalized so legacy rows still wrapped in a Scour redirect + // count as the outlet's site, not Scour's. + let raw: String = row.get("canonical_url"); + let site = crate::dedupe::canonical_url(&raw) + .and_then(|canonical| url::Url::parse(&canonical).ok()) + .and_then(|url| { + url.host_str() + .map(|host| host.trim_start_matches("www.").to_string()) + }) + .unwrap_or_default(); + let words = row + .get::, _>("title") + .unwrap_or_default() + .split(|c: char| !c.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect(); + Self { site, words } + } + + /// The same title anywhere (syndication), or mostly the same title on the + /// same site (a re-slugged republish). + fn same_as(&self, other: &Story) -> bool { + if self.words.is_empty() || other.words.is_empty() { + return false; + } + if self.words == other.words { + return true; + } + let shared = self.words.intersection(&other.words).count() as f64; + let union = self.words.union(&other.words).count() as f64; + !self.site.is_empty() + && self.site == other.site + && shared / union >= SAME_SITE_TITLE_OVERLAP + } } #[cfg(test)] @@ -365,8 +411,36 @@ mod tests { } #[test] - fn title_keys_ignore_case_and_punctuation() { - assert_eq!(title_key(Some("Foo: Bar — Baz!")), "foobarbaz"); - assert_eq!(title_key(None), ""); + fn stories_match_on_identical_titles_or_close_titles_from_one_site() { + let story = |site: &str, title: &str| Story { + site: site.into(), + words: title + .split(|c: char| !c.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect(), + }; + let accusations = story( + "the-independent.com", + "The bestselling novel pulled from prestigious literary prize after AI accusations", + ); + let allegations = story( + "the-independent.com", + "The bestselling novel pulled from prestigious literary prize after AI allegations", + ); + let conversation = story( + "theconversation.com", + "A bestselling novel was pulled from a French literary prize after AI accusations – what next?", + ); + assert!(accusations.same_as(&allegations), "a re-slugged republish"); + assert!( + !accusations.same_as(&conversation), + "other outlets' coverage stays" + ); + assert!( + story("a.com", "Foo: Bar").same_as(&story("b.com", "foo — bar")), + "syndicated copies match across sites" + ); + assert!(!story("a.com", "").same_as(&story("a.com", ""))); } }