Drop re-published copies of one story from related lists

The ingest dedupe only merges within one run, so a story an outlet
re-publishes on another day under a new URL and a reworded headline lands
in the archive twice and both copies showed up as related links. Links now
also skip an article from the same site as the pick or a closer link when
60% of their title words match, besides exact title matches anywhere.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AeJmv3WED4w8S3GV7mMjS7

src/related.rs

@@ -24,9 +24,13 @@ const PER_ARTICLE: usize = 5;
 const MIN_COSINE: f64 = 0.55;
 /// Embedding rows decoded per statement, so memory stays flat as the archive grows.
 const SCAN_PAGE: i64 = 1000;
-/// Candidates kept per pick between pages: enough headroom for the title
+/// Candidates kept per pick between pages: enough headroom for the
-/// de-duplication at the end to still fill [`PER_ARTICLE`].
+/// same-story filter at the end to still fill [`PER_ARTICLE`].
 const KEEP: usize = PER_ARTICLE * 4;
+/// Share of title words two articles from one site must have in common to be
+/// one story re-published under a new URL ("…after AI accusations" and
+/// "…after AI allegations").
+const SAME_SITE_TITLE_OVERLAP: f64 = 0.6;
 
 /// What [`refresh`] wrote.
 #[derive(Debug, Default, PartialEq)]
@@ -39,8 +43,8 @@ pub struct Refreshed {
 /// empty) in one pass over the embedding cache.
 ///
 /// Excluded: the pick itself, the rest of its issue, articles that only have a
-/// feed excerpt (their page would be near-empty), and articles whose title
+/// feed excerpt (their page would be near-empty), and articles that are the
-/// matches the pick's or a closer link's (syndicated copies of one story).
+/// pick's or a closer link's story again (see [`Story::same_as`]).
 pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<Refreshed> {
     let dates = if dates.is_empty() {
         sqlx::query_scalar::<_, String>("SELECT date FROM issues ORDER BY date")
@@ -53,11 +57,11 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
         dates.to_vec()
     };
 
-    // Every pick of every requested issue, with its title for de-duplication.
+    // Every pick of every requested issue, with its story key for de-duplication.
     let mut groups: Vec<Group> = Vec::with_capacity(dates.len());
     for date in &dates {
         let rows = sqlx::query(
-            "SELECT ia.article_id, a.title FROM issue_articles ia
+            "SELECT ia.article_id, a.title, a.canonical_url FROM issue_articles ia
              JOIN articles a ON a.id = ia.article_id
              WHERE ia.issue_date = ? ORDER BY ia.section, ia.position",
         )
@@ -71,7 +75,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
                 .iter()
                 .map(|row| Target {
                     article_id: row.get("article_id"),
-                    title_key: title_key(row.get::<Option<String>, _>("title").as_deref()),
+                    story: Story::from_row(row),
                     vector: Vec::new(),
                     best: Vec::new(),
                 })
@@ -99,7 +103,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
     let mut after = i64::MIN;
     loop {
         let rows = sqlx::query(
-            "SELECT ae.article_id, ae.embedding, a.title
+            "SELECT ae.article_id, ae.embedding, a.title, a.canonical_url
              FROM article_embeddings ae JOIN articles a ON a.id = ae.article_id
              WHERE ae.model = ? AND ae.dimension = ? AND a.excerpt_only = 0
                AND ae.article_id > ?
@@ -125,7 +129,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
                     continue;
                 }
             };
-            let title = title_key(row.get::<Option<String>, _>("title").as_deref());
+            let story = Story::from_row(row);
             for group in &mut groups {
                 if group.members.contains(&candidate) {
                     continue;
@@ -133,7 +137,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
                 for pick in &mut group.picks {
                     let cosine = dot(&pick.vector, &vector)?;
                     if cosine.is_finite() && cosine >= MIN_COSINE {
-                        pick.best.push((candidate, cosine, title.clone()));
+                        pick.best.push((candidate, cosine, story.clone()));
                     }
                 }
             }
@@ -161,16 +165,18 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
             .execute(&mut *tx)
             .await?;
         for pick in &group.picks {
-            let mut seen: HashSet<&str> = HashSet::new();
+            let mut seen = vec![&pick.story];
-            if !pick.title_key.is_empty() {
+            let mut chosen = Vec::new();
-                seen.insert(pick.title_key.as_str());
+            for (id, cosine, story) in &pick.best {
+                if chosen.len() == PER_ARTICLE {
+                    break;
                 }
-            let chosen = pick
+                if !seen.iter().any(|earlier| earlier.same_as(story)) {
-                .best
+                    seen.push(story);
-                .iter()
+                    chosen.push((id, cosine));
-                .filter(|(_, _, title)| title.is_empty() || seen.insert(title))
+                }
-                .take(PER_ARTICLE);
+            }
-            for (position, (related_id, cosine, _)) in chosen.enumerate() {
+            for (position, (related_id, cosine)) in chosen.into_iter().enumerate() {
                 sqlx::query(
                     "INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
                      VALUES (?, ?, ?, ?, ?)",
@@ -229,20 +235,60 @@ struct Group {
 
 struct Target {
     article_id: ArticleId,
-    title_key: String,
+    story: Story,
     vector: Vec<f32>,
-    /// (candidate, cosine, title key), closest first after each scan page.
+    /// (candidate, cosine, story), closest first after each scan page.
-    best: Vec<(ArticleId, f64, String)>,
+    best: Vec<(ArticleId, f64, Story)>,
 }
 
-/// Lowercased letters and digits only, so "Foo: Bar" and "foo — bar" collide.
+/// What decides whether two articles are one story told twice. The ingest
-fn title_key(title: Option<&str>) -> String {
+/// dedupe only merges within one run, so a story re-published on another day
-    title
+/// under a new URL reaches the archive as a second article.
+#[derive(Clone, Debug)]
+struct Story {
+    /// Canonical URL host without `www.`.
+    site: String,
+    /// Lowercased title words.
+    words: HashSet<String>,
+}
+
+impl Story {
+    fn from_row(row: &sqlx::sqlite::SqliteRow) -> Self {
+        // Re-canonicalized so legacy rows still wrapped in a Scour redirect
+        // count as the outlet's site, not Scour's.
+        let raw: String = row.get("canonical_url");
+        let site = crate::dedupe::canonical_url(&raw)
+            .and_then(|canonical| url::Url::parse(&canonical).ok())
+            .and_then(|url| {
+                url.host_str()
+                    .map(|host| host.trim_start_matches("www.").to_string())
+            })
+            .unwrap_or_default();
+        let words = row
+            .get::<Option<String>, _>("title")
             .unwrap_or_default()
-        .chars()
+            .split(|c: char| !c.is_alphanumeric())
-        .filter(|c| c.is_alphanumeric())
+            .filter(|word| !word.is_empty())
-        .flat_map(char::to_lowercase)
+            .map(str::to_lowercase)
-        .collect()
+            .collect();
+        Self { site, words }
+    }
+
+    /// The same title anywhere (syndication), or mostly the same title on the
+    /// same site (a re-slugged republish).
+    fn same_as(&self, other: &Story) -> bool {
+        if self.words.is_empty() || other.words.is_empty() {
+            return false;
+        }
+        if self.words == other.words {
+            return true;
+        }
+        let shared = self.words.intersection(&other.words).count() as f64;
+        let union = self.words.union(&other.words).count() as f64;
+        !self.site.is_empty()
+            && self.site == other.site
+            && shared / union >= SAME_SITE_TITLE_OVERLAP
+    }
 }
 
 #[cfg(test)]
@@ -365,8 +411,36 @@ mod tests {
     }
 
     #[test]
-    fn title_keys_ignore_case_and_punctuation() {
+    fn stories_match_on_identical_titles_or_close_titles_from_one_site() {
-        assert_eq!(title_key(Some("Foo: Bar — Baz!")), "foobarbaz");
+        let story = |site: &str, title: &str| Story {
-        assert_eq!(title_key(None), "");
+            site: site.into(),
+            words: title
+                .split(|c: char| !c.is_alphanumeric())
+                .filter(|word| !word.is_empty())
+                .map(str::to_lowercase)
+                .collect(),
+        };
+        let accusations = story(
+            "the-independent.com",
+            "The bestselling novel pulled from prestigious literary prize after AI accusations",
+        );
+        let allegations = story(
+            "the-independent.com",
+            "The bestselling novel pulled from prestigious literary prize after AI allegations",
+        );
+        let conversation = story(
+            "theconversation.com",
+            "A bestselling novel was pulled from a French literary prize after AI accusations – what next?",
+        );
+        assert!(accusations.same_as(&allegations), "a re-slugged republish");
+        assert!(
+            !accusations.same_as(&conversation),
+            "other outlets' coverage stays"
+        );
+        assert!(
+            story("a.com", "Foo: Bar").same_as(&story("b.com", "foo — bar")),
+            "syndicated copies match across sites"
+        );
+        assert!(!story("a.com", "").same_as(&story("a.com", "")));
     }
 }