thallada/the-daily-epub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify related articles: shared article header, ids only, no job

Browse Source 
- Both article pages render one `_article_head.html` partial from an
  `ArticleHead` built by a single constructor instead of two copies.
- `related::for_article` returns ids; the cosine stays stored for tuning.
- Title keys travel with each candidate, dropping the side map.
- Drop the `related-refresh` dashboard job: the backfill is a one-off and
  `features related` covers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AeJmv3WED4w8S3GV7mMjS7
This commit is contained in:
thallada
2026-10-12 16:11:43 +00:00
co-authored by Claude Opus 5.5
parent 03f0a454be
commit 08b15c42b4
7 changed files with 79 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/related.rs
+24 -64
View File
@@ -18,23 +18,16 @@ use crate::db::Db;
use crate::types::ArticleId;
/// Links shown under each article.
pub const PER_ARTICLE: usize = 5;
const PER_ARTICLE: usize = 5;
/// Below this cosine a neighbour is noise, not a related article (the same
/// cut the curation prompts use for "closest rated").
pub const MIN_COSINE: f64 = 0.55;
const MIN_COSINE: f64 = 0.55;
/// Embedding rows decoded per statement, so memory stays flat as the archive grows.
const SCAN_PAGE: i64 = 1000;
/// Candidates kept per pick between pages: enough headroom for the title
/// de-duplication at the end to still fill [`PER_ARTICLE`].
const KEEP: usize = PER_ARTICLE * 4;
/// One stored related link.
#[derive(Debug, Clone, PartialEq)]
pub struct Related {
pub article_id: ArticleId,
pub cosine: f64,
}
/// What [`refresh`] wrote.
#[derive(Debug, Default, PartialEq)]
pub struct Refreshed {
@@ -103,7 +96,6 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
});
}
let mut titles: HashMap<ArticleId, String> = HashMap::new();
let mut after = i64::MIN;
loop {
let rows = sqlx::query(
@@ -133,7 +125,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
continue;
}
};
let mut kept = false;
let title = title_key(row.get::<Option<String>, _>("title").as_deref());
for group in &mut groups {
if group.members.contains(&candidate) {
continue;
@@ -141,21 +133,20 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
for pick in &mut group.picks {
let cosine = dot(&pick.vector, &vector)?;
if cosine.is_finite() && cosine >= MIN_COSINE {
pick.best.push((candidate, cosine));
kept = true;
pick.best.push((candidate, cosine, title.clone()));
}
}
}
if kept {
titles.insert(
candidate,
title_key(row.get::<Option<String>, _>("title").as_deref()),
);
}
}
for group in &mut groups {
for pick in &mut group.picks {
rank(&mut pick.best);
// Highest cosine first; ties by id so reruns write the same rows.
pick.best.sort_by(|left, right| {
right
.1
.total_cmp(&left.1)
.then_with(|| left.0.cmp(&right.0))
});
pick.best.truncate(KEEP);
}
}
@@ -177,12 +168,9 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
let chosen = pick
.best
.iter()
.filter(|(id, _)| {
let key = titles.get(id).map_or("", String::as_str);
key.is_empty() || seen.insert(key)
})
.filter(|(_, _, title)| title.is_empty() || seen.insert(title))
.take(PER_ARTICLE);
for (position, (related_id, cosine)) in chosen.enumerate() {
for (position, (related_id, cosine, _)) in chosen.enumerate() {
sqlx::query(
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
VALUES (?, ?, ?, ?, ?)",
@@ -204,22 +192,15 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
}
/// The stored related links for one pick, closest first.
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<Related>> {
let rows = sqlx::query(
"SELECT related_id, cosine FROM issue_related
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<ArticleId>> {
Ok(sqlx::query_scalar(
"SELECT related_id FROM issue_related
WHERE issue_date = ? AND article_id = ? ORDER BY position",
)
.bind(date.to_string())
.bind(article_id)
.fetch_all(db.pool())
.await?;
Ok(rows
.iter()
.map(|row| Related {
article_id: row.get("related_id"),
cosine: row.get("cosine"),
})
.collect())
.await?)
}
/// The most recent issue each of `ids` was published in, for linking a related
@@ -250,17 +231,8 @@ struct Target {
article_id: ArticleId,
title_key: String,
vector: Vec<f32>,
best: Vec<(ArticleId, f64)>,
}
/// Highest cosine first; ties by id so reruns write the same rows.
fn rank(best: &mut [(ArticleId, f64)]) {
best.sort_by(|left, right| {
right
.1
.total_cmp(&left.1)
.then_with(|| left.0.cmp(&right.0))
});
/// (candidate, cosine, title key), closest first after each scan page.
best: Vec<(ArticleId, f64, String)>,
}
/// Lowercased letters and digits only, so "Foo: Bar" and "foo — bar" collide.
@@ -359,17 +331,10 @@ mod tests {
issue(&db, "2026-09-01", &[1, 2]).await;
let refreshed = refresh(&db, &voyage(), &[day("2026-09-01")]).await.unwrap();
let links = for_article(&db, day("2026-09-01"), 1).await.unwrap();
let ids: Vec<ArticleId> = links.iter().map(|link| link.article_id).collect();
assert_eq!(ids, vec![3, 6], "closest first, one copy of each story");
assert!(links[0].cosine > links[1].cosine);
// Article 2's neighbours: 3, 6 (5 shares 1's title but not 2's).
let two: Vec<ArticleId> = for_article(&db, day("2026-09-01"), 2)
.await
.unwrap()
.iter()
.map(|link| link.article_id)
.collect();
let one = for_article(&db, day("2026-09-01"), 1).await.unwrap();
assert_eq!(one, vec![3, 6], "closest first, one copy of each story");
// 5 only repeats article 1's title, so it is fair game for article 2.
let two = for_article(&db, day("2026-09-01"), 2).await.unwrap();
assert_eq!(two, vec![5, 3, 6]);
assert_eq!(
refreshed,
@@ -392,12 +357,7 @@ mod tests {
assert_eq!(refresh(&db, &voyage(), &[]).await.unwrap().issues, 2);
// A rerun rewrites rather than duplicates.
refresh(&db, &voyage(), &[]).await.unwrap();
let ids: Vec<ArticleId> = for_article(&db, day("2026-09-02"), 2)
.await
.unwrap()
.iter()
.map(|link| link.article_id)
.collect();
let ids = for_article(&db, day("2026-09-02"), 2).await.unwrap();
assert_eq!(ids, vec![1, 3]);
let issues = latest_issues(&db, &[1, 3]).await.unwrap();
assert_eq!(issues.get(&1), Some(&day("2026-09-01")));