From 08b15c42b496bf064208fc2aa417d27ede420ebf Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Mon, 12 Oct 2026 16:11:43 +0000 Subject: [PATCH] Simplify related articles: shared article header, ids only, no job - Both article pages render one `_article_head.html` partial from an `ArticleHead` built by a single constructor instead of two copies. - `related::for_article` returns ids; the cosine stays stored for tuning. - Title keys travel with each candidate, dropping the side map. - Drop the `related-refresh` dashboard job: the backfill is a one-off and `features related` covers it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AeJmv3WED4w8S3GV7mMjS7 --- src/jobs.rs | 15 +--- src/main.rs | 13 +--- src/related.rs | 88 ++++++--------------- src/web/issue.rs | 94 +++++++++++------------ src/web/templates/_article_head.html | 2 + src/web/templates/article.html | 5 +- src/web/templates/article_standalone.html | 5 +- 7 files changed, 79 insertions(+), 143 deletions(-) create mode 100644 src/web/templates/_article_head.html diff --git a/src/jobs.rs b/src/jobs.rs index 956e45d..dd8e1e1 100644 --- a/src/jobs.rs +++ b/src/jobs.rs @@ -48,13 +48,11 @@ pub enum Job { ImportRatings, /// `interests-categorize` → file uncategorized standing interests. InterestsCategorize, - /// `related-refresh` → `features related` (every issue). - RelatedRefresh, } impl Job { /// The catalogue in the order the Jobs page lists it. - pub const CATALOGUE: [Job; 9] = [ + pub const CATALOGUE: [Job; 8] = [ Job::Generate { date: None }, Job::DryRun, Job::ProfileRebuild, @@ -63,7 +61,6 @@ impl Job { Job::FeaturesPrune, Job::ImportRatings, Job::InterestsCategorize, - Job::RelatedRefresh, ]; /// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name @@ -91,7 +88,6 @@ impl Job { "features-prune" => Some(Job::FeaturesPrune), "import-ratings" => Some(Job::ImportRatings), "interests-categorize" => Some(Job::InterestsCategorize), - "related-refresh" => Some(Job::RelatedRefresh), _ => { let (rebuild, date) = match name.strip_prefix("generate-") { Some(date) => (false, date), @@ -120,7 +116,6 @@ impl Job { Job::FeaturesPrune => "features-prune".into(), Job::ImportRatings => "import-ratings".into(), Job::InterestsCategorize => "interests-categorize".into(), - Job::RelatedRefresh => "related-refresh".into(), } } @@ -155,9 +150,6 @@ impl Job { Job::InterestsCategorize => { "File uncategorized interests under categories with the bulk model, creating new ones only when needed." } - Job::RelatedRefresh => { - "Recompute the related articles listed under every issue's web articles." - } } } @@ -170,10 +162,7 @@ impl Job { Job::ProfileRebuild => Some("profile rebuild"), Job::FeaturesBackfill => Some("features backfill"), Job::BackfillSocial => Some("backfill-social"), - Job::FeaturesPrune - | Job::ImportRatings - | Job::InterestsCategorize - | Job::RelatedRefresh => None, + Job::FeaturesPrune | Job::ImportRatings | Job::InterestsCategorize => None, } } diff --git a/src/main.rs b/src/main.rs index 2216acc..6079e3a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,8 +80,8 @@ enum JobCommand { /// Run one catalogue job in-process and record it in the `jobs` table. Run { /// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`, - /// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings`, - /// `interests-categorize` or `related-refresh`. + /// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings` + /// or `interests-categorize`. name: String, }, } @@ -1248,15 +1248,6 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O )), jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)), jobs::Job::InterestsCategorize => Ok((interests::categorize(config, db).await?, None)), - jobs::Job::RelatedRefresh => Ok(( - cmd_features( - config, - db, - FeaturesCommand::Related(RelatedArgs { date: None }), - ) - .await?, - None, - )), } } diff --git a/src/related.rs b/src/related.rs index 71ae0e8..7e1b931 100644 --- a/src/related.rs +++ b/src/related.rs @@ -18,23 +18,16 @@ use crate::db::Db; use crate::types::ArticleId; /// Links shown under each article. -pub const PER_ARTICLE: usize = 5; +const PER_ARTICLE: usize = 5; /// Below this cosine a neighbour is noise, not a related article (the same /// cut the curation prompts use for "closest rated"). -pub const MIN_COSINE: f64 = 0.55; +const MIN_COSINE: f64 = 0.55; /// Embedding rows decoded per statement, so memory stays flat as the archive grows. const SCAN_PAGE: i64 = 1000; /// Candidates kept per pick between pages: enough headroom for the title /// de-duplication at the end to still fill [`PER_ARTICLE`]. const KEEP: usize = PER_ARTICLE * 4; -/// One stored related link. -#[derive(Debug, Clone, PartialEq)] -pub struct Related { - pub article_id: ArticleId, - pub cosine: f64, -} - /// What [`refresh`] wrote. #[derive(Debug, Default, PartialEq)] pub struct Refreshed { @@ -103,7 +96,6 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result = HashMap::new(); let mut after = i64::MIN; loop { let rows = sqlx::query( @@ -133,7 +125,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result, _>("title").as_deref()); for group in &mut groups { if group.members.contains(&candidate) { continue; @@ -141,21 +133,20 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result= MIN_COSINE { - pick.best.push((candidate, cosine)); - kept = true; + pick.best.push((candidate, cosine, title.clone())); } } } - if kept { - titles.insert( - candidate, - title_key(row.get::, _>("title").as_deref()), - ); - } } for group in &mut groups { for pick in &mut group.picks { - rank(&mut pick.best); + // Highest cosine first; ties by id so reruns write the same rows. + pick.best.sort_by(|left, right| { + right + .1 + .total_cmp(&left.1) + .then_with(|| left.0.cmp(&right.0)) + }); pick.best.truncate(KEEP); } } @@ -177,12 +168,9 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result Result Result> { - let rows = sqlx::query( - "SELECT related_id, cosine FROM issue_related +pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result> { + Ok(sqlx::query_scalar( + "SELECT related_id FROM issue_related WHERE issue_date = ? AND article_id = ? ORDER BY position", ) .bind(date.to_string()) .bind(article_id) .fetch_all(db.pool()) - .await?; - Ok(rows - .iter() - .map(|row| Related { - article_id: row.get("related_id"), - cosine: row.get("cosine"), - }) - .collect()) + .await?) } /// The most recent issue each of `ids` was published in, for linking a related @@ -250,17 +231,8 @@ struct Target { article_id: ArticleId, title_key: String, vector: Vec, - best: Vec<(ArticleId, f64)>, -} - -/// Highest cosine first; ties by id so reruns write the same rows. -fn rank(best: &mut [(ArticleId, f64)]) { - best.sort_by(|left, right| { - right - .1 - .total_cmp(&left.1) - .then_with(|| left.0.cmp(&right.0)) - }); + /// (candidate, cosine, title key), closest first after each scan page. + best: Vec<(ArticleId, f64, String)>, } /// Lowercased letters and digits only, so "Foo: Bar" and "foo — bar" collide. @@ -359,17 +331,10 @@ mod tests { issue(&db, "2026-09-01", &[1, 2]).await; let refreshed = refresh(&db, &voyage(), &[day("2026-09-01")]).await.unwrap(); - let links = for_article(&db, day("2026-09-01"), 1).await.unwrap(); - let ids: Vec = links.iter().map(|link| link.article_id).collect(); - assert_eq!(ids, vec![3, 6], "closest first, one copy of each story"); - assert!(links[0].cosine > links[1].cosine); - // Article 2's neighbours: 3, 6 (5 shares 1's title but not 2's). - let two: Vec = for_article(&db, day("2026-09-01"), 2) - .await - .unwrap() - .iter() - .map(|link| link.article_id) - .collect(); + let one = for_article(&db, day("2026-09-01"), 1).await.unwrap(); + assert_eq!(one, vec![3, 6], "closest first, one copy of each story"); + // 5 only repeats article 1's title, so it is fair game for article 2. + let two = for_article(&db, day("2026-09-01"), 2).await.unwrap(); assert_eq!(two, vec![5, 3, 6]); assert_eq!( refreshed, @@ -392,12 +357,7 @@ mod tests { assert_eq!(refresh(&db, &voyage(), &[]).await.unwrap().issues, 2); // A rerun rewrites rather than duplicates. refresh(&db, &voyage(), &[]).await.unwrap(); - let ids: Vec = for_article(&db, day("2026-09-02"), 2) - .await - .unwrap() - .iter() - .map(|link| link.article_id) - .collect(); + let ids = for_article(&db, day("2026-09-02"), 2).await.unwrap(); assert_eq!(ids, vec![1, 3]); let issues = latest_issues(&db, &[1, 3]).await.unwrap(); assert_eq!(issues.get(&1), Some(&day("2026-09-01"))); diff --git a/src/web/issue.rs b/src/web/issue.rs index bf30f7f..fb5d61f 100644 --- a/src/web/issue.rs +++ b/src/web/issue.rs @@ -943,19 +943,8 @@ struct ArticleLink { struct ArticleTemplate { page: Page, toc: Toc, - title: String, - source_url: String, - byline: Option, - source: Source, - meta_line: String, - social_line: Option, - understanding: chapters::Understanding, - summary: Option, - excerpt_only: bool, - body_html: String, + head: ArticleHead, discussion_html: Option, - read_online_url: String, - dashboard_href: String, rating: Option, previous: Option, next: Option, @@ -969,18 +958,50 @@ struct ArticleTemplate { #[template(path = "article_standalone.html")] struct StandaloneArticleTemplate { page: Page, + head: ArticleHead, +} + +/// What `_article_head.html` renders (header and body) plus the footer links +/// both article pages share. +struct ArticleHead { title: String, source_url: String, byline: Option, source: Source, meta_line: String, social_line: Option, + /// Issue-only: empty and `None` on the standalone page. + understanding: chapters::Understanding, + summary: Option, excerpt_only: bool, body_html: String, read_online_url: String, dashboard_href: String, } +impl ArticleHead { + fn new(article: &Article, config: &Config, is_admin: bool) -> Self { + Self { + title: article.title.clone(), + source_url: article.canonical_url.clone(), + byline: article.author.as_ref().map(|author| format!("By {author}")), + source: Source::new(article, config, is_admin), + meta_line: format!( + "{} words · ~{} min read", + thousands(article.word_count), + article.reading_minutes() + ), + social_line: chapters::social_line(&article.social), + understanding: chapters::Understanding::default(), + summary: None, + excerpt_only: article.excerpt_only, + body_html: prepare_body(&article.content_html), + read_online_url: article.url.clone(), + dashboard_href: format!("/dashboard/articles/{}", article.id), + } + } +} + /// One entry of the "Related" list under an issue article. #[derive(Debug)] struct RelatedView { @@ -1150,26 +1171,15 @@ pub async fn article( Ok(Html(ArticleTemplate { page, toc, - title: article.title.clone(), - source_url: article.canonical_url.clone(), - byline: article.author.as_ref().map(|author| format!("By {author}")), - source: Source::new(article, &config, is_admin), - meta_line: format!( - "{} words · ~{} min read", - thousands(article.word_count), - article.reading_minutes() - ), - social_line: chapters::social_line(&article.social), - understanding: chapters::understanding(pick), - summary: summary_for(&view.issue, pick).map(str::to_string), - excerpt_only: article.excerpt_only, - body_html: prepare_body(&article.content_html), + head: ArticleHead { + understanding: chapters::understanding(pick), + summary: summary_for(&view.issue, pick).map(str::to_string), + ..ArticleHead::new(article, &config, is_admin) + }, discussion_html: pick .discussion .as_ref() .map(|discussion| crate::comments::render_xhtml(discussion, &article.title)), - read_online_url: article.url.clone(), - dashboard_href: format!("/dashboard/articles/{}", article.id), rating: is_admin.then(|| { RatingWidget::for_issue( article.id, @@ -1217,20 +1227,7 @@ pub async fn standalone( page.flash = take_flash(&session).await?; Ok(Html(StandaloneArticleTemplate { page, - title: article.title.clone(), - source_url: article.canonical_url.clone(), - byline: article.author.as_ref().map(|author| format!("By {author}")), - source: Source::new(&article, &config, is_admin), - meta_line: format!( - "{} words · ~{} min read", - thousands(article.word_count), - article.reading_minutes() - ), - social_line: chapters::social_line(&article.social), - excerpt_only: article.excerpt_only, - body_html: prepare_body(&article.content_html), - read_online_url: article.url.clone(), - dashboard_href: format!("/dashboard/articles/{}", article.id), + head: ArticleHead::new(&article, &config, is_admin), }) .into_response()) } @@ -1244,18 +1241,17 @@ async fn related_views( date: Date, article_id: ArticleId, ) -> anyhow::Result> { - let links = crate::related::for_article(db, date, article_id).await?; - if links.is_empty() { + let ids = crate::related::for_article(db, date, article_id).await?; + if ids.is_empty() { return Ok(Vec::new()); } - let ids: Vec = links.iter().map(|link| link.article_id).collect(); let articles = db.get_articles(&ids).await?; let issues = crate::related::latest_issues(db, &ids).await?; - Ok(links + Ok(ids .iter() - .filter_map(|link| { - let article = articles.get(&link.article_id)?; - let href = match issues.get(&link.article_id) { + .filter_map(|id| { + let article = articles.get(id)?; + let href = match issues.get(id) { Some(date) => article_href(*date, article.id), None => standalone_href(article.id), }; diff --git a/src/web/templates/_article_head.html b/src/web/templates/_article_head.html new file mode 100644 index 0000000..c0bd103 --- /dev/null +++ b/src/web/templates/_article_head.html @@ -0,0 +1,2 @@ +

Article

{{ head.title }}

{% match head.byline %}{% when Some with (byline) %}

{{ byline }}

{% when None %}{% endmatch %}

{% let source = head.source %}{% include "_source.html" %} · {{ head.meta_line }}

{% let understanding = head.understanding %}{% include "_understanding.html" %}{% match head.summary %}{% when Some with (summary) %}

{{ summary }}

{% when None %}{% endmatch %}{% match head.social_line %}{% when Some with (social) %}

{{ social }}

{% when None %}{% endmatch %}{% if head.excerpt_only %}

Excerpt only — continue reading at the original site.

{% endif %}
+
{{ head.body_html|safe }}
diff --git a/src/web/templates/article.html b/src/web/templates/article.html index d95fcf2..2187a54 100644 --- a/src/web/templates/article.html +++ b/src/web/templates/article.html @@ -1,9 +1,8 @@ {% extends "layout.html" %}{% block ears %}{{ toc.short_date }}{% endblock %}{% block content %}
{% include "_toc.html" %}
-

Article

{{ title }}

{% match byline %}{% when Some with (byline) %}

{{ byline }}

{% when None %}{% endmatch %}

{% include "_source.html" %} · {{ meta_line }}

{% include "_understanding.html" %}{% match summary %}{% when Some with (summary) %}

{{ summary }}

{% when None %}{% endmatch %}{% match social_line %}{% when Some with (social) %}

{{ social }}

{% when None %}{% endmatch %}{% if excerpt_only %}

Excerpt only — continue reading at the original site.

{% endif %}
-
{{ body_html|safe }}
+ {% include "_article_head.html" %} {% match discussion_html %}{% when Some with (discussion) %}

Discussion

{{ discussion|safe }}
{% when None %}{% endmatch %} {% include "_related.html" %} - +
{% endblock %} diff --git a/src/web/templates/article_standalone.html b/src/web/templates/article_standalone.html index a5e2251..f35b11e 100644 --- a/src/web/templates/article_standalone.html +++ b/src/web/templates/article_standalone.html @@ -1,6 +1,5 @@ {% extends "layout.html" %}{% block ears %}Article{% endblock %}{% block content %}
-

Article

{{ title }}

{% match byline %}{% when Some with (byline) %}

{{ byline }}

{% when None %}{% endmatch %}

{% include "_source.html" %} · {{ meta_line }}

{% match social_line %}{% when Some with (social) %}

{{ social }}

{% when None %}{% endmatch %}{% if excerpt_only %}

Excerpt only — continue reading at the original site.

{% endif %}
-
{{ body_html|safe }}
- + {% include "_article_head.html" %} +
{% endblock %}