Simplify related articles: shared article header, ids only, no job

- Both article pages render one `_article_head.html` partial from an
  `ArticleHead` built by a single constructor instead of two copies.
- `related::for_article` returns ids; the cosine stays stored for tuning.
- Title keys travel with each candidate, dropping the side map.
- Drop the `related-refresh` dashboard job: the backfill is a one-off and
  `features related` covers it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AeJmv3WED4w8S3GV7mMjS7
This commit is contained in:
2026-10-12 16:11:43 +00:00
co-authored by Claude Opus 5.5
parent 03f0a454be
commit 08b15c42b4
7 changed files with 79 additions and 143 deletions
+2 -13
View File
@@ -48,13 +48,11 @@ pub enum Job {
ImportRatings,
/// `interests-categorize` → file uncategorized standing interests.
InterestsCategorize,
/// `related-refresh` → `features related` (every issue).
RelatedRefresh,
}
impl Job {
/// The catalogue in the order the Jobs page lists it.
pub const CATALOGUE: [Job; 9] = [
pub const CATALOGUE: [Job; 8] = [
Job::Generate { date: None },
Job::DryRun,
Job::ProfileRebuild,
@@ -63,7 +61,6 @@ impl Job {
Job::FeaturesPrune,
Job::ImportRatings,
Job::InterestsCategorize,
Job::RelatedRefresh,
];
/// `^[a-z0-9-]+$`: the only characters a job (and so a unit instance) name
@@ -91,7 +88,6 @@ impl Job {
"features-prune" => Some(Job::FeaturesPrune),
"import-ratings" => Some(Job::ImportRatings),
"interests-categorize" => Some(Job::InterestsCategorize),
"related-refresh" => Some(Job::RelatedRefresh),
_ => {
let (rebuild, date) = match name.strip_prefix("generate-") {
Some(date) => (false, date),
@@ -120,7 +116,6 @@ impl Job {
Job::FeaturesPrune => "features-prune".into(),
Job::ImportRatings => "import-ratings".into(),
Job::InterestsCategorize => "interests-categorize".into(),
Job::RelatedRefresh => "related-refresh".into(),
}
}
@@ -155,9 +150,6 @@ impl Job {
Job::InterestsCategorize => {
"File uncategorized interests under categories with the bulk model, creating new ones only when needed."
}
Job::RelatedRefresh => {
"Recompute the related articles listed under every issue's web articles."
}
}
}
@@ -170,10 +162,7 @@ impl Job {
Job::ProfileRebuild => Some("profile rebuild"),
Job::FeaturesBackfill => Some("features backfill"),
Job::BackfillSocial => Some("backfill-social"),
Job::FeaturesPrune
| Job::ImportRatings
| Job::InterestsCategorize
| Job::RelatedRefresh => None,
Job::FeaturesPrune | Job::ImportRatings | Job::InterestsCategorize => None,
}
}
+2 -11
View File
@@ -80,8 +80,8 @@ enum JobCommand {
/// Run one catalogue job in-process and record it in the `jobs` table.
Run {
/// `generate`, `generate-YYYY-MM-DD`, `dry-run`, `profile-rebuild`,
/// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings`,
/// `interests-categorize` or `related-refresh`.
/// `features-backfill`, `backfill-social`, `features-prune`, `import-ratings`
/// or `interests-categorize`.
name: String,
},
}
@@ -1248,15 +1248,6 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
)),
jobs::Job::ImportRatings => Ok((imports::run(config, db).await?, None)),
jobs::Job::InterestsCategorize => Ok((interests::categorize(config, db).await?, None)),
jobs::Job::RelatedRefresh => Ok((
cmd_features(
config,
db,
FeaturesCommand::Related(RelatedArgs { date: None }),
)
.await?,
None,
)),
}
}
+24 -64
View File
@@ -18,23 +18,16 @@ use crate::db::Db;
use crate::types::ArticleId;
/// Links shown under each article.
pub const PER_ARTICLE: usize = 5;
const PER_ARTICLE: usize = 5;
/// Below this cosine a neighbour is noise, not a related article (the same
/// cut the curation prompts use for "closest rated").
pub const MIN_COSINE: f64 = 0.55;
const MIN_COSINE: f64 = 0.55;
/// Embedding rows decoded per statement, so memory stays flat as the archive grows.
const SCAN_PAGE: i64 = 1000;
/// Candidates kept per pick between pages: enough headroom for the title
/// de-duplication at the end to still fill [`PER_ARTICLE`].
const KEEP: usize = PER_ARTICLE * 4;
/// One stored related link.
#[derive(Debug, Clone, PartialEq)]
pub struct Related {
pub article_id: ArticleId,
pub cosine: f64,
}
/// What [`refresh`] wrote.
#[derive(Debug, Default, PartialEq)]
pub struct Refreshed {
@@ -103,7 +96,6 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
});
}
let mut titles: HashMap<ArticleId, String> = HashMap::new();
let mut after = i64::MIN;
loop {
let rows = sqlx::query(
@@ -133,7 +125,7 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
continue;
}
};
let mut kept = false;
let title = title_key(row.get::<Option<String>, _>("title").as_deref());
for group in &mut groups {
if group.members.contains(&candidate) {
continue;
@@ -141,21 +133,20 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
for pick in &mut group.picks {
let cosine = dot(&pick.vector, &vector)?;
if cosine.is_finite() && cosine >= MIN_COSINE {
pick.best.push((candidate, cosine));
kept = true;
pick.best.push((candidate, cosine, title.clone()));
}
}
}
if kept {
titles.insert(
candidate,
title_key(row.get::<Option<String>, _>("title").as_deref()),
);
}
}
for group in &mut groups {
for pick in &mut group.picks {
rank(&mut pick.best);
// Highest cosine first; ties by id so reruns write the same rows.
pick.best.sort_by(|left, right| {
right
.1
.total_cmp(&left.1)
.then_with(|| left.0.cmp(&right.0))
});
pick.best.truncate(KEEP);
}
}
@@ -177,12 +168,9 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
let chosen = pick
.best
.iter()
.filter(|(id, _)| {
let key = titles.get(id).map_or("", String::as_str);
key.is_empty() || seen.insert(key)
})
.filter(|(_, _, title)| title.is_empty() || seen.insert(title))
.take(PER_ARTICLE);
for (position, (related_id, cosine)) in chosen.enumerate() {
for (position, (related_id, cosine, _)) in chosen.enumerate() {
sqlx::query(
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
VALUES (?, ?, ?, ?, ?)",
@@ -204,22 +192,15 @@ pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<R
}
/// The stored related links for one pick, closest first.
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<Related>> {
let rows = sqlx::query(
"SELECT related_id, cosine FROM issue_related
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<ArticleId>> {
Ok(sqlx::query_scalar(
"SELECT related_id FROM issue_related
WHERE issue_date = ? AND article_id = ? ORDER BY position",
)
.bind(date.to_string())
.bind(article_id)
.fetch_all(db.pool())
.await?;
Ok(rows
.iter()
.map(|row| Related {
article_id: row.get("related_id"),
cosine: row.get("cosine"),
})
.collect())
.await?)
}
/// The most recent issue each of `ids` was published in, for linking a related
@@ -250,17 +231,8 @@ struct Target {
article_id: ArticleId,
title_key: String,
vector: Vec<f32>,
best: Vec<(ArticleId, f64)>,
}
/// Highest cosine first; ties by id so reruns write the same rows.
fn rank(best: &mut [(ArticleId, f64)]) {
best.sort_by(|left, right| {
right
.1
.total_cmp(&left.1)
.then_with(|| left.0.cmp(&right.0))
});
/// (candidate, cosine, title key), closest first after each scan page.
best: Vec<(ArticleId, f64, String)>,
}
/// Lowercased letters and digits only, so "Foo: Bar" and "foo — bar" collide.
@@ -359,17 +331,10 @@ mod tests {
issue(&db, "2026-09-01", &[1, 2]).await;
let refreshed = refresh(&db, &voyage(), &[day("2026-09-01")]).await.unwrap();
let links = for_article(&db, day("2026-09-01"), 1).await.unwrap();
let ids: Vec<ArticleId> = links.iter().map(|link| link.article_id).collect();
assert_eq!(ids, vec![3, 6], "closest first, one copy of each story");
assert!(links[0].cosine > links[1].cosine);
// Article 2's neighbours: 3, 6 (5 shares 1's title but not 2's).
let two: Vec<ArticleId> = for_article(&db, day("2026-09-01"), 2)
.await
.unwrap()
.iter()
.map(|link| link.article_id)
.collect();
let one = for_article(&db, day("2026-09-01"), 1).await.unwrap();
assert_eq!(one, vec![3, 6], "closest first, one copy of each story");
// 5 only repeats article 1's title, so it is fair game for article 2.
let two = for_article(&db, day("2026-09-01"), 2).await.unwrap();
assert_eq!(two, vec![5, 3, 6]);
assert_eq!(
refreshed,
@@ -392,12 +357,7 @@ mod tests {
assert_eq!(refresh(&db, &voyage(), &[]).await.unwrap().issues, 2);
// A rerun rewrites rather than duplicates.
refresh(&db, &voyage(), &[]).await.unwrap();
let ids: Vec<ArticleId> = for_article(&db, day("2026-09-02"), 2)
.await
.unwrap()
.iter()
.map(|link| link.article_id)
.collect();
let ids = for_article(&db, day("2026-09-02"), 2).await.unwrap();
assert_eq!(ids, vec![1, 3]);
let issues = latest_issues(&db, &[1, 3]).await.unwrap();
assert_eq!(issues.get(&1), Some(&day("2026-09-01")));
+43 -47
View File
@@ -943,19 +943,8 @@ struct ArticleLink {
struct ArticleTemplate {
page: Page,
toc: Toc,
title: String,
source_url: String,
byline: Option<String>,
source: Source,
meta_line: String,
social_line: Option<String>,
understanding: chapters::Understanding,
summary: Option<String>,
excerpt_only: bool,
body_html: String,
head: ArticleHead,
discussion_html: Option<String>,
read_online_url: String,
dashboard_href: String,
rating: Option<RatingWidget>,
previous: Option<ArticleLink>,
next: Option<ArticleLink>,
@@ -969,18 +958,50 @@ struct ArticleTemplate {
#[template(path = "article_standalone.html")]
struct StandaloneArticleTemplate {
page: Page,
head: ArticleHead,
}
/// What `_article_head.html` renders (header and body) plus the footer links
/// both article pages share.
struct ArticleHead {
title: String,
source_url: String,
byline: Option<String>,
source: Source,
meta_line: String,
social_line: Option<String>,
/// Issue-only: empty and `None` on the standalone page.
understanding: chapters::Understanding,
summary: Option<String>,
excerpt_only: bool,
body_html: String,
read_online_url: String,
dashboard_href: String,
}
impl ArticleHead {
fn new(article: &Article, config: &Config, is_admin: bool) -> Self {
Self {
title: article.title.clone(),
source_url: article.canonical_url.clone(),
byline: article.author.as_ref().map(|author| format!("By {author}")),
source: Source::new(article, config, is_admin),
meta_line: format!(
"{} words · ~{} min read",
thousands(article.word_count),
article.reading_minutes()
),
social_line: chapters::social_line(&article.social),
understanding: chapters::Understanding::default(),
summary: None,
excerpt_only: article.excerpt_only,
body_html: prepare_body(&article.content_html),
read_online_url: article.url.clone(),
dashboard_href: format!("/dashboard/articles/{}", article.id),
}
}
}
/// One entry of the "Related" list under an issue article.
#[derive(Debug)]
struct RelatedView {
@@ -1150,26 +1171,15 @@ pub async fn article(
Ok(Html(ArticleTemplate {
page,
toc,
title: article.title.clone(),
source_url: article.canonical_url.clone(),
byline: article.author.as_ref().map(|author| format!("By {author}")),
source: Source::new(article, &config, is_admin),
meta_line: format!(
"{} words · ~{} min read",
thousands(article.word_count),
article.reading_minutes()
),
social_line: chapters::social_line(&article.social),
head: ArticleHead {
understanding: chapters::understanding(pick),
summary: summary_for(&view.issue, pick).map(str::to_string),
excerpt_only: article.excerpt_only,
body_html: prepare_body(&article.content_html),
..ArticleHead::new(article, &config, is_admin)
},
discussion_html: pick
.discussion
.as_ref()
.map(|discussion| crate::comments::render_xhtml(discussion, &article.title)),
read_online_url: article.url.clone(),
dashboard_href: format!("/dashboard/articles/{}", article.id),
rating: is_admin.then(|| {
RatingWidget::for_issue(
article.id,
@@ -1217,20 +1227,7 @@ pub async fn standalone(
page.flash = take_flash(&session).await?;
Ok(Html(StandaloneArticleTemplate {
page,
title: article.title.clone(),
source_url: article.canonical_url.clone(),
byline: article.author.as_ref().map(|author| format!("By {author}")),
source: Source::new(&article, &config, is_admin),
meta_line: format!(
"{} words · ~{} min read",
thousands(article.word_count),
article.reading_minutes()
),
social_line: chapters::social_line(&article.social),
excerpt_only: article.excerpt_only,
body_html: prepare_body(&article.content_html),
read_online_url: article.url.clone(),
dashboard_href: format!("/dashboard/articles/{}", article.id),
head: ArticleHead::new(&article, &config, is_admin),
})
.into_response())
}
@@ -1244,18 +1241,17 @@ async fn related_views(
date: Date,
article_id: ArticleId,
) -> anyhow::Result<Vec<RelatedView>> {
let links = crate::related::for_article(db, date, article_id).await?;
if links.is_empty() {
let ids = crate::related::for_article(db, date, article_id).await?;
if ids.is_empty() {
return Ok(Vec::new());
}
let ids: Vec<ArticleId> = links.iter().map(|link| link.article_id).collect();
let articles = db.get_articles(&ids).await?;
let issues = crate::related::latest_issues(db, &ids).await?;
Ok(links
Ok(ids
.iter()
.filter_map(|link| {
let article = articles.get(&link.article_id)?;
let href = match issues.get(&link.article_id) {
.filter_map(|id| {
let article = articles.get(id)?;
let href = match issues.get(id) {
Some(date) => article_href(*date, article.id),
None => standalone_href(article.id),
};
+2
View File
@@ -0,0 +1,2 @@
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ head.source_url }}">{{ head.title }}</a></h1>{% match head.byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% let source = head.source %}{% include "_source.html" %} · {{ head.meta_line }}</p>{% let understanding = head.understanding %}{% include "_understanding.html" %}{% match head.summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match head.social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if head.excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
<div class="prose-body mt-10 border-t border-rule pt-8">{{ head.body_html|safe }}</div>
+2 -3
View File
@@ -1,9 +1,8 @@
{% extends "layout.html" %}{% block ears %}<span class="sm:hidden">{{ toc.short_date }}</span><span class="hidden sm:inline">{{ toc.display_date }} · No. {{ toc.issue_number }}</span>{% endblock %}{% block content %}<div class="mx-auto max-w-7xl lg:grid lg:grid-cols-[15rem_minmax(0,1fr)] lg:gap-x-8 lg:px-6 xl:grid-cols-[18rem_minmax(0,1fr)] xl:gap-x-14">{% include "_toc.html" %}
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0" data-toc-scroll>
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ source_url }}">{{ title }}</a></h1>{% match byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% include "_source.html" %} · {{ meta_line }}</p>{% include "_understanding.html" %}{% match summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
<div class="prose-body mt-10 border-t border-rule pt-8">{{ body_html|safe }}</div>
{% include "_article_head.html" %}
{% match discussion_html %}{% when Some with (discussion) %}<section class="discussion prose-body mt-12 border-t border-rule pt-6 [&_.comment-meta]:font-sans [&_.comment-meta]:text-sm [&_.comment-meta]:text-muted [&_blockquote.reply]:ml-6"><h2 class="mb-6 border-0 pt-0 text-2xl">Discussion</h2>{{ discussion|safe }}</section>{% when None %}{% endmatch %}
{% include "_related.html" %}
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
</article>
</div>{% endblock %}
+2 -3
View File
@@ -1,6 +1,5 @@
{% extends "layout.html" %}{% block ears %}Article{% endblock %}{% block content %}
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0">
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ source_url }}">{{ title }}</a></h1>{% match byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% include "_source.html" %} · {{ meta_line }}</p>{% match social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
<div class="prose-body mt-10 border-t border-rule pt-8">{{ body_html|safe }}</div>
<footer class="mt-12 border-t border-rule pt-6"><p class="flex flex-wrap gap-2"><a class="btn" href="{{ read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ dashboard_href }}">Dashboard</a>{% endif %}</p></footer>
{% include "_article_head.html" %}
<footer class="mt-12 border-t border-rule pt-6"><p class="flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p></footer>
</article>{% endblock %}