Compare commits
9
Commits
13c81440ef
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddc98788c6 | ||
|
|
682fc4a6fb | ||
|
|
08b15c42b4 | ||
|
|
03f0a454be | ||
|
|
bf6a8d0129 | ||
|
|
7851d8e231 | ||
|
|
5abf062c24 | ||
|
|
9e521ddaef | ||
|
|
50551114cd |
No files matched your search
+1
-1
@@ -157,7 +157,7 @@ affinity_full = 40
|
||||
slop_author_penalty = 0.75 # blend and utility × 0.25 for authors with an AI slop verdict
|
||||
semantic_min_words = 300
|
||||
exploration_slots = 5
|
||||
embedding_retention_days = 120 # `features prune`: unrated, unpublished vectors
|
||||
embedding_retention_days = 0 # `features prune`: unrated, unpublished vectors; 0 = keep forever
|
||||
telemetry_retention_days = 180 # `features prune`: candidate_runs and article_assessments
|
||||
|
||||
[curation.ranking.quotas]
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
//! Preview an article the way the Xteink X4 will show it.
|
||||
//!
|
||||
//! Browsers are a poor stand-in for the X4: its pages are rendered ahead of
|
||||
//! time by the XTC converter's CREngine, which has its own defaults (`code` is a
|
||||
//! block, `pre` has no margin) and only four grey levels. This builds both
|
||||
//! EPUB editions around one article, runs the real converter on the X4 one and
|
||||
//! decodes every page to `page-NNN.png`.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --example x4_preview -- <article URL | HTML file> <out dir>
|
||||
//! cargo run --example x4_preview -- --convert <x4.epub> <out dir>
|
||||
//! ```
|
||||
//!
|
||||
//! A URL goes through the real extraction path; an HTML file is used as the
|
||||
//! stored body as-is. `--convert` skips the build, for iterating on a
|
||||
//! stylesheet inside an unzipped EPUB. Needs the converter at
|
||||
//! `/opt/epub-to-xtc-converter` and runs it with `xtc-settings.example.json`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use daily_epub::config::Config;
|
||||
use daily_epub::epub;
|
||||
use daily_epub::extract::Extractor;
|
||||
use daily_epub::types::Edition;
|
||||
|
||||
const CONVERTER: &str = "/opt/epub-to-xtc-converter/cli/index.js";
|
||||
const SETTINGS: &str = "xtc-settings.example.json";
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.as_slice() {
|
||||
[flag, epub, out] if flag == "--convert" => {
|
||||
let pages = convert(Path::new(epub), Path::new(out))?;
|
||||
println!("{pages} pages in {out}");
|
||||
}
|
||||
[source, out] => {
|
||||
let out = PathBuf::from(out);
|
||||
std::fs::create_dir_all(&out)?;
|
||||
let html = if source.starts_with("http") {
|
||||
let extractor = Extractor::new(reqwest::Client::new(), vec![]);
|
||||
let page = extractor.fetch_readable(source).await?;
|
||||
extractor.finish_readable(source, &page).content_html
|
||||
} else {
|
||||
std::fs::read_to_string(source)?
|
||||
};
|
||||
std::fs::write(out.join("stored.html"), &html)?;
|
||||
|
||||
let mut issue = epub::fixtures::issue();
|
||||
issue.lineup.picks.truncate(1);
|
||||
issue.lineup.picks[0].article.content_html = html;
|
||||
let config = Config::default();
|
||||
for edition in [Edition::Standard, Edition::X4] {
|
||||
let artifact =
|
||||
epub::build_edition_with_images(&issue, edition, &config, &out, &[])?;
|
||||
println!("{edition:?}: {}", artifact.path.display());
|
||||
if edition == Edition::X4 {
|
||||
let pages = convert(&artifact.path, &out)?;
|
||||
println!("{pages} X4 pages in {}", out.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => anyhow::bail!(
|
||||
"usage: x4_preview <URL | HTML file> <out dir> | --convert <epub> <out dir>"
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the converter on `epub` and decode its pages into `out`.
|
||||
fn convert(epub: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
std::fs::create_dir_all(out)?;
|
||||
let xtch = out.join("x4.xtch");
|
||||
let status = Command::new("node")
|
||||
.args([CONVERTER, "convert"])
|
||||
.arg(epub)
|
||||
.arg("-o")
|
||||
.arg(&xtch)
|
||||
.args(["-f", "xtch", "-c", SETTINGS])
|
||||
.stdout(Stdio::null())
|
||||
.status()?;
|
||||
anyhow::ensure!(status.success(), "converter failed: {status}");
|
||||
decode_xtch(&xtch, out)
|
||||
}
|
||||
|
||||
fn u16_at(bytes: &[u8], at: usize) -> usize {
|
||||
u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize
|
||||
}
|
||||
|
||||
fn u64_at(bytes: &[u8], at: usize) -> usize {
|
||||
u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize
|
||||
}
|
||||
|
||||
/// Decode every page of an XTCH container to a PNG.
|
||||
///
|
||||
/// Layout per the converter's `docs/xtc-format-spec.md`: the page index is at
|
||||
/// the offset in header bytes 0x18..0x20, 16 bytes per page (offset, size,
|
||||
/// width, height). Each page is a 22-byte header and two bit planes stored
|
||||
/// column by column, right to left, 8 vertical pixels per byte. The plane
|
||||
/// order follows the encoder (`cli/encoder.js`), not the spec's table: the
|
||||
/// first plane holds the level's low bit, and 0b01 is dark grey.
|
||||
fn decode_xtch(path: &Path, out: &Path) -> anyhow::Result<usize> {
|
||||
let bytes = std::fs::read(path)?;
|
||||
let pages = u16_at(&bytes, 0x06);
|
||||
let index = u64_at(&bytes, 0x18);
|
||||
for page in 0..pages {
|
||||
let entry = index + page * 16;
|
||||
let (offset, width, height) = (
|
||||
u64_at(&bytes, entry),
|
||||
u16_at(&bytes, entry + 12),
|
||||
u16_at(&bytes, entry + 14),
|
||||
);
|
||||
let plane = (width * height).div_ceil(8);
|
||||
let column = height.div_ceil(8);
|
||||
let data = &bytes[offset + 22..];
|
||||
let mut image = image::GrayImage::new(width as u32, height as u32);
|
||||
for x in 0..width {
|
||||
for y in 0..height {
|
||||
let byte = (width - 1 - x) * column + y / 8;
|
||||
let bit = 7 - (y % 8);
|
||||
let low = (data[byte] >> bit) & 1;
|
||||
let high = (data[plane + byte] >> bit) & 1;
|
||||
let shade = [255, 85, 170, 0][usize::from(high << 1 | low)];
|
||||
image.put_pixel(x as u32, y as u32, image::Luma([shade]));
|
||||
}
|
||||
}
|
||||
image.save(out.join(format!("page-{page:03}.png")))?;
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Related articles under each web issue article: the nearest embedded articles
|
||||
-- across the whole archive, computed when the issue is published (and by
|
||||
-- `features related` for older issues). Stored rather than computed per page
|
||||
-- view because finding them is a scan of every cached embedding.
|
||||
CREATE TABLE issue_related (
|
||||
issue_date TEXT NOT NULL REFERENCES issues(date) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL, -- 0 = closest
|
||||
related_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
cosine REAL NOT NULL,
|
||||
PRIMARY KEY (issue_date, article_id, position)
|
||||
);
|
||||
+4
-2
@@ -490,6 +490,8 @@ pub struct RankingConfig {
|
||||
pub slop_author_penalty: f64,
|
||||
pub semantic_min_words: i64,
|
||||
pub exploration_slots: usize,
|
||||
/// Days before `features prune` drops unrated, unpublished vectors; 0 keeps
|
||||
/// them forever (the default, so related articles span the whole archive).
|
||||
pub embedding_retention_days: i64,
|
||||
pub telemetry_retention_days: i64,
|
||||
pub quotas: RankingQuotas,
|
||||
@@ -517,7 +519,7 @@ impl Default for RankingConfig {
|
||||
slop_author_penalty: 0.75,
|
||||
semantic_min_words: 300,
|
||||
exploration_slots: 5,
|
||||
embedding_retention_days: 120,
|
||||
embedding_retention_days: 0,
|
||||
telemetry_retention_days: 180,
|
||||
quotas: RankingQuotas::default(),
|
||||
weights: RankingWeights::default(),
|
||||
@@ -2104,7 +2106,7 @@ mod tests {
|
||||
assert_eq!(ranking.weights.utility.knn, 0.10);
|
||||
assert_eq!(ranking.weights.utility.affinity, 0.05);
|
||||
assert_eq!(ranking.diversity.per_cluster_cap, 2);
|
||||
assert_eq!(ranking.embedding_retention_days, 120);
|
||||
assert_eq!(ranking.embedding_retention_days, 0);
|
||||
assert_eq!(ranking.telemetry_retention_days, 180);
|
||||
cfg.validate().unwrap();
|
||||
|
||||
|
||||
@@ -444,10 +444,10 @@ pub fn dot(left: &[f32], right: &[f32]) -> Result<f64, EmbeddingError> {
|
||||
|
||||
/// The `limit` closest compatible cached article vectors, highest cosine first.
|
||||
///
|
||||
/// A brute-force scan: the cache holds at most a few thousand rows after
|
||||
/// pruning, so one statement plus a sort is cheap. The caller supplies the
|
||||
/// target row's model, dimension and decoded vector so pages that already
|
||||
/// load that row do not query it a second time. Malformed candidate blobs are
|
||||
/// A brute-force scan of every compatible row: fine for the admin dashboard,
|
||||
/// too slow for reader pages (see [`crate::related`], which stores its
|
||||
/// results). The caller supplies the target row's model, dimension and decoded
|
||||
/// vector so pages that already load that row do not query it a second time. Malformed candidate blobs are
|
||||
/// ignored like malformed entries in the normal cache loader.
|
||||
pub async fn nearest_articles(
|
||||
db: &Db,
|
||||
|
||||
+49
-9
@@ -1058,7 +1058,7 @@ pub fn render_stats_text(data: &StatsData) -> String {
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Pruned {
|
||||
/// `article_embeddings` of unrated, unpublished articles past
|
||||
/// `embedding_retention_days`.
|
||||
/// `embedding_retention_days`; always 0 when that is 0 (keep forever).
|
||||
pub embeddings: u64,
|
||||
/// `candidate_runs` rows of runs past `telemetry_retention_days`.
|
||||
pub telemetry: u64,
|
||||
@@ -1067,7 +1067,8 @@ pub struct Pruned {
|
||||
}
|
||||
|
||||
/// Delete `article_embeddings` for articles neither rated nor published that
|
||||
/// are older than `embedding_retention_days`, `candidate_runs` rows whose run
|
||||
/// are older than `embedding_retention_days` (0 or less keeps every vector, so
|
||||
/// related articles can reach the whole archive), `candidate_runs` rows whose run
|
||||
/// started more than `telemetry_retention_days` ago, and `article_assessments`
|
||||
/// older than the same window (§7.1, §7.4). Runs from `features prune` and
|
||||
/// once per `generate` after publishing.
|
||||
@@ -1081,8 +1082,11 @@ pub async fn prune(
|
||||
now.checked_sub(jiff::Span::new().hours(days.max(0).saturating_mul(24)))
|
||||
.unwrap_or(Timestamp::UNIX_EPOCH)
|
||||
};
|
||||
let embeddings = sqlx::query(
|
||||
"DELETE FROM article_embeddings
|
||||
let embeddings = if embedding_retention_days <= 0 {
|
||||
0
|
||||
} else {
|
||||
sqlx::query(
|
||||
"DELETE FROM article_embeddings
|
||||
WHERE article_id IN (
|
||||
SELECT ae.article_id
|
||||
FROM article_embeddings ae JOIN articles a ON a.id = ae.article_id
|
||||
@@ -1090,11 +1094,12 @@ pub async fn prune(
|
||||
AND NOT EXISTS (SELECT 1 FROM rating_events re WHERE re.article_id = ae.article_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM issue_articles ia WHERE ia.article_id = ae.article_id)
|
||||
)",
|
||||
)
|
||||
.bind(fmt_ts(cutoff(embedding_retention_days)))
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected();
|
||||
)
|
||||
.bind(fmt_ts(cutoff(embedding_retention_days)))
|
||||
.execute(db.pool())
|
||||
.await?
|
||||
.rows_affected()
|
||||
};
|
||||
|
||||
let telemetry = sqlx::query(
|
||||
"DELETE FROM candidate_runs
|
||||
@@ -1942,4 +1947,39 @@ mean generation time: n/a (0 runs)
|
||||
// A second pass finds nothing left to remove.
|
||||
assert_eq!(prune(&db, 120, 180, now).await.unwrap(), Pruned::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_embedding_retention_keeps_every_vector() {
|
||||
let (_dir, db) = db_with_articles(&[1, 2]).await;
|
||||
let now = Timestamp::now();
|
||||
let old = fmt_ts(now - jiff::Span::new().hours(2000 * 24));
|
||||
sqlx::query("UPDATE articles SET first_seen = ?")
|
||||
.bind(&old)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let blob = encode_blob(&[0.5, 0.5]).unwrap();
|
||||
for id in 1..=2 {
|
||||
sqlx::query(
|
||||
"INSERT INTO article_embeddings
|
||||
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||
VALUES (?, 'voyage-4-lite', 2, 'h', ?, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&blob)
|
||||
.bind(&old)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// 0 must not mean "a cutoff of now", which would delete everything.
|
||||
let pruned = prune(&db, 0, 180, now).await.unwrap();
|
||||
assert_eq!(pruned.embeddings, 0);
|
||||
let remaining: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM article_embeddings")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 2);
|
||||
}
|
||||
}
|
||||
@@ -160,7 +160,16 @@ figcaption,
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
pre, code {
|
||||
/* The firmwares (CrossPoint, CrossInk) ignore `white-space` and treat `pre`
|
||||
as inline, so the X4 simplifier makes each listing line a `p.code-line`;
|
||||
the first and last lines carry the block's spacing, and a blank source
|
||||
line becomes a `p.code-gap` margin. `figure` centres for
|
||||
image captions, so lines reset alignment. The XTC converter's CREngine
|
||||
defaults `code` to a block. */
|
||||
pre {
|
||||
display: block;
|
||||
margin: 0;
|
||||
text-align: left;
|
||||
font-family: monospace;
|
||||
font-size: 0.85em;
|
||||
white-space: pre-wrap;
|
||||
@@ -169,24 +178,81 @@ pre, code {
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
/* Syntax tokens (src/highlight.rs), monochrome for e-ink: weight and slant
|
||||
carry the structure, one grey for comments. Flat selectors on purpose. */
|
||||
.hl-keyword,
|
||||
.hl-tag,
|
||||
p.code-line {
|
||||
margin: 0;
|
||||
text-indent: 0;
|
||||
text-align: left;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
p.code-first {
|
||||
margin-top: 0.6em;
|
||||
}
|
||||
|
||||
p.code-last {
|
||||
margin-bottom: 0.6em;
|
||||
}
|
||||
|
||||
/* Where the source had a blank line: half a line, not a whole one. */
|
||||
p.code-gap {
|
||||
margin-bottom: 0.5em;
|
||||
}
|
||||
|
||||
code {
|
||||
display: inline;
|
||||
font-family: monospace;
|
||||
font-size: 0.85em;
|
||||
hyphens: none;
|
||||
-webkit-hyphens: none;
|
||||
}
|
||||
|
||||
/* Syntax tokens (src/highlight.rs), as in style.css but on the X4's four
|
||||
levels only (#000, #555, #aaa, white), so glyphs are not dithered: ink for
|
||||
names, dark grey for keywords, literals and punctuation, light grey for
|
||||
comments. */
|
||||
.hl-function,
|
||||
.hl-macro,
|
||||
.hl-title,
|
||||
.hl-strong {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
font-style: italic;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.hl-type,
|
||||
.hl-namespace,
|
||||
.hl-constructor,
|
||||
.hl-attribute,
|
||||
.hl-emphasis {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.hl-keyword,
|
||||
.hl-tag {
|
||||
font-weight: bold;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-string {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-number,
|
||||
.hl-constant,
|
||||
.hl-label {
|
||||
font-style: italic;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-punctuation,
|
||||
.hl-operator {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
font-style: italic;
|
||||
color: #aaaaaa;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85em;
|
||||
|
||||
@@ -221,29 +221,65 @@ hr.rule {
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
/* CREngine readers (KOReader, PocketBook) default `code` to a block. */
|
||||
.article-body code {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
/* Left even inside a `<figure>`, which centres for image captions. */
|
||||
.article-body pre {
|
||||
margin: 0.8em 0;
|
||||
text-align: left;
|
||||
white-space: pre-wrap;
|
||||
word-wrap: break-word;
|
||||
border-left: 2px solid #cccccc;
|
||||
padding-left: 0.5em;
|
||||
}
|
||||
|
||||
/* Syntax tokens (src/highlight.rs), monochrome for e-ink: weight and slant
|
||||
carry the structure, one grey for comments. Flat selectors on purpose. */
|
||||
.hl-keyword,
|
||||
.hl-tag,
|
||||
/* Syntax tokens (src/highlight.rs), monochrome for e-ink, after zenbones and
|
||||
lackluster: what the code names stays in ink (functions bold, types italic),
|
||||
the language's own furniture steps back to grey (keywords bold grey,
|
||||
literals and punctuation grey), comments lightest. Flat selectors. */
|
||||
.hl-function,
|
||||
.hl-macro,
|
||||
.hl-title,
|
||||
.hl-strong {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
.hl-type,
|
||||
.hl-namespace,
|
||||
.hl-constructor,
|
||||
.hl-attribute,
|
||||
.hl-emphasis {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.hl-keyword,
|
||||
.hl-tag {
|
||||
font-weight: bold;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-string {
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-number,
|
||||
.hl-constant,
|
||||
.hl-label {
|
||||
font-style: italic;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.hl-emphasis {
|
||||
.hl-punctuation,
|
||||
.hl-operator {
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.hl-comment {
|
||||
font-style: italic;
|
||||
color: #888888;
|
||||
}
|
||||
|
||||
.article-body table {
|
||||
|
||||
+415
-7
@@ -124,10 +124,295 @@ fn is_dropped_declaration(decl: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Class of every paragraph [`paragraph_code_lines`] makes from a listing
|
||||
/// line; the first and last also get [`CODE_FIRST_CLASS`] / [`CODE_LAST_CLASS`].
|
||||
pub const CODE_LINE_CLASS: &str = "code-line";
|
||||
pub const CODE_FIRST_CLASS: &str = "code-first";
|
||||
pub const CODE_LAST_CLASS: &str = "code-last";
|
||||
/// Marks a line followed by blank lines in the source; it gets a short gap.
|
||||
pub const CODE_GAP_CLASS: &str = "code-gap";
|
||||
|
||||
/// Strip layout constructs the X4 renderer handles poorly from chapter markup,
|
||||
/// then soft-hyphenate anything too long for it to wrap (§3.10).
|
||||
pub fn simplify_xhtml(xhtml: &str) -> String {
|
||||
break_long_words(&strip_attributes(xhtml, DROPPED_ATTRIBUTES))
|
||||
paragraph_code_lines(&break_long_words(&strip_attributes(
|
||||
&flatten_lossy_tables(xhtml),
|
||||
DROPPED_ATTRIBUTES,
|
||||
)))
|
||||
}
|
||||
|
||||
/// Table elements dropped outright when an image table is flattened.
|
||||
const TABLE_FRAME_ELEMENTS: &[&str] =
|
||||
&["table", "thead", "tbody", "tfoot", "tr", "colgroup", "col"];
|
||||
|
||||
/// Table elements whose content becomes a `<div>` when a table is flattened.
|
||||
const TABLE_CELL_ELEMENTS: &[&str] = &["td", "th", "caption"];
|
||||
|
||||
/// Elements the X4 firmwares treat as images.
|
||||
const IMAGE_ELEMENTS: &[&str] = &["img", "image"];
|
||||
|
||||
/// Turn every table the X4 firmwares would lose content from into plain
|
||||
/// `<div>` blocks: one that holds an image or another table.
|
||||
///
|
||||
/// The firmwares (CrossPoint and CrossInk) never draw an image inside a table
|
||||
/// cell: they print its alt text instead. Publishers that lay out a figure as a
|
||||
/// one-cell table with the caption beneath the image (The Space Review, many
|
||||
/// older sites) would lose every figure. CrossInk also throws away all the text
|
||||
/// of a table nested in another. Flattening keeps each cell's content in order,
|
||||
/// which is how the firmwares already fall back for any table they cannot lay
|
||||
/// out as a grid. Tables of plain text stay tables.
|
||||
fn flatten_lossy_tables(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
if let Some(name) = raw_text_name(tag) {
|
||||
let close = find_close_tag(html, end, name).unwrap_or(html.len());
|
||||
out.push_str(&html[start..close]);
|
||||
cursor = close;
|
||||
continue;
|
||||
}
|
||||
if !is_element_tag(tag, "table", false) {
|
||||
out.push_str(tag);
|
||||
cursor = end;
|
||||
continue;
|
||||
}
|
||||
let table_end = matching_table_end(html, start);
|
||||
let table = &html[start..table_end];
|
||||
// Past the outer `<table` tag, so it doesn't count as its own nesting.
|
||||
let inner = &table[end - start..];
|
||||
if contains_element(inner, IMAGE_ELEMENTS) || contains_element(inner, &["table"]) {
|
||||
out.push_str(&flatten_table(table));
|
||||
} else {
|
||||
out.push_str(table);
|
||||
}
|
||||
cursor = table_end;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Byte offset just past the `</table>` that closes the table opened at
|
||||
/// `start`, counting nested tables; the end of `html` if it is never closed.
|
||||
fn matching_table_end(html: &str, start: usize) -> usize {
|
||||
let mut depth = 0usize;
|
||||
let mut cursor = start;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let tag_start = cursor + rel;
|
||||
let Some(end) = tag_end(html, tag_start) else {
|
||||
break;
|
||||
};
|
||||
let tag = &html[tag_start..end];
|
||||
if is_element_tag(tag, "table", false) && !tag.ends_with("/>") {
|
||||
depth += 1;
|
||||
} else if is_element_tag(tag, "table", true) {
|
||||
depth = depth.saturating_sub(1);
|
||||
if depth == 0 {
|
||||
return end;
|
||||
}
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
html.len()
|
||||
}
|
||||
|
||||
/// Whether any tag in `html` opens one of the named elements.
|
||||
fn contains_element(html: &str, names: &[&str]) -> bool {
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
return false;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
if names.iter().any(|name| is_element_tag(tag, name, false)) {
|
||||
return true;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Rewrite one table's markup (nested tables included) as `<div>` blocks.
|
||||
fn flatten_table(table: &str) -> String {
|
||||
let mut out = String::with_capacity(table.len());
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = table[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&table[cursor..start]);
|
||||
let Some(end) = tag_end(table, start) else {
|
||||
out.push_str(&table[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &table[start..end];
|
||||
cursor = end;
|
||||
let is = |names: &[&str], closing: bool| {
|
||||
names.iter().any(|name| is_element_tag(tag, name, closing))
|
||||
};
|
||||
if is(TABLE_FRAME_ELEMENTS, false) || is(TABLE_FRAME_ELEMENTS, true) {
|
||||
// A newline, not nothing: the firmwares buffer the last word
|
||||
// before an element boundary until whitespace arrives.
|
||||
out.push('\n');
|
||||
} else if is(TABLE_CELL_ELEMENTS, false) {
|
||||
out.push_str(if tag.ends_with("/>") {
|
||||
"<div></div>"
|
||||
} else {
|
||||
"<div>"
|
||||
});
|
||||
} else if is(TABLE_CELL_ELEMENTS, true) {
|
||||
out.push_str("</div>");
|
||||
} else {
|
||||
out.push_str(tag);
|
||||
}
|
||||
}
|
||||
out.push_str(&table[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Turn each line of every `<pre>` into its own `<p class="code-line">`.
|
||||
///
|
||||
/// The X4 firmwares (CrossPoint and forks such as CrossInk) ignore
|
||||
/// `white-space` and lay out only `p`, `div`, `li`, `blockquote`, lists and
|
||||
/// `br` as blocks: a `<pre>` is inline, so its margins never apply and its
|
||||
/// `<br/>` lines are handled by each firmware's own rules. CrossInk glues the
|
||||
/// first line onto the paragraph before, and empty `<br/>` lines count as scene
|
||||
/// breaks. A paragraph per line takes the one path every firmware lays out the
|
||||
/// same way. The first and last lines carry the block's margins; blank lines
|
||||
/// become a half-line gap (see [`code_paragraphs`]).
|
||||
///
|
||||
/// Inline elements open across a line break (a `<code>`, a multi-line
|
||||
/// comment's span) are closed at the end of the line and reopened at the start
|
||||
/// of the next, so every paragraph is well formed.
|
||||
fn paragraph_code_lines(html: &str) -> String {
|
||||
let mut out = String::with_capacity(html.len() + 256);
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
out.push_str(&html[cursor..start]);
|
||||
let Some(end) = tag_end(html, start) else {
|
||||
out.push_str(&html[start..]);
|
||||
return out;
|
||||
};
|
||||
let tag = &html[start..end];
|
||||
out.push_str(tag);
|
||||
cursor = end;
|
||||
if !is_element_tag(tag, "pre", false) {
|
||||
continue;
|
||||
}
|
||||
let Some(close) = find_close_tag(html, cursor, "pre") else {
|
||||
continue;
|
||||
};
|
||||
out.push_str(&code_paragraphs(&html[cursor..close]));
|
||||
cursor = close;
|
||||
}
|
||||
out.push_str(&html[cursor..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The paragraphs for one `<pre>` body whose line breaks are `<br/>`.
|
||||
fn code_paragraphs(body: &str) -> String {
|
||||
// Each line's markup, plus whether it shows any text.
|
||||
let mut lines: Vec<(String, bool)> = Vec::new();
|
||||
// Inline elements open at this point: (name, opening tag).
|
||||
let mut open: Vec<(String, &str)> = Vec::new();
|
||||
let mut line = String::new();
|
||||
let mut visible = false;
|
||||
let mut cursor = 0usize;
|
||||
let finish = |line: &mut String, visible: &mut bool, open: &[(String, &str)]| {
|
||||
let mut done = std::mem::take(line);
|
||||
if !*visible {
|
||||
done.push(NO_BREAK_SPACE);
|
||||
}
|
||||
for (name, _) in open.iter().rev() {
|
||||
done.push_str(&format!("</{name}>"));
|
||||
}
|
||||
let shown = std::mem::replace(visible, false);
|
||||
for (_, tag) in open {
|
||||
line.push_str(tag);
|
||||
}
|
||||
(done, shown)
|
||||
};
|
||||
while cursor < body.len() {
|
||||
let Some(rel) = body[cursor..].find('<') else {
|
||||
let text = &body[cursor..];
|
||||
visible |= shows_text(text);
|
||||
line.push_str(text);
|
||||
break;
|
||||
};
|
||||
let start = cursor + rel;
|
||||
let text = &body[cursor..start];
|
||||
visible |= shows_text(text);
|
||||
line.push_str(text);
|
||||
let Some(end) = tag_end(body, start) else {
|
||||
line.push_str(&body[start..]);
|
||||
break;
|
||||
};
|
||||
let tag = &body[start..end];
|
||||
cursor = end;
|
||||
let inner = tag.trim_start_matches('<').trim_end_matches('>');
|
||||
let name = crate::html::tag_name(inner);
|
||||
if name == "br" {
|
||||
lines.push(finish(&mut line, &mut visible, &open));
|
||||
} else if inner.starts_with('/') {
|
||||
if let Some(at) = open.iter().rposition(|(open_name, _)| *open_name == name) {
|
||||
open.remove(at);
|
||||
}
|
||||
line.push_str(tag);
|
||||
} else {
|
||||
if !inner.trim_end().ends_with('/') {
|
||||
open.push((name, tag));
|
||||
}
|
||||
line.push_str(tag);
|
||||
}
|
||||
}
|
||||
lines.push(finish(&mut line, &mut visible, &open));
|
||||
// Blank lines become a short gap after the line before them rather than
|
||||
// paragraphs of their own: with the firmware's paragraph spacing on, a
|
||||
// blank paragraph adds a full line plus two half-line gaps and reads as a
|
||||
// double newline. Leading, trailing and repeated blanks collapse.
|
||||
let mut kept: Vec<(String, bool)> = Vec::with_capacity(lines.len());
|
||||
for (line, shown) in lines {
|
||||
if shown {
|
||||
kept.push((line, false));
|
||||
} else if let Some(previous) = kept.last_mut() {
|
||||
previous.1 = true;
|
||||
}
|
||||
}
|
||||
if kept.is_empty() {
|
||||
// An all-blank listing still occupies one line.
|
||||
kept.push((NO_BREAK_SPACE.to_string(), false));
|
||||
}
|
||||
|
||||
let last = kept.len() - 1;
|
||||
let mut out = String::with_capacity(body.len() + kept.len() * 48);
|
||||
for (index, (line, gap_after)) in kept.into_iter().enumerate() {
|
||||
let mut class = String::from(CODE_LINE_CLASS);
|
||||
if index == 0 {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_FIRST_CLASS);
|
||||
}
|
||||
if index == last {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_LAST_CLASS);
|
||||
} else if gap_after {
|
||||
class.push(' ');
|
||||
class.push_str(CODE_GAP_CLASS);
|
||||
}
|
||||
out.push_str(&format!("<p class=\"{class}\">{line}</p>"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a run of character data shows anything but spaces.
|
||||
fn shows_text(text: &str) -> bool {
|
||||
text.chars()
|
||||
.any(|c| !c.is_whitespace() && c != NO_BREAK_SPACE)
|
||||
}
|
||||
|
||||
/// Insert soft hyphens into over-long words and make `<pre>` whitespace
|
||||
@@ -609,7 +894,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pre_whitespace_is_made_explicit_inside_nested_tags() {
|
||||
let input = "<pre><code>fn main() {\n if x < 2 {\n <em>return</em>;\n }\n <strong><a href=\"#\">// done</a></strong>\n}</code></pre>";
|
||||
let out = simplify_xhtml(input);
|
||||
let out = break_long_words(input);
|
||||
assert_eq!(
|
||||
out,
|
||||
concat!(
|
||||
@@ -622,10 +907,40 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_code_line_becomes_a_paragraph() {
|
||||
let input = "<p>Before.</p><figure><pre><code>let a = 1;\n\n let b = 2;</code></pre></figure><p>After.</p>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
concat!(
|
||||
"<p>Before.</p><figure><pre>",
|
||||
"<p class=\"code-line code-first code-gap\"><code>let a = 1;</code></p>",
|
||||
"<p class=\"code-line code-last\"><code>\u{00a0}\u{00a0}let b = 2;</code></p>",
|
||||
"</pre></figure><p>After.</p>"
|
||||
)
|
||||
);
|
||||
// `<pre` as a prefix of another name is not a code block.
|
||||
assert_eq!(simplify_xhtml("<prefix>x</prefix>"), "<prefix>x</prefix>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spans_open_across_a_line_break_are_closed_and_reopened() {
|
||||
let input = "<pre><code><span class=\"hl-comment\">/* one\ntwo */</span> x\n</code></pre>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
concat!(
|
||||
"<pre>",
|
||||
"<p class=\"code-line code-first\"><code><span class=\"hl-comment\">/* one</span></code></p>",
|
||||
"<p class=\"code-line code-last\"><code><span class=\"hl-comment\">two */</span> x</code></p>",
|
||||
"</pre>"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tabs_in_pre_become_four_no_break_spaces() {
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre><code>one\ttwo</code></pre>"),
|
||||
break_long_words("<pre><code>one\ttwo</code></pre>"),
|
||||
"<pre><code>one\u{00a0}\u{00a0}\u{00a0}\u{00a0}two</code></pre>"
|
||||
);
|
||||
}
|
||||
@@ -634,7 +949,7 @@ mod tests {
|
||||
fn whitespace_outside_pre_is_untouched() {
|
||||
let input = "<p>outside text\n next\tword</p><pre>x y</pre><p>tail text</p>";
|
||||
assert_eq!(
|
||||
simplify_xhtml(input),
|
||||
break_long_words(input),
|
||||
"<p>outside text\n next\tword</p><pre>x\u{00a0}\u{00a0}y</pre><p>tail text</p>"
|
||||
);
|
||||
}
|
||||
@@ -642,11 +957,11 @@ mod tests {
|
||||
#[test]
|
||||
fn newline_right_after_pre_open_is_not_a_blank_line() {
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre><code>\nlet a = 1;\n\nlet b = 2;</code></pre>"),
|
||||
break_long_words("<pre><code>\nlet a = 1;\n\nlet b = 2;</code></pre>"),
|
||||
"<pre><code>let a = 1;<br/><br/>let b = 2;</code></pre>"
|
||||
);
|
||||
assert_eq!(
|
||||
simplify_xhtml("<pre>\n x</pre>"),
|
||||
break_long_words("<pre>\n x</pre>"),
|
||||
"<pre>\u{00a0}\u{00a0}x</pre>"
|
||||
);
|
||||
}
|
||||
@@ -654,7 +969,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pre_without_a_trailing_newline_gets_no_extra_break() {
|
||||
let input = "<pre><code>let answer = 42;</code></pre>";
|
||||
assert_eq!(simplify_xhtml(input), input);
|
||||
assert_eq!(break_long_words(input), input);
|
||||
}
|
||||
|
||||
/// The shipped X4 stylesheet must already satisfy the X4 rules, so the
|
||||
@@ -794,4 +1109,97 @@ mod tests {
|
||||
"<?xml version=\"1.0\"?>\n<!DOCTYPE html>\n<html><body><p>2 < 3</p></body></html>";
|
||||
assert_eq!(simplify_xhtml(input), input);
|
||||
}
|
||||
|
||||
/// Whether any image in `html` sits inside a table, where the X4
|
||||
/// firmwares print its alt text instead of drawing it.
|
||||
fn image_inside_table(html: &str) -> bool {
|
||||
let mut depth = 0usize;
|
||||
let mut cursor = 0usize;
|
||||
while let Some(rel) = html[cursor..].find('<') {
|
||||
let start = cursor + rel;
|
||||
let end = tag_end(html, start).unwrap();
|
||||
let tag = &html[start..end];
|
||||
if is_element_tag(tag, "table", false) {
|
||||
depth += 1;
|
||||
} else if is_element_tag(tag, "table", true) {
|
||||
depth -= 1;
|
||||
} else if depth > 0 && IMAGE_ELEMENTS.iter().any(|n| is_element_tag(tag, n, false)) {
|
||||
return true;
|
||||
}
|
||||
cursor = end;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_figure_laid_out_as_a_table_becomes_divs() {
|
||||
// The Space Review's figure markup (issue 2026-10-01).
|
||||
let input = "<p>Before.</p><table><tbody><tr><td><img src=\"images/a.jpg\" alt=\"ELINT images\"/><p>During the 1970s. (credit: NRO)</p></td></tr></tbody></table><h3>Next</h3>";
|
||||
let out = simplify_xhtml(input);
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
assert!(!out.contains("<t"), "{out}");
|
||||
assert!(
|
||||
out.contains("<div><img src=\"images/a.jpg\" alt=\"ELINT images\"/><p>During the 1970s. (credit: NRO)</p></div>"),
|
||||
"{out}"
|
||||
);
|
||||
assert!(
|
||||
out.starts_with("<p>Before.</p>") && out.ends_with("<h3>Next</h3>"),
|
||||
"{out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_cell_of_an_image_table_keeps_its_content_in_order() {
|
||||
let input = "<table class=\"x\" width=\"600\"><caption>Cap</caption><thead><tr><th>A</th><th>B</th></tr></thead><tr><td colspan=\"2\"><img src=\"1.jpg\"/></td></tr><tr><td>left</td><td>right</td></tr></table>";
|
||||
let out = flatten_lossy_tables(input);
|
||||
assert_eq!(
|
||||
out.split_whitespace().collect::<String>(),
|
||||
"<div>Cap</div><div>A</div><div>B</div><div><imgsrc=\"1.jpg\"/></div><div>left</div><div>right</div>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_image_tables_flatten_from_the_outside_in() {
|
||||
for input in [
|
||||
// Image in the inner table only.
|
||||
"<table><tr><td><table><tr><td><img src=\"1.jpg\"/></td></tr></table></td></tr></table><p>after</p>",
|
||||
// Image in the outer table after a text-only inner table.
|
||||
"<table><tr><td><table><tr><td>x</td></tr></table><img src=\"1.jpg\"/></td></tr></table><p>after</p>",
|
||||
// An SVG `<image>` counts too.
|
||||
"<table><tr><td><svg><image href=\"1.jpg\"/></svg></td></tr></table><p>after</p>",
|
||||
] {
|
||||
let out = simplify_xhtml(input);
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
assert!(!out.contains("<table"), "{out}");
|
||||
assert!(out.ends_with("<p>after</p>"), "{out}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_nested_text_table_flattens_its_outer_table() {
|
||||
let input = "<table><tr><td>outer<table><tr><td>inner</td></tr></table></td></tr></table>";
|
||||
let out = flatten_lossy_tables(input);
|
||||
assert_eq!(
|
||||
out.split_whitespace().collect::<String>(),
|
||||
"<div>outer<div>inner</div></div>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_tables_stay_tables() {
|
||||
let input = "<table><tr><th>Year</th><td>1971</td></tr></table><p><img src=\"1.jpg\"/></p><table><tr><td>2</td></tr></table>";
|
||||
assert_eq!(flatten_lossy_tables(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_markup_inside_raw_text_is_left_alone() {
|
||||
let input = "<script>let t = '<table><td><img src=x></td></table>';</script>";
|
||||
assert_eq!(flatten_lossy_tables(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unclosed_image_table_still_flattens() {
|
||||
let out = flatten_lossy_tables("<table><tr><td><img src=\"1.jpg\"/>");
|
||||
assert!(!image_inside_table(&out), "{out}");
|
||||
}
|
||||
}
|
||||
+39
-6
@@ -32,6 +32,8 @@ pub const EXITED_BEFORE_START: &str = "unit exited before the job started; see t
|
||||
pub enum Job {
|
||||
/// `generate` (today) or `generate-YYYY-MM-DD`.
|
||||
Generate { date: Option<Date> },
|
||||
/// `rebuild-YYYY-MM-DD` → `rebuild --date`.
|
||||
Rebuild { date: Date },
|
||||
/// `dry-run` → `generate --dry-run`.
|
||||
DryRun,
|
||||
/// `profile-rebuild` → `profile rebuild`.
|
||||
@@ -70,8 +72,9 @@ impl Job {
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
|
||||
}
|
||||
|
||||
/// Parse a catalogue name or the dated `generate-YYYY-MM-DD` form; anything
|
||||
/// else (including `../x`, uppercase, unknown names) is `None`.
|
||||
/// Parse a catalogue name or a dated `generate-YYYY-MM-DD` or
|
||||
/// `rebuild-YYYY-MM-DD` form; anything else (including `../x`, uppercase,
|
||||
/// unknown names) is `None`.
|
||||
pub fn parse(name: &str) -> Option<Job> {
|
||||
if !Self::valid_name(name) {
|
||||
return None;
|
||||
@@ -86,10 +89,17 @@ impl Job {
|
||||
"import-ratings" => Some(Job::ImportRatings),
|
||||
"interests-categorize" => Some(Job::InterestsCategorize),
|
||||
_ => {
|
||||
let date = name.strip_prefix("generate-")?;
|
||||
let (rebuild, date) = match name.strip_prefix("generate-") {
|
||||
Some(date) => (false, date),
|
||||
None => (true, name.strip_prefix("rebuild-")?),
|
||||
};
|
||||
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
|
||||
let parsed: Date = date.parse().ok()?;
|
||||
(parsed.to_string() == date).then_some(Job::Generate { date: Some(parsed) })
|
||||
(parsed.to_string() == date).then_some(if rebuild {
|
||||
Job::Rebuild { date: parsed }
|
||||
} else {
|
||||
Job::Generate { date: Some(parsed) }
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,6 +108,7 @@ impl Job {
|
||||
match self {
|
||||
Job::Generate { date: None } => "generate".into(),
|
||||
Job::Generate { date: Some(date) } => format!("generate-{date}"),
|
||||
Job::Rebuild { date } => format!("rebuild-{date}"),
|
||||
Job::DryRun => "dry-run".into(),
|
||||
Job::ProfileRebuild => "profile-rebuild".into(),
|
||||
Job::FeaturesBackfill => "features-backfill".into(),
|
||||
@@ -121,6 +132,9 @@ impl Job {
|
||||
Job::Generate { date: Some(_) } => {
|
||||
"Build and publish the issue of a given date, republishing it if it exists."
|
||||
}
|
||||
Job::Rebuild { .. } => {
|
||||
"Re-render and republish a published issue's EPUBs from its stored content, without curating again."
|
||||
}
|
||||
Job::DryRun => "Run the whole pipeline but publish nothing and record no issue.",
|
||||
Job::ProfileRebuild => {
|
||||
"Regenerate the learned taste adjustments from ratings with the editor model."
|
||||
@@ -144,6 +158,7 @@ impl Job {
|
||||
pub fn takes_lock(&self) -> Option<&'static str> {
|
||||
match self {
|
||||
Job::Generate { .. } | Job::DryRun => Some("generate"),
|
||||
Job::Rebuild { .. } => Some("rebuild"),
|
||||
Job::ProfileRebuild => Some("profile rebuild"),
|
||||
Job::FeaturesBackfill => Some("features backfill"),
|
||||
Job::BackfillSocial => Some("backfill-social"),
|
||||
@@ -151,9 +166,9 @@ impl Job {
|
||||
}
|
||||
}
|
||||
|
||||
/// Needs a confirmation dialog: `generate` republishes an issue.
|
||||
/// Needs a confirmation dialog: `generate` and `rebuild` republish an issue.
|
||||
pub fn dangerous(&self) -> bool {
|
||||
matches!(self, Job::Generate { .. })
|
||||
matches!(self, Job::Generate { .. } | Job::Rebuild { .. })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -481,6 +496,24 @@ mod tests {
|
||||
assert!(!Job::parse("dry-run").unwrap().dangerous());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_the_dated_rebuild_form() {
|
||||
let job = Job::parse("rebuild-2026-10-01").unwrap();
|
||||
assert_eq!(
|
||||
job,
|
||||
Job::Rebuild {
|
||||
date: "2026-10-01".parse().unwrap()
|
||||
}
|
||||
);
|
||||
assert_eq!(job.name(), "rebuild-2026-10-01");
|
||||
assert_eq!(job.unit(), "daily-epub-job@rebuild-2026-10-01.service");
|
||||
assert_eq!(job.takes_lock(), Some("rebuild"));
|
||||
assert!(job.dangerous());
|
||||
for bad in ["rebuild", "rebuild-", "rebuild-2026-10-1", "rebuild-today"] {
|
||||
assert_eq!(Job::parse(bad), None, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_traversal_uppercase_and_unknown_names() {
|
||||
for name in [
|
||||
|
||||
@@ -35,6 +35,7 @@ pub mod miniflux;
|
||||
pub mod pipeline;
|
||||
pub mod publish;
|
||||
pub mod rate;
|
||||
pub mod related;
|
||||
pub mod report;
|
||||
pub mod server;
|
||||
pub mod social;
|
||||
|
||||
+84
-9
@@ -36,6 +36,8 @@ struct Cli {
|
||||
enum Command {
|
||||
/// Build (and publish) one issue.
|
||||
Generate(GenerateArgs),
|
||||
/// Rebuild and republish a published issue's EPUBs from its stored content.
|
||||
Rebuild(RebuildArgs),
|
||||
/// Run the rating endpoints, the OPDS catalog and downloads.
|
||||
Serve,
|
||||
/// Taste-profile maintenance.
|
||||
@@ -159,6 +161,13 @@ enum ConfigCommand {
|
||||
Check,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RebuildArgs {
|
||||
/// Date of the published issue to rebuild.
|
||||
#[arg(long, value_name = "YYYY-MM-DD")]
|
||||
date: jiff::civil::Date,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct GenerateArgs {
|
||||
/// Issue date in the configured timezone (defaults to today).
|
||||
@@ -330,6 +339,15 @@ enum FeaturesCommand {
|
||||
Backfill(BackfillArgs),
|
||||
/// Drop stale embeddings, old candidate telemetry and old assessments per the retention config.
|
||||
Prune,
|
||||
/// Recompute the related articles under each issue's web articles.
|
||||
Related(RelatedArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
struct RelatedArgs {
|
||||
/// Only this issue (YYYY-MM-DD); every issue when omitted.
|
||||
#[arg(long)]
|
||||
date: Option<jiff::civil::Date>,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Args)]
|
||||
@@ -382,6 +400,17 @@ async fn main() -> Result<()> {
|
||||
let outcome = pipeline::generate(&config, &db, &GenerateOptions::from(&args)).await?;
|
||||
print_outcome(&outcome);
|
||||
}
|
||||
Command::Rebuild(args) => {
|
||||
// No migration: this only re-renders what an existing schema holds.
|
||||
let db = Db::open(&config.database_path).await?;
|
||||
let published = pipeline::rebuild(&config, &db, args.date).await?;
|
||||
for artifact in &published.epubs {
|
||||
println!("{:?}: {}", artifact.edition, artifact.path.display());
|
||||
}
|
||||
if let Some(xtc) = &published.xtc {
|
||||
println!("XTC: {}", xtc.display());
|
||||
}
|
||||
}
|
||||
Command::Serve => {
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
let config_path = Config::resolve_path(cli.config.as_deref());
|
||||
@@ -451,6 +480,14 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
.description()
|
||||
);
|
||||
eprintln!(
|
||||
" {:<20} {}",
|
||||
"rebuild-YYYY-MM-DD",
|
||||
jobs::Job::Rebuild {
|
||||
date: jiff::civil::Date::default()
|
||||
}
|
||||
.description()
|
||||
);
|
||||
std::process::exit(2);
|
||||
};
|
||||
let db = Db::open_and_migrate(&config.database_path).await?;
|
||||
@@ -461,12 +498,13 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
/// The commands that write the database and provider budgets and so hold the
|
||||
/// run lock (§5): `generate`, `profile rebuild`, `features backfill`,
|
||||
/// run lock (§5): `generate`, `rebuild`, `profile rebuild`, `features backfill`,
|
||||
/// `backfill-social`, `interests backfill`, and a `job run` of any of them.
|
||||
/// Everything else is read-only or its own writer.
|
||||
fn lock_holder(command: &Command) -> Option<&'static str> {
|
||||
match command {
|
||||
Command::Generate(_) => Some("generate"),
|
||||
Command::Rebuild(_) => Some("rebuild"),
|
||||
// Writes the same tables the discovery stage of a run writes.
|
||||
Command::Feeds(FeedsCommand::Discover(_)) => Some("feeds discover"),
|
||||
Command::Profile(ProfileCommand::Rebuild) => Some("profile rebuild"),
|
||||
@@ -483,6 +521,7 @@ fn lock_holder(command: &Command) -> Option<&'static str> {
|
||||
| Command::Explain(_)
|
||||
| Command::Stats(_)
|
||||
| Command::Features(FeaturesCommand::Prune)
|
||||
| Command::Features(FeaturesCommand::Related(_))
|
||||
| Command::Db(_)
|
||||
| Command::Config(_)
|
||||
| Command::Users(_)
|
||||
@@ -1033,7 +1072,7 @@ async fn cmd_explain(db: &Db, args: ExplainArgs) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `features backfill` / `features prune`; returns the final summary line
|
||||
/// `features backfill` / `features prune` / `features related`; returns the final summary line
|
||||
/// (progress lines are printed as they happen).
|
||||
async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Result<String> {
|
||||
match command {
|
||||
@@ -1091,13 +1130,25 @@ async fn cmd_features(config: &Config, db: &Db, command: FeaturesCommand) -> Res
|
||||
jiff::Timestamp::now(),
|
||||
)
|
||||
.await?;
|
||||
let embeddings = if ranking.embedding_retention_days <= 0 {
|
||||
"kept every embedding (retention 0)".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"pruned {} embeddings older than {} days",
|
||||
pruned.embeddings, ranking.embedding_retention_days
|
||||
)
|
||||
};
|
||||
Ok(format!(
|
||||
"pruned {} embeddings older than {} days, {} candidate rows and {} assessments older than {} days",
|
||||
pruned.embeddings,
|
||||
ranking.embedding_retention_days,
|
||||
pruned.telemetry,
|
||||
pruned.assessments,
|
||||
ranking.telemetry_retention_days
|
||||
"{embeddings}, {} candidate rows and {} assessments older than {} days",
|
||||
pruned.telemetry, pruned.assessments, ranking.telemetry_retention_days
|
||||
))
|
||||
}
|
||||
FeaturesCommand::Related(args) => {
|
||||
let dates: Vec<_> = args.date.into_iter().collect();
|
||||
let refreshed = daily_epub::related::refresh(db, &config.voyage, &dates).await?;
|
||||
Ok(format!(
|
||||
"stored {} related links across {} issues",
|
||||
refreshed.links, refreshed.issues
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -1161,6 +1212,21 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
|
||||
jobs::Job::Generate { date } => {
|
||||
generate_job(config, db, date.map(|date| date.to_string()), false).await
|
||||
}
|
||||
jobs::Job::Rebuild { date } => {
|
||||
let published = pipeline::rebuild(config, db, *date).await?;
|
||||
let xtc = if published.xtc.is_some() {
|
||||
" and the XTC"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
Ok((
|
||||
format!(
|
||||
"rebuilt the {date} issue: {} EPUBs{xtc}",
|
||||
published.epubs.len()
|
||||
),
|
||||
None,
|
||||
))
|
||||
}
|
||||
jobs::Job::DryRun => generate_job(config, db, None, true).await,
|
||||
jobs::Job::ProfileRebuild => Ok((cmd_profile_rebuild(config, db).await?, None)),
|
||||
jobs::Job::FeaturesBackfill => {
|
||||
@@ -1450,7 +1516,7 @@ mod tests {
|
||||
row.message
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.starts_with("pruned 0 embeddings"),
|
||||
.starts_with("kept every embedding (retention 0)"),
|
||||
"{row:?}"
|
||||
);
|
||||
assert_eq!(row.run_id, None);
|
||||
@@ -1640,6 +1706,15 @@ mod tests {
|
||||
.command,
|
||||
Command::Features(FeaturesCommand::Prune)
|
||||
));
|
||||
match Cli::try_parse_from(["daily-epub", "features", "related", "--date", "2026-09-01"])
|
||||
.unwrap()
|
||||
.command
|
||||
{
|
||||
Command::Features(FeaturesCommand::Related(args)) => {
|
||||
assert_eq!(args.date, Some("2026-09-01".parse().unwrap()));
|
||||
}
|
||||
other => panic!("expected features related, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+140
@@ -268,6 +268,7 @@ pub async fn generate(config: &Config, db: &Db, opts: &GenerateOptions) -> Resul
|
||||
db.finish_run(run_id, &report).await?;
|
||||
if stages.published.is_some() {
|
||||
prune_retention(config, db).await;
|
||||
refresh_related(config, db, date).await;
|
||||
}
|
||||
// The issue row is written before the report is costed, so stamp the finished
|
||||
// report onto it now (the paths are preserved by `COALESCE`, §3.13).
|
||||
@@ -321,6 +322,15 @@ async fn prune_retention(config: &Config, db: &Db) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Store the issue's related-article links (best effort: a failure only
|
||||
/// leaves the web article pages without them until `features related`).
|
||||
async fn refresh_related(config: &Config, db: &Db, date: Date) {
|
||||
match crate::related::refresh(db, &config.voyage, &[date]).await {
|
||||
Ok(refreshed) => tracing::info!(links = refreshed.links, "related articles stored"),
|
||||
Err(error) => tracing::warn!(%error, "related articles failed; continuing"),
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`run_stages`] hands back; [`generate`] pairs it with the costed report.
|
||||
#[derive(Debug)]
|
||||
struct StageOutput {
|
||||
@@ -1095,6 +1105,51 @@ async fn persist_articles(db: &Db, articles: &mut [Article]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rebuild and republish a published issue's EPUBs (and XTC) from its stored
|
||||
/// snapshot, for when rendering changed after the issue went out.
|
||||
///
|
||||
/// The content is exactly the issue's: the `issues.issue_json` snapshot with
|
||||
/// each article's body restored from its row (the snapshot drops bodies).
|
||||
/// Unlike the web reader it does not refresh anything else from the live rows,
|
||||
/// so social lines and metadata stay as published. Nothing is curated or
|
||||
/// fetched except the images, which are downloaded again as in a run.
|
||||
pub async fn rebuild(config: &Config, db: &Db, date: Date) -> Result<Published> {
|
||||
let row = db
|
||||
.issue_by_date(date)
|
||||
.await?
|
||||
.with_context(|| format!("no issue for {date}"))?;
|
||||
let raw = row
|
||||
.issue_json
|
||||
.with_context(|| format!("the {date} issue predates issue snapshots"))?;
|
||||
let mut issue: Issue = serde_json::from_str(&raw).context("decoding issues.issue_json")?;
|
||||
let ids: Vec<ArticleId> = issue.lineup.picks.iter().map(|p| p.article.id).collect();
|
||||
let mut articles = db.get_articles(&ids).await?;
|
||||
for pick in &mut issue.lineup.picks {
|
||||
let article = articles
|
||||
.remove(&pick.article.id)
|
||||
.with_context(|| format!("article {} is gone", pick.article.id))?;
|
||||
pick.article.content_html = article.content_html;
|
||||
}
|
||||
|
||||
let (artifacts, _) = epub::build_all(&issue, config, &config.out_dir)
|
||||
.await
|
||||
.context("building the EPUB editions")?;
|
||||
let x4 = artifacts.iter().find(|a| a.edition == Edition::X4);
|
||||
let xtc = match x4 {
|
||||
Some(artifact) if config.xtc.enabled => {
|
||||
Some(epub::x4::convert(&config.xtc, &artifact.path, &config.out_dir).await?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let published = publish::publish_issue(config, &issue, &artifacts, xtc.as_deref())
|
||||
.await
|
||||
.context("publishing the issue")?;
|
||||
record_issue(db, &issue, &published)
|
||||
.await
|
||||
.context("recording the issue")?;
|
||||
Ok(published)
|
||||
}
|
||||
|
||||
/// Write the `issues` row and replace `issue_articles` for the date (notes §12).
|
||||
async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<()> {
|
||||
let path_for = |edition: Edition| {
|
||||
@@ -1346,6 +1401,91 @@ fn elapsed_ms(since: Timestamp) -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Read one entry of a zip archive as text.
|
||||
fn zip_entry(path: &std::path::Path, name: &str) -> String {
|
||||
use std::io::Read;
|
||||
let mut archive = zip::ZipArchive::new(std::fs::File::open(path).unwrap()).unwrap();
|
||||
let mut text = String::new();
|
||||
archive
|
||||
.by_name(name)
|
||||
.unwrap()
|
||||
.read_to_string(&mut text)
|
||||
.unwrap();
|
||||
text
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebuild_republishes_from_the_snapshot_without_touching_it() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut issue = crate::epub::fixtures::issue();
|
||||
for pick in &mut issue.lineup.picks {
|
||||
// No images: the rebuild must not need the network here.
|
||||
pick.article.content_html =
|
||||
"<p>Before.</p><pre data-lang=\"rust\"><code>fn main() {}</code></pre>".into();
|
||||
pick.article.image_count = 0;
|
||||
let article = &pick.article;
|
||||
db.upsert_entry(&crate::types::Entry {
|
||||
id: article.best_entry_id,
|
||||
feed_id: article.feed_id,
|
||||
feed_title: Some(article.feed_title.clone()),
|
||||
category: article.category.clone(),
|
||||
title: article.title.clone(),
|
||||
url: article.url.clone(),
|
||||
canonical_url: Some(article.canonical_url.clone()),
|
||||
author: article.author.clone(),
|
||||
published_at: article.published_at,
|
||||
comments_url: article.comments_url.clone(),
|
||||
raw_content: article.content_html.clone(),
|
||||
fetched_at: article.first_seen,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
pick.article.id = db.upsert_article(article).await.unwrap();
|
||||
}
|
||||
let mut snapshot = issue.clone();
|
||||
for pick in &mut snapshot.lineup.picks {
|
||||
pick.article.content_html.clear();
|
||||
}
|
||||
let issue_json = serde_json::to_string(&snapshot).unwrap();
|
||||
db.upsert_issue(
|
||||
issue.meta.date,
|
||||
issue.meta.issue_number,
|
||||
issue.meta.generated_at,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&issue.editorial.front_page_html),
|
||||
None,
|
||||
Some(&issue_json),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut config = Config {
|
||||
out_dir: dir.path().join("out"),
|
||||
..Config::default()
|
||||
};
|
||||
config.publish.epub_dir = dir.path().join("epubs");
|
||||
config.publish.xtc_dir = dir.path().join("xtc");
|
||||
config.xtc.enabled = false;
|
||||
let published = rebuild(&config, &db, issue.meta.date).await.unwrap();
|
||||
|
||||
assert_eq!(published.epubs.len(), 2);
|
||||
let row = db.issue_by_date(issue.meta.date).await.unwrap().unwrap();
|
||||
assert_eq!(row.issue_json.as_deref(), Some(issue_json.as_str()));
|
||||
let standard = &published.epubs[0];
|
||||
assert_eq!(
|
||||
row.epub_path.as_deref(),
|
||||
Some(standard.path.to_str().unwrap())
|
||||
);
|
||||
let chapter = zip_entry(&standard.path, "OEBPS/art-1001.xhtml");
|
||||
assert!(chapter.contains("<p>Before.</p>"), "body restored");
|
||||
assert!(chapter.contains("<span class=\"hl-keyword\">fn</span>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_date_matches_the_masthead_format() {
|
||||
let date: Date = "2026-08-15".parse().unwrap();
|
||||
|
||||
+446
@@ -0,0 +1,446 @@
|
||||
//! Related articles under each web issue article.
|
||||
//!
|
||||
//! For every pick, the nearest embedded articles across the whole archive
|
||||
//! (`article_embeddings`, kept forever by default). Finding them is a scan of
|
||||
//! every cached vector, so it runs once when an issue is published and once
|
||||
//! per issue from `features related`, and the result lives in `issue_related`;
|
||||
//! reader pages only read that table.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use anyhow::Result;
|
||||
use jiff::civil::Date;
|
||||
use sqlx::Row as _;
|
||||
|
||||
use crate::config::VoyageConfig;
|
||||
use crate::curate::embedding::{decode_blob, dot, load_article_embeddings};
|
||||
use crate::db::Db;
|
||||
use crate::types::ArticleId;
|
||||
|
||||
/// Links shown under each article.
|
||||
const PER_ARTICLE: usize = 5;
|
||||
/// Below this cosine a neighbour is noise, not a related article (the same
|
||||
/// cut the curation prompts use for "closest rated").
|
||||
const MIN_COSINE: f64 = 0.55;
|
||||
/// Embedding rows decoded per statement, so memory stays flat as the archive grows.
|
||||
const SCAN_PAGE: i64 = 1000;
|
||||
/// Candidates kept per pick between pages: enough headroom for the
|
||||
/// same-story filter at the end to still fill [`PER_ARTICLE`].
|
||||
const KEEP: usize = PER_ARTICLE * 4;
|
||||
/// Share of title words two articles from one site must have in common to be
|
||||
/// one story re-published under a new URL ("…after AI accusations" and
|
||||
/// "…after AI allegations").
|
||||
const SAME_SITE_TITLE_OVERLAP: f64 = 0.8;
|
||||
|
||||
/// What [`refresh`] wrote.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
pub struct Refreshed {
|
||||
pub issues: usize,
|
||||
pub links: usize,
|
||||
}
|
||||
|
||||
/// Recompute `issue_related` for the given issues (every issue when `dates` is
|
||||
/// empty) in one pass over the embedding cache.
|
||||
///
|
||||
/// Excluded: the pick itself, the rest of its issue, articles that only have a
|
||||
/// feed excerpt (their page would be near-empty), and articles that are the
|
||||
/// pick's or a closer link's story again (see [`Story::same_as`]).
|
||||
pub async fn refresh(db: &Db, voyage: &VoyageConfig, dates: &[Date]) -> Result<Refreshed> {
|
||||
let dates = if dates.is_empty() {
|
||||
sqlx::query_scalar::<_, String>("SELECT date FROM issues ORDER BY date")
|
||||
.fetch_all(db.pool())
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter_map(|raw| raw.parse().ok())
|
||||
.collect()
|
||||
} else {
|
||||
dates.to_vec()
|
||||
};
|
||||
|
||||
// Every pick of every requested issue, with its story key for de-duplication.
|
||||
let mut groups: Vec<Group> = Vec::with_capacity(dates.len());
|
||||
for date in &dates {
|
||||
let rows = sqlx::query(
|
||||
"SELECT ia.article_id, a.title, a.canonical_url FROM issue_articles ia
|
||||
JOIN articles a ON a.id = ia.article_id
|
||||
WHERE ia.issue_date = ? ORDER BY ia.section, ia.position",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
groups.push(Group {
|
||||
date: *date,
|
||||
members: rows.iter().map(|row| row.get("article_id")).collect(),
|
||||
picks: rows
|
||||
.iter()
|
||||
.map(|row| Target {
|
||||
article_id: row.get("article_id"),
|
||||
story: Story::from_row(row),
|
||||
vector: Vec::new(),
|
||||
best: Vec::new(),
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
let ids: Vec<ArticleId> = groups
|
||||
.iter()
|
||||
.flat_map(|group| group.picks.iter().map(|pick| pick.article_id))
|
||||
.collect();
|
||||
let mut vectors = load_article_embeddings(db, voyage, &ids).await?;
|
||||
for group in &mut groups {
|
||||
// A pick without a cached vector simply gets no related list.
|
||||
group
|
||||
.picks
|
||||
.retain_mut(|pick| match vectors.remove(&pick.article_id) {
|
||||
Some(vector) => {
|
||||
pick.vector = vector;
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
});
|
||||
}
|
||||
|
||||
let mut after = i64::MIN;
|
||||
loop {
|
||||
let rows = sqlx::query(
|
||||
"SELECT ae.article_id, ae.embedding, a.title, a.canonical_url
|
||||
FROM article_embeddings ae JOIN articles a ON a.id = ae.article_id
|
||||
WHERE ae.model = ? AND ae.dimension = ? AND a.excerpt_only = 0
|
||||
AND ae.article_id > ?
|
||||
ORDER BY ae.article_id LIMIT ?",
|
||||
)
|
||||
.bind(&voyage.model)
|
||||
.bind(voyage.output_dimension as i64)
|
||||
.bind(after)
|
||||
.bind(SCAN_PAGE)
|
||||
.fetch_all(db.pool())
|
||||
.await?;
|
||||
let Some(last) = rows.last() else { break };
|
||||
after = last.get("article_id");
|
||||
for row in &rows {
|
||||
let candidate: ArticleId = row.get("article_id");
|
||||
let vector = match decode_blob(
|
||||
&row.get::<Vec<u8>, _>("embedding"),
|
||||
voyage.output_dimension,
|
||||
) {
|
||||
Ok(vector) => vector,
|
||||
Err(error) => {
|
||||
tracing::warn!(article_id = candidate, %error, "ignoring a malformed embedding");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let story = Story::from_row(row);
|
||||
for group in &mut groups {
|
||||
if group.members.contains(&candidate) {
|
||||
continue;
|
||||
}
|
||||
for pick in &mut group.picks {
|
||||
let cosine = dot(&pick.vector, &vector)?;
|
||||
if cosine.is_finite() && cosine >= MIN_COSINE {
|
||||
pick.best.push((candidate, cosine, story.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for group in &mut groups {
|
||||
for pick in &mut group.picks {
|
||||
// Highest cosine first; ties by id so reruns write the same rows.
|
||||
pick.best.sort_by(|left, right| {
|
||||
right
|
||||
.1
|
||||
.total_cmp(&left.1)
|
||||
.then_with(|| left.0.cmp(&right.0))
|
||||
});
|
||||
pick.best.truncate(KEEP);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut refreshed = Refreshed::default();
|
||||
let mut tx = db.pool().begin().await?;
|
||||
for group in &groups {
|
||||
let date = group.date.to_string();
|
||||
sqlx::query("DELETE FROM issue_related WHERE issue_date = ?")
|
||||
.bind(&date)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
for pick in &group.picks {
|
||||
let mut seen = vec![&pick.story];
|
||||
let mut chosen = Vec::new();
|
||||
for (id, cosine, story) in &pick.best {
|
||||
if chosen.len() == PER_ARTICLE {
|
||||
break;
|
||||
}
|
||||
if !seen.iter().any(|earlier| earlier.same_as(story)) {
|
||||
seen.push(story);
|
||||
chosen.push((id, cosine));
|
||||
}
|
||||
}
|
||||
for (position, (related_id, cosine)) in chosen.into_iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind(&date)
|
||||
.bind(pick.article_id)
|
||||
.bind(position as i64)
|
||||
.bind(related_id)
|
||||
.bind(cosine)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
refreshed.links += 1;
|
||||
}
|
||||
}
|
||||
refreshed.issues += 1;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(refreshed)
|
||||
}
|
||||
|
||||
/// The stored related links for one pick, closest first.
|
||||
pub async fn for_article(db: &Db, date: Date, article_id: ArticleId) -> Result<Vec<ArticleId>> {
|
||||
Ok(sqlx::query_scalar(
|
||||
"SELECT related_id FROM issue_related
|
||||
WHERE issue_date = ? AND article_id = ? ORDER BY position",
|
||||
)
|
||||
.bind(date.to_string())
|
||||
.bind(article_id)
|
||||
.fetch_all(db.pool())
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// The most recent issue each of `ids` was published in, for linking a related
|
||||
/// article to its issue page rather than the standalone one.
|
||||
pub async fn latest_issues(db: &Db, ids: &[ArticleId]) -> Result<HashMap<ArticleId, Date>> {
|
||||
let mut issues = HashMap::new();
|
||||
for id in ids {
|
||||
let date: Option<String> =
|
||||
sqlx::query_scalar("SELECT MAX(issue_date) FROM issue_articles WHERE article_id = ?")
|
||||
.bind(id)
|
||||
.fetch_one(db.pool())
|
||||
.await?;
|
||||
if let Some(date) = date.and_then(|raw| raw.parse().ok()) {
|
||||
issues.insert(*id, date);
|
||||
}
|
||||
}
|
||||
Ok(issues)
|
||||
}
|
||||
|
||||
struct Group {
|
||||
date: Date,
|
||||
/// Every pick of the issue, embedded or not: none of them is "related".
|
||||
members: HashSet<ArticleId>,
|
||||
picks: Vec<Target>,
|
||||
}
|
||||
|
||||
struct Target {
|
||||
article_id: ArticleId,
|
||||
story: Story,
|
||||
vector: Vec<f32>,
|
||||
/// (candidate, cosine, story), closest first after each scan page.
|
||||
best: Vec<(ArticleId, f64, Story)>,
|
||||
}
|
||||
|
||||
/// What decides whether two articles are one story told twice. The ingest
|
||||
/// dedupe only merges within one run, so a story re-published on another day
|
||||
/// under a new URL reaches the archive as a second article.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Story {
|
||||
/// Canonical URL host without `www.`.
|
||||
site: String,
|
||||
/// Lowercased title words.
|
||||
words: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Story {
|
||||
fn from_row(row: &sqlx::sqlite::SqliteRow) -> Self {
|
||||
// Re-canonicalized so legacy rows still wrapped in a Scour redirect
|
||||
// count as the outlet's site, not Scour's.
|
||||
let raw: String = row.get("canonical_url");
|
||||
let site = crate::dedupe::canonical_url(&raw)
|
||||
.and_then(|canonical| url::Url::parse(&canonical).ok())
|
||||
.and_then(|url| {
|
||||
url.host_str()
|
||||
.map(|host| host.trim_start_matches("www.").to_string())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let words = row
|
||||
.get::<Option<String>, _>("title")
|
||||
.unwrap_or_default()
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|word| !word.is_empty())
|
||||
.map(str::to_lowercase)
|
||||
.collect();
|
||||
Self { site, words }
|
||||
}
|
||||
|
||||
/// The same title anywhere (syndication), or mostly the same title on the
|
||||
/// same site (a re-slugged republish).
|
||||
fn same_as(&self, other: &Story) -> bool {
|
||||
if self.words.is_empty() || other.words.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if self.words == other.words {
|
||||
return true;
|
||||
}
|
||||
let shared = self.words.intersection(&other.words).count() as f64;
|
||||
let union = self.words.union(&other.words).count() as f64;
|
||||
!self.site.is_empty()
|
||||
&& self.site == other.site
|
||||
&& shared / union >= SAME_SITE_TITLE_OVERLAP
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::curate::embedding::encode_blob;
|
||||
|
||||
fn voyage() -> VoyageConfig {
|
||||
VoyageConfig {
|
||||
model: "voyage-4-lite".into(),
|
||||
output_dimension: 2,
|
||||
..VoyageConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn harness() -> (tempfile::TempDir, Db) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = Db::open_and_migrate(&dir.path().join("related.db"))
|
||||
.await
|
||||
.unwrap();
|
||||
(dir, db)
|
||||
}
|
||||
|
||||
async fn article(db: &Db, id: ArticleId, title: &str, vector: &[f32], excerpt_only: bool) {
|
||||
sqlx::query(
|
||||
"INSERT INTO articles (id, canonical_url, title, excerpt_only, first_seen)
|
||||
VALUES (?, ?, ?, ?, '2026-08-15T00:00:00Z')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(format!("https://example.com/{id}"))
|
||||
.bind(title)
|
||||
.bind(excerpt_only)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let norm = vector.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||
let unit: Vec<f32> = vector.iter().map(|v| v / norm).collect();
|
||||
sqlx::query(
|
||||
"INSERT INTO article_embeddings
|
||||
(article_id, model, dimension, input_hash, embedding, created_at)
|
||||
VALUES (?, 'voyage-4-lite', 2, 'h', ?, '2026-08-15T00:00:00Z')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(encode_blob(&unit).unwrap())
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn issue(db: &Db, date: &str, picks: &[ArticleId]) {
|
||||
sqlx::query("INSERT INTO issues (date, issue_number, generated_at) VALUES (?, 1, ?)")
|
||||
.bind(date)
|
||||
.bind("2026-09-01T00:00:00Z")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
for (position, id) in picks.iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_articles (issue_date, article_id, section, position)
|
||||
VALUES (?, ?, 'Top Stories', ?)",
|
||||
)
|
||||
.bind(date)
|
||||
.bind(id)
|
||||
.bind(position as i64)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn day(raw: &str) -> Date {
|
||||
raw.parse().unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn links_skip_the_issue_excerpts_duplicates_and_far_articles() {
|
||||
let (_dir, db) = harness().await;
|
||||
article(&db, 1, "Rust borrow checker", &[1.0, 0.0], false).await;
|
||||
article(&db, 2, "Same issue neighbour", &[1.0, 0.01], false).await;
|
||||
article(&db, 3, "Lifetimes explained", &[1.0, 0.1], false).await;
|
||||
article(&db, 4, "Paywalled", &[1.0, 0.05], true).await;
|
||||
article(&db, 5, "Rust Borrow-Checker!", &[1.0, 0.02], false).await;
|
||||
article(&db, 6, "Ownership deep dive", &[1.0, 0.3], false).await;
|
||||
article(&db, 7, "Ownership: deep dive", &[1.0, 0.31], false).await;
|
||||
article(&db, 8, "Gardening", &[0.0, 1.0], false).await;
|
||||
issue(&db, "2026-09-01", &[1, 2]).await;
|
||||
|
||||
let refreshed = refresh(&db, &voyage(), &[day("2026-09-01")]).await.unwrap();
|
||||
let one = for_article(&db, day("2026-09-01"), 1).await.unwrap();
|
||||
assert_eq!(one, vec![3, 6], "closest first, one copy of each story");
|
||||
// 5 only repeats article 1's title, so it is fair game for article 2.
|
||||
let two = for_article(&db, day("2026-09-01"), 2).await.unwrap();
|
||||
assert_eq!(two, vec![5, 3, 6]);
|
||||
assert_eq!(
|
||||
refreshed,
|
||||
Refreshed {
|
||||
issues: 1,
|
||||
links: 5
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshing_every_issue_replaces_rows_and_links_earlier_picks() {
|
||||
let (_dir, db) = harness().await;
|
||||
article(&db, 1, "One", &[1.0, 0.0], false).await;
|
||||
article(&db, 2, "Two", &[1.0, 0.1], false).await;
|
||||
article(&db, 3, "Three", &[1.0, 0.3], false).await;
|
||||
issue(&db, "2026-09-01", &[1]).await;
|
||||
issue(&db, "2026-09-02", &[2]).await;
|
||||
|
||||
assert_eq!(refresh(&db, &voyage(), &[]).await.unwrap().issues, 2);
|
||||
// A rerun rewrites rather than duplicates.
|
||||
refresh(&db, &voyage(), &[]).await.unwrap();
|
||||
let ids = for_article(&db, day("2026-09-02"), 2).await.unwrap();
|
||||
assert_eq!(ids, vec![1, 3]);
|
||||
let issues = latest_issues(&db, &[1, 3]).await.unwrap();
|
||||
assert_eq!(issues.get(&1), Some(&day("2026-09-01")));
|
||||
assert_eq!(issues.get(&3), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stories_match_on_identical_titles_or_close_titles_from_one_site() {
|
||||
let story = |site: &str, title: &str| Story {
|
||||
site: site.into(),
|
||||
words: title
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|word| !word.is_empty())
|
||||
.map(str::to_lowercase)
|
||||
.collect(),
|
||||
};
|
||||
let accusations = story(
|
||||
"the-independent.com",
|
||||
"The bestselling novel pulled from prestigious literary prize after AI accusations",
|
||||
);
|
||||
let allegations = story(
|
||||
"the-independent.com",
|
||||
"The bestselling novel pulled from prestigious literary prize after AI allegations",
|
||||
);
|
||||
let conversation = story(
|
||||
"theconversation.com",
|
||||
"A bestselling novel was pulled from a French literary prize after AI accusations – what next?",
|
||||
);
|
||||
assert!(accusations.same_as(&allegations), "a re-slugged republish");
|
||||
assert!(
|
||||
!accusations.same_as(&conversation),
|
||||
"other outlets' coverage stays"
|
||||
);
|
||||
assert!(
|
||||
story("a.com", "Foo: Bar").same_as(&story("b.com", "foo — bar")),
|
||||
"syndicated copies match across sites"
|
||||
);
|
||||
assert!(!story("a.com", "").same_as(&story("a.com", "")));
|
||||
}
|
||||
}
|
||||
@@ -353,7 +353,7 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
||||
("curation.ranking.affinity_full", "Interest-attributable ratings at which affinity reaches full weight. Must be > affinity_floor."),
|
||||
("curation.ranking.semantic_min_words", "Bodies shorter than this are not embedded."),
|
||||
("curation.ranking.exploration_slots", "Shortlist slots reserved for exploration picks."),
|
||||
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted."),
|
||||
("curation.ranking.embedding_retention_days", "features prune: unrated, unpublished vectors older than this are deleted; 0 keeps them forever."),
|
||||
("curation.ranking.telemetry_retention_days", "features prune: candidate_runs and article_assessments older than this are deleted."),
|
||||
("curation.ranking.quotas.triage", "Deep-set slots filled by triage score."),
|
||||
("curation.ranking.quotas.interest", "Deep-set slots filled by interest similarity."),
|
||||
|
||||
+238
-21
@@ -943,23 +943,74 @@ struct ArticleLink {
|
||||
struct ArticleTemplate {
|
||||
page: Page,
|
||||
toc: Toc,
|
||||
head: ArticleHead,
|
||||
discussion_html: Option<String>,
|
||||
rating: Option<RatingWidget>,
|
||||
previous: Option<ArticleLink>,
|
||||
next: Option<ArticleLink>,
|
||||
issue_href: String,
|
||||
related: Vec<RelatedView>,
|
||||
}
|
||||
|
||||
/// An archive article that never ran in an issue, reached from a related list:
|
||||
/// the stored body with no summary, rating or issue navigation.
|
||||
#[derive(Template)]
|
||||
#[template(path = "article_standalone.html")]
|
||||
struct StandaloneArticleTemplate {
|
||||
page: Page,
|
||||
head: ArticleHead,
|
||||
}
|
||||
|
||||
/// What `_article_head.html` renders (header and body) plus the footer links
|
||||
/// both article pages share.
|
||||
struct ArticleHead {
|
||||
title: String,
|
||||
source_url: String,
|
||||
byline: Option<String>,
|
||||
source: Source,
|
||||
meta_line: String,
|
||||
social_line: Option<String>,
|
||||
/// Issue-only: empty and `None` on the standalone page.
|
||||
understanding: chapters::Understanding,
|
||||
summary: Option<String>,
|
||||
excerpt_only: bool,
|
||||
body_html: String,
|
||||
discussion_html: Option<String>,
|
||||
read_online_url: String,
|
||||
dashboard_href: String,
|
||||
rating: Option<RatingWidget>,
|
||||
previous: Option<ArticleLink>,
|
||||
next: Option<ArticleLink>,
|
||||
issue_href: String,
|
||||
}
|
||||
|
||||
impl ArticleHead {
|
||||
fn new(article: &Article, config: &Config, is_admin: bool) -> Self {
|
||||
Self {
|
||||
title: article.title.clone(),
|
||||
source_url: article.canonical_url.clone(),
|
||||
byline: article.author.as_ref().map(|author| format!("By {author}")),
|
||||
source: Source::new(article, config, is_admin),
|
||||
meta_line: format!(
|
||||
"{} words · ~{} min read",
|
||||
thousands(article.word_count),
|
||||
article.reading_minutes()
|
||||
),
|
||||
social_line: chapters::social_line(&article.social),
|
||||
understanding: chapters::Understanding::default(),
|
||||
summary: None,
|
||||
excerpt_only: article.excerpt_only,
|
||||
body_html: prepare_body(&article.content_html),
|
||||
read_online_url: article.url.clone(),
|
||||
dashboard_href: format!("/dashboard/articles/{}", article.id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of the "Related" list under an issue article.
|
||||
#[derive(Debug)]
|
||||
struct RelatedView {
|
||||
title: String,
|
||||
href: String,
|
||||
author: Option<String>,
|
||||
source: Source,
|
||||
reading_minutes: i64,
|
||||
date: String,
|
||||
}
|
||||
|
||||
#[derive(Template)]
|
||||
@@ -1114,31 +1165,21 @@ pub async fn article(
|
||||
let is_admin = viewer.role == crate::web::users::Role::Admin;
|
||||
let config = state.config();
|
||||
let active_nav = if view.is_latest { "latest" } else { "archive" };
|
||||
let related = related_views(&state.db, &config, is_admin, date, article_id).await?;
|
||||
let mut page = Page::new(article.title.clone(), Some(viewer.clone()), active_nav);
|
||||
page.flash = take_flash(&session).await?;
|
||||
Ok(Html(ArticleTemplate {
|
||||
page,
|
||||
toc,
|
||||
title: article.title.clone(),
|
||||
source_url: article.canonical_url.clone(),
|
||||
byline: article.author.as_ref().map(|author| format!("By {author}")),
|
||||
source: Source::new(article, &config, is_admin),
|
||||
meta_line: format!(
|
||||
"{} words · ~{} min read",
|
||||
thousands(article.word_count),
|
||||
article.reading_minutes()
|
||||
),
|
||||
social_line: chapters::social_line(&article.social),
|
||||
understanding: chapters::understanding(pick),
|
||||
summary: summary_for(&view.issue, pick).map(str::to_string),
|
||||
excerpt_only: article.excerpt_only,
|
||||
body_html: prepare_body(&article.content_html),
|
||||
head: ArticleHead {
|
||||
understanding: chapters::understanding(pick),
|
||||
summary: summary_for(&view.issue, pick).map(str::to_string),
|
||||
..ArticleHead::new(article, &config, is_admin)
|
||||
},
|
||||
discussion_html: pick
|
||||
.discussion
|
||||
.as_ref()
|
||||
.map(|discussion| crate::comments::render_xhtml(discussion, &article.title)),
|
||||
read_online_url: article.url.clone(),
|
||||
dashboard_href: format!("/dashboard/articles/{}", article.id),
|
||||
rating: is_admin.then(|| {
|
||||
RatingWidget::for_issue(
|
||||
article.id,
|
||||
@@ -1150,10 +1191,90 @@ pub async fn article(
|
||||
previous,
|
||||
next,
|
||||
issue_href: issue_href(date),
|
||||
related,
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `/articles/{id}`: any stored article on its own page. One that ran in an
|
||||
/// issue redirects to its (latest) issue page, which carries the summary and
|
||||
/// related list.
|
||||
pub async fn standalone(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthSession,
|
||||
Extension(session): Extension<Session>,
|
||||
Path(article_id): Path<ArticleId>,
|
||||
) -> Result<Response, WebError> {
|
||||
let viewer = auth
|
||||
.user()
|
||||
.await
|
||||
.map(Viewer::from)
|
||||
.ok_or_else(|| WebError::Unauthenticated {
|
||||
next: standalone_href(article_id),
|
||||
})?;
|
||||
let Some(article) = state.db.get_article(article_id).await? else {
|
||||
return Err(WebError::NotFound);
|
||||
};
|
||||
if let Some(date) = crate::related::latest_issues(&state.db, &[article_id])
|
||||
.await?
|
||||
.remove(&article_id)
|
||||
{
|
||||
return Ok(Redirect::to(&article_href(date, article_id)).into_response());
|
||||
}
|
||||
let is_admin = viewer.role == crate::web::users::Role::Admin;
|
||||
let config = state.config();
|
||||
let mut page = Page::new(article.title.clone(), Some(viewer), "archive");
|
||||
page.flash = take_flash(&session).await?;
|
||||
Ok(Html(StandaloneArticleTemplate {
|
||||
page,
|
||||
head: ArticleHead::new(&article, &config, is_admin),
|
||||
})
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// The stored related links of one pick, each pointing at the issue page it
|
||||
/// ran on or, failing that, its standalone page.
|
||||
async fn related_views(
|
||||
db: &Db,
|
||||
config: &Config,
|
||||
is_admin: bool,
|
||||
date: Date,
|
||||
article_id: ArticleId,
|
||||
) -> anyhow::Result<Vec<RelatedView>> {
|
||||
let ids = crate::related::for_article(db, date, article_id).await?;
|
||||
if ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let articles = db.get_articles(&ids).await?;
|
||||
let issues = crate::related::latest_issues(db, &ids).await?;
|
||||
Ok(ids
|
||||
.iter()
|
||||
.filter_map(|id| {
|
||||
let article = articles.get(id)?;
|
||||
let href = match issues.get(id) {
|
||||
Some(date) => article_href(*date, article.id),
|
||||
None => standalone_href(article.id),
|
||||
};
|
||||
Some(RelatedView {
|
||||
title: article.title.clone(),
|
||||
href,
|
||||
author: article.author.clone(),
|
||||
source: Source::new(article, config, is_admin),
|
||||
reading_minutes: article.reading_minutes(),
|
||||
date: short_date(article.published_at.unwrap_or(article.first_seen), config),
|
||||
})
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// "Aug 15, 2026" in the configured zone.
|
||||
fn short_date(timestamp: jiff::Timestamp, config: &Config) -> String {
|
||||
match config.tz() {
|
||||
Ok(tz) => timestamp.to_zoned(tz).strftime("%b %-d, %Y").to_string(),
|
||||
Err(_) => timestamp.strftime("%b %-d, %Y").to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn world(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthSession,
|
||||
@@ -1335,6 +1456,10 @@ fn article_href(date: Date, article_id: ArticleId) -> String {
|
||||
format!("/issues/{date}/articles/{article_id}")
|
||||
}
|
||||
|
||||
fn standalone_href(article_id: ArticleId) -> String {
|
||||
format!("/articles/{article_id}")
|
||||
}
|
||||
|
||||
fn issue_href(date: Date) -> String {
|
||||
format!("/issues/{date}")
|
||||
}
|
||||
@@ -2198,6 +2323,98 @@ mod tests {
|
||||
assert!(!reader.contains(&feed_href));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn related_links_reach_issue_pages_and_standalone_archive_articles() {
|
||||
let (_dir, db, source) = seeded_issue(true).await;
|
||||
crate::web::users::add(&db, "reader", "correct horse battery", false)
|
||||
.await
|
||||
.unwrap();
|
||||
let pick = &source.lineup.picks[0].article;
|
||||
let sibling = &source.lineup.picks[1].article;
|
||||
let mut archived = pick.clone();
|
||||
archived.id = 0;
|
||||
archived.canonical_url = "https://example.com/archived-story".into();
|
||||
archived.title = "An archived story nobody picked".into();
|
||||
archived.content_html = "<p>Archived body text.</p>".into();
|
||||
let archived_id = db.upsert_article(&archived).await.unwrap();
|
||||
for (position, related_id) in [archived_id, sibling.id].into_iter().enumerate() {
|
||||
sqlx::query(
|
||||
"INSERT INTO issue_related (issue_date, article_id, position, related_id, cosine)
|
||||
VALUES (?, ?, ?, ?, 0.8)",
|
||||
)
|
||||
.bind(source.meta.date.to_string())
|
||||
.bind(pick.id)
|
||||
.bind(position as i64)
|
||||
.bind(related_id)
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let app = crate::server::router(crate::server::AppState::new(
|
||||
db,
|
||||
crate::config::Config::default(),
|
||||
None,
|
||||
));
|
||||
let cookie = login_cookie(&app, "reader", "correct horse battery").await;
|
||||
let get = |uri: String, cookie: Option<String>| {
|
||||
let app = app.clone();
|
||||
async move {
|
||||
let mut request = Request::builder().uri(uri);
|
||||
if let Some(cookie) = cookie {
|
||||
request = request.header(header::COOKIE, cookie);
|
||||
}
|
||||
app.oneshot(request.body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
|
||||
let page = response_text(
|
||||
get(
|
||||
format!("/issues/{}/articles/{}", source.meta.date, pick.id),
|
||||
Some(cookie.clone()),
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
let related = page
|
||||
.split_once("id=\"related-heading\"")
|
||||
.expect("related list")
|
||||
.1;
|
||||
let archived_at = related
|
||||
.find(&format!("href=\"/articles/{archived_id}\""))
|
||||
.expect("unpicked article links to its standalone page");
|
||||
let sibling_at = related
|
||||
.find(&format!(
|
||||
"href=\"/issues/{}/articles/{}\"",
|
||||
source.meta.date, sibling.id
|
||||
))
|
||||
.expect("a picked article links to its issue page");
|
||||
assert!(archived_at < sibling_at, "stored order is kept");
|
||||
|
||||
let standalone = get(format!("/articles/{archived_id}"), Some(cookie.clone())).await;
|
||||
assert_eq!(standalone.status(), StatusCode::OK);
|
||||
let standalone = response_text(standalone).await;
|
||||
assert!(standalone.contains("An archived story nobody picked"));
|
||||
assert!(standalone.contains("Archived body text."));
|
||||
|
||||
let picked = get(format!("/articles/{}", pick.id), Some(cookie)).await;
|
||||
assert_eq!(picked.status(), StatusCode::SEE_OTHER);
|
||||
assert_eq!(
|
||||
picked.headers()[header::LOCATION],
|
||||
format!("/issues/{}/articles/{}", source.meta.date, pick.id)
|
||||
);
|
||||
|
||||
let anonymous = get(format!("/articles/{archived_id}"), None).await;
|
||||
assert!(anonymous.status().is_redirection());
|
||||
assert!(
|
||||
anonymous.headers()[header::LOCATION]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("/login")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fallback_full_issue_recovers_colophon_world_and_behind() {
|
||||
let (_dir, db, source) = seeded_issue(false).await;
|
||||
|
||||
@@ -681,6 +681,7 @@ pub fn router(config: &crate::config::Config) -> axum::Router<crate::server::App
|
||||
.route_layer(from_fn(session::require_password_change));
|
||||
let full_issues = axum::Router::new()
|
||||
.route("/issues/{date}/articles/{article_id}", get(issue::article))
|
||||
.route("/articles/{article_id}", get(issue::standalone))
|
||||
.route("/issues/{date}/world", get(issue::world))
|
||||
.route("/issues/{date}/behind", get(issue::behind))
|
||||
.route("/issues/{date}/read", get(issue::read))
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
/*! tailwindcss v4.3.3 | MIT License | https://tailwindcss.com */
|
||||
@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-duration:initial}}}@layer theme{:root,:host{--spacing:.25rem;--container-md:28rem;--container-lg:32rem;--container-2xl:42rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-normal:0em;--leading-tight:1.25;--leading-snug:1.375;--leading-normal:1.5;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;--default-mono-font-family:ui-monospace, SFMono-Regular, Menlo, monospace}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{paddiLine truncated
|
||||
@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-duration:initial}}}@layer theme{:root,:host{--spacing:.25rem;--container-md:28rem;--container-lg:32rem;--container-2xl:42rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-normal:0em;--leading-tight:1.25;--leading-snug:1.375;--leading-normal:1.5;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;--default-mono-font-family:ui-monospace, SFMono-Regular, Menlo, monospace}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{paddiLine truncated
|
||||
@@ -0,0 +1,2 @@
|
||||
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ head.source_url }}">{{ head.title }}</a></h1>{% match head.byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% let source = head.source %}{% include "_source.html" %} · {{ head.meta_line }}</p>{% let understanding = head.understanding %}{% include "_understanding.html" %}{% match head.summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match head.social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if head.excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
|
||||
<div class="prose-body mt-10 border-t border-rule pt-8">{{ head.body_html|safe }}</div>
|
||||
@@ -0,0 +1 @@
|
||||
{% if !related.is_empty() %}<section class="mt-12" aria-labelledby="related-heading"><h2 id="related-heading" class="reader-section-heading">Related</h2><ul class="m-0 list-none p-0">{% for item in related %}<li class="border-b border-rule py-5"><h3 class="font-serif text-xl font-semibold leading-[1.15] tracking-[-0.01em]"><a class="text-ink no-underline hover:text-accent" href="{{ item.href }}">{{ item.title }}</a></h3><p class="mt-1.5 font-sans text-sm text-muted">{% match item.author %}{% when Some with (author) %}{{ author }} · {% when None %}{% endmatch %}{% let source = item.source %}{% include "_source.html" %} · {{ item.reading_minutes }} min read · {{ item.date }}</p></li>{% endfor %}</ul></section>{% endif %}
|
||||
@@ -1,8 +1,8 @@
|
||||
{% extends "layout.html" %}{% block ears %}<span class="sm:hidden">{{ toc.short_date }}</span><span class="hidden sm:inline">{{ toc.display_date }} · No. {{ toc.issue_number }}</span>{% endblock %}{% block content %}<div class="mx-auto max-w-7xl lg:grid lg:grid-cols-[15rem_minmax(0,1fr)] lg:gap-x-8 lg:px-6 xl:grid-cols-[18rem_minmax(0,1fr)] xl:gap-x-14">{% include "_toc.html" %}
|
||||
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0" data-toc-scroll>
|
||||
<header><p class="font-sans text-[0.72rem] uppercase tracking-[0.12em] text-muted">Article</p><h1 class="mt-3 text-4xl font-semibold leading-[1.1] tracking-[-0.01em] sm:text-5xl"><a class="text-ink no-underline hover:text-accent" href="{{ source_url }}">{{ title }}</a></h1>{% match byline %}{% when Some with (byline) %}<p class="mt-5 font-sans text-sm font-medium text-ink-2">{{ byline }}</p>{% when None %}{% endmatch %}<p class="mt-1 font-sans text-sm text-muted">{% include "_source.html" %} · {{ meta_line }}</p>{% include "_understanding.html" %}{% match summary %}{% when Some with (summary) %}<p class="mt-7 text-xl italic leading-relaxed text-ink-2">{{ summary }}</p>{% when None %}{% endmatch %}{% match social_line %}{% when Some with (social) %}<p class="mt-4 font-sans text-sm text-muted">{{ social }}</p>{% when None %}{% endmatch %}{% if excerpt_only %}<p class="notice mt-6">Excerpt only — continue reading at the original site.</p>{% endif %}</header>
|
||||
<div class="prose-body mt-10 border-t border-rule pt-8">{{ body_html|safe }}</div>
|
||||
{% include "_article_head.html" %}
|
||||
{% match discussion_html %}{% when Some with (discussion) %}<section class="discussion prose-body mt-12 border-t border-rule pt-6 [&_.comment-meta]:font-sans [&_.comment-meta]:text-sm [&_.comment-meta]:text-muted [&_blockquote.reply]:ml-6"><h2 class="mb-6 border-0 pt-0 text-2xl">Discussion</h2>{{ discussion|safe }}</section>{% when None %}{% endmatch %}
|
||||
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
|
||||
{% include "_related.html" %}
|
||||
<footer class="mt-12 border-t border-rule pt-6">{% match rating %}{% when Some with (widget) %}{% include "_rating_widget.html" %}{% when None %}{% endmatch %}<p class="mt-6 flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p><nav class="mt-10 grid grid-cols-1 gap-3 font-sans sm:grid-cols-2" aria-label="Adjacent articles">{% match previous %}{% when Some with (previous) %}<a class="group min-h-24 border border-rule p-4 text-ink no-underline hover:border-accent" rel="prev" href="{{ previous.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Previous</span><span class="mt-2 block leading-snug group-hover:text-accent">← {{ previous.title }}</span></a>{% when None %}{% endmatch %}{% match next %}{% when Some with (next) %}<a class="group min-h-24 border border-rule p-4 text-right text-ink no-underline hover:border-accent sm:col-start-2" rel="next" href="{{ next.href }}"><span class="block text-[0.68rem] uppercase tracking-[0.12em] text-muted">Next</span><span class="mt-2 block leading-snug group-hover:text-accent">{{ next.title }} →</span></a>{% when None %}{% endmatch %}</nav><p class="mt-8 font-sans text-sm"><a href="{{ issue_href }}">← Back to this issue</a></p></footer>
|
||||
</article>
|
||||
</div>{% endblock %}
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends "layout.html" %}{% block ears %}Article{% endblock %}{% block content %}
|
||||
<article class="reader-page mx-auto mt-10 w-full max-w-[68ch] px-4 sm:px-6 lg:mt-12 lg:px-0">
|
||||
{% include "_article_head.html" %}
|
||||
<footer class="mt-12 border-t border-rule pt-6"><p class="flex flex-wrap gap-2"><a class="btn" href="{{ head.read_online_url }}">Read online ↗</a>{% if page.is_admin() %}<a class="btn" href="{{ head.dashboard_href }}">Dashboard</a>{% endif %}</p></footer>
|
||||
</article>{% endblock %}
|
||||
Reference in new issue
Block a user