Add rebuild: republish an issue's EPUBs from its stored content

`daily-epub rebuild --date D` (and the rebuild-YYYY-MM-DD job) takes the
issues.issue_json snapshot and restores each article's body from its row.
It then builds both editions and the XTC, publishes them under the same
names and records the issue as a run would. Nothing is curated again or
refreshed from live rows, so headlines, summaries, social lines and rating
links stay as published; only images are downloaded again. It takes the
run lock, opens the database without migrating, and the job is marked
dangerous like generate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-01 19:53:06 +00:00
co-authored by Claude Opus 5.5
parent 9e521ddaef
commit 5abf062c24
3 changed files with 214 additions and 7 deletions
+39 -6
View File
@@ -32,6 +32,8 @@ pub const EXITED_BEFORE_START: &str = "unit exited before the job started; see t
pub enum Job {
/// `generate` (today) or `generate-YYYY-MM-DD`.
Generate { date: Option<Date> },
/// `rebuild-YYYY-MM-DD` → `rebuild --date`.
Rebuild { date: Date },
/// `dry-run` → `generate --dry-run`.
DryRun,
/// `profile-rebuild` → `profile rebuild`.
@@ -70,8 +72,9 @@ impl Job {
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
}
/// Parse a catalogue name or the dated `generate-YYYY-MM-DD` form; anything
/// else (including `../x`, uppercase, unknown names) is `None`.
/// Parse a catalogue name or a dated `generate-YYYY-MM-DD` or
/// `rebuild-YYYY-MM-DD` form; anything else (including `../x`, uppercase,
/// unknown names) is `None`.
pub fn parse(name: &str) -> Option<Job> {
if !Self::valid_name(name) {
return None;
@@ -86,10 +89,17 @@ impl Job {
"import-ratings" => Some(Job::ImportRatings),
"interests-categorize" => Some(Job::InterestsCategorize),
_ => {
let date = name.strip_prefix("generate-")?;
let (rebuild, date) = match name.strip_prefix("generate-") {
Some(date) => (false, date),
None => (true, name.strip_prefix("rebuild-")?),
};
// Exactly `YYYY-MM-DD`; the round trip rejects `2026-9-3`.
let parsed: Date = date.parse().ok()?;
(parsed.to_string() == date).then_some(Job::Generate { date: Some(parsed) })
(parsed.to_string() == date).then_some(if rebuild {
Job::Rebuild { date: parsed }
} else {
Job::Generate { date: Some(parsed) }
})
}
}
}
@@ -98,6 +108,7 @@ impl Job {
match self {
Job::Generate { date: None } => "generate".into(),
Job::Generate { date: Some(date) } => format!("generate-{date}"),
Job::Rebuild { date } => format!("rebuild-{date}"),
Job::DryRun => "dry-run".into(),
Job::ProfileRebuild => "profile-rebuild".into(),
Job::FeaturesBackfill => "features-backfill".into(),
@@ -121,6 +132,9 @@ impl Job {
Job::Generate { date: Some(_) } => {
"Build and publish the issue of a given date, republishing it if it exists."
}
Job::Rebuild { .. } => {
"Re-render and republish a published issue's EPUBs from its stored content, without curating again."
}
Job::DryRun => "Run the whole pipeline but publish nothing and record no issue.",
Job::ProfileRebuild => {
"Regenerate the learned taste adjustments from ratings with the editor model."
@@ -144,6 +158,7 @@ impl Job {
pub fn takes_lock(&self) -> Option<&'static str> {
match self {
Job::Generate { .. } | Job::DryRun => Some("generate"),
Job::Rebuild { .. } => Some("rebuild"),
Job::ProfileRebuild => Some("profile rebuild"),
Job::FeaturesBackfill => Some("features backfill"),
Job::BackfillSocial => Some("backfill-social"),
@@ -151,9 +166,9 @@ impl Job {
}
}
/// Needs a confirmation dialog: `generate` republishes an issue.
/// Needs a confirmation dialog: `generate` and `rebuild` republish an issue.
pub fn dangerous(&self) -> bool {
matches!(self, Job::Generate { .. })
matches!(self, Job::Generate { .. } | Job::Rebuild { .. })
}
}
@@ -481,6 +496,24 @@ mod tests {
assert!(!Job::parse("dry-run").unwrap().dangerous());
}
#[test]
fn parse_accepts_the_dated_rebuild_form() {
let job = Job::parse("rebuild-2026-10-01").unwrap();
assert_eq!(
job,
Job::Rebuild {
date: "2026-10-01".parse().unwrap()
}
);
assert_eq!(job.name(), "rebuild-2026-10-01");
assert_eq!(job.unit(), "daily-epub-job@rebuild-2026-10-01.service");
assert_eq!(job.takes_lock(), Some("rebuild"));
assert!(job.dangerous());
for bad in ["rebuild", "rebuild-", "rebuild-2026-10-1", "rebuild-today"] {
assert_eq!(Job::parse(bad), None, "{bad}");
}
}
#[test]
fn parse_rejects_traversal_uppercase_and_unknown_names() {
for name in [
+45 -1
View File
@@ -36,6 +36,8 @@ struct Cli {
enum Command {
/// Build (and publish) one issue.
Generate(GenerateArgs),
/// Rebuild and republish a published issue's EPUBs from its stored content.
Rebuild(RebuildArgs),
/// Run the rating endpoints, the OPDS catalog and downloads.
Serve,
/// Taste-profile maintenance.
@@ -159,6 +161,13 @@ enum ConfigCommand {
Check,
}
#[derive(Debug, clap::Args)]
struct RebuildArgs {
/// Date of the published issue to rebuild.
#[arg(long, value_name = "YYYY-MM-DD")]
date: jiff::civil::Date,
}
#[derive(Debug, clap::Args)]
struct GenerateArgs {
/// Issue date in the configured timezone (defaults to today).
@@ -382,6 +391,17 @@ async fn main() -> Result<()> {
let outcome = pipeline::generate(&config, &db, &GenerateOptions::from(&args)).await?;
print_outcome(&outcome);
}
Command::Rebuild(args) => {
// No migration: this only re-renders what an existing schema holds.
let db = Db::open(&config.database_path).await?;
let published = pipeline::rebuild(&config, &db, args.date).await?;
for artifact in &published.epubs {
println!("{:?}: {}", artifact.edition, artifact.path.display());
}
if let Some(xtc) = &published.xtc {
println!("XTC: {}", xtc.display());
}
}
Command::Serve => {
let db = Db::open_and_migrate(&config.database_path).await?;
let config_path = Config::resolve_path(cli.config.as_deref());
@@ -451,6 +471,14 @@ async fn main() -> Result<()> {
}
.description()
);
eprintln!(
" {:<20} {}",
"rebuild-YYYY-MM-DD",
jobs::Job::Rebuild {
date: jiff::civil::Date::default()
}
.description()
);
std::process::exit(2);
};
let db = Db::open_and_migrate(&config.database_path).await?;
@@ -461,12 +489,13 @@ async fn main() -> Result<()> {
}
/// The commands that write the database and provider budgets and so hold the
/// run lock (§5): `generate`, `profile rebuild`, `features backfill`,
/// run lock (§5): `generate`, `rebuild`, `profile rebuild`, `features backfill`,
/// `backfill-social`, `interests backfill`, and a `job run` of any of them.
/// Everything else is read-only or its own writer.
fn lock_holder(command: &Command) -> Option<&'static str> {
match command {
Command::Generate(_) => Some("generate"),
Command::Rebuild(_) => Some("rebuild"),
// Writes the same tables the discovery stage of a run writes.
Command::Feeds(FeedsCommand::Discover(_)) => Some("feeds discover"),
Command::Profile(ProfileCommand::Rebuild) => Some("profile rebuild"),
@@ -1161,6 +1190,21 @@ async fn run_job(config: &Config, db: &Db, job: &jobs::Job) -> Result<(String, O
jobs::Job::Generate { date } => {
generate_job(config, db, date.map(|date| date.to_string()), false).await
}
jobs::Job::Rebuild { date } => {
let published = pipeline::rebuild(config, db, *date).await?;
let xtc = if published.xtc.is_some() {
" and the XTC"
} else {
""
};
Ok((
format!(
"rebuilt the {date} issue: {} EPUBs{xtc}",
published.epubs.len()
),
None,
))
}
jobs::Job::DryRun => generate_job(config, db, None, true).await,
jobs::Job::ProfileRebuild => Ok((cmd_profile_rebuild(config, db).await?, None)),
jobs::Job::FeaturesBackfill => {
+130
View File
@@ -1095,6 +1095,51 @@ async fn persist_articles(db: &Db, articles: &mut [Article]) -> Result<()> {
Ok(())
}
/// Rebuild and republish a published issue's EPUBs (and XTC) from its stored
/// snapshot, for when rendering changed after the issue went out.
///
/// The content is exactly the issue's: the `issues.issue_json` snapshot with
/// each article's body restored from its row (the snapshot drops bodies).
/// Unlike the web reader it does not refresh anything else from the live rows,
/// so social lines and metadata stay as published. Nothing is curated or
/// fetched except the images, which are downloaded again as in a run.
pub async fn rebuild(config: &Config, db: &Db, date: Date) -> Result<Published> {
let row = db
.issue_by_date(date)
.await?
.with_context(|| format!("no issue for {date}"))?;
let raw = row
.issue_json
.with_context(|| format!("the {date} issue predates issue snapshots"))?;
let mut issue: Issue = serde_json::from_str(&raw).context("decoding issues.issue_json")?;
let ids: Vec<ArticleId> = issue.lineup.picks.iter().map(|p| p.article.id).collect();
let mut articles = db.get_articles(&ids).await?;
for pick in &mut issue.lineup.picks {
let article = articles
.remove(&pick.article.id)
.with_context(|| format!("article {} is gone", pick.article.id))?;
pick.article.content_html = article.content_html;
}
let (artifacts, _) = epub::build_all(&issue, config, &config.out_dir)
.await
.context("building the EPUB editions")?;
let x4 = artifacts.iter().find(|a| a.edition == Edition::X4);
let xtc = match x4 {
Some(artifact) if config.xtc.enabled => {
Some(epub::x4::convert(&config.xtc, &artifact.path, &config.out_dir).await?)
}
_ => None,
};
let published = publish::publish_issue(config, &issue, &artifacts, xtc.as_deref())
.await
.context("publishing the issue")?;
record_issue(db, &issue, &published)
.await
.context("recording the issue")?;
Ok(published)
}
/// Write the `issues` row and replace `issue_articles` for the date (notes §12).
async fn record_issue(db: &Db, issue: &Issue, published: &Published) -> Result<()> {
let path_for = |edition: Edition| {
@@ -1346,6 +1391,91 @@ fn elapsed_ms(since: Timestamp) -> i64 {
mod tests {
use super::*;
/// Read one entry of a zip archive as text.
fn zip_entry(path: &std::path::Path, name: &str) -> String {
use std::io::Read;
let mut archive = zip::ZipArchive::new(std::fs::File::open(path).unwrap()).unwrap();
let mut text = String::new();
archive
.by_name(name)
.unwrap()
.read_to_string(&mut text)
.unwrap();
text
}
#[tokio::test]
async fn rebuild_republishes_from_the_snapshot_without_touching_it() {
let dir = tempfile::tempdir().unwrap();
let db = Db::open_and_migrate(&dir.path().join("db.sqlite"))
.await
.unwrap();
let mut issue = crate::epub::fixtures::issue();
for pick in &mut issue.lineup.picks {
// No images: the rebuild must not need the network here.
pick.article.content_html =
"<p>Before.</p><pre data-lang=\"rust\"><code>fn main() {}</code></pre>".into();
pick.article.image_count = 0;
let article = &pick.article;
db.upsert_entry(&crate::types::Entry {
id: article.best_entry_id,
feed_id: article.feed_id,
feed_title: Some(article.feed_title.clone()),
category: article.category.clone(),
title: article.title.clone(),
url: article.url.clone(),
canonical_url: Some(article.canonical_url.clone()),
author: article.author.clone(),
published_at: article.published_at,
comments_url: article.comments_url.clone(),
raw_content: article.content_html.clone(),
fetched_at: article.first_seen,
})
.await
.unwrap();
pick.article.id = db.upsert_article(article).await.unwrap();
}
let mut snapshot = issue.clone();
for pick in &mut snapshot.lineup.picks {
pick.article.content_html.clear();
}
let issue_json = serde_json::to_string(&snapshot).unwrap();
db.upsert_issue(
issue.meta.date,
issue.meta.issue_number,
issue.meta.generated_at,
None,
None,
None,
Some(&issue.editorial.front_page_html),
None,
Some(&issue_json),
)
.await
.unwrap();
let mut config = Config {
out_dir: dir.path().join("out"),
..Config::default()
};
config.publish.epub_dir = dir.path().join("epubs");
config.publish.xtc_dir = dir.path().join("xtc");
config.xtc.enabled = false;
let published = rebuild(&config, &db, issue.meta.date).await.unwrap();
assert_eq!(published.epubs.len(), 2);
let row = db.issue_by_date(issue.meta.date).await.unwrap().unwrap();
assert_eq!(row.issue_json.as_deref(), Some(issue_json.as_str()));
let standard = &published.epubs[0];
assert_eq!(
row.epub_path.as_deref(),
Some(standard.path.to_str().unwrap())
);
let chapter = zip_entry(&standard.path, "OEBPS/art-1001.xhtml");
assert!(chapter.contains("<p>Before.</p>"), "body restored");
assert!(chapter.contains("<span class=\"hl-keyword\">fn</span>"));
}
#[test]
fn display_date_matches_the_masthead_format() {
let date: Date = "2026-08-15".parse().unwrap();