Revert "Add a [cdn] section and purge the edge after publishing"
This reverts commit 41fe61a691.
This commit is contained in:
@@ -220,19 +220,3 @@ journal_lines = 300 # job-page journal tail; valid range 10..=5
|
|||||||
# routes remain public. A signed-in web user can download without Basic auth.
|
# routes remain public. A signed-in web user can download without Basic auth.
|
||||||
# basic_auth_user = "daily"
|
# basic_auth_user = "daily"
|
||||||
# basic_auth_pass = "..."
|
# basic_auth_pass = "..."
|
||||||
|
|
||||||
# [cdn]
|
|
||||||
# The CDN in front of the origin. Leave the whole table out (the default) and
|
|
||||||
# nothing here applies: no purge is attempted and no token is needed.
|
|
||||||
#
|
|
||||||
# provider = "cloudflare" # the only recognised value today
|
|
||||||
# cloudflare_zone_id = "..." # zone id from the Cloudflare dashboard overview
|
|
||||||
# purge_after_publish = true # purge the edge after `generate` publishes (dry runs never do)
|
|
||||||
#
|
|
||||||
# api_token via DAILY_EPUB_CDN__API_TOKEN env; never put it in this file.
|
|
||||||
# The token needs exactly one permission — Zone -> Cache Purge — scoped to this
|
|
||||||
# one zone. Nothing else is called, so nothing else should be granted.
|
|
||||||
#
|
|
||||||
# Setting `provider` without both the zone id and the token is a config error.
|
|
||||||
# See docs/runbooks/cdn-rollout.md for the zone settings and cache rules that
|
|
||||||
# make the origin's Cache-Control headers actually apply at the edge.
|
|
||||||
|
|||||||
-368
@@ -1,368 +0,0 @@
|
|||||||
//! CDN cache purge (spec §3.12).
|
|
||||||
//!
|
|
||||||
//! The origin sends `s-maxage=86400` on the public pages and the feed, so the
|
|
||||||
//! edge holds a copy for a day. That is only safe because publishing a new
|
|
||||||
//! issue purges the edge immediately afterwards.
|
|
||||||
//!
|
|
||||||
//! The purge is deliberately a *purge everything*, not a list of URLs: a new
|
|
||||||
//! issue changes more than its own page. `/` becomes the new issue, `/issues`
|
|
||||||
//! grows a row, `/feed.xml` and `/issues.json` change, and the *previous*
|
|
||||||
//! issue's page changes too — its "latest" nav marker moves. Enumerating that
|
|
||||||
//! set correctly is exactly the kind of thing that silently rots. Everything
|
|
||||||
//! expensive at the edge (`/static/*`) is content-hashed and carries a
|
|
||||||
//! versioned URL, so re-fetching it after a purge costs one origin hit.
|
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use crate::config::{CDN_CLOUDFLARE, Config};
|
|
||||||
use crate::http::{RetryPolicy, build_client};
|
|
||||||
|
|
||||||
/// Cloudflare's API root; overridden in tests to point at a loopback server.
|
|
||||||
pub const CLOUDFLARE_API_BASE: &str = "https://api.cloudflare.com/client/v4";
|
|
||||||
|
|
||||||
/// A purge is a single small request; do not let it hold up a finished run.
|
|
||||||
const PURGE_TIMEOUT: Duration = Duration::from_secs(10);
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
|
||||||
pub enum CdnError {
|
|
||||||
#[error("cdn.provider {0:?} is not recognised")]
|
|
||||||
UnknownProvider(String),
|
|
||||||
#[error("cdn.cloudflare_zone_id is not set")]
|
|
||||||
MissingZoneId,
|
|
||||||
#[error("no CDN API token; set {0}")]
|
|
||||||
MissingToken(String),
|
|
||||||
#[error("building the HTTP client failed: {0}")]
|
|
||||||
Client(#[source] reqwest::Error),
|
|
||||||
#[error("the cache purge request failed: {0}")]
|
|
||||||
Request(#[source] reqwest::Error),
|
|
||||||
#[error("the cache purge was rejected (HTTP {status}): {message}")]
|
|
||||||
Api { status: u16, message: String },
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What [`purge_all`] did.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum PurgeOutcome {
|
|
||||||
/// No `cdn.provider` is configured; nothing was called.
|
|
||||||
Disabled,
|
|
||||||
/// The provider accepted the purge.
|
|
||||||
Purged {
|
|
||||||
provider: &'static str,
|
|
||||||
/// The provider's own id for the purge, when it returns one.
|
|
||||||
id: Option<String>,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for PurgeOutcome {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
match self {
|
|
||||||
PurgeOutcome::Disabled => write!(f, "cdn purge disabled (no cdn.provider configured)"),
|
|
||||||
PurgeOutcome::Purged { provider, id } => match id {
|
|
||||||
Some(id) => write!(f, "purged the whole {provider} cache (id {id})"),
|
|
||||||
None => write!(f, "purged the whole {provider} cache"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Purge the CDN's entire cache for the configured zone.
|
|
||||||
///
|
|
||||||
/// Returns [`PurgeOutcome::Disabled`] without touching the network when no
|
|
||||||
/// provider is configured, so callers need no `if enabled` of their own.
|
|
||||||
pub async fn purge_all(cfg: &Config) -> Result<PurgeOutcome, CdnError> {
|
|
||||||
purge_all_at(cfg, CLOUDFLARE_API_BASE).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// [`purge_all`] against an explicit API root (tests point this at loopback).
|
|
||||||
pub async fn purge_all_at(cfg: &Config, api_base: &str) -> Result<PurgeOutcome, CdnError> {
|
|
||||||
let Some(provider) = cfg.cdn.provider_name() else {
|
|
||||||
return Ok(PurgeOutcome::Disabled);
|
|
||||||
};
|
|
||||||
if provider != CDN_CLOUDFLARE {
|
|
||||||
return Err(CdnError::UnknownProvider(provider));
|
|
||||||
}
|
|
||||||
let zone = cfg.cdn.zone_id().ok_or(CdnError::MissingZoneId)?;
|
|
||||||
let token = cfg
|
|
||||||
.cdn
|
|
||||||
.api_token()
|
|
||||||
.ok_or_else(|| CdnError::MissingToken(crate::config::CdnConfig::api_token_env_var()))?;
|
|
||||||
|
|
||||||
let client = build_client(PURGE_TIMEOUT).map_err(CdnError::Client)?;
|
|
||||||
let url = format!(
|
|
||||||
"{}/zones/{zone}/purge_cache",
|
|
||||||
api_base.trim_end_matches('/')
|
|
||||||
);
|
|
||||||
let policy = RetryPolicy {
|
|
||||||
max_attempts: 3,
|
|
||||||
base_delay: Duration::from_millis(500),
|
|
||||||
max_delay: Duration::from_secs(5),
|
|
||||||
};
|
|
||||||
let id = policy
|
|
||||||
.run("cloudflare cache purge", is_retryable, || async {
|
|
||||||
let response = client
|
|
||||||
.post(&url)
|
|
||||||
.bearer_auth(token)
|
|
||||||
.json(&serde_json::json!({ "purge_everything": true }))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(CdnError::Request)?;
|
|
||||||
let status = response.status();
|
|
||||||
let body = response.text().await.unwrap_or_default();
|
|
||||||
interpret(status.as_u16(), &body)
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
Ok(PurgeOutcome::Purged {
|
|
||||||
provider: CDN_CLOUDFLARE,
|
|
||||||
id,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Cloudflare answers `{"success": …, "errors": [{"code", "message"}], "result": …}`.
|
|
||||||
/// A 200 with `success: false` is still a failure, so the status alone is not
|
|
||||||
/// enough to go on.
|
|
||||||
fn interpret(status: u16, body: &str) -> Result<Option<String>, CdnError> {
|
|
||||||
let json: Option<serde_json::Value> = serde_json::from_str(body).ok();
|
|
||||||
let success = json
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|value| value.get("success"))
|
|
||||||
.and_then(serde_json::Value::as_bool);
|
|
||||||
if (200..300).contains(&status) && success != Some(false) {
|
|
||||||
let id = json
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|value| value.pointer("/result/id"))
|
|
||||||
.and_then(serde_json::Value::as_str)
|
|
||||||
.map(str::to_string);
|
|
||||||
return Ok(id);
|
|
||||||
}
|
|
||||||
Err(CdnError::Api {
|
|
||||||
status,
|
|
||||||
message: api_message(json.as_ref(), body),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The API's own `errors[].message` text, falling back to the raw body.
|
|
||||||
fn api_message(json: Option<&serde_json::Value>, body: &str) -> String {
|
|
||||||
let messages: Vec<String> = json
|
|
||||||
.and_then(|value| value.get("errors"))
|
|
||||||
.and_then(serde_json::Value::as_array)
|
|
||||||
.map(|errors| {
|
|
||||||
errors
|
|
||||||
.iter()
|
|
||||||
.filter_map(|error| {
|
|
||||||
let message = error.get("message").and_then(serde_json::Value::as_str)?;
|
|
||||||
Some(
|
|
||||||
match error.get("code").and_then(serde_json::Value::as_i64) {
|
|
||||||
Some(code) => format!("{message} (code {code})"),
|
|
||||||
None => message.to_string(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
if !messages.is_empty() {
|
|
||||||
return messages.join("; ");
|
|
||||||
}
|
|
||||||
let trimmed = body.trim();
|
|
||||||
if trimmed.is_empty() {
|
|
||||||
"no response body".into()
|
|
||||||
} else {
|
|
||||||
trimmed.chars().take(300).collect()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Retry transport failures and 5xx/429; a 4xx (bad token, wrong zone) is the
|
|
||||||
/// operator's problem and retrying only delays the message.
|
|
||||||
fn is_retryable(error: &CdnError) -> bool {
|
|
||||||
match error {
|
|
||||||
CdnError::Request(source) => crate::http::is_retryable(source),
|
|
||||||
CdnError::Api { status, .. } => *status >= 500 || *status == 429,
|
|
||||||
_ => false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
|
|
||||||
use axum::Router;
|
|
||||||
use axum::extract::State;
|
|
||||||
use axum::http::{HeaderMap, StatusCode};
|
|
||||||
use axum::routing::post;
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
|
||||||
struct Fake {
|
|
||||||
scripted: Arc<Mutex<std::collections::VecDeque<(StatusCode, serde_json::Value)>>>,
|
|
||||||
seen: Arc<Mutex<Vec<(HeaderMap, serde_json::Value)>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Fake {
|
|
||||||
fn push(&self, status: StatusCode, body: serde_json::Value) {
|
|
||||||
self.scripted
|
|
||||||
.lock()
|
|
||||||
.expect("script lock")
|
|
||||||
.push_back((status, body));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn requests(&self) -> Vec<(HeaderMap, serde_json::Value)> {
|
|
||||||
self.seen.lock().expect("seen lock").clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle(
|
|
||||||
State(fake): State<Fake>,
|
|
||||||
headers: HeaderMap,
|
|
||||||
axum::Json(body): axum::Json<serde_json::Value>,
|
|
||||||
) -> (StatusCode, axum::Json<serde_json::Value>) {
|
|
||||||
fake.seen.lock().expect("seen lock").push((headers, body));
|
|
||||||
let (status, body) = fake
|
|
||||||
.scripted
|
|
||||||
.lock()
|
|
||||||
.expect("script lock")
|
|
||||||
.pop_front()
|
|
||||||
.unwrap_or((
|
|
||||||
StatusCode::INTERNAL_SERVER_ERROR,
|
|
||||||
json!({"success": false, "errors": [{"message": "unscripted"}]}),
|
|
||||||
));
|
|
||||||
(status, axum::Json(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn serve(fake: Fake) -> String {
|
|
||||||
let app = Router::new()
|
|
||||||
.route("/zones/{zone}/purge_cache", post(handle))
|
|
||||||
.with_state(fake);
|
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
||||||
.await
|
|
||||||
.expect("loopback listener");
|
|
||||||
let addr = listener.local_addr().expect("local addr");
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let _ = axum::serve(listener, app).await;
|
|
||||||
});
|
|
||||||
format!("http://{addr}")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn configured() -> Config {
|
|
||||||
let mut config = Config::default();
|
|
||||||
config.cdn.provider = Some("cloudflare".into());
|
|
||||||
config.cdn.cloudflare_zone_id = Some("zone-abc".into());
|
|
||||||
config.cdn.api_token = Some("token-never-logged".into());
|
|
||||||
config
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn no_provider_means_no_request() {
|
|
||||||
let config = Config::default();
|
|
||||||
assert!(!config.cdn.is_enabled());
|
|
||||||
assert_eq!(
|
|
||||||
purge_all_at(&config, "http://127.0.0.1:1/never-called")
|
|
||||||
.await
|
|
||||||
.unwrap(),
|
|
||||||
PurgeOutcome::Disabled
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_successful_purge_sends_the_documented_request() {
|
|
||||||
let fake = Fake::default();
|
|
||||||
fake.push(
|
|
||||||
StatusCode::OK,
|
|
||||||
json!({"success": true, "errors": [], "messages": [], "result": {"id": "zone-abc"}}),
|
|
||||||
);
|
|
||||||
let base = serve(fake.clone()).await;
|
|
||||||
|
|
||||||
let outcome = purge_all_at(&configured(), &base).await.expect("purge");
|
|
||||||
assert_eq!(
|
|
||||||
outcome,
|
|
||||||
PurgeOutcome::Purged {
|
|
||||||
provider: "cloudflare",
|
|
||||||
id: Some("zone-abc".into()),
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
let requests = fake.requests();
|
|
||||||
assert_eq!(requests.len(), 1);
|
|
||||||
let (headers, body) = &requests[0];
|
|
||||||
assert_eq!(
|
|
||||||
headers.get("authorization").and_then(|v| v.to_str().ok()),
|
|
||||||
Some("Bearer token-never-logged")
|
|
||||||
);
|
|
||||||
assert_eq!(body, &json!({"purge_everything": true}));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_success_false_body_is_an_error_carrying_the_api_message() {
|
|
||||||
let fake = Fake::default();
|
|
||||||
fake.push(
|
|
||||||
StatusCode::OK,
|
|
||||||
json!({
|
|
||||||
"success": false,
|
|
||||||
"errors": [{"code": 10000, "message": "Authentication error"}],
|
|
||||||
"result": null
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
let base = serve(fake).await;
|
|
||||||
let error = purge_all_at(&configured(), &base).await.unwrap_err();
|
|
||||||
let text = error.to_string();
|
|
||||||
assert!(text.contains("Authentication error"), "{text}");
|
|
||||||
assert!(text.contains("10000"), "{text}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_4xx_is_not_retried_and_a_5xx_is() {
|
|
||||||
let fake = Fake::default();
|
|
||||||
fake.push(
|
|
||||||
StatusCode::FORBIDDEN,
|
|
||||||
json!({"success": false, "errors": [{"code": 9109, "message": "Invalid access token"}]}),
|
|
||||||
);
|
|
||||||
let base = serve(fake.clone()).await;
|
|
||||||
let error = purge_all_at(&configured(), &base).await.unwrap_err();
|
|
||||||
assert!(error.to_string().contains("Invalid access token"));
|
|
||||||
assert_eq!(fake.requests().len(), 1, "a 403 must not be retried");
|
|
||||||
|
|
||||||
let fake = Fake::default();
|
|
||||||
fake.push(
|
|
||||||
StatusCode::BAD_GATEWAY,
|
|
||||||
json!({"success": false, "errors": [{"message": "bad gateway"}]}),
|
|
||||||
);
|
|
||||||
fake.push(
|
|
||||||
StatusCode::OK,
|
|
||||||
json!({"success": true, "errors": [], "result": {"id": "zone-abc"}}),
|
|
||||||
);
|
|
||||||
let base = serve(fake.clone()).await;
|
|
||||||
purge_all_at(&configured(), &base)
|
|
||||||
.await
|
|
||||||
.expect("the retry succeeds");
|
|
||||||
assert_eq!(fake.requests().len(), 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn config_validation_demands_a_zone_and_a_token() {
|
|
||||||
let mut config = Config::default();
|
|
||||||
config.validate().expect("no cdn section is valid");
|
|
||||||
|
|
||||||
config.cdn.provider = Some("cloudflare".into());
|
|
||||||
let error = config.validate().unwrap_err().to_string();
|
|
||||||
assert!(error.contains("cloudflare_zone_id"), "{error}");
|
|
||||||
|
|
||||||
config.cdn.cloudflare_zone_id = Some("zone-abc".into());
|
|
||||||
let error = config.validate().unwrap_err().to_string();
|
|
||||||
assert!(error.contains("DAILY_EPUB_CDN__API_TOKEN"), "{error}");
|
|
||||||
|
|
||||||
config.cdn.api_token = Some("token".into());
|
|
||||||
config.validate().expect("fully configured");
|
|
||||||
|
|
||||||
config.cdn.provider = Some("fastly".into());
|
|
||||||
let error = config.validate().unwrap_err().to_string();
|
|
||||||
assert!(error.contains("not recognised"), "{error}");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_token_is_stripped_from_a_redacted_copy() {
|
|
||||||
let config = configured();
|
|
||||||
let redacted = config.cdn.redacted();
|
|
||||||
assert!(redacted.api_token.is_none());
|
|
||||||
assert_eq!(redacted.cloudflare_zone_id.as_deref(), Some("zone-abc"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-141
@@ -82,8 +82,6 @@ pub struct Config {
|
|||||||
pub publish: PublishConfig,
|
pub publish: PublishConfig,
|
||||||
pub xtc: XtcConfig,
|
pub xtc: XtcConfig,
|
||||||
pub server: ServerConfig,
|
pub server: ServerConfig,
|
||||||
/// `[cdn]` — the CDN in front of the origin, and the post-publish purge.
|
|
||||||
pub cdn: CdnConfig,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for Config {
|
impl Default for Config {
|
||||||
@@ -108,7 +106,6 @@ impl Default for Config {
|
|||||||
publish: PublishConfig::default(),
|
publish: PublishConfig::default(),
|
||||||
xtc: XtcConfig::default(),
|
xtc: XtcConfig::default(),
|
||||||
server: ServerConfig::default(),
|
server: ServerConfig::default(),
|
||||||
cdn: CdnConfig::default(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -731,111 +728,6 @@ impl Default for ServerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The only CDN provider the purge knows how to talk to.
|
|
||||||
pub const CDN_CLOUDFLARE: &str = "cloudflare";
|
|
||||||
|
|
||||||
/// `[cdn]` — the CDN in front of the origin (§3.12).
|
|
||||||
///
|
|
||||||
/// Everything here is off by default: with no `provider` the app behaves
|
|
||||||
/// exactly as it did before a CDN existed. The one thing the origin does when
|
|
||||||
/// a provider *is* configured is purge the edge right after a successful
|
|
||||||
/// publish, so the new issue (and the previous issue's "latest" nav marker)
|
|
||||||
/// stop being served stale for the `s-maxage` day.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
|
||||||
#[serde(deny_unknown_fields, default)]
|
|
||||||
pub struct CdnConfig {
|
|
||||||
/// `"cloudflare"`, or unset to disable the purge entirely.
|
|
||||||
pub provider: Option<String>,
|
|
||||||
/// The zone the site lives in, from the Cloudflare dashboard's overview.
|
|
||||||
pub cloudflare_zone_id: Option<String>,
|
|
||||||
/// Supply only via `DAILY_EPUB_CDN__API_TOKEN`; never put it in the TOML.
|
|
||||||
///
|
|
||||||
/// The token needs a single permission — `Zone → Cache Purge` — scoped to
|
|
||||||
/// the one zone.
|
|
||||||
pub api_token: Option<String>,
|
|
||||||
/// Purge the edge after `generate` publishes an issue (dry runs never do).
|
|
||||||
pub purge_after_publish: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for CdnConfig {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
provider: None,
|
|
||||||
cloudflare_zone_id: None,
|
|
||||||
api_token: None,
|
|
||||||
purge_after_publish: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CdnConfig {
|
|
||||||
/// The only place the token may come from.
|
|
||||||
pub fn api_token_env_var() -> String {
|
|
||||||
format!("{ENV_PREFIX}CDN{ENV_SPLIT}API_TOKEN")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The provider name, trimmed and lowercased, when one is configured.
|
|
||||||
pub fn provider_name(&self) -> Option<String> {
|
|
||||||
self.provider
|
|
||||||
.as_deref()
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|name| !name.is_empty())
|
|
||||||
.map(str::to_lowercase)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The zone id, trimmed, when one is configured.
|
|
||||||
pub fn zone_id(&self) -> Option<&str> {
|
|
||||||
self.cloudflare_zone_id
|
|
||||||
.as_deref()
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|id| !id.is_empty())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The token, trimmed, when one is configured.
|
|
||||||
pub fn api_token(&self) -> Option<&str> {
|
|
||||||
self.api_token
|
|
||||||
.as_deref()
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|token| !token.is_empty())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// True when a recognised provider is configured.
|
|
||||||
pub fn is_enabled(&self) -> bool {
|
|
||||||
self.provider_name().is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A copy safe to log or persist: the token is stripped.
|
|
||||||
pub fn redacted(&self) -> Self {
|
|
||||||
Self {
|
|
||||||
api_token: None,
|
|
||||||
..self.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate(&self) -> Result<(), ConfigError> {
|
|
||||||
let Some(provider) = self.provider_name() else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
if provider != CDN_CLOUDFLARE {
|
|
||||||
return Err(ConfigError::Invalid(format!(
|
|
||||||
"cdn.provider {provider:?} is not recognised; the only supported value is \"{CDN_CLOUDFLARE}\""
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
if self.zone_id().is_none() {
|
|
||||||
return Err(ConfigError::Invalid(
|
|
||||||
"cdn.provider = \"cloudflare\" requires cdn.cloudflare_zone_id".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if self.api_token().is_none() {
|
|
||||||
return Err(ConfigError::Invalid(format!(
|
|
||||||
"cdn.provider = \"cloudflare\" requires an API token; set {}",
|
|
||||||
Self::api_token_env_var()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Config keys that moved from `[deepseek]` to `[llm]`; anywhere else they are
|
/// Config keys that moved from `[deepseek]` to `[llm]`; anywhere else they are
|
||||||
/// a stale-configuration error.
|
/// a stale-configuration error.
|
||||||
const LLM_ROLE_KEYS: &[&str] = &[
|
const LLM_ROLE_KEYS: &[&str] = &[
|
||||||
@@ -1048,19 +940,6 @@ impl Config {
|
|||||||
));
|
));
|
||||||
lines.push(file_line("publish.epub_dir", &self.publish.epub_dir));
|
lines.push(file_line("publish.epub_dir", &self.publish.epub_dir));
|
||||||
lines.push(file_line("publish.xtc_dir", &self.publish.xtc_dir));
|
lines.push(file_line("publish.xtc_dir", &self.publish.xtc_dir));
|
||||||
lines.push(match self.cdn.provider_name() {
|
|
||||||
None => "cdn: disabled (no cdn.provider)".to_string(),
|
|
||||||
Some(provider) => format!(
|
|
||||||
"cdn: {provider} · zone {} · {} · purge_after_publish {}",
|
|
||||||
self.cdn.zone_id().unwrap_or("MISSING"),
|
|
||||||
if self.cdn.api_token().is_some() {
|
|
||||||
"token present".to_string()
|
|
||||||
} else {
|
|
||||||
format!("token MISSING (set {})", CdnConfig::api_token_env_var())
|
|
||||||
},
|
|
||||||
self.cdn.purge_after_publish,
|
|
||||||
),
|
|
||||||
});
|
|
||||||
lines
|
lines
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1193,7 +1072,6 @@ impl Config {
|
|||||||
"curation.sections must not be empty".into(),
|
"curation.sections must not be empty".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
self.cdn.validate()?;
|
|
||||||
self.tz()?;
|
self.tz()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1413,10 +1291,6 @@ mod tests {
|
|||||||
command = "node"
|
command = "node"
|
||||||
args = ["/opt/epub-to-xtc-converter/cli/index.js", "convert"]
|
args = ["/opt/epub-to-xtc-converter/cli/index.js", "convert"]
|
||||||
format = "xtc"
|
format = "xtc"
|
||||||
|
|
||||||
[cdn]
|
|
||||||
provider = "cloudflare"
|
|
||||||
cloudflare_zone_id = "zone-abc"
|
|
||||||
"#,
|
"#,
|
||||||
)?;
|
)?;
|
||||||
jail.set_env("DAILY_EPUB_MINIFLUX__API_KEY", "secret-token");
|
jail.set_env("DAILY_EPUB_MINIFLUX__API_KEY", "secret-token");
|
||||||
@@ -1426,7 +1300,6 @@ mod tests {
|
|||||||
jail.set_env("DAILY_EPUB_VOYAGE__ENABLED", "false");
|
jail.set_env("DAILY_EPUB_VOYAGE__ENABLED", "false");
|
||||||
jail.set_env("DAILY_EPUB_PROVIDERS__GEMINI__API_KEY", "gemini-key");
|
jail.set_env("DAILY_EPUB_PROVIDERS__GEMINI__API_KEY", "gemini-key");
|
||||||
jail.set_env("DAILY_EPUB_LLM__EDITOR", "gemini");
|
jail.set_env("DAILY_EPUB_LLM__EDITOR", "gemini");
|
||||||
jail.set_env("DAILY_EPUB_CDN__API_TOKEN", "cdn-token");
|
|
||||||
|
|
||||||
let c = Config::load(None).map_err(|e| figment::Error::from(e.to_string()))?;
|
let c = Config::load(None).map_err(|e| figment::Error::from(e.to_string()))?;
|
||||||
assert_eq!(c.voyage.api_key.as_deref(), Some("voyage-key"));
|
assert_eq!(c.voyage.api_key.as_deref(), Some("voyage-key"));
|
||||||
@@ -1451,20 +1324,6 @@ mod tests {
|
|||||||
assert_eq!(c.miniflux.api_key.as_deref(), Some("secret-token"));
|
assert_eq!(c.miniflux.api_key.as_deref(), Some("secret-token"));
|
||||||
assert_eq!(c.target_article_count, 12);
|
assert_eq!(c.target_article_count, 12);
|
||||||
assert_eq!(c.server.hmac_secret.as_deref(), Some("hunter2"));
|
assert_eq!(c.server.hmac_secret.as_deref(), Some("hunter2"));
|
||||||
// The CDN token arrives only through the environment, and the whole
|
|
||||||
// section validates as a unit (provider ⇒ zone id ⇒ token).
|
|
||||||
assert_eq!(c.cdn.api_token(), Some("cdn-token"));
|
|
||||||
assert_eq!(c.cdn.zone_id(), Some("zone-abc"));
|
|
||||||
assert!(c.cdn.purge_after_publish);
|
|
||||||
assert!(
|
|
||||||
c.check_report(None)
|
|
||||||
.iter()
|
|
||||||
.any(|line| line.starts_with("cdn: cloudflare ")
|
|
||||||
&& line.contains("token present")
|
|
||||||
&& !line.contains("cdn-token")),
|
|
||||||
"{:?}",
|
|
||||||
c.check_report(None)
|
|
||||||
);
|
|
||||||
// untouched default
|
// untouched default
|
||||||
assert_eq!(c.retention_days, 21);
|
assert_eq!(c.retention_days, 21);
|
||||||
assert_eq!(c.timezone, "America/New_York");
|
assert_eq!(c.timezone, "America/New_York");
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
pub mod auth;
|
pub mod auth;
|
||||||
pub mod cdn;
|
|
||||||
pub mod comments;
|
pub mod comments;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod curate;
|
pub mod curate;
|
||||||
|
|||||||
-22
@@ -64,15 +64,6 @@ enum Command {
|
|||||||
/// Operator jobs (what `daily-epub-job@<name>.service` runs).
|
/// Operator jobs (what `daily-epub-job@<name>.service` runs).
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
Job(JobCommand),
|
Job(JobCommand),
|
||||||
/// CDN cache maintenance.
|
|
||||||
#[command(subcommand)]
|
|
||||||
Cdn(CdnCommand),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Subcommand)]
|
|
||||||
enum CdnCommand {
|
|
||||||
/// Purge the whole edge cache for the configured zone.
|
|
||||||
Purge,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Subcommand)]
|
#[derive(Debug, Subcommand)]
|
||||||
@@ -378,17 +369,6 @@ async fn main() -> Result<()> {
|
|||||||
db.migrate().await?;
|
db.migrate().await?;
|
||||||
println!("migrations up to date: {}", config.database_path.display());
|
println!("migrations up to date: {}", config.database_path.display());
|
||||||
}
|
}
|
||||||
Command::Cdn(CdnCommand::Purge) => {
|
|
||||||
// No database, no files, no provider budgets: nothing the run lock
|
|
||||||
// protects, so `generate` and a purge may safely overlap.
|
|
||||||
match daily_epub::cdn::purge_all(&config).await {
|
|
||||||
Ok(outcome) => println!("{outcome}"),
|
|
||||||
Err(error) => {
|
|
||||||
eprintln!("cdn purge failed: {error}");
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Command::Config(ConfigCommand::Check) => {
|
Command::Config(ConfigCommand::Check) => {
|
||||||
// Reaching here means `Config::load` already validated it; a bad
|
// Reaching here means `Config::load` already validated it; a bad
|
||||||
// config exited non-zero above. Nothing is opened, nothing locked.
|
// config exited non-zero above. Nothing is opened, nothing locked.
|
||||||
@@ -446,7 +426,6 @@ fn lock_holder(command: &Command) -> Option<&'static str> {
|
|||||||
| Command::Features(FeaturesCommand::Prune)
|
| Command::Features(FeaturesCommand::Prune)
|
||||||
| Command::Db(_)
|
| Command::Db(_)
|
||||||
| Command::Config(_)
|
| Command::Config(_)
|
||||||
| Command::Cdn(_)
|
|
||||||
| Command::Users(_) => None,
|
| Command::Users(_) => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1122,7 +1101,6 @@ mod tests {
|
|||||||
vec!["db", "migrate"],
|
vec!["db", "migrate"],
|
||||||
vec!["features", "prune"],
|
vec!["features", "prune"],
|
||||||
vec!["config", "check"],
|
vec!["config", "check"],
|
||||||
vec!["cdn", "purge"],
|
|
||||||
vec!["job", "run", "features-prune"],
|
vec!["job", "run", "features-prune"],
|
||||||
vec!["job", "run", "not-a-job"],
|
vec!["job", "run", "not-a-job"],
|
||||||
] {
|
] {
|
||||||
|
|||||||
@@ -786,11 +786,6 @@ async fn run_stages(
|
|||||||
record_issue(db, &issue, &published)
|
record_issue(db, &issue, &published)
|
||||||
.await
|
.await
|
||||||
.context("recording the issue")?;
|
.context("recording the issue")?;
|
||||||
// The edge holds the public pages for `s-maxage=86400`, so it has to be
|
|
||||||
// told the day changed. Best effort: the paper is already published and
|
|
||||||
// recorded, and a stale edge for a few hours is not worth failing a run
|
|
||||||
// that otherwise succeeded (§3.12).
|
|
||||||
purge_cdn(config).await;
|
|
||||||
Some(published)
|
Some(published)
|
||||||
};
|
};
|
||||||
report.timings.record("publish", elapsed_ms(stage));
|
report.timings.record("publish", elapsed_ms(stage));
|
||||||
@@ -803,23 +798,6 @@ async fn run_stages(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Purge the CDN after a successful publish, never failing the run.
|
|
||||||
async fn purge_cdn(config: &Config) {
|
|
||||||
if !config.cdn.purge_after_publish {
|
|
||||||
tracing::debug!("cdn purge skipped: cdn.purge_after_publish is false");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
match crate::cdn::purge_all(config).await {
|
|
||||||
Ok(crate::cdn::PurgeOutcome::Disabled) => {
|
|
||||||
tracing::debug!("cdn purge skipped: no cdn.provider configured");
|
|
||||||
}
|
|
||||||
Ok(outcome) => tracing::info!("{outcome}"),
|
|
||||||
Err(error) => {
|
|
||||||
tracing::warn!(%error, "the CDN cache purge failed; the edge may serve the previous issue until its s-maxage expires")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Near misses listed in the "Behind the paper" chapter (§15.1).
|
/// Near misses listed in the "Behind the paper" chapter (§15.1).
|
||||||
const NEAR_MISSES_IN_PAPER: usize = 10;
|
const NEAR_MISSES_IN_PAPER: usize = 10;
|
||||||
|
|
||||||
|
|||||||
@@ -218,12 +218,9 @@ const OPTIONAL_KEYS: &[(&str, FieldKind)] = &[
|
|||||||
("server.basic_auth_user", FieldKind::Text),
|
("server.basic_auth_user", FieldKind::Text),
|
||||||
("server.basic_auth_pass", FieldKind::Secret),
|
("server.basic_auth_pass", FieldKind::Secret),
|
||||||
("xtc.settings", FieldKind::Path),
|
("xtc.settings", FieldKind::Path),
|
||||||
("cdn.provider", FieldKind::Text),
|
|
||||||
("cdn.cloudflare_zone_id", FieldKind::Text),
|
|
||||||
("cdn.api_token", FieldKind::Secret),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
const SECRET_SUFFIXES: &[&str] = &["api_key", "hmac_secret", "basic_auth_pass", "api_token"];
|
const SECRET_SUFFIXES: &[&str] = &["api_key", "hmac_secret", "basic_auth_pass"];
|
||||||
|
|
||||||
const PATH_KEYS: &[&str] = &[
|
const PATH_KEYS: &[&str] = &[
|
||||||
"database_path",
|
"database_path",
|
||||||
@@ -258,7 +255,6 @@ const GROUP_ORDER: &[&str] = &[
|
|||||||
"publish",
|
"publish",
|
||||||
"xtc",
|
"xtc",
|
||||||
"server",
|
"server",
|
||||||
"cdn",
|
|
||||||
"miniflux",
|
"miniflux",
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -371,10 +367,6 @@ pub const SETTINGS_HELP: &[(&str, &str)] = &[
|
|||||||
("server.login_window_minutes", "Length of the login throttle window."),
|
("server.login_window_minutes", "Length of the login throttle window."),
|
||||||
("server.jobs_enabled", "Allow the dashboard to start the fixed systemd job catalogue."),
|
("server.jobs_enabled", "Allow the dashboard to start the fixed systemd job catalogue."),
|
||||||
("server.journal_lines", "Journal lines shown on a dashboard job page (10-5000)."),
|
("server.journal_lines", "Journal lines shown on a dashboard job page (10-5000)."),
|
||||||
("cdn.provider", "cloudflare, or empty for no CDN integration at all. Setting it requires both cdn.cloudflare_zone_id and the API token."),
|
|
||||||
("cdn.cloudflare_zone_id", "Zone id from the Cloudflare dashboard overview for the site's zone."),
|
|
||||||
("cdn.api_token", "Environment only (DAILY_EPUB_CDN__API_TOKEN). Needs exactly one permission: Zone -> Cache Purge, scoped to that one zone."),
|
|
||||||
("cdn.purge_after_publish", "Purge the whole edge cache after generate publishes an issue. A purge failure is logged and does not fail the run; dry runs never purge."),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
/// `DAILY_EPUB_` + the path upper-cased with `.` → `__` (§13.1 item 2).
|
/// `DAILY_EPUB_` + the path upper-cased with `.` → `__` (§13.1 item 2).
|
||||||
@@ -1691,7 +1683,6 @@ mod tests {
|
|||||||
"publish",
|
"publish",
|
||||||
"xtc",
|
"xtc",
|
||||||
"server",
|
"server",
|
||||||
"cdn",
|
|
||||||
"miniflux",
|
"miniflux",
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user