//! Provider-neutral LLM clients, transports, retry, and token accounting (§4, §5). //! //! Two transports speak to the wire directly through the shared `reqwest` //! client (no vendor SDK; implementation notes, cross-cutting item 5 is stale): //! //! - [`DeepseekBackend`]: the OpenAI-compatible chat-completions endpoint. The //! system prompt is the first message so DeepSeek's prefix cache hits. //! - [`AnthropicBackend`]: `POST /v1/messages` with the system prompt as one //! `cache_control: ephemeral` block, `output_config.effort`, and server-side //! `fallbacks: "default"` (§4.2). No sampling parameters, no `thinking`, no //! prefill — Opus 5 rejects them. A `stop_reason: "refusal"` (HTTP 200) is //! [`LlmError::Refusal`], which the callers use to fall back to the bulk client. //! //! Every call goes through [`LlmClient`], which sends the byte-identical system //! prompt on every request, folds token usage into a per-provider [`UsageMeter`] //! priced by a [`PriceTable`], and refuses further work once that provider's //! `max_daily_usd` is spent. [`Llms`] pairs the bulk and editor clients. //! //! Tests inject [`MockBackend`] or a loopback `axum` listener; nothing here //! touches the network under test. use std::future::Future; use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use serde::Deserialize; use serde_json::json; use crate::config::{AnthropicConfig, DeepseekConfig}; use crate::http::RetryPolicy; use crate::types::TokenUsage; pub const JSON_OBJECT: &str = "json_object"; const DEEPSEEK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180); const ANTHROPIC_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); const ANTHROPIC_VERSION: &str = "2023-06-01"; const ANTHROPIC_BETA: &str = "server-side-fallback-2026-07-01"; #[derive(Debug, thiserror::Error)] pub enum LlmError { #[error("{provider} api key is not configured (set {env_var})")] MissingApiKey { provider: &'static str, env_var: &'static str, }, #[error("{provider} request failed: {message}")] Api { provider: &'static str, message: String, }, #[error("{provider} request failed (transient): {message}")] Transient { provider: &'static str, message: String, }, #[error("{provider} returned a refusal")] Refusal { provider: &'static str }, #[error("{provider} returned an empty completion")] EmptyResponse { provider: &'static str }, #[error("llm returned unparseable JSON: {0}")] Json(#[from] serde_json::Error), #[error("daily cost ceiling of ${limit:.2} reached (spent ${spent:.4})")] BudgetExceeded { spent: f64, limit: f64 }, } impl LlmError { pub fn is_transient(&self) -> bool { matches!(self, LlmError::Transient { .. }) } fn api(provider: &'static str, message: impl Into) -> Self { Self::Api { provider, message: message.into(), } } fn transient(provider: &'static str, message: impl Into) -> Self { Self::Transient { provider, message: message.into(), } } } #[derive(Debug, Clone, Copy, PartialEq)] pub struct PriceTable { pub input_per_mtok: f64, pub cache_write_per_mtok: f64, pub cache_read_per_mtok: f64, pub output_per_mtok: f64, } impl PriceTable { pub fn deepseek(cfg: &DeepseekConfig) -> Self { Self { input_per_mtok: cfg.price_input_per_mtok, cache_write_per_mtok: 0.0, cache_read_per_mtok: cfg.price_cached_input_per_mtok, output_per_mtok: cfg.price_output_per_mtok, } } pub fn anthropic(cfg: &AnthropicConfig) -> Self { Self { input_per_mtok: cfg.price_input_per_mtok, cache_write_per_mtok: cfg.price_cache_write_per_mtok, cache_read_per_mtok: cfg.price_cache_read_per_mtok, output_per_mtok: cfg.price_output_per_mtok, } } } #[derive(Debug, Clone)] pub struct UsageMeter { inner: Arc>, exceeded: Arc, prior_spend_usd: Arc>, limit_usd: f64, prices: PriceTable, } impl UsageMeter { /// A meter priced from the `[deepseek]` table; the other providers build /// theirs with [`UsageMeter::with_prices`]. pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self { Self::with_prices(PriceTable::deepseek(cfg), limit_usd) } pub fn with_prices(prices: PriceTable, limit_usd: f64) -> Self { Self { inner: Arc::new(Mutex::new(TokenUsage::default())), exceeded: Arc::new(AtomicBool::new(false)), prior_spend_usd: Arc::new(Mutex::new(0.0)), limit_usd, prices, } } pub fn preload_cost(&self, spent_usd: f64) { let spent_usd = spent_usd.max(0.0); match self.prior_spend_usd.lock() { Ok(mut guard) => *guard = spent_usd, Err(poisoned) => *poisoned.into_inner() = spent_usd, } if self.limit_usd > 0.0 && spent_usd >= self.limit_usd { self.trip("prior spend for the run's UTC day reached the ceiling"); } } pub fn record(&self, usage: TokenUsage) -> TokenUsage { let total = match self.inner.lock() { Ok(mut guard) => { guard.add(usage); *guard } Err(poisoned) => { let mut guard = poisoned.into_inner(); guard.add(usage); *guard } }; let spent = self.spent_usd(); tracing::debug!( input = usage.input_tokens, cache_write = usage.cache_write_tokens, cache_read = usage.cached_tokens, output = usage.output_tokens, live_cost_usd = self.cost_usd(), day_spend_usd = spent, "recorded llm usage" ); if self.limit_usd > 0.0 && spent > self.limit_usd { self.trip("token spend crossed the ceiling"); } total } fn trip(&self, why: &str) { self.exceeded.store(true, Ordering::SeqCst); tracing::error!( spent_usd = self.spent_usd(), limit_usd = self.limit_usd, "LLM budget exceeded ({why}); remaining calls for this provider are skipped" ); } pub fn total(&self) -> TokenUsage { match self.inner.lock() { Ok(guard) => *guard, Err(poisoned) => *poisoned.into_inner(), } } pub fn cost_of(&self, usage: TokenUsage) -> f64 { usage.cost_usd( self.prices.input_per_mtok, self.prices.cache_write_per_mtok, self.prices.cache_read_per_mtok, self.prices.output_per_mtok, ) } pub fn cost_usd(&self) -> f64 { self.cost_of(self.total()) } pub fn spent_usd(&self) -> f64 { let prior = match self.prior_spend_usd.lock() { Ok(guard) => *guard, Err(poisoned) => *poisoned.into_inner(), }; prior + self.cost_usd() } pub fn limit_usd(&self) -> f64 { self.limit_usd } pub fn budget_exceeded(&self) -> bool { self.exceeded.load(Ordering::SeqCst) } pub fn check_budget(&self) -> Result<(), LlmError> { if self.budget_exceeded() || (self.limit_usd > 0.0 && self.spent_usd() >= self.limit_usd) { self.exceeded.store(true, Ordering::SeqCst); return Err(LlmError::BudgetExceeded { spent: self.spent_usd(), limit: self.limit_usd, }); } Ok(()) } } #[derive(Debug, Clone)] pub struct ChatRequest { pub model: String, pub system: Arc, pub user: String, pub temperature: f32, pub json: bool, pub effort: Option, } #[derive(Debug, Clone, Default)] pub struct ChatCompletion { pub content: String, pub usage: TokenUsage, } type BoxFuture<'a, T> = Pin + Send + 'a>>; pub trait ChatBackend: std::fmt::Debug + Send + Sync { fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result>; } #[derive(Debug, Clone)] pub struct DeepseekBackend { http: reqwest::Client, endpoint: String, api_key: String, } impl DeepseekBackend { pub fn new(cfg: &DeepseekConfig) -> Result { let api_key = cfg .api_key .as_deref() .map(str::trim) .filter(|key| !key.is_empty()) .ok_or(LlmError::MissingApiKey { provider: "deepseek", env_var: "DAILY_EPUB_DEEPSEEK__API_KEY", })? .to_string(); let http = crate::http::build_client(DEEPSEEK_TIMEOUT) .map_err(|error| LlmError::api("deepseek", format!("building http client: {error}")))?; Ok(Self { http, endpoint: format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')), api_key, }) } } impl ChatBackend for DeepseekBackend { fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result> { Box::pin(async move { let mut body = json!({ "model": req.model, "messages": [ {"role": "system", "content": req.system.as_str()}, {"role": "user", "content": req.user}, ], "temperature": req.temperature, "stream": false, }); if req.json && let Some(object) = body.as_object_mut() { object.insert("response_format".into(), json!({"type": JSON_OBJECT})); } let response = self .http .post(&self.endpoint) .bearer_auth(&self.api_key) .json(&body) .send() .await .map_err(|error| classify_reqwest_error("deepseek", error))?; let status = response.status(); if !status.is_success() { return Err(classify_status("deepseek", status, response).await); } let parsed: DeepseekResponse = response.json().await.map_err(|error| { LlmError::api("deepseek", format!("decoding chat completion: {error}")) })?; let content = parsed .choices .into_iter() .next() .and_then(|choice| choice.message.content) .filter(|content| !content.trim().is_empty()) .ok_or(LlmError::EmptyResponse { provider: "deepseek", })?; let usage = parsed.usage.map(deepseek_usage).unwrap_or_default(); Ok(ChatCompletion { content, usage }) }) } } #[derive(Debug, Deserialize)] struct DeepseekResponse { #[serde(default)] choices: Vec, #[serde(default)] usage: Option, } #[derive(Debug, Deserialize)] struct DeepseekChoice { message: DeepseekMessage, } #[derive(Debug, Deserialize)] struct DeepseekMessage { #[serde(default)] content: Option, } #[derive(Debug, Default, Deserialize)] struct DeepseekUsage { #[serde(default)] prompt_tokens: i64, #[serde(default)] completion_tokens: i64, #[serde(default)] prompt_cache_hit_tokens: Option, #[serde(default)] prompt_tokens_details: Option, } #[derive(Debug, Default, Deserialize)] struct PromptTokenDetails { #[serde(default)] cached_tokens: Option, } fn deepseek_usage(usage: DeepseekUsage) -> TokenUsage { let cached = usage .prompt_tokens_details .as_ref() .and_then(|details| details.cached_tokens) .or(usage.prompt_cache_hit_tokens) .unwrap_or(0) .max(0); let prompt = usage.prompt_tokens.max(0); let cached = cached.min(prompt); TokenUsage { input_tokens: prompt - cached, cached_tokens: cached, cache_write_tokens: 0, output_tokens: usage.completion_tokens.max(0), } } #[derive(Debug, Clone)] pub struct AnthropicBackend { http: reqwest::Client, endpoint: String, api_key: String, } impl AnthropicBackend { pub fn new(cfg: &AnthropicConfig) -> Result { let api_key = cfg .api_key .as_deref() .map(str::trim) .filter(|key| !key.is_empty()) .ok_or(LlmError::MissingApiKey { provider: "anthropic", env_var: "DAILY_EPUB_ANTHROPIC__API_KEY", })? .to_string(); let http = crate::http::build_client(ANTHROPIC_TIMEOUT).map_err(|error| { LlmError::api("anthropic", format!("building http client: {error}")) })?; Ok(Self { http, endpoint: format!("{}/v1/messages", cfg.base_url.trim_end_matches('/')), api_key, }) } } impl ChatBackend for AnthropicBackend { fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result> { Box::pin(async move { let body = json!({ "model": req.model, "max_tokens": 16_000, "system": [{ "type": "text", "text": req.system.as_str(), "cache_control": {"type": "ephemeral"}, }], "messages": [{"role": "user", "content": req.user}], "output_config": {"effort": req.effort.as_deref().unwrap_or("high")}, "fallbacks": "default", }); let response = self .http .post(&self.endpoint) .header("x-api-key", &self.api_key) .header("anthropic-version", ANTHROPIC_VERSION) .header("anthropic-beta", ANTHROPIC_BETA) .header(reqwest::header::CONTENT_TYPE, "application/json") .json(&body) .send() .await .map_err(|error| classify_reqwest_error("anthropic", error))?; let status = response.status(); if !status.is_success() { return Err(classify_status("anthropic", status, response).await); } let parsed: AnthropicResponse = response.json().await.map_err(|error| { LlmError::api("anthropic", format!("decoding messages response: {error}")) })?; if parsed.stop_reason.as_deref() == Some("refusal") { return Err(LlmError::Refusal { provider: "anthropic", }); } let content = parsed .content .into_iter() .filter(|block| block.kind == "text") .filter_map(|block| block.text) .collect::>() .join(""); if content.trim().is_empty() { return Err(LlmError::EmptyResponse { provider: "anthropic", }); } Ok(ChatCompletion { content, usage: anthropic_usage(parsed.usage), }) }) } } #[derive(Debug, Deserialize)] struct AnthropicResponse { #[serde(default)] content: Vec, #[serde(default)] stop_reason: Option, #[serde(default)] usage: AnthropicUsage, } #[derive(Debug, Deserialize)] struct AnthropicContent { #[serde(rename = "type")] kind: String, #[serde(default)] text: Option, } #[derive(Debug, Default, Deserialize)] struct AnthropicUsage { #[serde(default)] input_tokens: i64, #[serde(default)] cache_creation_input_tokens: i64, #[serde(default)] cache_read_input_tokens: i64, #[serde(default)] output_tokens: i64, } fn anthropic_usage(usage: AnthropicUsage) -> TokenUsage { TokenUsage { input_tokens: usage.input_tokens.max(0), cached_tokens: usage.cache_read_input_tokens.max(0), cache_write_tokens: usage.cache_creation_input_tokens.max(0), output_tokens: usage.output_tokens.max(0), } } async fn classify_status( provider: &'static str, status: reqwest::StatusCode, response: reqwest::Response, ) -> LlmError { let detail = response.text().await.unwrap_or_default(); let message = format!("{status}: {}", detail.chars().take(500).collect::()); if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { LlmError::transient(provider, message) } else { LlmError::api(provider, message) } } fn classify_reqwest_error(provider: &'static str, error: reqwest::Error) -> LlmError { if crate::http::is_retryable(&error) { LlmError::transient(provider, error.to_string()) } else { LlmError::api(provider, error.to_string()) } } #[derive(Debug, Clone)] pub struct LlmClient { pub provider: &'static str, pub system_prompt: Arc, pub model: String, pub effort: Option, pub meter: UsageMeter, backend: Arc, retry: RetryPolicy, } impl LlmClient { pub fn new( cfg: &DeepseekConfig, system_prompt: String, meter: UsageMeter, ) -> Result { let backend = DeepseekBackend::new(cfg)?; Ok(Self::with_backend_options( "deepseek", &cfg.model, system_prompt, None, meter, Arc::new(backend), )) } pub fn new_anthropic( cfg: &AnthropicConfig, system_prompt: String, meter: UsageMeter, ) -> Result { let backend = AnthropicBackend::new(cfg)?; Ok(Self::with_backend_options( "anthropic", &cfg.model, system_prompt, Some(cfg.effort.clone()), meter, Arc::new(backend), )) } pub fn with_backend( model: &str, system_prompt: String, meter: UsageMeter, backend: Arc, ) -> Self { Self::with_backend_options("mock", model, system_prompt, None, meter, backend) } pub fn with_backend_options( provider: &'static str, model: &str, system_prompt: String, effort: Option, meter: UsageMeter, backend: Arc, ) -> Self { Self { provider, system_prompt: Arc::new(system_prompt), model: model.to_string(), effort, meter, backend, retry: RetryPolicy::default(), } } #[cfg(test)] fn with_retry(mut self, retry: RetryPolicy) -> Self { self.retry = retry; self } pub async fn complete( &self, user_prompt: &str, temperature: f32, json: bool, ) -> Result { self.meter.check_budget()?; let request = ChatRequest { model: self.model.clone(), system: Arc::clone(&self.system_prompt), user: user_prompt.to_string(), temperature, json, effort: self.effort.clone(), }; let completion = self .retry .run( &format!("{} chat completion", self.provider), LlmError::is_transient, || self.backend.complete(request.clone()), ) .await?; self.meter.record(completion.usage); Ok(completion.content) } pub async fn complete_json( &self, user_prompt: &str, temperature: f32, ) -> Result { let raw = self.complete(user_prompt, temperature, true).await?; let cleaned = strip_code_fence(&raw); match serde_json::from_str(cleaned) { Ok(value) => Ok(value), Err(error) => { tracing::warn!( provider = self.provider, %error, preview = %cleaned.chars().take(400).collect::(), "llm returned malformed JSON" ); Err(LlmError::Json(error)) } } } pub async fn complete_text( &self, user_prompt: &str, temperature: f32, ) -> Result { self.complete(user_prompt, temperature, false).await } } /// The two provider clients the pipeline works with (§4.2). /// /// Both share the exact same system prompt string (§8.4). Each has its own /// [`UsageMeter`] with its own price table and `max_daily_usd` (§5). #[derive(Debug, Clone, Default)] pub struct Llms { /// DeepSeek — scoring, and the fallback for every editor call. pub bulk: Option, /// Claude — selection, summaries, the brief, the profile rebuild. pub editor: Option, } impl Llms { /// Build both clients from config with one shared system prompt. /// /// A missing key or `anthropic.enabled = false` leaves that slot `None` with /// a log line; nothing here is fatal because the paper always publishes (§17). pub fn from_config( deepseek: &DeepseekConfig, anthropic: &AnthropicConfig, system_prompt: String, bulk_meter: UsageMeter, editor_meter: UsageMeter, ) -> Self { let bulk = match LlmClient::new(deepseek, system_prompt.clone(), bulk_meter) { Ok(client) => Some(client), Err(error) => { tracing::warn!(%error, "DeepSeek (bulk) is unavailable"); None } }; let editor = if anthropic.enabled { match LlmClient::new_anthropic(anthropic, system_prompt, editor_meter) { Ok(client) => Some(client), Err(error) => { tracing::warn!(%error, "Anthropic (editor) is unavailable; editor work falls back to bulk"); None } } } else { tracing::info!("anthropic.enabled = false; editor work runs on the bulk provider"); None }; Self { bulk, editor } } /// The editor when configured and its meter is not tripped, else bulk. pub fn editor_or_bulk(&self) -> Option<&LlmClient> { self.editor .as_ref() .filter(|client| !client.meter.budget_exceeded()) .or(self.bulk.as_ref()) } /// True when no provider at all is available (`--skip-llm` or no keys). pub fn is_empty(&self) -> bool { self.bulk.is_none() && self.editor.is_none() } } pub fn strip_code_fence(raw: &str) -> &str { let trimmed = raw.trim(); let Some(rest) = trimmed.strip_prefix("```") else { return trimmed; }; let rest = rest.strip_prefix("json").unwrap_or(rest); rest.trim_start_matches(['\n', '\r']) .trim_end() .trim_end_matches("```") .trim() } #[derive(Debug, Default)] pub struct MockBackend { scripted: Mutex>>, pub seen: Mutex>, } impl MockBackend { pub fn new() -> Self { Self::default() } pub fn push(&self, content: impl Into, usage: TokenUsage) { if let Ok(mut queue) = self.scripted.lock() { queue.push_back(Ok(ChatCompletion { content: content.into(), usage, })); } } pub fn push_error(&self, message: impl Into) { self.push_llm_error(LlmError::api("mock", message)); } pub fn push_llm_error(&self, error: LlmError) { if let Ok(mut queue) = self.scripted.lock() { queue.push_back(Err(error)); } } pub fn calls(&self) -> usize { self.seen.lock().map(|seen| seen.len()).unwrap_or(0) } pub fn prompts(&self) -> Vec { self.seen .lock() .map(|seen| seen.clone()) .unwrap_or_default() } } impl ChatBackend for MockBackend { fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result> { Box::pin(async move { let next = self .scripted .lock() .ok() .and_then(|mut queue| queue.pop_front()); if let Ok(mut seen) = self.seen.lock() { seen.push(req); } next.unwrap_or_else(|| Err(LlmError::api("mock", "mock backend ran out of responses"))) }) } } #[cfg(test)] mod tests { use super::*; use std::collections::VecDeque; use std::time::Duration; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::routing::post; use axum::{Json, Router}; fn cfg() -> DeepseekConfig { DeepseekConfig::default() } pub(crate) fn tokens(input: i64, cached: i64, output: i64) -> TokenUsage { TokenUsage { input_tokens: input, cached_tokens: cached, cache_write_tokens: 0, output_tokens: output, } } // ----------------------------------------------------------------------- // Meter and pricing // ----------------------------------------------------------------------- #[test] fn meter_accumulates_and_prices() { let meter = UsageMeter::new(&cfg(), 2.0); meter.record(tokens(1_000_000, 0, 0)); meter.record(tokens(0, 1_000_000, 1_000_000)); let total = meter.total(); assert_eq!(total.input_tokens, 1_000_000); assert_eq!(total.cached_tokens, 1_000_000); assert_eq!(total.output_tokens, 1_000_000); // The DeepSeek path prices exactly as before the cache-write counter. assert!((meter.cost_usd() - 0.4228).abs() < 1e-9); assert!(!meter.budget_exceeded()); assert!(meter.check_budget().is_ok()); } #[test] fn anthropic_price_table_charges_cache_reads_and_writes() { let meter = UsageMeter::with_prices(PriceTable::anthropic(&AnthropicConfig::default()), 100.0); meter.record(TokenUsage { input_tokens: 1_000_000, cached_tokens: 1_000_000, cache_write_tokens: 1_000_000, output_tokens: 1_000_000, }); // 5 + 0.5 + 6.25 + 25 assert!((meter.cost_usd() - 36.75).abs() < 1e-9); } #[test] fn meter_trips_the_budget_flag_and_stays_tripped() { // Ceiling of $0.10; 1M cache-miss input tokens costs $0.14. let meter = UsageMeter::new(&cfg(), 0.10); meter.record(tokens(1_000_000, 0, 0)); assert!(meter.budget_exceeded()); assert!(matches!( meter.check_budget(), Err(LlmError::BudgetExceeded { .. }) )); // Cloned meters share the flag. assert!(meter.clone().budget_exceeded()); } #[test] fn preloaded_daily_spend_trips_the_flag() { let meter = UsageMeter::new(&cfg(), 1.0); meter.preload_cost(0.5); assert!(!meter.budget_exceeded()); assert!((meter.spent_usd() - 0.5).abs() < 1e-9); meter.preload_cost(1.5); assert!(meter.budget_exceeded()); } #[test] fn deepseek_usage_split_uses_prompt_token_details() { let u: DeepseekUsage = serde_json::from_str( r#"{"prompt_tokens": 1000, "completion_tokens": 120, "total_tokens": 1120, "prompt_tokens_details": {"cached_tokens": 800}}"#, ) .expect("fixture usage"); assert_eq!(deepseek_usage(u), tokens(200, 800, 120)); } #[test] fn deepseek_usage_falls_back_to_native_cache_fields() { let u: DeepseekUsage = serde_json::from_str( r#"{"prompt_tokens": 500, "completion_tokens": 40, "prompt_cache_hit_tokens": 448, "prompt_cache_miss_tokens": 52}"#, ) .expect("fixture usage"); assert_eq!(deepseek_usage(u), tokens(52, 448, 40)); // Missing usage is not an error, just zero. let empty: DeepseekUsage = serde_json::from_str("{}").expect("empty usage"); assert_eq!(deepseek_usage(empty), TokenUsage::default()); } #[test] fn anthropic_usage_maps_cache_fields() { let u: AnthropicUsage = serde_json::from_str( r#"{"input_tokens": 120, "cache_creation_input_tokens": 3000, "cache_read_input_tokens": 0, "output_tokens": 800}"#, ) .expect("fixture usage"); assert_eq!( anthropic_usage(u), TokenUsage { input_tokens: 120, cached_tokens: 0, cache_write_tokens: 3000, output_tokens: 800, } ); } #[test] fn code_fences_are_stripped() { assert_eq!(strip_code_fence("{\"a\":1}"), "{\"a\":1}"); assert_eq!(strip_code_fence("```json\n{\"a\":1}\n```"), "{\"a\":1}"); assert_eq!(strip_code_fence("```\n{\"a\":1}\n```"), "{\"a\":1}"); } // ----------------------------------------------------------------------- // LlmClient over the mock backend // ----------------------------------------------------------------------- fn client(backend: Arc, limit: f64) -> LlmClient { LlmClient::with_backend( "deepseek-v4-flash", "SYSTEM PROMPT".into(), UsageMeter::new(&cfg(), limit), backend, ) } #[tokio::test] async fn json_completion_records_usage_and_sends_system_prompt_first() { let backend = Arc::new(MockBackend::new()); backend.push(r#"{"value": 42}"#, tokens(10, 90, 5)); let llm = client(Arc::clone(&backend), 2.0); #[derive(serde::Deserialize)] struct Out { value: i64, } let out: Out = llm .complete_json("score these", 0.3) .await .expect("mock completion"); assert_eq!(out.value, 42); assert_eq!(llm.meter.total(), tokens(10, 90, 5)); let prompts = backend.prompts(); assert_eq!(prompts.len(), 1); assert_eq!(prompts[0].system.as_str(), "SYSTEM PROMPT"); assert!(prompts[0].json); assert_eq!(prompts[0].user, "score these"); } #[tokio::test] async fn identical_system_prompt_bytes_across_calls() { let backend = Arc::new(MockBackend::new()); backend.push("{}", TokenUsage::default()); backend.push("{}", TokenUsage::default()); let llm = client(Arc::clone(&backend), 2.0); let _: serde_json::Value = llm.complete_json("a", 0.3).await.expect("first"); let _: serde_json::Value = llm.complete_json("b", 0.3).await.expect("second"); let prompts = backend.prompts(); assert_eq!(prompts[0].system.as_bytes(), prompts[1].system.as_bytes()); } #[tokio::test] async fn calls_are_refused_once_the_budget_is_gone() { let backend = Arc::new(MockBackend::new()); backend.push("{}", tokens(1_000_000, 0, 0)); let llm = client(Arc::clone(&backend), 0.01); let _: serde_json::Value = llm.complete_json("first", 0.3).await.expect("first call"); let err = llm .complete_text("second", 0.3) .await .expect_err("budget must be enforced"); assert!(matches!(err, LlmError::BudgetExceeded { .. })); // The refused call never reached the backend. assert_eq!(backend.calls(), 1); } #[tokio::test] async fn malformed_json_surfaces_as_json_error() { let backend = Arc::new(MockBackend::new()); backend.push("not json at all", TokenUsage::default()); let llm = client(backend, 2.0); let out: Result = llm.complete_json("x", 0.3).await; assert!(matches!(out, Err(LlmError::Json(_)))); } #[tokio::test] async fn refusals_are_not_retried_and_keep_their_variant() { let backend = Arc::new(MockBackend::new()); backend.push_llm_error(LlmError::Refusal { provider: "anthropic", }); backend.push("{}", TokenUsage::default()); let llm = client(Arc::clone(&backend), 2.0).with_retry(RetryPolicy { max_attempts: 3, base_delay: Duration::from_millis(1), max_delay: Duration::from_millis(2), }); let err = llm.complete_text("x", 0.3).await.expect_err("refusal"); assert!(matches!( err, LlmError::Refusal { provider: "anthropic" } )); assert_eq!(backend.calls(), 1, "a refusal is terminal for that client"); } #[test] fn missing_api_keys_name_their_provider() { let deepseek = DeepseekConfig { api_key: Some(" ".into()), ..DeepseekConfig::default() }; let err = DeepseekBackend::new(&deepseek).expect_err("blank key"); assert!(matches!( err, LlmError::MissingApiKey { provider: "deepseek", .. } )); assert!(err.to_string().contains("DAILY_EPUB_DEEPSEEK__API_KEY")); let anthropic = AnthropicConfig::default(); let err = AnthropicBackend::new(&anthropic).expect_err("no key"); assert!(matches!( err, LlmError::MissingApiKey { provider: "anthropic", .. } )); assert!(err.to_string().contains("DAILY_EPUB_ANTHROPIC__API_KEY")); } // ----------------------------------------------------------------------- // Llms // ----------------------------------------------------------------------- fn mock_client(provider: &'static str, limit: f64) -> (LlmClient, Arc) { let backend = Arc::new(MockBackend::new()); let prices = if provider == "anthropic" { PriceTable::anthropic(&AnthropicConfig::default()) } else { PriceTable::deepseek(&cfg()) }; let client = LlmClient::with_backend_options( provider, "model", "SYSTEM".into(), None, UsageMeter::with_prices(prices, limit), Arc::clone(&backend) as Arc, ); (client, backend) } #[test] fn editor_or_bulk_prefers_an_untripped_editor() { let (bulk, _) = mock_client("deepseek", 2.0); let (editor, _) = mock_client("anthropic", 3.0); let llms = Llms { bulk: Some(bulk), editor: Some(editor), }; assert_eq!(llms.editor_or_bulk().map(|c| c.provider), Some("anthropic")); // Trip the editor's meter: bulk takes over. llms.editor .as_ref() .expect("editor") .meter .preload_cost(10.0); assert_eq!(llms.editor_or_bulk().map(|c| c.provider), Some("deepseek")); // No bulk and a tripped editor means no client at all. let only_editor = Llms { bulk: None, editor: llms.editor.clone(), }; assert!(only_editor.editor_or_bulk().is_none()); assert!(Llms::default().is_empty()); assert!(Llms::default().editor_or_bulk().is_none()); } #[test] fn from_config_without_keys_yields_no_clients() { let llms = Llms::from_config( &cfg(), &AnthropicConfig::default(), "SYSTEM".into(), UsageMeter::new(&cfg(), 1.0), UsageMeter::with_prices(PriceTable::anthropic(&AnthropicConfig::default()), 1.0), ); assert!(llms.is_empty()); } // ----------------------------------------------------------------------- // AnthropicBackend against a loopback listener (§4.2, §20) // ----------------------------------------------------------------------- #[derive(Clone, Default)] struct FakeAnthropic { seen: Arc>>, scripted: Arc>>, } impl FakeAnthropic { fn push(&self, status: StatusCode, body: serde_json::Value) { self.scripted .lock() .expect("script lock") .push_back((status, body)); } fn requests(&self) -> Vec<(HeaderMap, serde_json::Value)> { self.seen.lock().expect("seen lock").clone() } } async fn handle( State(fake): State, headers: HeaderMap, Json(body): Json, ) -> (StatusCode, Json) { fake.seen.lock().expect("seen lock").push((headers, body)); let (status, body) = fake .scripted .lock() .expect("script lock") .pop_front() .unwrap_or(( StatusCode::INTERNAL_SERVER_ERROR, json!({"error": "unscripted"}), )); (status, Json(body)) } async fn serve(fake: FakeAnthropic) -> String { let app = Router::new() .route("/v1/messages", post(handle)) .with_state(fake); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("loopback listener"); let addr = listener.local_addr().expect("local addr"); tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); format!("http://{addr}") } async fn anthropic_client(fake: FakeAnthropic, limit: f64) -> LlmClient { let base_url = serve(fake).await; let config = AnthropicConfig { base_url, api_key: Some("test-key-never-logged".into()), effort: "medium".into(), ..AnthropicConfig::default() }; LlmClient::new_anthropic( &config, "PROFILE SYSTEM PROMPT".into(), UsageMeter::with_prices(PriceTable::anthropic(&config), limit), ) .expect("client") .with_retry(RetryPolicy { max_attempts: 3, base_delay: Duration::from_millis(1), max_delay: Duration::from_millis(2), }) } fn ok_message(text: &str, stop_reason: &str) -> serde_json::Value { json!({ "id": "msg_01", "type": "message", "role": "assistant", "model": "claude-opus-5", "content": [ {"type": "thinking", "thinking": ""}, {"type": "text", "text": text} ], "stop_reason": stop_reason, "usage": { "input_tokens": 1_000_000, "cache_creation_input_tokens": 1_000_000, "cache_read_input_tokens": 1_000_000, "output_tokens": 1_000_000 } }) } #[tokio::test] async fn anthropic_request_has_the_documented_shape() { let fake = FakeAnthropic::default(); fake.push( StatusCode::OK, ok_message("```json\n{\"ok\": true}\n```", "end_turn"), ); let llm = anthropic_client(fake.clone(), 100.0).await; let out: serde_json::Value = llm .complete_json("the task", 0.7) .await .expect("completion"); assert_eq!(out, json!({"ok": true})); let requests = fake.requests(); assert_eq!(requests.len(), 1); let (headers, body) = &requests[0]; assert_eq!( headers.get("x-api-key").and_then(|v| v.to_str().ok()), Some("test-key-never-logged") ); assert_eq!( headers .get("anthropic-version") .and_then(|v| v.to_str().ok()), Some("2023-06-01") ); assert_eq!( headers.get("anthropic-beta").and_then(|v| v.to_str().ok()), Some("server-side-fallback-2026-07-01") ); assert!( headers .get("content-type") .and_then(|v| v.to_str().ok()) .is_some_and(|v| v.starts_with("application/json")) ); assert_eq!(body["model"], "claude-opus-5"); assert_eq!(body["max_tokens"], 16_000); assert_eq!(body["system"][0]["type"], "text"); assert_eq!(body["system"][0]["text"], "PROFILE SYSTEM PROMPT"); assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral"); assert_eq!(body["messages"][0]["role"], "user"); assert_eq!(body["messages"][0]["content"], "the task"); assert_eq!(body["output_config"]["effort"], "medium"); assert_eq!(body["fallbacks"], "default"); for forbidden in [ "temperature", "top_p", "top_k", "thinking", "response_format", ] { assert!( body.get(forbidden).is_none(), "{forbidden} must not be sent" ); } assert_eq!( body["messages"].as_array().map(Vec::len), Some(1), "no prefill" ); // Usage was priced with the cache read/write rates: 5 + 6.25 + 0.5 + 25. assert_eq!( llm.meter.total(), TokenUsage { input_tokens: 1_000_000, cached_tokens: 1_000_000, cache_write_tokens: 1_000_000, output_tokens: 1_000_000, } ); assert!((llm.meter.cost_usd() - 36.75).abs() < 1e-9); } #[tokio::test] async fn anthropic_refusal_surfaces_as_the_fallback_error() { let fake = FakeAnthropic::default(); fake.push( StatusCode::OK, json!({ "content": [], "stop_reason": "refusal", "stop_details": {"type": "refusal", "category": "cyber"}, "usage": {"input_tokens": 0, "output_tokens": 0} }), ); let llm = anthropic_client(fake.clone(), 100.0).await; let err = llm.complete_text("x", 0.3).await.expect_err("refusal"); assert!(matches!( err, LlmError::Refusal { provider: "anthropic" } )); assert!(!err.is_transient()); assert_eq!(fake.requests().len(), 1, "a refusal is never retried"); } #[tokio::test] async fn anthropic_429_is_retried_but_400_is_not() { let fake = FakeAnthropic::default(); fake.push( StatusCode::TOO_MANY_REQUESTS, json!({"type": "error", "error": {"type": "rate_limit_error"}}), ); fake.push( StatusCode::OK, ok_message("{\"after\": \"retry\"}", "end_turn"), ); let llm = anthropic_client(fake.clone(), 100.0).await; let text = llm.complete_text("x", 0.3).await.expect("second attempt"); assert_eq!(text, "{\"after\": \"retry\"}"); assert_eq!(fake.requests().len(), 2); let fake = FakeAnthropic::default(); fake.push( StatusCode::BAD_REQUEST, json!({"type": "error", "error": {"type": "invalid_request_error", "message": "nope"}}), ); let llm = anthropic_client(fake.clone(), 100.0).await; let err = llm.complete_text("x", 0.3).await.expect_err("400"); assert!(matches!( err, LlmError::Api { provider: "anthropic", .. } )); assert!(err.to_string().contains("400")); assert_eq!(fake.requests().len(), 1, "400 is never retried"); } #[tokio::test] async fn anthropic_concatenates_text_blocks_and_rejects_empty_output() { let fake = FakeAnthropic::default(); fake.push( StatusCode::OK, json!({ "content": [ {"type": "text", "text": "{\"a\": "}, {"type": "thinking", "thinking": "..."}, {"type": "text", "text": "1}"} ], "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 5} }), ); fake.push( StatusCode::OK, json!({ "content": [{"type": "thinking", "thinking": ""}], "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 0} }), ); let llm = anthropic_client(fake.clone(), 100.0).await; let out: serde_json::Value = llm.complete_json("x", 0.3).await.expect("joined"); assert_eq!(out, json!({"a": 1})); let err = llm.complete_text("y", 0.3).await.expect_err("empty"); assert!(matches!( err, LlmError::EmptyResponse { provider: "anthropic" } )); } }