thalladaandClaude Fable 5.1
d403c51edf
Dead code and stale v1 comments removed (clippy -W dead_code clean, the three world.rs warnings fixed), the Brief chapter's TOC title renamed from "From the Editor", features prune now also sweeps article_assessments and generate runs the sweep once after publishing, the example config is tested key-for-key against Config::default(), README commands match --help, and docs/plans/2026-08-15-implementation-notes.md records the Anthropic and Voyage facts, the new tables, the budget-day rule and the lock. Implemented by a Claude agent from docs/plans/curation-v2-briefs/step7.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1rCLQeKBgnBo3oTgHuTMe
1384 lines
44 KiB
Rust
1384 lines
44 KiB
Rust
//! Provider-neutral LLM clients, transports, retry, and token accounting (§4, §5).
|
|
//!
|
|
//! Two transports speak to the wire directly through the shared `reqwest`
|
|
//! client (no vendor SDK; implementation notes, cross-cutting item 5 is stale):
|
|
//!
|
|
//! - [`DeepseekBackend`]: the OpenAI-compatible chat-completions endpoint. The
|
|
//! system prompt is the first message so DeepSeek's prefix cache hits.
|
|
//! - [`AnthropicBackend`]: `POST /v1/messages` with the system prompt as one
|
|
//! `cache_control: ephemeral` block, `output_config.effort`, and server-side
|
|
//! `fallbacks: "default"` (§4.2). No sampling parameters, no `thinking`, no
|
|
//! prefill — Opus 5 rejects them. A `stop_reason: "refusal"` (HTTP 200) is
|
|
//! [`LlmError::Refusal`], which the callers use to fall back to the bulk client.
|
|
//!
|
|
//! Every call goes through [`LlmClient`], which sends the byte-identical system
|
|
//! prompt on every request, folds token usage into a per-provider [`UsageMeter`]
|
|
//! priced by a [`PriceTable`], and refuses further work once that provider's
|
|
//! `max_daily_usd` is spent. [`Llms`] pairs the bulk and editor clients.
|
|
//!
|
|
//! Tests inject [`MockBackend`] or a loopback `axum` listener; nothing here
|
|
//! touches the network under test.
|
|
|
|
use std::future::Future;
|
|
use std::pin::Pin;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use serde::Deserialize;
|
|
use serde_json::json;
|
|
|
|
use crate::config::{AnthropicConfig, DeepseekConfig};
|
|
use crate::http::RetryPolicy;
|
|
use crate::types::TokenUsage;
|
|
|
|
pub const JSON_OBJECT: &str = "json_object";
|
|
const DEEPSEEK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(180);
|
|
const ANTHROPIC_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
|
|
const ANTHROPIC_VERSION: &str = "2023-06-01";
|
|
const ANTHROPIC_BETA: &str = "server-side-fallback-2026-07-01";
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum LlmError {
|
|
#[error("{provider} api key is not configured (set {env_var})")]
|
|
MissingApiKey {
|
|
provider: &'static str,
|
|
env_var: &'static str,
|
|
},
|
|
#[error("{provider} request failed: {message}")]
|
|
Api {
|
|
provider: &'static str,
|
|
message: String,
|
|
},
|
|
#[error("{provider} request failed (transient): {message}")]
|
|
Transient {
|
|
provider: &'static str,
|
|
message: String,
|
|
},
|
|
#[error("{provider} returned a refusal")]
|
|
Refusal { provider: &'static str },
|
|
#[error("{provider} returned an empty completion")]
|
|
EmptyResponse { provider: &'static str },
|
|
#[error("llm returned unparseable JSON: {0}")]
|
|
Json(#[from] serde_json::Error),
|
|
#[error("daily cost ceiling of ${limit:.2} reached (spent ${spent:.4})")]
|
|
BudgetExceeded { spent: f64, limit: f64 },
|
|
}
|
|
|
|
impl LlmError {
|
|
pub fn is_transient(&self) -> bool {
|
|
matches!(self, LlmError::Transient { .. })
|
|
}
|
|
|
|
fn api(provider: &'static str, message: impl Into<String>) -> Self {
|
|
Self::Api {
|
|
provider,
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
fn transient(provider: &'static str, message: impl Into<String>) -> Self {
|
|
Self::Transient {
|
|
provider,
|
|
message: message.into(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
|
pub struct PriceTable {
|
|
pub input_per_mtok: f64,
|
|
pub cache_write_per_mtok: f64,
|
|
pub cache_read_per_mtok: f64,
|
|
pub output_per_mtok: f64,
|
|
}
|
|
|
|
impl PriceTable {
|
|
pub fn deepseek(cfg: &DeepseekConfig) -> Self {
|
|
Self {
|
|
input_per_mtok: cfg.price_input_per_mtok,
|
|
cache_write_per_mtok: 0.0,
|
|
cache_read_per_mtok: cfg.price_cached_input_per_mtok,
|
|
output_per_mtok: cfg.price_output_per_mtok,
|
|
}
|
|
}
|
|
|
|
pub fn anthropic(cfg: &AnthropicConfig) -> Self {
|
|
Self {
|
|
input_per_mtok: cfg.price_input_per_mtok,
|
|
cache_write_per_mtok: cfg.price_cache_write_per_mtok,
|
|
cache_read_per_mtok: cfg.price_cache_read_per_mtok,
|
|
output_per_mtok: cfg.price_output_per_mtok,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct UsageMeter {
|
|
inner: Arc<Mutex<TokenUsage>>,
|
|
exceeded: Arc<AtomicBool>,
|
|
prior_spend_usd: Arc<Mutex<f64>>,
|
|
limit_usd: f64,
|
|
prices: PriceTable,
|
|
}
|
|
|
|
impl UsageMeter {
|
|
/// A meter priced from the `[deepseek]` table; the other providers build
|
|
/// theirs with [`UsageMeter::with_prices`].
|
|
pub fn new(cfg: &DeepseekConfig, limit_usd: f64) -> Self {
|
|
Self::with_prices(PriceTable::deepseek(cfg), limit_usd)
|
|
}
|
|
|
|
pub fn with_prices(prices: PriceTable, limit_usd: f64) -> Self {
|
|
Self {
|
|
inner: Arc::new(Mutex::new(TokenUsage::default())),
|
|
exceeded: Arc::new(AtomicBool::new(false)),
|
|
prior_spend_usd: Arc::new(Mutex::new(0.0)),
|
|
limit_usd,
|
|
prices,
|
|
}
|
|
}
|
|
|
|
pub fn preload_cost(&self, spent_usd: f64) {
|
|
let spent_usd = spent_usd.max(0.0);
|
|
match self.prior_spend_usd.lock() {
|
|
Ok(mut guard) => *guard = spent_usd,
|
|
Err(poisoned) => *poisoned.into_inner() = spent_usd,
|
|
}
|
|
if self.limit_usd > 0.0 && spent_usd >= self.limit_usd {
|
|
self.trip("prior spend for the run's UTC day reached the ceiling");
|
|
}
|
|
}
|
|
|
|
pub fn record(&self, usage: TokenUsage) -> TokenUsage {
|
|
let total = match self.inner.lock() {
|
|
Ok(mut guard) => {
|
|
guard.add(usage);
|
|
*guard
|
|
}
|
|
Err(poisoned) => {
|
|
let mut guard = poisoned.into_inner();
|
|
guard.add(usage);
|
|
*guard
|
|
}
|
|
};
|
|
let spent = self.spent_usd();
|
|
tracing::debug!(
|
|
input = usage.input_tokens,
|
|
cache_write = usage.cache_write_tokens,
|
|
cache_read = usage.cached_tokens,
|
|
output = usage.output_tokens,
|
|
live_cost_usd = self.cost_usd(),
|
|
day_spend_usd = spent,
|
|
"recorded llm usage"
|
|
);
|
|
if self.limit_usd > 0.0 && spent > self.limit_usd {
|
|
self.trip("token spend crossed the ceiling");
|
|
}
|
|
total
|
|
}
|
|
|
|
fn trip(&self, why: &str) {
|
|
self.exceeded.store(true, Ordering::SeqCst);
|
|
tracing::error!(
|
|
spent_usd = self.spent_usd(),
|
|
limit_usd = self.limit_usd,
|
|
"LLM budget exceeded ({why}); remaining calls for this provider are skipped"
|
|
);
|
|
}
|
|
|
|
pub fn total(&self) -> TokenUsage {
|
|
match self.inner.lock() {
|
|
Ok(guard) => *guard,
|
|
Err(poisoned) => *poisoned.into_inner(),
|
|
}
|
|
}
|
|
|
|
pub fn cost_of(&self, usage: TokenUsage) -> f64 {
|
|
usage.cost_usd(
|
|
self.prices.input_per_mtok,
|
|
self.prices.cache_write_per_mtok,
|
|
self.prices.cache_read_per_mtok,
|
|
self.prices.output_per_mtok,
|
|
)
|
|
}
|
|
|
|
pub fn cost_usd(&self) -> f64 {
|
|
self.cost_of(self.total())
|
|
}
|
|
|
|
pub fn spent_usd(&self) -> f64 {
|
|
let prior = match self.prior_spend_usd.lock() {
|
|
Ok(guard) => *guard,
|
|
Err(poisoned) => *poisoned.into_inner(),
|
|
};
|
|
prior + self.cost_usd()
|
|
}
|
|
|
|
pub fn limit_usd(&self) -> f64 {
|
|
self.limit_usd
|
|
}
|
|
|
|
pub fn budget_exceeded(&self) -> bool {
|
|
self.exceeded.load(Ordering::SeqCst)
|
|
}
|
|
|
|
pub fn check_budget(&self) -> Result<(), LlmError> {
|
|
if self.budget_exceeded() || (self.limit_usd > 0.0 && self.spent_usd() >= self.limit_usd) {
|
|
self.exceeded.store(true, Ordering::SeqCst);
|
|
return Err(LlmError::BudgetExceeded {
|
|
spent: self.spent_usd(),
|
|
limit: self.limit_usd,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ChatRequest {
|
|
pub model: String,
|
|
pub system: Arc<String>,
|
|
pub user: String,
|
|
pub temperature: f32,
|
|
pub json: bool,
|
|
pub effort: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct ChatCompletion {
|
|
pub content: String,
|
|
pub usage: TokenUsage,
|
|
}
|
|
|
|
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
|
|
|
pub trait ChatBackend: std::fmt::Debug + Send + Sync {
|
|
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>>;
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct DeepseekBackend {
|
|
http: reqwest::Client,
|
|
endpoint: String,
|
|
api_key: String,
|
|
}
|
|
|
|
impl DeepseekBackend {
|
|
pub fn new(cfg: &DeepseekConfig) -> Result<Self, LlmError> {
|
|
let api_key = cfg
|
|
.api_key
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|key| !key.is_empty())
|
|
.ok_or(LlmError::MissingApiKey {
|
|
provider: "deepseek",
|
|
env_var: "DAILY_EPUB_DEEPSEEK__API_KEY",
|
|
})?
|
|
.to_string();
|
|
let http = crate::http::build_client(DEEPSEEK_TIMEOUT)
|
|
.map_err(|error| LlmError::api("deepseek", format!("building http client: {error}")))?;
|
|
Ok(Self {
|
|
http,
|
|
endpoint: format!("{}/chat/completions", cfg.base_url.trim_end_matches('/')),
|
|
api_key,
|
|
})
|
|
}
|
|
}
|
|
|
|
impl ChatBackend for DeepseekBackend {
|
|
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>> {
|
|
Box::pin(async move {
|
|
let mut body = json!({
|
|
"model": req.model,
|
|
"messages": [
|
|
{"role": "system", "content": req.system.as_str()},
|
|
{"role": "user", "content": req.user},
|
|
],
|
|
"temperature": req.temperature,
|
|
"stream": false,
|
|
});
|
|
if req.json
|
|
&& let Some(object) = body.as_object_mut()
|
|
{
|
|
object.insert("response_format".into(), json!({"type": JSON_OBJECT}));
|
|
}
|
|
let response = self
|
|
.http
|
|
.post(&self.endpoint)
|
|
.bearer_auth(&self.api_key)
|
|
.json(&body)
|
|
.send()
|
|
.await
|
|
.map_err(|error| classify_reqwest_error("deepseek", error))?;
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
return Err(classify_status("deepseek", status, response).await);
|
|
}
|
|
let parsed: DeepseekResponse = response.json().await.map_err(|error| {
|
|
LlmError::api("deepseek", format!("decoding chat completion: {error}"))
|
|
})?;
|
|
let content = parsed
|
|
.choices
|
|
.into_iter()
|
|
.next()
|
|
.and_then(|choice| choice.message.content)
|
|
.filter(|content| !content.trim().is_empty())
|
|
.ok_or(LlmError::EmptyResponse {
|
|
provider: "deepseek",
|
|
})?;
|
|
let usage = parsed.usage.map(deepseek_usage).unwrap_or_default();
|
|
Ok(ChatCompletion { content, usage })
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct DeepseekResponse {
|
|
#[serde(default)]
|
|
choices: Vec<DeepseekChoice>,
|
|
#[serde(default)]
|
|
usage: Option<DeepseekUsage>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct DeepseekChoice {
|
|
message: DeepseekMessage,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct DeepseekMessage {
|
|
#[serde(default)]
|
|
content: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Default, Deserialize)]
|
|
struct DeepseekUsage {
|
|
#[serde(default)]
|
|
prompt_tokens: i64,
|
|
#[serde(default)]
|
|
completion_tokens: i64,
|
|
#[serde(default)]
|
|
prompt_cache_hit_tokens: Option<i64>,
|
|
#[serde(default)]
|
|
prompt_tokens_details: Option<PromptTokenDetails>,
|
|
}
|
|
|
|
#[derive(Debug, Default, Deserialize)]
|
|
struct PromptTokenDetails {
|
|
#[serde(default)]
|
|
cached_tokens: Option<i64>,
|
|
}
|
|
|
|
fn deepseek_usage(usage: DeepseekUsage) -> TokenUsage {
|
|
let cached = usage
|
|
.prompt_tokens_details
|
|
.as_ref()
|
|
.and_then(|details| details.cached_tokens)
|
|
.or(usage.prompt_cache_hit_tokens)
|
|
.unwrap_or(0)
|
|
.max(0);
|
|
let prompt = usage.prompt_tokens.max(0);
|
|
let cached = cached.min(prompt);
|
|
TokenUsage {
|
|
input_tokens: prompt - cached,
|
|
cached_tokens: cached,
|
|
cache_write_tokens: 0,
|
|
output_tokens: usage.completion_tokens.max(0),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct AnthropicBackend {
|
|
http: reqwest::Client,
|
|
endpoint: String,
|
|
api_key: String,
|
|
}
|
|
|
|
impl AnthropicBackend {
|
|
pub fn new(cfg: &AnthropicConfig) -> Result<Self, LlmError> {
|
|
let api_key = cfg
|
|
.api_key
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|key| !key.is_empty())
|
|
.ok_or(LlmError::MissingApiKey {
|
|
provider: "anthropic",
|
|
env_var: "DAILY_EPUB_ANTHROPIC__API_KEY",
|
|
})?
|
|
.to_string();
|
|
let http = crate::http::build_client(ANTHROPIC_TIMEOUT).map_err(|error| {
|
|
LlmError::api("anthropic", format!("building http client: {error}"))
|
|
})?;
|
|
Ok(Self {
|
|
http,
|
|
endpoint: format!("{}/v1/messages", cfg.base_url.trim_end_matches('/')),
|
|
api_key,
|
|
})
|
|
}
|
|
}
|
|
|
|
impl ChatBackend for AnthropicBackend {
|
|
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>> {
|
|
Box::pin(async move {
|
|
let body = json!({
|
|
"model": req.model,
|
|
"max_tokens": 16_000,
|
|
"system": [{
|
|
"type": "text",
|
|
"text": req.system.as_str(),
|
|
"cache_control": {"type": "ephemeral"},
|
|
}],
|
|
"messages": [{"role": "user", "content": req.user}],
|
|
"output_config": {"effort": req.effort.as_deref().unwrap_or("high")},
|
|
"fallbacks": "default",
|
|
});
|
|
let response = self
|
|
.http
|
|
.post(&self.endpoint)
|
|
.header("x-api-key", &self.api_key)
|
|
.header("anthropic-version", ANTHROPIC_VERSION)
|
|
.header("anthropic-beta", ANTHROPIC_BETA)
|
|
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
|
.json(&body)
|
|
.send()
|
|
.await
|
|
.map_err(|error| classify_reqwest_error("anthropic", error))?;
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
return Err(classify_status("anthropic", status, response).await);
|
|
}
|
|
let parsed: AnthropicResponse = response.json().await.map_err(|error| {
|
|
LlmError::api("anthropic", format!("decoding messages response: {error}"))
|
|
})?;
|
|
if parsed.stop_reason.as_deref() == Some("refusal") {
|
|
return Err(LlmError::Refusal {
|
|
provider: "anthropic",
|
|
});
|
|
}
|
|
let content = parsed
|
|
.content
|
|
.into_iter()
|
|
.filter(|block| block.kind == "text")
|
|
.filter_map(|block| block.text)
|
|
.collect::<Vec<_>>()
|
|
.join("");
|
|
if content.trim().is_empty() {
|
|
return Err(LlmError::EmptyResponse {
|
|
provider: "anthropic",
|
|
});
|
|
}
|
|
Ok(ChatCompletion {
|
|
content,
|
|
usage: anthropic_usage(parsed.usage),
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct AnthropicResponse {
|
|
#[serde(default)]
|
|
content: Vec<AnthropicContent>,
|
|
#[serde(default)]
|
|
stop_reason: Option<String>,
|
|
#[serde(default)]
|
|
usage: AnthropicUsage,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct AnthropicContent {
|
|
#[serde(rename = "type")]
|
|
kind: String,
|
|
#[serde(default)]
|
|
text: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Default, Deserialize)]
|
|
struct AnthropicUsage {
|
|
#[serde(default)]
|
|
input_tokens: i64,
|
|
#[serde(default)]
|
|
cache_creation_input_tokens: i64,
|
|
#[serde(default)]
|
|
cache_read_input_tokens: i64,
|
|
#[serde(default)]
|
|
output_tokens: i64,
|
|
}
|
|
|
|
fn anthropic_usage(usage: AnthropicUsage) -> TokenUsage {
|
|
TokenUsage {
|
|
input_tokens: usage.input_tokens.max(0),
|
|
cached_tokens: usage.cache_read_input_tokens.max(0),
|
|
cache_write_tokens: usage.cache_creation_input_tokens.max(0),
|
|
output_tokens: usage.output_tokens.max(0),
|
|
}
|
|
}
|
|
|
|
async fn classify_status(
|
|
provider: &'static str,
|
|
status: reqwest::StatusCode,
|
|
response: reqwest::Response,
|
|
) -> LlmError {
|
|
let detail = response.text().await.unwrap_or_default();
|
|
let message = format!("{status}: {}", detail.chars().take(500).collect::<String>());
|
|
if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
|
LlmError::transient(provider, message)
|
|
} else {
|
|
LlmError::api(provider, message)
|
|
}
|
|
}
|
|
|
|
fn classify_reqwest_error(provider: &'static str, error: reqwest::Error) -> LlmError {
|
|
if crate::http::is_retryable(&error) {
|
|
LlmError::transient(provider, error.to_string())
|
|
} else {
|
|
LlmError::api(provider, error.to_string())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct LlmClient {
|
|
pub provider: &'static str,
|
|
pub system_prompt: Arc<String>,
|
|
pub model: String,
|
|
pub effort: Option<String>,
|
|
pub meter: UsageMeter,
|
|
backend: Arc<dyn ChatBackend>,
|
|
retry: RetryPolicy,
|
|
}
|
|
|
|
impl LlmClient {
|
|
pub fn new(
|
|
cfg: &DeepseekConfig,
|
|
system_prompt: String,
|
|
meter: UsageMeter,
|
|
) -> Result<Self, LlmError> {
|
|
let backend = DeepseekBackend::new(cfg)?;
|
|
Ok(Self::with_backend_options(
|
|
"deepseek",
|
|
&cfg.model,
|
|
system_prompt,
|
|
None,
|
|
meter,
|
|
Arc::new(backend),
|
|
))
|
|
}
|
|
|
|
pub fn new_anthropic(
|
|
cfg: &AnthropicConfig,
|
|
system_prompt: String,
|
|
meter: UsageMeter,
|
|
) -> Result<Self, LlmError> {
|
|
let backend = AnthropicBackend::new(cfg)?;
|
|
Ok(Self::with_backend_options(
|
|
"anthropic",
|
|
&cfg.model,
|
|
system_prompt,
|
|
Some(cfg.effort.clone()),
|
|
meter,
|
|
Arc::new(backend),
|
|
))
|
|
}
|
|
|
|
pub fn with_backend(
|
|
model: &str,
|
|
system_prompt: String,
|
|
meter: UsageMeter,
|
|
backend: Arc<dyn ChatBackend>,
|
|
) -> Self {
|
|
Self::with_backend_options("mock", model, system_prompt, None, meter, backend)
|
|
}
|
|
|
|
pub fn with_backend_options(
|
|
provider: &'static str,
|
|
model: &str,
|
|
system_prompt: String,
|
|
effort: Option<String>,
|
|
meter: UsageMeter,
|
|
backend: Arc<dyn ChatBackend>,
|
|
) -> Self {
|
|
Self {
|
|
provider,
|
|
system_prompt: Arc::new(system_prompt),
|
|
model: model.to_string(),
|
|
effort,
|
|
meter,
|
|
backend,
|
|
retry: RetryPolicy::default(),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn with_retry(mut self, retry: RetryPolicy) -> Self {
|
|
self.retry = retry;
|
|
self
|
|
}
|
|
|
|
pub async fn complete(
|
|
&self,
|
|
user_prompt: &str,
|
|
temperature: f32,
|
|
json: bool,
|
|
) -> Result<String, LlmError> {
|
|
self.meter.check_budget()?;
|
|
let request = ChatRequest {
|
|
model: self.model.clone(),
|
|
system: Arc::clone(&self.system_prompt),
|
|
user: user_prompt.to_string(),
|
|
temperature,
|
|
json,
|
|
effort: self.effort.clone(),
|
|
};
|
|
let completion = self
|
|
.retry
|
|
.run(
|
|
&format!("{} chat completion", self.provider),
|
|
LlmError::is_transient,
|
|
|| self.backend.complete(request.clone()),
|
|
)
|
|
.await?;
|
|
self.meter.record(completion.usage);
|
|
Ok(completion.content)
|
|
}
|
|
|
|
pub async fn complete_json<T: serde::de::DeserializeOwned>(
|
|
&self,
|
|
user_prompt: &str,
|
|
temperature: f32,
|
|
) -> Result<T, LlmError> {
|
|
let raw = self.complete(user_prompt, temperature, true).await?;
|
|
let cleaned = strip_code_fence(&raw);
|
|
match serde_json::from_str(cleaned) {
|
|
Ok(value) => Ok(value),
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
provider = self.provider,
|
|
%error,
|
|
preview = %cleaned.chars().take(400).collect::<String>(),
|
|
"llm returned malformed JSON"
|
|
);
|
|
Err(LlmError::Json(error))
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn complete_text(
|
|
&self,
|
|
user_prompt: &str,
|
|
temperature: f32,
|
|
) -> Result<String, LlmError> {
|
|
self.complete(user_prompt, temperature, false).await
|
|
}
|
|
}
|
|
|
|
/// The two provider clients the pipeline works with (§4.2).
|
|
///
|
|
/// Both share the exact same system prompt string (§8.4). Each has its own
|
|
/// [`UsageMeter`] with its own price table and `max_daily_usd` (§5).
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct Llms {
|
|
/// DeepSeek — scoring, and the fallback for every editor call.
|
|
pub bulk: Option<LlmClient>,
|
|
/// Claude — selection, summaries, the brief, the profile rebuild.
|
|
pub editor: Option<LlmClient>,
|
|
}
|
|
|
|
impl Llms {
|
|
/// Build both clients from config with one shared system prompt.
|
|
///
|
|
/// A missing key or `anthropic.enabled = false` leaves that slot `None` with
|
|
/// a log line; nothing here is fatal because the paper always publishes (§17).
|
|
pub fn from_config(
|
|
deepseek: &DeepseekConfig,
|
|
anthropic: &AnthropicConfig,
|
|
system_prompt: String,
|
|
bulk_meter: UsageMeter,
|
|
editor_meter: UsageMeter,
|
|
) -> Self {
|
|
let bulk = match LlmClient::new(deepseek, system_prompt.clone(), bulk_meter) {
|
|
Ok(client) => Some(client),
|
|
Err(error) => {
|
|
tracing::warn!(%error, "DeepSeek (bulk) is unavailable");
|
|
None
|
|
}
|
|
};
|
|
let editor = if anthropic.enabled {
|
|
match LlmClient::new_anthropic(anthropic, system_prompt, editor_meter) {
|
|
Ok(client) => Some(client),
|
|
Err(error) => {
|
|
tracing::warn!(%error, "Anthropic (editor) is unavailable; editor work falls back to bulk");
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
tracing::info!("anthropic.enabled = false; editor work runs on the bulk provider");
|
|
None
|
|
};
|
|
Self { bulk, editor }
|
|
}
|
|
|
|
/// The editor when configured and its meter is not tripped, else bulk.
|
|
pub fn editor_or_bulk(&self) -> Option<&LlmClient> {
|
|
self.editor
|
|
.as_ref()
|
|
.filter(|client| !client.meter.budget_exceeded())
|
|
.or(self.bulk.as_ref())
|
|
}
|
|
|
|
/// True when no provider at all is available (`--skip-llm` or no keys).
|
|
pub fn is_empty(&self) -> bool {
|
|
self.bulk.is_none() && self.editor.is_none()
|
|
}
|
|
}
|
|
|
|
pub fn strip_code_fence(raw: &str) -> &str {
|
|
let trimmed = raw.trim();
|
|
let Some(rest) = trimmed.strip_prefix("```") else {
|
|
return trimmed;
|
|
};
|
|
let rest = rest.strip_prefix("json").unwrap_or(rest);
|
|
rest.trim_start_matches(['\n', '\r'])
|
|
.trim_end()
|
|
.trim_end_matches("```")
|
|
.trim()
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
pub struct MockBackend {
|
|
scripted: Mutex<std::collections::VecDeque<Result<ChatCompletion, LlmError>>>,
|
|
pub seen: Mutex<Vec<ChatRequest>>,
|
|
}
|
|
|
|
impl MockBackend {
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
pub fn push(&self, content: impl Into<String>, usage: TokenUsage) {
|
|
if let Ok(mut queue) = self.scripted.lock() {
|
|
queue.push_back(Ok(ChatCompletion {
|
|
content: content.into(),
|
|
usage,
|
|
}));
|
|
}
|
|
}
|
|
|
|
pub fn push_error(&self, message: impl Into<String>) {
|
|
self.push_llm_error(LlmError::api("mock", message));
|
|
}
|
|
|
|
pub fn push_llm_error(&self, error: LlmError) {
|
|
if let Ok(mut queue) = self.scripted.lock() {
|
|
queue.push_back(Err(error));
|
|
}
|
|
}
|
|
|
|
pub fn calls(&self) -> usize {
|
|
self.seen.lock().map(|seen| seen.len()).unwrap_or(0)
|
|
}
|
|
|
|
pub fn prompts(&self) -> Vec<ChatRequest> {
|
|
self.seen
|
|
.lock()
|
|
.map(|seen| seen.clone())
|
|
.unwrap_or_default()
|
|
}
|
|
}
|
|
|
|
impl ChatBackend for MockBackend {
|
|
fn complete<'a>(&'a self, req: ChatRequest) -> BoxFuture<'a, Result<ChatCompletion, LlmError>> {
|
|
Box::pin(async move {
|
|
let next = self
|
|
.scripted
|
|
.lock()
|
|
.ok()
|
|
.and_then(|mut queue| queue.pop_front());
|
|
if let Ok(mut seen) = self.seen.lock() {
|
|
seen.push(req);
|
|
}
|
|
next.unwrap_or_else(|| Err(LlmError::api("mock", "mock backend ran out of responses")))
|
|
})
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::collections::VecDeque;
|
|
use std::time::Duration;
|
|
|
|
use axum::extract::State;
|
|
use axum::http::{HeaderMap, StatusCode};
|
|
use axum::routing::post;
|
|
use axum::{Json, Router};
|
|
|
|
fn cfg() -> DeepseekConfig {
|
|
DeepseekConfig::default()
|
|
}
|
|
|
|
pub(crate) fn tokens(input: i64, cached: i64, output: i64) -> TokenUsage {
|
|
TokenUsage {
|
|
input_tokens: input,
|
|
cached_tokens: cached,
|
|
cache_write_tokens: 0,
|
|
output_tokens: output,
|
|
}
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Meter and pricing
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn meter_accumulates_and_prices() {
|
|
let meter = UsageMeter::new(&cfg(), 2.0);
|
|
meter.record(tokens(1_000_000, 0, 0));
|
|
meter.record(tokens(0, 1_000_000, 1_000_000));
|
|
let total = meter.total();
|
|
assert_eq!(total.input_tokens, 1_000_000);
|
|
assert_eq!(total.cached_tokens, 1_000_000);
|
|
assert_eq!(total.output_tokens, 1_000_000);
|
|
// The DeepSeek path prices exactly as before the cache-write counter.
|
|
assert!((meter.cost_usd() - 0.4228).abs() < 1e-9);
|
|
assert!(!meter.budget_exceeded());
|
|
assert!(meter.check_budget().is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_price_table_charges_cache_reads_and_writes() {
|
|
let meter =
|
|
UsageMeter::with_prices(PriceTable::anthropic(&AnthropicConfig::default()), 100.0);
|
|
meter.record(TokenUsage {
|
|
input_tokens: 1_000_000,
|
|
cached_tokens: 1_000_000,
|
|
cache_write_tokens: 1_000_000,
|
|
output_tokens: 1_000_000,
|
|
});
|
|
// 5 + 0.5 + 6.25 + 25
|
|
assert!((meter.cost_usd() - 36.75).abs() < 1e-9);
|
|
}
|
|
|
|
#[test]
|
|
fn meter_trips_the_budget_flag_and_stays_tripped() {
|
|
// Ceiling of $0.10; 1M cache-miss input tokens costs $0.14.
|
|
let meter = UsageMeter::new(&cfg(), 0.10);
|
|
meter.record(tokens(1_000_000, 0, 0));
|
|
assert!(meter.budget_exceeded());
|
|
assert!(matches!(
|
|
meter.check_budget(),
|
|
Err(LlmError::BudgetExceeded { .. })
|
|
));
|
|
// Cloned meters share the flag.
|
|
assert!(meter.clone().budget_exceeded());
|
|
}
|
|
|
|
#[test]
|
|
fn preloaded_daily_spend_trips_the_flag() {
|
|
let meter = UsageMeter::new(&cfg(), 1.0);
|
|
meter.preload_cost(0.5);
|
|
assert!(!meter.budget_exceeded());
|
|
assert!((meter.spent_usd() - 0.5).abs() < 1e-9);
|
|
meter.preload_cost(1.5);
|
|
assert!(meter.budget_exceeded());
|
|
}
|
|
|
|
#[test]
|
|
fn deepseek_usage_split_uses_prompt_token_details() {
|
|
let u: DeepseekUsage = serde_json::from_str(
|
|
r#"{"prompt_tokens": 1000, "completion_tokens": 120, "total_tokens": 1120,
|
|
"prompt_tokens_details": {"cached_tokens": 800}}"#,
|
|
)
|
|
.expect("fixture usage");
|
|
assert_eq!(deepseek_usage(u), tokens(200, 800, 120));
|
|
}
|
|
|
|
#[test]
|
|
fn deepseek_usage_falls_back_to_native_cache_fields() {
|
|
let u: DeepseekUsage = serde_json::from_str(
|
|
r#"{"prompt_tokens": 500, "completion_tokens": 40,
|
|
"prompt_cache_hit_tokens": 448, "prompt_cache_miss_tokens": 52}"#,
|
|
)
|
|
.expect("fixture usage");
|
|
assert_eq!(deepseek_usage(u), tokens(52, 448, 40));
|
|
// Missing usage is not an error, just zero.
|
|
let empty: DeepseekUsage = serde_json::from_str("{}").expect("empty usage");
|
|
assert_eq!(deepseek_usage(empty), TokenUsage::default());
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_usage_maps_cache_fields() {
|
|
let u: AnthropicUsage = serde_json::from_str(
|
|
r#"{"input_tokens": 120, "cache_creation_input_tokens": 3000,
|
|
"cache_read_input_tokens": 0, "output_tokens": 800}"#,
|
|
)
|
|
.expect("fixture usage");
|
|
assert_eq!(
|
|
anthropic_usage(u),
|
|
TokenUsage {
|
|
input_tokens: 120,
|
|
cached_tokens: 0,
|
|
cache_write_tokens: 3000,
|
|
output_tokens: 800,
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn code_fences_are_stripped() {
|
|
assert_eq!(strip_code_fence("{\"a\":1}"), "{\"a\":1}");
|
|
assert_eq!(strip_code_fence("```json\n{\"a\":1}\n```"), "{\"a\":1}");
|
|
assert_eq!(strip_code_fence("```\n{\"a\":1}\n```"), "{\"a\":1}");
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// LlmClient over the mock backend
|
|
// -----------------------------------------------------------------------
|
|
|
|
fn client(backend: Arc<MockBackend>, limit: f64) -> LlmClient {
|
|
LlmClient::with_backend(
|
|
"deepseek-v4-flash",
|
|
"SYSTEM PROMPT".into(),
|
|
UsageMeter::new(&cfg(), limit),
|
|
backend,
|
|
)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn json_completion_records_usage_and_sends_system_prompt_first() {
|
|
let backend = Arc::new(MockBackend::new());
|
|
backend.push(r#"{"value": 42}"#, tokens(10, 90, 5));
|
|
let llm = client(Arc::clone(&backend), 2.0);
|
|
|
|
#[derive(serde::Deserialize)]
|
|
struct Out {
|
|
value: i64,
|
|
}
|
|
let out: Out = llm
|
|
.complete_json("score these", 0.3)
|
|
.await
|
|
.expect("mock completion");
|
|
assert_eq!(out.value, 42);
|
|
assert_eq!(llm.meter.total(), tokens(10, 90, 5));
|
|
|
|
let prompts = backend.prompts();
|
|
assert_eq!(prompts.len(), 1);
|
|
assert_eq!(prompts[0].system.as_str(), "SYSTEM PROMPT");
|
|
assert!(prompts[0].json);
|
|
assert_eq!(prompts[0].user, "score these");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn identical_system_prompt_bytes_across_calls() {
|
|
let backend = Arc::new(MockBackend::new());
|
|
backend.push("{}", TokenUsage::default());
|
|
backend.push("{}", TokenUsage::default());
|
|
let llm = client(Arc::clone(&backend), 2.0);
|
|
let _: serde_json::Value = llm.complete_json("a", 0.3).await.expect("first");
|
|
let _: serde_json::Value = llm.complete_json("b", 0.3).await.expect("second");
|
|
let prompts = backend.prompts();
|
|
assert_eq!(prompts[0].system.as_bytes(), prompts[1].system.as_bytes());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn calls_are_refused_once_the_budget_is_gone() {
|
|
let backend = Arc::new(MockBackend::new());
|
|
backend.push("{}", tokens(1_000_000, 0, 0));
|
|
let llm = client(Arc::clone(&backend), 0.01);
|
|
let _: serde_json::Value = llm.complete_json("first", 0.3).await.expect("first call");
|
|
let err = llm
|
|
.complete_text("second", 0.3)
|
|
.await
|
|
.expect_err("budget must be enforced");
|
|
assert!(matches!(err, LlmError::BudgetExceeded { .. }));
|
|
// The refused call never reached the backend.
|
|
assert_eq!(backend.calls(), 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn malformed_json_surfaces_as_json_error() {
|
|
let backend = Arc::new(MockBackend::new());
|
|
backend.push("not json at all", TokenUsage::default());
|
|
let llm = client(backend, 2.0);
|
|
let out: Result<serde_json::Value, _> = llm.complete_json("x", 0.3).await;
|
|
assert!(matches!(out, Err(LlmError::Json(_))));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn refusals_are_not_retried_and_keep_their_variant() {
|
|
let backend = Arc::new(MockBackend::new());
|
|
backend.push_llm_error(LlmError::Refusal {
|
|
provider: "anthropic",
|
|
});
|
|
backend.push("{}", TokenUsage::default());
|
|
let llm = client(Arc::clone(&backend), 2.0).with_retry(RetryPolicy {
|
|
max_attempts: 3,
|
|
base_delay: Duration::from_millis(1),
|
|
max_delay: Duration::from_millis(2),
|
|
});
|
|
let err = llm.complete_text("x", 0.3).await.expect_err("refusal");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::Refusal {
|
|
provider: "anthropic"
|
|
}
|
|
));
|
|
assert_eq!(backend.calls(), 1, "a refusal is terminal for that client");
|
|
}
|
|
|
|
#[test]
|
|
fn missing_api_keys_name_their_provider() {
|
|
let deepseek = DeepseekConfig {
|
|
api_key: Some(" ".into()),
|
|
..DeepseekConfig::default()
|
|
};
|
|
let err = DeepseekBackend::new(&deepseek).expect_err("blank key");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::MissingApiKey {
|
|
provider: "deepseek",
|
|
..
|
|
}
|
|
));
|
|
assert!(err.to_string().contains("DAILY_EPUB_DEEPSEEK__API_KEY"));
|
|
|
|
let anthropic = AnthropicConfig::default();
|
|
let err = AnthropicBackend::new(&anthropic).expect_err("no key");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::MissingApiKey {
|
|
provider: "anthropic",
|
|
..
|
|
}
|
|
));
|
|
assert!(err.to_string().contains("DAILY_EPUB_ANTHROPIC__API_KEY"));
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Llms
|
|
// -----------------------------------------------------------------------
|
|
|
|
fn mock_client(provider: &'static str, limit: f64) -> (LlmClient, Arc<MockBackend>) {
|
|
let backend = Arc::new(MockBackend::new());
|
|
let prices = if provider == "anthropic" {
|
|
PriceTable::anthropic(&AnthropicConfig::default())
|
|
} else {
|
|
PriceTable::deepseek(&cfg())
|
|
};
|
|
let client = LlmClient::with_backend_options(
|
|
provider,
|
|
"model",
|
|
"SYSTEM".into(),
|
|
None,
|
|
UsageMeter::with_prices(prices, limit),
|
|
Arc::clone(&backend) as Arc<dyn ChatBackend>,
|
|
);
|
|
(client, backend)
|
|
}
|
|
|
|
#[test]
|
|
fn editor_or_bulk_prefers_an_untripped_editor() {
|
|
let (bulk, _) = mock_client("deepseek", 2.0);
|
|
let (editor, _) = mock_client("anthropic", 3.0);
|
|
let llms = Llms {
|
|
bulk: Some(bulk),
|
|
editor: Some(editor),
|
|
};
|
|
assert_eq!(llms.editor_or_bulk().map(|c| c.provider), Some("anthropic"));
|
|
// Trip the editor's meter: bulk takes over.
|
|
llms.editor
|
|
.as_ref()
|
|
.expect("editor")
|
|
.meter
|
|
.preload_cost(10.0);
|
|
assert_eq!(llms.editor_or_bulk().map(|c| c.provider), Some("deepseek"));
|
|
// No bulk and a tripped editor means no client at all.
|
|
let only_editor = Llms {
|
|
bulk: None,
|
|
editor: llms.editor.clone(),
|
|
};
|
|
assert!(only_editor.editor_or_bulk().is_none());
|
|
assert!(Llms::default().is_empty());
|
|
assert!(Llms::default().editor_or_bulk().is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn from_config_without_keys_yields_no_clients() {
|
|
let llms = Llms::from_config(
|
|
&cfg(),
|
|
&AnthropicConfig::default(),
|
|
"SYSTEM".into(),
|
|
UsageMeter::new(&cfg(), 1.0),
|
|
UsageMeter::with_prices(PriceTable::anthropic(&AnthropicConfig::default()), 1.0),
|
|
);
|
|
assert!(llms.is_empty());
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// AnthropicBackend against a loopback listener (§4.2, §20)
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[derive(Clone, Default)]
|
|
struct FakeAnthropic {
|
|
seen: Arc<Mutex<Vec<(HeaderMap, serde_json::Value)>>>,
|
|
scripted: Arc<Mutex<VecDeque<(StatusCode, serde_json::Value)>>>,
|
|
}
|
|
|
|
impl FakeAnthropic {
|
|
fn push(&self, status: StatusCode, body: serde_json::Value) {
|
|
self.scripted
|
|
.lock()
|
|
.expect("script lock")
|
|
.push_back((status, body));
|
|
}
|
|
|
|
fn requests(&self) -> Vec<(HeaderMap, serde_json::Value)> {
|
|
self.seen.lock().expect("seen lock").clone()
|
|
}
|
|
}
|
|
|
|
async fn handle(
|
|
State(fake): State<FakeAnthropic>,
|
|
headers: HeaderMap,
|
|
Json(body): Json<serde_json::Value>,
|
|
) -> (StatusCode, Json<serde_json::Value>) {
|
|
fake.seen.lock().expect("seen lock").push((headers, body));
|
|
let (status, body) = fake
|
|
.scripted
|
|
.lock()
|
|
.expect("script lock")
|
|
.pop_front()
|
|
.unwrap_or((
|
|
StatusCode::INTERNAL_SERVER_ERROR,
|
|
json!({"error": "unscripted"}),
|
|
));
|
|
(status, Json(body))
|
|
}
|
|
|
|
async fn serve(fake: FakeAnthropic) -> String {
|
|
let app = Router::new()
|
|
.route("/v1/messages", post(handle))
|
|
.with_state(fake);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("loopback listener");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
tokio::spawn(async move {
|
|
let _ = axum::serve(listener, app).await;
|
|
});
|
|
format!("http://{addr}")
|
|
}
|
|
|
|
async fn anthropic_client(fake: FakeAnthropic, limit: f64) -> LlmClient {
|
|
let base_url = serve(fake).await;
|
|
let config = AnthropicConfig {
|
|
base_url,
|
|
api_key: Some("test-key-never-logged".into()),
|
|
effort: "medium".into(),
|
|
..AnthropicConfig::default()
|
|
};
|
|
LlmClient::new_anthropic(
|
|
&config,
|
|
"PROFILE SYSTEM PROMPT".into(),
|
|
UsageMeter::with_prices(PriceTable::anthropic(&config), limit),
|
|
)
|
|
.expect("client")
|
|
.with_retry(RetryPolicy {
|
|
max_attempts: 3,
|
|
base_delay: Duration::from_millis(1),
|
|
max_delay: Duration::from_millis(2),
|
|
})
|
|
}
|
|
|
|
fn ok_message(text: &str, stop_reason: &str) -> serde_json::Value {
|
|
json!({
|
|
"id": "msg_01",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"model": "claude-opus-5",
|
|
"content": [
|
|
{"type": "thinking", "thinking": ""},
|
|
{"type": "text", "text": text}
|
|
],
|
|
"stop_reason": stop_reason,
|
|
"usage": {
|
|
"input_tokens": 1_000_000,
|
|
"cache_creation_input_tokens": 1_000_000,
|
|
"cache_read_input_tokens": 1_000_000,
|
|
"output_tokens": 1_000_000
|
|
}
|
|
})
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn anthropic_request_has_the_documented_shape() {
|
|
let fake = FakeAnthropic::default();
|
|
fake.push(
|
|
StatusCode::OK,
|
|
ok_message("```json\n{\"ok\": true}\n```", "end_turn"),
|
|
);
|
|
let llm = anthropic_client(fake.clone(), 100.0).await;
|
|
|
|
let out: serde_json::Value = llm
|
|
.complete_json("the task", 0.7)
|
|
.await
|
|
.expect("completion");
|
|
assert_eq!(out, json!({"ok": true}));
|
|
|
|
let requests = fake.requests();
|
|
assert_eq!(requests.len(), 1);
|
|
let (headers, body) = &requests[0];
|
|
assert_eq!(
|
|
headers.get("x-api-key").and_then(|v| v.to_str().ok()),
|
|
Some("test-key-never-logged")
|
|
);
|
|
assert_eq!(
|
|
headers
|
|
.get("anthropic-version")
|
|
.and_then(|v| v.to_str().ok()),
|
|
Some("2023-06-01")
|
|
);
|
|
assert_eq!(
|
|
headers.get("anthropic-beta").and_then(|v| v.to_str().ok()),
|
|
Some("server-side-fallback-2026-07-01")
|
|
);
|
|
assert!(
|
|
headers
|
|
.get("content-type")
|
|
.and_then(|v| v.to_str().ok())
|
|
.is_some_and(|v| v.starts_with("application/json"))
|
|
);
|
|
|
|
assert_eq!(body["model"], "claude-opus-5");
|
|
assert_eq!(body["max_tokens"], 16_000);
|
|
assert_eq!(body["system"][0]["type"], "text");
|
|
assert_eq!(body["system"][0]["text"], "PROFILE SYSTEM PROMPT");
|
|
assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
|
|
assert_eq!(body["messages"][0]["role"], "user");
|
|
assert_eq!(body["messages"][0]["content"], "the task");
|
|
assert_eq!(body["output_config"]["effort"], "medium");
|
|
assert_eq!(body["fallbacks"], "default");
|
|
for forbidden in [
|
|
"temperature",
|
|
"top_p",
|
|
"top_k",
|
|
"thinking",
|
|
"response_format",
|
|
] {
|
|
assert!(
|
|
body.get(forbidden).is_none(),
|
|
"{forbidden} must not be sent"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
body["messages"].as_array().map(Vec::len),
|
|
Some(1),
|
|
"no prefill"
|
|
);
|
|
|
|
// Usage was priced with the cache read/write rates: 5 + 6.25 + 0.5 + 25.
|
|
assert_eq!(
|
|
llm.meter.total(),
|
|
TokenUsage {
|
|
input_tokens: 1_000_000,
|
|
cached_tokens: 1_000_000,
|
|
cache_write_tokens: 1_000_000,
|
|
output_tokens: 1_000_000,
|
|
}
|
|
);
|
|
assert!((llm.meter.cost_usd() - 36.75).abs() < 1e-9);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn anthropic_refusal_surfaces_as_the_fallback_error() {
|
|
let fake = FakeAnthropic::default();
|
|
fake.push(
|
|
StatusCode::OK,
|
|
json!({
|
|
"content": [],
|
|
"stop_reason": "refusal",
|
|
"stop_details": {"type": "refusal", "category": "cyber"},
|
|
"usage": {"input_tokens": 0, "output_tokens": 0}
|
|
}),
|
|
);
|
|
let llm = anthropic_client(fake.clone(), 100.0).await;
|
|
let err = llm.complete_text("x", 0.3).await.expect_err("refusal");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::Refusal {
|
|
provider: "anthropic"
|
|
}
|
|
));
|
|
assert!(!err.is_transient());
|
|
assert_eq!(fake.requests().len(), 1, "a refusal is never retried");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn anthropic_429_is_retried_but_400_is_not() {
|
|
let fake = FakeAnthropic::default();
|
|
fake.push(
|
|
StatusCode::TOO_MANY_REQUESTS,
|
|
json!({"type": "error", "error": {"type": "rate_limit_error"}}),
|
|
);
|
|
fake.push(
|
|
StatusCode::OK,
|
|
ok_message("{\"after\": \"retry\"}", "end_turn"),
|
|
);
|
|
let llm = anthropic_client(fake.clone(), 100.0).await;
|
|
let text = llm.complete_text("x", 0.3).await.expect("second attempt");
|
|
assert_eq!(text, "{\"after\": \"retry\"}");
|
|
assert_eq!(fake.requests().len(), 2);
|
|
|
|
let fake = FakeAnthropic::default();
|
|
fake.push(
|
|
StatusCode::BAD_REQUEST,
|
|
json!({"type": "error", "error": {"type": "invalid_request_error", "message": "nope"}}),
|
|
);
|
|
let llm = anthropic_client(fake.clone(), 100.0).await;
|
|
let err = llm.complete_text("x", 0.3).await.expect_err("400");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::Api {
|
|
provider: "anthropic",
|
|
..
|
|
}
|
|
));
|
|
assert!(err.to_string().contains("400"));
|
|
assert_eq!(fake.requests().len(), 1, "400 is never retried");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn anthropic_concatenates_text_blocks_and_rejects_empty_output() {
|
|
let fake = FakeAnthropic::default();
|
|
fake.push(
|
|
StatusCode::OK,
|
|
json!({
|
|
"content": [
|
|
{"type": "text", "text": "{\"a\": "},
|
|
{"type": "thinking", "thinking": "..."},
|
|
{"type": "text", "text": "1}"}
|
|
],
|
|
"stop_reason": "end_turn",
|
|
"usage": {"input_tokens": 10, "output_tokens": 5}
|
|
}),
|
|
);
|
|
fake.push(
|
|
StatusCode::OK,
|
|
json!({
|
|
"content": [{"type": "thinking", "thinking": ""}],
|
|
"stop_reason": "end_turn",
|
|
"usage": {"input_tokens": 10, "output_tokens": 0}
|
|
}),
|
|
);
|
|
let llm = anthropic_client(fake.clone(), 100.0).await;
|
|
let out: serde_json::Value = llm.complete_json("x", 0.3).await.expect("joined");
|
|
assert_eq!(out, json!({"a": 1}));
|
|
let err = llm.complete_text("y", 0.3).await.expect_err("empty");
|
|
assert!(matches!(
|
|
err,
|
|
LlmError::EmptyResponse {
|
|
provider: "anthropic"
|
|
}
|
|
));
|
|
}
|
|
}
|