feat(llm-client): add bounded upstream retries

Cargo.toml

@@ -26,6 +26,7 @@ async-stream = "0.3"
 async-trait = "0.1"
 futures = "0.3"
 futures-util = "0.3"
+httpdate = "1"
 parking_lot = "0.12"
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots", "stream"] }

crates/libsy-llm-client/Cargo.toml

@@ -18,9 +18,11 @@ reqwest.workspace = true
 async-trait.workspace = true
 futures-util.workspace = true
 opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
+httpdate.workspace = true
 serde_json.workspace = true
+tokio.workspace = true
+tracing.workspace = true
 
 [dev-dependencies]
 futures.workspace = true
-tokio.workspace = true
 wiremock = "0.6"

crates/libsy-llm-client/README.md

@@ -22,7 +22,7 @@ provider SDK.
   when set, otherwise the model's default backend.
 - **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or
   `Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,
-  static `extra_headers`). The variant fixes the URL path and auth scheme
+  static `extra_headers`, `max_retries`). The variant fixes the URL path and auth scheme
   (Bearer vs `x-api-key` + `anthropic-version`).
 - **Model rewrite.** The resolved model name is both the map key and the model id
   sent upstream — it overwrites whatever `model` the request arrived with.
@@ -56,6 +56,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
         base_url: "https://api.openai.com/v1".to_string(),
         api_key: std::env::var("OPENAI_API_KEY").ok(),
         extra_headers: BTreeMap::new(),
+        max_retries: 2,
     };
 
     let models = [ModelConfig::new(
@@ -169,6 +170,14 @@ fn build_multi_format_client(
   `authorization` / `x-api-key` / `anthropic-version` / `content-type`. So a
   caller's placeholder credential never overrides the backend's real key.
 - Per-backend static headers go in `HttpBackendConfig::extra_headers`.
+- `HttpBackendConfig::max_retries` controls additional attempts after retryable
+  transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body
+  transport failures are retried; streaming body failures are not replayed after
+  the response has been returned.
+
+Retries replay the same upstream request. A transport failure can therefore
+duplicate a request that the provider processed but did not finish returning,
+and the retry budget plus capped `Retry-After` delays determines total latency.
 
 ## Errors
 

crates/libsy-llm-client/src/backend.rs

@@ -12,6 +12,9 @@ use crate::error::is_overflow_body;
 
 const ANTHROPIC_VERSION: &str = "2023-06-01";
 
+/// Default number of retries for server-configured upstream calls.
+pub const DEFAULT_MAX_RETRIES: u32 = 2;
+
 // Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
 // provider-wrap is a one-line entry here, not a fork of the parsing logic.
 const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
@@ -39,6 +42,8 @@ pub struct HttpBackendConfig {
     pub api_key: Option<String>,
     /// Static headers added to every outbound call to this backend.
     pub extra_headers: BTreeMap<String, String>,
+    /// Additional attempts after the initial upstream request.
+    pub max_retries: u32,
 }
 
 impl fmt::Debug for HttpBackendConfig {
@@ -47,6 +52,7 @@ impl fmt::Debug for HttpBackendConfig {
             .field("base_url", &self.base_url)
             .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
             .field("extra_headers", &self.extra_headers)
+            .field("max_retries", &self.max_retries)
             .finish()
     }
 }
@@ -124,6 +130,11 @@ impl Backend {
         &self.config().extra_headers
     }
 
+    /// Additional attempts allowed after the initial request.
+    pub fn max_retries(&self) -> u32 {
+        self.config().max_retries
+    }
+
     /// Whether this backend speaks the Anthropic Messages wire format — the only
     /// one with a `count_tokens` endpoint.
     pub fn is_anthropic(&self) -> bool {
@@ -186,6 +197,7 @@ mod tests {
             base_url: base_url.to_string(),
             api_key: Some("secret".to_string()),
             extra_headers: BTreeMap::new(),
+            max_retries: 0,
         }
     }