From ac8d0fa34164b07b555a42e023258c8b019834ce Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 28 Jul 2026 06:22:27 +0000 Subject: [PATCH] refactor(routing)!: remove latency-aware router Signed-off-by: Eric Liu --- AGENTS.md | 7 +- CHANGELOG.md | 14 + .../tests/stats_accumulator.rs | 6 +- .../src/core_bindings/context.rs | 9 +- .../src/core_bindings/response.rs | 6 +- docs/internal/latency_service_routing.md | 609 ----- docs/internal/metrics_reference.md | 68 +- examples/latency_service_caller_required.yaml | 70 - examples/prometheus/README.md | 14 +- examples/prometheus/prometheus.yml | 11 +- examples/prometheus/switchyard.rules.yaml | 61 +- switchyard/__init__.py | 15 - switchyard/cli/launch_command.py | 2 +- .../cli/launchers/claude_code_launcher.py | 4 +- switchyard/cli/route_bundle.py | 133 -- switchyard/cli/switchyard_cli.py | 2 +- switchyard/lib/affinity_pin_store.py | 4 +- switchyard/lib/backends/__init__.py | 12 - switchyard/lib/backends/health_poller.py | 181 -- .../backends/latency_service_llm_backend.py | 1098 --------- switchyard/lib/config/__init__.py | 6 - .../config/latency_service_backend_config.py | 168 -- .../lib/endpoints/prometheus_emitter.py | 5 +- switchyard/lib/endpoints/stats_endpoint.py | 7 +- switchyard/lib/endpoints/upstream_error.py | 11 +- switchyard/lib/profiles/__init__.py | 2 - .../escalation_router_profile_config.py | 6 +- switchyard/lib/profiles/latency_service.py | 36 - switchyard/lib/proxy_context.py | 4 +- switchyard/lib/redis_pin_store.py | 4 +- switchyard/lib/session_affinity.py | 4 +- switchyard/lib/tracing.py | 4 +- tests/e2e/test_latency_service_llm_backend.py | 427 ---- tests/readme/test_readme.py | 7 +- tests/test_error_source_annotation.py | 6 +- tests/test_inference_e2e.py | 95 + tests/test_latency_service_health_metrics.py | 311 --- tests/test_latency_service_llm_backend.py | 1976 ----------------- tests/test_latency_service_spans.py | 273 --- tests/test_latency_service_stats_metrics.py | 273 --- tests/test_outcome_metrics.py | 408 +--- tests/test_passthrough_profile_e2e.py | 2 +- tests/test_redis_pin_store.py | 29 +- tests/test_route_bundle.py | 149 +- tests/test_route_selection_headers.py | 272 +-- tests/test_session_affinity.py | 2 +- tests/test_stream_close_chain.py | 2 +- tests/test_stream_leak_repro.py | 2 +- tests/test_upstream_error_passthrough.py | 732 +----- 49 files changed, 275 insertions(+), 7274 deletions(-) delete mode 100644 docs/internal/latency_service_routing.md delete mode 100644 examples/latency_service_caller_required.yaml delete mode 100644 switchyard/lib/backends/health_poller.py delete mode 100644 switchyard/lib/backends/latency_service_llm_backend.py delete mode 100644 switchyard/lib/config/latency_service_backend_config.py delete mode 100644 switchyard/lib/profiles/latency_service.py delete mode 100644 tests/e2e/test_latency_service_llm_backend.py delete mode 100644 tests/test_latency_service_health_metrics.py delete mode 100644 tests/test_latency_service_llm_backend.py delete mode 100644 tests/test_latency_service_spans.py delete mode 100644 tests/test_latency_service_stats_metrics.py diff --git a/AGENTS.md b/AGENTS.md index 0a3117ad..e214dc9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,12 +182,10 @@ switchyard/ │ │ ├── openai_llm_backend.py # OpenAiPassthroughBackend │ │ ├── openai_native_backend.py # OpenAiNativeBackend │ │ ├── anthropic_native_llm_backend.py # AnthropicNativeBackend -│ │ ├── latency_service_llm_backend.py # LatencyServiceLLMBackend │ │ ├── llm_target.py # LlmTarget, BackendFormat │ │ ├── multi_llm_backend.py # MultiLlmBackend helpers │ │ ├── stats_llm_backend.py # StatsLlmBackend -│ │ ├── backend_format_resolver.py # BackendFormatResolver -│ │ └── health_poller.py # HealthPoller, EndpointHealthStatus +│ │ └── backend_format_resolver.py # BackendFormatResolver │ ├── processors/ # Request-side / response-side component implementations │ │ ├── format_translate.py │ │ ├── random_routing_request_processor.py @@ -205,8 +203,7 @@ switchyard/ │ │ ├── sse_helpers.py │ │ └── base.py │ └── config/ -│ ├── intake_sink_config.py -│ └── latency_service_backend_config.py +│ └── intake_sink_config.py ├── cli/ # CLI (requires `nemo-switchyard[cli]`) │ ├── switchyard_cli.py # `switchyard` entry point │ ├── launch_command.py # `switchyard launch claude/codex` diff --git a/CHANGELOG.md b/CHANGELOG.md index a711afd1..bd9d04b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to Switchyard are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Removed + +- **Latency-aware router** — the `latency_service` route type and its + `LatencyServiceLLMBackend`, `LatencyServiceBackendConfig`, + `LatencyServiceEndpoint`, and `LatencyServiceProfileConfig` public API are + removed. It depended on NVIDIA Inference Hub's latency endpoint and schema. + Deployments that need multi-endpoint, load- or latency-aware routing should + migrate to [Dynamo](https://github.com/ai-dynamo/dynamo) (backend-load / + KV-cache-aware routing with request failover) or an external load balancer + such as [Traefik](https://doc.traefik.io/traefik/reference/routing-configuration/http/load-balancing/service/) + or HAProxy. + ## [0.1.0] — Initial release First public release of Switchyard — a typed, composable control plane for LLM diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rs index d6cd3740..1f2e5134 100644 --- a/crates/switchyard-components/tests/stats_accumulator.rs +++ b/crates/switchyard-components/tests/stats_accumulator.rs @@ -751,7 +751,7 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> { accumulator.record_routing_decision("stage_router", "dimensions")?; accumulator.record_routing_decision("stage_router", "dimensions")?; accumulator.record_routing_decision("stage_router", "llm-classifier")?; - accumulator.record_routing_decision("latency-service", "health")?; + accumulator.record_routing_decision("escalation_router", "judge")?; let snapshot = accumulator.snapshot()?; assert_eq!( @@ -771,8 +771,8 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> { assert_eq!( snapshot .routing_decisions - .get("latency-service") - .and_then(|sources| sources.get("health")), + .get("escalation_router") + .and_then(|sources| sources.get("judge")), Some(&1) ); diff --git a/crates/switchyard-py/src/core_bindings/context.rs b/crates/switchyard-py/src/core_bindings/context.rs index 57c8fd4b..1b361f81 100644 --- a/crates/switchyard-py/src/core_bindings/context.rs +++ b/crates/switchyard-py/src/core_bindings/context.rs @@ -614,11 +614,10 @@ impl PyProxyContext { /// Records the measured backend-call latency in ms. /// /// The Rust ``StatsLlmBackend`` wraps native backends and writes this - /// slot automatically. Python-only backends (e.g. ``LatencyServiceLLMBackend``) - /// that can't be wrapped record their measurement here so the - /// downstream ``StatsResponseProcessor`` can compute - /// ``routing_overhead_ms = total_latency - backend_latency`` and emit - /// it on ``/metrics``. Setting ``None`` clears the slot. + /// slot automatically. Python-only backends that can't be wrapped record + /// their measurement here so the downstream ``StatsResponseProcessor`` can + /// compute ``routing_overhead_ms = total_latency - backend_latency`` and + /// emit it on ``/metrics``. Setting ``None`` clears the slot. #[setter] fn set_backend_call_latency_ms(&self, value: Option) -> PyResult<()> { let mut inner = self.lock()?; diff --git a/crates/switchyard-py/src/core_bindings/response.rs b/crates/switchyard-py/src/core_bindings/response.rs index 85fcb444..e9a02ede 100644 --- a/crates/switchyard-py/src/core_bindings/response.rs +++ b/crates/switchyard-py/src/core_bindings/response.rs @@ -492,9 +492,9 @@ impl PyResponseStream { /// ``Switchyard.call`` round trip (``source`` is ``None``): the source was /// moved onto the core stream by ``take_core_stream`` as a /// ``SourceClosingStream``; dropping the inner adapter here drops that - /// wrapper, which closes the source on drop. This is the latency-router - /// production path, where response processors re-wrap the core stream and - /// ``source`` cannot be recovered on this object. + /// wrapper, which closes the source on drop. This is the production path + /// where response processors re-wrap the core stream and ``source`` + /// cannot be recovered on this object. fn aclose<'py>(&self, py: Python<'py>) -> PyResult> { self.consumed.store(true, Ordering::Release); let stream = Arc::clone(&self.stream); diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md deleted file mode 100644 index d57b731b..00000000 --- a/docs/internal/latency_service_routing.md +++ /dev/null @@ -1,609 +0,0 @@ -# Latency Service Routing - -The **latency-aware router** (`type: latency_service`) routes each request to one of -several equivalent endpoints, biasing traffic toward whichever endpoint a central -**Latency Service** most recently observed as healthy and fast. It is the production -routing policy for OpenRouter-backed deployments, where a separate Latency Service -owns heartbeat probing and statistical latency profiling, and Switchyard consumes its -verdicts to make per-request steering decisions. - -This document explains how the router works, the design choices behind it, and how to -configure and operate it. - -## When to use it - -Use `latency_service` when you have **multiple interchangeable endpoints** serving the -same logical model (e.g. the same model behind different providers or regions) and you -want Switchyard to: - -- bias traffic toward the lowest-latency healthy endpoint, -- automatically steer away from degraded endpoints, and -- retry on a different endpoint when one fails, without the client ever seeing it. - -If you only have a single backend, use `passthrough`. If you want a fixed weighted -strong/weak split for benchmarking, use -[Random Routing](../routing_algorithms/random_routing.md). - -## Architecture - -The router is implemented as a single custom `LLMBackend`, -`LatencyServiceLLMBackend` (`switchyard/lib/backends/latency_service_llm_backend.py`), -rather than a monolithic routing strategy. It slots into the standard four-role chain: - -``` -[RequestProcessor*] → LatencyServiceLLMBackend → [ResponseProcessor*] → TranslationEngine -``` - -The backend owns four things: - -1. **A pool of `OpenAILLMClient` instances**, one per configured endpoint, keyed by the - endpoint's `model` ID. -2. **A thread-safe in-memory health cache** (`dict[str, EndpointHealth]` guarded by a - `threading.Lock`). -3. **A background `HealthPoller` daemon thread** - (`switchyard/lib/backends/health_poller.py`) that refreshes the cache from the - Latency Service. -4. **Endpoint-selection + retry logic** on the request hot path. - -Inbound format translation (Anthropic Messages / OpenAI Responses → OpenAI Chat) is -delegated to the Rust `TranslationEngine`, so the backend transparently accepts any -inbound wire format and only ever speaks OpenAI Chat Completions upstream. - -### Two-clock design: poll loop vs. hot path - -The defining property of this router is that **health polling and request serving run on -separate clocks**: - -- The **`HealthPoller` daemon thread** is the only thing that ever talks to the Latency - Service. It polls on a fixed interval and *writes* verdicts into the cache. -- The **request hot path** only ever *reads* the cache (under the lock). It never makes a - network call to the Latency Service. - -Health awareness adds **zero per-request latency**. Selection is a dictionary -read and a weighted random pick. The poller is a plain daemon thread (not an asyncio -task), so its lifecycle is trivial. It starts when the backend is constructed and dies -with the process. Its synchronous `httpx.Client` never contends with the event loop -serving requests. - -## The health poller - -`HealthPoller` polls `{latency_service_url}/v1/endpoints/health` every `poll_interval_s` -seconds (default 10s), passing the configured endpoint IDs as `endpoint_ids` query -params. The Latency Service responds with a verdict and a latency sample per endpoint: - -```json -{ - "endpoint_health": { - "openai/gpt-5.5": {"status": "healthy", "last_latency_ms": 420.0}, - "azure_openai/gpt-5.5": {"status": "degraded", "last_latency_ms": 980.0} - } -} -``` - -Each cache entry is an `EndpointHealth` snapshot carrying the discrete `status` and the -most recent `last_latency_ms`. The status enum is the **shared contract** between -Switchyard and the Latency Service: both sides must agree on these three string values: - -| Status | Meaning | -|---|---| -| `healthy` | Endpoint is up and within latency expectations. | -| `degraded` | Endpoint is reachable but slow or partially failing. | -| `unknown` | No current verdict (warming up, or fallback after a poll failure). | - -### Failure handling: fall back to UNKNOWN, not stale data - -If a poll fails for any reason (DNS, network, timeout, non-2xx, or a schema mismatch on -the response), the poller **resets every endpoint in the cache to `UNKNOWN`** rather than -leaving stale verdicts in place. With every endpoint at `UNKNOWN`, the router degrades to -uniform random routing across the full pool. This is the safe default when health -information is unavailable. Acting on stale "healthy" data after the Latency Service has -gone dark would be strictly worse. - -`last_latency_ms` is `None` before the first successful poll and whenever the Latency -Service reports it as null. - -## Endpoint selection - -On each request, `_select_endpoint()` picks a `model` ID from the cache in two steps: - -### 1. Tier preference: `HEALTHY > UNKNOWN > DEGRADED` - -Endpoints are bucketed by status, and the router picks from the **best non-empty tier**. -`UNKNOWN` is deliberately preferred over `DEGRADED`: an endpoint we have no verdict for is -a better bet than one the Latency Service explicitly flagged as degraded. - -### 2. Within a tier: inverse-latency weighted random - -Within the chosen tier, `_pick_by_latency()` selects an endpoint with probability -proportional to `1 / last_latency_ms`, so an endpoint the Latency Service recently saw -at 200 ms receives roughly twice the traffic of one at 400 ms. This is **weighted random, -not always-pick-the-fastest**, which spreads load and avoids stampeding a single endpoint. - -The picker falls back to **uniform random** within the tier when: - -- there is only one candidate, or -- any candidate's `last_latency_ms` is unknown (`None`), or -- any sample is non-positive (defensive: the Latency Service should never report ≤ 0). - -This keeps behavior predictable while the poller is warming up or when the upstream -reports nulls. - -### 3. Optional: session affinity (sticky routing) - -By default the router picks per **turn**. Every request runs the tier + latency steps -above independently. For a multi-turn conversation this interleaves endpoints, which lets -each endpoint's upstream prompt/KV cache lapse and forces expensive cache *re-writes* on -the next turn that lands there. - -Set `session_affinity: true` to route per **conversation** instead. The latency-aware -picker decides only the **first** turn, then pins that conversation to the endpoint -that served it. Every later turn reuses that endpoint (the affinity fast-path bypasses -tier + latency selection entirely), so the upstream cache stays warm for the life of -the conversation. - -- **Conversation identity.** A streaming proxy holds no session ID. The client re-sends - the full history each turn. `session_key_from_body()` - (`switchyard/lib/session_key.py`) derives a stable key by hashing the stable harness - prefix made from the system prompt + the first user message. Later turns only append, so - every turn of one conversation hashes the same; distinct conversations differ. -- **Health overrides the pin.** A pin is reused only while its endpoint is `HEALTHY` or - `UNKNOWN`. If it goes `DEGRADED` (or the call fails this request), the next turn - re-routes through normal selection and re-pins to whatever serves it. Locality yields - to health, so a session is never funneled into a failing endpoint. -- **Bounded memory.** Pins live in an in-process LRU map (the L1) capped at - `affinity_max_sessions` (default `10_000`); the least-recently-used pin is evicted past - the cap. -- **Cross-worker stickiness (optional shared store).** The L1 map is per process, so a - multi-worker/multi-pod deployment may pin the same conversation differently per worker, - and pins are lost on pod churn. Set `affinity_store: "redis"` (with `affinity_store_url`) - to add a shared, persistent L2: on an L1 miss the pin is read through from the store and - warmed back into L1, and every pin is written through to it. Pins then survive across - replicas and pod restarts (bounded by `affinity_store_ttl_seconds`, refreshed on each - write). The L2 is **best-effort** — an error or timeout falls back to L1 / unpinned and - never fails a request; fail-open operations are counted on - `switchyard_affinity_l2_errors_total`, and a circuit breaker (3 consecutive failures → - skip the store for 10 s, then probe; `switchyard_affinity_l2_breaker_open` gauge) keeps - a store outage from taxing every request with timeout waits. The store connection is released by the backend's - `shutdown()` (the server lifespan teardown hook awaits it). Requires the - `switchyard[affinity-redis]` extra. (Alternatively, front the workers with a - session-affinity load balancer.) - -The `switchyard.route_decision` span tags each pick with `switchyard.affinity_hit` so you -can measure the warm-reuse rate. Affinity is **off by default**; existing per-turn latency -routing is unchanged unless opted in. - -#### Deploying the shared store: the operational contract - -What an operator needs to know to provision and run the Redis L2. Everything below is -verified against the implementation (`RedisPinStore`, `SessionAffinity`). - -**Topology.** A **standalone Redis endpoint only** (the client is -`redis.asyncio.from_url`): a single instance, a primary endpoint of a replicated setup, -or a managed single-endpoint tier. **Redis Cluster and Sentinel URLs are not -supported** — cluster `MOVED` redirects surface as fail-open errors, meaning requests -keep succeeding while stickiness silently degrades. A sustained -`switchyard_affinity_l2_errors_total` rate from the first request is the -mis-provisioning signal; watch it when first enabling the store. - -**Auth and TLS.** URL-driven: `rediss://` for TLS, credentials inside the URL. Route -YAML expands `${ENV}` references in every string value, so keep the URL in a secret and -configure `affinity_store_url: "${AFFINITY_REDIS_URL}"` rather than an inline literal. - -**What is stored.** Key = `affinity_key_prefix` + a 16-hex-char hash of the -conversation's stable prefix (system prompt + first user message); value = the pinned -endpoint id string. **No prompt text, message content, or user identifier leaves the -process** — a pin is ~50–100 bytes. - -**Traffic and sizing.** One `GET` per request whose conversation isn't in the local L1 -(first turn on a pod), and one `SET` (which refreshes the TTL) per successful request. -Working-set memory ≈ active conversations × ~150 B, bounded by the sliding -`affinity_store_ttl_seconds` (default 1 h). **Persistence (RDB/AOF) and HA are optional**: -losing the store only resets stickiness — conversations re-route cold and re-pin — it -never affects correctness. `maxmemory-policy allkeys-lru` is the recommended eviction -policy; an evicted pin costs one cold re-route. - -**Failure envelope.** Every store operation is bounded by a 0.1 s socket/connect -timeout (a colocated Redis answers in well under 1 ms, so this is ~100× headroom) and -fails open — requests never fail because of the store. A **circuit breaker** caps the -outage tax: after 3 consecutive failures, L2 operations are skipped without a network -attempt for a 10 s cooldown, then one operation probes; success closes the breaker, -failure re-arms it. Worst case per pod during an outage is therefore a brief window of -up to ~0.2 s added per request (one `GET` on the L1 miss + one `SET` after success) -before the breaker opens, then ~zero added latency with one 0.1 s probe per 10 s — -stickiness degrades to per-pod L1 throughout. Alert on -`switchyard_affinity_l2_breaker_open == 1` (the outage signal) and on -`rate(switchyard_affinity_l2_errors_total[5m])` (degradation without a full trip — -note the rate drops to ~0.1/s per pod while the breaker is open, since only probes -fail). Pod readiness (`/health`, `is_ready()`) never depends on Redis. - -**Isolation.** Deployments or routes sharing one Redis should set distinct -`affinity_key_prefix` values (or separate logical databases). A pin is honored whenever -its stored endpoint id exists in the reading route's endpoint set, so shared prefixes -cross-pollinate stickiness between routes that reuse endpoint ids. - -**Config changes, upgrades, rollback.** A persisted pin whose endpoint id is no longer -in the route's endpoint list is ignored gracefully (the next turn re-routes and -re-pins) — shrinking or renaming the endpoint set needs no store cleanup. The -conversation-key derivation is not a cross-version contract: a Switchyard upgrade may -invalidate existing pins (one-time cold re-route, self-healing). To roll back, set -`affinity_store: "memory"`; stale keys expire via TTL, and `FLUSHDB` is always safe -(stickiness reset only). - -The deterministic LLM-classifier route also supports `affinity_warmup_turns` to delay -when a tier becomes sticky. Latency-service routing does **not** use that knob. Its -affinity objective is endpoint cache reuse, so it pins after the first successful -endpoint selection. - -## Retry and failover - -A single client request may try up to `1 + max_retries` endpoints (default `max_retries` -is 2, so 3 attempts). On each attempt: - -1. Select an endpoint. If selection returns one already tried this request (and untried - endpoints remain), pick uniformly at random from the **untried** set. Dedup prevents - wasting an attempt re-hitting an endpoint that just failed. -2. Stamp the endpoint's `upstream_model` into `body["model"]` and call the upstream. -3. On a **transient** failure such as `429` (rate limit), `408` (request timeout), any - `5xx`, or a network / pre-status / SDK error, log it, record the error in stats, increment - the outcome metric, and continue to the next endpoint. These are exactly the faults a - health-aware router should absorb by trying a different endpoint. -4. On a **4xx client error** (`400`, `401`, `403`, `404`, `409`, `413`, `415`, `422`, …) - the loop **fails fast**: it does *not* retry. The request itself is malformed or - unauthorized, so every replica rejects the same payload identically; retrying only adds - latency before surfacing the same status. (Context-window overflow surfaces as a `400` - here and is also passed straight through. The `latency_service` route does not - participate in the chain-level evict-and-retry described in - [Context-Window Handling](../operations/context_window.md).) - -If an attempt succeeds **after** at least one failure, the router records a -`retry_recovered` outcome. This is direct evidence the steering logic rescued a request that -would otherwise have failed. - -Whenever the request ends on an upstream HTTP error, either a fail-fast 4xx or an -exhausted retryable error (e.g. a `503` after retries are exhausted), the backend records the -upstream status code and body on `ctx` so the endpoint passes the real upstream status -through to the client rather than masking it as a generic `500`. - -## Configuration - -`LatencyServiceBackendConfig` owns the router's policy, retry, and optional -session-affinity settings. In CLI YAML, configure the Python -`type: latency_service` router in a `routes:` bundle; there are no -policy-specific CLI flags. Run it with: - -```bash -switchyard --routing-profiles my_routes.yaml -- serve --port 4100 -``` - -### Config schema - -`LatencyServiceBackendConfig` -(`switchyard/lib/config/latency_service_backend_config.py`): - -| Field | Default | Purpose | -|---|---|---| -| `latency_service_url` | `""` | Base URL of the Latency Service. | -| `endpoints` | `[]` | The endpoints to route across (see below). At least one is required. | -| `poll_interval_s` | `10.0` | How often the background poller refreshes health. | -| `poll_timeout_s` | `5.0` | Timeout for each health API call. | -| `max_retries` | `2` | Extra endpoints to try on failure (total attempts = `1 + max_retries`). | -| `credential_policy` | `"configured_endpoint"` | Credential precedence for upstream calls. The default uses each endpoint's configured `api_key`; set `"caller_override"` only for BYO-key deployments where caller headers should replace endpoint keys. | -| `session_affinity` | `false` | Pin each conversation to one endpoint after the first turn (sticky routing: keeps the upstream cache warm). See [Endpoint selection §3](#3-optional-session-affinity-sticky-routing). | -| `affinity_max_sessions` | `10_000` | LRU cap on the in-process pin map (L1) when `session_affinity` is on. | -| `affinity_store` | `"memory"` | Shared L2 pin store behind the L1. `"memory"` = per-process only; `"redis"` shares pins across workers/pods and persists them across pod churn (requires `session_affinity: true`, `affinity_store_url`, and the `affinity-redis` extra). | -| `affinity_store_url` | `None` | Connection URL for the shared store (e.g. `redis://host:6379/0`). Required when `affinity_store` is `"redis"`. | -| `affinity_store_ttl_seconds` | `3_600` | Expiry for a shared pin; refreshed on each write, so an active conversation slides its TTL. | -| `affinity_key_prefix` | `"swyd:pin:"` | Namespace prefix for shared-store keys. | -| `enable_stats` | `true` | Wire the stats processors + `/metrics` exposition. | - -Each `LatencyServiceEndpoint`: - -| Field | Required | Purpose | -|---|---|---| -| `model` | yes | **Latency Service lookup key**: must be unique across the endpoint list. Also the value stamped into `body["model"]` unless `upstream_model` is set. | -| `upstream_model` | no | Override for `body["model"]` sent upstream. Defaults to `model`. | -| `api_key` | no | API key for the backing LLM API. This is the default upstream credential. | -| `base_url` | no | Base URL for the backing LLM API (include `/v1`). | -| `timeout` | no | Per-request timeout in seconds. | - -### Why two `model` fields? - -The Latency Service registers each endpoint under an ID *it* picked (e.g. -`openrouter-primary/openai/gpt-4o`). The upstream gateway may expect the OpenRouter -model name for the same destination (e.g. `openai/gpt-4o`). So: - -- **`model`** is the Latency Service lookup key (and the metrics label). -- **`upstream_model`** is what gets stamped into `body["model"]` on the outbound call. - -When they're equal, `upstream_model` can be omitted. - -### Example - -Create an OpenRouter key at and export it as -`OPENROUTER_API_KEY` before running this example. - -```yaml -routes: - gpt-4o: - type: latency_service - latency_service_url: https://latency-service.example.com - poll_interval_s: 10.0 - poll_timeout_s: 5.0 - max_retries: 2 - endpoints: - - model: openrouter-primary/openai/gpt-4o # Latency Service endpoint_id - upstream_model: openai/gpt-4o # OpenRouter model name to call - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - - model: openrouter-backup/openai/gpt-4o - upstream_model: openai/gpt-4o - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} -``` - -A runnable version of this config, with extensive inline notes, lives at -[`experiments/latency_routing.yaml`](../../experiments/latency_routing.yaml). -A production-style Kubernetes manifest lives at -[`examples/k8s/latency-aware-deployment.yaml`](../../examples/k8s/latency-aware-deployment.yaml). - -### BYO-key (multi-tenant) mode - -By default, caller `Authorization: Bearer ` and `x-api-key` headers do not replace -the configured endpoint credentials. This keeps server-owned latency-service deployments -from accidentally forwarding a client placeholder or unrelated proxy credential upstream. - -For a multi-tenant BYO-key deployment, set `credential_policy: caller_override` on the -latency-service route. In that mode, a non-empty caller key is forwarded as the upstream -credential for that request, and the endpoint `api_key` is used only when the caller omits -the header. Credential precedence is **`Authorization` > `x-api-key` > endpoint -`api_key`**. - -## Response fidelity (Responses API) - -For an endpoint configured `request_type: openai_responses`, the backend calls the -upstream `/v1/responses` surface and returns the upstream payload **verbatim** in both -modes: - -- **Non-streaming** calls return the upstream's exact JSON body — the raw HTTP response - is used instead of round-tripping through the OpenAI SDK's typed model. -- **Streaming** calls forward the upstream's SSE frames as **verbatim strings** - (`RawSSEFrameStream` in `switchyard/lib/llm_client.py`): no typed-event parse happens, - so per-event provider extras, explicit nulls, event names, and comment keep-alives all - survive. Frames are byte-equivalent modulo CRLF → LF line normalization. Usage - accounting still works — the stats layers parse the frame's `data:` payload to find - `response.completed` usage without altering what flows to the client. - -When the inbound request is also Responses-format, the terminal translation -short-circuits (same format in and out), so the client receives the upstream payload -as-is. **Cross-format** calls (e.g. a chat client served by a Responses endpoint, or -vice versa) are *re-synthesized* by the translation engine and are inherently lossy — -fields the upstream never produced in the target format cannot be invented. For -full-fidelity Responses routing, configure every endpoint on the route with -`request_type: openai_responses`. - -### Field-fidelity matrix (Responses API) - -How each field class fares per path. "Exact" means byte-equivalent to the upstream -(streaming: modulo CRLF normalization); "synthesized" means the translation engine -rebuilds the payload from its neutral representation and only mapped fields survive. - -| Field class (examples) | Same-format, non-streaming | Same-format, streaming | Cross-format (either direction) | -|---|---|---|---| -| Standard scalars (`id`, `model`, `status`, `created_at`) | exact | exact | synthesized (`id`/`model` mapped; `created_at` → chat `created` defaults to `0`) | -| Sampling/config echoes (`temperature`, `top_p`, `store`, `text`) | exact | exact | dropped (no chat-format equivalent) | -| `reasoning` config + explicit-`null` fields | exact (nulls preserved) | exact (nulls preserved) | dropped | -| `usage` (incl. `*_tokens_details`) | exact | exact | token counts + reasoning detail mapped; cached-token detail drops | -| Output content (`output[]`, text deltas) | exact | exact | mapped (text/tool calls translate; unmappable block types drop) | -| Provider extras (unknown keys, e.g. Azure `content_filter_results`) | exact | exact | dropped | -| SSE event names / keep-alive comments | n/a | exact | re-framed to the target format's event contract | -| `metadata`, `previous_response_id` | exact | exact | dropped | - -Wire-level proofs live in -`tests/test_upstream_error_passthrough.py` -(`test_responses_body_returned_exactly_on_the_wire`, -`test_responses_stream_forwarded_verbatim_on_the_wire`, -`test_azure_flavored_responses_stream_preserved_on_the_wire`) against -OpenAI-shaped and Azure-shaped samples; validation against live production -captures remains a deployment-side step. - -## Failure-source annotation - -When a request fails, the error response carries two headers so clients and -observability can tell **which layer** the failure came from without parsing bodies: - -| Header | Values | Meaning | -|---|---|---| -| `x-switchyard-error-source` | `switchyard` \| `provider` | `provider`: an upstream LLM failure passed through (HTTP error, or the 500 rendered for a status-less network fault). `switchyard`: this proxy rejected or failed the request itself — credential-policy 401 (`missing_caller_api_key`), translation 400 (`invalid_value`), model-not-found 404, context-window 400, body validation 400, unexpected internal 500. | -| `x-switchyard-upstream-model` | model id | The model actually attempted upstream when the failure happened. Present only when a routing selection took place. | - -The provider error **body** still passes through verbatim — the annotation is -header-only, deliberately, so the transparent-proxy contract on bodies is preserved. -The same vocabulary appears as `switchyard.error_source` / `switchyard.upstream_model` -tags on failed `switchyard.upstream_attempt` spans and as `error_source` / -`upstream_model` fields on the per-event error log (below). - -Layering note: Switchyard can only distinguish *itself* from *its upstreams*. A proxy in -front of Switchyard (e.g. LiteLLM) should tag its own failures the same way and -propagate these headers from below. Mid-stream failures after HTTP 200 has been -committed cannot carry headers and are not annotated today. - -## Route-selection spend attribution - -For tokenomics reporting, a LiteLLM front proxy needs to tie its provider spend-log -rows back to the Switchyard logical route that selected them. The backend stamps -**every upstream attempt** with a spend-logs metadata request header (LiteLLM copies -its JSON value into the provider spend-log DB row): - -```text -x-litellm-spend-logs-metadata: {"router_model": "", - "router_strategy": "latency", - "router_selected_endpoint": "", - "router_selected_model": "", - "router_selected_provider": "", - "router_correlation_id": ""} -``` - -One correlation id is generated per client request and shared by every failover -attempt; the selection fields are re-stamped per attempt, so each provider row -records the endpoint it actually hit. `router_model` is the model the client asked -for, falling back to the configured `route_model` when the body carried no model. - -**Successful** responses then return the selection that served the request, so the -front proxy can enrich its own (parent) spend-log row with the same correlation id -the provider row received: - -| Header | Value | -|---|---| -| `x-switchyard-router-model` | `router_model` from the selection | -| `x-switchyard-selected-model` | `router_selected_model` | -| `x-switchyard-selected-provider` | `router_selected_provider` | -| `x-switchyard-router-correlation-id` | `router_correlation_id` | - -Streaming responses carry the same headers (the upstream call completes before the -SSE response is committed). Error responses carry the failure-source headers above; -the selection headers join them only when an upstream call already **succeeded** -before the failure (e.g. a response-translation error after a billed 200), so the -provider row's correlation id stays joinable — a request that never reached a -successful upstream claims no selection. Because the client controls `router_model`, -header values are re-validated as header material: a value that is not -latin-1-encodable or contains control characters is omitted rather than breaking -(or splitting) the response; the outbound JSON header is immune (`json.dumps` -escapes it) and still records the raw value. The cross-component contract is -`CTX_ROUTE_SELECTION` in `switchyard/lib/proxy_context.py` (written by the backend, -read by the endpoint layer via `switchyard/lib/endpoints/route_selection.py`); -wire-level proofs live in `tests/test_route_selection_headers.py`. - -## Observability - -When `enable_stats` is `true` (the default), `LatencyServiceProfileConfig` wires a -`StatsRequestProcessor` + `StatsResponseProcessor` pair sharing one `StatsAccumulator`. -Because `LatencyServiceLLMBackend` is a Python-only backend, profile assembly cannot wrap -it with the Rust-native `StatsLlmBackend`; instead it records success / error / -call-latency in-place into the same accumulator the response processor records token usage -into. It also sets `ctx.selected_model` and `ctx.backend_call_latency_ms` so per-endpoint -attribution and routing-overhead figures land correctly on `/metrics`. - -The backend publishes its in-memory health cache and poll-loop health to `/metrics`: - -| Metric | Type | Meaning | -|---|---|---| -| `switchyard_endpoint_status{model,status}` | gauge | Current verdict per endpoint; exactly one status row per model is `1`. | -| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample per endpoint. Absent when the upstream reported null. | -| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. | -| `switchyard_latency_service_poll_age_seconds` | gauge | Seconds since the last successful poll. Absent before the first success. | -| `switchyard_latency_service_polls_total` | counter | Total successful polls. | -| `switchyard_latency_service_poll_failures_total` | counter | Total failed polls; each resets the cache to `unknown`. | -| `switchyard_affinity_hits_total` | counter | Turns served by an existing session-affinity pin (warm reuse). **Emitted only when `session_affinity` is on.** | -| `switchyard_affinity_misses_total` | counter | First/unpinnable turns routed by the latency-aware picker while affinity was on. Reuse rate = `hits / (hits + misses)`. **Emitted only when `session_affinity` is on.** | -| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) store after an in-process miss — cross-worker/churn warm reuse. **Emitted only when a shared affinity store is configured.** | -| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. Alert on a sustained rate — the store is degraded while requests keep succeeding. **Emitted only when a shared affinity store is configured.** | - -See the [Metrics Reference](metrics_reference.md) for full label semantics, PromQL recipes -(traffic share per endpoint, retry-rescue rate), and a triage cheatsheet. - -### Expected routing overhead - -`switchyard_routing_overhead_ms{quantile}` is the source of truth for what Switchyard -itself adds to a request: per request it records `total_latency − backend_call_latency`, -which bundles format translation, endpoint selection, body mutation, response wrapping, -and retry wall time — everything except the upstream call. - -Reference figures, measured on an idle dev box over loopback (stub upstream + stub -Latency Service, non-streaming `/v1/chat/completions`, sequential; commit `0289a415`): - -| Figure | Value | -|---|---| -| `switchyard_routing_overhead_ms{quantile="0.5"}` | ~0.3 ms | -| `switchyard_routing_overhead_ms{quantile="0.99"}` | ~7 ms | -| Mean overhead (`_sum / _count`) | ~2 ms | -| Client-observed added wall time vs calling the upstream directly | ~2 ms mean | - -Expected order of magnitude is therefore **single-digit milliseconds**. When an -end-to-end comparison ("direct provider call" vs "through the stack") shows a much -larger delta — e.g. ~100 ms — the difference lives **outside** this metric: - -* the extra network hop and TLS handshake from the client to the Switchyard pod, -* any fronting layer (LiteLLM / inference-gateway) in front of Switchyard, -* per-pod queueing under concurrency — a single event loop serializes CPU work, so - client wall time grows with in-flight load while per-request overhead does not - (at loopback concurrency 10 the same box shows ~22 ms client p50 with an unchanged - overhead summary), -* upstream time-to-first-token variance between the two measurement runs. - -Attribute before optimizing: read the deployment's own summary first. - -```promql -# Median / p99 Switchyard-internal overhead -switchyard_routing_overhead_ms{quantile="0.5"} -switchyard_routing_overhead_ms{quantile="0.99"} - -# Mean over a window -rate(switchyard_routing_overhead_ms_sum[5m]) / rate(switchyard_routing_overhead_ms_count[5m]) -``` - -Two caveats. For streaming responses `switchyard_total_latency_ms` (and hence the -overhead summary) measures the **full turn**, not time-to-first-token — TTFT is what -interactive users feel, so don't read a long generation as router overhead. And the -summary is per-process: in a multi-pod deployment each pod reports its own. - -The repeatable harness is the CI perf benchmark (`.github/workflows/perf.yml`): aiperf -against a no-op route measures the full server path with zero backend cost, non-streaming -and streaming, at configurable concurrency. - -### Per-event error log (Loki) - -`/metrics` answers *how many* of each error code, but Prometheus stores aggregates. It -carries no per-event timestamps. For audit, replay, or a "what failed and when" panel, -each failed upstream attempt also emits one structured JSON line on the -`switchyard.upstream_errors` logger -(`switchyard/lib/endpoints/upstream_error_log.py`): - -```json -{"event":"upstream_attempt_failed","timestamp":"2026-05-29T12:00:00+00:00","model":"openai/gpt-5.5","upstream_model":"gpt-5.5","attempt":1,"status_code":429,"code":"429","outcome":"retryable_error","error_source":"provider","error_type":"APIStatusError","error":"..."} -``` - -`code` and `outcome` mirror the labels on `switchyard_upstream_attempts_total`, so the log -joins cleanly to the counter; `status_code` is `null` (and `code="none"`) for a non-HTTP -failure. Because the deployment uses plain-text logging, the message *is* the JSON object. -A Loki query parses it with `| json` and zero formatter config: - -```logql -{app="switchyard"} | json | event="upstream_attempt_failed" -``` - -`is_ready()` returns `True` once the poller has completed at least one successful poll, -which makes it useful as a readiness gate during startup. - -## Testing the router - -Three tiers, cheapest first. The `switchyard-latency-router-e2e` skill carries the -runnable commands and helper scripts for each. - -1. **Offline unit suite** (no keys): `tests/test_latency_service_*.py`, - `tests/test_session_key.py`, `tests/test_outcome_metrics.py`, and - `tests/test_upstream_error_passthrough.py`: full coverage of selection, tiering, - inverse-latency weighting, retry/fail-fast, session affinity, the health poller, - metrics, and spans. -2. **In-repo integration** (`tests/e2e/test_latency_service_llm_backend.py`, marked - `integration`): an in-process mock Latency Service feeds verdicts while the LLM - calls hit OpenRouter. Requires `OPENROUTER_API_KEY` (skips without it). -3. **Real-world**: point a real backend at a mock or live Latency Service and confirm - it routes. A mock LS pointed at `openrouter.ai` proves the - poll → tier → weighting path with no API key — a keyless dispatch returns `401`, - which confirms traffic reached the host. - -## Source map - -| File | Responsibility | -|---|---| -| `switchyard/lib/backends/latency_service_llm_backend.py` | Selection, session-affinity pins, retry, stats/metrics emission, per-attempt error log. | -| `switchyard/lib/session_key.py` | `session_key_from_body()`: stable per-conversation key for session affinity. | -| `switchyard/lib/session_affinity.py` | `SessionAffinity`: L1 (in-process LRU) + optional best-effort L2 pin store. | -| `switchyard/lib/affinity_pin_store.py` | `AffinityPinStore`: async protocol for a shared/persistent L2 store. | -| `switchyard/lib/redis_pin_store.py` | `RedisPinStore`: Redis-backed shared L2 (lazy `redis`, `affinity-redis` extra). | -| `switchyard/lib/endpoints/upstream_error_log.py` | Structured per-attempt failure log (`switchyard.upstream_errors`, Loki). | -| `switchyard/lib/backends/health_poller.py` | `EndpointHealthStatus`, `EndpointHealth`, `HealthPoller` daemon. | -| `switchyard/lib/config/latency_service_backend_config.py` | `LatencyServiceEndpoint`, `LatencyServiceBackendConfig`. | -| `switchyard/lib/profiles/latency_service.py` | `LatencyServiceProfileConfig`: profile assembly + stats wiring. | -| `tests/test_latency_service_llm_backend.py` | Selection, retry, tier-preference tests. | -| `tests/test_latency_service_health_metrics.py` | Poller and metrics tests. | diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index cdba2a31..e8097479 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -12,7 +12,7 @@ a drop-in scrape config and starter alert rules. | Content-Type | `text/plain; version=0.0.4; charset=utf-8` | | Format | Prometheus text format 0.0.4 | | Auth | None. Designed as a public scrape endpoint | -| Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) | +| Default scrape interval | 15s | `GET /metrics` is served by the Python route-bundle server started with `switchyard --routing-profiles PATH -- serve`. @@ -29,12 +29,11 @@ A JSON variant of the same underlying data lives at `GET /v1/stats`, with ## Per-endpoint counters -The `model` label is the latency-service endpoint id (`openai/gpt-5.5`, +The `model` label is the configured endpoint id (`openai/gpt-5.5`, `azure_openai/gpt-5.5`, etc.). The `tier` label is optional: present only when a routing factory -supplied one. The `random_routing` factory emits `tier="strong"|"weak"`; -the `latency_service` factory does not. +supplied one. The `random_routing` factory emits `tier="strong"|"weak"`. | Metric | Type | Meaning | |---|---|---| @@ -64,44 +63,6 @@ Each summary emits `{quantile="0.5"}` and `{quantile="0.99"}` rows plus Healthy traffic typically sits at p50 ≈ 0.4 ms, p99 ≈ 0.6 ms. -## Latency Service state (gauges: latency-service chains only) - -Published from the in-memory health cache the `LatencyServiceLLMBackend` -maintains, refreshed on each successful poll of the Latency Service. - -| Metric | Type | Meaning | -|---|---|---| -| `switchyard_endpoint_status{model,status}` | gauge | Current Latency Service verdict per endpoint. `status` is one of `healthy`, `degraded`, `unknown`. Exactly one row per `model` is `1`; the rest are `0`, so `sum by (status)` gives a clean count of endpoints in each state. | -| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample the Latency Service reported for this endpoint. **Absent** when the upstream reported `null`, and absence is meaningful. | - -## Latency Service poll health (no labels: latency-service chains only) - -| Metric | Type | Meaning | -|---|---|---| -| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. `0` means the next routing decisions are based on the cache-reset-to-unknown fallback state. | -| `switchyard_latency_service_poll_age_seconds` | gauge | Monotonic seconds since the last successful poll. **Absent** before the first success; combined with `poll_ok=0`, this distinguishes "never polled" from "polled but recently failed". | -| `switchyard_latency_service_polls_total` | counter | Total successful polls since process start. | -| `switchyard_latency_service_poll_failures_total` | counter | Total failed poll attempts. Each failure resets every endpoint in the cache to `unknown`. | - -## Session affinity (no labels: latency-service chains with `session_affinity` on) - -These counters are **absent unless `session_affinity: true`**, so the metric surface stays clean for the default per-turn-routing case. - -| Metric | Type | Meaning | -|---|---|---| -| `switchyard_affinity_hits_total` | counter | Conversation turns served by an existing session-affinity pin, meaning the upstream prompt/KV cache was reused. Counted once per request (first attempt only), so failover retries don't inflate it. | -| `switchyard_affinity_misses_total` | counter | First turns of a conversation, or turns whose pin was broken by a health verdict, routed by the latency-aware picker. | -| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) affinity store after an in-process (L1) miss — cross-worker/churn warm reuse. **Also requires a shared store** (`affinity_store: "redis"`). | -| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. A sustained rate means the store is degraded while requests keep succeeding — this is the alerting signal. While the breaker is open, only recovery probes fail, so the rate drops to ~1 per cooldown per pod. **Also requires a shared store.** | -| `switchyard_affinity_l2_breaker_open` | gauge | `1` while the shared-store circuit breaker is open (operations skipped without a network attempt after 3 consecutive failures; one probe per 10 s cooldown), `0` when closed. The unambiguous store-outage signal. **Also requires a shared store.** | - -Warm-reuse rate (the fraction of turns that hit a warm endpoint): - -```promql -rate(switchyard_affinity_hits_total[5m]) - / (rate(switchyard_affinity_hits_total[5m]) + rate(switchyard_affinity_misses_total[5m])) -``` - ## Outcome counters for error-rate ratios The `outcome` label takes exactly three values: @@ -115,7 +76,6 @@ The `outcome` label takes exactly three values: | `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. | | `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). | | `switchyard_router_retry_recovered_total` | counter | Requests whose first upstream attempt failed but a subsequent attempt succeeded. This is direct evidence the routing logic rescued the request. | -| `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` | counter | Latency-service chains only: the per-model complement of `switchyard_upstream_attempts_total` (which stays model-free). `requested_model` is the client-supplied model bounded to a config-derived id — the route id (`route_model`, e.g. `nvidia/switchyard/gpt-5.4`) or a configured endpoint id — with `other` as the fallback sentinel. `upstream_model` is the selected endpoint's upstream name. Answers "for route X, which upstream failed or succeeded". | ### The `code` label on `switchyard_upstream_attempts_total` @@ -154,7 +114,7 @@ error_rate_reduction = direct_error_rate − router_error_rate # Live Endpoint Routing rescue rate rate(switchyard_router_retry_recovered_total[5m]) -# Traffic share per endpoint (sanity-check inverse-latency weighting) +# Traffic share per endpoint sum by (model) (rate(switchyard_requests_total[5m])) / ignoring(model) group_left sum(rate(switchyard_requests_total[5m])) @@ -165,12 +125,6 @@ sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) / ignoring(code) group_left sum (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) - -# Per-upstream outcome breakdown for one route — "for nvidia/switchyard/gpt-5.4, -# which upstream provider failed or succeeded?" (latency-service chains) -sum by (upstream_model, outcome, code) ( - rate(switchyard_latency_upstream_attempts_total{requested_model="nvidia/switchyard/gpt-5.4"}[5m]) -) ``` > **Note:** because `switchyard_upstream_attempts_total` now carries the @@ -189,18 +143,10 @@ into label space. | Label | Values | Where | |---|---|---| | `model` | One per configured endpoint, typically 2–6 per deployment. | All per-endpoint metrics. | -| `status` | Exactly 3: `healthy`, `degraded`, `unknown`. | `switchyard_endpoint_status` | | `outcome` | Exactly 3: `success`, `retryable_error`, `other_error`. | Outcome counters | | `code` | Bounded: the known-code allowlist (`200`, `400`, `401`, `403`, `404`, `408`, `409`, `422`, `429`, `500`, `502`, `503`, `504`), plus `none` and the per-class buckets `1xx`/`2xx`/`3xx`/`4xx`/`5xx`/`other`. About 20 values max. | `switchyard_upstream_attempts_total` | -| `requested_model` | Bounded to config-derived ids: the route id (`route_model`) plus configured endpoint ids, else the `other` sentinel. | `switchyard_latency_upstream_attempts_total` | -| `upstream_model` | One per configured endpoint (the endpoint's upstream name). | `switchyard_latency_upstream_attempts_total` | | `quantile` | Exactly 2: `0.5`, `0.99`. | All summaries | -| `tier` | Small enumerated set, optional. Not emitted by latency-service chains. | Per-endpoint counters/summaries on routing chains that supply it | - -For a latency-service deployment with `N` endpoints, expect roughly -`11N + 17` series at startup (five `code` series are seeded), growing by at -most a dozen more as additional upstream status codes appear. That is about 39 -series for two endpoints. Well within single-Prometheus capacity. +| `tier` | Small enumerated set, optional. | Per-endpoint counters/summaries on routing chains that supply it | ## Triage cheatsheet @@ -208,7 +154,5 @@ series for two endpoints. Well within single-Prometheus capacity. |---|---| | `model=""` rows appear | The per-endpoint attribution wiring regressed. `ctx.selected_model` is not being set by the backend. | | All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. | -| `switchyard_latency_service_poll_ok` stuck at `0` | DNS / network unreachability to the Latency Service, **or** a schema mismatch on the LS response. Check server logs for `"Health poller: failed to reach Latency Service"`. | -| `switchyard_endpoint_last_latency_ms{...}` absent for an endpoint | The Latency Service reported `last_latency_ms: null`, or the endpoint was not returned in the most recent poll response. | | `switchyard_routing_overhead_ms_count` stuck at `0` | The backend is not publishing `ctx.backend_call_latency_ms` (regression of the Python-backend routing-overhead wiring). | -| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky (cross-check `switchyard_endpoint_status`), or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). | +| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky, or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). | diff --git a/examples/latency_service_caller_required.yaml b/examples/latency_service_caller_required.yaml deleted file mode 100644 index 3d6f78cf..00000000 --- a/examples/latency_service_caller_required.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# ============================================================================= -# Latency-aware routing with per-caller key attribution (BYO key, required) -# ============================================================================= -# -# Use this when every upstream call must be billed to the *caller's* own API key -# (e.g. per-user spend attribution in multi-tenant gateways) and the service-level key must -# NEVER authenticate caller inference. -# -# What `credential_policy: caller_required` does: -# * The caller key is read from the `x-switchyard-api-key` request header -# (preferred), then `Authorization: Bearer` / `x-api-key`. -# * Present -> forwarded upstream as `Authorization: Bearer `; spend is -# attributed to that caller. -# * Missing -> the request is rejected with HTTP 401 BEFORE any upstream call; -# the configured endpoint key is never used. -# (Applies to every ingress path: /v1/chat/completions, /v1/responses, /v1/messages.) -# -# Why a dedicated header? A proxy in front of Switchyard (e.g. LiteLLM) consumes -# the `Authorization` header for its own auth and strips it before the upstream -# call. The custom `x-switchyard-api-key` header passes through untouched. -# -# How clients send the header: -# * Codex CLI: env_http_headers / http_headers -# * OpenCode: provider options.headers -# * OpenAI / Anthropic SDK: extra / default headers -# * Claude Code: limited (can't easily send two distinct auth values) -# -- may need a wrapper/gateway unless one key serves both. -# -# Rollout order (IMPORTANT -- avoid 401s): -# 1. Deploy this build first with the route still on its current policy -# (`configured_endpoint` default, or `caller_override`) -- no behavior change. -# 2. Update clients to send `x-switchyard-api-key` (harmless under those policies). -# 3. Only after clients are confirmed sending it, switch to `caller_required`. -# -# Run: -# uv run switchyard serve \ -# --routing-profiles examples/latency_service_caller_required.yaml --port 4100 -# -# # Caller key present -> billed to that key: -# curl -s -X POST http://127.0.0.1:4100/v1/chat/completions \ -# -H 'Content-Type: application/json' \ -# -H "x-switchyard-api-key: $USER_UPSTREAM_API_KEY" \ -# -d '{"model":"my-model","messages":[{"role":"user","content":"hi"}]}' -# -# # Header omitted -> HTTP 401: -# # "missing caller API key; supply it via the x-switchyard-api-key header" -# # (the service key is never used for caller inference). -# -# Substitute your own Latency Service URL, gateway base URL, and model IDs below. -# ============================================================================= - -routes: - my-model: - type: latency_service - latency_service_url: https://your-latency-service.example.com - # Require a caller key; 401 when missing; never fall back to a service key. - credential_policy: caller_required - poll_interval_s: 10.0 - max_retries: 2 - endpoints: - # No `api_key:` on the endpoints: `caller_required` forwards the caller's - # key and never falls back, so there is no service credential to leak. - - model: primary/my-model # Latency Service endpoint_id - upstream_model: provider/my-model # model name the upstream expects - base_url: https://your-inference-gateway.example.com/v1 - # request_type: openai_responses # set for Responses-only upstreams - # (e.g. gpt-5.1-codex-mini) - - model: backup/my-model - upstream_model: provider/my-model - base_url: https://your-inference-gateway.example.com/v1 diff --git a/examples/prometheus/README.md b/examples/prometheus/README.md index 77311cab..ceea201f 100644 --- a/examples/prometheus/README.md +++ b/examples/prometheus/README.md @@ -25,12 +25,9 @@ Prometheus configuration (adjust the `targets:` list to point at your Switchyard pods or hosts). The endpoint is plain `GET /metrics` over HTTP, no auth. -If you're running Switchyard on Kubernetes via -[`examples/k8s/latency-aware-deployment.yaml`](../k8s/latency-aware-deployment.yaml), -the service-level DNS is -`switchyard-latency.llm-routing.svc.cluster.local`. The example scrape -job uses static targets; a `kubernetes_sd_configs` variant is a one-step -swap for cluster-native discovery. +If you're running Switchyard on Kubernetes, the example scrape job uses +static targets; a `kubernetes_sd_configs` variant is a one-step swap for +cluster-native discovery. ### 2. Load the alert rules @@ -61,8 +58,7 @@ series: ``` You should see the families documented in `metrics_reference.md` — -top-line gauges, per-endpoint counters/summaries, Latency Service state, -poll health, and outcome counters. +top-line gauges, per-endpoint counters/summaries, and outcome counters. ## Alert tuning notes @@ -71,8 +67,6 @@ conservative — tuned to avoid pages during routine scrape-config rolls or single-poll blips. Review them against your SLOs before enabling notifications: -* `LatencyServiceUnreachable` fires after 60s of `poll_ok=0`. Bring this - down if your routing logic should react faster than one poll cycle. * `RouterOverheadHigh` is set at the success-criterion threshold (p99 > 1 ms for 5 m). Retry-heavy workloads inflate this — cross-check `switchyard_upstream_attempts_total{outcome="retryable_error"}` rate diff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.yml index 229a0831..cbecb672 100644 --- a/examples/prometheus/prometheus.yml +++ b/examples/prometheus/prometheus.yml @@ -16,16 +16,11 @@ scrape_configs: - job_name: switchyard metrics_path: /metrics scheme: http - # Default scrape every 15s — the Latency Service poll cycle defaults - # to 10s, so finer scrape resolution captures no additional state. scrape_interval: 15s static_configs: - # Replace with your Switchyard host(s) and port(s). On Kubernetes - # with the manifest at examples/k8s/latency-aware-deployment.yaml - # the in-cluster DNS resolves to - # switchyard-latency.llm-routing.svc.cluster.local on port 80 - # targeting container port 4100. Swap this static_configs block - # for kubernetes_sd_configs in a cluster-native deployment. + # Replace with your Switchyard host(s) and port(s). Swap this + # static_configs block for kubernetes_sd_configs in a + # cluster-native deployment. - targets: - switchyard.example.internal:4100 diff --git a/examples/prometheus/switchyard.rules.yaml b/examples/prometheus/switchyard.rules.yaml index dfc8378f..3b340005 100644 --- a/examples/prometheus/switchyard.rules.yaml +++ b/examples/prometheus/switchyard.rules.yaml @@ -9,60 +9,6 @@ # examples/prometheus/README.md before wiring to a pager. groups: - - name: switchyard.latency_service.health - rules: - - alert: LatencyServiceUnreachable - expr: switchyard_latency_service_poll_ok == 0 - for: 60s - labels: - severity: warning - component: switchyard - annotations: - summary: "Switchyard cannot reach the Latency Service" - description: | - switchyard_latency_service_poll_ok has been 0 for at least - 60s. Routing has fallen back to uniform random across all - endpoints (every verdict reset to unknown). Check the - Switchyard pod logs for "Health poller: failed to reach - Latency Service" and confirm DNS / network reachability to - the upstream LS. - - - alert: LatencyServicePollStale - # No `for:` — the age gauge only grows while polls keep failing, so - # the >60s threshold is already debounced; a `for:` would only delay - # the alert past the documented 60s staleness budget. - expr: switchyard_latency_service_poll_age_seconds > 60 - labels: - severity: warning - component: switchyard - annotations: - summary: "Switchyard health verdicts are stale" - description: | - The most recent successful Latency Service poll was more - than 60 s ago. Routing decisions are still using cached - verdicts but they are increasingly out of date. Compare - switchyard_latency_service_poll_failures_total against - switchyard_latency_service_polls_total to see whether the - poller is failing intermittently or hanging entirely. - - - alert: LatencyServiceEndpointFlapping - expr: | - sum by (model) ( - changes(switchyard_endpoint_status{status="healthy"}[10m]) - ) > 5 - for: 0s - labels: - severity: info - component: switchyard - annotations: - summary: "Endpoint {{ $labels.model }} is flapping" - description: | - switchyard_endpoint_status{model="{{ $labels.model }}"} has - transitioned in or out of "healthy" more than 5 times in - the last 10 minutes. Likely a real upstream instability — - cross-check switchyard_endpoint_last_latency_ms for that - model and the LS upstream's own dashboards. - - name: switchyard.routing.evidence rules: - alert: RouterNotRescuing @@ -84,7 +30,7 @@ groups: / 504) but no request has been rescued by retry in the same 5 m window. Likely cause: every configured endpoint is degraded simultaneously, so the retry loop has nowhere to - steer. Check switchyard_endpoint_status by model. + steer. - alert: RouterOverheadHigh expr: switchyard_routing_overhead_ms{quantile="0.99"} > 1.0 @@ -127,6 +73,5 @@ groups: clients is at least as high as the per-attempt upstream retryable_error rate. The router is not absorbing failures via retry — either retries are exhausting or every - endpoint is producing the same failure mode. Look at - switchyard_endpoint_status and the per-model breakdown of - switchyard_errors_total{model}. + endpoint is producing the same failure mode. Look at the + per-model breakdown of switchyard_errors_total{model}. diff --git a/switchyard/__init__.py b/switchyard/__init__.py index 34b5c35d..5fc93f89 100644 --- a/switchyard/__init__.py +++ b/switchyard/__init__.py @@ -13,10 +13,6 @@ from switchyard.lib.backends import ( AnthropicNativeBackend, - EndpointHealth, - EndpointHealthStatus, - HealthPoller, - LatencyServiceLLMBackend, OpenAiNativeBackend, ) from switchyard.lib.backends.llm_target import ( @@ -42,8 +38,6 @@ ) from switchyard.lib.config import ( IntakeSinkConfig, - LatencyServiceBackendConfig, - LatencyServiceEndpoint, ) from switchyard.lib.processors.intake_payload_builder import IntakePayloadBuilder from switchyard.lib.processors.intake_request_processor import IntakeRequestProcessor @@ -58,7 +52,6 @@ DeterministicRoutingProfileConfig, EscalationRouterConfig, EscalationRouterProfileConfig, - LatencyServiceProfileConfig, NoopProfile, NoopProfileConfig, PassthroughProfileConfig, @@ -156,7 +149,6 @@ def __getattr__(name: str) -> Any: "DeterministicRoutingPresets", "EscalationRouterConfig", "EscalationRouterProfileConfig", - "LatencyServiceProfileConfig", "PassthroughProfileConfig", "Profile", "ProfileConfig", @@ -182,13 +174,6 @@ def __getattr__(name: str) -> Any: "build_switchyard_app", # Route dispatch table "RouteTable", - # Latency Service usage case - "EndpointHealth", - "EndpointHealthStatus", - "HealthPoller", - "LatencyServiceBackendConfig", - "LatencyServiceEndpoint", - "LatencyServiceLLMBackend", "IntakePayloadBuilder", "IntakeRequestProcessor", "IntakeResponseProcessor", diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py index b79a5164..435e1738 100644 --- a/switchyard/cli/launch_command.py +++ b/switchyard/cli/launch_command.py @@ -490,7 +490,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None: * ``--routing-profiles FILE`` (global flag) — YAML-driven multi-chain table. ``--model`` is optional; falls back to the first YAML route. - Random / latency-aware routing live in the YAML schema. + Random and other multi-chain routing live in the YAML schema. """ if getattr(args, "startup_timing", False): startup_timing.enable() diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.py index 148764c1..c71eedc3 100644 --- a/switchyard/cli/launchers/claude_code_launcher.py +++ b/switchyard/cli/launchers/claude_code_launcher.py @@ -298,8 +298,8 @@ def _run_claude_with_switchyard( Takes a pre-built :class:`Switchyard` and a display-only model name. The caller decides what's in the chain (single-model - passthrough, random routing across a preset, latency-service - pool, …) — this function owns only the uvicorn-in-a-thread + + passthrough, random routing across a preset, …) — this function + owns only the uvicorn-in-a-thread + ``claude`` supervision + env-var injection boilerplate that is identical across those modes. diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 727c648b..a465a390 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -34,13 +34,11 @@ from switchyard.cli.model_catalog.model_discovery import fetch_model_ids from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target -from switchyard.lib.config import LatencyServiceBackendConfig, LatencyServiceEndpoint from switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARS from switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIES from switchyard.lib.profiles import ( DeterministicRoutingProfileConfig, EscalationRouterProfileConfig, - LatencyServiceProfileConfig, ProfileSwitchyard, StageRouterProfileConfig, ) @@ -193,32 +191,6 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: | frozenset({"defaults", "enable_stats"}) | _PASSTHROUGH_SETTING_KEYS ) -_LATENCY_ENDPOINT_DEFAULT_KEYS = frozenset({ - "api_key", - "base_url", - "timeout", - "timeout_secs", -}) -_LATENCY_ENDPOINT_KEYS = _LATENCY_ENDPOINT_DEFAULT_KEYS | frozenset( - {"model", "upstream_model", "request_type"} -) -_LATENCY_SERVICE_ROUTE_KEYS = _ROUTE_METADATA_KEYS | frozenset({ - "defaults", - "endpoints", - "latency_service_url", - "latency_url", - "poll_interval_s", - "poll_timeout_s", - "max_retries", - "credential_policy", - "enable_stats", - "session_affinity", - "affinity_max_sessions", - "affinity_store", - "affinity_store_url", - "affinity_store_ttl_seconds", - "affinity_key_prefix", -}) | _LATENCY_ENDPOINT_DEFAULT_KEYS _NOOP_ROUTE_KEYS = _ROUTE_METADATA_KEYS _DETERMINISTIC_ROUTE_KEYS = ( _ROUTE_METADATA_KEYS @@ -318,7 +290,6 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: _ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = { "model": _MODEL_ROUTE_KEYS, "random_routing": _RANDOM_ROUTING_ROUTE_KEYS, - "latency_service": _LATENCY_SERVICE_ROUTE_KEYS, "noop": _NOOP_ROUTE_KEYS, "passthrough": _PASSTHROUGH_ROUTE_KEYS, "deterministic": _DETERMINISTIC_ROUTE_KEYS, @@ -328,7 +299,6 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: _DEFAULT_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = { "model": _TARGET_DEFAULT_KEYS, "random_routing": _TARGET_DEFAULT_KEYS, - "latency_service": _LATENCY_ENDPOINT_DEFAULT_KEYS, "passthrough": _PASSTHROUGH_SETTING_KEYS, "noop": frozenset(), "deterministic": _TARGET_DEFAULT_KEYS, @@ -869,16 +839,6 @@ def _build_route_chain( extra_response_processors=extra_response_processors, ) - if route_type == "latency_service": - return _latency_service_switchyard( - model_id, - route, - target_defaults, - stats=stats, - extra_request_processors=pre_routing_request_processors, - extra_response_processors=extra_response_processors, - ) - if route_type == "noop": from switchyard.lib.profiles.noop import NoopProfileConfig @@ -1305,62 +1265,6 @@ def _passthrough_target( ) -def _latency_service_switchyard( - model_id: str, - route: Mapping[str, object], - target_defaults: Mapping[str, object], - stats: StatsAccumulator, - extra_request_processors: Sequence[Any] = (), - extra_response_processors: Sequence[Any] = (), -) -> ChainRuntime: - endpoints_raw = _require_sequence(route.get("endpoints"), "latency_service.endpoints") - endpoints = [ - _latency_endpoint(value, target_defaults, index=index) - for index, value in enumerate(endpoints_raw) - ] - enable_stats = _optional_bool(route.get("enable_stats"), default=True) - credential_policy = ( - _optional_str(route["credential_policy"]) - if "credential_policy" in route - else "configured_endpoint" - ) - config = LatencyServiceBackendConfig.model_validate({ - "latency_service_url": _required_str( - route.get("latency_service_url", route.get("latency_url")), - "latency_service.latency_service_url", - ), - "endpoints": endpoints, - # The YAML route key is what clients send as ``model``; hand it to the - # backend so the per-model metric can attribute route-key traffic. - "route_model": model_id, - "poll_interval_s": _optional_float(route.get("poll_interval_s"), default=10.0), - "poll_timeout_s": _optional_float(route.get("poll_timeout_s"), default=5.0), - "max_retries": _optional_int(route.get("max_retries"), default=2), - "credential_policy": credential_policy, - "enable_stats": enable_stats, - "session_affinity": _optional_bool(route.get("session_affinity"), default=False), - "affinity_max_sessions": _optional_int( - route.get("affinity_max_sessions"), default=10_000 - ), - "affinity_store": _optional_str(route.get("affinity_store")) or "memory", - "affinity_store_url": _optional_str(route.get("affinity_store_url")), - "affinity_store_ttl_seconds": _optional_int( - route.get("affinity_store_ttl_seconds"), default=3_600 - ), - "affinity_key_prefix": _optional_str(route.get("affinity_key_prefix")) or "swyd:pin:", - }) - return ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components( - stats_accumulator=stats, - enable_stats=config.enable_stats, - pre_request_processors=extra_request_processors, - response_processors=extra_response_processors, - ) - ) - - def _route_config( route: Mapping[str, object], target_defaults: Mapping[str, object], @@ -1438,33 +1342,6 @@ def _target_mapping( return target -def _latency_endpoint( - raw: object, - defaults: Mapping[str, object], - index: int, -) -> LatencyServiceEndpoint: - data = dict(defaults) - where = f"latency_service.endpoints[{index}]" - if isinstance(raw, str): - data["model"] = raw - elif isinstance(raw, Mapping): - endpoint_mapping = _require_mapping(raw, where) - _validate_allowed_keys(endpoint_mapping, _LATENCY_ENDPOINT_KEYS, where) - data.update(endpoint_mapping) - else: - raise RouteBundleConfigError(f"{where} must be a string or mapping") - - if "timeout" not in data and "timeout_secs" in data: - data["timeout"] = data["timeout_secs"] - - endpoint_data = { - key: data[key] - for key in ("model", "upstream_model", "api_key", "base_url", "timeout", "request_type") - if key in data - } - return LatencyServiceEndpoint.model_validate(endpoint_data) - - def _target_defaults( bundle_defaults: Mapping[str, object], route: Mapping[str, object], @@ -1492,8 +1369,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str: if raw_type is None: if not route: return "noop" - if "latency_service_url" in route or "latency_url" in route: - return "latency_service" if "strong" in route and "weak" in route: return "random_routing" if "target" in route or "model" in route: @@ -1512,8 +1387,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str: "target": "model", "random": "random_routing", "random_routing": "random_routing", - "latency": "latency_service", - "latency_service": "latency_service", "noop": "noop", "no_op": "noop", "passthrough": "passthrough", @@ -1615,12 +1488,6 @@ def _optional_mapping(value: object, where: str) -> dict[str, object]: return _require_mapping(value, where) -def _require_sequence(value: object, where: str) -> Sequence[object]: - if not isinstance(value, list): - raise RouteBundleConfigError(f"{where} must be a list") - return value - - def _required_str(value: object, where: str) -> str: if not isinstance(value, str) or not value: raise RouteBundleConfigError(f"{where} must be a non-empty string") diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py index 59c4de40..b0fdaf78 100644 --- a/switchyard/cli/switchyard_cli.py +++ b/switchyard/cli/switchyard_cli.py @@ -44,7 +44,7 @@ switchyard --routing-profiles profiles.yaml -- launch claude -- --no-auto-approve Routing policies that used to be top-level CLI verbs (``passthrough``, -``random-routing``, ``latency-service``) and launcher flags +``random-routing``) and launcher flags (``--routing``, ``--weak-model``, ``--strong-probability``, ``--preset``) are expressed in routing-profile YAML files. ``serve`` and launchers still parse the YAML into profile-backed runtimes. diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.py index 1654bfeb..39b76689 100644 --- a/switchyard/lib/affinity_pin_store.py +++ b/switchyard/lib/affinity_pin_store.py @@ -20,8 +20,8 @@ class AffinityPinStore(Protocol): """Shared, out-of-process store for conversation→decision pins. - The pinned value is opaque to the store (an endpoint id for the latency - backend, a tier label for the classifier router). Implementations are + The pinned value is opaque to the store (a tier label for the classifier + or escalation router). Implementations are called on the request path, so they should be fast, and :class:`SessionAffinity` treats them as **best-effort**: an error or timeout must never break request routing (the caller falls back to L1 / unpinned). diff --git a/switchyard/lib/backends/__init__.py b/switchyard/lib/backends/__init__.py index 6251d02d..53c20ae6 100644 --- a/switchyard/lib/backends/__init__.py +++ b/switchyard/lib/backends/__init__.py @@ -11,14 +11,6 @@ BackendFormatResolution, BackendFormatResolver, ) -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) from switchyard.lib.backends.stats_llm_backend import ( StatsLlmBackend, ) @@ -32,10 +24,6 @@ "AnthropicNativeBackend", "BackendFormatResolution", "BackendFormatResolver", - "EndpointHealth", - "EndpointHealthStatus", - "HealthPoller", - "LatencyServiceLLMBackend", "OpenAiPassthroughBackend", "OpenAiNativeBackend", "StatsLlmBackend", diff --git a/switchyard/lib/backends/health_poller.py b/switchyard/lib/backends/health_poller.py deleted file mode 100644 index 5393e568..00000000 --- a/switchyard/lib/backends/health_poller.py +++ /dev/null @@ -1,181 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Health enum and background poller for the Latency Service usage case. - -The Latency Service exposes a bulk health endpoint that returns one of -three states per endpoint. :class:`EndpointHealthStatus` is the shared -contract between Switchyard and the Latency Service — both sides must -agree on these string values. - -:class:`HealthPoller` is a daemon thread that periodically pulls fresh -verdicts from the Latency Service and writes them into an in-memory -cache shared with :class:`LatencyServiceLLMBackend`. The backend's -request hot path only ever *reads* the cache (under a lock), so health -polling adds zero per-request latency. - -Each cache entry is an :class:`EndpointHealth` snapshot carrying both -the discrete ``status`` and the most recent ``last_latency_ms`` sample -reported by the Latency Service. The status drives tier selection -(HEALTHY > UNKNOWN > DEGRADED); the latency drives inverse-latency -weighted selection *within* a tier. - -Design properties: - -- **Daemon thread, not asyncio task.** Simple lifecycle: starts when the - backend is constructed, dies with the process. The sync ``httpx.Client`` - used inside the thread avoids event-loop contention for other work. -- **Fallback to UNKNOWN on poll failure.** If the Latency Service is - unreachable, all endpoints reset to ``UNKNOWN`` so the backend falls - back to random routing rather than acting on stale health data. -- **Graceful stop via ``stop()``.** Tests and ``shutdown()`` paths set - ``_stop_event`` to break the polling loop promptly. -""" - -from __future__ import annotations - -import enum -import logging -import threading -import time -from typing import NamedTuple - -import httpx - -logger = logging.getLogger(__name__) - - -class EndpointHealthStatus(enum.Enum): - """Shared contract between Switchyard and the Latency Service. - - Three states are sufficient for routing decisions — the Latency - Service may track finer-grained failure reasons internally for - observability but exposes only these values in the health API. - """ - - HEALTHY = "healthy" - DEGRADED = "degraded" - UNKNOWN = "unknown" - - -class EndpointHealth(NamedTuple): - """Snapshot of one endpoint's health, written by :class:`HealthPoller`. - - ``status`` drives tier-based routing (HEALTHY > UNKNOWN > DEGRADED); - ``last_latency_ms`` — when present — drives inverse-latency weighted - selection within whichever tier is chosen. ``last_latency_ms`` is - ``None`` before the first successful poll and whenever the Latency - Service reports it as null. - """ - - status: EndpointHealthStatus - last_latency_ms: float | None = None - - -class HealthPoller(threading.Thread): - """Background daemon thread that refreshes the backend's health cache. - - Polls ``{latency_service_url}/v1/endpoints/health`` every - ``poll_interval_s`` seconds and writes verdicts into - ``health_cache`` under ``cache_lock``. Holds references to the - same dict and lock the backend reads from — no message passing, no - extra copies. - """ - - def __init__( - self, - latency_service_url: str, - model_ids: list[str], - health_cache: dict[str, EndpointHealth], - cache_lock: threading.Lock, - poll_interval_s: float, - poll_timeout_s: float, - ) -> None: - super().__init__(daemon=True, name="latency-health-poller") - self._url = latency_service_url.rstrip("/") + "/v1/endpoints/health" - self._model_ids = model_ids - self._health_cache = health_cache - self._cache_lock = cache_lock - self._poll_interval_s = poll_interval_s - self._http_client = httpx.Client(timeout=poll_timeout_s) - self._stop_event = threading.Event() - self._poll_count = 0 - # Failure counter and last-success timestamp drive the /metrics - # poll-health gauges; tracked here because the poll loop is the - # single point where outcomes are observable. - self._poll_failures = 0 - self._last_poll_ok = False - self._last_success_at: float | None = None - - def run(self) -> None: - while not self._stop_event.is_set(): - try: - resp = self._http_client.get( - self._url, - params=[("endpoint_ids", mid) for mid in self._model_ids], - ) - resp.raise_for_status() - data = resp.json() - with self._cache_lock: - for mid, info in data["endpoint_health"].items(): - if mid in self._health_cache: - self._health_cache[mid] = EndpointHealth( - status=EndpointHealthStatus(info["status"]), - last_latency_ms=info.get("last_latency_ms"), - ) - self._poll_count += 1 - self._last_poll_ok = True - self._last_success_at = time.monotonic() - except Exception: - logger.warning( - "Health poller: failed to reach Latency Service, " - "resetting all endpoints to UNKNOWN (random routing)" - ) - with self._cache_lock: - for mid in self._health_cache: - self._health_cache[mid] = EndpointHealth( - status=EndpointHealthStatus.UNKNOWN, - ) - self._poll_failures += 1 - self._last_poll_ok = False - self._stop_event.wait(timeout=self._poll_interval_s) - - @property - def has_polled(self) -> bool: - """True once at least one poll has successfully updated the cache.""" - return self._poll_count > 0 - - @property - def poll_successes(self) -> int: - """Total number of successful polls since the poller started.""" - return self._poll_count - - @property - def poll_failures(self) -> int: - """Total number of poll attempts that failed (including timeouts). - - On every failure, all cached endpoint verdicts are reset to UNKNOWN - so routing falls back to random selection instead of acting on - stale health data. - """ - return self._poll_failures - - @property - def last_poll_ok(self) -> bool: - """True iff the most recent poll attempt succeeded.""" - return self._last_poll_ok - - @property - def seconds_since_last_success(self) -> float | None: - """Monotonic seconds since the last successful poll. - - ``None`` before the first success — useful as a "never polled" - signal on /metrics scrapes during startup or sustained outage. - """ - if self._last_success_at is None: - return None - return time.monotonic() - self._last_success_at - - def stop(self) -> None: - """Signal the polling loop to exit at the next iteration boundary.""" - self._stop_event.set() diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py deleted file mode 100644 index 1942df79..00000000 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ /dev/null @@ -1,1098 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""``LLMBackend`` that routes across many endpoints by Latency Service verdicts. - -This is the usage case for Inference Hub deployments where a central -Latency Service owns heartbeat probing and statistical profiling. The -backend holds a pool of ``OpenAILLMClient`` instances keyed by model ID, -reads health verdicts from a locally-cached map maintained by a -:class:`HealthPoller` daemon thread, and picks a healthy endpoint on -every request. - -Chain integration:: - - request-side work → LatencyServiceLLMBackend → response-side work → TranslationEngine - -Request-format translation is delegated to -:class:`switchyard_rust.translation.TranslationEngine` after an endpoint is -selected, so Chat endpoints keep receiving Chat Completions while -Responses-mode endpoints receive the OpenAI Responses API natively. -""" - -import json -import logging -import random -import threading -import time -import uuid -from collections.abc import Mapping -from dataclasses import dataclass - -from openai import APIStatusError, AsyncStream - -from switchyard.lib.affinity_pin_store import AffinityPinStore -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.chat_response.openai_chat import ResponseStream -from switchyard.lib.chat_response.openai_responses import ResponsesApiStream -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, -) -from switchyard.lib.endpoints import outcome_metrics, prometheus_emitter -from switchyard.lib.endpoints.upstream_error_log import log_upstream_attempt_failure -from switchyard.lib.llm_client import OpenAILLMClient -from switchyard.lib.prometheus_exposition import format_number, render_labels -from switchyard.lib.proxy_context import ( - CTX_CALLER_API_KEY, - CTX_ERROR_SOURCE, - CTX_ROUTE_SELECTION, - CTX_UPSTREAM_ATTEMPTS_RECORDED, - CTX_UPSTREAM_HTTP_BODY, - CTX_UPSTREAM_HTTP_STATUS, - CTX_UPSTREAM_MODEL, - ERROR_SOURCE_PROVIDER, - ERROR_SOURCE_SWITCHYARD, - ProxyContext, -) -from switchyard.lib.roles import LLMBackend -from switchyard.lib.session_affinity import SessionAffinity -from switchyard.lib.stats_accumulator import StatsAccumulator -from switchyard.lib.tracing import routing_span, set_tags -from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse, request_type_matches -from switchyard_rust.translation import TranslationEngine - -log = logging.getLogger(__name__) - -#: Request header read by a LiteLLM front proxy; its JSON value is copied into -#: the provider spend-log DB row, tying that row back to the Switchyard route -#: that selected it (see :data:`CTX_ROUTE_SELECTION` for the payload contract). -SPEND_LOGS_METADATA_HEADER = "x-litellm-spend-logs-metadata" - -#: ``router_strategy`` value identifying this backend's selection algorithm. -ROUTER_STRATEGY_LATENCY = "latency" - - -@dataclass(frozen=True) -class _RouteDecision: - """One endpoint-selection outcome, carried to the ``route_decision`` span. - - ``candidates`` is the set the picker chose among (the winning health tier), - and ``was_fastest`` is whether ``selected`` had the lowest known latency in - that set. ``affinity`` is ``True`` when ``selected`` came from a session - pin (reused endpoint) rather than the latency-aware picker. - """ - - selected: str - candidates: tuple[str, ...] - was_fastest: bool - affinity: bool = False - - -class LatencyServiceLLMBackend(LLMBackend): - """Routes to healthy endpoints based on Latency Service health verdicts. - - On construction, builds a pool of ``OpenAILLMClient`` instances - (one per configured endpoint, keyed by ``model``) and starts a - :class:`HealthPoller` daemon thread that refreshes the in-memory - health cache every ``poll_interval_s`` seconds. - - On each ``call()``: - - 1. Pick an endpoint from the health cache — ``HEALTHY`` preferred, - then ``UNKNOWN``, then ``DEGRADED``. Within the chosen tier, - selection is weighted by inverse ``last_latency_ms`` when every - candidate has a known sample; otherwise it falls back to uniform - random. - 2. Translate the request to that endpoint's configured OpenAI API - surface and override the body's ``model`` with the selected endpoint ID. - 3. Call ``OpenAILLMClient`` on the configured API surface. On a transient error - (429 / 408 / 5xx / network), retry with a different endpoint - (dedup prevents re-selecting the same one within a single - request). On a 4xx client error the request is rejected - identically by every replica, so the loop fails fast and passes - the upstream status through. - 4. Wrap the response into a Rust-owned ``ChatResponse``. - """ - - def __init__( - self, - config: LatencyServiceBackendConfig, - *, - stats_accumulator: StatsAccumulator | None = None, - ) -> None: - if not config.endpoints: - raise ValueError("At least one endpoint must be configured") - - self._config = config - self._translation = TranslationEngine() - self._clients: dict[str, OpenAILLMClient] = {} - self._upstream_models: dict[str, str] = {} - self._request_types: dict[str, ChatRequestType] = {} - self._health_cache: dict[str, EndpointHealth] = {} - self._cache_lock = threading.Lock() - # When provided, the backend records success/error/latency directly - # into the accumulator on each attempt. This mirrors what the Rust - # ``StatsLlmBackend`` does for native backends; the Python-only - # latency-service backend can't be wrapped by it, so we record - # in-place to keep ``/metrics`` populated. - self._stats = stats_accumulator - - # Session affinity: pins a conversation to the endpoint that last served - # it so its upstream prompt/KV cache stays warm. Shared coordinator with - # the classifier router; the latency-specific reuse policy (health-gated, - # failover-aware) lives in ``_select_endpoint_decision``. An optional - # shared L2 (Redis) lets pins outlive this worker and be read by peers, - # so a conversation stays pinned across replicas and pod churn. - self._affinity = SessionAffinity( - enabled=config.session_affinity, - max_sessions=config.affinity_max_sessions, - l2=_build_affinity_l2(config), - ) - # Cumulative warm-reuse counters, published on /metrics when affinity is - # enabled. A "hit" is a turn served by an existing pin; a "miss" is a - # first/unpinnable turn routed by the latency-aware picker. Counted once - # per request (first attempt only) so failover retries don't inflate - # them. Incremented and read only on the event loop, so no lock needed. - self._affinity_hits = 0 - self._affinity_misses = 0 - - # Per-model upstream-attempt counts for the latency-service-scoped - # /metrics breakdown, keyed by (requested model, selected upstream model, - # outcome, HTTP code). Kept separate from the layer-aggregate - # ``switchyard_upstream_attempts_total`` so that counter stays model-free. - # Cardinality is bounded: ``upstream_model`` comes from config, and - # ``requested_model`` is normalized to a config-derived id (route id or - # endpoint id) or the ``"other"`` sentinel in ``_record_model_attempt`` - # so a caller-supplied model string can't grow the series set. Mutated - # and read only on the event loop (like the affinity counters), so no - # lock. - self._upstream_attempts_by_model: dict[tuple[str, str, str, str], int] = {} - - for ep_cfg in config.endpoints: - model_id = ep_cfg.model - if not model_id: - raise ValueError( - "Every endpoint must have a 'model' field — " - "this is the key used by the Latency Service" - ) - if model_id in self._clients: - raise ValueError(f"Duplicate model ID: {model_id}") - - self._clients[model_id] = OpenAILLMClient( - api_key=ep_cfg.api_key, - base_url=ep_cfg.base_url, - timeout=ep_cfg.timeout, - # The ``call`` retry loop already retries on a *different* - # endpoint, which is what a health-aware router wants. Letting - # the SDK also retry (default 2) on the *same* endpoint stacks - # multiplicatively (up to (1+max_retries) x 3 attempts) and adds - # exponential-backoff sleeps that hold the request — and its - # buffered body — alive longer, amplifying connection-pool - # pressure during an upstream incident. Disable it here. - max_retries=0, - ) - self._upstream_models[model_id] = ep_cfg.upstream_model or model_id - self._request_types[model_id] = _latency_endpoint_request_type( - ep_cfg.request_type - ) - self._health_cache[model_id] = EndpointHealth( - status=EndpointHealthStatus.UNKNOWN, - ) - log.info( - "LatencyServiceLLMBackend endpoint: model=%s upstream_model=%s " - "request_type=%s base_url=%s", - model_id, - self._upstream_models[model_id], - ep_cfg.request_type, - ep_cfg.base_url, - ) - - # Bounded ``requested_model`` label set: the configured endpoint ids - # plus the client-facing route id when the deployment supplied one - # (IH clients request the route key, e.g. ``nvidia/switchyard/gpt-5.4``, - # never an endpoint id). Both sources are config-derived, so metric - # cardinality stays bounded. - self._requested_model_ids = frozenset(self._clients) | ( - frozenset((config.route_model,)) if config.route_model else frozenset() - ) - - self._poller = HealthPoller( - latency_service_url=config.latency_service_url, - model_ids=list(self._clients.keys()), - health_cache=self._health_cache, - cache_lock=self._cache_lock, - poll_interval_s=config.poll_interval_s, - poll_timeout_s=config.poll_timeout_s, - ) - # Publish LS verdicts + poll health on /metrics. Registering here - # (and unregistering in ``shutdown``) ties emitter lifetime to the - # backend's lifetime, so a re-built chain doesn't leak a closure - # over a torn-down cache. - prometheus_emitter.register(self._render_prometheus_lines) - self._poller.start() - - @property - def supported_request_types(self) -> list[ChatRequestType]: - """OpenAI request types accepted by the configured endpoints.""" - ordered = [ChatRequestType.OPENAI_CHAT, ChatRequestType.OPENAI_RESPONSES] - configured = set(self._request_types.values()) - return [request_type for request_type in ordered if request_type in configured] - - # -- Endpoint selection (reads from cache, never blocks on network) ----- - - def _select_endpoint(self) -> str: - """Pick a model ID — see :meth:`_select_endpoint_decision`.""" - return self._select_endpoint_decision().selected - - def _select_endpoint_decision( - self, - pinned_endpoint: str | None = None, - tried: frozenset[str] = frozenset(), - ) -> _RouteDecision: - """Pick a model ID and report the candidate set it chose among. - - When ``pinned_endpoint`` is set (this conversation's session-affinity - pin) and still usable — HEALTHY/UNKNOWN and not already failed this - request — it is reused directly, bypassing the latency-aware pick so the - conversation sticks to one endpoint and its upstream cache stays warm. - - Otherwise tier priority is HEALTHY > UNKNOWN > DEGRADED. Within the - chosen tier, selection is **inverse-latency weighted** when every - candidate has a known ``last_latency_ms`` — endpoints that the Latency - Service most recently saw as faster receive proportionally more - traffic. If any candidate's latency is unknown, the picker falls back - to uniform random for that tier; this keeps behaviour predictable while - the poller is warming up or when the upstream reports nulls. - - Returns the selected endpoint plus the candidates considered and - whether the pick was the lowest-latency one — the signals the - ``switchyard.route_decision`` span reports. - """ - with self._cache_lock: - snapshot = dict(self._health_cache) - - # -- Affinity fast-path: reuse the pinned endpoint while it stays - # healthy and hasn't already failed this request. Skips latency - # weighting entirely — the whole point is to not chase a marginally - # faster endpoint at the cost of cache locality. - if ( - pinned_endpoint is not None - and pinned_endpoint not in tried - and _affinity_usable(snapshot.get(pinned_endpoint)) - ): - return _RouteDecision( - selected=pinned_endpoint, - candidates=(pinned_endpoint,), - was_fastest=False, - affinity=True, - ) - - by_health: dict[EndpointHealthStatus, list[str]] = { - h: [] for h in EndpointHealthStatus - } - for mid, hp in snapshot.items(): - by_health[hp.status].append(mid) - - for tier in ( - EndpointHealthStatus.HEALTHY, - EndpointHealthStatus.UNKNOWN, - EndpointHealthStatus.DEGRADED, - ): - if by_health[tier]: - candidates = by_health[tier] - selected = self._pick_by_latency(candidates, snapshot) - return _RouteDecision( - selected=selected, - candidates=tuple(candidates), - was_fastest=self._is_fastest(selected, candidates, snapshot), - ) - - candidates = list(self._clients.keys()) - selected = random.choice(candidates) - return _RouteDecision( - selected=selected, - candidates=tuple(candidates), - was_fastest=self._is_fastest(selected, candidates, snapshot), - ) - - @staticmethod - def _is_fastest( - selected: str, - candidates: list[str], - snapshot: dict[str, EndpointHealth], - ) -> bool: - """Whether *selected* had the lowest known ``last_latency_ms``. - - ``False`` when no candidate has a usable latency sample (selection was - uniform, so "fastest" is undefined). Over many requests the fraction of - ``True`` shows how often inverse-latency weighting picked the front-runner. - """ - known: dict[str, float] = {} - for mid in candidates: - latency = snapshot[mid].last_latency_ms - if latency is not None and latency > 0: - known[mid] = latency - if not known or selected not in known: - return False - return known[selected] == min(known.values()) - - def _poll_age_ms(self) -> float | None: - """Milliseconds since the last successful Latency Service poll, or ``None``.""" - age_s = self._poller.seconds_since_last_success - return age_s * 1000.0 if age_s is not None else None - - @staticmethod - def _pick_by_latency( - candidates: list[str], - snapshot: dict[str, EndpointHealth], - ) -> str: - """Inverse-latency weighted random pick across ``candidates``. - - Returns a uniform random pick when only one candidate exists, when - any candidate's ``last_latency_ms`` is unknown, or when any sample - is non-positive (treated as bogus — the Latency Service should - never report 0 or negative, but be defensive). - """ - if len(candidates) == 1: - return candidates[0] - latencies = [snapshot[c].last_latency_ms for c in candidates] - if any(lat is None or lat <= 0 for lat in latencies): - return random.choice(candidates) - weights = [1.0 / lat for lat in latencies] # type: ignore[operator] - return random.choices(candidates, weights=weights, k=1)[0] - - # -- Request processing (hot path — no Latency Service call) ------------ - - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: - """Serve *request* on the best healthy endpoint, failing over on error. - - Candidates are ordered by health and observed latency (with optional - session affinity); every upstream attempt is stamped with the - spend-logs metadata header, and the attempt that succeeds records its - route selection on *ctx* (see :data:`CTX_ROUTE_SELECTION`). Exhausting - all candidates re-raises the last upstream error, annotated for the - client-facing error envelope. - """ - # This backend records its own per-attempt ``outcome_metrics`` counters - # below (one per failover attempt). Claim attempt accounting for this - # request so the endpoint-layer fallback in ``dispatch`` / - # ``handle_chain_exception`` does not double-count the retry fan-out. - ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] = True - api_key_override = self._api_key_override_for_policy( - ctx.metadata.get(CTX_CALLER_API_KEY) - ) - # ``caller_required`` never falls back to the configured endpoint key: - # reject before any upstream call so the service credential can't - # authenticate caller inference — and before the affinity lookup, so a - # doomed request never pays a pin resolution (possibly a shared-store - # round-trip). The rejection deliberately stays invisible to - # ``switchyard_upstream_attempts_total`` (the accounting flag above is - # already claimed): no upstream attempt happened, and counting a - # synthetic 401 would skew the direct-to-endpoint baseline error rate. - # ``api_key_override`` is ``None`` here only when no usable caller key - # was supplied. - if self._config.credential_policy == "caller_required" and api_key_override is None: - _reject_missing_caller_api_key(ctx) - - # Captured before the per-attempt ``body["model"]`` override so the span - # records the model the client asked for, not the selected endpoint. - incoming_model = request.model - # One correlation id per client request, shared by every failover - # attempt's spend-logs header and by the response headers — the join - # key between a front proxy's spend-log row and the provider rows. - correlation_id = str(uuid.uuid4()) - # Resolve the session-affinity pin once (keyed on the stable conversation - # prefix). ``None`` when affinity is disabled or the conversation isn't - # pinned yet, so every attempt routes purely by health + latency. - pinned_endpoint = await self._affinity.pinned(ctx, request) - - last_exc: Exception | None = None - # Upstream model of the last failed attempt, surfaced on the error - # envelope/span/log so operators can tell *which* endpoint the - # provider-originated failure came from. - last_upstream_model: str | None = None - tried: set[str] = set() - - for attempt in range(1 + self._config.max_retries): - # -- Route decision span: which endpoint, out of which candidates -- - with routing_span("switchyard.route_decision") as route_span: - decision = self._select_endpoint_decision(pinned_endpoint, frozenset(tried)) - model_id = decision.selected - candidates = decision.candidates - was_fastest = decision.was_fastest - if model_id in tried and len(tried) < len(self._clients): - remaining = [m for m in self._clients if m not in tried] - model_id = random.choice(remaining) - # A forced dedup pick, not the latency-weighted choice. - candidates = tuple(remaining) - was_fastest = False - tried.add(model_id) - # Count the warm-reuse outcome once per request (the first - # decision); later attempts are failover, not an affinity signal. - if self._affinity.enabled and attempt == 0: - if decision.affinity: - self._affinity_hits += 1 - else: - self._affinity_misses += 1 - set_tags(route_span, { - "switchyard.model": incoming_model, - "switchyard.candidate_endpoints": ",".join(candidates), - "switchyard.selected_endpoint": model_id, - "switchyard.was_fastest_selected": was_fastest, - "switchyard.affinity_hit": decision.affinity, - "switchyard.latency_service_poll_age_ms": self._poll_age_ms(), - }) - - upstream_model = self._upstream_models[model_id] - target_request_type = self._request_types[model_id] - body = self._body_for_endpoint_request_type(ctx, request, target_request_type) - body["model"] = upstream_model - # Per-attempt route-selection record: each upstream call is stamped - # with the endpoint it actually hits, so a failover retry's - # spend-log row doesn't inherit the failed attempt's selection. - route_selection = self._route_selection( - incoming_model=incoming_model, - model_id=model_id, - upstream_model=upstream_model, - correlation_id=correlation_id, - ) - log.debug( - "LatencyServiceLLMBackend: attempt=%d model=%s upstream=%s " - "request_type=%s stream=%s", - attempt + 1, - model_id, - upstream_model, - target_request_type.value, - body.get("stream"), - ) - - # -- Upstream attempt span: outcome of this one upstream call ----- - with routing_span("switchyard.upstream_attempt") as attempt_span: - set_tags(attempt_span, { - "switchyard.model": incoming_model, - "switchyard.selected_endpoint": model_id, - "switchyard.retry_count": attempt, - }) - started_at = time.monotonic() - try: - result = await self._call_endpoint( - model_id, - target_request_type, - api_key=api_key_override, - body=body, - extra_headers={ - SPEND_LOGS_METADATA_HEADER: json.dumps(route_selection), - }, - ) - except APIStatusError as exc: - set_tags(attempt_span, { - "switchyard.outcome": outcome_metrics.classify(exc.status_code), - "switchyard.upstream_status_code": exc.status_code, - "switchyard.error_code": outcome_metrics.code_label(exc.status_code), - "switchyard.error_source": ERROR_SOURCE_PROVIDER, - "switchyard.upstream_model": upstream_model, - }) - if self._stats is not None: - await self._stats.record_error(model_id) - outcome_metrics.record_upstream_attempt(exc.status_code) - self._record_model_attempt( - incoming_model, - upstream_model, - outcome_metrics.classify(exc.status_code), - outcome_metrics.code_label(exc.status_code), - ) - # Per-event structured log (Loki) — the timestamped complement - # to the aggregate outcome counter recorded above. - log_upstream_attempt_failure( - model=model_id, - attempt=attempt + 1, - status_code=exc.status_code, - error=exc, - upstream_model=upstream_model, - ) - last_exc = exc - last_upstream_model = upstream_model - # A client error (4xx other than 408/429) is deterministic — - # replicas reject the same payload identically — so fail fast - # instead of burning attempts; the post-loop passthrough - # surfaces the upstream status to the client. - if not _is_retryable_status(exc.status_code): - break - continue - except Exception as exc: - set_tags(attempt_span, { - "switchyard.outcome": "retryable_error", - "switchyard.error_code": outcome_metrics.NO_STATUS_CODE, - "switchyard.error_source": ERROR_SOURCE_PROVIDER, - "switchyard.upstream_model": upstream_model, - }) - if self._stats is not None: - await self._stats.record_error(model_id) - # Non-HTTP failure (network, pre-status timeout, SDK error) — - # treated as a retryable_error in the outcome ratio: those - # are exactly the faults a health-aware router should absorb - # by selecting a different endpoint on the next attempt. - outcome_metrics.record_upstream_attempt(None) - self._record_model_attempt( - incoming_model, - upstream_model, - "retryable_error", - outcome_metrics.NO_STATUS_CODE, - ) - # status_code=None → logged as code="none" (no HTTP status). - log_upstream_attempt_failure( - model=model_id, - attempt=attempt + 1, - status_code=None, - error=exc, - upstream_model=upstream_model, - ) - last_exc = exc - last_upstream_model = upstream_model - continue - - backend_latency_ms = (time.monotonic() - started_at) * 1000.0 - set_tags(attempt_span, { - "switchyard.outcome": "success", - "switchyard.upstream_status_code": 200, - }) - if self._stats is not None: - await self._stats.record_success(model_id, backend_latency_ms) - outcome_metrics.record_upstream_attempt(200) - self._record_model_attempt( - incoming_model, upstream_model, "success", "200", - ) - # A successful attempt after at least one failure is direct - # evidence the steering logic rescued this request. Counted - # once per client request, not per recovered retry. - if attempt > 0: - outcome_metrics.record_retry_recovered() - - # ``ctx.selected_model`` is the cross-language hook the Rust - # ``StatsResponseProcessor`` reads to attribute token usage and - # end-to-end latency per endpoint. Without it, the response - # processor sees no ``BackendSelection`` and buckets every call - # to ``model=""`` on /metrics. - ctx.selected_model = model_id - # ``backend_call_latency_ms`` lets the Rust ``StatsResponseProcessor`` - # compute ``routing_overhead_ms = total_latency - backend_latency`` - # for this Python-only backend, which can't be wrapped by the - # Rust ``StatsLlmBackend`` that normally publishes this signal. - ctx.backend_call_latency_ms = backend_latency_ms - - # The selection that actually served the request, surfaced to - # the endpoint layer as ``x-switchyard-*`` response headers so - # a front proxy can enrich its own spend-log row with the same - # correlation id the provider row received. - ctx.metadata[CTX_ROUTE_SELECTION] = route_selection - - # Pin this conversation to the endpoint that served it so later - # turns reuse it (warm cache). Re-pinning on every success also - # follows a recovery: if the previous pin degraded and we - # re-routed, the endpoint that worked becomes the new pin. - # (No-op when affinity is disabled.) - await self._affinity.pin(ctx, request, model_id) - - return _chat_response_for_request_type(target_request_type, result) - - # All attempts failed. If the last failure was an upstream HTTP - # error (e.g. 401 from a bad API key), record the status code and - # body on ctx so the endpoint can passthrough the upstream status - # rather than masking it as a 500. The exception itself still - # propagates — the chain is errored — but the endpoint reads ctx - # to decide the response code. - if isinstance(last_exc, APIStatusError): - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = last_exc.status_code - upstream_body = _extract_upstream_body(last_exc) - if upstream_body is not None: - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = upstream_body - - # Failure-origin annotation for the client-facing error envelope: - # every exhausted attempt failed at a selected upstream, so what the - # endpoint surfaces — the HTTP passthrough above, or the 500 for - # status-less network failures — is provider-originated. (The stash - # helpers below mark Switchyard-originated rejections instead.) - ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_PROVIDER - if last_upstream_model is not None: - ctx.metadata[CTX_UPSTREAM_MODEL] = last_upstream_model - - raise last_exc # type: ignore[misc] - - def _body_for_endpoint_request_type( - self, - ctx: ProxyContext, - request: ChatRequest, - target_request_type: ChatRequestType, - ) -> dict[str, object]: - try: - normalized = self._translation.request_to_any_of( - request, [target_request_type], - ) - except ValueError as exc: - # Transparent-router contract: a payload the upstream provider would - # reject (e.g. an unsupported message role like "api") must surface - # as a provider-compatible 400, not a generic 500. The Rust - # translation layer raises a ``ValueError`` whose message is - # prefixed with the stable error kind; an invalid-value rejection is - # recorded as an upstream 400 so the endpoint passes it through. - _stash_invalid_request_error(ctx, exc) - raise - if not request_type_matches(normalized, target_request_type): - raise TypeError( - "LatencyServiceLLMBackend expected request type " - f"{target_request_type.value} after translation" - ) - return dict(normalized.body) - - def _route_selection( - self, - *, - incoming_model: str | None, - model_id: str, - upstream_model: str, - correlation_id: str, - ) -> dict[str, str | None]: - """Build the route-selection payload for one upstream attempt. - - This is the JSON value of the outbound ``x-litellm-spend-logs-metadata`` - header and, for the attempt that succeeds, the - :data:`CTX_ROUTE_SELECTION` record behind the ``x-switchyard-*`` - response headers. ``router_model`` falls back to the configured route id - when the client body carried no model; ``router_selected_provider`` is - the upstream model's leading path segment (IH/LiteLLM naming, e.g. - ``"openai/openai/gpt-5.4"`` → ``"openai"``). - """ - return { - "router_model": incoming_model or self._config.route_model, - "router_strategy": ROUTER_STRATEGY_LATENCY, - "router_selected_endpoint": model_id, - "router_selected_model": upstream_model, - "router_selected_provider": upstream_model.split("/", 1)[0], - "router_correlation_id": correlation_id, - } - - async def _call_endpoint( - self, - model_id: str, - target_request_type: ChatRequestType, - *, - api_key: str | None, - body: dict[str, object], - extra_headers: dict[str, str], - ) -> object: - """Make one upstream SDK call with the protected *extra_headers* stamped on. - - ``extra_headers`` rides the client wrapper's ``**kwargs`` into the - OpenAI SDK's per-request header merge (over ``default_headers``). A - passthrough body may itself carry an SDK-style ``extra_headers`` field - (shape-preserving translation keeps unknown keys); it is still - forwarded — it used to ride ``**body`` into the same SDK parameter — - but the protected headers must win the merge so callers can't spoof - them, and header names are case-insensitive on the wire while dict - keys are not, so every case variant of a protected name is stripped - from the client mapping (a client-cased duplicate would otherwise - reach the wire alongside ours and win first-match parsing on the - receiving proxy). A non-mapping value is dropped: it could only ever - have broken the SDK call. - """ - client_extra = body.pop("extra_headers", None) - protected = {name.lower() for name in extra_headers} - headers: dict[str, object] = ( - { - name: value - for name, value in client_extra.items() - if not (isinstance(name, str) and name.lower() in protected) - } - if isinstance(client_extra, Mapping) - else {} - ) - headers.update(extra_headers) - if target_request_type == ChatRequestType.OPENAI_RESPONSES: - return await self._clients[model_id].aresponses( - api_key=api_key, - extra_headers=headers, - **body, - ) - return await self._clients[model_id].acompletion( - api_key=api_key, - extra_headers=headers, - **body, - ) - - def _api_key_override_for_policy(self, caller_api_key: object) -> str | None: - """Per-request key to forward upstream, or ``None`` to use the endpoint key. - - ``configured_endpoint`` never forwards a caller key. ``caller_override`` - and ``caller_required`` forward a usable caller key; they differ only in - the no-key case, which ``call`` handles before any upstream attempt - (``caller_override`` falls back to the endpoint key, ``caller_required`` - returns 401). - """ - if self._config.credential_policy == "configured_endpoint": - return None - if isinstance(caller_api_key, str) and caller_api_key.strip(): - return caller_api_key - return None - - def _record_model_attempt( - self, - requested_model: str | None, - upstream_model: str, - outcome: str, - code: str, - ) -> None: - """Count one upstream attempt for the per-model /metrics breakdown. - - Event-loop only (no lock). ``requested_model`` is the client-supplied - model; it is bounded to a config-derived id — the route id - (``config.route_model``) or a configured endpoint id — with the - ``"other"`` sentinel as fallback before becoming a Prometheus label, so - caller-controlled input can't create unbounded metric-series - cardinality. - """ - requested = ( - requested_model if requested_model in self._requested_model_ids else "other" - ) - key = (requested, upstream_model, outcome, code) - self._upstream_attempts_by_model[key] = ( - self._upstream_attempts_by_model.get(key, 0) + 1 - ) - - # -- Lifecycle ---------------------------------------------------------- - - async def shutdown(self) -> None: - """Stop the background :class:`HealthPoller` daemon and release the - shared session-affinity store, when one is configured. - - Picked up automatically by the server's component teardown hook - (``_run_lifecycle_method``), which awaits awaitable ``shutdown`` - results. Safe to call multiple times. - """ - prometheus_emitter.unregister(self._render_prometheus_lines) - self._poller.stop() - await self._affinity.aclose() - - def is_ready(self) -> bool: - """True once the background poller has completed at least one successful poll.""" - return self._poller.has_polled - - # -- Metrics emitter ---------------------------------------------------- - - def _render_prometheus_lines(self) -> list[str]: - """Emit per-endpoint health verdicts and poll-loop health gauges. - - Snapshotted under ``self._cache_lock`` so a poll concurrent with a - scrape can't produce a mixed view across endpoints. Output is - Prometheus exposition lines without trailing newline — composed - by :mod:`switchyard.lib.endpoints.prometheus_emitter`. - """ - with self._cache_lock: - snapshot = dict(self._health_cache) - - lines: list[str] = [] - lines.append( - "# HELP switchyard_endpoint_status " - "Latency-Service verdict per endpoint (1 = current status; " - "exactly one status row per model is non-zero)." - ) - lines.append("# TYPE switchyard_endpoint_status gauge") - for model_id, health in sorted(snapshot.items()): - current = health.status.value - for status in EndpointHealthStatus: - value = 1 if status.value == current else 0 - labels = render_labels({"model": model_id, "status": status.value}) - lines.append(f"switchyard_endpoint_status{labels} {value}") - - lines.append( - "# HELP switchyard_endpoint_last_latency_ms " - "Last latency sample (ms) reported by the Latency Service per endpoint. " - "Absent until the first poll publishes a non-null sample." - ) - lines.append("# TYPE switchyard_endpoint_last_latency_ms gauge") - for model_id, health in sorted(snapshot.items()): - if health.last_latency_ms is None: - continue - labels = render_labels({"model": model_id}) - lines.append( - f"switchyard_endpoint_last_latency_ms{labels} " - f"{format_number(health.last_latency_ms)}" - ) - - lines.append( - "# HELP switchyard_latency_service_poll_ok " - "1 when the last poll succeeded, 0 when it failed or has not yet run." - ) - lines.append("# TYPE switchyard_latency_service_poll_ok gauge") - last_age = self._poller.seconds_since_last_success - # ``poll_ok`` reflects the latest poll attempt. Combined with - # ``poll_age_seconds``, scrapers can tell "never polled" (no age line) - # from "polled, but the latest attempt failed" (age present, ok=0). - poll_ok = 1 if self._poller.last_poll_ok else 0 - lines.append(f"switchyard_latency_service_poll_ok {poll_ok}") - - lines.append( - "# HELP switchyard_latency_service_poll_age_seconds " - "Monotonic seconds since the last successful poll. Absent before the " - "first success." - ) - lines.append("# TYPE switchyard_latency_service_poll_age_seconds gauge") - if last_age is not None: - lines.append( - "switchyard_latency_service_poll_age_seconds " - f"{format_number(last_age)}" - ) - - lines.append( - "# HELP switchyard_latency_service_polls_total " - "Total successful health polls since the poller started." - ) - lines.append("# TYPE switchyard_latency_service_polls_total counter") - lines.append( - f"switchyard_latency_service_polls_total {self._poller.poll_successes}" - ) - - lines.append( - "# HELP switchyard_latency_service_poll_failures_total " - "Total failed health polls; each failure resets every endpoint to " - "UNKNOWN." - ) - lines.append("# TYPE switchyard_latency_service_poll_failures_total counter") - lines.append( - "switchyard_latency_service_poll_failures_total " - f"{self._poller.poll_failures}" - ) - - # Warm-reuse counters — only meaningful (and only emitted) when session - # affinity is enabled, so the metric surface stays clean for the common - # per-turn-routing case. Reuse rate = hits / (hits + misses). - if self._config.session_affinity: - lines.append( - "# HELP switchyard_affinity_hits_total " - "Conversation turns served by an existing session-affinity pin " - "(warm endpoint reuse)." - ) - lines.append("# TYPE switchyard_affinity_hits_total counter") - lines.append(f"switchyard_affinity_hits_total {self._affinity_hits}") - - lines.append( - "# HELP switchyard_affinity_misses_total " - "First or unpinnable turns routed by the latency-aware picker " - "while session affinity was enabled." - ) - lines.append("# TYPE switchyard_affinity_misses_total counter") - lines.append(f"switchyard_affinity_misses_total {self._affinity_misses}") - - # Shared-store (L2) counters — emitted only when a shared pin store - # is configured. Hits measure cross-worker/churn pin reuse; errors - # count fail-open store operations (the alerting signal for a store - # that is degraded while requests keep succeeding). - if self._affinity.l2_enabled: - lines.append( - "# HELP switchyard_affinity_l2_hits_total " - "Pins resolved from the shared (L2) affinity store after an " - "in-process (L1) miss — cross-worker warm reuse." - ) - lines.append("# TYPE switchyard_affinity_l2_hits_total counter") - lines.append( - f"switchyard_affinity_l2_hits_total {self._affinity.l2_hits}" - ) - - lines.append( - "# HELP switchyard_affinity_l2_errors_total " - "Shared (L2) affinity-store operations that failed open " - "(get or put); routing fell back to in-process pins only." - ) - lines.append("# TYPE switchyard_affinity_l2_errors_total counter") - lines.append( - f"switchyard_affinity_l2_errors_total {self._affinity.l2_errors}" - ) - - lines.append( - "# HELP switchyard_affinity_l2_breaker_open " - "1 while the shared-store circuit breaker is open " - "(operations skipped without a network attempt); 0 when closed." - ) - lines.append("# TYPE switchyard_affinity_l2_breaker_open gauge") - lines.append( - "switchyard_affinity_l2_breaker_open " - f"{int(self._affinity.l2_breaker_open)}" - ) - - # Per-model upstream-attempt breakdown — the latency-service-scoped - # complement to the layer-aggregate ``switchyard_upstream_attempts_total``. - # Series are created lazily per (requested_model, upstream_model, outcome, - # code), so the surface stays empty until traffic flows. Snapshotted to a - # plain dict for a stable view across the emission loop. - attempts_by_model = dict(self._upstream_attempts_by_model) - if attempts_by_model: - lines.append( - "# HELP switchyard_latency_upstream_attempts_total " - "Upstream call attempts from the latency-service backend, labelled " - "by requested (route) model, selected upstream model, outcome, and " - "HTTP status code (code=\"none\" for non-HTTP failures). Per-model " - "complement to the layer-aggregate switchyard_upstream_attempts_total; " - "cardinality is bounded by the configured endpoint set." - ) - lines.append("# TYPE switchyard_latency_upstream_attempts_total counter") - # Sorted for deterministic exposition order across scrapes. - for (requested_model, upstream_model, outcome, code), count in sorted( - attempts_by_model.items() - ): - labels = render_labels({ - "requested_model": requested_model, - "upstream_model": upstream_model, - "outcome": outcome, - "code": code, - }) - lines.append( - f"switchyard_latency_upstream_attempts_total{labels} {count}" - ) - return lines - - -def _latency_endpoint_request_type(value: str) -> ChatRequestType: - if value == "openai_responses": - return ChatRequestType.OPENAI_RESPONSES - return ChatRequestType.OPENAI_CHAT - - -def _chat_response_for_request_type( - request_type: ChatRequestType, - result: object, -) -> ChatResponse: - if request_type == ChatRequestType.OPENAI_RESPONSES: - # Streaming yields raw SSE frame strings (``RawSSEFrameStream``) so the - # upstream events pass through verbatim; anything async-iterable is a - # stream, a plain mapping is the exact non-streaming JSON body. - if isinstance(result, AsyncStream) or hasattr(result, "__aiter__"): - return ChatResponse.openai_responses_stream(ResponsesApiStream(result)) - return ChatResponse.openai_responses_completion(result) - if isinstance(result, AsyncStream): - return ChatResponse.openai_stream(ResponseStream(result)) - return ChatResponse.openai_completion(result) - - -def _is_retryable_status(status_code: int) -> bool: - """Whether an upstream status warrants failing over to a different endpoint. - - Retries 5xx + 408 + 429 (transient/capacity/server-side); other 4xx are the - client's payload, which every replica rejects identically, so fail fast. - Deliberately broader than ``outcome_metrics.RETRYABLE_STATUSES`` (metrics - bucketing) — e.g. 502/503 must fail over but aren't metrics-retryable. - """ - return status_code >= 500 or status_code in (408, 429) - - -def _affinity_usable(health: EndpointHealth | None) -> bool: - """Whether a pinned endpoint may still serve its session. - - The pin holds while the endpoint is HEALTHY or UNKNOWN — the same tiers the - latency-aware picker trusts. A DEGRADED verdict (or an endpoint that - vanished from the health cache) breaks the pin so the next turn re-routes: - locality must yield to health, or sticky routing would funnel a session - into a failing endpoint. - """ - return health is not None and health.status in ( - EndpointHealthStatus.HEALTHY, - EndpointHealthStatus.UNKNOWN, - ) - - -def _build_affinity_l2(config: LatencyServiceBackendConfig) -> AffinityPinStore | None: - """Build the optional shared L2 pin store from config (``None`` = L1-only). - - ``affinity_store="memory"`` (the default) keeps pins per process. ``"redis"`` - imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays - optional. Config validation guarantees a URL when the store is Redis. - """ - if config.affinity_store != "redis": - return None - from switchyard.lib.redis_pin_store import RedisPinStore - - assert config.affinity_store_url is not None # enforced by config validator - return RedisPinStore( - config.affinity_store_url, - ttl_seconds=config.affinity_store_ttl_seconds, - key_prefix=config.affinity_key_prefix, - ) - - -# Prefix of the stringified translation error for a client-invalid payload. -# ``TranslationError::kind()`` is the stable FFI signal for the error variant -# (``crates/switchyard-translation/src/error.rs``); ``py_translation_error`` -# formats translation errors as ``"{kind}: {message}"``, so an invalid-value -# rejection (e.g. an unsupported message role) is recognizable by this prefix. -_TRANSLATION_INVALID_VALUE_PREFIX = "InvalidValue:" - - -def _stash_invalid_request_error(ctx: ProxyContext, error: Exception) -> None: - """Record a translation invalid-value error as an upstream HTTP 400 on ``ctx``. - - Mirrors the upstream-status passthrough used for provider HTTP errors: the - Compatibility execution flattens the raised exception to a generic error - string, but ``ctx.metadata`` survives the error path, so the endpoint's - ``upstream_response_from_ctx`` can return a provider-compatible 400. This is - a no-op for any other error (it keeps propagating and FastAPI maps it to a - 500 as before). - """ - message = str(error) - if not message.startswith(_TRANSLATION_INVALID_VALUE_PREFIX): - return - detail = message[len(_TRANSLATION_INVALID_VALUE_PREFIX) :].strip() - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 400 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = { - "error": { - "message": detail or "invalid request payload", - "type": "invalid_request_error", - "code": "invalid_value", - } - } - # This 400 rides the upstream-status channel but is Switchyard's own - # translation rejection — label it so the error-source header is truthful. - ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD - - -def _reject_missing_caller_api_key(ctx: ProxyContext) -> None: - """Reject a ``caller_required`` request that carries no caller API key. - - Stashes an HTTP 401 and a provider-compatible error envelope on ``ctx`` the - same way upstream-status passthrough does, then raises so the chain errors - out before any upstream call. No upstream is contacted or counted; - ``upstream_response_from_ctx`` reads the stashed status to return a 401. This - is what keeps the configured endpoint key from ever authenticating caller - inference under the ``caller_required`` policy. - """ - ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 - ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = { - "error": { - "message": "missing caller API key; supply it via the x-switchyard-api-key header", - "type": "invalid_request_error", - "code": "missing_caller_api_key", - } - } - # Rides the upstream-status channel but no upstream was contacted — this - # is Switchyard's own credential-policy rejection. - ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD - raise PermissionError("caller_required policy: missing caller API key") - - -def _extract_upstream_body(error: APIStatusError) -> object | None: - """Best-effort extraction of the upstream response body for passthrough. - - Prefers the structured ``body`` attribute (set by the OpenAI SDK for - JSON responses). Falls back to ``response.text`` for responses the - SDK couldn't decode. Returns ``None`` when nothing usable is present - — the endpoint then synthesizes a generic error envelope. - """ - body: object | None = getattr(error, "body", None) - if body is not None: - return body - response = getattr(error, "response", None) - if response is None: - return None - text: object | None = getattr(response, "text", None) - return text if text else None diff --git a/switchyard/lib/config/__init__.py b/switchyard/lib/config/__init__.py index f21a9bf7..1c74cba1 100644 --- a/switchyard/lib/config/__init__.py +++ b/switchyard/lib/config/__init__.py @@ -7,14 +7,8 @@ IntakeQueueFullPolicy, IntakeSinkConfig, ) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) __all__ = [ "IntakeQueueFullPolicy", "IntakeSinkConfig", - "LatencyServiceBackendConfig", - "LatencyServiceEndpoint", ] diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py deleted file mode 100644 index f52da2c3..00000000 --- a/switchyard/lib/config/latency_service_backend_config.py +++ /dev/null @@ -1,168 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Configuration models for the Latency Service usage case. - -``LatencyServiceEndpoint`` describes one LLM backend monitored by the -Latency Service. ``LatencyServiceBackendConfig`` bundles the full -backend configuration — URL of the Latency Service, the endpoint list, -and polling/retry parameters. - -The ``model`` field on each endpoint doubles as the endpoint ID used by -the Latency Service's health API — mirroring the routing-by-model-name -convention the rest of the library already follows. -""" - -from typing import Literal, Self - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -LatencyServiceRequestType = Literal["openai_chat", "openai_responses"] -LatencyServiceCredentialPolicy = Literal[ - "configured_endpoint", "caller_override", "caller_required" -] - - -class LatencyServiceEndpoint(BaseModel): - """One LLM backend registered with the Latency Service. - - The ``model`` field is the endpoint ID used by the Latency Service — - it must be unique across the endpoint list and it is the value the - Latency Service returns health verdicts under. By default it is also - the value written into ``body["model"]`` when calling the upstream; - set ``upstream_model`` when the upstream expects a different name - (e.g. routing the latency-service key ``"openai/gpt-5.5"`` through an - IH gateway that expects ``"openai/openai/gpt-5.5"``). - - Attributes: - model: Latency-service lookup key. Must be unique across the - endpoint list. Also used as ``body["model"]`` unless - ``upstream_model`` is set. - upstream_model: Optional override for ``body["model"]`` sent to - the upstream LLM. Defaults to ``model`` when ``None``. - api_key: API key for the backing LLM API. - base_url: Base URL for the backing LLM API (include ``/v1``). - timeout: Request timeout in seconds, forwarded to the underlying - ``OpenAILLMClient``. ``None`` uses the client default. - request_type: Upstream OpenAI API surface used for this endpoint. - ``"openai_chat"`` sends ``/v1/chat/completions``; ``"openai_responses"`` - sends ``/v1/responses`` for Responses-only upstream models. - """ - - model_config = ConfigDict(frozen=True) - - model: str - upstream_model: str | None = None - api_key: str | None = None - base_url: str | None = None - timeout: float | None = None - request_type: LatencyServiceRequestType = "openai_chat" - - @field_validator("request_type", mode="before") - @classmethod - def _normalize_request_type(cls, value: object) -> object: - if value == "chat": - return "openai_chat" - if value == "responses": - return "openai_responses" - return value - - -class LatencyServiceBackendConfig(BaseModel): - """Configuration for :class:`LatencyServiceLLMBackend`. - - Attributes: - latency_service_url: Base URL of the Latency Service - (e.g. ``"http://latency-service.inference-hub.svc:8080"``). - endpoints: LLM backends to route across. Each must have a - unique ``model`` — this is the routing + health-lookup key. - poll_interval_s: How often the background poller refreshes - health from the Latency Service. Health is cached between - polls; the request hot path never blocks on a network call. - poll_timeout_s: Timeout for the health API call. - max_retries: On error, retry on a different endpoint up to this - many times. Dedup prevents re-selecting an endpoint that - already failed for the same request. - route_model: Client-facing route id this backend serves (the - route-table / YAML route key, e.g. - ``"nvidia/switchyard/gpt-5.4"``). Metrics-only: it joins the - bounded ``requested_model`` label set on - ``switchyard_latency_upstream_attempts_total`` so route-key - traffic is attributed instead of collapsing to ``"other"``. - Has no effect on routing. - credential_policy: Which credential authenticates the upstream call. - The caller key is read from the ``x-switchyard-api-key`` header - (preferred — it survives proxies such as LiteLLM that strip - ``Authorization``), then ``Authorization: Bearer`` / ``x-api-key``, on - every ingress path (``/chat/completions``, ``/responses``, - ``/messages``). ``"configured_endpoint"`` always uses each endpoint's - configured ``api_key`` and ignores any caller key. ``"caller_override"`` - opts into BYO-key forwarding: a caller key is used when present, else - the call falls back to the configured ``api_key``. ``"caller_required"`` - forwards the caller key but never falls back — a request with no caller - key is rejected with HTTP 401 and the configured ``api_key`` is never - used for upstream inference (use this for per-user spend attribution, - e.g. multi-tenant gateway routes). - session_affinity: When ``True``, pin each conversation to the endpoint - that first served it (cache stays warm); a pin is broken only when - its endpoint degrades or the call fails. Per process. Default off. - affinity_max_sessions: Bounded-LRU cap on pinned conversations; ignored - when ``session_affinity`` is off. - affinity_store: Shared L2 pin store behind the in-process LRU. ``"memory"`` - (default) keeps pins per process; ``"redis"`` shares them across - workers/pods and persists them across pod churn. The store is - best-effort — an L2 error never fails a request. - affinity_store_url: Connection URL for the shared store (e.g. - ``"redis://host:6379/0"``). Required when ``affinity_store`` is - ``"redis"``. - affinity_store_ttl_seconds: Expiry for a shared pin. The backend re-pins - on every successful turn, so an active conversation slides its TTL. - affinity_key_prefix: Namespace prefix for shared-store keys. - enable_stats: When ``True`` (default), the factory wires a - :class:`StatsRequestProcessor` + :class:`StatsResponseProcessor` - pair sharing one :class:`StatsAccumulator` and wraps the - backend in :class:`StatsLlmBackend`, so the chain contributes - ``GET /metrics``, ``GET /v1/stats``, and the legacy - ``GET /v1/routing/stats`` aliases via the standard - ``get_endpoint()`` mechanism in :func:`build_switchyard_app`. - """ - - model_config = ConfigDict(frozen=True, extra="forbid") - - latency_service_url: str = "" - endpoints: list[LatencyServiceEndpoint] = Field(default_factory=list) - route_model: str | None = None - poll_interval_s: float = 10.0 - poll_timeout_s: float = 5.0 - max_retries: int = 2 - credential_policy: LatencyServiceCredentialPolicy = "configured_endpoint" - session_affinity: bool = False - affinity_max_sessions: int = Field(default=10_000, ge=0) - affinity_store: Literal["memory", "redis"] = "memory" - affinity_store_url: str | None = None - affinity_store_ttl_seconds: int = Field(default=3_600, gt=0) - affinity_key_prefix: str = "swyd:pin:" - enable_stats: bool = True - - @model_validator(mode="after") - def _affinity_capacity_nonzero_when_enabled(self) -> Self: - # A zero-capacity affinity store retains nothing — silently non-sticky. - if self.session_affinity and self.affinity_max_sessions == 0: - raise ValueError( - "affinity_max_sessions must be > 0 when session_affinity is enabled" - ) - return self - - @model_validator(mode="after") - def _redis_store_requires_url_and_affinity(self) -> Self: - # A shared store is dead config unless affinity is on and reachable. - if self.affinity_store == "redis": - if not self.session_affinity: - raise ValueError( - 'affinity_store="redis" requires session_affinity to be enabled' - ) - if not self.affinity_store_url: - raise ValueError( - 'affinity_store="redis" requires affinity_store_url to be set' - ) - return self diff --git a/switchyard/lib/endpoints/prometheus_emitter.py b/switchyard/lib/endpoints/prometheus_emitter.py index e4929928..aafe0274 100644 --- a/switchyard/lib/endpoints/prometheus_emitter.py +++ b/switchyard/lib/endpoints/prometheus_emitter.py @@ -6,9 +6,8 @@ The accumulator-derived exposition rendered by :func:`switchyard.lib.prometheus_exposition.render_prometheus` covers request flow (counts, tokens, latency). Components that own state which -is not request-derived — Latency Service verdicts, poll-loop health, -endpoint-level live samples — register an emitter here so their lines -appear on the same ``/metrics`` scrape rather than a sidecar URL. +is not request-derived register an emitter here so their lines appear on +the same ``/metrics`` scrape rather than a sidecar URL. Single-process table by design: a Switchyard server is one process, emitters are owned by component lifetimes, and the table is diff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.py index 31d96f94..3ffdc267 100644 --- a/switchyard/lib/endpoints/stats_endpoint.py +++ b/switchyard/lib/endpoints/stats_endpoint.py @@ -68,10 +68,9 @@ async def reset_stats() -> dict[str, str]: async def get_metrics() -> Response: """Prometheus text-format exposition of the shared stats snapshot. - Components that own non-request-derived state (Latency Service - verdicts, poll-loop health) contribute extra lines via - :mod:`switchyard.lib.endpoints.prometheus_emitter` so a single - ``/metrics`` scrape carries both surfaces. + Components that own non-request-derived state contribute extra + lines via :mod:`switchyard.lib.endpoints.prometheus_emitter` so a + single ``/metrics`` scrape carries both surfaces. """ snapshot = await stats.snapshot() outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n" diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.py index 285eda79..66de361d 100644 --- a/switchyard/lib/endpoints/upstream_error.py +++ b/switchyard/lib/endpoints/upstream_error.py @@ -3,9 +3,9 @@ """Endpoint-side helper: convert chain exceptions into upstream-status responses. -Python LLM backends (e.g. :class:`LatencyServiceLLMBackend`) stash the -upstream HTTP status / body into ``ctx.metadata`` before raising the -upstream provider's exception. Rust backends attach typed +Python LLM backends stash the upstream HTTP status / body into +``ctx.metadata`` before raising the upstream provider's exception. Rust +backends attach typed ``status_code`` and ``body`` attributes to ``SwitchyardUpstreamError`` so endpoints can preserve provider failures without parsing exception text. """ @@ -53,9 +53,8 @@ def record_upstream_attempt_success(ctx: ProxyContext) -> None: to one successful attempt observable here. No-op when a backend already counted its own attempts - (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — e.g. - :class:`LatencyServiceLLMBackend`, whose retry fan-out must not be - double-counted here. + (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — its retry fan-out must not + be double-counted here. """ if ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED): return diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py index b7d3de6d..1b06d983 100644 --- a/switchyard/lib/profiles/__init__.py +++ b/switchyard/lib/profiles/__init__.py @@ -21,7 +21,6 @@ HeaderRoutingDecision, HeaderRoutingProfile, ) -from switchyard.lib.profiles.latency_service import LatencyServiceProfileConfig from switchyard.lib.profiles.noop import NoopProfile, NoopProfileConfig from switchyard.lib.profiles.passthrough import PassthroughProfileConfig from switchyard.lib.profiles.protocols import ( @@ -61,7 +60,6 @@ "HeaderRoutingConfig", "HeaderRoutingDecision", "HeaderRoutingProfile", - "LatencyServiceProfileConfig", "NoopProfile", "NoopProfileConfig", "PassthroughProfileConfig", diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py index 96c59db2..f6f5b4ce 100644 --- a/switchyard/lib/profiles/escalation_router_profile_config.py +++ b/switchyard/lib/profiles/escalation_router_profile_config.py @@ -138,9 +138,9 @@ def build(self) -> ComponentChainProfile: def _build_affinity_l2(config: EscalationRouterConfig) -> AffinityPinStore | None: """Build the optional shared L2 latch store (``None`` = L1-only). - Mirrors the latency route's ``affinity_store`` semantics; ``"redis"`` - imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays - optional. Config validation guarantees a URL when the store is Redis. + Uses the shared ``affinity_store`` semantics; ``"redis"`` imports + :class:`RedisPinStore` lazily so the ``redis`` dependency stays optional. + Config validation guarantees a URL when the store is Redis. """ if config.affinity_store != "redis": return None diff --git a/switchyard/lib/profiles/latency_service.py b/switchyard/lib/profiles/latency_service.py deleted file mode 100644 index abdb144d..00000000 --- a/switchyard/lib/profiles/latency_service.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Profile-owned latency-service routing construction.""" - -from __future__ import annotations - -from typing import Self - -from switchyard.lib.config import LatencyServiceBackendConfig -from switchyard.lib.profiles.chain import ComponentChainProfile -from switchyard.lib.profiles.table import profile_config - - -@profile_config("latency_service") -class LatencyServiceProfileConfig: - """Profile config wrapper for health-aware latency-service profiles.""" - - config: LatencyServiceBackendConfig - - @classmethod - def from_config(cls, config: LatencyServiceBackendConfig) -> Self: - """Create a profile config from the validated parsing model.""" - return cls(config=config) - - def build(self) -> ComponentChainProfile: - """Build the latency-service profile runtime.""" - from switchyard.lib.backends import LatencyServiceLLMBackend - - backend = LatencyServiceLLMBackend(self.config) - return ComponentChainProfile( - backend=backend, - ) - - -__all__ = ["LatencyServiceProfileConfig"] diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py index 864ddfb6..01c9b78a 100644 --- a/switchyard/lib/proxy_context.py +++ b/switchyard/lib/proxy_context.py @@ -58,8 +58,8 @@ #: Set truthy by a backend that records its own per-attempt #: ``switchyard.lib.endpoints.outcome_metrics`` upstream-attempt counters -#: (e.g. :class:`LatencyServiceLLMBackend`, which retries across endpoints and -#: must count each attempt). When present, the endpoint layer skips its +#: (e.g. a backend that retries across endpoints and must count each +#: attempt). When present, the endpoint layer skips its #: single-attempt fallback recording for this request so retry fan-out is not #: double-counted. Absent for the Rust native / passthrough / multi backends, #: which issue exactly one upstream attempt per call and have no Python retry diff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.py index 0cc19354..0b1b2491 100644 --- a/switchyard/lib/redis_pin_store.py +++ b/switchyard/lib/redis_pin_store.py @@ -6,8 +6,8 @@ Implements :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` against a standalone Redis so pins are visible to every Switchyard worker/pod and survive pod churn. Keys are namespaced ``{key_prefix}{session_key}`` and expire after -``ttl_seconds``; the latency backend re-pins on every successful turn, so an -active conversation slides its own TTL. +``ttl_seconds``; callers re-pin on every successful turn, so an active +conversation slides its own TTL. The ``redis`` dependency is optional (``switchyard[affinity-redis]``) and is imported lazily, so the default install never pulls it in. Socket timeouts are diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py index a4ef6e57..1d5f48d7 100644 --- a/switchyard/lib/session_affinity.py +++ b/switchyard/lib/session_affinity.py @@ -71,8 +71,8 @@ class SessionAffinity: stickiness: derive a stable conversation key (system prompt + first user message, memoized on the request context) and store/look up a pinned value in a bounded LRU. The pinned value is opaque — a tier label for the - classifier router, an endpoint id for the latency backend. Each caller keeps - its own *policy* (when to write a pin, whether to honor one) on top. + classifier or escalation router. Each caller keeps its own *policy* + (when to write a pin, whether to honor one) on top. The in-process ``SessionCache`` is the L1. An optional ``l2`` implementing :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` is a shared, diff --git a/switchyard/lib/tracing.py b/switchyard/lib/tracing.py index ff295487..5f80a7f1 100644 --- a/switchyard/lib/tracing.py +++ b/switchyard/lib/tracing.py @@ -64,8 +64,8 @@ def routing_span(name: str) -> Iterator[Span]: def set_tags(span: Span, tags: Mapping[str, Any]) -> None: """Set each non-``None`` tag on *span*. - ``None`` values are skipped so an unavailable signal (e.g. no latency-service - poll has completed yet) leaves no empty/misleading tag on the span. + ``None`` values are skipped so an unavailable signal (e.g. a signal that + has not been recorded yet) leaves no empty/misleading tag on the span. """ for key, value in tags.items(): if value is not None: diff --git a/tests/e2e/test_latency_service_llm_backend.py b/tests/e2e/test_latency_service_llm_backend.py deleted file mode 100644 index fbd93dc5..00000000 --- a/tests/e2e/test_latency_service_llm_backend.py +++ /dev/null @@ -1,427 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Integration tests for :class:`LatencyServiceLLMBackend`. - -These tests make real LLM calls through the latency-service backend -to validate end-to-end routing against an external health source. - -A lightweight mock Latency Service runs in-process to supply health -verdicts. The LLM calls go to a real backend (OpenRouter by default, -with NVIDIA Inference Hub as a fallback). - -Prerequisites: - - OPENROUTER_API_KEY or NVIDIA_API_KEY environment variable - -Run with: - OPENROUTER_API_KEY=sk-or-... pytest tests/e2e/test_latency_service_llm_backend.py -v -""" - -import json -import os -import threading -import time -from http.server import BaseHTTPRequestHandler, HTTPServer -from urllib.parse import parse_qs, urlparse - -import pytest - -from switchyard.lib.backends.health_poller import ( - EndpointHealthStatus, -) -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.core import ( - ChatRequest, - ChatResponseType, - response_type_matches, -) - -from .conftest import find_free_port, get_nvidia_config - -pytestmark = pytest.mark.integration - -_nvidia = get_nvidia_config() -_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set" - -BACKEND_BASE_URL = _nvidia["base_url"] - -if _nvidia["provider"] == "openrouter": - MODEL_A = _nvidia["model"] - MODEL_B = os.environ.get("OPENROUTER_MODEL_B") or "anthropic/claude-opus-4.7" -else: - MODEL_A = "openai/openai/gpt-5.2" - MODEL_B = "azure/openai/gpt-5.2" - -SIMPLE_MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello! Can you help me?"}, -] - - -# --------------------------------------------------------------------------- -# Mock Latency Service -# --------------------------------------------------------------------------- - - -class _HealthState: - """Thread-safe mutable health state for the mock Latency Service.""" - - def __init__(self) -> None: - self._lock = threading.Lock() - self._health: dict[str, str] = {} - self.request_count = 0 - - def set_health(self, model_id: str, status: str) -> None: - with self._lock: - self._health[model_id] = status - - def get_response(self, requested_ids: list[str]) -> dict: - with self._lock: - self.request_count += 1 - return { - "endpoint_health": { - mid: { - "status": self._health.get(mid, "unknown"), - "last_latency_ms": None, - } - for mid in requested_ids - } - } - - -def _make_handler(state: _HealthState): - """Create an HTTP request handler class bound to the given health state.""" - - class _Handler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - parsed = urlparse(self.path) - if parsed.path == "/v1/endpoints/health": - qs = parse_qs(parsed.query) - endpoint_ids = qs.get("endpoint_ids", []) - body = json.dumps(state.get_response(endpoint_ids)) - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(body.encode()) - else: - self.send_response(404) - self.end_headers() - - def log_message(self, format, *args) -> None: # noqa: A002 - pass # Suppress request logging in test output - - return _Handler - - -class MockLatencyService: - """Manages a mock Latency Service HTTP server for testing.""" - - def __init__(self) -> None: - self.state = _HealthState() - self.port = find_free_port() - handler = _make_handler(self.state) - self._server = HTTPServer(("127.0.0.1", self.port), handler) - self._thread = threading.Thread( - target=self._server.serve_forever, - daemon=True, - name="mock-latency-service", - ) - self._thread.start() - - @property - def url(self) -> str: - return f"http://127.0.0.1:{self.port}" - - def set_health(self, model_id: str, status: str) -> None: - self.state.set_health(model_id, status) - - def shutdown(self) -> None: - self._server.shutdown() - self._thread.join(timeout=5.0) - - -@pytest.fixture() -def mock_latency_service(): - """Provide a mock Latency Service for the duration of a test.""" - svc = MockLatencyService() - yield svc - svc.shutdown() - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _endpoint(model: str) -> LatencyServiceEndpoint: - return LatencyServiceEndpoint( - model=model, - api_key=_nvidia["api_key"], - base_url=BACKEND_BASE_URL, - ) - - -def _make_backend( - latency_service_url: str, - models: list[str], - poll_interval_s: float = 1.0, - **kwargs, -) -> LatencyServiceLLMBackend: - config = LatencyServiceBackendConfig( - latency_service_url=latency_service_url, - endpoints=[_endpoint(m) for m in models], - poll_interval_s=poll_interval_s, - **kwargs, - ) - return LatencyServiceLLMBackend(config) - - -def _chat_request(stream: bool = False) -> ChatRequest: - # ``max_tokens`` is deliberately generous so reasoning-capable - # backends (e.g. the GPT-5 / Qwen 3.5 families) have room to spend - # the first several hundred tokens on internal reasoning before - # emitting user-visible ``content``. Matches the value used by - # the sibling passthrough e2e suites. - body: dict = { - "messages": SIMPLE_MESSAGES, - "max_tokens": 2048, - "stream": stream, - } - return ChatRequest.openai_chat(body) # type: ignore[arg-type] - - -def _wait_for_first_poll( - backend: LatencyServiceLLMBackend, - timeout: float = 10.0, -) -> None: - """Block until the background poller completes at least one poll.""" - deadline = time.monotonic() + timeout - while not backend.is_ready(): - if time.monotonic() > deadline: - raise TimeoutError("Poller did not complete first poll within timeout") - time.sleep(0.05) - - -# ================================================================== # -# Non-streaming tests -# ================================================================== # - - -@pytest.mark.skipif(not _nvidia["api_key"], reason=_skip_reason) -class TestLatencyServiceBackendNonStreaming: - """Non-streaming requests through the latency-service backend.""" - - async def test_healthy_endpoint_serves_request(self, mock_latency_service): - """A HEALTHY endpoint should successfully serve a non-streaming request.""" - mock_latency_service.set_health(MODEL_A, "healthy") - backend = _make_backend(mock_latency_service.url, [MODEL_A]) - try: - _wait_for_first_poll(backend) - - ctx = ProxyContext() - response = await backend.call(ctx, _chat_request()) - - assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION) - assert response.body["choices"][0]["message"]["content"] - assert response.body["choices"][0]["message"]["role"] == "assistant" - assert ctx.selected_model == MODEL_A - finally: - await backend.shutdown() - - async def test_response_has_usage(self, mock_latency_service): - """Response should include token usage statistics.""" - mock_latency_service.set_health(MODEL_A, "healthy") - backend = _make_backend(mock_latency_service.url, [MODEL_A]) - try: - _wait_for_first_poll(backend) - - ctx = ProxyContext() - response = await backend.call(ctx, _chat_request()) - - assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION) - assert response.body["usage"] is not None - assert response.body["usage"]["prompt_tokens"] > 0 - assert response.body["usage"]["completion_tokens"] > 0 - finally: - await backend.shutdown() - - -# ================================================================== # -# Streaming tests -# ================================================================== # - - -@pytest.mark.skipif(not _nvidia["api_key"], reason=_skip_reason) -class TestLatencyServiceBackendStreaming: - """Streaming requests through the latency-service backend.""" - - async def test_streaming_yields_chunks(self, mock_latency_service): - """Streaming request should yield content chunks.""" - mock_latency_service.set_health(MODEL_A, "healthy") - backend = _make_backend(mock_latency_service.url, [MODEL_A]) - try: - _wait_for_first_poll(backend) - - ctx = ProxyContext() - response = await backend.call(ctx, _chat_request(stream=True)) - assert response_type_matches(response, ChatResponseType.OPENAI_STREAM) - - chunks = [chunk async for chunk in response.stream] - assert len(chunks) >= 2, "Expected at least one content chunk + stop chunk" - - text_parts = [ - c.choices[0].delta.content - for c in chunks - if c.choices and c.choices[0].delta and c.choices[0].delta.content - ] - full_text = "".join(text_parts) - assert len(full_text) > 0 - finally: - await backend.shutdown() - - -# ================================================================== # -# Health-based routing -# ================================================================== # - - -@pytest.mark.skipif(not _nvidia["api_key"], reason=_skip_reason) -class TestLatencyServiceBackendHealth: - """Verify health-based endpoint selection with real LLM calls.""" - - async def test_healthy_preferred_over_degraded(self, mock_latency_service): - """When one endpoint is HEALTHY and another DEGRADED, all traffic - should go to the HEALTHY one.""" - mock_latency_service.set_health(MODEL_A, "healthy") - mock_latency_service.set_health(MODEL_B, "degraded") - backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B]) - try: - _wait_for_first_poll(backend) - - selected_models = set() - for _ in range(3): - ctx = ProxyContext() - await backend.call(ctx, _chat_request()) - selected_models.add(ctx.selected_model) - - assert selected_models == {MODEL_A}, ( - f"Expected all traffic to HEALTHY model, got: {selected_models}" - ) - finally: - await backend.shutdown() - - async def test_unknown_falls_back_to_random(self, mock_latency_service): - """When all endpoints are UNKNOWN, the backend should distribute - traffic randomly across all endpoints.""" - mock_latency_service.set_health(MODEL_A, "unknown") - mock_latency_service.set_health(MODEL_B, "unknown") - backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B]) - try: - _wait_for_first_poll(backend) - - selected_models = set() - for _ in range(6): - ctx = ProxyContext() - await backend.call(ctx, _chat_request()) - selected_models.add(ctx.selected_model) - - assert len(selected_models) > 1, ( - f"Expected random distribution across models, got: {selected_models}" - ) - finally: - await backend.shutdown() - - async def test_poller_receives_health_updates(self, mock_latency_service): - """The background poller should pick up health changes from the - Latency Service and reflect them in the health cache.""" - mock_latency_service.set_health(MODEL_A, "healthy") - backend = _make_backend( - mock_latency_service.url, [MODEL_A], poll_interval_s=0.5, - ) - try: - _wait_for_first_poll(backend) - assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.HEALTHY - - mock_latency_service.set_health(MODEL_A, "degraded") - time.sleep(1.5) # Wait for at least one poll cycle - - assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.DEGRADED - finally: - await backend.shutdown() - - -# ================================================================== # -# Readiness -# ================================================================== # - - -@pytest.mark.skipif(not _nvidia["api_key"], reason=_skip_reason) -class TestLatencyServiceBackendReadiness: - """Verify is_ready() behavior with a real poller.""" - - async def test_ready_after_first_poll(self, mock_latency_service): - """is_ready() should return True once the poller has fetched health.""" - mock_latency_service.set_health(MODEL_A, "healthy") - backend = _make_backend(mock_latency_service.url, [MODEL_A]) - try: - assert not backend.is_ready() - _wait_for_first_poll(backend) - assert backend.is_ready() - finally: - await backend.shutdown() - - -# ================================================================== # -# Session affinity (sticky routing) -# ================================================================== # - - -@pytest.mark.skipif(not _nvidia["api_key"], reason=_skip_reason) -class TestLatencyServiceBackendStickiness: - """Verify session affinity pins a multi-turn conversation to one endpoint.""" - - async def test_conversation_sticks_to_one_endpoint(self, mock_latency_service): - """With ``session_affinity`` on, every turn of one conversation routes - to the same endpoint — even though both endpoints are HEALTHY with no - latency sample and would otherwise be picked at random (the foil is - ``test_unknown_falls_back_to_random``, which spreads without affinity).""" - mock_latency_service.set_health(MODEL_A, "healthy") - mock_latency_service.set_health(MODEL_B, "healthy") - backend = _make_backend( - mock_latency_service.url, [MODEL_A, MODEL_B], session_affinity=True, - ) - try: - _wait_for_first_poll(backend) - - # One conversation that grows each turn but keeps a stable prefix - # (same system + first user message → same session key). - messages = list(SIMPLE_MESSAGES) - selected: list[str] = [] - for turn in range(6): - ctx = ProxyContext() - request = ChatRequest.openai_chat( - {"messages": messages, "max_tokens": 128}, # type: ignore[arg-type] - ) - response = await backend.call(ctx, request) - assert response_type_matches( - response, ChatResponseType.OPENAI_COMPLETION - ) - selected.append(ctx.selected_model) - messages = messages + [ - {"role": "assistant", "content": "Sure."}, - {"role": "user", "content": f"Follow-up {turn + 1}?"}, - ] - - assert len(set(selected)) == 1, ( - f"affinity should pin the conversation to one endpoint, got {selected}" - ) - assert selected[0] in {MODEL_A, MODEL_B} - finally: - await backend.shutdown() diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.py index 6af37699..b1686e62 100644 --- a/tests/readme/test_readme.py +++ b/tests/readme/test_readme.py @@ -71,10 +71,9 @@ def test_python_snippet_tripwire(readme_text: str) -> None: def _validate_route_blocks(text: str, source: Path) -> int: # Schema/key validation, not a full chain build: building documented routes # is NOT hermetic — `passthrough` with `discover: true` does a live catalog - # fetch and `latency_service` polls. The - # schema layer (route type + per-type key allowlist) is what we can check - # offline, and it catches the likeliest drift: a renamed `type:` or a key - # that no longer exists on that type. + # fetch. The schema layer (route type + per-type key allowlist) is what we + # can check offline, and it catches the likeliest drift: a renamed `type:` + # or a key that no longer exists on that type. blocks = _code_blocks(text, "yaml") validated_routes = 0 for idx, block in enumerate(blocks): diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.py index 6613fd25..57710da9 100644 --- a/tests/test_error_source_annotation.py +++ b/tests/test_error_source_annotation.py @@ -5,9 +5,9 @@ Every client-facing error carries ``x-switchyard-error-source`` naming the layer that originated it (``switchyard`` | ``provider``), plus -``x-switchyard-upstream-model`` when a routing selection had happened. The -backend-side ctx stamps are covered in ``test_latency_service_llm_backend.py``; -these tests cover the endpoint layer that renders them. +``x-switchyard-upstream-model`` when a routing selection had happened. These +tests cover the endpoint layer that renders those annotations from the ctx +stamps a backend sets. """ from __future__ import annotations diff --git a/tests/test_inference_e2e.py b/tests/test_inference_e2e.py index 2164ede1..f8420b46 100644 --- a/tests/test_inference_e2e.py +++ b/tests/test_inference_e2e.py @@ -13,10 +13,12 @@ import json from collections.abc import AsyncIterator +from unittest.mock import AsyncMock, patch import httpx import pytest from fastapi import FastAPI +from fastapi.testclient import TestClient from openai.types.chat import ChatCompletion, ChatCompletionChunk from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice @@ -93,6 +95,23 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: raise self._exc +class _TranslatingMockLLMBackend(LLMBackend): + """Chat-only backend that translates the inbound request to Chat at the top of + ``call`` — mirroring any real chat-completions backend (native / passthrough). + + An unsupported inbound field (e.g. a bad message role) is rejected by the + translation engine here, before any upstream call, exactly as it would be for + a production chat backend. + """ + + def supported_request_types(self) -> list[ChatRequestType]: + return [ChatRequestType.OPENAI_CHAT] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + request = TranslationEngine().request_to_any_of(request, [ChatRequestType.OPENAI_CHAT]) + return ChatResponse.openai_completion(_make_completion()) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -257,6 +276,17 @@ async def raising_client(raising_app: FastAPI) -> AsyncIterator[httpx.AsyncClien yield c +@pytest.fixture +def translating_app() -> FastAPI: + """App whose chat-only backend translates the inbound request to Chat, so an + invalid inbound role is rejected during translation before any upstream call.""" + switchyard = Switchyard( + backend=_TranslatingMockLLMBackend(), + translator=TranslationEngine(), + ) + return build_switchyard_app(switchyard) + + # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- @@ -502,3 +532,68 @@ async def test_anthropic_tool_use_translation( block = tool_use_blocks[0] assert block["name"] == "get_weather" assert block["input"] == {"city": "Paris"} + + +class TestEndpointErrorContract: + """Endpoint error-mapping contracts previously covered only through the + removed latency-service backend, re-vehicled onto in-process backends.""" + + def test_post_dispatch_exception_returns_json_500(self, app: FastAPI) -> None: + """An exception raised AFTER dispatch (e.g. during result serialization) + must surface as a JSON 500 envelope, not FastAPI's plain-text 500.""" + + class _Unserializable: + def model_dump(self) -> None: + raise RuntimeError("serialization exploded") + + with patch( + "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request", + new_callable=AsyncMock, + return_value=_Unserializable(), + ): + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={"model": "any-model", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 500 + assert response.headers["content-type"].startswith("application/json") + body = response.json() + assert body["error"]["type"] == "internal_error" + assert body["error"]["code"] == "internal_chain_error" + assert "serialization exploded" in body["error"]["message"] + + @pytest.mark.parametrize( + ("path", "payload"), + [ + ( + "/v1/responses", + {"model": "m", "input": [{"type": "message", "role": "api", "content": "ping"}]}, + ), + ( + "/v1/messages", + {"model": "m", "max_tokens": 16, "messages": [{"role": "api", "content": "ping"}]}, + ), + ], + ) + def test_invalid_inbound_role_surfaces_as_internal_error( + self, translating_app: FastAPI, path: str, payload: dict[str, object] + ) -> None: + """An unsupported inbound message role is rejected by the translation engine + when a chat-only backend translates the request, before any upstream call. + + NOTE: this currently surfaces as a 500 ``internal_error``. The removed + latency-service backend special-cased translation ``invalid_value`` errors + into a provider-compatible 400; no surviving backend reproduces that, so + the standalone-server contract is now a 500. Update this test (and the + endpoint/translation layer) if a 400 is restored. + """ + with TestClient(translating_app, raise_server_exceptions=False) as client: + response = client.post(path, json=payload) + + assert response.status_code == 500 + body = response.json() + assert body["error"]["type"] == "internal_error" + assert '"api"' in body["error"]["message"] + assert "role" in body["error"]["message"] diff --git a/tests/test_latency_service_health_metrics.py b/tests/test_latency_service_health_metrics.py deleted file mode 100644 index 9329137e..00000000 --- a/tests/test_latency_service_health_metrics.py +++ /dev/null @@ -1,311 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end gates for latency-service routing-state metrics. - -The Prometheus exposition rendered from the accumulator covers request -flow but says nothing about *why* a request landed on a given endpoint. -The latency-service backend now contributes per-endpoint verdict gauges -and poll-loop health gauges so dashboards can see the routing inputs, -not just outputs. -""" - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest -from fastapi.testclient import TestClient - -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.endpoints import prometheus_emitter -from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard -from switchyard.server.switchyard_app import build_switchyard_app - - -@pytest.fixture(autouse=True) -def _clean_table(): - prometheus_emitter._clear_for_tests() - yield - prometheus_emitter._clear_for_tests() - - -def _config(*models: str) -> LatencyServiceBackendConfig: - return LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model=model, - base_url=f"http://llm-{model}.test/v1", - api_key="test-key", - ) - for model in models - ], - ) - - -def _latency_service_switchyard( - config: LatencyServiceBackendConfig, -) -> ProfileSwitchyard: - """Build the latency-service profile-backed serving adapter.""" - return ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components(enable_stats=config.enable_stats) - ) - - -def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend: - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"): - return LatencyServiceLLMBackend(config) - - -def _emit(backend: LatencyServiceLLMBackend) -> str: - """Render this backend's emitter output as a single string.""" - return "\n".join(backend._render_prometheus_lines()) - - -class TestEndpointStatusGauge: - def test_emits_one_row_per_status_per_endpoint(self) -> None: - """Three-state encoding lets `sum by (status)` give a clean histogram.""" - backend = _make_backend(_config("model-A", "model-B")) - with backend._cache_lock: - backend._health_cache["model-A"] = EndpointHealth( - EndpointHealthStatus.HEALTHY, 100.0, - ) - backend._health_cache["model-B"] = EndpointHealth( - EndpointHealthStatus.DEGRADED, 800.0, - ) - - out = _emit(backend) - assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 1' in out - assert 'switchyard_endpoint_status{model="model-A",status="degraded"} 0' in out - assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 0' in out - assert 'switchyard_endpoint_status{model="model-B",status="degraded"} 1' in out - assert 'switchyard_endpoint_status{model="model-B",status="healthy"} 0' in out - - def test_unknown_default_before_first_poll(self) -> None: - """New backends start with every endpoint UNKNOWN.""" - backend = _make_backend(_config("model-A")) - out = _emit(backend) - assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 1' in out - assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 0' in out - - -class TestEndpointLatencyGauge: - def test_emits_only_for_endpoints_with_sample(self) -> None: - """Absence is meaningful: don't emit a zero where no sample exists.""" - backend = _make_backend(_config("with-sample", "no-sample")) - with backend._cache_lock: - backend._health_cache["with-sample"] = EndpointHealth( - EndpointHealthStatus.HEALTHY, 250.5, - ) - backend._health_cache["no-sample"] = EndpointHealth( - EndpointHealthStatus.HEALTHY, None, - ) - - out = _emit(backend) - assert ( - 'switchyard_endpoint_last_latency_ms{model="with-sample"} 250.5' in out - ) - assert 'switchyard_endpoint_last_latency_ms{model="no-sample"}' not in out - - -class TestPollHealthGauges: - def test_before_first_poll_signals_never_polled(self) -> None: - """poll_ok=0 + no poll_age line lets a scraper detect "never polled".""" - backend = _make_backend(_config("model-A")) - out = _emit(backend) - assert "switchyard_latency_service_poll_ok 0" in out - assert "switchyard_latency_service_poll_age_seconds" not in out.replace( - "# HELP switchyard_latency_service_poll_age_seconds", "" - ).replace( - "# TYPE switchyard_latency_service_poll_age_seconds", "" - ) - assert "switchyard_latency_service_polls_total 0" in out - assert "switchyard_latency_service_poll_failures_total 0" in out - - def test_after_successful_poll_emits_age_and_ok(self) -> None: - backend = _make_backend(_config("model-A")) - backend._poller._poll_count = 3 - backend._poller._last_poll_ok = True - backend._poller._last_success_at = __import__("time").monotonic() - 1.0 - - out = _emit(backend) - assert "switchyard_latency_service_poll_ok 1" in out - assert "switchyard_latency_service_polls_total 3" in out - # Age value is positive — exact value depends on test wall time. - age_lines = [ - line for line in out.splitlines() - if line.startswith("switchyard_latency_service_poll_age_seconds ") - ] - assert len(age_lines) == 1 - age_value = float(age_lines[0].split()[-1]) - assert age_value > 0 - - def test_poll_failure_flips_ok_to_zero(self) -> None: - """Even with prior successes, a recorded failure flips poll_ok off - so dashboards alarm on the *latest* poll result, not history.""" - backend = _make_backend(_config("model-A")) - backend._poller._poll_count = 5 - backend._poller._last_success_at = __import__("time").monotonic() - 1.0 - backend._poller._poll_failures = 1 - backend._poller._last_poll_ok = False - - out = _emit(backend) - assert "switchyard_latency_service_poll_ok 0" in out - assert "switchyard_latency_service_poll_failures_total 1" in out - - def test_success_after_prior_failure_emits_ok(self) -> None: - """Prior failures are historical counters; poll_ok tracks latest outcome.""" - backend = _make_backend(_config("model-A")) - backend._poller._poll_count = 5 - backend._poller._poll_failures = 1 - backend._poller._last_poll_ok = True - backend._poller._last_success_at = __import__("time").monotonic() - 1.0 - - out = _emit(backend) - assert "switchyard_latency_service_poll_ok 1" in out - assert "switchyard_latency_service_poll_failures_total 1" in out - - -class TestEmitterLifecycle: - async def test_construction_registers_emitter(self) -> None: - assert prometheus_emitter._EMITTERS == [] - backend = _make_backend(_config("model-A")) - assert len(prometheus_emitter._EMITTERS) == 1 - # Confirm table output includes this backend's lines. - assert "switchyard_endpoint_status" in prometheus_emitter.render() - await backend.shutdown() - assert prometheus_emitter._EMITTERS == [] - - async def test_shutdown_idempotent(self) -> None: - """Lifespan tear-down may call shutdown more than once.""" - backend = _make_backend(_config("model-A")) - await backend.shutdown() - await backend.shutdown() - assert prometheus_emitter._EMITTERS == [] - - -class TestRoutingOverhead: - async def test_metrics_record_routing_overhead_for_python_backend(self) -> None: - """End-to-end: a request through the latency-service chain must - publish ``switchyard_routing_overhead_ms`` with non-zero count. - - Before ``ctx.backend_call_latency_ms`` existed, the Rust response - processor had no backend-latency reading to subtract from total, - and the summary stayed at ``count=0`` even under heavy load. - """ - from unittest.mock import AsyncMock - - from openai.types.chat import ChatCompletion - from openai.types.chat.chat_completion import Choice - from openai.types.chat.chat_completion_message import ChatCompletionMessage - - from switchyard_rust.core import ChatRequest - - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard( - LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="test-key", - base_url="http://llm.test/v1", - ), - ], - ) - ) - - backend = next( - component - for component in switchyard.iter_components() - if hasattr(component, "_clients") - ) - completion = ChatCompletion( - id="cmpl-test", - object="chat.completion", - created=1700000000, - model="model-A", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="ok"), - finish_reason="stop", - ) - ], - ) - backend._clients["model-A"].acompletion = AsyncMock(return_value=completion) - - await switchyard.call(ChatRequest.openai_chat({ - "model": "model-A", - "messages": [{"role": "user", "content": "hi"}], - })) - - app = build_switchyard_app(switchyard) - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - - assert metrics.status_code == 200 - body = metrics.text - count_line = next( - line for line in body.splitlines() - if line.startswith("switchyard_routing_overhead_ms_count") - ) - count = int(count_line.split()[-1]) - assert count >= 1, f"expected non-zero routing_overhead samples, body=\n{body}" - - -class TestEndToEnd: - def test_metrics_endpoint_includes_health_lines(self) -> None: - """Wire the full chain via the recipe and verify /metrics carries - the new lines on top of the standard accumulator exposition.""" - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard( - LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="test-key", - base_url="http://llm.test/v1", - ), - ], - ) - ) - app = build_switchyard_app(switchyard) - - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - - assert metrics.status_code == 200 - body = metrics.text - # Both surfaces coexist on the same scrape. - assert "switchyard_requests_total" in body - assert "switchyard_endpoint_status" in body - assert "switchyard_latency_service_poll_ok" in body - assert "switchyard_latency_service_polls_total" in body diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py deleted file mode 100644 index 00e9adda..00000000 --- a/tests/test_latency_service_llm_backend.py +++ /dev/null @@ -1,1976 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for :class:`LatencyServiceLLMBackend` (usage case).""" - -import json -import random -import threading -import time -import uuid -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import openai -import pytest -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage -from pydantic import ValidationError - -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.backends.latency_service_llm_backend import ( - SPEND_LOGS_METADATA_HEADER, - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.proxy_context import ( - CTX_ERROR_SOURCE, - CTX_ROUTE_SELECTION, - CTX_UPSTREAM_HTTP_STATUS, - CTX_UPSTREAM_MODEL, - ProxyContext, -) -from switchyard.lib.switchyard import Switchyard -from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ( - ChatRequest, - ChatRequestType, - ChatResponseType, - response_type_matches, -) -from switchyard_rust.translation import TranslationEngine - -LATENCY_SERVICE_URL = "http://latency-service.test:8080" - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _ep( - model: str, - base_url: str | None = None, - request_type: str = "openai_chat", -) -> LatencyServiceEndpoint: - return LatencyServiceEndpoint( - model=model, - base_url=base_url or f"http://llm-{model}.test", - api_key="test-key", - request_type=request_type, # type: ignore[arg-type] - ) - - -def _config(*models: str, **kwargs) -> LatencyServiceBackendConfig: - request_type = str(kwargs.pop("request_type", "openai_chat")) - return LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep(m, request_type=request_type) for m in models], - **kwargs, - ) - - -def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend: - """Construct a backend with a mocked OpenAILLMClient and stopped poller.""" - with patch( - "switchyard.lib.backends" - ".latency_service_llm_backend.OpenAILLMClient" - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock( - name=f"client-{kw.get('base_url')}" - ) - with patch.object(HealthPoller, "start"): - backend = LatencyServiceLLMBackend(config) - return backend - - -def _set_health( - backend: LatencyServiceLLMBackend, - health_map: dict[str, EndpointHealthStatus | EndpointHealth], -) -> None: - """Write directly to the backend's health cache for deterministic tests. - - Accepts either a bare ``EndpointHealthStatus`` (auto-wrapped with no - latency sample) or a full ``EndpointHealth`` snapshot. - """ - with backend._cache_lock: - for model_id, value in health_map.items(): - if isinstance(value, EndpointHealthStatus): - value = EndpointHealth(status=value) - backend._health_cache[model_id] = value - - -def _make_completion( - *, model: str = "test-model", content: str = "hello", -) -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model=model, - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - ) - - -def _openai_request(**overrides) -> ChatRequest: - body: dict = { - "model": "incoming-model", - "messages": [{"role": "user", "content": "hi"}], - } - body.update(overrides) - return ChatRequest.openai_chat(body) # type: ignore[arg-type] - - -def _api_status_error(status_code: int) -> openai.APIStatusError: - """Synthetic APIStatusError carrying the given upstream status code.""" - response = httpx.Response( - status_code, - request=httpx.Request("POST", "http://llm.test/v1/chat/completions"), - json={"error": {"message": "synthetic"}}, - ) - return openai.APIStatusError( - "synthetic", response=response, body={"error": "synthetic"} - ) - - -# --------------------------------------------------------------------------- -# Health poller helpers -# --------------------------------------------------------------------------- - - -def _make_poller( - model_ids: list[str], - health_cache: dict[str, EndpointHealth], - poll_interval_s: float = 100.0, -) -> HealthPoller: - cache_lock = threading.Lock() - return HealthPoller( - latency_service_url=LATENCY_SERVICE_URL, - model_ids=model_ids, - health_cache=health_cache, - cache_lock=cache_lock, - poll_interval_s=poll_interval_s, - poll_timeout_s=5.0, - ) - - -def _health_response(status_code: int, **kwargs) -> httpx.Response: - return httpx.Response( - status_code, - request=httpx.Request("GET", LATENCY_SERVICE_URL + "/v1/endpoints/health"), - **kwargs, - ) - - -def _mock_health_response(poller: HealthPoller, response: httpx.Response) -> None: - mock_client = MagicMock() - mock_client.get.return_value = response - poller._http_client = mock_client - - -def _run_one_poll(poller: HealthPoller) -> None: - """Execute exactly one poll iteration and then exit the run loop.""" - original_wait = poller._stop_event.wait - call_count = 0 - - def _wait_then_stop(timeout=None): - nonlocal call_count - call_count += 1 - if call_count >= 1: - poller._stop_event.set() - return original_wait(timeout=0) - - poller._stop_event.wait = _wait_then_stop # type: ignore[method-assign] - poller.run() - - -# --------------------------------------------------------------------------- -# Config validation -# --------------------------------------------------------------------------- - - -class TestConfigValidation: - def test_supported_request_types(self): - backend = _make_backend(_config("model-A")) - assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT] - - def test_responses_endpoint_supported_request_types(self): - backend = _make_backend(_config("model-A", request_type="openai_responses")) - assert backend.supported_request_types == [ChatRequestType.OPENAI_RESPONSES] - - def test_mixed_endpoint_supported_request_types_are_stable(self): - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[ - _ep("chat-model"), - _ep("responses-model", request_type="openai_responses"), - ], - ) - backend = _make_backend(config) - assert backend.supported_request_types == [ - ChatRequestType.OPENAI_CHAT, - ChatRequestType.OPENAI_RESPONSES, - ] - - def test_request_type_aliases_normalize(self): - assert LatencyServiceEndpoint(model="m", request_type="chat").request_type == "openai_chat" - assert ( - LatencyServiceEndpoint(model="m", request_type="responses").request_type - == "openai_responses" - ) - - def test_unknown_request_type_raises(self): - with pytest.raises(ValidationError): - LatencyServiceEndpoint(model="m", request_type="anthropic") # type: ignore[arg-type] - - def test_unknown_backend_config_field_raises(self): - with pytest.raises(ValidationError): - LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep("model-A")], - credential_polciy="caller_override", - ) - - def test_no_endpoints_raises(self): - with pytest.raises(ValueError, match="At least one endpoint"): - _make_backend(_config()) - - def test_single_endpoint_ok(self): - backend = _make_backend(_config("model-A")) - assert "model-A" in backend._clients - - def test_missing_model_raises(self): - # dataclass enforces ``model`` as required, but an empty string - # should still be rejected by the backend as a routing key. - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[LatencyServiceEndpoint(model="")], - ) - with pytest.raises(ValueError, match="must have a 'model' field"): - _make_backend(config) - - def test_duplicate_model_raises(self): - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep("model-A"), _ep("model-A", base_url="http://other")], - ) - with pytest.raises(ValueError, match="Duplicate model ID"): - _make_backend(config) - - -# --------------------------------------------------------------------------- -# Client construction -# --------------------------------------------------------------------------- - - -class TestClientConstruction: - def test_disables_sdk_retries(self): - """The SDK retry layer is disabled so the health-aware ``call`` loop is - the single source of retries. - - The loop already retries on a *different* endpoint; letting the SDK also - retry the *same* endpoint stacks multiplicatively and adds backoff - sleeps that hold each request (and its buffered body) alive longer, - amplifying connection-pool pressure during an upstream incident. - """ - with patch( - "switchyard.lib.backends" - ".latency_service_llm_backend.OpenAILLMClient" - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock() - with patch.object(HealthPoller, "start"): - LatencyServiceLLMBackend(_config("model-A", "model-B")) - - assert mock_cls.call_args_list, "expected one OpenAILLMClient per endpoint" - for call in mock_cls.call_args_list: - assert call.kwargs.get("max_retries") == 0 - - -# --------------------------------------------------------------------------- -# Tiered endpoint selection -# --------------------------------------------------------------------------- - - -class TestEndpointSelection: - def test_healthy_preferred_over_unknown(self): - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.UNKNOWN, - }) - picks = {backend._select_endpoint() for _ in range(50)} - assert picks == {"model-A"} - - def test_unknown_preferred_over_degraded(self): - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealthStatus.UNKNOWN, - "model-B": EndpointHealthStatus.DEGRADED, - }) - picks = {backend._select_endpoint() for _ in range(50)} - assert picks == {"model-A"} - - def test_all_healthy_random_distribution(self): - backend = _make_backend(_config("model-A", "model-B", "model-C")) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - "model-C": EndpointHealthStatus.HEALTHY, - }) - picks = {backend._select_endpoint() for _ in range(200)} - assert len(picks) > 1 - - def test_all_degraded_still_picks(self): - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealthStatus.DEGRADED, - "model-B": EndpointHealthStatus.DEGRADED, - }) - pick = backend._select_endpoint() - assert pick in {"model-A", "model-B"} - - def test_initial_state_is_unknown(self): - backend = _make_backend(_config("model-A", "model-B")) - assert backend._health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN - assert backend._health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN - assert backend._health_cache["model-A"].last_latency_ms is None - - -# --------------------------------------------------------------------------- -# Inverse-latency weighted selection within a tier -# --------------------------------------------------------------------------- - - -class TestLatencyWeightedSelection: - def test_picks_skew_toward_lower_latency(self): - """Two HEALTHY endpoints; the faster one should attract most traffic.""" - backend = _make_backend(_config("fast", "slow")) - _set_health(backend, { - "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0), - "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0), - }) - random.seed(0) - picks = [backend._select_endpoint() for _ in range(2000)] - fast = picks.count("fast") - slow = picks.count("slow") - # weights are 1/50 vs 1/500 → expected 10:1; bound loosely to avoid flake. - assert fast > slow * 5, f"fast={fast} slow={slow}" - - def test_unknown_latency_falls_back_to_uniform(self): - """If any candidate's last_latency_ms is None, the tier picks uniformly.""" - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0), - "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, None), - }) - random.seed(0) - picks = [backend._select_endpoint() for _ in range(2000)] - a = picks.count("model-A") - b = picks.count("model-B") - # Uniform within ±15% of 50/50 over 2000 draws is comfortably non-flaky. - assert 850 < a < 1150, f"model-A picked {a}/2000" - assert 850 < b < 1150, f"model-B picked {b}/2000" - - def test_zero_latency_falls_back_to_uniform(self): - """Non-positive samples are treated as bogus and trigger uniform fallback.""" - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 0.0), - "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0), - }) - # Must not raise (no 1/0); just exercise selection a few times. - picks = {backend._select_endpoint() for _ in range(50)} - assert picks <= {"model-A", "model-B"} - - def test_weighting_only_within_winning_tier(self): - """A faster DEGRADED endpoint must not beat a slower HEALTHY one.""" - backend = _make_backend(_config("slow-healthy", "fast-degraded")) - _set_health(backend, { - "slow-healthy": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0), - "fast-degraded": EndpointHealth(EndpointHealthStatus.DEGRADED, 10.0), - }) - picks = {backend._select_endpoint() for _ in range(50)} - assert picks == {"slow-healthy"} - - -# --------------------------------------------------------------------------- -# Retry with dedup -# --------------------------------------------------------------------------- - - -class TestRetryDedup: - async def test_retry_avoids_same_endpoint(self): - backend = _make_backend(_config("model-A", "model-B", max_retries=1)) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - }) - - fail_mock = AsyncMock(side_effect=RuntimeError("down")) - success_mock = AsyncMock(return_value=_make_completion()) - backend._clients["model-A"].acompletion = fail_mock - backend._clients["model-B"].acompletion = success_mock - - ctx = ProxyContext() - result = await backend.call(ctx, _openai_request()) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - assert success_mock.called - - async def test_all_retries_exhausted_raises(self): - backend = _make_backend(_config("model-A", "model-B", max_retries=1)) - - for mid in backend._clients: - backend._clients[mid].acompletion = AsyncMock( - side_effect=RuntimeError("down") - ) - - ctx = ProxyContext() - with pytest.raises(RuntimeError, match="down"): - await backend.call(ctx, _openai_request()) - - -# --------------------------------------------------------------------------- -# Retry policy: transient errors retry, 4xx client errors fail fast -# --------------------------------------------------------------------------- - - -class TestRetryPolicy: - @pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 413, 415, 422]) - def test_client_errors_are_not_retryable(self, status): - from switchyard.lib.backends.latency_service_llm_backend import ( - _is_retryable_status, - ) - - assert _is_retryable_status(status) is False - - @pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504]) - def test_transient_errors_are_retryable(self, status): - from switchyard.lib.backends.latency_service_llm_backend import ( - _is_retryable_status, - ) - - assert _is_retryable_status(status) is True - - async def test_400_fails_fast_without_failover(self): - """A 400 on the first endpoint must not retry the second.""" - from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS - - backend = _make_backend(_config("model-A", "model-B", max_retries=2)) - # model-A is the sole HEALTHY endpoint → tried first deterministically. - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.UNKNOWN, - }) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(400) - ) - backend._clients["model-B"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - with pytest.raises(openai.APIStatusError): - await backend.call(ctx, _openai_request()) - - # Failed once on model-A, never failed over to model-B. - assert backend._clients["model-A"].acompletion.call_count == 1 - assert backend._clients["model-B"].acompletion.call_count == 0 - # Upstream status is stashed so the endpoint passes the 400 through. - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400 - - async def test_5xx_retries_to_another_endpoint(self): - """A 503 on the first endpoint fails over and recovers on the second.""" - backend = _make_backend(_config("model-A", "model-B", max_retries=2)) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.UNKNOWN, - }) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(503) - ) - backend._clients["model-B"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - result = await backend.call(ctx, _openai_request()) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - assert backend._clients["model-B"].acompletion.call_count == 1 - assert ctx.selected_model == "model-B" - - -# --------------------------------------------------------------------------- -# Session affinity (pin a conversation to one endpoint) -# --------------------------------------------------------------------------- - - -def _conv_request(text: str) -> ChatRequest: - """A request whose first user message anchors a distinct session.""" - return _openai_request(messages=[{"role": "user", "content": text}]) - - -class TestSessionAffinity: - async def test_disabled_by_default_pins_nothing(self): - """Without session_affinity, the affinity map stays empty.""" - backend = _make_backend(_config("model-A", "model-B")) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - }) - for mid in backend._clients: - backend._clients[mid].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), _conv_request("task")) - assert len(backend._affinity) == 0 - - async def test_first_turn_latency_aware_then_sticks(self): - """Turn 1 uses the latency-aware pick; later turns reuse that endpoint - even after latencies flip to favour the other one.""" - backend = _make_backend(_config("fast", "slow", session_affinity=True)) - for mid in backend._clients: - backend._clients[mid].acompletion = AsyncMock( - return_value=_make_completion() - ) - # Turn 1: only "fast" is selectable (slow is DEGRADED) → deterministic pin. - _set_health(backend, { - "fast": EndpointHealthStatus.HEALTHY, - "slow": EndpointHealthStatus.DEGRADED, - }) - req = _conv_request("the task") - ctx1 = ProxyContext() - await backend.call(ctx1, req) - assert ctx1.selected_model == "fast" - - # Flip: "slow" is now healthy AND the latency winner. Affinity must - # still keep the conversation on "fast". - _set_health(backend, { - "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 1000.0), - "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 1.0), - }) - for _ in range(20): - ctx = ProxyContext() - await backend.call(ctx, req) - assert ctx.selected_model == "fast" - - async def test_distinct_conversations_pin_independently(self): - """Two conversations resolve to independent pins.""" - backend = _make_backend(_config("model-A", "model-B", session_affinity=True)) - for mid in backend._clients: - backend._clients[mid].acompletion = AsyncMock( - return_value=_make_completion() - ) - - # Conversation 1 pins to model-A (only A selectable). - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.DEGRADED, - }) - req_a = _conv_request("conversation one") - ctx_a = ProxyContext() - await backend.call(ctx_a, req_a) - assert ctx_a.selected_model == "model-A" - - # Conversation 2 pins to model-B (only B selectable). - _set_health(backend, { - "model-A": EndpointHealthStatus.DEGRADED, - "model-B": EndpointHealthStatus.HEALTHY, - }) - req_b = _conv_request("conversation two") - ctx_b = ProxyContext() - await backend.call(ctx_b, req_b) - assert ctx_b.selected_model == "model-B" - - # Independent pins: each conversation resolves to its own endpoint. - assert len(backend._affinity) == 2 - assert await backend._affinity.pinned(ProxyContext(), req_a) == "model-A" - assert await backend._affinity.pinned(ProxyContext(), req_b) == "model-B" - - async def test_degraded_pin_reroutes_and_repins(self): - """When the pinned endpoint degrades, the next turn re-routes to a - healthy endpoint and the pin follows.""" - backend = _make_backend(_config("model-A", "model-B", session_affinity=True)) - for mid in backend._clients: - backend._clients[mid].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.DEGRADED, - }) - req = _conv_request("task") - await backend.call(ProxyContext(), req) - assert await backend._affinity.pinned(ProxyContext(), req) == "model-A" - - # model-A degrades; model-B becomes the only healthy endpoint. - _set_health(backend, { - "model-A": EndpointHealthStatus.DEGRADED, - "model-B": EndpointHealthStatus.HEALTHY, - }) - ctx2 = ProxyContext() - await backend.call(ctx2, req) - assert ctx2.selected_model == "model-B" - assert await backend._affinity.pinned(ProxyContext(), req) == "model-B" - - async def test_pin_follows_recovery_after_call_failure(self): - """If the first-turn endpoint fails the call, the pin records the - endpoint that actually served the request.""" - backend = _make_backend( - _config("model-A", "model-B", max_retries=1, session_affinity=True) - ) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - }) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("down") - ) - backend._clients["model-B"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - req = _conv_request("task") - ctx = ProxyContext() - await backend.call(ctx, req) - # Regardless of which endpoint was tried first, only model-B succeeds. - assert ctx.selected_model == "model-B" - assert await backend._affinity.pinned(ProxyContext(), req) == "model-B" - - async def test_lru_eviction_bounds_map(self): - """The affinity map never exceeds affinity_max_sessions.""" - backend = _make_backend( - _config("model-A", session_affinity=True, affinity_max_sessions=2) - ) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - for i in range(3): - await backend.call(ProxyContext(), _conv_request(f"conversation {i}")) - - assert len(backend._affinity) == 2 - - def test_affinity_config_reaches_backend(self): - """A config dict's affinity settings reach the backend's pin map.""" - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[{"model": "model-A", "base_url": "http://a.test", "api_key": "k"}], - session_affinity=True, - affinity_max_sessions=5, - ) - assert config.session_affinity is True - assert config.affinity_max_sessions == 5 - - backend = _make_backend(config) - assert backend._affinity.max_sessions == 5 - - async def test_affinity_counters_rendered_on_metrics(self): - """hits/misses counters appear on /metrics when affinity is enabled.""" - backend = _make_backend(_config("model-A", session_affinity=True)) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - req = _conv_request("the task") - await backend.call(ProxyContext(), req) # first turn → miss (no pin yet) - await backend.call(ProxyContext(), req) # reuse → hit - await backend.call(ProxyContext(), req) # reuse → hit - - out = "\n".join(backend._render_prometheus_lines()) - assert "switchyard_affinity_hits_total 2" in out - assert "switchyard_affinity_misses_total 1" in out - - async def test_affinity_counters_absent_when_disabled(self): - """The warm-reuse counters stay off the metric surface by default.""" - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - await backend.call(ProxyContext(), _conv_request("task")) - - out = "\n".join(backend._render_prometheus_lines()) - assert "switchyard_affinity_hits_total" not in out - assert "switchyard_affinity_misses_total" not in out - - def test_affinity_l2_counters_rendered_when_shared_store_configured(self): - """L2 hit/error counters appear on /metrics when a shared store is on. - - Constructing the Redis store never connects (the client is lazy), so - this stays hermetic. - """ - backend = _make_backend(_config( - "model-A", - session_affinity=True, - affinity_store="redis", - affinity_store_url="redis://cache.test:6379/0", - )) - - out = "\n".join(backend._render_prometheus_lines()) - assert "switchyard_affinity_l2_hits_total 0" in out - assert "switchyard_affinity_l2_errors_total 0" in out - assert "switchyard_affinity_l2_breaker_open 0" in out - - def test_affinity_l2_breaker_gauge_reads_1_while_open(self): - """The breaker gauge flips to 1 once the failure streak opens it.""" - backend = _make_backend(_config( - "model-A", - session_affinity=True, - affinity_store="redis", - affinity_store_url="redis://cache.test:6379/0", - )) - affinity = backend._affinity - for _ in range(affinity._l2_breaker_threshold): - affinity._record_l2_failure("test-induced failure") - - out = "\n".join(backend._render_prometheus_lines()) - assert "switchyard_affinity_l2_breaker_open 1" in out - - def test_affinity_l2_counters_absent_for_memory_store(self): - """Without a shared store the L2 counters stay off the metric surface.""" - backend = _make_backend(_config("model-A", session_affinity=True)) - - out = "\n".join(backend._render_prometheus_lines()) - assert "switchyard_affinity_l2_hits_total" not in out - assert "switchyard_affinity_l2_errors_total" not in out - assert "switchyard_affinity_l2_breaker_open" not in out - - def test_negative_affinity_max_rejected_at_construction(self): - """A negative cap is rejected when the config is built — otherwise the - LRU eviction loop in SessionCache.put would pop past an empty map.""" - from pydantic import ValidationError - - with pytest.raises(ValidationError): - LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep("model-A")], - session_affinity=True, - affinity_max_sessions=-1, - ) - - def test_zero_affinity_max_rejected_when_enabled(self): - """A zero cap with affinity on is rejected — it would retain nothing.""" - from pydantic import ValidationError - - with pytest.raises(ValidationError): - LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep("model-A")], - session_affinity=True, - affinity_max_sessions=0, - ) - - -# --------------------------------------------------------------------------- -# Request processing -# --------------------------------------------------------------------------- - - -class TestCall: - async def test_non_streaming_returns_completion_chat_response(self): - backend = _make_backend(_config("model-A")) - completion = _make_completion(content="world") - backend._clients["model-A"].acompletion = AsyncMock(return_value=completion) - - ctx = ProxyContext() - result = await backend.call(ctx, _openai_request()) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - assert result.body["choices"][0]["message"]["content"] == "world" - # ``selected_model`` is the cross-language ctx field the Rust - # ``StatsResponseProcessor`` reads to attribute tokens and - # end-to-end latency per endpoint on /metrics. - assert ctx.selected_model == "model-A" - # ``backend_call_latency_ms`` is the second cross-language hook — - # the response processor uses it to compute ``routing_overhead_ms``. - # Mocked upstream returns instantly so latency is small but non-None. - assert ctx.backend_call_latency_ms is not None - assert ctx.backend_call_latency_ms >= 0.0 - - async def test_responses_endpoint_dispatches_responses_natively(self): - backend = _make_backend(_config("model-A", request_type="openai_responses")) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={ - "id": "resp-test", - "object": "response", - "model": "model-A", - "output": [], - } - ) - backend._clients["model-A"].acompletion = AsyncMock() - - ctx = ProxyContext() - result = await backend.call( - ctx, - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - backend._clients["model-A"].aresponses.assert_awaited_once() - backend._clients["model-A"].acompletion.assert_not_called() - call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs - assert call_kwargs["model"] == "model-A" - assert call_kwargs["input"] == "hi" - assert "messages" not in call_kwargs - assert ctx.selected_model == "model-A" - - async def test_responses_endpoint_preserves_exact_upstream_body(self): - """Same-format Responses passthrough returns the upstream JSON untouched. - - Provider-specific extras and explicit-null fields must survive the - wrap into ``ChatResponse`` — nothing may be normalized, - re-synthesized, or dropped on this path. - """ - upstream_body = { - "id": "resp-raw", - "object": "response", - "created_at": 1719890000, - "model": "model-A", - "status": "completed", - "output": [], - "store": False, - "temperature": 1.0, - "top_p": 0.9, - "previous_response_id": None, - "provider_meta": {"region": "eastus"}, - } - backend = _make_backend(_config("model-A", request_type="openai_responses")) - backend._clients["model-A"].aresponses = AsyncMock(return_value=dict(upstream_body)) - - result = await backend.call( - ProxyContext(), - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION) - assert result.body == upstream_body - - async def test_chat_endpoint_translates_responses_to_chat_fallback(self): - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - backend._clients["model-A"].aresponses = AsyncMock() - - result = await backend.call( - ProxyContext(), - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - backend._clients["model-A"].acompletion.assert_awaited_once() - backend._clients["model-A"].aresponses.assert_not_called() - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["model"] == "model-A" - assert call_kwargs["messages"][0]["content"] == "hi" - - async def test_streaming_wraps_into_streaming_chat_response(self): - backend = _make_backend(_config("model-A")) - stream_mock = MagicMock(spec=openai.AsyncStream) - backend._clients["model-A"].acompletion = AsyncMock(return_value=stream_mock) - - result = await backend.call(ProxyContext(), _openai_request(stream=True)) - assert response_type_matches(result, ChatResponseType.OPENAI_STREAM) - - async def test_responses_streaming_wraps_into_responses_stream(self): - backend = _make_backend(_config("model-A", request_type="openai_responses")) - stream_mock = MagicMock(spec=openai.AsyncStream) - backend._clients["model-A"].aresponses = AsyncMock(return_value=stream_mock) - - result = await backend.call( - ProxyContext(), - ChatRequest.openai_responses({ - "model": "incoming-model", - "input": "hi", - "stream": True, - }), - ) - - assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM) - - async def test_model_id_overrides_incoming_model(self): - """The selected endpoint model ID should replace the request body's model.""" - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), _openai_request(model="incoming-model")) - - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["model"] == "model-A" - - async def test_accepts_anthropic_request_via_translation(self): - """Non-OpenAI inbound formats go through TranslationEngine.""" - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - anthropic_req = ChatRequest.anthropic({ - "model": "claude-sonnet-4-20250514", - "messages": [{"role": "user", "content": "Hi"}], - "max_tokens": 1024, - }) - result = await backend.call(ProxyContext(), anthropic_req) - - assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION) - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["model"] == "model-A" - assert "messages" in call_kwargs - - async def test_upstream_model_override_used_in_body(self): - """When upstream_model is set, it's the value sent in body['model'].""" - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[ - LatencyServiceEndpoint( - model="openai/gpt-5.5", - upstream_model="openai/openai/gpt-5.5", - base_url="https://inference-api.nvidia.com/v1", - api_key="nvapi-test", - ), - ], - ) - backend = _make_backend(config) - backend._clients["openai/gpt-5.5"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - await backend.call(ctx, _openai_request(model="incoming-model")) - - call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs - assert call_kwargs["model"] == "openai/openai/gpt-5.5" - assert ctx.selected_model == "openai/gpt-5.5" - - async def test_upstream_model_default_falls_back_to_model(self): - """Without upstream_model, body['model'] should be the lookup key.""" - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[ - LatencyServiceEndpoint( - model="openai/gpt-5.5", - base_url="http://llm.test", - api_key="k", - ), - ], - ) - backend = _make_backend(config) - backend._clients["openai/gpt-5.5"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), _openai_request()) - - call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs - assert call_kwargs["model"] == "openai/gpt-5.5" - - -# --------------------------------------------------------------------------- -# Route-selection spend-logs metadata (tokenomics attribution) -# --------------------------------------------------------------------------- - - -def _spend_logs_payload(mock: AsyncMock, call_index: int = -1) -> dict[str, object]: - """Parse the spend-logs metadata header stamped on one recorded SDK call.""" - call_kwargs = mock.call_args_list[call_index].kwargs - header_value = call_kwargs["extra_headers"][SPEND_LOGS_METADATA_HEADER] - payload = json.loads(header_value) - assert isinstance(payload, dict) - return payload - - -class TestRouteSelectionSpendLogs: - """Every upstream attempt is stamped with route-selection metadata. - - The ``x-litellm-spend-logs-metadata`` request header ties a LiteLLM - provider spend-log row back to the Switchyard route that selected it, and - the successful attempt's selection is recorded on - ``ctx.metadata[CTX_ROUTE_SELECTION]`` so the endpoint layer can return the - matching ``x-switchyard-*`` response headers. - """ - - async def test_outbound_header_carries_route_selection(self): - """The outbound header records the full selection for the chosen endpoint.""" - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[ - LatencyServiceEndpoint( - model="openai/gpt-5.5", - upstream_model="openai/openai/gpt-5.5", - base_url="https://inference-api.test/v1", - api_key="k", - ), - ], - ) - backend = _make_backend(config) - backend._clients["openai/gpt-5.5"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.5")) - - payload = _spend_logs_payload(backend._clients["openai/gpt-5.5"].acompletion) - assert payload["router_model"] == "nvidia/switchyard/gpt-5.5" - assert payload["router_strategy"] == "latency" - assert payload["router_selected_endpoint"] == "openai/gpt-5.5" - assert payload["router_selected_model"] == "openai/openai/gpt-5.5" - assert payload["router_selected_provider"] == "openai" - # Must parse as a UUID — the join key between spend-log rows. - uuid.UUID(str(payload["router_correlation_id"])) - - async def test_ctx_records_selection_matching_outbound_header(self): - """ctx records exactly the payload the wire header carried.""" - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - await backend.call(ctx, _openai_request()) - - payload = _spend_logs_payload(backend._clients["model-A"].acompletion) - assert ctx.metadata[CTX_ROUTE_SELECTION] == payload - # Endpoint id without a provider prefix: the provider segment - # degenerates to the whole id (split on the first "/"). - assert payload["router_selected_provider"] == "model-A" - - async def test_failover_restamps_selection_and_keeps_correlation_id(self): - """Each attempt is stamped with its own endpoint under one correlation id.""" - backend = _make_backend(_config("model-A", "model-B")) - _set_health( - backend, - { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.DEGRADED, - }, - ) - failing = AsyncMock(side_effect=_api_status_error(500)) - succeeding = AsyncMock(return_value=_make_completion()) - backend._clients["model-A"].acompletion = failing - backend._clients["model-B"].acompletion = succeeding - - ctx = ProxyContext() - await backend.call(ctx, _openai_request()) - - first = _spend_logs_payload(failing) - second = _spend_logs_payload(succeeding) - # Each attempt records the endpoint it actually hit… - assert first["router_selected_endpoint"] == "model-A" - assert second["router_selected_endpoint"] == "model-B" - # …while sharing one correlation id per client request. - assert first["router_correlation_id"] == second["router_correlation_id"] - # ctx carries the selection that served the request, not the failure. - assert ctx.metadata[CTX_ROUTE_SELECTION] == second - - async def test_correlation_id_is_fresh_per_request(self): - """Two client requests never share a correlation id.""" - backend = _make_backend(_config("model-A")) - mock = AsyncMock(return_value=_make_completion()) - backend._clients["model-A"].acompletion = mock - - await backend.call(ProxyContext(), _openai_request()) - await backend.call(ProxyContext(), _openai_request()) - - first = _spend_logs_payload(mock, call_index=0) - second = _spend_logs_payload(mock, call_index=1) - assert first["router_correlation_id"] != second["router_correlation_id"] - - async def test_router_model_falls_back_to_configured_route_model(self): - """A model-less client body attributes to the configured route id.""" - backend = _make_backend( - _config("model-A", route_model="nvidia/switchyard/gpt-5.5") - ) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - body: dict = {"messages": [{"role": "user", "content": "hi"}]} - await backend.call(ProxyContext(), ChatRequest.openai_chat(body)) - - payload = _spend_logs_payload(backend._clients["model-A"].acompletion) - assert payload["router_model"] == "nvidia/switchyard/gpt-5.5" - - async def test_no_selection_recorded_when_all_attempts_fail(self): - """No billed success → no selection recorded on ctx.""" - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401), - ) - - ctx = ProxyContext() - with pytest.raises(openai.APIStatusError): - await backend.call(ctx, _openai_request()) - - assert CTX_ROUTE_SELECTION not in ctx.metadata - - async def test_responses_surface_is_stamped_too(self): - """The Responses-API surface is stamped like the Chat surface.""" - backend = _make_backend(_config("model-A", request_type="openai_responses")) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={"id": "resp-test", "object": "response", "output": []} - ) - - await backend.call( - ProxyContext(), - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - payload = _spend_logs_payload(backend._clients["model-A"].aresponses) - assert payload["router_selected_endpoint"] == "model-A" - assert payload["router_strategy"] == "latency" - - async def test_cased_spoof_of_spend_logs_header_is_stripped(self): - """A differently-cased spoof key cannot ride client extra_headers to the wire. - - Header names are case-insensitive on the wire while dict merges are - not: exactly one instance of the protected header — ours — may reach - the SDK, while benign client headers still pass through. - """ - backend = _make_backend(_config("model-A")) - mock = AsyncMock(return_value=_make_completion()) - backend._clients["model-A"].acompletion = mock - - await backend.call( - ProxyContext(), - _openai_request( - extra_headers={ - "X-LiteLLM-Spend-Logs-Metadata": "spoofed", - "x-client-tag": "42", - }, - ), - ) - - headers = mock.call_args.kwargs["extra_headers"] - spoof_keys = [k for k in headers if k.lower() == SPEND_LOGS_METADATA_HEADER] - assert spoof_keys == [SPEND_LOGS_METADATA_HEADER] - payload = _spend_logs_payload(mock) - assert payload["router_selected_endpoint"] == "model-A" - assert headers["x-client-tag"] == "42" - - -# --------------------------------------------------------------------------- -# Credential policy -# --------------------------------------------------------------------------- - - -class TestCallerApiKey: - """Per-request caller-key forwarding is opt-in for multi-tenant deployments. - - When the HTTP endpoint extracts an ``Authorization: Bearer ...`` header - and writes the value into ``ctx.metadata[CTX_CALLER_API_KEY]``, the - backend ignores that key by default. ``credential_policy="caller_override"`` - makes the caller key a per-call SDK override. - """ - - async def test_default_policy_ignores_caller_key_for_chat_sdk(self): - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" - await backend.call(ctx, _openai_request()) - - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] is None - - async def test_caller_override_policy_passes_caller_key_to_chat_sdk(self): - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - backend = _make_backend(_config("model-A", credential_policy="caller_override")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" - await backend.call(ctx, _openai_request()) - - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] == "nvapi-caller-supplied" - - async def test_default_policy_ignores_caller_key_for_responses_sdk(self): - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - backend = _make_backend(_config("model-A", request_type="openai_responses")) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={"id": "resp-test", "output": []} - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" - await backend.call( - ctx, - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs - assert call_kwargs["api_key"] is None - - async def test_caller_override_policy_passes_caller_key_to_responses_sdk(self): - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - backend = _make_backend(_config( - "model-A", - request_type="openai_responses", - credential_policy="caller_override", - )) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={"id": "resp-test", "output": []} - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" - await backend.call( - ctx, - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs - assert call_kwargs["api_key"] == "nvapi-caller-supplied" - - async def test_missing_caller_key_falls_back_to_config(self): - """ctx without a caller key sends api_key=None to OpenAILLMClient. - - ``OpenAILLMClient.acompletion`` then uses the construction-time - config key (no ``with_options`` override). The test only asserts - the contract at the seam — that we pass ``api_key=None`` so the - client falls back, rather than passing a stale or empty string. - """ - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), _openai_request()) - - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] is None - - async def test_missing_caller_key_falls_back_to_config_for_responses(self): - backend = _make_backend(_config("model-A", request_type="openai_responses")) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={"id": "resp-test", "output": []} - ) - - await backend.call( - ProxyContext(), - ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), - ) - - call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs - assert call_kwargs["api_key"] is None - - async def test_caller_required_policy_passes_caller_key_to_chat_sdk(self): - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - backend = _make_backend(_config("model-A", credential_policy="caller_required")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" - await backend.call(ctx, _openai_request()) - - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] == "nvapi-caller-supplied" - - async def test_caller_required_policy_rejects_missing_caller_key(self): - """No caller key under caller_required → 401 stashed, no upstream call.""" - from switchyard.lib.proxy_context import ( - CTX_UPSTREAM_ATTEMPTS_RECORDED, - CTX_UPSTREAM_HTTP_STATUS, - ) - - backend = _make_backend(_config("model-A", credential_policy="caller_required")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - with pytest.raises(PermissionError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - backend._clients["model-A"].acompletion.assert_not_called() - # Attempt accounting stays claimed: no upstream attempt happened, so - # the endpoint fallback must not count this 401 in - # ``switchyard_upstream_attempts_total``. - assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True - assert "switchyard_latency_upstream_attempts_total" not in "\n".join( - backend._render_prometheus_lines() - ) - - async def test_caller_required_policy_rejects_blank_caller_key(self): - """A blank caller key is unusable → 401, never the configured service key.""" - from switchyard.lib.proxy_context import ( - CTX_CALLER_API_KEY, - CTX_UPSTREAM_ATTEMPTS_RECORDED, - CTX_UPSTREAM_HTTP_STATUS, - ) - - backend = _make_backend(_config("model-A", credential_policy="caller_required")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = " " - with pytest.raises(PermissionError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - backend._clients["model-A"].acompletion.assert_not_called() - assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True - - async def test_caller_required_rejection_skips_affinity_lookup(self): - """The credential check runs before pin resolution, so a doomed - request never touches the session-affinity store.""" - backend = _make_backend(_config( - "model-A", credential_policy="caller_required", session_affinity=True, - )) - backend._affinity.pinned = AsyncMock() - - with pytest.raises(PermissionError): - await backend.call(ProxyContext(), _openai_request()) - - backend._affinity.pinned.assert_not_awaited() - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "incoming-model", "input": "hi"}), - ], -) -async def test_non_chat_http_endpoints_default_to_endpoint_credentials( - path: str, - body: dict[str, object], -) -> None: - """HTTP ingress may attach caller keys, but the default backend policy ignores them.""" - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion(model="model-A") - ) - app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine())) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), - base_url="http://test", - ) as client: - response = await client.post( - path, - json=body, - headers={"Authorization": "Bearer CALLER-BYO-KEY"}, - ) - - assert response.status_code == 200, response.text - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] is None - - -@pytest.mark.parametrize( - ("path", "body"), - [ - ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}), - ("/v1/responses", {"model": "incoming-model", "input": "hi"}), - ], -) -async def test_non_chat_http_endpoints_forward_caller_key_in_byo_mode( - path: str, - body: dict[str, object], -) -> None: - """Opt-in BYO mode preserves caller keys across Messages/Responses ingress.""" - backend = _make_backend(_config("model-A", credential_policy="caller_override")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion(model="model-A") - ) - app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine())) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), - base_url="http://test", - ) as client: - response = await client.post( - path, - json=body, - headers={"Authorization": "Bearer CALLER-BYO-KEY"}, - ) - - assert response.status_code == 200, response.text - call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs - assert call_kwargs["api_key"] == "CALLER-BYO-KEY" - - -async def test_responses_http_endpoint_can_use_native_latency_responses_mode() -> None: - backend = _make_backend(_config("model-A", request_type="openai_responses")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].aresponses = AsyncMock( - return_value={ - "id": "resp-test", - "object": "response", - "model": "model-A", - "output": [], - } - ) - backend._clients["model-A"].acompletion = AsyncMock() - app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine())) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), - base_url="http://test", - ) as client: - response = await client.post( - "/v1/responses", - json={"model": "incoming-model", "input": "hi"}, - headers={"Authorization": "Bearer CALLER-BYO-KEY"}, - ) - - assert response.status_code == 200, response.text - backend._clients["model-A"].aresponses.assert_awaited_once() - backend._clients["model-A"].acompletion.assert_not_called() - call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs - assert call_kwargs["api_key"] is None - assert call_kwargs["input"] == "hi" - - -class TestEndpointApiKeyReachesUpstream: - """End-to-end auth: which credential the upstream HTTP call carries. - - The seam test above only asserts the ``api_key`` value handed to - ``acompletion``; it cannot catch a regression where a blank caller key - clobbers the configured endpoint key and the upstream call goes out - unauthenticated. These tests run the real - ``OpenAILLMClient`` against a mocked HTTP transport and inspect the - ``Authorization`` header that actually reaches the wire. - """ - - @staticmethod - def _backend_with_captured_auth( - captured: dict[str, str | None], - credential_policy: str = "configured_endpoint", - ) -> LatencyServiceLLMBackend: - """Real-client backend whose upstream auth header is captured.""" - - def handler(request: httpx.Request) -> httpx.Response: - captured["authorization"] = request.headers.get("authorization") - return httpx.Response( - 200, - json={ - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1700000000, - "model": "model-A", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "ok"}, - "finish_reason": "stop", - }], - }, - ) - - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[LatencyServiceEndpoint( - model="model-A", - base_url="http://llm.test/v1", - api_key="ENDPOINT-CONFIGURED-KEY", - )], - credential_policy=credential_policy, - ) - with patch.object(HealthPoller, "start"): - backend = LatencyServiceLLMBackend(config) - backend._clients["model-A"].async_client._client = httpx.AsyncClient( - transport=httpx.MockTransport(handler) - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - return backend - - async def test_no_caller_key_uses_configured_endpoint_key(self): - """No caller key → the endpoint's configured key authenticates upstream.""" - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth(captured) - - await backend.call(ProxyContext(), _openai_request()) - - assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY" - - async def test_default_policy_ignores_caller_key(self): - """Default policy keeps using the endpoint key even when a caller key exists.""" - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth(captured) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY" - await backend.call(ctx, _openai_request()) - - assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY" - - async def test_caller_override_policy_uses_caller_key(self): - """BYO mode lets a caller-supplied key override the endpoint key.""" - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth( - captured, - credential_policy="caller_override", - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY" - await backend.call(ctx, _openai_request()) - - assert captured["authorization"] == "Bearer CALLER-BYO-KEY" - - async def test_blank_caller_key_does_not_clobber_endpoint_key(self): - """A blank caller key in BYO mode still falls back to the configured key.""" - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth( - captured, - credential_policy="caller_override", - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = " " - await backend.call(ctx, _openai_request()) - - assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY" - - async def test_caller_required_policy_uses_caller_key(self): - """caller_required forwards the caller key to the upstream wire.""" - from switchyard.lib.proxy_context import CTX_CALLER_API_KEY - - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth( - captured, - credential_policy="caller_required", - ) - - ctx = ProxyContext() - ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY" - await backend.call(ctx, _openai_request()) - - assert captured["authorization"] == "Bearer CALLER-BYO-KEY" - - async def test_caller_required_rejects_missing_key_without_upstream_call(self): - """caller_required with no caller key 401s before reaching the upstream. - - The configured ``ENDPOINT-CONFIGURED-KEY`` must never authenticate the - call — the mock transport handler should not run at all. - """ - from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS - - captured: dict[str, str | None] = {} - backend = self._backend_with_captured_auth( - captured, - credential_policy="caller_required", - ) - - ctx = ProxyContext() - with pytest.raises(PermissionError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - assert captured == {} - - -# --------------------------------------------------------------------------- -# Per-model upstream-attempt metrics -# --------------------------------------------------------------------------- - - -class TestPerModelUpstreamMetrics: - """``switchyard_latency_upstream_attempts_total`` breaks attempts out by - requested route model, selected upstream model, outcome, and HTTP code — - the per-model complement to the layer-aggregate counter.""" - - async def test_success_attempt_recorded_per_model(self): - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - # Request a configured endpoint id so ``requested_model`` is preserved. - await backend.call(ProxyContext(), _openai_request(model="model-A")) - - out = "\n".join(backend._render_prometheus_lines()) - assert ( - 'switchyard_latency_upstream_attempts_total{' - 'requested_model="model-A",upstream_model="model-A",' - 'outcome="success",code="200"} 1' - ) in out - - async def test_error_attempt_recorded_per_model(self): - backend = _make_backend(_config("model-A", max_retries=0)) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(500) - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - with pytest.raises(openai.APIStatusError): - await backend.call(ProxyContext(), _openai_request(model="model-A")) - - out = "\n".join(backend._render_prometheus_lines()) - assert ( - 'switchyard_latency_upstream_attempts_total{' - 'requested_model="model-A",upstream_model="model-A",' - 'outcome="retryable_error",code="500"} 1' - ) in out - - async def test_route_model_preserved_as_requested_model_label(self): - # Clients of a deployed route send the route key (e.g. - # ``nvidia/switchyard/gpt-5.4``), never an endpoint id. The route id is - # config-derived, so it joins the bounded label set instead of - # collapsing to the ``"other"`` sentinel. - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[_ep("azure/openai/gpt-5.4")], - route_model="nvidia/switchyard/gpt-5.4", - ) - backend = _make_backend(config) - backend._clients["azure/openai/gpt-5.4"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"azure/openai/gpt-5.4": EndpointHealthStatus.HEALTHY}) - - await backend.call( - ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.4") - ) - - out = "\n".join(backend._render_prometheus_lines()) - assert ( - 'switchyard_latency_upstream_attempts_total{' - 'requested_model="nvidia/switchyard/gpt-5.4",' - 'upstream_model="azure/openai/gpt-5.4",' - 'outcome="success",code="200"} 1' - ) in out - assert 'requested_model="other"' not in out - - async def test_unknown_requested_model_normalized_to_sentinel(self): - # A client-supplied model that is not a configured endpoint id must not - # become a raw Prometheus label (unbounded cardinality); it collapses to - # the ``"other"`` sentinel. - backend = _make_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - await backend.call(ProxyContext(), _openai_request()) # model="incoming-model" - - out = "\n".join(backend._render_prometheus_lines()) - assert 'requested_model="other"' in out - assert 'requested_model="incoming-model"' not in out - - async def test_upstream_model_override_labels_the_upstream_name(self): - config = LatencyServiceBackendConfig( - latency_service_url=LATENCY_SERVICE_URL, - endpoints=[LatencyServiceEndpoint( - model="model-A", - base_url="http://a.test", - api_key="k", - upstream_model="azure/openai/gpt-5.1-codex-mini", - )], - ) - backend = _make_backend(config) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - - await backend.call(ProxyContext(), _openai_request()) - - out = "\n".join(backend._render_prometheus_lines()) - assert 'upstream_model="azure/openai/gpt-5.1-codex-mini"' in out - - async def test_metric_absent_until_traffic(self): - backend = _make_backend(_config("model-A")) - - out = "\n".join(backend._render_prometheus_lines()) - - assert "switchyard_latency_upstream_attempts_total" not in out - - -# --------------------------------------------------------------------------- -# Failure-source annotation -# --------------------------------------------------------------------------- - - -class TestErrorSourceAnnotation: - """The backend stamps the failure origin + upstream model on ctx.""" - - async def test_http_failure_stamps_provider_source_and_upstream_model(self): - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401) - ) - - ctx = ProxyContext() - with pytest.raises(openai.APIStatusError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_ERROR_SOURCE] == "provider" - assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A" - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - - async def test_network_failure_stamps_provider_without_status(self): - """A status-less upstream fault is still provider-originated, so the - endpoint's 500 envelope can say so instead of implying an internal bug.""" - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("connection reset") - ) - - ctx = ProxyContext() - with pytest.raises(RuntimeError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_ERROR_SOURCE] == "provider" - assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A" - assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata - - async def test_caller_required_rejection_is_switchyard_source(self): - """The caller_required 401 rides the upstream-status channel but is - Switchyard's own rejection — the source label must say so.""" - backend = _make_backend( - _config("model-A", credential_policy="caller_required") - ) - - ctx = ProxyContext() - with pytest.raises(PermissionError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard" - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - assert CTX_UPSTREAM_MODEL not in ctx.metadata - - def test_translation_rejection_is_switchyard_source(self): - """The translation invalid-value 400 is likewise Switchyard-originated.""" - from switchyard.lib.backends.latency_service_llm_backend import ( - _stash_invalid_request_error, - ) - - ctx = ProxyContext() - _stash_invalid_request_error( - ctx, ValueError("InvalidValue: unsupported message role 'api'") - ) - - assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard" - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400 - - -# --------------------------------------------------------------------------- -# Readiness and shutdown -# --------------------------------------------------------------------------- - - -class TestReadinessAndShutdown: - def test_not_ready_before_first_poll(self): - backend = _make_backend(_config("model-A")) - assert backend.is_ready() is False - - def test_ready_after_first_poll(self): - backend = _make_backend(_config("model-A")) - backend._poller._poll_count = 1 - assert backend.is_ready() is True - - async def test_shutdown_stops_poller(self): - backend = _make_backend(_config("model-A")) - await backend.shutdown() - assert backend._poller._stop_event.is_set() - - async def test_shutdown_closes_shared_affinity_store(self): - """Backend teardown releases the shared (L2) pin store's connections.""" - - class _ClosableL2: - def __init__(self) -> None: - self.closed = False - - async def get(self, key: str) -> str | None: - return None - - async def put(self, key: str, value: str) -> None: - return None - - async def aclose(self) -> None: - self.closed = True - - backend = _make_backend(_config("model-A", session_affinity=True)) - l2 = _ClosableL2() - backend._affinity._l2 = l2 - - await backend.shutdown() - assert l2.closed is True - - -# --------------------------------------------------------------------------- -# Health poller -# --------------------------------------------------------------------------- - - -class TestHealthPoller: - def test_poll_updates_cache(self): - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - "model-B": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A", "model-B"], health_cache) - - response_data = { - "endpoint_health": { - "model-A": {"status": "healthy", "last_latency_ms": 100.0}, - "model-B": {"status": "degraded", "last_latency_ms": 1500.0}, - } - } - _mock_health_response(poller, _health_response(200, json=response_data)) - _run_one_poll(poller) - - assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY - assert health_cache["model-A"].last_latency_ms == 100.0 - assert health_cache["model-B"].status == EndpointHealthStatus.DEGRADED - assert health_cache["model-B"].last_latency_ms == 1500.0 - assert poller.has_polled - - def test_poll_captures_null_latency(self): - """``last_latency_ms`` is nullable in the spec; absence becomes None.""" - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A"], health_cache) - - response_data = { - "endpoint_health": { - "model-A": {"status": "healthy", "last_latency_ms": None}, - } - } - _mock_health_response(poller, _health_response(200, json=response_data)) - _run_one_poll(poller) - - assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY - assert health_cache["model-A"].last_latency_ms is None - - def test_poll_failure_resets_to_unknown(self): - """If the Latency Service is unreachable, all endpoints reset to UNKNOWN - so the backend falls back to random routing rather than stale data.""" - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0), - "model-B": EndpointHealth(EndpointHealthStatus.DEGRADED, 800.0), - } - poller = _make_poller(["model-A", "model-B"], health_cache) - _mock_health_response(poller, _health_response(500)) - _run_one_poll(poller) - - assert health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN - assert health_cache["model-A"].last_latency_ms is None - assert health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN - assert health_cache["model-B"].last_latency_ms is None - assert not poller.has_polled - - def test_poll_ignores_unknown_model_ids(self): - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A"], health_cache) - - response_data = { - "endpoint_health": { - "model-A": {"status": "healthy", "last_latency_ms": 100.0}, - "model-UNKNOWN": {"status": "degraded", "last_latency_ms": 999.0}, - } - } - _mock_health_response(poller, _health_response(200, json=response_data)) - _run_one_poll(poller) - - assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY - assert "model-UNKNOWN" not in health_cache - - def test_success_increments_polls_and_records_timestamp(self): - """``/metrics`` reads these fields to expose poll-loop health.""" - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A"], health_cache) - _mock_health_response( - poller, - _health_response(200, json={"endpoint_health": { - "model-A": {"status": "healthy", "last_latency_ms": 50.0}, - }}), - ) - _run_one_poll(poller) - - assert poller.poll_successes == 1 - assert poller.poll_failures == 0 - assert poller.last_poll_ok is True - assert poller.seconds_since_last_success is not None - assert poller.seconds_since_last_success >= 0 - - def test_failure_increments_failures_and_leaves_age_unset(self): - """Pre-success failure leaves ``seconds_since_last_success`` None so - scrapers can distinguish "never polled" from "polled but stale".""" - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A"], health_cache) - _mock_health_response(poller, _health_response(500)) - _run_one_poll(poller) - - assert poller.poll_successes == 0 - assert poller.poll_failures == 1 - assert poller.last_poll_ok is False - assert poller.seconds_since_last_success is None - - def test_stop_event_terminates_poller(self): - health_cache: dict[str, EndpointHealth] = { - "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN), - } - poller = _make_poller(["model-A"], health_cache, poll_interval_s=0.05) - _mock_health_response( - poller, - _health_response(200, json={"endpoint_health": { - "model-A": {"status": "healthy", "last_latency_ms": 50.0}, - }}), - ) - - poller.start() - time.sleep(0.15) - poller.stop() - poller.join(timeout=2.0) - - assert not poller.is_alive() - assert poller.has_polled diff --git a/tests/test_latency_service_spans.py b/tests/test_latency_service_spans.py deleted file mode 100644 index 09d7be99..00000000 --- a/tests/test_latency_service_spans.py +++ /dev/null @@ -1,273 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Datadog routing spans/tags emitted by ``LatencyServiceLLMBackend``. - -The backend emits a ``switchyard.route_decision`` span around endpoint selection -and a ``switchyard.upstream_attempt`` span around each upstream call. These tests -install a fake tracer (so no ddtrace install is needed) and assert the tag set. -""" - -from __future__ import annotations - -import time -from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import openai -import pytest -from openai.types.chat import ChatCompletion -from openai.types.chat.chat_completion import Choice -from openai.types.chat.chat_completion_message import ChatCompletionMessage - -from switchyard.lib import tracing -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.proxy_context import ProxyContext -from switchyard_rust.core import ChatRequest - -# --------------------------------------------------------------------------- -# Fake tracer (stands in for ddtrace so no install is required) -# --------------------------------------------------------------------------- - - -class _FakeSpan: - def __init__(self, name: str) -> None: - self.name = name - self.tags: dict[str, Any] = {} - - def set_tag(self, key: str, value: Any) -> None: - self.tags[key] = value - - def __enter__(self) -> _FakeSpan: - return self - - def __exit__(self, *_args: object) -> bool: - return False - - -class _FakeTracer: - def __init__(self) -> None: - self.spans: list[_FakeSpan] = [] - - def trace(self, name: str) -> _FakeSpan: - span = _FakeSpan(name) - self.spans.append(span) - return span - - def named(self, name: str) -> list[_FakeSpan]: - return [span for span in self.spans if span.name == name] - - -@pytest.fixture -def tracer(monkeypatch: pytest.MonkeyPatch) -> _FakeTracer: - fake = _FakeTracer() - monkeypatch.setattr(tracing, "_dd_tracer", fake) - return fake - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _config(*models: str, **kwargs: Any) -> LatencyServiceBackendConfig: - return LatencyServiceBackendConfig( - latency_service_url="http://ls.test:8080", - endpoints=[ - LatencyServiceEndpoint(model=m, base_url=f"http://{m}.test", api_key="k") - for m in models - ], - **kwargs, - ) - - -def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend: - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient" - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock() - with patch.object(HealthPoller, "start"): - return LatencyServiceLLMBackend(config) - - -def _set_health( - backend: LatencyServiceLLMBackend, - mapping: dict[str, EndpointHealthStatus | EndpointHealth], -) -> None: - with backend._cache_lock: - for mid, value in mapping.items(): - if isinstance(value, EndpointHealthStatus): - value = EndpointHealth(status=value) - backend._health_cache[mid] = value - - -def _completion(content: str = "ok") -> ChatCompletion: - return ChatCompletion( - id="chatcmpl-test", - object="chat.completion", - created=1700000000, - model="m", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content=content), - finish_reason="stop", - ) - ], - ) - - -def _request(**overrides: Any) -> ChatRequest: - body: dict[str, Any] = {"model": "incoming", "messages": [{"role": "user", "content": "hi"}]} - body.update(overrides) - return ChatRequest.openai_chat(body) # type: ignore[arg-type] - - -def _api_status_error(status_code: int) -> openai.APIStatusError: - response = httpx.Response( - status_code, request=httpx.Request("POST", "http://upstream.test/v1/chat/completions") - ) - return openai.APIStatusError("upstream error", response=response, body=None) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -async def test_success_emits_route_and_attempt_spans(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0)}) - backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion()) - - await backend.call(ProxyContext(), _request(model="incoming")) - - route = tracer.named("switchyard.route_decision") - attempt = tracer.named("switchyard.upstream_attempt") - assert len(route) == 1 - assert len(attempt) == 1 - - # poll-age is absent (poller never recorded a success) — None tags are skipped. - assert route[0].tags == { - "switchyard.model": "incoming", - "switchyard.candidate_endpoints": "model-A", - "switchyard.selected_endpoint": "model-A", - "switchyard.was_fastest_selected": True, - "switchyard.affinity_hit": False, - } - - assert attempt[0].tags == { - "switchyard.model": "incoming", - "switchyard.selected_endpoint": "model-A", - "switchyard.retry_count": 0, - "switchyard.outcome": "success", - "switchyard.upstream_status_code": 200, - } - - -async def test_poll_age_tag_present_after_a_poll(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._poller._last_success_at = time.monotonic() - backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion()) - - await backend.call(ProxyContext(), _request()) - - route = tracer.named("switchyard.route_decision")[0] - assert route.tags["switchyard.latency_service_poll_age_ms"] >= 0.0 - - -async def test_api_status_error_tags(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(429) - ) - - with pytest.raises(openai.APIStatusError): - await backend.call(ProxyContext(), _request()) - - attempt = tracer.named("switchyard.upstream_attempt")[0] - assert attempt.tags["switchyard.upstream_status_code"] == 429 - assert attempt.tags["switchyard.outcome"] == "retryable_error" - assert attempt.tags["switchyard.error_code"] == "429" - # Failure-source annotation: the attempt failed upstream. - assert attempt.tags["switchyard.error_source"] == "provider" - assert attempt.tags["switchyard.upstream_model"] == "model-A" - - -async def test_generic_error_tags(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down")) - - with pytest.raises(RuntimeError, match="down"): - await backend.call(ProxyContext(), _request()) - - attempt = tracer.named("switchyard.upstream_attempt")[0] - assert attempt.tags["switchyard.outcome"] == "retryable_error" - assert attempt.tags["switchyard.error_code"] == "none" - # A non-HTTP failure has no status line. - assert "switchyard.upstream_status_code" not in attempt.tags - # A network-level fault is still an upstream-side failure. - assert attempt.tags["switchyard.error_source"] == "provider" - assert attempt.tags["switchyard.upstream_model"] == "model-A" - - -async def test_retry_count_is_sequential_and_ends_success(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A", "model-B", max_retries=1)) - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - }) - backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down")) - backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion()) - - await backend.call(ProxyContext(), _request()) - - attempts = tracer.named("switchyard.upstream_attempt") - # retry_count is the 0-based attempt index, monotonically increasing. - assert [a.tags["switchyard.retry_count"] for a in attempts] == list(range(len(attempts))) - assert attempts[-1].tags["switchyard.outcome"] == "success" - - -async def test_was_fastest_false_when_latency_unknown(tracer: _FakeTracer) -> None: - backend = _make_backend(_config("model-A", "model-B")) - # Both HEALTHY but no latency samples -> uniform pick, "fastest" undefined. - _set_health(backend, { - "model-A": EndpointHealthStatus.HEALTHY, - "model-B": EndpointHealthStatus.HEALTHY, - }) - selected = {"model-A", "model-B"} - backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion()) - backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion()) - - await backend.call(ProxyContext(), _request()) - - route = tracer.named("switchyard.route_decision")[0] - assert route.tags["switchyard.was_fastest_selected"] is False - assert route.tags["switchyard.selected_endpoint"] in selected - - -async def test_call_works_without_tracer() -> None: - # No ``tracer`` fixture: _dd_tracer is None (ddtrace absent in dev), so the - # spans are no-ops and call() must proceed normally. - backend = _make_backend(_config("model-A")) - _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) - backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion()) - - ctx = ProxyContext() - await backend.call(ctx, _request()) - assert ctx.selected_model == "model-A" diff --git a/tests/test_latency_service_stats_metrics.py b/tests/test_latency_service_stats_metrics.py deleted file mode 100644 index ec011c91..00000000 --- a/tests/test_latency_service_stats_metrics.py +++ /dev/null @@ -1,273 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""End-to-end gate that a latency-service-routed chain exposes ``/metrics``. - -Pins the fix for the production 404: the profile-backed latency service chain -must contribute a stats response processor so Datadog / OTel scrapers do not -see 404 on every scrape. -""" - -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -from fastapi.testclient import TestClient - -from switchyard.lib.backends.health_poller import HealthPoller -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.processors.stats_request_processor import ( - StatsRequestProcessor, -) -from switchyard.lib.processors.stats_response_processor_accumulator import ( - StatsResponseProcessor, -) -from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard -from switchyard.lib.stats_accumulator import StatsAccumulator -from switchyard.server.switchyard_app import build_switchyard_app - - -def _config(*, enable_stats: bool = True) -> LatencyServiceBackendConfig: - return LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="test-key", - base_url="http://llm.test/v1", - ), - ], - enable_stats=enable_stats, - ) - - -def _build_switchyard( - config: LatencyServiceBackendConfig, - *, - stats_accumulator: StatsAccumulator | None = None, -) -> ProfileSwitchyard: - """Build a latency-service profile without starting real poller threads.""" - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - return ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components( - stats_accumulator=stats_accumulator, - enable_stats=config.enable_stats, - ) - ) - - -# --------------------------------------------------------------------------- -# Profile component assembly -# --------------------------------------------------------------------------- - - -def test_profile_wires_stats_processors_by_default() -> None: - switchyard = _build_switchyard(_config()) - - components = list(switchyard.iter_components()) - - assert any(isinstance(p, StatsRequestProcessor) for p in components), ( - "expected StatsRequestProcessor in the latency-service profile" - ) - assert any(isinstance(p, StatsResponseProcessor) for p in components), ( - "expected StatsResponseProcessor in the latency-service profile" - ) - - -def test_profile_skips_stats_when_disabled() -> None: - switchyard = _build_switchyard(_config(enable_stats=False)) - components = list(switchyard.iter_components()) - - assert not any(isinstance(p, StatsRequestProcessor) for p in components) - assert not any(isinstance(p, StatsResponseProcessor) for p in components) - - -async def test_profile_shares_one_accumulator_across_components() -> None: - """Response processor and backend must record into the same accumulator. - - The shared bucket is how ``/metrics`` ends up with both per-call counts - (recorded by the backend) and token usage (recorded by the response - processor) under the same model label. The Rust binding's - ``accumulator`` getter returns a fresh Python wrapper each time, so - we verify shared identity by writing through one surface and reading - through the other. - """ - switchyard = _build_switchyard(_config()) - components = list(switchyard.iter_components()) - - stats_processor = next( - p for p in components - if isinstance(p, StatsResponseProcessor) - ) - backend = next( - p for p in components - if isinstance(p, LatencyServiceLLMBackend) - ) - - # Write through the backend's accumulator handle, then read through the - # response processor's accumulator handle. If construction wired - # two separate accumulators, the read would not see the write. - assert backend._stats is not None - await backend._stats.record_success("model-A", 12.5) - - snapshot = await stats_processor.accumulator.snapshot() - assert snapshot["total_requests"] == 1 - assert snapshot["models"]["model-A"]["calls"] == 1 - - assert stats_processor.get_endpoint() is not None - - -def test_profile_honors_externally_provided_stats_accumulator() -> None: - shared = StatsAccumulator() - switchyard = _build_switchyard(_config(), stats_accumulator=shared) - - stats_processor = next( - p for p in switchyard.iter_components() - if isinstance(p, StatsResponseProcessor) - ) - # The processor's stats source must be the shared accumulator so an - # externally-mounted StatsEndpoint surfaces the same data. - assert stats_processor.get_endpoint() is not None - - -# --------------------------------------------------------------------------- -# End-to-end: build_switchyard_app exposes /metrics + /v1/stats -# --------------------------------------------------------------------------- - - -def _build_app(*, enable_stats: bool): - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(_config(enable_stats=enable_stats)) - .build() - .with_runtime_components(enable_stats=enable_stats) - ) - return build_switchyard_app(switchyard) - - -def test_recipe_app_exposes_metrics_endpoint() -> None: - """Regression: latency-service deployments must serve ``/metrics``. - - Before this fix, Datadog/OTel scrapers got 404 on every poll because the - latency-service chain contributed no ``StatsEndpoint`` via - :func:`build_switchyard_app`'s component iteration. - """ - app = _build_app(enable_stats=True) - - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - stats = client.get("/v1/stats") - routing_stats = client.get("/v1/routing/stats") - - assert metrics.status_code == 200, ( - f"/metrics should be 200 on a latency-service chain, got {metrics.status_code}" - ) - assert metrics.headers["content-type"].startswith("text/plain") - # Prometheus exposition includes the canonical counter family even at zero. - assert "switchyard_requests_total" in metrics.text - - assert stats.status_code == 200 - assert routing_stats.status_code == 200 - - -def test_recipe_app_omits_metrics_when_stats_disabled() -> None: - """``enable_stats=False`` keeps ``/metrics`` unmounted — opt-out is intentional.""" - app = _build_app(enable_stats=False) - - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - stats = client.get("/v1/stats") - - assert metrics.status_code == 404 - assert stats.status_code == 404 - - -# --------------------------------------------------------------------------- -# YAML route-bundle path -# --------------------------------------------------------------------------- - - -def test_route_bundle_latency_service_exposes_metrics() -> None: - """Deployment path (YAML bundle) must also surface ``/metrics``.""" - from switchyard.cli.route_bundle import build_route_bundle_table - - bundle = { - "routes": { - "ls-route": { - "type": "latency_service", - "latency_service_url": "http://latency-service.test:8080", - "endpoints": [ - { - "model": "model-A", - "api_key": "test-key", - "base_url": "http://llm.test/v1", - }, - ], - }, - }, - } - - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - table = build_route_bundle_table(bundle) - app = build_switchyard_app(table) - - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - - assert metrics.status_code == 200 - assert "switchyard_requests_total" in metrics.text - - -def test_route_bundle_latency_service_honors_enable_stats_false() -> None: - """YAML ``enable_stats: false`` opts out of the metrics surface.""" - from switchyard.cli.route_bundle import build_route_bundle_table - - bundle = { - "routes": { - "ls-route": { - "type": "latency_service", - "latency_service_url": "http://latency-service.test:8080", - "enable_stats": False, - "endpoints": [ - { - "model": "model-A", - "api_key": "test-key", - "base_url": "http://llm.test/v1", - }, - ], - }, - }, - } - - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - table = build_route_bundle_table(bundle) - app = build_switchyard_app(table) - - with TestClient(app, raise_server_exceptions=False) as client: - metrics = client.get("/metrics") - - assert metrics.status_code == 404 diff --git a/tests/test_outcome_metrics.py b/tests/test_outcome_metrics.py index 9cb27d23..3d839778 100644 --- a/tests/test_outcome_metrics.py +++ b/tests/test_outcome_metrics.py @@ -5,39 +5,33 @@ from __future__ import annotations -import json -import logging -from unittest.mock import AsyncMock, MagicMock, patch - -import openai import pytest from fastapi.testclient import TestClient from openai.types.chat import ChatCompletion from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_message import ChatCompletionMessage +from openai.types.completion_usage import CompletionUsage -from switchyard.lib.backends.health_poller import ( - EndpointHealth, - EndpointHealthStatus, - HealthPoller, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) from switchyard.lib.endpoints import outcome_metrics from switchyard.lib.endpoints.upstream_error import ( record_upstream_attempt_failure, record_upstream_attempt_success, ) -from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard from switchyard.lib.proxy_context import ( CTX_UPSTREAM_ATTEMPTS_RECORDED, CTX_UPSTREAM_HTTP_STATUS, ProxyContext, ) +from switchyard.lib.roles import LLMBackend +from switchyard.lib.switchyard import Switchyard from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ChatRequest, SwitchyardUpstreamError +from switchyard_rust.core import ( + ChatRequest, + ChatRequestType, + ChatResponse, + SwitchyardUpstreamError, +) +from switchyard_rust.translation import TranslationEngine @pytest.fixture(autouse=True) @@ -47,17 +41,6 @@ def _reset_counters(): outcome_metrics._reset_for_tests() -def _latency_service_switchyard( - config: LatencyServiceBackendConfig, -) -> ProfileSwitchyard: - """Build the latency-service profile-backed serving adapter.""" - return ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components(enable_stats=config.enable_stats) - ) - - # --------------------------------------------------------------------------- # Classification # --------------------------------------------------------------------------- @@ -194,295 +177,6 @@ def test_distinct_codes_get_distinct_series(self) -> None: ) -# --------------------------------------------------------------------------- -# Latency-service backend wiring -# --------------------------------------------------------------------------- - - -def _config(*models: str) -> LatencyServiceBackendConfig: - return LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model=model, - base_url=f"http://llm-{model}.test/v1", - api_key="test-key", - ) - for model in models - ], - max_retries=2, - ) - - -def _build_backend(config: LatencyServiceBackendConfig): - from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, - ) - - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"): - return LatencyServiceLLMBackend(config) - - -def _make_completion() -> ChatCompletion: - return ChatCompletion( - id="cmpl-test", - object="chat.completion", - created=1700000000, - model="any", - choices=[ - Choice( - index=0, - message=ChatCompletionMessage(role="assistant", content="ok"), - finish_reason="stop", - ) - ], - ) - - -def _api_status_error(status_code: int) -> openai.APIStatusError: - """Build a synthetic APIStatusError carrying the given status code. - - Mirrors what ``OpenAILLMClient.acompletion`` raises when the upstream - HTTP call returns a non-2xx response. - """ - import httpx - - response = httpx.Response( - status_code, - request=httpx.Request("POST", "http://llm.test/v1/chat/completions"), - json={"error": {"message": "synthetic"}}, - ) - return openai.APIStatusError( - "synthetic", response=response, body={"error": "synthetic"} - ) - - -class TestBackendCounters: - async def test_success_records_one_attempt_success(self) -> None: - backend = _build_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion() - ) - - await backend.call(ProxyContext(), ChatRequest.openai_chat({ - "model": "model-A", - "messages": [{"role": "user", "content": "hi"}], - })) - - out = "\n".join(outcome_metrics.render_lines()) - assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0' - in out - ) - assert "switchyard_router_retry_recovered_total 0" in out - - async def test_429_then_success_increments_recovered(self) -> None: - """First attempt 429, retry succeeds — the steering signal we care about.""" - backend = _build_backend(_config("model-A", "model-B")) - # Pin model-A as the sole HEALTHY endpoint so it is tried first - # deterministically; otherwise selection is a random coin between two - # UNKNOWN endpoints and the 429-then-recover path only fires by chance. - with backend._cache_lock: - backend._health_cache["model-A"] = EndpointHealth( - status=EndpointHealthStatus.HEALTHY, - ) - backend._health_cache["model-B"] = EndpointHealth( - status=EndpointHealthStatus.UNKNOWN, - ) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(429), - ) - backend._clients["model-B"].acompletion = AsyncMock( - return_value=_make_completion(), - ) - with backend._cache_lock: - backend._health_cache["model-A"] = EndpointHealth( - EndpointHealthStatus.HEALTHY, - 10.0, - ) - backend._health_cache["model-B"] = EndpointHealth( - EndpointHealthStatus.DEGRADED, - 100.0, - ) - - await backend.call(ProxyContext(), ChatRequest.openai_chat({ - "model": "x", - "messages": [{"role": "user", "content": "hi"}], - })) - - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 1' - in out - ) - assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out - assert "switchyard_router_retry_recovered_total 1" in out - - async def test_401_does_not_count_as_retryable(self) -> None: - """A 401 (bad key) is ``other_error`` and is not retried — fail fast.""" - backend = _build_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401), - ) - - import pytest as _pytest - with _pytest.raises(openai.APIStatusError): - await backend.call(ProxyContext(), ChatRequest.openai_chat({ - "model": "x", - "messages": [{"role": "user", "content": "hi"}], - })) - - # A 4xx client error is deterministic, so the loop fails fast: exactly - # one attempt, no failover retries. - assert backend._clients["model-A"].acompletion.call_count == 1 - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="other_error",code="401"} 1' - in out - ) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0' - in out - ) - assert "switchyard_router_retry_recovered_total 0" in out - - async def test_network_error_counts_as_retryable(self) -> None: - """Non-HTTP exceptions (network, pre-status timeout) map to retryable_error.""" - backend = _build_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("connection refused"), - ) - - import pytest as _pytest - with _pytest.raises(RuntimeError): - await backend.call(ProxyContext(), ChatRequest.openai_chat({ - "model": "x", - "messages": [{"role": "user", "content": "hi"}], - })) - - out = "\n".join(outcome_metrics.render_lines()) - assert ( - 'switchyard_upstream_attempts_total{outcome="retryable_error",code="none"} 3' - in out - ) - - async def test_failure_emits_structured_error_log( - self, caplog: pytest.LogCaptureFixture - ) -> None: - """Every failed attempt emits one per-event structured log for Loki. - - Single endpoint that always 429s — deterministic, unlike a - multi-endpoint setup whose first pick is a random choice. - """ - backend = _build_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(429), - ) - - with caplog.at_level(logging.WARNING, logger="switchyard.upstream_errors"): - with pytest.raises(openai.APIStatusError): - await backend.call(ProxyContext(), ChatRequest.openai_chat({ - "model": "x", - "messages": [{"role": "user", "content": "hi"}], - })) - - records = [ - json.loads(r.getMessage()) - for r in caplog.records - if r.name == "switchyard.upstream_errors" - ] - # 3 attempts (1 + max_retries=2), all 429 → 3 structured records, - # attempt numbers 1-based and increasing. - assert len(records) == 3 - assert all(r["event"] == "upstream_attempt_failed" for r in records) - assert all(r["status_code"] == 429 and r["code"] == "429" for r in records) - assert all(r["model"] == "model-A" for r in records) - assert [r["attempt"] for r in records] == [1, 2, 3] - - -# --------------------------------------------------------------------------- -# FastAPI middleware — client-side counter -# --------------------------------------------------------------------------- - - -class TestClientResponseMiddleware: - def test_successful_chat_completion_counts_as_success(self) -> None: - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard( - LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="test-key", - base_url="http://llm.test/v1", - ), - ], - ) - ) - backend = next( - component for component in switchyard.iter_components() - if hasattr(component, "_clients") - ) - backend._clients["model-A"].acompletion = AsyncMock( - return_value=_make_completion(), - ) - - app = build_switchyard_app(switchyard) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": "model-A", - "messages": [{"role": "user", "content": "hi"}], - }, - ) - assert response.status_code == 200 - metrics = client.get("/metrics").text - - assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics - assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics - - def test_metrics_route_is_not_counted_as_client_response(self) -> None: - """Only the LLM routes feed the client outcome counter. Otherwise a - scraper polling /metrics every 10s would inflate the success count.""" - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard( - LatencyServiceBackendConfig( - latency_service_url="http://latency.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="test-key", - base_url="http://llm.test/v1", - ), - ], - ) - ) - app = build_switchyard_app(switchyard) - with TestClient(app, raise_server_exceptions=False) as client: - for _ in range(5): - client.get("/metrics") - client.get("/health") - client.get("/v1/models") - metrics = client.get("/metrics").text - - assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics - - # --------------------------------------------------------------------------- # Endpoint-layer fallback — wires the upstream-attempt counter for backends # (Rust native / passthrough / multi) that issue one attempt and can't reach @@ -540,17 +234,71 @@ def test_dedup_flag_suppresses_fallback(self) -> None: assert f"{_upstream_count(out, 'success', '200')} 0" in out assert f"{_upstream_count(out, 'retryable_error', 'none')} 0" in out - async def test_latency_service_backend_sets_dedup_flag(self) -> None: - """The latency-service backend claims attempt accounting on its ctx.""" - backend = _build_backend(_config("model-A")) - backend._clients["model-A"].acompletion = AsyncMock(return_value=_make_completion()) - ctx = ProxyContext() - await backend.call(ctx, ChatRequest.openai_chat({ - "model": "model-A", - "messages": [{"role": "user", "content": "hi"}], - })) - assert ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED) is True - # It recorded exactly one attempt itself — and the flag would stop the - # endpoint fallback from adding a second. - out = "\n".join(outcome_metrics.render_lines()) - assert f"{_upstream_count(out, 'success', '200')} 1" in out + +# --------------------------------------------------------------------------- +# Client-response middleware — only the /v1/* LLM routes feed the client +# outcome counter; the operational routes (/metrics, /health, /v1/models) do +# not, so a scraper polling them cannot inflate the success count. +# --------------------------------------------------------------------------- + + +class _CannedBackend(LLMBackend): + """Passthrough-style backend: returns one canned OpenAI completion per call.""" + + def supported_request_types(self) -> list[ChatRequestType]: + return [ + ChatRequestType.OPENAI_CHAT, + ChatRequestType.OPENAI_RESPONSES, + ChatRequestType.ANTHROPIC, + ] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + return ChatResponse.openai_completion( + ChatCompletion( + id="chatcmpl-test", + object="chat.completion", + created=1700000000, + model="mock-model", + choices=[ + Choice( + index=0, + message=ChatCompletionMessage(role="assistant", content="hi"), + finish_reason="stop", + ) + ], + usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8), + ) + ) + + +def _canned_app() -> object: + return build_switchyard_app( + Switchyard(backend=_CannedBackend(), translator=TranslationEngine()) + ) + + +class TestClientResponseMiddleware: + def test_successful_chat_completion_counts_as_success(self) -> None: + """A served /v1/chat/completions success increments the client outcome counter.""" + with TestClient(_canned_app(), raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={"model": "mock-model", "messages": [{"role": "user", "content": "hi"}]}, + ) + assert response.status_code == 200 + + metrics = "\n".join(outcome_metrics.render_lines()) + assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics + assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics + + def test_operational_routes_are_not_counted_as_client_responses(self) -> None: + """Polling /metrics, /health, /v1/models must not feed the client counter — + otherwise a scraper on a fixed cadence would inflate the success count.""" + with TestClient(_canned_app(), raise_server_exceptions=False) as client: + for _ in range(5): + client.get("/metrics") + client.get("/health") + client.get("/v1/models") + + metrics = "\n".join(outcome_metrics.render_lines()) + assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics diff --git a/tests/test_passthrough_profile_e2e.py b/tests/test_passthrough_profile_e2e.py index 1e96a287..3a42ee28 100644 --- a/tests/test_passthrough_profile_e2e.py +++ b/tests/test_passthrough_profile_e2e.py @@ -399,7 +399,7 @@ async def test_upstream_401_does_not_return_200( class TestPassthroughProfileUpstreamAttemptCounters: - """`switchyard_upstream_attempts_total` must populate for non-latency chains. + """`switchyard_upstream_attempts_total` must populate for passthrough chains. The endpoint-layer fallback records one upstream attempt per request for backends (here the Rust ``OpenAiPassthroughBackend``) that issue exactly diff --git a/tests/test_redis_pin_store.py b/tests/test_redis_pin_store.py index 49a7269d..1ccf5faf 100644 --- a/tests/test_redis_pin_store.py +++ b/tests/test_redis_pin_store.py @@ -14,11 +14,8 @@ import pytest from pydantic import ValidationError -from switchyard.lib.backends.latency_service_llm_backend import _build_affinity_l2 -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) +from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig +from switchyard.lib.profiles.escalation_router_profile_config import _build_affinity_l2 from switchyard.lib.redis_pin_store import RedisPinStore @@ -86,15 +83,17 @@ async def test_aclose_before_first_use_is_noop() -> None: # --- config validation + backend wiring ------------------------------------- -def _redis_config(**overrides: object) -> LatencyServiceBackendConfig: +def _redis_config(**overrides: object) -> EscalationRouterConfig: base: dict[str, object] = { - "endpoints": [LatencyServiceEndpoint(model="m")], - "session_affinity": True, + "strong": {"model": "strong-model"}, + "weak": {"model": "weak-model"}, + "judge": {"model": "judge-model"}, + "fallback_target_on_evict": "weak", "affinity_store": "redis", "affinity_store_url": "redis://cache:6379/0", } base.update(overrides) - return LatencyServiceBackendConfig(**base) # type: ignore[arg-type] + return EscalationRouterConfig(**base) # type: ignore[arg-type] def test_redis_store_requires_url() -> None: @@ -102,13 +101,13 @@ def test_redis_store_requires_url() -> None: _redis_config(affinity_store_url=None) -def test_redis_store_requires_session_affinity() -> None: - with pytest.raises(ValidationError): - _redis_config(session_affinity=False) - - def test_memory_is_the_default_store() -> None: - cfg = LatencyServiceBackendConfig(endpoints=[LatencyServiceEndpoint(model="m")]) + cfg = EscalationRouterConfig( + strong={"model": "strong-model"}, + weak={"model": "weak-model"}, + judge={"model": "judge-model"}, + fallback_target_on_evict="weak", + ) assert cfg.affinity_store == "memory" assert _build_affinity_l2(cfg) is None diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index b1a7e68a..2f736554 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -50,18 +50,6 @@ def _random_processor(chain: object) -> Any: ) -def _latency_backend(chain: object) -> Any: - from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, - ) - - return next( - component - for component in chain.iter_components() - if isinstance(component, LatencyServiceLLMBackend) - ) - - class _NoopRequestProcessor: async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest: return request @@ -1486,115 +1474,15 @@ def test_route_bundle_keeps_first_multi_target_route_as_default() -> None: "strong": {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"}, "weak": {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"}, } -_AFFINITY_LAT_ROUTE = { - "type": "latency_service", - "latency_service_url": "http://ls.test:8080", - "session_affinity": True, - "affinity_max_sessions": 50, - "endpoints": [ - {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"}, - {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"}, - ], -} -@pytest.mark.parametrize("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE]) +@pytest.mark.parametrize("route", [_AFFINITY_DET_ROUTE]) def test_session_affinity_keys_accepted_by_route_bundle(route: dict[str, Any]) -> None: - """session_affinity / affinity_max_sessions are valid keys on both route types.""" + """session_affinity / affinity_max_sessions are valid keys on the deterministic route.""" table = build_route_bundle_table({"routes": {"r": route}}) assert isinstance(table, RouteTable) -def test_latency_service_credential_policy_defaults_to_endpoint_keys() -> None: - """Omitting credential_policy keeps the server-configured endpoint keys authoritative.""" - table = build_route_bundle_table({"routes": {"r": _AFFINITY_LAT_ROUTE}}) - - backend = _latency_backend(table.lookup_switchyard("r")) - - assert backend._config.credential_policy == "configured_endpoint" - - -def test_latency_service_credential_policy_reaches_backend_config() -> None: - """YAML credential_policy opts into BYO-key caller overrides.""" - table = build_route_bundle_table({ - "routes": { - "r": { - **_AFFINITY_LAT_ROUTE, - "credential_policy": "caller_override", - }, - }, - }) - - backend = _latency_backend(table.lookup_switchyard("r")) - - assert backend._config.credential_policy == "caller_override" - - -def test_latency_service_invalid_credential_policy_rejected_via_bundle() -> None: - """Invalid latency-service credential policy values fail closed.""" - from pydantic import ValidationError - - with pytest.raises(ValidationError): - build_route_bundle_table({ - "routes": { - "r": { - **_AFFINITY_LAT_ROUTE, - "credential_policy": "caller-overrides", - }, - }, - }) - - -def test_latency_endpoint_request_type_reaches_backend_config() -> None: - """Endpoint-level request_type in YAML selects the upstream API surface.""" - table = build_route_bundle_table({ - "routes": { - "r": { - "type": "latency_service", - "latency_service_url": "http://ls.test:8080", - "endpoints": [ - { - "model": "codex-mini", - "api_key": "k", - "base_url": "https://ls.test/v1", - "request_type": "openai_responses", - }, - {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"}, - ], - }, - }, - }) - - backend = _latency_backend(table.lookup_switchyard("r")) - - by_model = {endpoint.model: endpoint for endpoint in backend._config.endpoints} - assert by_model["codex-mini"].request_type == "openai_responses" - assert by_model["w"].request_type == "openai_chat" - - -def test_latency_route_key_reaches_backend_as_route_model() -> None: - """The YAML route key becomes the backend's metrics route_model id.""" - table = build_route_bundle_table({ - "routes": { - "nvidia/switchyard/gpt-5.4": { - "type": "latency_service", - "latency_service_url": "http://ls.test:8080", - "endpoints": [ - { - "model": "azure/openai/gpt-5.4", - "api_key": "k", - "base_url": "https://ls.test/v1", - }, - ], - }, - }, - }) - - backend = _latency_backend(table.lookup_switchyard("nvidia/switchyard/gpt-5.4")) - - assert backend._config.route_model == "nvidia/switchyard/gpt-5.4" - - def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None: """affinity_warmup_turns is a deterministic-route knob.""" table = build_route_bundle_table({ @@ -1603,7 +1491,7 @@ def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None: assert isinstance(table, RouteTable) -@pytest.mark.parametrize("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE]) +@pytest.mark.parametrize("route", [_AFFINITY_DET_ROUTE]) def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> None: """A zero cap with affinity on is rejected — proving the keys reach the config.""" from pydantic import ValidationError @@ -1612,37 +1500,6 @@ def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> No build_route_bundle_table({"routes": {"r": {**route, "affinity_max_sessions": 0}}}) -def test_latency_affinity_redis_reaches_backend_via_bundle() -> None: - """Redis L2 keys parse and construct a RedisPinStore on the latency backend.""" - from switchyard.lib.redis_pin_store import RedisPinStore - - table = build_route_bundle_table({ - "routes": { - "r": { - **_AFFINITY_LAT_ROUTE, - "affinity_store": "redis", - "affinity_store_url": "redis://cache:6379/0", - "affinity_store_ttl_seconds": 120, - "affinity_key_prefix": "k:", - }, - }, - }) - backend = _latency_backend(table.lookup_switchyard("r")) - assert backend._config.affinity_store == "redis" - assert backend._config.affinity_store_url == "redis://cache:6379/0" - assert isinstance(backend._affinity._l2, RedisPinStore) - - -def test_latency_affinity_redis_requires_url_via_bundle() -> None: - """affinity_store=redis without a URL fails closed at config load.""" - from pydantic import ValidationError - - with pytest.raises(ValidationError): - build_route_bundle_table({ - "routes": {"r": {**_AFFINITY_LAT_ROUTE, "affinity_store": "redis"}}, - }) - - def test_negative_affinity_warmup_turns_rejected_via_bundle() -> None: """The deterministic config validates affinity_warmup_turns as non-negative.""" from pydantic import ValidationError diff --git a/tests/test_route_selection_headers.py b/tests/test_route_selection_headers.py index 350498f4..4b1dabac 100644 --- a/tests/test_route_selection_headers.py +++ b/tests/test_route_selection_headers.py @@ -1,43 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""End-to-end gate for route-selection spend-attribution headers. +"""Route-selection spend-attribution response headers. For tokenomics reporting, a front proxy (e.g. LiteLLM) must be able to tie its provider spend-log rows back to the Switchyard logical route that -selected them: - -* outbound — every upstream attempt carries an ``x-litellm-spend-logs-metadata`` - request header whose JSON payload records the route selection plus a - per-request correlation id; -* inbound — the Switchyard response returns ``x-switchyard-*`` headers with - the successful attempt's selection and the same correlation id, so the - parent spend-log row can be enriched to match the provider row. - -The app-level tests run the real ``OpenAILLMClient`` against a loopback HTTP -stub, so the outbound header is asserted on the wire — not on a mock. +selected them: the Switchyard response returns ``x-switchyard-*`` headers with +the successful attempt's selection plus a per-request correlation id, so the +parent spend-log row can be enriched to match the provider row. """ from __future__ import annotations import json -from collections.abc import Iterator from typing import Any -from fastapi import FastAPI from fastapi.responses import JSONResponse -from fastapi.testclient import TestClient -from pytest import fixture -from pytest_mock import MockerFixture -from switchyard.lib.backends.health_poller import HealthPoller -from switchyard.lib.backends.latency_service_llm_backend import ( - SPEND_LOGS_METADATA_HEADER, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) from switchyard.lib.endpoints.dispatch import serialize_chain_result from switchyard.lib.endpoints.route_selection import ( ROUTER_CORRELATION_ID_HEADER, @@ -47,10 +26,7 @@ route_selection_headers, ) from switchyard.lib.endpoints.upstream_error import handle_chain_exception -from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard from switchyard.lib.proxy_context import CTX_ROUTE_SELECTION, ProxyContext -from switchyard.server.switchyard_app import build_switchyard_app -from tests._chain_test_helpers import _backend_payload, _OpenAICompatStub, _sse_body, _stream_chunk ROUTE_MODEL = "nvidia/switchyard/test-route" ENDPOINT_ID = "openai/test-model" @@ -58,10 +34,9 @@ def _selection(**overrides: object) -> dict[str, object]: - """Selection record as the latency backend writes it, with *overrides* applied.""" + """Recorded route selection, with *overrides* applied.""" payload: dict[str, object] = { "router_model": ROUTE_MODEL, - "router_strategy": "latency", "router_selected_endpoint": ENDPOINT_ID, "router_selected_model": UPSTREAM_MODEL, "router_selected_provider": "openai", @@ -256,240 +231,3 @@ def test_failure_without_selection_has_no_selection_headers(self) -> None: assert response.status_code == 500 assert ROUTER_CORRELATION_ID_HEADER not in response.headers - - -# --------------------------------------------------------------------------- -# End-to-end: HTTP in → wire header out → response headers back -# --------------------------------------------------------------------------- - - -@fixture -def latency_app(mocker: MockerFixture) -> Iterator[tuple[FastAPI, _OpenAICompatStub]]: - """Latency-service app whose single endpoint targets a loopback stub. - - The backend uses the real ``OpenAILLMClient``/OpenAI SDK, so the stub - records the actual HTTP headers the upstream would receive. Only the - health poller is stubbed out (no Latency Service in tests). - """ - with _OpenAICompatStub() as stub: - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - route_model=ROUTE_MODEL, - endpoints=[ - LatencyServiceEndpoint( - model=ENDPOINT_ID, - upstream_model=UPSTREAM_MODEL, - base_url=stub.base_url, - api_key="test-key", - ), - ], - ) - mocker.patch.object(HealthPoller, "start") - mocker.patch.object(HealthPoller, "stop") - switchyard = ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components(enable_stats=config.enable_stats) - ) - yield build_switchyard_app(switchyard), stub - - -def _wire_spend_logs_payload(stub: _OpenAICompatStub) -> dict[str, object]: - """Parse the spend-logs metadata header the stub received on the wire.""" - headers = stub.requests[-1]["headers"] - assert isinstance(headers, dict) - payload = json.loads(str(headers[SPEND_LOGS_METADATA_HEADER])) - assert isinstance(payload, dict) - return payload - - -def test_chat_completion_roundtrips_selection_and_correlation_id( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """The wire spend-logs header and the response headers share one selection.""" - app, stub = latency_app - stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL)) - - with TestClient(app) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": ROUTE_MODEL, - "messages": [{"role": "user", "content": "hi"}], - }, - ) - - assert response.status_code == 200 - payload = _wire_spend_logs_payload(stub) - assert payload["router_model"] == ROUTE_MODEL - assert payload["router_strategy"] == "latency" - assert payload["router_selected_endpoint"] == ENDPOINT_ID - assert payload["router_selected_model"] == UPSTREAM_MODEL - assert payload["router_selected_provider"] == "openai" - assert response.headers[ROUTER_MODEL_HEADER] == ROUTE_MODEL - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - assert response.headers[SELECTED_PROVIDER_HEADER] == "openai" - # The join key: the response header must equal the id the provider row got. - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"] - - -def test_streaming_chat_completion_carries_selection_headers( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """A streaming completion returns the same selection headers before the body.""" - app, stub = latency_app - stub.respond_sse(_sse_body([_stream_chunk(content="hel"), _stream_chunk(finish="stop")])) - - with TestClient(app) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": ROUTE_MODEL, - "messages": [{"role": "user", "content": "hi"}], - "stream": True, - }, - ) - - assert response.status_code == 200 - assert response.headers["content-type"].startswith("text/event-stream") - assert "data:" in response.text - payload = _wire_spend_logs_payload(stub) - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"] - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - - -def test_anthropic_messages_endpoint_carries_selection_headers( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """Anthropic inbound rides the same dispatch path and gets the same headers.""" - app, stub = latency_app - stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL)) - - with TestClient(app) as client: - response = client.post( - "/v1/messages", - json={ - "model": ROUTE_MODEL, - "max_tokens": 64, - "messages": [{"role": "user", "content": "hi"}], - }, - ) - - assert response.status_code == 200 - payload = _wire_spend_logs_payload(stub) - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"] - assert response.headers[SELECTED_PROVIDER_HEADER] == "openai" - - -def test_responses_endpoint_carries_selection_headers( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """OpenAI Responses inbound rides the same dispatch path and gets the same headers.""" - app, stub = latency_app - stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL)) - - with TestClient(app) as client: - response = client.post( - "/v1/responses", - json={"model": ROUTE_MODEL, "input": "hi"}, - ) - - assert response.status_code == 200 - payload = _wire_spend_logs_payload(stub) - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"] - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - - -def test_upstream_error_response_has_no_selection_headers( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """No upstream call succeeded — the error response must not claim a selection. - - The failed attempt itself is still stamped on the wire (the provider row - for the failure keeps its attribution); the client-facing headers are - reserved for a recorded selection, i.e. a billed upstream success — which - never happened here. (A failure *after* a billed success does carry them; - see TestErrorPathSelectionHeaders.) - """ - app, stub = latency_app - stub.respond_json({"error": {"message": "bad key"}}, status=401) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": ROUTE_MODEL, - "messages": [{"role": "user", "content": "hi"}], - }, - ) - - assert response.status_code == 401 - assert ROUTER_CORRELATION_ID_HEADER not in response.headers - assert SPEND_LOGS_METADATA_HEADER in stub.requests[-1]["headers"] # type: ignore[operator] - - -def test_hostile_model_string_never_breaks_the_response( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """A model string that is not legal header material must not 500 or inject. - - Single-chain profile serving forwards any client model string, so - router_model is client-controlled; the billed 200 must survive, with the - unsafe header skipped and the config-derived headers intact. - """ - app, stub = latency_app - - with TestClient(app) as client: - for hostile_model in ("gpt-4中文", "gpt\r\nx-evil: 1"): - stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL)) - response = client.post( - "/v1/chat/completions", - json={ - "model": hostile_model, - "messages": [{"role": "user", "content": "hi"}], - }, - ) - - assert response.status_code == 200 - assert ROUTER_MODEL_HEADER not in response.headers - assert "x-evil" not in response.headers - assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL - # The outbound header is JSON (ensure_ascii escapes control and - # non-ASCII chars), so the wire payload still records the model. - assert _wire_spend_logs_payload(stub)["router_model"] == hostile_model - - -def test_client_body_extra_headers_forwarded_not_collided( - latency_app: tuple[FastAPI, _OpenAICompatStub], -) -> None: - """A passthrough body carrying an SDK-style extra_headers field keeps working. - - Pre-spend-logs it rode ``**body`` into the SDK's header kwarg; it must - still reach the wire — and must not be able to spoof the spend-logs - header under any key casing, since header names are case-insensitive on - the wire. - """ - app, stub = latency_app - stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL)) - - with TestClient(app) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": ROUTE_MODEL, - "messages": [{"role": "user", "content": "hi"}], - "extra_headers": { - "x-client-tag": "42", - SPEND_LOGS_METADATA_HEADER: "spoofed", - "X-LiteLLM-Spend-Logs-Metadata": "spoofed-cased", - }, - }, - ) - - assert response.status_code == 200 - headers = stub.requests[-1]["headers"] - assert isinstance(headers, dict) - assert headers["x-client-tag"] == "42" - payload = _wire_spend_logs_payload(stub) - assert payload["router_selected_endpoint"] == ENDPOINT_ID # not "spoofed" - assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"] diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.py index 26cd8897..d06a8ac5 100644 --- a/tests/test_session_affinity.py +++ b/tests/test_session_affinity.py @@ -3,7 +3,7 @@ """Unit tests for SessionAffinity's L1 (in-process) + optional L2 (shared) tiering. -The latency backend and classifier router integration tests cover affinity +The classifier and escalation router integration tests cover affinity *policy*; these tests cover the store mechanics in isolation: write-through, read-through with L1 warming, and best-effort (fail-open) degradation when the L2 errors. diff --git a/tests/test_stream_close_chain.py b/tests/test_stream_close_chain.py index 13484499..81b6e085 100644 --- a/tests/test_stream_close_chain.py +++ b/tests/test_stream_close_chain.py @@ -3,7 +3,7 @@ """Close ownership survives the Python -> Rust-core -> Python round trip. -The latency-router production path is: a Python backend returns +The production path is: a Python backend returns ``ChatResponse.openai_stream(ResponseStream(sdk_stream))``; the backend adapter converts it to a Rust-core stream via ``take_core``; response processors run in core; the terminal translator rebuilds a Python ``ChatResponseStream`` via diff --git a/tests/test_stream_leak_repro.py b/tests/test_stream_leak_repro.py index ec00b6f9..22c1f571 100644 --- a/tests/test_stream_leak_repro.py +++ b/tests/test_stream_leak_repro.py @@ -195,7 +195,7 @@ async def test_our_close_releases_pool_connection() -> None: class _RealSdkStreamingBackend(LLMBackend): """Backend that issues a real streaming SDK call and wraps the AsyncStream. - Mirrors the latency-service production shape: ``call`` returns + Mirrors the production shape: ``call`` returns ``ChatResponse.openai_stream(ResponseStream(sdk_stream))`` where ``sdk_stream`` is a genuine OpenAI ``AsyncStream`` holding a pooled httpx connection. Running this through ``Switchyard`` exercises the real diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py index e2be77f5..5952906d 100644 --- a/tests/test_upstream_error_passthrough.py +++ b/tests/test_upstream_error_passthrough.py @@ -19,99 +19,23 @@ from __future__ import annotations import json -from collections.abc import AsyncIterator -from unittest.mock import AsyncMock, MagicMock, patch -import httpx -import openai -import pytest -from fastapi import FastAPI from fastapi.testclient import TestClient from switchyard.cli.route_bundle import build_route_bundle_table -from switchyard.lib.backends.health_poller import HealthPoller -from switchyard.lib.backends.latency_service_llm_backend import ( - LatencyServiceLLMBackend, -) -from switchyard.lib.config.latency_service_backend_config import ( - LatencyServiceBackendConfig, - LatencyServiceEndpoint, -) -from switchyard.lib.endpoints.error_envelope import ( - ERROR_SOURCE_HEADER, - UPSTREAM_MODEL_HEADER, -) +from switchyard.lib.endpoints.error_envelope import ERROR_SOURCE_HEADER from switchyard.lib.endpoints.upstream_error import ( internal_chain_error_response, upstream_response_from_ctx, ) -from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard from switchyard.lib.proxy_context import ( CTX_UPSTREAM_HTTP_BODY, CTX_UPSTREAM_HTTP_STATUS, ProxyContext, ) from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import ChatRequest from tests._chain_test_helpers import _OpenAICompatStub -# --------------------------------------------------------------------------- -# Helper -# --------------------------------------------------------------------------- - - -def _api_status_error(status: int, body: object = None) -> openai.APIStatusError: - """Build an ``openai.APIStatusError`` with a realistic response object.""" - request = httpx.Request("POST", "http://upstream.test/v1/chat/completions") - response = httpx.Response( - status_code=status, - json=body if body is not None else {"error": {"message": f"upstream {status}"}}, - request=request, - ) - return openai.APIStatusError( - message=f"upstream returned {status}", - response=response, - body=body, - ) - - -def _latency_service_switchyard( - config: LatencyServiceBackendConfig, -) -> ProfileSwitchyard: - """Build the latency-service profile-backed serving adapter.""" - return ProfileSwitchyard( - LatencyServiceProfileConfig.from_config(config) - .build() - .with_runtime_components(enable_stats=config.enable_stats) - ) - - -def _make_backend() -> LatencyServiceLLMBackend: - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="bad-key", - base_url="http://llm.test/v1", - ), - ], - ) - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"): - return LatencyServiceLLMBackend(config) - - -def _openai_request() -> ChatRequest: - return ChatRequest.openai_chat({ - "model": "model-A", - "messages": [{"role": "user", "content": "hi"}], - }) - - # --------------------------------------------------------------------------- # Helper unit tests # --------------------------------------------------------------------------- @@ -228,266 +152,11 @@ def test_internal_chain_error_truncates_long_repr(self) -> None: assert len(body["error"]["message"]) <= 200 -# --------------------------------------------------------------------------- -# Backend records status on ctx before raising -# --------------------------------------------------------------------------- - - -class TestLatencyServiceBackendStashesStatus: - async def test_401_recorded_on_ctx_before_raise(self) -> None: - backend = _make_backend() - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401, {"error": {"message": "bad key"}}), - ) - - ctx = ProxyContext() - with pytest.raises(openai.APIStatusError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 - body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY] - assert isinstance(body, dict) - assert body["error"]["message"] == "bad key" - - async def test_429_recorded_on_ctx_before_raise(self) -> None: - backend = _make_backend() - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(429, {"error": {"message": "slow down"}}), - ) - - ctx = ProxyContext() - with pytest.raises(openai.APIStatusError): - await backend.call(ctx, _openai_request()) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 429 - - async def test_non_http_error_leaves_ctx_clean(self) -> None: - """Network errors etc shouldn't trip the upstream-status passthrough.""" - backend = _make_backend() - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("connection refused"), - ) - - ctx = ProxyContext() - with pytest.raises(RuntimeError, match="connection refused"): - await backend.call(ctx, _openai_request()) - - assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata - assert CTX_UPSTREAM_HTTP_BODY not in ctx.metadata - - -# --------------------------------------------------------------------------- -# End-to-end: HTTP request through the recipe → upstream 401 → client 401 -# --------------------------------------------------------------------------- - - -@pytest.fixture -async def latency_service_app() -> AsyncIterator[tuple[FastAPI, LatencyServiceLLMBackend]]: - """Build a latency-service-backed FastAPI app sharing one backend instance. - - Returns ``(app, backend)`` so each test can patch ``backend._clients[...].acompletion`` - to simulate a specific upstream response. - """ - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="bad-key", - base_url="http://llm.test/v1", - ), - ], - ) - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard(config) - app = build_switchyard_app(switchyard) - # Reach into the chain to find the LatencyServiceLLMBackend - # instance so tests can stub the upstream call. - backend = _find_latency_backend(switchyard) - yield app, backend - - -def _find_latency_backend(switchyard: object) -> LatencyServiceLLMBackend: - iter_components = getattr(switchyard, "iter_components", None) - assert iter_components is not None - for component in iter_components(): - # The latency-service backend may be wrapped (it isn't today, but - # be tolerant) — match by isinstance, with attribute-walk fallback. - if isinstance(component, LatencyServiceLLMBackend): - return component - inner = getattr(component, "_inner", None) or getattr(component, "inner", None) - if isinstance(inner, LatencyServiceLLMBackend): - return inner - raise AssertionError("LatencyServiceLLMBackend not found in chain") - - -def test_upstream_401_passes_through_as_401( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """A 401 from upstream must become a 401 to the client, not a 500.""" - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error( - 401, {"error": {"message": "bad key", "type": "invalid_api_key"}}, - ), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 401, ( - f"expected upstream 401 to pass through; got {response.status_code}, " - f"body={response.text!r}" - ) - body = response.json() - assert body["error"]["message"] == "bad key" - assert body["error"]["type"] == "invalid_api_key" - assert body["error"]["code"] == "invalid_api_key" - - -def test_upstream_429_passes_through_as_429( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error( - 429, {"error": {"message": "rate limit", "type": "rate_limit_exceeded"}}, - ), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 429 - assert response.json() == { - "error": { - "message": "rate limit", - "type": "rate_limit_exceeded", - "code": "rate_limit_exceeded", - } - } - - -def test_non_http_error_returns_wrapped_500( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """Errors without an upstream HTTP status still return a JSON error envelope.""" - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("connection refused"), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 500 - assert response.headers["content-type"].startswith("application/json") - body = response.json() - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - assert "connection refused" in body["error"]["message"] - - -def test_post_dispatch_exception_returns_json_500( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """Exception raised after dispatch (e.g. during result serialization) must not - fall through to FastAPI's plain-text 500 handler.""" - app, _ = latency_service_app - - class _UnserializableResult: - def model_dump(self) -> None: - raise RuntimeError("serialization exploded") - - with patch( - "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request", - new_callable=AsyncMock, - return_value=_UnserializableResult(), - ): - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 500 - assert response.headers["content-type"].startswith("application/json") - body = response.json() - assert body["error"]["type"] == "internal_error" - assert body["error"]["code"] == "internal_chain_error" - assert "serialization exploded" in body["error"]["message"] - - # --------------------------------------------------------------------------- # Anthropic and Responses endpoints share the same helper # --------------------------------------------------------------------------- -def test_anthropic_endpoint_passes_through_upstream_401( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401, {"error": {"message": "bad key"}}), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/messages", - json={ - "model": "model-A", - "max_tokens": 16, - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert response.status_code == 401 - assert response.json() == { - "error": { - "message": "bad key", - "type": "upstream_error", - "code": "upstream_error", - } - } - - -def test_responses_endpoint_passes_through_upstream_401( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401, {"error": {"message": "bad key"}}), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={"model": "model-A", "input": "ping"}, - ) - - assert response.status_code == 401 - assert response.json() == { - "error": { - "message": "bad key", - "type": "upstream_error", - "code": "upstream_error", - } - } - - def test_rust_openai_route_upstream_401_returns_structured_openai_error() -> None: """Rust OpenAI-native backend errors must not fall through to FastAPI's 500.""" with _OpenAICompatStub() as upstream: @@ -609,111 +278,6 @@ def test_rust_openai_route_upstream_401_returns_same_error_shape_for_responses_i } -# --------------------------------------------------------------------------- -# Invalid request payloads -# --------------------------------------------------------------------------- -# A transparent router must reject the same payloads the upstream provider -# would. An unsupported message role (e.g. "api") was previously coerced to -# "user" and returned 200; it now surfaces as a provider-compatible 400 via the -# same ctx-stash → endpoint-passthrough mechanism used for upstream HTTP errors. - - -class TestInvalidRoleRejection: - async def test_invalid_role_stashes_400_without_upstream_call(self) -> None: - """An unsupported role is rejected at translation, before any upstream call.""" - backend = _make_backend() - backend._clients["model-A"].acompletion = AsyncMock() - - ctx = ProxyContext() - request = ChatRequest.openai_responses({ - "model": "model-A", - "input": [{"type": "message", "role": "api", "content": "hi"}], - }) - with pytest.raises(ValueError): - await backend.call(ctx, request) - - assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400 - body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY] - assert isinstance(body, dict) - assert body["error"]["code"] == "invalid_value" - # The upstream must never be called for an invalid payload. - backend._clients["model-A"].acompletion.assert_not_awaited() - - async def test_valid_role_passes_translation_and_reaches_upstream(self) -> None: - """A valid role must not be over-rejected — translation succeeds and the - request proceeds to the upstream call (no invalid-value 400 stashed).""" - backend = _make_backend() - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("reached upstream"), - ) - - ctx = ProxyContext() - request = ChatRequest.openai_responses({ - "model": "model-A", - "input": [{"type": "message", "role": "user", "content": "hi"}], - }) - with pytest.raises(RuntimeError, match="reached upstream"): - await backend.call(ctx, request) - - assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata - - -def test_responses_invalid_role_returns_400( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """role:"api" on the Responses endpoint must return 400, not a coerced 200/500.""" - app, backend = latency_service_app - # No upstream stub: the request must be rejected before any upstream call. - backend._clients["model-A"].acompletion = AsyncMock() - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={ - "model": "model-A", - "input": [{"type": "message", "role": "api", "content": "ping"}], - }, - ) - - assert response.status_code == 400, ( - f"expected invalid role to be rejected with 400; got {response.status_code}, " - f"body={response.text!r}" - ) - body = response.json() - assert body["error"]["code"] == "invalid_value" - backend._clients["model-A"].acompletion.assert_not_awaited() - - -def test_anthropic_invalid_role_returns_400( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """role:"api" on the Anthropic endpoint is rejected with a 400. - - The Anthropic inbound format must be translated to OpenAI Chat for the - latency backend, so the request is decoded and the unsupported role is - rejected before any upstream call. (A native OpenAI-Chat request is a - passthrough for this backend, so its contract is enforced upstream.) - """ - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock() - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/messages", - json={ - "model": "model-A", - "max_tokens": 16, - "messages": [{"role": "api", "content": "ping"}], - }, - ) - - assert response.status_code == 400, ( - f"expected invalid role to be rejected with 400; got {response.status_code}, " - f"body={response.text!r}" - ) - backend._clients["model-A"].acompletion.assert_not_awaited() - - # --------------------------------------------------------------------------- # Failure-source headers on the wire # --------------------------------------------------------------------------- @@ -722,71 +286,6 @@ def test_anthropic_invalid_role_returns_400( # the HTTP response a client receives. -def test_upstream_401_header_labels_provider_on_the_wire( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=_api_status_error(401, {"error": {"message": "bad key"}}), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 401 - assert response.headers[ERROR_SOURCE_HEADER] == "provider" - assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A" - # The provider body still passes through unmodified — annotation is header-only. - assert response.json()["error"]["message"] == "bad key" - - -def test_network_failure_500_header_labels_provider_on_the_wire( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """A status-less upstream fault renders as the internal 500 envelope but - the header says provider — clients can tell it wasn't a Switchyard bug.""" - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock( - side_effect=RuntimeError("connection refused"), - ) - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, - ) - - assert response.status_code == 500 - assert response.json()["error"]["code"] == "internal_chain_error" - assert response.headers[ERROR_SOURCE_HEADER] == "provider" - assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A" - - -def test_invalid_role_400_header_labels_switchyard_on_the_wire( - latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], -) -> None: - """The translation rejection rides the upstream-status channel but must - surface as switchyard-originated on the wire.""" - app, backend = latency_service_app - backend._clients["model-A"].acompletion = AsyncMock() - - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={ - "model": "model-A", - "input": [{"type": "message", "role": "api", "content": "ping"}], - }, - ) - - assert response.status_code == 400 - assert response.json()["error"]["code"] == "invalid_value" - assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" - - def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None: """RouteTable dispatch 404s carry the switchyard source header.""" with _OpenAICompatStub() as upstream: @@ -816,232 +315,3 @@ def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None: assert response.status_code == 404 assert response.json()["error"]["code"] == "model_not_found" assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" - - -def test_caller_required_401_header_labels_switchyard_on_the_wire() -> None: - """The caller_required rejection is Switchyard's own 401: same status as a - provider auth failure, distinguishable by the source header on the wire.""" - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="svc-key", # pragma: allowlist secret - base_url="http://llm.test/v1", - ), - ], - credential_policy="caller_required", - ) - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - app = build_switchyard_app(_latency_service_switchyard(config)) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": "model-A", - "messages": [{"role": "user", "content": "ping"}], - }, - ) - - assert response.status_code == 401 - assert response.json()["error"]["code"] == "missing_caller_api_key" - assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" - assert UPSTREAM_MODEL_HEADER not in response.headers - - -# --------------------------------------------------------------------------- -# Responses API fidelity on the wire -# --------------------------------------------------------------------------- - - -def test_responses_body_returned_exactly_on_the_wire() -> None: - """A Responses request through the full app returns the upstream JSON - byte-for-byte: provider extras and explicit-null fields included. - - This is the composed fidelity proof — backend raw wrap, the - terminal translation short-circuit, and endpoint serialization together. - """ - upstream_body = { - "id": "resp-e2e", - "object": "response", - "created_at": 1719890000, - "model": "model-A", - "status": "completed", - "output": [ - { - "type": "message", - "id": "msg-1", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "hello", "annotations": []}], - } - ], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - "store": False, - "temperature": 1.0, - "top_p": 0.9, - "text": {"format": {"type": "text"}}, - "reasoning": {"effort": None, "summary": None}, - "previous_response_id": None, - "provider_meta": {"azure_region": "eastus"}, - } - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="k", - base_url="http://llm.test/v1", - request_type="openai_responses", - ), - ], - ) - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard(config) - backend = _find_latency_backend(switchyard) - backend._clients["model-A"].aresponses = AsyncMock( - return_value=dict(upstream_body) - ) - app = build_switchyard_app(switchyard) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={"model": "model-A", "input": "ping"}, - ) - - assert response.status_code == 200, response.text - assert response.json() == upstream_body - - -def _latency_responses_app(frames: list[str]) -> FastAPI: - """Full app over one ``openai_responses`` endpoint streaming *frames*.""" - config = LatencyServiceBackendConfig( - latency_service_url="http://latency-service.test:8080", - endpoints=[ - LatencyServiceEndpoint( - model="model-A", - api_key="k", - base_url="http://llm.test/v1", - request_type="openai_responses", - ), - ], - ) - with patch( - "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", - ) as mock_cls: - mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") - with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): - switchyard = _latency_service_switchyard(config) - backend = _find_latency_backend(switchyard) - - async def _frame_iter() -> AsyncIterator[str]: - for frame in frames: - yield frame - - backend._clients["model-A"].aresponses = AsyncMock( - return_value=_frame_iter() - ) - return build_switchyard_app(switchyard) - - -def test_responses_stream_forwarded_verbatim_on_the_wire() -> None: - """A same-format streaming Responses request returns the upstream SSE - frames byte-for-byte: unknown provider fields, explicit nulls, event - names, and comment keep-alives all survive (streaming fidelity leg).""" - frames = [ - ( - 'event: response.created\n' - 'data: {"type":"response.created","response":{"id":"resp_s1","store":false,' - '"temperature":1.0,"reasoning":{"effort":null,"summary":null}}}\n\n' - ), - ": keep-alive\n\n", - ( - 'event: response.output_text.delta\n' - 'data: {"type":"response.output_text.delta","delta":"hello","vendor_extra":1}\n\n' - ), - ( - 'event: response.completed\n' - 'data: {"type":"response.completed","response":{"id":"resp_s1","usage":' - '{"input_tokens":3,"output_tokens":2}}}\n\n' - ), - ] - app = _latency_responses_app(frames) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={"model": "model-A", "input": "ping", "stream": True}, - ) - - assert response.status_code == 200, response.text - assert response.text == "".join(frames) - - -def test_azure_flavored_responses_stream_preserved_on_the_wire() -> None: - """Azure-shaped Responses events (content-filter annotations and other - provider extras) pass through the streaming path unmodified.""" - frames = [ - ( - 'event: response.output_text.delta\n' - 'data: {"type":"response.output_text.delta","delta":"hi",' - '"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}\n\n' - ), - ( - 'event: response.completed\n' - 'data: {"type":"response.completed","response":{"id":"resp_az",' - '"prompt_filter_results":[{"prompt_index":0}],"usage":' - '{"input_tokens":1,"output_tokens":1},"previous_response_id":null}}\n\n' - ), - ] - app = _latency_responses_app(frames) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/responses", - json={"model": "model-A", "input": "ping", "stream": True}, - ) - - assert response.status_code == 200, response.text - assert response.text == "".join(frames) - - -def test_responses_raw_frames_translate_for_chat_client_on_the_wire() -> None: - """Cross-format regression: a chat client served by a Responses endpoint - still gets translated chat chunks — raw frame strings are parsed back into - events for the translator instead of passing through.""" - frames = [ - ( - 'event: response.created\n' - 'data: {"type":"response.created","response":{"id":"resp_x","model":"m"}}\n\n' - ), - ( - 'event: response.output_text.delta\n' - 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n' - ), - ( - 'event: response.completed\n' - 'data: {"type":"response.completed","response":{"id":"resp_x","usage":' - '{"input_tokens":3,"output_tokens":2}}}\n\n' - ), - ] - app = _latency_responses_app(frames) - with TestClient(app, raise_server_exceptions=False) as client: - response = client.post( - "/v1/chat/completions", - json={ - "model": "model-A", - "messages": [{"role": "user", "content": "ping"}], - "stream": True, - }, - ) - - assert response.status_code == 200, response.text - assert "chat.completion.chunk" in response.text - assert "hello" in response.text - assert "response.output_text.delta" not in response.text