From 0e5b488f2b3fbb97e32dccbd27e8ce3635d9ddea Mon Sep 17 00:00:00 2001 From: Ryan Lempka Date: Mon, 20 Jul 2026 12:55:37 -0500 Subject: [PATCH] fix(launch): tier backend formats come from the preset, not launcher overrides Signed-off-by: Ryan Lempka --- switchyard/cli/launch_command.py | 9 -- switchyard/cli/routing/route_builder.py | 12 +- .../profiles/deterministic_routing_presets.py | 3 +- tests/test_deterministic_routing_profile.py | 6 +- tests/test_launch_preset_cache_e2e.py | 121 ++++++++++++++++++ tests/test_launch_route_builder.py | 9 +- 6 files changed, 134 insertions(+), 26 deletions(-) create mode 100644 tests/test_launch_preset_cache_e2e.py diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py index b79a5164..8b33c42b 100644 --- a/switchyard/cli/launch_command.py +++ b/switchyard/cli/launch_command.py @@ -37,7 +37,6 @@ require_route_model, ) from switchyard.lib import startup_timing -from switchyard.lib.backends.llm_target import BackendFormat from switchyard.lib.profiles.random_routing import ( RandomRoutingConfig, ) @@ -612,8 +611,6 @@ def cmd_launch_claude(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, - strong_backend_format=BackendFormat.AUTO, timeout=args.timeout, ) dry_run_route = LaunchRouteConfig( @@ -668,8 +665,6 @@ def cmd_launch_claude(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, - strong_backend_format=BackendFormat.AUTO, timeout=args.timeout, ) raise SystemExit(launch_claude_deterministic_routing( @@ -811,7 +806,6 @@ def cmd_launch_codex(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, timeout=args.timeout, ) dry_run_route = LaunchRouteConfig( @@ -865,7 +859,6 @@ def cmd_launch_codex(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, timeout=args.timeout, ) raise SystemExit(launch_codex_deterministic_routing( @@ -1024,7 +1017,6 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, timeout=args.timeout, ) dry_run_route = LaunchRouteConfig( @@ -1078,7 +1070,6 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None: classifier_min_confidence=getattr( args, "classifier_min_confidence", None, ), - backend_format=BackendFormat.OPENAI, timeout=args.timeout, ) raise SystemExit(launch_openclaw_deterministic_routing( diff --git a/switchyard/cli/routing/route_builder.py b/switchyard/cli/routing/route_builder.py index 5e86d449..c36bb119 100644 --- a/switchyard/cli/routing/route_builder.py +++ b/switchyard/cli/routing/route_builder.py @@ -90,9 +90,7 @@ def build_deterministic_routing_config( classifier_model: str | None, profile_name: str | None, classifier_min_confidence: float | None, - backend_format: BackendFormat, timeout: float | None, - strong_backend_format: BackendFormat | None = None, ) -> DeterministicRoutingConfig: """Layer LLM-as-classifier CLI overrides on top of the shipping preset. @@ -116,10 +114,6 @@ def build_deterministic_routing_config( / ``coding_agent`` / ``openclaw``). classifier_min_confidence: User override for the tier selector's confidence floor. - backend_format: Wire format for the weak + classifier tiers (and the - strong tier when ``strong_backend_format`` is ``None``). - strong_backend_format: Strong-tier wire format override; ``AUTO`` - probes the backend for Anthropic Messages support at runtime. timeout: Per-request HTTP timeout for the tier backends (seconds). """ @@ -158,7 +152,7 @@ def build_deterministic_routing_config( strong_target = LlmTarget( id="strong", model=strong_model, - format=strong_backend_format or backend_format, + format=preset.strong.format, api_key=primary.api_key, base_url=primary.base_url, timeout_secs=timeout, @@ -166,7 +160,7 @@ def build_deterministic_routing_config( weak_target = LlmTarget( id="weak", model=weak_model, - format=backend_format, + format=preset.weak.format, api_key=weak.api_key, base_url=weak.base_url, timeout_secs=timeout, @@ -174,7 +168,7 @@ def build_deterministic_routing_config( classifier_target = LlmTarget( id="classifier", model=classifier_model_resolved, - format=backend_format, + format=preset.classifier.format, api_key=primary.api_key, base_url=primary.base_url, timeout_secs=preset.classifier_timeout_s, diff --git a/switchyard/lib/profiles/deterministic_routing_presets.py b/switchyard/lib/profiles/deterministic_routing_presets.py index 3e6b4624..1cdc1e5f 100644 --- a/switchyard/lib/profiles/deterministic_routing_presets.py +++ b/switchyard/lib/profiles/deterministic_routing_presets.py @@ -71,7 +71,8 @@ def coding_agent_default( strong=LlmTarget( id="strong", model=_MODEL_OPUS_4_7, - format=BackendFormat.OPENAI, + # AUTO probes for Anthropic Messages; keeps cache_control on Claude. + format=BackendFormat.AUTO, api_key=api_key, base_url=base_url, timeout_secs=timeout_secs, diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py index 331b3bb2..2c1ad026 100644 --- a/tests/test_deterministic_routing_profile.py +++ b/tests/test_deterministic_routing_profile.py @@ -458,10 +458,10 @@ def test_preset_metadata_round_trips(self) -> None: assert config.weak.model == "moonshotai/kimi-k2.6" assert config.classifier.model == "google/gemini-3.5-flash" - def test_preset_uses_openai_compatible_formats(self) -> None: - # OpenRouter's default surface is OpenAI-compatible chat completions. + def test_preset_owns_per_tier_formats(self) -> None: + # Strong is a Claude model: AUTO probes so cache_control survives. config = DeterministicRoutingPresets.coding_agent_default(api_key="nvapi-test") - assert config.strong.format is BackendFormat.OPENAI + assert config.strong.format is BackendFormat.AUTO assert config.weak.format is BackendFormat.OPENAI assert config.classifier.format is BackendFormat.OPENAI diff --git a/tests/test_launch_preset_cache_e2e.py b/tests/test_launch_preset_cache_e2e.py new file mode 100644 index 00000000..8f1e9e9f --- /dev/null +++ b/tests/test_launch_preset_cache_e2e.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Zero-flag launcher configs keep Anthropic prompt caching on the wire. + +The preset's strong tier is a Claude model with ``format=auto``. Built through +the launcher route builder and the deterministic profile, its outbound +``/v1/messages`` body must carry ``cache_control`` markers even for an +OpenAI-shaped harness; the weak tier stays OpenAI format with no markers. +""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from switchyard.cli.config.user_config import LaunchRouteConfig +from switchyard.cli.routing.route_builder import ( + LaunchTierConnectivity, + build_deterministic_routing_config, +) +from switchyard.lib.backends.anthropic_cache_breakpoint_backend import ( + AnthropicCacheBreakpointBackend, +) +from switchyard.lib.backends.deterministic_routing_llm_backend import ( + DeterministicRoutingLLMBackend, +) +from switchyard.lib.profiles.deterministic_routing_profile_config import ( + DeterministicRoutingProfileConfig, +) +from switchyard.lib.proxy_context import ProxyContext +from switchyard_rust.core import ChatRequest + +CAPTURED = [] + + +class Upstream(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("content-length", 0)))) + CAPTURED.append({"path": self.path, "body": body}) + if self.path.endswith("/v1/messages"): + resp = {"id": "m1", "type": "message", "role": "assistant", "model": body.get("model"), + "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}} + else: + resp = {"id": "c1", "object": "chat.completion", "model": body.get("model"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}} + data = json.dumps(resp).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *a): + pass + + +@pytest.fixture(scope="module") +def upstream(): + server = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + t = threading.Thread(target=server.serve_forever, daemon=True) + t.start() + yield f"http://127.0.0.1:{server.server_port}/v1" + server.shutdown() + + + + +async def test_launcher_preset_path_emits_cache_control_on_strong(upstream) -> None: + config = build_deterministic_routing_config( + LaunchRouteConfig(type="deterministic"), + primary=LaunchTierConnectivity(base_url=upstream, api_key="sk-test"), + weak=LaunchTierConnectivity(base_url=upstream, api_key="sk-test"), + classifier_model=None, + profile_name=None, + classifier_min_confidence=None, + timeout=30.0, + ) + assert config.strong.model == "anthropic/claude-opus-4.7" + assert str(config.strong.format).lower().endswith("auto") + + profile = ( + DeterministicRoutingProfileConfig.from_config(config) + .build() + .with_runtime_components(enable_stats=False) + ) + backend = next( + c for c in profile.iter_components() + if isinstance(c, DeterministicRoutingLLMBackend) + ) + assert isinstance(backend._backends["strong"], AnthropicCacheBreakpointBackend) + assert not isinstance(backend._backends["weak"], AnthropicCacheBreakpointBackend) + + CAPTURED.clear() + request = ChatRequest.openai_chat({ + "model": "inbound", + "messages": [ + {"role": "system", "content": "You are a coding agent."}, + {"role": "user", "content": "hello"}, + ], + }) + ctx = ProxyContext() + response = await backend._backends["strong"].call(ctx, request) + assert response is not None + + strong_calls = [c for c in CAPTURED if c["path"].endswith("/v1/messages")] + assert strong_calls, f"no anthropic call captured; saw {[c['path'] for c in CAPTURED]}" + body = strong_calls[-1]["body"] + flat = json.dumps(body) + assert '"cache_control"' in flat and '"ephemeral"' in flat, ( + f"cache_control missing from wire body: {flat[:400]}" + ) + + CAPTURED.clear() + await backend._backends["weak"].call(ctx, request) + weak_calls = [c for c in CAPTURED if c["path"].endswith("/v1/chat/completions")] + assert weak_calls, f"no chat call captured; saw {[c['path'] for c in CAPTURED]}" + assert "cache_control" not in json.dumps(weak_calls[-1]["body"]) diff --git a/tests/test_launch_route_builder.py b/tests/test_launch_route_builder.py index 3879d3f0..eab3f12c 100644 --- a/tests/test_launch_route_builder.py +++ b/tests/test_launch_route_builder.py @@ -125,7 +125,6 @@ def test_deterministic_launch_routes_use_preset_defaults() -> None: classifier_model=None, profile_name=None, classifier_min_confidence=None, - backend_format=BackendFormat.OPENAI, timeout=600.0, ) assert config.strong.model == "anthropic/claude-opus-4.7" @@ -134,6 +133,10 @@ def test_deterministic_launch_routes_use_preset_defaults() -> None: assert config.profile_name == "coding_agent" assert config.classifier_min_confidence == 0.0 assert config.preset == "coding_agent_default" + # Tier formats are preset-owned. + assert config.strong.format == BackendFormat.AUTO + assert config.weak.format == BackendFormat.OPENAI + assert config.classifier.format == BackendFormat.OPENAI def test_deterministic_launch_routes_apply_user_overrides() -> None: @@ -150,7 +153,6 @@ def test_deterministic_launch_routes_apply_user_overrides() -> None: classifier_model="nvidia/nvidia/nemotron-3-super-v3", profile_name="general", classifier_min_confidence=0.55, - backend_format=BackendFormat.OPENAI, timeout=600.0, ) assert config.strong.model == "openai/openai/gpt-5.2" @@ -159,6 +161,7 @@ def test_deterministic_launch_routes_apply_user_overrides() -> None: assert config.profile_name == "general" assert config.classifier_min_confidence == 0.55 assert config.preset is None + assert config.strong.format == BackendFormat.AUTO def test_deterministic_launch_routes_reject_invalid_profile() -> None: @@ -171,7 +174,6 @@ def test_deterministic_launch_routes_reject_invalid_profile() -> None: classifier_model=None, profile_name="invented_profile", classifier_min_confidence=None, - backend_format=BackendFormat.OPENAI, timeout=600.0, ) @@ -186,7 +188,6 @@ def test_deterministic_launch_routes_reject_out_of_range_confidence() -> None: classifier_model=None, profile_name=None, classifier_min_confidence=1.5, - backend_format=BackendFormat.OPENAI, timeout=600.0, )