Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions switchyard/cli/launch_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
require_route_model,
)
from switchyard.lib import startup_timing
from switchyard.lib.backends.llm_target import BackendFormat
from switchyard.lib.profiles.random_routing import (
RandomRoutingConfig,
)
Expand Down Expand Up @@ -612,8 +611,6 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
strong_backend_format=BackendFormat.AUTO,
timeout=args.timeout,
)
dry_run_route = LaunchRouteConfig(
Expand Down Expand Up @@ -668,8 +665,6 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
strong_backend_format=BackendFormat.AUTO,
timeout=args.timeout,
)
raise SystemExit(launch_claude_deterministic_routing(
Expand Down Expand Up @@ -811,7 +806,6 @@ def cmd_launch_codex(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
timeout=args.timeout,
)
dry_run_route = LaunchRouteConfig(
Expand Down Expand Up @@ -865,7 +859,6 @@ def cmd_launch_codex(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
timeout=args.timeout,
)
raise SystemExit(launch_codex_deterministic_routing(
Expand Down Expand Up @@ -1024,7 +1017,6 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
timeout=args.timeout,
)
dry_run_route = LaunchRouteConfig(
Expand Down Expand Up @@ -1078,7 +1070,6 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None:
classifier_min_confidence=getattr(
args, "classifier_min_confidence", None,
),
backend_format=BackendFormat.OPENAI,
timeout=args.timeout,
)
raise SystemExit(launch_openclaw_deterministic_routing(
Expand Down
12 changes: 3 additions & 9 deletions switchyard/cli/routing/route_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,7 @@ def build_deterministic_routing_config(
classifier_model: str | None,
profile_name: str | None,
classifier_min_confidence: float | None,
backend_format: BackendFormat,
timeout: float | None,
strong_backend_format: BackendFormat | None = None,
) -> DeterministicRoutingConfig:
"""Layer LLM-as-classifier CLI overrides on top of the shipping preset.

Expand All @@ -116,10 +114,6 @@ def build_deterministic_routing_config(
/ ``coding_agent`` / ``openclaw``).
classifier_min_confidence: User override for the tier selector's
confidence floor.
backend_format: Wire format for the weak + classifier tiers (and the
strong tier when ``strong_backend_format`` is ``None``).
strong_backend_format: Strong-tier wire format override; ``AUTO``
probes the backend for Anthropic Messages support at runtime.
timeout: Per-request HTTP timeout for the tier backends
(seconds).
"""
Expand Down Expand Up @@ -158,23 +152,23 @@ def build_deterministic_routing_config(
strong_target = LlmTarget(
id="strong",
model=strong_model,
format=strong_backend_format or backend_format,
format=preset.strong.format,
Comment thread
ryan-lempka marked this conversation as resolved.
api_key=primary.api_key,
base_url=primary.base_url,
timeout_secs=timeout,
)
weak_target = LlmTarget(
id="weak",
model=weak_model,
format=backend_format,
format=preset.weak.format,
api_key=weak.api_key,
base_url=weak.base_url,
timeout_secs=timeout,
)
classifier_target = LlmTarget(
id="classifier",
model=classifier_model_resolved,
format=backend_format,
format=preset.classifier.format,
api_key=primary.api_key,
base_url=primary.base_url,
timeout_secs=preset.classifier_timeout_s,
Expand Down
3 changes: 2 additions & 1 deletion switchyard/lib/profiles/deterministic_routing_presets.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ def coding_agent_default(
strong=LlmTarget(
id="strong",
model=_MODEL_OPUS_4_7,
format=BackendFormat.OPENAI,
# AUTO probes for Anthropic Messages; keeps cache_control on Claude.
format=BackendFormat.AUTO,
api_key=api_key,
base_url=base_url,
timeout_secs=timeout_secs,
Expand Down
6 changes: 3 additions & 3 deletions tests/test_deterministic_routing_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,10 +458,10 @@ def test_preset_metadata_round_trips(self) -> None:
assert config.weak.model == "moonshotai/kimi-k2.6"
assert config.classifier.model == "google/gemini-3.5-flash"

def test_preset_uses_openai_compatible_formats(self) -> None:
# OpenRouter's default surface is OpenAI-compatible chat completions.
def test_preset_owns_per_tier_formats(self) -> None:
# Strong is a Claude model: AUTO probes so cache_control survives.
config = DeterministicRoutingPresets.coding_agent_default(api_key="nvapi-test")
assert config.strong.format is BackendFormat.OPENAI
assert config.strong.format is BackendFormat.AUTO
assert config.weak.format is BackendFormat.OPENAI
assert config.classifier.format is BackendFormat.OPENAI

Expand Down
121 changes: 121 additions & 0 deletions tests/test_launch_preset_cache_e2e.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Zero-flag launcher configs keep Anthropic prompt caching on the wire.

The preset's strong tier is a Claude model with ``format=auto``. Built through
the launcher route builder and the deterministic profile, its outbound
``/v1/messages`` body must carry ``cache_control`` markers even for an
OpenAI-shaped harness; the weak tier stays OpenAI format with no markers.
"""
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

import pytest

from switchyard.cli.config.user_config import LaunchRouteConfig
from switchyard.cli.routing.route_builder import (
LaunchTierConnectivity,
build_deterministic_routing_config,
)
from switchyard.lib.backends.anthropic_cache_breakpoint_backend import (
AnthropicCacheBreakpointBackend,
)
from switchyard.lib.backends.deterministic_routing_llm_backend import (
DeterministicRoutingLLMBackend,
)
from switchyard.lib.profiles.deterministic_routing_profile_config import (
DeterministicRoutingProfileConfig,
)
from switchyard.lib.proxy_context import ProxyContext
from switchyard_rust.core import ChatRequest

CAPTURED = []


class Upstream(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers.get("content-length", 0))))
CAPTURED.append({"path": self.path, "body": body})
if self.path.endswith("/v1/messages"):
resp = {"id": "m1", "type": "message", "role": "assistant", "model": body.get("model"),
"content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1}}
else:
resp = {"id": "c1", "object": "chat.completion", "model": body.get("model"),
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
data = json.dumps(resp).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(data)))
self.end_headers()
self.wfile.write(data)

def log_message(self, *a):
pass


@pytest.fixture(scope="module")
def upstream():
server = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
t = threading.Thread(target=server.serve_forever, daemon=True)
t.start()
yield f"http://127.0.0.1:{server.server_port}/v1"
server.shutdown()




async def test_launcher_preset_path_emits_cache_control_on_strong(upstream) -> None:
config = build_deterministic_routing_config(
LaunchRouteConfig(type="deterministic"),
primary=LaunchTierConnectivity(base_url=upstream, api_key="sk-test"),
weak=LaunchTierConnectivity(base_url=upstream, api_key="sk-test"),
classifier_model=None,
profile_name=None,
classifier_min_confidence=None,
timeout=30.0,
)
assert config.strong.model == "anthropic/claude-opus-4.7"
assert str(config.strong.format).lower().endswith("auto")

profile = (
DeterministicRoutingProfileConfig.from_config(config)
.build()
.with_runtime_components(enable_stats=False)
)
backend = next(
c for c in profile.iter_components()
if isinstance(c, DeterministicRoutingLLMBackend)
)
assert isinstance(backend._backends["strong"], AnthropicCacheBreakpointBackend)
assert not isinstance(backend._backends["weak"], AnthropicCacheBreakpointBackend)

CAPTURED.clear()
request = ChatRequest.openai_chat({
"model": "inbound",
"messages": [
{"role": "system", "content": "You are a coding agent."},
{"role": "user", "content": "hello"},
],
})
ctx = ProxyContext()
response = await backend._backends["strong"].call(ctx, request)
Comment thread
ryan-lempka marked this conversation as resolved.
assert response is not None

strong_calls = [c for c in CAPTURED if c["path"].endswith("/v1/messages")]
assert strong_calls, f"no anthropic call captured; saw {[c['path'] for c in CAPTURED]}"
body = strong_calls[-1]["body"]
flat = json.dumps(body)
assert '"cache_control"' in flat and '"ephemeral"' in flat, (
f"cache_control missing from wire body: {flat[:400]}"
)

CAPTURED.clear()
await backend._backends["weak"].call(ctx, request)
weak_calls = [c for c in CAPTURED if c["path"].endswith("/v1/chat/completions")]
assert weak_calls, f"no chat call captured; saw {[c['path'] for c in CAPTURED]}"
assert "cache_control" not in json.dumps(weak_calls[-1]["body"])
9 changes: 5 additions & 4 deletions tests/test_launch_route_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,6 @@ def test_deterministic_launch_routes_use_preset_defaults() -> None:
classifier_model=None,
profile_name=None,
classifier_min_confidence=None,
backend_format=BackendFormat.OPENAI,
timeout=600.0,
)
assert config.strong.model == "anthropic/claude-opus-4.7"
Expand All @@ -134,6 +133,10 @@ def test_deterministic_launch_routes_use_preset_defaults() -> None:
assert config.profile_name == "coding_agent"
assert config.classifier_min_confidence == 0.0
assert config.preset == "coding_agent_default"
# Tier formats are preset-owned.
assert config.strong.format == BackendFormat.AUTO
assert config.weak.format == BackendFormat.OPENAI
assert config.classifier.format == BackendFormat.OPENAI


def test_deterministic_launch_routes_apply_user_overrides() -> None:
Expand All @@ -150,7 +153,6 @@ def test_deterministic_launch_routes_apply_user_overrides() -> None:
classifier_model="nvidia/nvidia/nemotron-3-super-v3",
profile_name="general",
classifier_min_confidence=0.55,
backend_format=BackendFormat.OPENAI,
timeout=600.0,
)
assert config.strong.model == "openai/openai/gpt-5.2"
Expand All @@ -159,6 +161,7 @@ def test_deterministic_launch_routes_apply_user_overrides() -> None:
assert config.profile_name == "general"
assert config.classifier_min_confidence == 0.55
assert config.preset is None
assert config.strong.format == BackendFormat.AUTO


def test_deterministic_launch_routes_reject_invalid_profile() -> None:
Expand All @@ -171,7 +174,6 @@ def test_deterministic_launch_routes_reject_invalid_profile() -> None:
classifier_model=None,
profile_name="invented_profile",
classifier_min_confidence=None,
backend_format=BackendFormat.OPENAI,
timeout=600.0,
)

Expand All @@ -186,7 +188,6 @@ def test_deterministic_launch_routes_reject_out_of_range_confidence() -> None:
classifier_model=None,
profile_name=None,
classifier_min_confidence=1.5,
backend_format=BackendFormat.OPENAI,
timeout=600.0,
)

Expand Down
Loading