Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/getting-started.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,15 @@ jobs:
- name: Exercise the guide
# Single invocation: pytest picks up tests/getting_started/ (CLI, YAML,
# serve lifecycle) AND --markdown-docs execs the Python snippet from
# the guide. The passthrough→noop fixture lives in
# the guide. The local mock-upstream fixture lives in
# tests/getting_started/conftest.py so it only loads when collecting
# from that subtree.
run: uv run pytest tests/getting_started --markdown-docs docs/getting_started.md -v
env:
# The server-lifecycle test interpolates ${OPENAI_API_KEY} into a
# noop-backed routes.yaml; the value is never sent upstream. Stub
# so the bundle loader's env-var resolution doesn't fail on a
# clean runner.
# The server-lifecycle test serves a type: model route pointed at a
# local in-process mock OpenAI upstream; these keys are never sent
# upstream. Stub so the bundle loader's env-var resolution doesn't
# fail on a clean runner.
OPENAI_API_KEY: sk-test
NVIDIA_API_KEY: nvapi-test
ANTHROPIC_API_KEY: sk-ant-test
102 changes: 92 additions & 10 deletions .github/workflows/perf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@ permissions:

jobs:
proxy-perf:
name: Proxy overhead (No-Op backend)
name: Proxy overhead (local stub backend)
runs-on: ubuntu-latest

env:
PROXY_PORT: 4000
PERF_MODEL: noop
STUB_PORT: 9000
# A local zero-latency stub replaces the removed noop route; the proxy
# forwards to it over loopback, which adds a small constant overhead.
PERF_MODEL: mock-model
# gpt2 tokenizer is ~500 KB and downloads in < 2 s on Actions runners.
# It is only used for dataset-manager token counting; actual inference
# token counts come from the server (--use-server-token-count).
Expand All @@ -52,14 +55,91 @@ jobs:
run: uv pip install aiperf

# ------------------------------------------------------------------
# Start the proxy with the No-Op backend
# Start a local zero-latency mock OpenAI upstream. This replaces the
# removed noop route: the proxy serves a real `type: model` chain that
# forwards to this loopback stub, which returns a fixed completion
# instantly. The extra loopback hop adds a small constant overhead.
# ------------------------------------------------------------------
- name: Start switchyard noop proxy
- name: Start local mock OpenAI upstream
run: |
cat > bench.yaml <<'YAML'
cat > perf_stub.py <<'PY'
"""Zero-latency OpenAI-compatible chat.completions stub (loopback only)."""
import json
import os
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

_COMPLETION = {
"id": "chatcmpl-perf-stub",
"object": "chat.completion",
"created": 1700000000,
"model": "mock-model",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "4"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
}

_CHUNK = {
"id": "chatcmpl-perf-stub",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "mock-model",
"choices": [
{"index": 0, "delta": {"content": "4"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
}


class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"

def do_POST(self):
length = int(self.headers.get("content-length", "0"))
body = json.loads(self.rfile.read(length) or b"{}")
if body.get("stream"):
payload = (
f"data: {json.dumps(_CHUNK)}\n\n".encode()
+ b"data: [DONE]\n\n"
)
content_type = "text/event-stream"
else:
payload = json.dumps(_COMPLETION).encode()
content_type = "application/json"
self.send_response(200)
self.send_header("content-type", content_type)
self.send_header("content-length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)

def log_message(self, *_args):
return None


port = int(os.environ["STUB_PORT"])
ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()
PY
python3 perf_stub.py &
echo "STUB_PID=$!" >> "$GITHUB_ENV"

# ------------------------------------------------------------------
# Start the proxy: a `type: model` route pointed at the local stub.
# ------------------------------------------------------------------
- name: Start switchyard proxy
run: |
cat > bench.yaml <<YAML
defaults:
api_key: dummy
base_url: http://localhost:$STUB_PORT/v1
format: openai
routes:
noop:
type: noop
mock-model:
type: model
model: mock-model
YAML
uv run switchyard --routing-profiles bench.yaml -- serve --port $PROXY_PORT &
echo "PROXY_PID=$!" >> "$GITHUB_ENV"
Expand Down Expand Up @@ -128,11 +208,13 @@ jobs:
--output-artifact-dir perf-results/streaming

# ------------------------------------------------------------------
# Stop proxy
# Stop proxy and local stub
# ------------------------------------------------------------------
- name: Stop proxy
- name: Stop proxy and stub
if: always()
run: kill "$PROXY_PID" || true
run: |
kill "$PROXY_PID" || true
kill "$STUB_PID" || true

# ------------------------------------------------------------------
# Upload results as a build artifact for comparison over time
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/readme.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ jobs:
- name: Exercise the README
# Single invocation: pytest picks up tests/readme/ (YAML schema + CLI
# checks) AND --markdown-docs execs the Python snippet from the README.
# The passthrough→noop fixture lives in tests/readme/conftest.py so it
# only loads when collecting from that subtree.
# The local mock-upstream fixture lives in tests/readme/conftest.py so
# it only loads when collecting from that subtree.
run: uv run pytest tests/readme --markdown-docs README.md -v
env:
# The YAML examples reference ${NVIDIA_API_KEY} etc.; stub so the
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
KV-cache-aware routing with request failover) or an external load balancer
such as [Traefik](https://doc.traefik.io/traefik/reference/routing-configuration/http/load-balancing/service/)
or HAProxy.
- **Public `type: noop` and `type: passthrough` route types** — removed from
route bundles. Use `type: model` to register a single explicit model target.
Catalog auto-discovery via a bare `type: passthrough` route is gone; there is
no `type: model` equivalent, so list the model ids you want as explicit
`type: model` routes.

## [0.1.0] — Initial release

Expand Down
2 changes: 1 addition & 1 deletion docs/cli_reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ on the launchers; `serve` uses `--routing-profiles`:
| Flag | Purpose |
|---|---|
| `--model ID` | Single-model passthrough. Every request is rewritten to `model=ID` and forwarded to `--base-url`. |
| `--routing-profiles PATH` | Path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `passthrough`, `random_routing`, `stage_router`, `escalation_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. |
| `--routing-profiles PATH` | Path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `random_routing`, `stage_router`, `escalation_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. |

On the launchers, the two flags are mutually exclusive: pass one or the other, not both.

Expand Down
5 changes: 3 additions & 2 deletions docs/core_concepts.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ has a complete runnable bundle.

Every route name is registered as a model ID and listed on `GET /v1/models`.
Clients select a route by putting that name in the request's `model` field.
Some route types also discover or register direct upstream model IDs.
Some route types also discover a tier's upstream catalog from its
`GET /v1/models` and register those model IDs directly.

## Tiers and routing strategies

Expand All @@ -46,7 +47,7 @@ is more capable and more expensive, and a weak target that is cheaper and
faster. A tier is a role assigned inside a route, not a fixed property of a
model.

A route's `type` sets the strategy. `model` and `passthrough` call one target.
A route's `type` sets the strategy. `model` calls one target.
`random_routing` splits traffic on a fixed probability. `deterministic` asks a
classifier model to pick a tier. `stage_router` uses agent-progress signals,
with an optional classifier fallback. `escalation_router` starts on the weak
Expand Down
2 changes: 1 addition & 1 deletion docs/operations/context_window.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ format.

Any multi-target route (stage-router, random_routing, or deterministic) supports
this. Set `fallback_target_on_evict` on the route. Single-target routes
(`type: passthrough`, `type: model`) have no alternative target, so the original
(`type: model`) have no alternative target, so the original
overflow propagates unchanged.

## Configuration
Expand Down
11 changes: 5 additions & 6 deletions docs/routing_algorithms/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,13 @@ The examples use model IDs from the
[OpenRouter model catalog](https://openrouter.ai/api/v1/models). Select IDs
available to your account before deploying; catalog availability can change.

## Model and passthrough routes
## Model routes

- `type: model` registers one explicit model alias without model discovery.
- `type: passthrough` queries the upstream model catalog and registers the
discovered models.

Both create direct, single-target chains. Use a routing policy when requests
must be split or classified across targets.
This creates a direct, single-target chain. Use a routing policy when requests
must be split or classified across targets. There is no catalog auto-discovery,
so to expose several upstream models, add one `type: model` route per model id.

## Self-hosted targets

Expand Down Expand Up @@ -133,7 +132,7 @@ Claude Code lineage signals in either direction: `false` keeps the request on
normal routing even when delegated-work headers are present, and `true` marks a
request as a sub-agent even when no harness headers appear.

The key applies to `model`, `passthrough`, `deterministic`, `escalation_router`,
The key applies to `model`, `deterministic`, `escalation_router`,
and `stage_router` routes. `random_routing` expands into its table entries on a
separate path and does not consume it.

Expand Down
2 changes: 1 addition & 1 deletion docs/routing_algorithms/stage_router_routing.md
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ curl -s http://localhost:4000/v1/stats > routing_stats_final.json

## When *not* to use stage-router

- **Single-model deployments.** Use a `model` or `passthrough` route instead.
- **Single-model deployments.** Use a `model` route instead.
- **Probabilistic A/B splits.** Use
[Random Routing](random_routing.md) (`type: random_routing`).
The stage-router's signals are wasted on a fixed traffic ratio.
Expand Down
6 changes: 0 additions & 6 deletions examples/route.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,13 @@ routes:
type: model
target: moonshotai/kimi-k2.6

openrouter-catalog: # `type: passthrough` — auto-discovers available models via GET /v1/models
type: passthrough

random-router: # `type: random_routing` — weighted coin
type: random_routing
strong: anthropic/claude-opus-4.7
weak: moonshotai/kimi-k2.6
strong_probability: 0.3
fallback_target_on_evict: strong

bench: # `type: noop` — for benchmarking
type: noop

llm-classifier: # `type: deterministic` — LLM-as-classifier
type: deterministic
profile: coding_agent # general | coding_agent | openclaw
Expand Down
4 changes: 0 additions & 4 deletions switchyard/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,6 @@
DeterministicRoutingProfileConfig,
EscalationRouterConfig,
EscalationRouterProfileConfig,
NoopProfile,
NoopProfileConfig,
PassthroughProfileConfig,
Profile,
ProfileConfig,
Expand Down Expand Up @@ -138,8 +136,6 @@ def __getattr__(name: str) -> Any:
# Chain infrastructure
"Switchyard",
"LLMBackend",
"NoopProfile",
"NoopProfileConfig",
"StageRouterConfig",
"StageRouterProfileConfig",
"ClassifierConfig",
Expand Down
2 changes: 1 addition & 1 deletion switchyard/cli/launchers/launcher_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ def _classifier_part(r: _Mapping) -> str: # type: ignore[type-arg]
if route_type == "random_routing":
p = r.get("strong_probability", "")
return f"random-routing: strong={_model(r.get('strong'))}, weak={_model(r.get('weak'))}, p_strong={p}"
if route_type in ("model", "passthrough"):
if route_type == "model":
target = r.get("model") or r.get("target") or route_key
return f"passthrough → {target}"
return f"{route_type}: {route_key}"
Expand Down
Loading
Loading