From d1f56728da22f5c4f5e484be98c527a728f66d04 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Sat, 18 Jul 2026 21:24:02 +0200 Subject: [PATCH 1/2] feat(docs): serving anthropic messages api e2e Signed-off-by: Christopher Haar --- .gitignore | 5 +- .../examples/anthropic-messages-api.md | 100 ++++++++++++++++++ .../inference-class.yaml | 37 +++++++ .../inference-cluster.yaml | 25 +++++ .../inference-gateway.yaml | 23 ++++ .../model-deployment.yaml | 67 ++++++++++++ .../anthropic-messages-api/model-service.yaml | 16 +++ .../compose-model-replica/function/routing.py | 8 +- nix.sh | 17 ++- 9 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 docs/content/examples/anthropic-messages-api.md create mode 100644 docs/manifests/examples/anthropic-messages-api/inference-class.yaml create mode 100644 docs/manifests/examples/anthropic-messages-api/inference-cluster.yaml create mode 100644 docs/manifests/examples/anthropic-messages-api/inference-gateway.yaml create mode 100644 docs/manifests/examples/anthropic-messages-api/model-deployment.yaml create mode 100644 docs/manifests/examples/anthropic-messages-api/model-service.yaml diff --git a/.gitignore b/.gitignore index 6bf96b8b9..1759fe187 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,7 @@ docs/utils/webpack/node_modules # Local superpowers planning artifacts (specs/plans/notes) — not repo content docs/superpowers/ -examples/ \ No newline at end of file + +# Local demo manifests at the repo root (may hold real credentials) — only the +# top-level directory, not docs/manifests/examples or docs/content/examples. +/examples/ diff --git a/docs/content/examples/anthropic-messages-api.md b/docs/content/examples/anthropic-messages-api.md new file mode 100644 index 000000000..0c704218d --- /dev/null +++ b/docs/content/examples/anthropic-messages-api.md @@ -0,0 +1,100 @@ +--- +title: Anthropic Messages API +weight: 70 +description: Serve a model on the Anthropic Messages API and drive it from Claude Code. +--- + +A vLLM server registers the Anthropic Messages API at `/v1/messages` alongside +its OpenAI routes, with no extra flag. Modelplane's route matches the +`///` prefix and preserves the path below it, so the same +service URL answers both `/v1/chat/completions` and `/v1/messages`. A client that +speaks the Messages API, including Claude Code via `ANTHROPIC_BASE_URL`, talks to +the deployment directly. See +[Alternate APIs]({{< ref "/models/model-service.md" >}}) for the routing detail. + +This recipe serves Qwen3-8B on a single NVIDIA H100 on Nebius, with tool calling +on: `--enable-auto-tool-choice` and `--tool-call-parser=hermes` are what let +Claude Code's tool use work. An 8B model needs a fraction of an H100, so the GPU +has ample headroom. Apply the platform side first, then the ML side. + +## Platform + +{{< manifests "examples/anthropic-messages-api/inference-class.yaml" >}} + +{{< manifests "examples/anthropic-messages-api/inference-cluster.yaml" >}} + +## Deployment + +{{< manifests "examples/anthropic-messages-api/model-deployment.yaml" >}} + +{{< manifests "examples/anthropic-messages-api/model-service.yaml" >}} + +## Send a request + +Read the endpoint's public address from the `ModelService` status: + +```bash +ADDRESS=$(kubectl get ms qwen3-8b -n ml-team -o jsonpath='{.status.address}') +``` + +Post to `/v1/messages` in the Messages API shape. The `model` field is the +engine's `--served-model-name` (`qwen`); `max_tokens` is required: + +```bash +curl "$ADDRESS/v1/messages" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "qwen", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +`status.address` is the in-cluster gateway address, so run the request from +inside the cluster if your shell can't reach it directly: + +```bash +kubectl run -i --rm curl-test \ + --image=curlimages/curl \ + --restart=Never \ + --env="ADDRESS=$ADDRESS" \ + -- sh -c 'curl -s "$ADDRESS/v1/messages" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + -d "{\"model\":\"qwen\",\"max_tokens\":1024,\"messages\":[{\"role\":\"user\",\"content\":\"Hello!\"}]}"' +``` + +## Point Claude Code at it + +Claude Code appends `/v1/messages` to `ANTHROPIC_BASE_URL`, so point it at the +service address and map its model tiers onto the served name. vLLM doesn't check +the auth token, so any non-empty value works. Claude Code reserves 32000 output +tokens by default, which alone leaves little context room on a small model; cap +it with `CLAUDE_CODE_MAX_OUTPUT_TOKENS` so the input and output fit under the +engine's `--max-model-len` (40960 here): + +```bash +export ANTHROPIC_BASE_URL="$ADDRESS" +export ANTHROPIC_AUTH_TOKEN=dummy +export ANTHROPIC_DEFAULT_OPUS_MODEL=qwen +export ANTHROPIC_DEFAULT_SONNET_MODEL=qwen +export ANTHROPIC_DEFAULT_HAIKU_MODEL=qwen +export CLAUDE_CODE_MAX_OUTPUT_TOKENS=8192 +claude +``` + +The gateway must be reachable from wherever `claude` runs. If `$ADDRESS` is +in-cluster only, forward the Traefik gateway service to a local port: + +```bash +kubectl -n traefik-system port-forward svc/traefik 8080:80 +``` + +Then point `ANTHROPIC_BASE_URL` at the local port, keeping the service's +`//` path prefix so Claude Code's `/v1/messages` still routes: + +```bash +export ANTHROPIC_BASE_URL="http://localhost:8080/ml-team/qwen3-8b" +``` + diff --git a/docs/manifests/examples/anthropic-messages-api/inference-class.yaml b/docs/manifests/examples/anthropic-messages-api/inference-class.yaml new file mode 100644 index 000000000..1867da889 --- /dev/null +++ b/docs/manifests/examples/anthropic-messages-api/inference-class.yaml @@ -0,0 +1,37 @@ +# InferenceClass for a single-H100 Nebius shape, serving Qwen3-8B. +# +# Nebius sizes nodes by platform + preset rather than an instance type; the +# preset determines the GPU, vCPU, and memory shape of each node. The devices +# block describes what a node of this class has, DRA-style - used by the +# scheduler to match models to clusters, and to form DRA ResourceClaims for +# claim: DRA devices. An 8B model needs a fraction of an H100, so this shape has +# ample headroom; a single L40S (gpu-l40s-a) is the economical alternative. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: h100-1x +spec: + description: "Nebius gpu-h100-sxm, 1x NVIDIA H100 80GB" + provisioning: + provider: Nebius + nebius: + platform: gpu-h100-sxm + preset: 1gpu-16vcpu-200gb + diskSizeGb: 200 + driversPreset: cuda13.0 + accelerator: + type: nvidia-h100 + count: 1 + devices: + - name: gpu + claim: DRA + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 1 + attributes: + architecture: { string: Hopper } + cudaComputeCapability: { version: "9.0.0" } + capacity: + # The H100 80GB's real usable VRAM, what the NVIDIA DRA driver reports, + # not its nominal 80GB. A nodeSelector asking for >= 80Gi would never bind. + memory: { value: "81559Mi" } diff --git a/docs/manifests/examples/anthropic-messages-api/inference-cluster.yaml b/docs/manifests/examples/anthropic-messages-api/inference-cluster.yaml new file mode 100644 index 000000000..71ea88d7d --- /dev/null +++ b/docs/manifests/examples/anthropic-messages-api/inference-cluster.yaml @@ -0,0 +1,25 @@ +# An InferenceCluster backed by a Nebius mk8s cluster. Modelplane provisions the +# full mk8s cluster and installs the inference stack; only the GPU node pool - +# referencing the InferenceClass above - is declared here. It authenticates to +# the cluster with the credentials of the Nebius ClusterProviderConfig named +# default, the same identity that provisions it. +# +# Delete with foreground cascading deletion for a clean teardown, so the +# inference stack uninstalls before the cluster's API server goes away: +# kubectl delete inferencecluster nebius-eu-north --cascade=foreground +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: nebius-eu-north + labels: + modelplane.ai/region: eu-north +spec: + cluster: + source: Nebius + nebius: {} + nodePools: + - name: gpu-h100 + className: h100-1x + nodeCount: 1 + minNodeCount: 1 + maxNodeCount: 1 diff --git a/docs/manifests/examples/anthropic-messages-api/inference-gateway.yaml b/docs/manifests/examples/anthropic-messages-api/inference-gateway.yaml new file mode 100644 index 000000000..9bba907d4 --- /dev/null +++ b/docs/manifests/examples/anthropic-messages-api/inference-gateway.yaml @@ -0,0 +1,23 @@ +# The InferenceGateway creates a unified, OpenAI-compatible endpoint on the +# control plane cluster. It installs Traefik Proxy and creates a Gateway that +# routes traffic to model replicas on remote inference clusters. +# +# Create one InferenceGateway per control plane. It must be named "default". +# +# For kind or bare-metal clusters, set loadBalancer to MetalLB and configure an +# address pool. For cloud clusters with native LoadBalancer support, omit the +# loadBalancer field entirely. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceGateway +metadata: + name: default +spec: + backend: Traefik + traefik: + version: "40.2.0" + + # Remove the loadBalancer section if your cluster supports LoadBalancer + # services natively (e.g. GKE, EKS). + loadBalancer: MetalLB + metallb: + addressPool: "172.18.255.200-172.18.255.250" diff --git a/docs/manifests/examples/anthropic-messages-api/model-deployment.yaml b/docs/manifests/examples/anthropic-messages-api/model-deployment.yaml new file mode 100644 index 000000000..b24263671 --- /dev/null +++ b/docs/manifests/examples/anthropic-messages-api/model-deployment.yaml @@ -0,0 +1,67 @@ +# Qwen3-8B served on a single NVIDIA H100 (Nebius) by vLLM, exposed on the +# Anthropic Messages API. vLLM's server registers /v1/messages (and +# /v1/messages/count_tokens) alongside its OpenAI routes with no extra flag, so +# this is an ordinary serve: the Anthropic surface comes for free, and +# Modelplane preserves the path below the service prefix to reach it. +# +# The tool-calling flags are what make it usable from Claude Code, not decoration: +# +# --enable-auto-tool-choice with +# --tool-call-parser=hermes parse the model's tool calls so Claude Code's +# tool use works (qwen3_xml is for Qwen3-Coder, +# not this dense model). Qwen3's tool-use +# template ships in the tokenizer, so no +# --chat-template is needed. +# --reasoning-parser=qwen3 with +# --default-chat-template-kwargs turns thinking off. Qwen3 thinks by default, +# burying a one-line answer under a +# block and forbidding greedy decode. +# --max-model-len=40960 Qwen3-8B's native context (past it needs YaRN). +# Claude Code reserves 32000 output tokens, so a +# smaller window 500s ("max_completion_tokens +# cannot be greater than max_model_len", then +# input+output overflow). The client must also +# cap output with CLAUDE_CODE_MAX_OUTPUT_TOKENS +# so input+output fit here. H100 has KV room. +# --gpu-memory-utilization headroom, not correctness. +# +# The v0.23.0 image is >0.17.1, so vLLM handles Claude Code's attribution header +# without breaking prefix caching. No --port or --host: Modelplane's routing +# expects the engine on its default :8000 with a /health probe, and passes args +# through verbatim. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen3-8b + namespace: ml-team +spec: + replicas: 1 + template: + spec: + # No clusterSelector: the single Nebius cluster is matched on device + # capacity alone. + engines: + - name: qwen3-8b + members: + - role: Standalone + nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: | + device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai:v0.23.0 + args: + - "--model=Qwen/Qwen3-8B" + - "--served-model-name=qwen" + - "--max-model-len=40960" + - "--gpu-memory-utilization=0.92" + - "--reasoning-parser=qwen3" + - "--default-chat-template-kwargs={\"enable_thinking\": false}" + - "--enable-auto-tool-choice" + - "--tool-call-parser=hermes" diff --git a/docs/manifests/examples/anthropic-messages-api/model-service.yaml b/docs/manifests/examples/anthropic-messages-api/model-service.yaml new file mode 100644 index 000000000..130c9a2ed --- /dev/null +++ b/docs/manifests/examples/anthropic-messages-api/model-service.yaml @@ -0,0 +1,16 @@ +# Exposes the qwen3-8b deployment as a single URL. The route matches the +# /// prefix and preserves the path below it, so the engine's +# Anthropic Messages API at .../v1/messages rides the same address as its +# OpenAI-compatible .../v1/chat/completions. Read the public address from +# status.address: +# kubectl get ms qwen3-8b -n ml-team -o jsonpath='{.status.address}' +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen3-8b + namespace: ml-team +spec: + endpoints: + - selector: + matchLabels: + modelplane.ai/deployment: qwen3-8b diff --git a/functions/compose-model-replica/function/routing.py b/functions/compose-model-replica/function/routing.py index 733872692..9e01b6cff 100644 --- a/functions/compose-model-replica/function/routing.py +++ b/functions/compose-model-replica/function/routing.py @@ -57,8 +57,8 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: return meta.namespace -_EPP_IMAGE = "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0" -_SIDECAR_IMAGE = "ghcr.io/llm-d/llm-d-routing-sidecar:v0.8.0" +_EPP_IMAGE = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" +_SIDECAR_IMAGE = "ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0" # ConfigMap key and mount-path filename for each strategy's EPP config. # ConfigMap key and mount-path filename for the EPP config. One name for both @@ -109,7 +109,7 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: # silently degrades (#179). It's derived best-effort from the engine flags via # _kv_block_size() (BLOCK_SIZE_TOKENS placeholder), defaulting to vLLM's 16. _EPP_CONFIG_TEMPLATE = """\ -apiVersion: inference.networking.x-k8s.io/v1alpha1 +apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer @@ -160,7 +160,7 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str: # diverge, when the queue score breaks the tie. It's a starting default, not a # tuned one. _UNIFIED_EPP_CONFIG_TEMPLATE = """\ -apiVersion: inference.networking.x-k8s.io/v1alpha1 +apiVersion: llm-d.ai/v1alpha1 kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer diff --git a/nix.sh b/nix.sh index bfa938520..a4d90bd39 100755 --- a/nix.sh +++ b/nix.sh @@ -137,9 +137,22 @@ if [ -n "${DOCKER_CONFIG:-}" ]; then DOCKER_CONFIG_FLAGS="-v ${DOCKER_CONFIG}:/dockercfg:ro -e DOCKER_CONFIG=/dockercfg" fi +# Publish ports from the container to the host so services in the in-container +# kind cluster can be reached from the local host. The kind cluster runs in the +# container's Docker-in-Docker daemon, so its ports aren't visible on the host +# by default. Set NIX_SH_PORTS to a space-separated list of docker -p specs, +# e.g. NIX_SH_PORTS="8080:8080" to expose a `kubectl port-forward` bound to +# 0.0.0.0:8080 inside the container. +PORT_FLAGS="" +if [ -n "${NIX_SH_PORTS:-}" ]; then + for _p in ${NIX_SH_PORTS}; do + PORT_FLAGS="${PORT_FLAGS} -p ${_p}" + done +fi + # Run with --privileged for Docker-in-Docker (required for composition tests). -# shellcheck disable=SC2086 # INTERACTIVE_FLAGS and DOCKER_CONFIG_FLAGS are intentionally word-split. -docker run --rm --privileged --cgroupns=host ${INTERACTIVE_FLAGS} ${DOCKER_CONFIG_FLAGS} \ +# shellcheck disable=SC2086 # INTERACTIVE_FLAGS, DOCKER_CONFIG_FLAGS, and PORT_FLAGS are intentionally word-split. +docker run --rm --privileged --cgroupns=host ${INTERACTIVE_FLAGS} ${DOCKER_CONFIG_FLAGS} ${PORT_FLAGS} \ -v "$(pwd):/modelplane" \ -v "modelplane-nix:/nix" \ -w /modelplane \ From a8a1d13a2c736a6623798b20dfb30ec84d20ef88 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Tue, 21 Jul 2026 11:47:22 +0200 Subject: [PATCH 2/2] feat(test): add unit test for disaggregated and unified routing Signed-off-by: Christopher Haar --- .../tests/test_backends.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index 7ab3cd7d2..892b97133 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -751,6 +751,26 @@ def test_epp_config_arms_the_pd_decider(self) -> None: self.assertNotIn("nonCachedTokens: 0", cfg) self.assertNotIn("prepareDataPlugins", cfg) + def test_epp_and_sidecar_images_and_config_group_are_pinned(self) -> None: + """Lock the picker + sidecar images and the EndpointPickerConfig API group. + + Nothing else asserts these, so a wrong tag/registry path or a stale config + group passes CI and only surfaces as an EPP/sidecar crashloop at deploy. + These are deliberate literals, not routing._* constants: comparing to the + constant would be tautological (it can't catch a typo in the constant), and + a literal forces a bump to show up here and be reviewed. + """ + out = self._apply() + epp = out["epp"].spec.forProvider.manifest["spec"]["template"]["spec"]["containers"] + self.assertEqual( + next(c["image"] for c in epp if c["name"] == "epp"), + "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0", + ) + sidecar = next(c for c in self._serving_pod(out, "decode")["spec"]["containers"] if c["name"] == "pd-sidecar") + self.assertEqual(sidecar["image"], "ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0") + cfg = out["epp-config"].spec.forProvider.manifest["data"]["epp-config.yaml"] + self.assertIn("apiVersion: llm-d.ai/v1alpha1", cfg) + def test_epp_role_watches_inferenceobjectives(self) -> None: """The picker watches InferenceObjectives (GIE x-k8s.io group); the Role must allow it.""" rules = self._apply()["epp-role"].spec.forProvider.manifest["rules"] @@ -889,6 +909,21 @@ def test_epp_config_is_unified_not_disaggregated(self) -> None: self.assertNotIn("prefill", cfg) self.assertNotIn("decider", cfg) + def test_epp_image_and_config_group_are_pinned(self) -> None: + """Lock the picker image and the EndpointPickerConfig API group for the + unified path too. A deliberate literal (not routing._EPP_IMAGE) so a wrong + tag/registry or a stale config group is caught in review, not as a + deploy-time crashloop. Unified has no sidecar, so only the EPP is checked. + """ + out = self._apply() + epp = out["epp"].spec.forProvider.manifest["spec"]["template"]["spec"]["containers"] + self.assertEqual( + next(c["image"] for c in epp if c["name"] == "epp"), + "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0", + ) + cfg = out["epp-config"].spec.forProvider.manifest["data"]["epp-config.yaml"] + self.assertIn("apiVersion: llm-d.ai/v1alpha1", cfg) + def test_epp_pod_carries_config_checksum(self) -> None: """The EPP reads its config once at startup, so a config change must roll the pod. The pod template carries a sha256 of the rendered config to drive