feat(docs): Serve the Anthropic Messages API (Claude Code) end to end

.gitignore

@@ -20,4 +20,7 @@ docs/utils/webpack/node_modules
 
 # Local superpowers planning artifacts (specs/plans/notes) — not repo content
 docs/superpowers/
-examples/
+
+# Local demo manifests at the repo root (may hold real credentials) — only the
+# top-level directory, not docs/manifests/examples or docs/content/examples.
+/examples/

docs/content/examples/anthropic-messages-api.md

@@ -0,0 +1,100 @@
+---
+title: Anthropic Messages API
+weight: 70
+description: Serve a model on the Anthropic Messages API and drive it from Claude Code.
+---
+<!-- vale write-good.Passive = NO -->
+A vLLM server registers the Anthropic Messages API at `/v1/messages` alongside
+its OpenAI routes, with no extra flag. Modelplane's route matches the
+`/<namespace>/<service>/` prefix and preserves the path below it, so the same
+service URL answers both `/v1/chat/completions` and `/v1/messages`. A client that
+speaks the Messages API, including Claude Code via `ANTHROPIC_BASE_URL`, talks to
+the deployment directly. See
+[Alternate APIs]({{< ref "/models/model-service.md" >}}) for the routing detail.
+
+This recipe serves Qwen3-8B on a single NVIDIA H100 on Nebius, with tool calling
+on: `--enable-auto-tool-choice` and `--tool-call-parser=hermes` are what let
+Claude Code's tool use work. An 8B model needs a fraction of an H100, so the GPU
+has ample headroom. Apply the platform side first, then the ML side.
+
+## Platform
+
+{{< manifests "examples/anthropic-messages-api/inference-class.yaml" >}}
+
+{{< manifests "examples/anthropic-messages-api/inference-cluster.yaml" >}}
+
+## Deployment
+
+{{< manifests "examples/anthropic-messages-api/model-deployment.yaml" >}}
+
+{{< manifests "examples/anthropic-messages-api/model-service.yaml" >}}
+
+## Send a request
+
+Read the endpoint's public address from the `ModelService` status:
+
+```bash
+ADDRESS=$(kubectl get ms qwen3-8b -n ml-team -o jsonpath='{.status.address}')
+```
+
+Post to `/v1/messages` in the Messages API shape. The `model` field is the
+engine's `--served-model-name` (`qwen`); `max_tokens` is required:
+
+```bash
+curl "$ADDRESS/v1/messages" \
+  -H "Content-Type: application/json" \
+  -H "anthropic-version: 2023-06-01" \
+  -d '{
+    "model": "qwen",
+    "max_tokens": 1024,
+    "messages": [{"role": "user", "content": "Hello!"}]
+  }'
+```
+
+`status.address` is the in-cluster gateway address, so run the request from
+inside the cluster if your shell can't reach it directly:
+
+```bash
+kubectl run -i --rm curl-test \
+  --image=curlimages/curl \
+  --restart=Never \
+  --env="ADDRESS=$ADDRESS" \
+  -- sh -c 'curl -s "$ADDRESS/v1/messages" \
+  -H "Content-Type: application/json" \
+  -H "anthropic-version: 2023-06-01" \
+  -d "{\"model\":\"qwen\",\"max_tokens\":1024,\"messages\":[{\"role\":\"user\",\"content\":\"Hello!\"}]}"'
+```
+
+## Point Claude Code at it
+
+Claude Code appends `/v1/messages` to `ANTHROPIC_BASE_URL`, so point it at the
+service address and map its model tiers onto the served name. vLLM doesn't check
+the auth token, so any non-empty value works. Claude Code reserves 32000 output
+tokens by default, which alone leaves little context room on a small model; cap
+it with `CLAUDE_CODE_MAX_OUTPUT_TOKENS` so the input and output fit under the
+engine's `--max-model-len` (40960 here):
+
+```bash
+export ANTHROPIC_BASE_URL="$ADDRESS"
+export ANTHROPIC_AUTH_TOKEN=dummy
+export ANTHROPIC_DEFAULT_OPUS_MODEL=qwen
+export ANTHROPIC_DEFAULT_SONNET_MODEL=qwen
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=qwen
+export CLAUDE_CODE_MAX_OUTPUT_TOKENS=8192
+claude
+```
+
+The gateway must be reachable from wherever `claude` runs. If `$ADDRESS` is
+in-cluster only, forward the Traefik gateway service to a local port:
+
+```bash
+kubectl -n traefik-system port-forward svc/traefik 8080:80
+```
+
+Then point `ANTHROPIC_BASE_URL` at the local port, keeping the service's
+`/<namespace>/<service>` path prefix so Claude Code's `/v1/messages` still routes:
+
+```bash
+export ANTHROPIC_BASE_URL="http://localhost:8080/ml-team/qwen3-8b"
+```
+<!-- vale write-good.Passive = YES -->

docs/manifests/examples/anthropic-messages-api/inference-class.yaml

@@ -0,0 +1,37 @@
+# InferenceClass for a single-H100 Nebius shape, serving Qwen3-8B.
+#
+# Nebius sizes nodes by platform + preset rather than an instance type; the
+# preset determines the GPU, vCPU, and memory shape of each node. The devices
+# block describes what a node of this class has, DRA-style - used by the
+# scheduler to match models to clusters, and to form DRA ResourceClaims for
+# claim: DRA devices. An 8B model needs a fraction of an H100, so this shape has
+# ample headroom; a single L40S (gpu-l40s-a) is the economical alternative.
+apiVersion: modelplane.ai/v1alpha1
+kind: InferenceClass
+metadata:
+  name: h100-1x
+spec:
+  description: "Nebius gpu-h100-sxm, 1x NVIDIA H100 80GB"
+  provisioning:
+    provider: Nebius
+    nebius:
+      platform: gpu-h100-sxm
+      preset: 1gpu-16vcpu-200gb
+      diskSizeGb: 200
+      driversPreset: cuda13.0
+      accelerator:
+        type: nvidia-h100
+        count: 1
+  devices:
+  - name: gpu
+    claim: DRA
+    driver: gpu.nvidia.com
+    deviceClassName: gpu.nvidia.com
+    count: 1
+    attributes:
+      architecture: { string: Hopper }
+      cudaComputeCapability: { version: "9.0.0" }
+    capacity:
+      # The H100 80GB's real usable VRAM, what the NVIDIA DRA driver reports,
+      # not its nominal 80GB. A nodeSelector asking for >= 80Gi would never bind.
+      memory: { value: "81559Mi" }

docs/manifests/examples/anthropic-messages-api/inference-cluster.yaml

@@ -0,0 +1,25 @@
+# An InferenceCluster backed by a Nebius mk8s cluster. Modelplane provisions the
+# full mk8s cluster and installs the inference stack; only the GPU node pool -
+# referencing the InferenceClass above - is declared here. It authenticates to
+# the cluster with the credentials of the Nebius ClusterProviderConfig named
+# default, the same identity that provisions it.
+#
+# Delete with foreground cascading deletion for a clean teardown, so the
+# inference stack uninstalls before the cluster's API server goes away:
+#   kubectl delete inferencecluster nebius-eu-north --cascade=foreground
+apiVersion: modelplane.ai/v1alpha1
+kind: InferenceCluster
+metadata:
+  name: nebius-eu-north
+  labels:
+    modelplane.ai/region: eu-north
+spec:
+  cluster:
+    source: Nebius
+    nebius: {}
+  nodePools:
+  - name: gpu-h100
+    className: h100-1x
+    nodeCount: 1
+    minNodeCount: 1
+    maxNodeCount: 1

docs/manifests/examples/anthropic-messages-api/inference-gateway.yaml

@@ -0,0 +1,23 @@
+# The InferenceGateway creates a unified, OpenAI-compatible endpoint on the
+# control plane cluster. It installs Traefik Proxy and creates a Gateway that
+# routes traffic to model replicas on remote inference clusters.
+#
+# Create one InferenceGateway per control plane. It must be named "default".
+#
+# For kind or bare-metal clusters, set loadBalancer to MetalLB and configure an
+# address pool. For cloud clusters with native LoadBalancer support, omit the
+# loadBalancer field entirely.
+apiVersion: modelplane.ai/v1alpha1
+kind: InferenceGateway
+metadata:
+  name: default
+spec:
+  backend: Traefik
+  traefik:
+    version: "40.2.0"
+
+    # Remove the loadBalancer section if your cluster supports LoadBalancer
+    # services natively (e.g. GKE, EKS).
+    loadBalancer: MetalLB
+    metallb:
+      addressPool: "172.18.255.200-172.18.255.250"

docs/manifests/examples/anthropic-messages-api/model-deployment.yaml

@@ -0,0 +1,67 @@
+# Qwen3-8B served on a single NVIDIA H100 (Nebius) by vLLM, exposed on the
+# Anthropic Messages API. vLLM's server registers /v1/messages (and
+# /v1/messages/count_tokens) alongside its OpenAI routes with no extra flag, so
+# this is an ordinary serve: the Anthropic surface comes for free, and
+# Modelplane preserves the path below the service prefix to reach it.
+#
+# The tool-calling flags are what make it usable from Claude Code, not decoration:
+#
+#   --enable-auto-tool-choice with
+#   --tool-call-parser=hermes        parse the model's tool calls so Claude Code's
+#                                    tool use works (qwen3_xml is for Qwen3-Coder,
+#                                    not this dense model). Qwen3's tool-use
+#                                    template ships in the tokenizer, so no
+#                                    --chat-template is needed.
+#   --reasoning-parser=qwen3 with
+#   --default-chat-template-kwargs   turns thinking off. Qwen3 thinks by default,
+#                                    burying a one-line answer under a <think>
+#                                    block and forbidding greedy decode.
+#   --max-model-len=40960            Qwen3-8B's native context (past it needs YaRN).
+#                                    Claude Code reserves 32000 output tokens, so a
+#                                    smaller window 500s ("max_completion_tokens
+#                                    cannot be greater than max_model_len", then
+#                                    input+output overflow). The client must also
+#                                    cap output with CLAUDE_CODE_MAX_OUTPUT_TOKENS
+#                                    so input+output fit here. H100 has KV room.
+#   --gpu-memory-utilization         headroom, not correctness.
+#
+# The v0.23.0 image is >0.17.1, so vLLM handles Claude Code's attribution header
+# without breaking prefix caching. No --port or --host: Modelplane's routing
+# expects the engine on its default :8000 with a /health probe, and passes args
+# through verbatim.
+apiVersion: modelplane.ai/v1alpha1
+kind: ModelDeployment
+metadata:
+  name: qwen3-8b
+  namespace: ml-team
+spec:
+  replicas: 1
+  template:
+    spec:
+      # No clusterSelector: the single Nebius cluster is matched on device
+      # capacity alone.
+      engines:
+      - name: qwen3-8b
+        members:
+        - role: Standalone
+          nodeSelector:
+            devices:
+            - name: gpu
+              count: 1
+              selectors:
+              - cel: |
+                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
+          template:
+            spec:
+              containers:
+              - name: engine
+                image: vllm/vllm-openai:v0.23.0
+                args:
+                - "--model=Qwen/Qwen3-8B"
+                - "--served-model-name=qwen"
+                - "--max-model-len=40960"
+                - "--gpu-memory-utilization=0.92"
+                - "--reasoning-parser=qwen3"
+                - "--default-chat-template-kwargs={\"enable_thinking\": false}"
+                - "--enable-auto-tool-choice"
+                - "--tool-call-parser=hermes"

docs/manifests/examples/anthropic-messages-api/model-service.yaml

@@ -0,0 +1,16 @@
+# Exposes the qwen3-8b deployment as a single URL. The route matches the
+# /<namespace>/<service>/ prefix and preserves the path below it, so the engine's
+# Anthropic Messages API at .../v1/messages rides the same address as its
+# OpenAI-compatible .../v1/chat/completions. Read the public address from
+# status.address:
+#   kubectl get ms qwen3-8b -n ml-team -o jsonpath='{.status.address}'
+apiVersion: modelplane.ai/v1alpha1
+kind: ModelService
+metadata:
+  name: qwen3-8b
+  namespace: ml-team
+spec:
+  endpoints:
+  - selector:
+      matchLabels:
+        modelplane.ai/deployment: qwen3-8b

functions/compose-model-replica/function/routing.py

@@ -57,8 +57,8 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str:
     return meta.namespace
 
 
-_EPP_IMAGE = "ghcr.io/llm-d/llm-d-inference-scheduler:v0.8.0"
-_SIDECAR_IMAGE = "ghcr.io/llm-d/llm-d-routing-sidecar:v0.8.0"
+_EPP_IMAGE = "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0"
+_SIDECAR_IMAGE = "ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0"
 
 # ConfigMap key and mount-path filename for each strategy's EPP config.
 # ConfigMap key and mount-path filename for the EPP config. One name for both
@@ -109,7 +109,7 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str:
 # silently degrades (#179). It's derived best-effort from the engine flags via
 # _kv_block_size() (BLOCK_SIZE_TOKENS placeholder), defaulting to vLLM's 16.
 _EPP_CONFIG_TEMPLATE = """\
-apiVersion: inference.networking.x-k8s.io/v1alpha1
+apiVersion: llm-d.ai/v1alpha1
 kind: EndpointPickerConfig
 plugins:
 - type: approx-prefix-cache-producer
@@ -160,7 +160,7 @@ def _namespace(meta: metav1.ObjectMeta | None) -> str:
 # diverge, when the queue score breaks the tie. It's a starting default, not a
 # tuned one.
 _UNIFIED_EPP_CONFIG_TEMPLATE = """\
-apiVersion: inference.networking.x-k8s.io/v1alpha1
+apiVersion: llm-d.ai/v1alpha1
 kind: EndpointPickerConfig
 plugins:
 - type: approx-prefix-cache-producer

functions/compose-model-replica/tests/test_backends.py

@@ -751,6 +751,26 @@ def test_epp_config_arms_the_pd_decider(self) -> None:
         self.assertNotIn("nonCachedTokens: 0", cfg)
         self.assertNotIn("prepareDataPlugins", cfg)
 
+    def test_epp_and_sidecar_images_and_config_group_are_pinned(self) -> None:
+        """Lock the picker + sidecar images and the EndpointPickerConfig API group.
+
+        Nothing else asserts these, so a wrong tag/registry path or a stale config
+        group passes CI and only surfaces as an EPP/sidecar crashloop at deploy.
+        These are deliberate literals, not routing._* constants: comparing to the
+        constant would be tautological (it can't catch a typo in the constant), and
+        a literal forces a bump to show up here and be reviewed.
+        """
+        out = self._apply()
+        epp = out["epp"].spec.forProvider.manifest["spec"]["template"]["spec"]["containers"]
+        self.assertEqual(
+            next(c["image"] for c in epp if c["name"] == "epp"),
+            "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0",
+        )
+        sidecar = next(c for c in self._serving_pod(out, "decode")["spec"]["containers"] if c["name"] == "pd-sidecar")
+        self.assertEqual(sidecar["image"], "ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0")
+        cfg = out["epp-config"].spec.forProvider.manifest["data"]["epp-config.yaml"]
+        self.assertIn("apiVersion: llm-d.ai/v1alpha1", cfg)
+
     def test_epp_role_watches_inferenceobjectives(self) -> None:
         """The picker watches InferenceObjectives (GIE x-k8s.io group); the Role must allow it."""
         rules = self._apply()["epp-role"].spec.forProvider.manifest["rules"]
@@ -889,6 +909,21 @@ def test_epp_config_is_unified_not_disaggregated(self) -> None:
         self.assertNotIn("prefill", cfg)
         self.assertNotIn("decider", cfg)
 
+    def test_epp_image_and_config_group_are_pinned(self) -> None:
+        """Lock the picker image and the EndpointPickerConfig API group for the
+        unified path too. A deliberate literal (not routing._EPP_IMAGE) so a wrong
+        tag/registry or a stale config group is caught in review, not as a
+        deploy-time crashloop. Unified has no sidecar, so only the EPP is checked.
+        """
+        out = self._apply()
+        epp = out["epp"].spec.forProvider.manifest["spec"]["template"]["spec"]["containers"]
+        self.assertEqual(
+            next(c["image"] for c in epp if c["name"] == "epp"),
+            "ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0",
+        )
+        cfg = out["epp-config"].spec.forProvider.manifest["data"]["epp-config.yaml"]
+        self.assertIn("apiVersion: llm-d.ai/v1alpha1", cfg)
+
     def test_epp_pod_carries_config_checksum(self) -> None:
         """The EPP reads its config once at startup, so a config change must roll
         the pod. The pod template carries a sha256 of the rendered config to drive

nix.sh

@@ -137,9 +137,22 @@ if [ -n "${DOCKER_CONFIG:-}" ]; then
 	DOCKER_CONFIG_FLAGS="-v ${DOCKER_CONFIG}:/dockercfg:ro -e DOCKER_CONFIG=/dockercfg"
 fi
 
+# Publish ports from the container to the host so services in the in-container
+# kind cluster can be reached from the local host. The kind cluster runs in the
+# container's Docker-in-Docker daemon, so its ports aren't visible on the host
+# by default. Set NIX_SH_PORTS to a space-separated list of docker -p specs,
+# e.g. NIX_SH_PORTS="8080:8080" to expose a `kubectl port-forward` bound to
+# 0.0.0.0:8080 inside the container.
+PORT_FLAGS=""
+if [ -n "${NIX_SH_PORTS:-}" ]; then
+	for _p in ${NIX_SH_PORTS}; do
+		PORT_FLAGS="${PORT_FLAGS} -p ${_p}"
+	done
+fi
+
 # Run with --privileged for Docker-in-Docker (required for composition tests).
-# shellcheck disable=SC2086  # INTERACTIVE_FLAGS and DOCKER_CONFIG_FLAGS are intentionally word-split.
-docker run --rm --privileged --cgroupns=host ${INTERACTIVE_FLAGS} ${DOCKER_CONFIG_FLAGS} \
+# shellcheck disable=SC2086  # INTERACTIVE_FLAGS, DOCKER_CONFIG_FLAGS, and PORT_FLAGS are intentionally word-split.
+docker run --rm --privileged --cgroupns=host ${INTERACTIVE_FLAGS} ${DOCKER_CONFIG_FLAGS} ${PORT_FLAGS} \
 	-v "$(pwd):/modelplane" \
 	-v "modelplane-nix:/nix" \
 	-w /modelplane \