diff --git a/.gitignore b/.gitignore index 6bf96b8b9..862b241db 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,7 @@ docs/utils/webpack/node_modules # Local superpowers planning artifacts (specs/plans/notes) — not repo content docs/superpowers/ -examples/ \ No newline at end of file + +# Local demo manifests at the repo root (may hold real credentials) — only the +# top-level directory, not docs/manifests/examples or docs/content/examples. +/examples/ \ No newline at end of file diff --git a/docs/content/examples/qwen2.5-72b.md b/docs/content/examples/qwen2.5-72b.md new file mode 100644 index 000000000..3679f51e3 --- /dev/null +++ b/docs/content/examples/qwen2.5-72b.md @@ -0,0 +1,60 @@ +--- +title: Qwen2.5-72B +weight: 35 +description: A 72B dense chat model (AWQ INT4) on a single 80 GB GPU, on AKS and Nebius. +--- + +A 72B dense chat model served from an AWQ INT4 quantization on a single 80 GB +GPU per replica: one `Standalone` engine fed by a `ModelCache`. The platform +side comes in two shapes - an A100 on AKS and an H100 on Nebius - and the ML +side is the same manifest for both. The deployment carries no +`clusterSelector` and two replicas, and each pool holds exactly one GPU, so +with both platforms applied one replica runs on each. The service then splits +traffic between the two GPUs by weight, which the last section uses to compare +them. To serve on just one platform, apply one tab and drop `replicas` to 1. + +These manifests mirror the repository's AKS and Nebius demos. Apply the +platform side first, then the ML side. + +## Platform + +{{< tabs >}} +{{< tab "AKS" >}} +{{< manifests "examples/qwen2.5-72b/inference-class-aks.yaml" >}} + +{{< manifests "examples/qwen2.5-72b/inference-cluster-aks.yaml" >}} +{{< /tab >}} +{{< tab "Nebius" >}} +{{< manifests "examples/qwen2.5-72b/inference-class-nebius.yaml" >}} + +{{< manifests "examples/qwen2.5-72b/inference-cluster-nebius.yaml" >}} +{{< /tab >}} +{{< /tabs >}} + +## Deployment + +{{< manifests "examples/qwen2.5-72b/model-cache.yaml" >}} + +{{< manifests "examples/qwen2.5-72b/model-deployment.yaml" >}} + +{{< manifests "examples/qwen2.5-72b/model-service.yaml" >}} + +## Compare the A100 and the H100 + +Replicas are fleet-wide, not per-cluster: the deployment's `replicas: 2` means +two complete serving instances, and because each pool holds a single 80 GB GPU +they land one on the A100 and one on the H100. Modelplane labels each +replica's endpoint with the cluster it runs on, so the service can split +traffic between the platforms by weight. This service pairs the deployment +label with each cluster label and gives each GPU half of the live traffic +behind the same URL: + +{{< manifests "examples/qwen2.5-72b/model-service-split.yaml" >}} + +Both GPUs now serve the same workload, so their engine metrics give a direct +performance comparison: scrape each replica's latency and throughput as in +[Collecting engine metrics]({{< ref "collecting-engine-metrics.md" >}}) and +read the two side by side. Weights are relative, so once one platform wins, +shift the 50/50 toward it - 80/20, and as far as 100/0 - without touching the +deployment. + diff --git a/docs/manifests/examples/qwen2.5-72b/inference-class-aks.yaml b/docs/manifests/examples/qwen2.5-72b/inference-class-aks.yaml new file mode 100644 index 000000000..c67c210a6 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/inference-class-aks.yaml @@ -0,0 +1,31 @@ +# InferenceClass for the A100 shape on AKS. One NVIDIA A100 80GB on a +# Standard_NC24ads_A100_v4. The GPU is declared as a DRA device: the scheduler +# matches a ModelDeployment's nodeSelector against this capacity, then DRA +# binds the physical GPU to the serving pod. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: aks-a100-1x-nc24ads +spec: + description: "AKS Standard_NC24ads_A100_v4, 1x NVIDIA A100 80GB" + provisioning: + provider: AKS + aks: + vmSize: Standard_NC24ads_A100_v4 + diskSizeGb: 200 + accelerator: + type: nvidia-a100 + count: 1 + devices: + - name: gpu + claim: DRA + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 1 + attributes: + architecture: { string: Ampere } + cudaComputeCapability: { version: "8.0.0" } + capacity: + # The H100's usable VRAM as the NVIDIA DRA driver reports it, not the + # nominal 80GB. + memory: { value: "81920Mi" } diff --git a/docs/manifests/examples/qwen2.5-72b/inference-class-nebius.yaml b/docs/manifests/examples/qwen2.5-72b/inference-class-nebius.yaml new file mode 100644 index 000000000..d0df32b0c --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/inference-class-nebius.yaml @@ -0,0 +1,34 @@ +# InferenceClass for the H100 shape on Nebius. One NVIDIA H100 80GB on the +# gpu-h100-sxm platform, with the cuda13.0 driver preset mk8s preinstalls on +# the pool's nodes. The GPU is declared as a DRA device: the scheduler matches +# a ModelDeployment's nodeSelector against this capacity, then DRA binds the +# physical GPU to the serving pod. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceClass +metadata: + name: nebius-h100-1x-sxm +spec: + description: "Nebius gpu-h100-sxm, 1x NVIDIA H100 80GB" + provisioning: + provider: Nebius + nebius: + platform: gpu-h100-sxm + preset: 1gpu-16vcpu-200gb + diskSizeGb: 200 + driversPreset: cuda13.0 + accelerator: + type: nvidia-h100 + count: 1 + devices: + - name: gpu + claim: DRA + driver: gpu.nvidia.com + deviceClassName: gpu.nvidia.com + count: 1 + attributes: + architecture: { string: Hopper } + cudaComputeCapability: { version: "9.0.0" } + capacity: + # The H100's usable VRAM as the NVIDIA DRA driver reports it, not the + # nominal 80GB. + memory: { value: "81559Mi" } diff --git a/docs/manifests/examples/qwen2.5-72b/inference-cluster-aks.yaml b/docs/manifests/examples/qwen2.5-72b/inference-cluster-aks.yaml new file mode 100644 index 000000000..7262b8955 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/inference-cluster-aks.yaml @@ -0,0 +1,20 @@ +# AKS InferenceCluster with one A100 node pool in swedencentral. No +# clusterSelector targets it; the ModelDeployment matches on device capacity +# alone, so it lands here or on any other compatible cluster in the fleet. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: aks-a100-single + labels: + modelplane.ai/region: swedencentral +spec: + cluster: + source: AKS + aks: + location: swedencentral + nodePools: + - name: gpua100 + className: aks-a100-1x-nc24ads + nodeCount: 1 + minNodeCount: 1 + maxNodeCount: 1 diff --git a/docs/manifests/examples/qwen2.5-72b/inference-cluster-nebius.yaml b/docs/manifests/examples/qwen2.5-72b/inference-cluster-nebius.yaml new file mode 100644 index 000000000..73da5e764 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/inference-cluster-nebius.yaml @@ -0,0 +1,18 @@ +# Nebius InferenceCluster with one H100 node pool that scales from zero: no +# minNodeCount, so the autoscaling floor defaults to nodeCount (0) and the GPU +# node only appears when a deployment's pod goes Pending. +apiVersion: modelplane.ai/v1alpha1 +kind: InferenceCluster +metadata: + name: nebius-h100-single + labels: + modelplane.ai/region: eu-north +spec: + cluster: + source: Nebius + nebius: {} + nodePools: + - name: gpu-h100 + className: nebius-h100-1x-sxm + nodeCount: 0 + maxNodeCount: 1 diff --git a/docs/manifests/examples/qwen2.5-72b/model-cache.yaml b/docs/manifests/examples/qwen2.5-72b/model-cache.yaml new file mode 100644 index 000000000..ef2ed9610 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/model-cache.yaml @@ -0,0 +1,12 @@ +# The cache the engine serves from, hydrated once per matched cluster. The AWQ +# quant repo is public, so no authSecret is needed. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelCache +metadata: + name: qwen-72b-awq + namespace: ml-team +spec: + source: HuggingFace + huggingFace: + repo: Qwen/Qwen2.5-72B-Instruct-AWQ + sizeGiB: 150 diff --git a/docs/manifests/examples/qwen2.5-72b/model-deployment.yaml b/docs/manifests/examples/qwen2.5-72b/model-deployment.yaml new file mode 100644 index 000000000..293ef5406 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/model-deployment.yaml @@ -0,0 +1,41 @@ +# Qwen2.5-72B Instruct (AWQ INT4) served on a single 80GB GPU per replica by +# vLLM. The model layer is cloud-agnostic: the >=70Gi CEL selector matches the +# A100 class on AKS (81920Mi) and the H100 class on Nebius (81559Mi) alike. +# Replicas are fleet-wide, not per-cluster, and each pool holds exactly one +# such GPU, so with both platforms applied the two replicas land one on the +# A100 and one on the H100. With a single platform applied, drop replicas to 1. +# +# 72B in AWQ INT4 is ~40Gi of weights, leaving room for the KV cache on an +# 80GB GPU. --max-model-len caps the context at 8192 - raise it only as far as +# the leftover VRAM allows. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: qwen-72b + namespace: ml-team +spec: + replicas: 2 + template: + spec: + modelCacheRef: + name: qwen-72b-awq + engines: + - name: qwen-72b + members: + - role: Standalone + nodeSelector: + devices: + - name: gpu + count: 1 + selectors: + - cel: | + device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("70Gi")) >= 0 + template: + spec: + containers: + - name: engine + image: vllm/vllm-openai:v0.23.0 + args: + - --model=/mnt/models + - --served-model-name=qwen-72b + - --max-model-len=8192 diff --git a/docs/manifests/examples/qwen2.5-72b/model-service-split.yaml b/docs/manifests/examples/qwen2.5-72b/model-service-split.yaml new file mode 100644 index 000000000..efba3d925 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/model-service-split.yaml @@ -0,0 +1,21 @@ +# Splits the service's traffic between the two platforms. Each entry pairs the +# deployment label with the cluster label Modelplane puts on every endpoint, and +# the weights fix each side's share of requests regardless of how many replicas +# match. 50/50 sends both GPUs the same workload; shift the ratio to favor one. +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen-72b + namespace: ml-team +spec: + endpoints: + - weight: 50 + selector: + matchLabels: + modelplane.ai/deployment: qwen-72b + modelplane.ai/cluster: aks-a100-single + - weight: 50 + selector: + matchLabels: + modelplane.ai/deployment: qwen-72b + modelplane.ai/cluster: nebius-h100-single diff --git a/docs/manifests/examples/qwen2.5-72b/model-service.yaml b/docs/manifests/examples/qwen2.5-72b/model-service.yaml new file mode 100644 index 000000000..e8de94370 --- /dev/null +++ b/docs/manifests/examples/qwen2.5-72b/model-service.yaml @@ -0,0 +1,15 @@ +# Exposes the qwen-72b deployment's endpoints as a single OpenAI-compatible +# URL. Modelplane labels each composed ModelEndpoint with the deployment name, +# so this selector reaches every replica. Read the public address from +# status.address: +# kubectl get ms qwen-72b -n ml-team -o jsonpath='{.status.address}' +apiVersion: modelplane.ai/v1alpha1 +kind: ModelService +metadata: + name: qwen-72b + namespace: ml-team +spec: + endpoints: + - selector: + matchLabels: + modelplane.ai/deployment: qwen-72b diff --git a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt index 37bcac31b..d0a8082c3 100644 --- a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt +++ b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt @@ -155,6 +155,7 @@ llms-full\.txt # ML and AI Qwen +Qwen2.5-72B Qwen3-8B Qwen3-Coder Kimi-K2 @@ -234,6 +235,7 @@ EFA BF16 FP8 INT4 +AWQ InfiniBand CEL CLI