diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/PROFILES-old-vs-new.txt b/examples/gb10-dgx-spark-nvfp4-optimized/PROFILES-old-vs-new.txt new file mode 100644 index 0000000..3e1d350 --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/PROFILES-old-vs-new.txt @@ -0,0 +1,24 @@ +########## qwen27b ########## +--- OLD serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_FLASHINFER_SAMPLER=0 /home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm serve nvidia/Qwen3.6-27B-NVFP4 --host 127.0.0.1 --port 9104 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.4 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend marlin --no-enable-prefix-caching --quantization modelopt --trust-remote-code --load-format fastsafetensors --enable-chunked-prefill --no-async-scheduling --no-enable-prefix-caching --mamba-cache-mode none --language-model-only --mm-processor-cache-gb 0 --enforce-eager --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice +--- NEW serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_FLASHINFER_SAMPLER=0 /home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm serve nvidia/Qwen3.6-27B-NVFP4 --host 127.0.0.1 --port 9104 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.4 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend marlin --no-enable-prefix-caching --quantization modelopt --trust-remote-code --load-format fastsafetensors --enable-chunked-prefill --no-enable-prefix-caching --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + +########## qwen35b ########## +--- OLD serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_FLASHINFER_SAMPLER=0 /home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 --host 127.0.0.1 --port 9204 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 2048 --gpu-memory-utilization 0.4 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend marlin --no-enable-prefix-caching --quantization modelopt --trust-remote-code --load-format fastsafetensors --enable-chunked-prefill --no-async-scheduling --no-enable-prefix-caching --mamba-cache-mode none --language-model-only --mm-processor-cache-gb 0 --enforce-eager --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice +--- NEW serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_FLASHINFER_SAMPLER=0 /home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 --host 127.0.0.1 --port 9204 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.4 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend marlin --no-enable-prefix-caching --quantization modelopt --trust-remote-code --load-format fastsafetensors --enable-chunked-prefill --no-enable-prefix-caching --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --reasoning-parser qwen3 --tool-call-parser qwen3_xml --enable-auto-tool-choice + +########## gemma4-26b ########## +--- OLD serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_V2_MODEL_RUNNER=1 /home/bob/runtimes/vllm/current/.venv/bin/vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 --host 127.0.0.1 --port 9304 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 8192 --gpu-memory-utilization 0.3 --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend cutlass --no-enable-prefix-caching --trust-remote-code --enable-chunked-prefill --reasoning-parser gemma4 --enable-auto-tool-choice --tool-call-parser gemma4 --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --override-generation-config '{"max_new_tokens":null}' --kv-cache-memory-bytes 12884901888 +--- NEW serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_V2_MODEL_RUNNER=1 /home/bob/runtimes/vllm/current/.venv/bin/vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 --host 127.0.0.1 --port 9304 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.3 --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend cutlass --no-enable-prefix-caching --trust-remote-code --enable-chunked-prefill --reasoning-parser gemma4 --enable-auto-tool-choice --tool-call-parser gemma4 --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --override-generation-config '{"max_new_tokens":null}' --kv-cache-memory-bytes 12884901888 + +########## gemma4-31b ########## +--- OLD serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_V2_MODEL_RUNNER=1 /home/bob/runtimes/vllm/current/.venv/bin/vllm serve nvidia/Gemma-4-31B-IT-NVFP4 --host 127.0.0.1 --port 9304 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 8192 --gpu-memory-utilization 0.6 --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --no-enable-prefix-caching --trust-remote-code --enable-chunked-prefill --reasoning-parser gemma4 --enable-auto-tool-choice --tool-call-parser gemma4 --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --override-generation-config '{"max_new_tokens":null}' --kv-cache-memory-bytes 10737418240 +--- NEW serve (4k-reference profile) --- + CUTE_DSL_ARCH=sm_121a FLASHINFER_DISABLE_VERSION_CHECK=1 VLLM_SERVER_DEV_MODE=1 VLLM_USE_V2_MODEL_RUNNER=1 /home/bob/runtimes/vllm/current/.venv/bin/vllm serve nvidia/Gemma-4-31B-IT-NVFP4 --host 127.0.0.1 --port 9304 --max-model-len 4096 --max-num-seqs 32 --max-num-batched-tokens 16384 --gpu-memory-utilization 0.6 --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --no-enable-prefix-caching --trust-remote-code --enable-chunked-prefill --reasoning-parser gemma4 --enable-auto-tool-choice --tool-call-parser gemma4 --language-model-only --mm-processor-cache-gb 0 --default-chat-template-kwargs '{"enable_thinking":false}' --no-enable-log-requests --no-enable-flashinfer-autotune --override-generation-config '{"max_new_tokens":null}' --kv-cache-memory-bytes 10737418240 + diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/README.md b/examples/gb10-dgx-spark-nvfp4-optimized/README.md new file mode 100644 index 0000000..01ec987 --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/README.md @@ -0,0 +1,58 @@ +--- +title: GB10 / DGX Spark NVFP4 Optimized Sweep Specs +author: Bob +date: 2026-07-07 +--- + +# GB10 / DGX Spark NVFP4 Optimized Sweep Specs + +Throughput-tuned default-sweep specs for four NVFP4 models on a GB10 +(DGX Spark, aarch64, ~122 GiB unified memory), vLLM 0.24.0: + +- `qwen27b.json` — `nvidia/Qwen3.6-27B-NVFP4` (hybrid linear-attention) +- `qwen35b.json` — `nvidia/Qwen3.6-35B-A3B-NVFP4` (hybrid linear-attention MoE) +- `gemma4-26b.json` — `nvidia/Gemma-4-26B-A4B-NVFP4` +- `gemma4-31b.json` — `nvidia/Gemma-4-31B-IT-NVFP4` (dense) + +These are machine-specific: the `runner.vllm_command`, env +(`CUTE_DSL_ARCH=sm_121a`, etc.), and ports point at a particular runtime. +Edit them before use elsewhere. `PROFILES-old-vs-new.txt` records each +model's old serve command vs the tuned one. + +## What was tuned (and why) + +Measured against a public DGX Spark benchmark, the original hand-authored +specs served vLLM in a decode-hostile way. The tuned specs drop three flags +and raise one: + +- **Drop `--mamba-cache-mode none`** — the Qwen3.6 models are hybrid + linear-attention (only every 4th layer is full attention; the rest keep a + recurrent SSM state). `none` disables caching that state, so every decode + step recomputes it over the whole sequence and decode throughput falls with + context. Default (cache on) keeps it flat. +- **Drop `--enforce-eager`** — enables CUDA graphs. +- **Drop `--no-async-scheduling`** — enables scheduler/compute overlap. +- **Raise `--max-num-batched-tokens` to 16384** — large-context prefill was + step-throttled at 2048. + +Result on qwen27b: aggregate output throughput at concurrency rose ~20–30% +(4k max-throughput reference c32: 157 → 186 tok/s; decode-8k c32: 61 → 80; +decode-16k c16: 31 → 40). The Gemma specs were already free of the bad flags. + +## Notes on other levers (tested, not adopted here) + +- **MTP / speculative decoding** (`--speculative-config '{"method":"mtp", + "num_speculative_tokens":1}'`) roughly 1.6x's *single-stream* decode on the + Qwen models (they ship an MTP head), but adds overhead and memory pressure + under concurrency, so it is left off for max throughput. Enable it when + single-request latency matters more than aggregate throughput. +- **flashinfer autotune + sampler** boot fine on sm_121 but gave no aggregate + throughput gain here, so they stay disabled. + +## Usage + +```sh +localperf bench run --spec qwen27b.json \ + --run-dir runs/qwen27b --artifact runs/qwen27b.sqlite --timeout 6h +localperf artifact render runs/qwen27b.sqlite +``` diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-26b.json b/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-26b.json new file mode 100644 index 0000000..b7b09dc --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-26b.json @@ -0,0 +1,1818 @@ +{ + "version": "1", + "name": "gemma4-26b-optimized", + "description": "Default active-context single-wave sweep for Gemma 4 26B A4B NVFP4. Uses current v0.23 Gemma-compatible runtime, 12 GiB manual KV block, and prefix caching disabled.", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "engines": [ + { + "name": "vllm", + "type": "vllm-managed", + "command": "vllm", + "bench_command": "vllm" + } + ], + "runner": { + "vllm_command": "/home/bob/runtimes/vllm/current/.venv/bin/vllm", + "vllm_bench_command": "/home/bob/runtimes/vllm/current/.venv/bin/vllm", + "one_awake_profile": true, + "preboot_profiles": false, + "stop_managed_on_exit": true, + "append_timestamp_to_run": false + }, + "safety": { + "min_mem_available_gib": 24, + "poll_interval_millis": 1000, + "startup_timeout_sec": 1200, + "workload_timeout_sec": 7200, + "http_timeout_sec": 60 + }, + "warmup": { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 128, + "random_output_len": 16, + "random_range_ratio": "0", + "enabled": true, + "num_prompts": 1, + "max_concurrency": 1 + }, + "profiles": [ + { + "name": "4k-reference", + "engine": "vllm", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "host": "127.0.0.1", + "port": 9304, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 4096, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.3, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "cutlass", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "12884901888" + ], + "enable_prefix_caching": false + }, + { + "name": "8k", + "engine": "vllm", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "host": "127.0.0.1", + "port": 9308, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 8192, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.3, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "cutlass", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "12884901888" + ], + "enable_prefix_caching": false + }, + { + "name": "16k", + "engine": "vllm", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "host": "127.0.0.1", + "port": 9316, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 16384, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.25, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "cutlass", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "12884901888" + ], + "enable_prefix_caching": false + }, + { + "name": "32k", + "engine": "vllm", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "host": "127.0.0.1", + "port": 9332, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 32768, + "max_num_seqs": 16, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.22, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "cutlass", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "12884901888" + ], + "enable_prefix_caching": false + }, + { + "name": "64k", + "engine": "vllm", + "model": "nvidia/Gemma-4-26B-A4B-NVFP4", + "host": "127.0.0.1", + "port": 9364, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 65536, + "max_num_seqs": 8, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.2, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "cutlass", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "12884901888" + ], + "enable_prefix_caching": false + } + ], + "workloads": [ + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c1", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c4", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c8", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c16", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c32", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c1", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c4", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c8", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c16", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c32", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c1", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c4", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c8", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c16", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c32", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c1", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c4", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c8", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c16", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c32", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c1", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c4", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c8", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c16", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c32", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c1", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c4", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c8", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c16", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c1", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c4", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c8", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c16", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c1", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c4", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c8", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c1", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c4", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c8", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + } + ], + "output_dir": "/home/bob/localperf-runs/default-sweep-current-20260703", + "env": { + "VLLM_SERVER_DEV_MODE": "1", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "VLLM_USE_V2_MODEL_RUNNER": "1" + } +} \ No newline at end of file diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-31b.json b/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-31b.json new file mode 100644 index 0000000..af8e8fc --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/gemma4-31b.json @@ -0,0 +1,1818 @@ +{ + "version": "1", + "name": "gemma4-31b-optimized", + "description": "Default active-context single-wave sweep for Gemma 4 26B A4B NVFP4. Uses current v0.23 Gemma-compatible runtime, 12 GiB manual KV block, and prefix caching disabled.", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "engines": [ + { + "name": "vllm", + "type": "vllm-managed", + "command": "vllm", + "bench_command": "vllm" + } + ], + "runner": { + "vllm_command": "/home/bob/runtimes/vllm/current/.venv/bin/vllm", + "vllm_bench_command": "/home/bob/runtimes/vllm/current/.venv/bin/vllm", + "one_awake_profile": true, + "preboot_profiles": false, + "stop_managed_on_exit": true, + "append_timestamp_to_run": false + }, + "safety": { + "min_mem_available_gib": 24, + "poll_interval_millis": 1000, + "startup_timeout_sec": 1200, + "workload_timeout_sec": 7200, + "http_timeout_sec": 60 + }, + "warmup": { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 128, + "random_output_len": 16, + "random_range_ratio": "0", + "enabled": true, + "num_prompts": 1, + "max_concurrency": 1 + }, + "profiles": [ + { + "name": "4k-reference", + "engine": "vllm", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "host": "127.0.0.1", + "port": 9304, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 4096, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.6, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "10737418240" + ], + "enable_prefix_caching": false + }, + { + "name": "8k", + "engine": "vllm", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "host": "127.0.0.1", + "port": 9308, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 8192, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.6, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "10737418240" + ], + "enable_prefix_caching": false + }, + { + "name": "16k", + "engine": "vllm", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "host": "127.0.0.1", + "port": 9316, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 16384, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.6, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "10737418240" + ], + "enable_prefix_caching": false + }, + { + "name": "32k", + "engine": "vllm", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "host": "127.0.0.1", + "port": 9332, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 32768, + "max_num_seqs": 16, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.6, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "10737418240" + ], + "enable_prefix_caching": false + }, + { + "name": "64k", + "engine": "vllm", + "model": "nvidia/Gemma-4-31B-IT-NVFP4", + "host": "127.0.0.1", + "port": 9364, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 65536, + "max_num_seqs": 8, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.6, + "kv_cache_dtype": "fp8", + "attention_backend": "TRITON_ATTN", + "moe_backend": "", + "args": [ + "--trust-remote-code", + "--enable-chunked-prefill", + "--reasoning-parser", + "gemma4", + "--enable-auto-tool-choice", + "--tool-call-parser", + "gemma4", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--override-generation-config", + "{\"max_new_tokens\":null}" + ], + "engine_args": [ + "--kv-cache-memory-bytes", + "10737418240" + ], + "enable_prefix_caching": false + } + ], + "workloads": [ + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c1", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c4", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c8", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c16", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c32", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c1", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c4", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c8", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c16", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c32", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c1", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c4", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c8", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c16", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c32", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c1", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c4", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c8", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c16", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c32", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c1", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c4", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c8", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c16", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c32", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c1", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c4", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c8", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c16", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c1", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c4", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c8", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c16", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c1", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c4", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c8", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c1", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c4", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c8", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=gemma4-26b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + } + ], + "output_dir": "/home/bob/localperf-runs/default-sweep-current-20260703", + "env": { + "VLLM_SERVER_DEV_MODE": "1", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "VLLM_USE_V2_MODEL_RUNNER": "1" + } +} \ No newline at end of file diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/qwen27b.json b/examples/gb10-dgx-spark-nvfp4-optimized/qwen27b.json new file mode 100644 index 0000000..92bd0ea --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/qwen27b.json @@ -0,0 +1,1813 @@ +{ + "version": "1", + "name": "qwen27b-optimized", + "description": "Default active-context single-wave sweep generated from the localperf default sweep shape. One request per target user for each concurrency point. Prefix caching disabled.", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "engines": [ + { + "name": "vllm", + "type": "vllm-managed", + "command": "vllm", + "bench_command": "vllm" + } + ], + "runner": { + "vllm_command": "/home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm", + "vllm_bench_command": "/home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm", + "one_awake_profile": true, + "preboot_profiles": false, + "stop_managed_on_exit": true, + "append_timestamp_to_run": false + }, + "safety": { + "min_mem_available_gib": 24, + "poll_interval_millis": 1000, + "startup_timeout_sec": 1200, + "workload_timeout_sec": 7200, + "http_timeout_sec": 60 + }, + "warmup": { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 128, + "random_output_len": 16, + "random_range_ratio": "0", + "enabled": true, + "num_prompts": 1, + "max_concurrency": 1 + }, + "profiles": [ + { + "name": "4k-reference", + "engine": "vllm", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "host": "127.0.0.1", + "port": 9104, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 4096, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "8k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "host": "127.0.0.1", + "port": 9108, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 8192, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "16k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "host": "127.0.0.1", + "port": 9116, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 16384, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "32k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "host": "127.0.0.1", + "port": 9132, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 32768, + "max_num_seqs": 16, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "64k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-27B-NVFP4", + "host": "127.0.0.1", + "port": 9164, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 65536, + "max_num_seqs": 8, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + } + ], + "workloads": [ + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c1", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c4", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c8", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c16", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c32", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c1", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c4", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c8", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c16", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c32", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c1", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c4", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c8", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c16", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c32", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c1", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c4", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c8", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c16", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c32", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c1", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c4", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c8", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c16", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c32", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c1", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c4", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c8", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c16", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c1", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c4", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c8", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c16", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c1", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c4", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c8", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c1", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c4", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c8", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen27b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + } + ], + "output_dir": "/home/bob/localperf-runs/default-sweep-current-20260703", + "env": { + "VLLM_SERVER_DEV_MODE": "1", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "VLLM_USE_FLASHINFER_SAMPLER": "0" + } +} \ No newline at end of file diff --git a/examples/gb10-dgx-spark-nvfp4-optimized/qwen35b.json b/examples/gb10-dgx-spark-nvfp4-optimized/qwen35b.json new file mode 100644 index 0000000..fc19cc0 --- /dev/null +++ b/examples/gb10-dgx-spark-nvfp4-optimized/qwen35b.json @@ -0,0 +1,1973 @@ +{ + "version": "1", + "name": "qwen35b-optimized", + "description": "Default active-context single-wave sweep for Qwen 35B A3B NVFP4. One request per target user; practical-context profiles use a 16 GiB manual KV block. Prefix caching disabled.", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "engines": [ + { + "name": "vllm", + "type": "vllm-managed", + "command": "vllm", + "bench_command": "vllm" + } + ], + "runner": { + "vllm_command": "/home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm", + "vllm_bench_command": "/home/bob/runtimes/vllm/versions/vllm-0.24.0-qwen36-sm121/.venv/bin/vllm", + "one_awake_profile": true, + "preboot_profiles": false, + "stop_managed_on_exit": true, + "append_timestamp_to_run": false + }, + "safety": { + "min_mem_available_gib": 24, + "poll_interval_millis": 1000, + "startup_timeout_sec": 1200, + "workload_timeout_sec": 7200, + "http_timeout_sec": 60 + }, + "warmup": { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 128, + "random_output_len": 16, + "random_range_ratio": "0", + "enabled": true, + "num_prompts": 1, + "max_concurrency": 1 + }, + "profiles": [ + { + "name": "4k-reference", + "engine": "vllm", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "host": "127.0.0.1", + "port": 9204, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 4096, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "8k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "host": "127.0.0.1", + "port": 9208, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 8192, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "16k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "host": "127.0.0.1", + "port": 9216, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 16384, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "32k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "host": "127.0.0.1", + "port": 9232, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 32768, + "max_num_seqs": 32, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + }, + { + "name": "64k", + "engine": "vllm", + "model": "nvidia/Qwen3.6-35B-A3B-NVFP4", + "host": "127.0.0.1", + "port": 9264, + "managed": true, + "health_path": "/v1/models", + "serve": {}, + "max_model_len": 65536, + "max_num_seqs": 16, + "enable_sleep_mode": false, + "max_num_batched_tokens": 16384, + "gpu_memory_utilization": 0.4, + "kv_cache_dtype": "fp8", + "attention_backend": "flashinfer", + "moe_backend": "marlin", + "args": [ + "--quantization", + "modelopt", + "--trust-remote-code", + "--load-format", + "fastsafetensors", + "--enable-chunked-prefill", + "--no-enable-prefix-caching", + "--language-model-only", + "--mm-processor-cache-gb", + "0", + "--default-chat-template-kwargs", + "{\"enable_thinking\":false}", + "--no-enable-log-requests", + "--no-enable-flashinfer-autotune", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_xml", + "--enable-auto-tool-choice" + ], + "engine_args": [] + } + ], + "workloads": [ + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c1", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c4", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c8", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c16", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 1024, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "max-throughput-reference-c32", + "phase": "decode", + "context_target": 4096, + "context_semantics": "capacity", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "4k-reference" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c1", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c4", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c8", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c16", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 8063, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-8k-c32", + "phase": "prefill", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c1", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c4", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c8", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c16", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 7040, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-8k-c32", + "phase": "decode", + "context_target": 8192, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "8k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c1", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c4", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c8", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c16", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 16127, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-16k-c32", + "phase": "prefill", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c1", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c4", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c8", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c16", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 15104, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-16k-c32", + "phase": "decode", + "context_target": 16384, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "16k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c1", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c4", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c8", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c16", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 32255, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-32k-c32", + "phase": "prefill", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c1", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c4", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c8", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c16", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 31232, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-32k-c32", + "phase": "decode", + "context_target": 32768, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "32k" + ], + "num_prompts": 32, + "repeats": 1, + "max_concurrency": [ + 32 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 32, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c1", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c4", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c8", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 64511, + "random_output_len": 1, + "random_range_ratio": "0", + "name": "prefill-64k-c16", + "phase": "prefill", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c1", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 1, + "repeats": 1, + "max_concurrency": [ + 1 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 1, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c4", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 4, + "repeats": 1, + "max_concurrency": [ + 4 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 4, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c8", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 8, + "repeats": 1, + "max_concurrency": [ + 8 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 8, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + }, + { + "backend": "openai-chat", + "endpoint": "/v1/chat/completions", + "dataset_name": "random", + "request_rate": "inf", + "seed": 0, + "random_input_len": 63488, + "random_output_len": 1024, + "random_range_ratio": "0", + "name": "decode-64k-c16", + "phase": "decode", + "context_target": 65536, + "context_semantics": "active", + "load_generator": "localperf_http", + "dataset": { + "prepared": {} + }, + "request": {}, + "load": {}, + "profiles": [ + "64k" + ], + "num_prompts": 16, + "repeats": 1, + "max_concurrency": [ + 16 + ], + "traffic": {}, + "ignore_eos": true, + "temperature": 0, + "samples": 16, + "save_detailed": true, + "capture_payload_artifacts": false, + "metadata": [ + "model_slug=qwen35b", + "samples=one_wave", + "default_sweep=20260704-single-wave-noprefix", + "generated_by=localperf-default-sweep-refresh" + ] + } + ], + "output_dir": "/home/bob/localperf-runs/default-sweep-current-20260703", + "env": { + "VLLM_SERVER_DEV_MODE": "1", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_DISABLE_VERSION_CHECK": "1", + "VLLM_USE_FLASHINFER_SAMPLER": "0" + } +} \ No newline at end of file