diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml old mode 100644 new mode 100755 index 5551860f2..e4e82c16d --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -228,7 +228,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 + image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index f77390707..70759d5fe 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -19,24 +19,37 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} +CHUNK_SIZE=8192 # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor -python3 -m sglang.launch_server \ +set -x +sglang serve \ --attention-backend triton \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ - --mem-fraction-static 0.8 > $SERVER_LOG 2>&1 & + --mem-fraction-static $MEM_FRAC_STATIC \ + --kv-cache-dtype fp8_e4m3 \ + --mamba-ssm-dtype bfloat16 \ + --cuda-graph-max-bs $CONC \ + --max-running-requests $CONC \ + --chunked-prefill-size $CHUNK_SIZE \ + --max-prefill-tokens $CHUNK_SIZE \ + --disable-radix-cache \ + --num-continuous-decode-steps 2 \ + --tokenizer-worker-num $TP \ + > $SERVER_LOG 2>&1 & SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - +export PYTHONDONTWRITEBYTECODE=1 run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml old mode 100644 new mode 100755 index 2090a1b25..c7182d4c7 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -961,3 +961,12 @@ description: - "Extend MiniMax M2.5 FP8 single-node config for H200 with vLLM v0.16.0 (TP8)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/869 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + description: + - "Improved fp8 performance" + - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" + - "Relevant PR: https://github.com/sgl-project/sglang/pull/19826" + - "Image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/910