From a11cb7ab53438bf104838bc239bb6329f81a7d20 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Mar 2026 05:02:24 +0000 Subject: [PATCH 1/7] qwen3.5 fp8 mi355x optimization Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 3 ++- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 .github/configs/amd-master.yaml diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml old mode 100644 new mode 100755 index 337047e57..6e1e22f17 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -228,7 +228,8 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 + #TODO: srok, waiting for the image + #image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index f77390707..5612b1499 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -30,7 +30,9 @@ python3 -m sglang.launch_server \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ - --mem-fraction-static 0.8 > $SERVER_LOG 2>&1 & + --mem-fraction-static 0.8 \ + --kv-cache-dtype fp8_e4m3 \ + > $SERVER_LOG 2>&1 & SERVER_PID=$! From a706a84ec7e9aa94a04393039eef8c6fa160c63c Mon Sep 17 00:00:00 2001 From: seungrokj Date: Fri, 13 Mar 2026 05:03:42 +0000 Subject: [PATCH 2/7] qwen3.5 fp8 mi355x optimization Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 6e1e22f17..ea691dc3c 100755 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -230,6 +230,7 @@ qwen3.5-fp8-mi325x-sglang: qwen3.5-fp8-mi355x-sglang: #TODO: srok, waiting for the image #image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 + image: TBD model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x From d26ce9c105bb07fcddc8397371b7962f8f14ff63 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 16 Mar 2026 09:38:15 +0000 Subject: [PATCH 3/7] qwen3.5 fp8 mi355x Signed-off-by: seungrokj --- .github/configs/amd-master.yaml | 4 +--- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index ea691dc3c..79f05c813 100755 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -228,9 +228,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - #TODO: srok, waiting for the image - #image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 - image: TBD + image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index 5612b1499..795b32611 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -19,19 +19,30 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} +CHUNK_SIZE=8192 # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor -python3 -m sglang.launch_server \ +set -x +sglang serve \ --attention-backend triton \ --model-path $MODEL \ --host=0.0.0.0 \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ - --mem-fraction-static 0.8 \ + --mem-fraction-static $MEM_FRAC_STATIC \ --kv-cache-dtype fp8_e4m3 \ + --mamba-ssm-dtype bfloat16 \ + --cuda-graph-max-bs $CONC \ + --max-running-requests $CONC \ + --chunked-prefill-size $CHUNK_SIZE \ + --max-prefill-tokens $CHUNK_SIZE \ + --disable-radix-cache \ + --num-continuous-decode-steps 2 \ + --tokenizer-worker-num $TP \ > $SERVER_LOG 2>&1 & SERVER_PID=$! From 28030dfce5200236a260b46b49dd3cabea8e0f7e Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 16 Mar 2026 09:48:54 +0000 Subject: [PATCH 4/7] qwen3.5 fp8 perf changelog Signed-off-by: seungrokj --- perf-changelog.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 403dfd7dd..8f1623dae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -947,3 +947,12 @@ - "Image: vllm/vllm-openai:v0.17.0" - "TP=2 and TP=4, concurrency 4-64 for 1k1k, 1k8k, and 8k1k sequence lengths" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/757 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang + description: + - "Improve fp8 performance + - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" + - "Relevant PR: https://github.com/sgl-project/sglang/pull/19826" + - "Image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/910 From fd5c8d69df363933789a7af87a93f7fe5a0b1ff1 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 16 Mar 2026 09:52:31 +0000 Subject: [PATCH 5/7] qwen3.5 fp8 perf changelog Signed-off-by: seungrokj --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7f2f649fe..41ad83857 100755 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -965,7 +965,7 @@ - config-keys: - qwen3.5-fp8-mi355x-sglang description: - - "Improve fp8 performance + - "Improved fp8 performance" - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" - "Relevant PR: https://github.com/sgl-project/sglang/pull/19826" - "Image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315" From dcf43371e1dcb5256078b03c6a4fa17a7fbd2c42 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 16 Mar 2026 09:53:27 +0000 Subject: [PATCH 6/7] qwen3.5 fp8 perf changelog Signed-off-by: seungrokj --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 41ad83857..c7182d4c7 100755 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -969,4 +969,4 @@ - "Relevant Issue: https://github.com/sgl-project/sglang/issues/19633" - "Relevant PR: https://github.com/sgl-project/sglang/pull/19826" - "Image: rocm/sgl-dev:v0.5.9-rocm720-mi35x-20260315" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/910 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/910 From 3335a1822cc98aea2e01c917dbdd2551a342e102 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 18 Mar 2026 02:21:49 +0000 Subject: [PATCH 7/7] prevent pythonbytecode Signed-off-by: seungrokj --- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index 795b32611..70759d5fe 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -49,7 +49,7 @@ SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - +export PYTHONDONTWRITEBYTECODE=1 run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \