From a635a75f8465c596d2079e93c4056e58973949cb Mon Sep 17 00:00:00 2001 From: shifengmin Date: Tue, 7 Jul 2026 16:06:02 +0800 Subject: [PATCH] docs: add prefix-cache-affinity DP rank routing documents. --- src/content/docs/en/cli_reference.md | 3 +++ src/content/docs/en/features/prefix_cache.md | 12 ++++++++++++ src/content/docs/zh/cli_reference.md | 3 +++ src/content/docs/zh/features/prefix_cache.md | 12 ++++++++++++ 4 files changed, 30 insertions(+) diff --git a/src/content/docs/en/cli_reference.md b/src/content/docs/en/cli_reference.md index 93577f6..1d450c1 100644 --- a/src/content/docs/en/cli_reference.md +++ b/src/content/docs/en/cli_reference.md @@ -66,6 +66,9 @@ xLLM uses gflags to manage service startup parameters. `--model ` is the o | `max_memory_utilization` | `double` | `0.8` | Fraction of GPU memory used for model inference, including model weights and KV Cache. | | `kv_cache_dtype` | `string` | `"auto"` | KV Cache dtype for quantization. `auto` aligns with model dtype and disables quantization. `int8` enables INT8 quantization and is only supported on the MLU backend. | | `enable_prefix_cache` | `bool` | `true` | Whether to enable prefix cache in the block manager. See [Prefix Cache](/en/features/prefix_cache/). | +| `enable_prefix_cache_aware_dp_routing` | `bool` | `false` | Whether to enable prefix-cache-aware DP rank routing. Requires `enable_prefix_cache=true` and `dp_size>1`. | +| `prefix_cache_aware_dp_match_threshold` | `double` | `0.5` | Minimum prefix block hit ratio for cache-aware DP affinity routing. Below this threshold, routing falls back to free-block balancing. | +| `prefix_cache_aware_dp_imbalance_threshold` | `double` | `0.1` | Maximum cross-rank KV utilization gap before cache-aware affinity routing is disabled. Computed as `(max_used-min_used)/total_blocks`. | | `xxh3_128bits_seed` | `uint32` | `1024` | Default XXH3 128-bit hash seed. | | `enable_xtensor` | `bool` | `false` | Whether to enable XTensor for model weights with the physical page pool. | | `phy_page_granularity_size` | `int64` | `2097152` | Granularity size of one physical page in bytes, default 2 MiB, for continuous KV Cache. | diff --git a/src/content/docs/en/features/prefix_cache.md b/src/content/docs/en/features/prefix_cache.md index 88c98f3..83e6601 100644 --- a/src/content/docs/en/features/prefix_cache.md +++ b/src/content/docs/en/features/prefix_cache.md @@ -15,6 +15,18 @@ The prefix cache is implemented in xLLM and exposed through gflags parameters to --enable_prefix_cache=true ``` +## Cache-Aware DP Routing + +When running with data parallelism (`dp_size > 1`), requests can be routed to the DP rank that holds the longest prefix-cache hit, improving KV cache reuse across ranks. + +- Enable cache-aware DP routing: +``` +--enable_prefix_cache=true --enable_prefix_cache_aware_dp_routing=true +``` + +- `prefix_cache_aware_dp_match_threshold` (default `0.5`): minimum prefix block hit ratio to prefer the cache-affinity rank. Below this threshold, routing falls back to free-block balancing. +- `prefix_cache_aware_dp_imbalance_threshold` (default `0.1`): maximum cross-rank KV utilization gap (`(max_used-min_used)/total_blocks`). Exceeding this disables affinity routing and selects the least-loaded rank. + ## Performance Impact After enabling prefix cache, on the Qwen3-8B model with a TPOT constraint of 50ms, the E2E latency **decreased by 10%**. diff --git a/src/content/docs/zh/cli_reference.md b/src/content/docs/zh/cli_reference.md index 15d5657..2d1b271 100644 --- a/src/content/docs/zh/cli_reference.md +++ b/src/content/docs/zh/cli_reference.md @@ -66,6 +66,9 @@ xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填 | `max_memory_utilization` | `double` | `0.8` | 模型推理可使用的 GPU 显存比例，包括模型权重和 KV Cache。 | | `kv_cache_dtype` | `string` | `"auto"` | KV Cache 量化数据类型；`auto` 表示与模型 dtype 对齐且不量化，`int8` 表示启用 INT8 量化，仅 MLU 后端支持。 | | `enable_prefix_cache` | `bool` | `true` | 是否在 block manager 中启用 prefix cache；详见 [Prefix Cache](/zh/features/prefix_cache/)。 | +| `enable_prefix_cache_aware_dp_routing` | `bool` | `false` | 是否启用基于 prefix cache 亲和性的 DP rank 路由。需要 `enable_prefix_cache=true` 且 `dp_size>1`。 | +| `prefix_cache_aware_dp_match_threshold` | `double` | `0.5` | cache 感知路由的最低 prefix block 命中比例；低于该阈值时回退到空闲 block 均衡。 | +| `prefix_cache_aware_dp_imbalance_threshold` | `double` | `0.1` | 跨 rank KV 利用率差异上限；计算方式为 `(max_used-min_used)/total_blocks`，超过此值时路由到负载最低的 rank。 | | `xxh3_128bits_seed` | `uint32` | `1024` | XXH3 128-bit 哈希的默认 seed。 | | `enable_xtensor` | `bool` | `false` | 是否为模型权重启用基于物理页池的 XTensor。 | | `phy_page_granularity_size` | `int64` | `2097152` | 单个物理页的粒度大小，单位 byte，默认 2 MiB；用于连续 KV Cache。 | diff --git a/src/content/docs/zh/features/prefix_cache.md b/src/content/docs/zh/features/prefix_cache.md index 5fb28d4..1767252 100644 --- a/src/content/docs/zh/features/prefix_cache.md +++ b/src/content/docs/zh/features/prefix_cache.md @@ -16,6 +16,18 @@ prefix_cache已在xLLM实现，并向外暴露gflag参数，控制功能的开 --enable_prefix_cache=true ``` +## Cache 感知 DP 路由 + +当使用数据并行（`dp_size > 1`）时，请求可被路由到持有最长 prefix cache 命中的 DP rank，提高跨 rank 的 KV cache 复用率。 + +- 开启 cache 感知 DP 路由： +``` +--enable_prefix_cache=true --enable_prefix_cache_aware_dp_routing=true +``` + +- `prefix_cache_aware_dp_match_threshold`（默认 `0.5`）：选择 cache 亲和 rank 所需的最低 prefix block 命中比例，低于该阈值时回退到空闲 block 均衡。 +- `prefix_cache_aware_dp_imbalance_threshold`（默认 `0.1`）：跨 rank KV 利用率差异上限（`(max_used-min_used)/total_blocks`），超过此值时路由到负载最低的 rank。 + ## 性能效果 开启prefix_cache之后，在Qwen3-8B模型上，限制TPOT50ms，E2E时延 **下降10%**。