feat(ds4): Lucebox heterogeneous expert parallel + DSpark verify

server/CMakeLists.txt

@@ -260,6 +260,7 @@ add_library(dflash_common STATIC
     src/deepseek4/deepseek4_target_shard_ipc_daemon.cpp
     src/deepseek4/deepseek4_dspark.cpp
     src/deepseek4/deepseek4_dspark_spec.cpp
+    src/deepseek4/deepseek4_dspark_draft_ipc_daemon.cpp
     src/flashprefill_q8.cpp
     src/kv_cache.cpp
     src/kv_quant.cpp
@@ -894,6 +895,20 @@ if(DFLASH27B_TESTS)
         endif()
         add_test(NAME deepseek4_unit COMMAND test_deepseek4_unit)
     endif()
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ggml_rmsnorm_batch.cpp")
+        add_executable(test_ggml_rmsnorm_batch test/test_ggml_rmsnorm_batch.cpp)
+        target_include_directories(test_ggml_rmsnorm_batch PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS})
+        target_link_libraries(test_ggml_rmsnorm_batch PRIVATE ggml ${DFLASH27B_GGML_BACKEND_TARGET} ggml-base)
+        if(DFLASH27B_GPU_BACKEND STREQUAL "hip")
+            target_compile_definitions(test_ggml_rmsnorm_batch PRIVATE GGML_USE_HIP)
+            target_link_libraries(test_ggml_rmsnorm_batch PRIVATE hip::host)
+        else()
+            target_compile_definitions(test_ggml_rmsnorm_batch PRIVATE GGML_USE_CUDA)
+            find_package(CUDAToolkit REQUIRED)
+            target_link_libraries(test_ggml_rmsnorm_batch PRIVATE CUDA::cudart)
+        endif()
+        add_test(NAME ggml_rmsnorm_batch COMMAND test_ggml_rmsnorm_batch)
+    endif()
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_ds4_dspark_load.cpp")
         add_executable(test_ds4_dspark_load test/test_ds4_dspark_load.cpp)
         target_include_directories(test_ds4_dspark_load PRIVATE ${DFLASH27B_SRC_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/include)

server/deps/llama.cpp/ggml/include/ggml-backend.h

@@ -323,6 +323,9 @@ extern "C" {
 
     // Get the number of splits of the last graph
     GGML_API int                  ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
+    GGML_API int                  ggml_backend_sched_get_n_splits_for_backend(
+                                            ggml_backend_sched_t sched,
+                                            ggml_backend_t backend);
     GGML_API int                  ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
 
     GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);
@@ -340,6 +343,13 @@ extern "C" {
     GGML_API enum ggml_status     ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
     GGML_API void                 ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
 
+    // The caller has synchronized all backends and is recording this compute
+    // into an outer device graph.  Buffer-reuse waits may be omitted, while
+    // any copy requiring a synchronous fallback becomes an explicit error.
+    GGML_API void                 ggml_backend_sched_set_whole_graph_capture(
+                                            ggml_backend_sched_t sched,
+                                            bool enabled);
+
     // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
     // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
     // The correct way to use this API is to discard the deallocated tensors and create new ones.
@@ -348,6 +358,25 @@ extern "C" {
     // Set a callback to be called for each resulting node during graph compute
     GGML_API void                 ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
 
+    // End a backend split before a late node introduces a new cross-backend
+    // input, so independent earlier nodes can be enqueued before the copy/event
+    // wait. Must be configured before allocating the graph.
+    GGML_API void                 ggml_backend_sched_set_late_cross_input_split(ggml_backend_sched_t sched, bool enabled);
+
+    // Apply the same split rule only at an explicitly marked node. This keeps
+    // unrelated cross-backend consumers in their established scheduler
+    // segments. Nodes must be registered before allocating the graph.
+    GGML_API void                 ggml_backend_sched_add_late_cross_input_split_node(
+                                            ggml_backend_sched_t sched,
+                                            const struct ggml_tensor * node);
+
+    // Mark a GGML_OP_MOE_FUSED mode -3 node as a deferred peer copy. The
+    // scheduler keeps src[0] on its owner backend, records a dedicated event
+    // after the producing split, and injects the native handle into the op.
+    GGML_API void                 ggml_backend_sched_add_deferred_peer_copy_node(
+                                            ggml_backend_sched_t sched,
+                                            struct ggml_tensor * node);
+
     //
     // Meta backend
     //

server/deps/llama.cpp/ggml/include/ggml-cuda.h

@@ -24,6 +24,34 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device);
 
 GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);
 
+// Configure streams lazily created by this backend context at the device's
+// lowest scheduling priority. Must be called before the backend first submits
+// work. Other backend contexts on the same device are unaffected.
+GGML_BACKEND_API bool ggml_backend_cuda_set_low_priority_stream(
+    ggml_backend_t backend);
+
+// Skip the expensive per-node CUDA/HIP graph property comparison on the
+// calling thread once a stable graph has already been captured.  Callers must
+// bracket only immutable-topology graphs whose tensor addresses and shapes do
+// not change; input contents may still be updated in place.
+GGML_BACKEND_API void ggml_backend_cuda_set_skip_props_check(bool skip);
+
+// Capture a scheduler-wide device program around multiple ordinary backend
+// graph submissions.  The handle is opaque and belongs to `backend`; callers
+// must free it before freeing that backend.  These hooks are intentionally
+// opt-in so normal per-split CUDA/HIP graph behavior is unchanged.
+typedef void * ggml_backend_cuda_whole_graph_t;
+GGML_BACKEND_API bool ggml_backend_cuda_whole_graph_capture_prepare(
+    ggml_backend_t backend0, ggml_backend_t backend1);
+GGML_BACKEND_API bool ggml_backend_cuda_whole_graph_capture_begin(
+    ggml_backend_t backend);
+GGML_BACKEND_API ggml_backend_cuda_whole_graph_t
+ggml_backend_cuda_whole_graph_capture_end(ggml_backend_t backend);
+GGML_BACKEND_API bool ggml_backend_cuda_whole_graph_launch(
+    ggml_backend_t backend, ggml_backend_cuda_whole_graph_t graph);
+GGML_BACKEND_API void ggml_backend_cuda_whole_graph_free(
+    ggml_backend_t backend, ggml_backend_cuda_whole_graph_t graph);
+
 // device buffer
 GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
 
@@ -40,6 +68,12 @@ GGML_BACKEND_API int  ggml_backend_cuda_get_device_count(void);
 GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
 GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
 
+// Override the plain quantized MUL_MAT MMVQ column ceiling on the calling
+// thread. Pass zero to restore LUCE_MMVQ_MAX_NCOLS. This is intentionally
+// thread-local so one graph builder can select a safe topology without
+// changing concurrent requests or other CUDA/HIP backends.
+GGML_BACKEND_API void ggml_backend_cuda_set_mmvq_max_ncols_override(int max_ncols);
+
 GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
 GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);
 

server/deps/llama.cpp/ggml/include/ggml-rpc.h

@@ -8,10 +8,10 @@ extern "C" {
 
 #define RPC_PROTO_MAJOR_VERSION    3
 #define RPC_PROTO_MINOR_VERSION    6
-#define RPC_PROTO_PATCH_VERSION    2
+#define RPC_PROTO_PATCH_VERSION    4
 
 #ifdef  __cplusplus
-static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
+static_assert(GGML_OP_COUNT == 104, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
 #endif
 
 #define GGML_RPC_MAX_SERVERS       16

server/deps/llama.cpp/ggml/include/ggml.h

@@ -603,6 +603,16 @@ extern "C" {
 
         GGML_OP_DS4_HC,  // Fused DeepSeek4 hyper-connection pre/post/out mixing
 
+        GGML_OP_DS4_INDEXER_QAT,  // DS4 indexer Hadamard + FP4 activation simulation
+
+        GGML_OP_DS4_INDEXER_SCORE,  // Fused DS4 indexer dot/ReLU/head reduction
+
+        GGML_OP_DS4_INDEXER_MASK,  // Apply per-token DS4 indexer top-k to an attention mask
+
+        // Keep extension operations appended so established GGML op ordinals
+        // remain stable within a protocol generation.
+        GGML_OP_MUL_MAT_GROUPED_SRC,
+
         GGML_OP_COUNT,
     };
 
@@ -1446,6 +1456,23 @@ extern "C" {
             struct ggml_tensor  * a,
             struct ggml_tensor  * b);
 
+    // Matrix multiply where b is physically [K/group, N, group] but is
+    // consumed as logical [K, N]. This is a distinct operation so backends
+    // that do not implement the grouped layout reject it instead of silently
+    // treating the physical storage as an ordinary flattened matrix.
+    // CUDA/HIP MMQ fuses the gather into its Q8 activation quantizer; the CPU
+    // implementation materializes each logical activation row in scratch.
+    GGML_API struct ggml_tensor * ggml_mul_mat_grouped_src(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * a,
+            struct ggml_tensor  * b);
+
+    GGML_API bool ggml_mul_mat_is_grouped_src(
+            const struct ggml_tensor * tensor);
+
+    GGML_API int64_t ggml_mul_mat_grouped_src_groups(
+            const struct ggml_tensor * tensor);
+
     // change the precision of a matrix multiplication
     // set to GGML_PREC_F32 for higher precision (useful for phi-2)
     GGML_API void ggml_mul_mat_set_prec(
@@ -2377,6 +2404,40 @@ extern "C" {
             struct ggml_tensor * a,
             enum ggml_prec       prec);
 
+    // DS4 layout and block-sparse policy for flash_attn_ext. raw_window is the
+    // maximum visible span inside the raw-row region. Compressed rows are
+    // selected in fixed-size blocks, capped to keep_rows. Zero leaves the
+    // operation dense. A negative value means the mask already contains an
+    // exact row selection and abs(keep_rows) is its maximum compressed width.
+    GGML_API void ggml_flash_attn_ext_set_ds4_sparse(
+            struct ggml_tensor * a,
+            int                  raw_rows,
+            int                  raw_window,
+            int                  keep_rows,
+            int                  block_size);
+
+    // Fuse DS4's inverse 64-d tail RoPE into the D=512 flash-attention
+    // writeback. q_unrotated additionally asks the kernel to apply the forward
+    // tail RoPE to Q from shared F32. This is exact-only plumbing: both paths
+    // retain the explicit F32 rounding boundary and YaRN arithmetic used by
+    // GGML_OP_ROPE.
+    GGML_API void ggml_flash_attn_ext_set_ds4_inverse_rope(
+            struct ggml_tensor * a,
+            int                  kv_start,
+            float                freq_base,
+            float                freq_scale,
+            float                ext_factor,
+            float                attn_factor,
+            float                beta_fast,
+            float                beta_slow,
+            int                  n_ctx_orig,
+            bool                 q_unrotated);
+
+    // True when flash_attn_ext carries the DS4 sparse-layout or fused-RoPE
+    // contract. Backends must implement that complete contract or reject it.
+    GGML_API bool ggml_flash_attn_ext_is_ds4(
+            const struct ggml_tensor * a);
+
     GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
             const struct ggml_tensor * a);
 
@@ -2426,9 +2487,66 @@ extern "C" {
             struct ggml_tensor  * experts,
             struct ggml_tensor  * expert_weights);
 
-    // Fused DeepSeek4 hyper-connection helpers (decode, n_tokens == 1).
-    // ggml_ds4_hc_pre: mix[2*n_hc+n_hc^2] + base + hc_state[n_embd*n_hc] ->
-    //   dst[n_embd + 2*n_hc + n_hc^2] = { working, split(pre,post,comb) }
+    // Reduce packed owner-local expert rows directly back to token order.
+    // inverse_routes maps [route, token] to a row in packed_experts; routes
+    // with zero weight are ignored before the row is read.
+    GGML_API struct ggml_tensor * ggml_laguna_moe_packed_combine(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * packed_experts,
+            struct ggml_tensor  * inverse_routes,
+            struct ggml_tensor  * expert_weights);
+
+    // Coarse DeepSeek-V4 routed-owner op. gate_up contains concatenated gate
+    // and up output rows; the backend performs fused gate/up MMVQ + clamped
+    // SwiGLU, down MMVQ, route weighting, and local reduction.
+    GGML_API struct ggml_tensor * ggml_ds4_moe_owner(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * input,
+            struct ggml_tensor  * gate_up_w,
+            struct ggml_tensor  * down_w,
+            struct ggml_tensor  * expert_ids,
+            struct ggml_tensor  * expert_weights,
+            int64_t               ff_dim,
+            float                 swiglu_clamp,
+            float                 down_scale);
+
+    // Coarse owner variant for checkpoints that store gate and up tensors
+    // separately. The backend fuses their MMVQ traversal and SwiGLU while
+    // preserving the checkpoint's three external value scales.
+    GGML_API struct ggml_tensor * ggml_ds4_moe_owner_split(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * input,
+            struct ggml_tensor  * gate_w,
+            struct ggml_tensor  * up_w,
+            struct ggml_tensor  * down_w,
+            struct ggml_tensor  * expert_ids,
+            struct ggml_tensor  * expert_weights,
+            int64_t               ff_dim,
+            float                 swiglu_clamp,
+            float                 gate_scale,
+            float                 up_scale,
+            float                 down_scale);
+
+    // Reorder a small [route, token] owner-local ID matrix so occurrences of
+    // the same expert across verification tokens share a kernel block. Each
+    // encoded ID retains its original route slot for exact output scatter.
+    GGML_API struct ggml_tensor * ggml_ds4_moe_align_ids(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * expert_ids);
+
+    // Copy an F32 peer-GPU tensor only after a scheduler-provided device event
+    // has completed. The scheduler writes the native event handle into the op
+    // parameters and deliberately leaves src on its owner backend.
+    GGML_API struct ggml_tensor * ggml_ds4_deferred_peer_copy(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * src);
+
+    // Fused DeepSeek4 hyper-connection helpers. The first dimension is the
+    // per-token payload; an optional second dimension batches independent
+    // tokens without changing their arithmetic.
+    // ggml_ds4_hc_pre: mix[mix_dim,n_tokens] + base[mix_dim] +
+    //   hc_state[n_embd*n_hc,n_tokens] ->
+    //   dst[n_embd+mix_dim,n_tokens] = { working, split(pre,post,comb) }
     GGML_API struct ggml_tensor * ggml_ds4_hc_pre(
             struct ggml_context * ctx,
             struct ggml_tensor  * mix,
@@ -2440,15 +2558,28 @@ extern "C" {
             float                 post_scale,
             float                 comb_scale);
 
-    // ggml_ds4_hc_post: residual hc_state + block_out + split -> new hc_state
+    // ggml_ds4_hc_post: residual hc_state + block_out + split -> new hc_state;
+    // all non-base tensors may carry the same n_tokens second dimension.
     GGML_API struct ggml_tensor * ggml_ds4_hc_post(
             struct ggml_context * ctx,
             struct ggml_tensor  * residual_hc,
             struct ggml_tensor  * block_out,
             struct ggml_tensor  * split,
             int                   n_hc);
 
-    // ggml_ds4_hc_out: output-stage merge of hc streams into one embedding
+    // Exact mode-1 variant for heterogeneous MoE verification. It computes
+    // block_out[d] = peer_block[d] + main_block[d] inside the HC-post kernel,
+    // eliminating the standalone reduction and tokenwise CONT copies.
+    GGML_API struct ggml_tensor * ggml_ds4_hc_post_split(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * residual_hc,
+            struct ggml_tensor  * main_block,
+            struct ggml_tensor  * peer_block,
+            struct ggml_tensor  * split,
+            int                   n_hc);
+
+    // ggml_ds4_hc_out: output-stage merge of hc streams into one embedding per
+    // token. mix and hc_state may carry the same n_tokens second dimension.
     GGML_API struct ggml_tensor * ggml_ds4_hc_out(
             struct ggml_context * ctx,
             struct ggml_tensor  * mix,
@@ -2457,6 +2588,36 @@ extern "C" {
             int                   n_hc,
             float                 pre_scale);
 
+    // Official DS4 ratio-4 indexer transform. Each contiguous 128-wide F32
+    // row is Hadamard-rotated and passed through the model's blockwise FP4
+    // activation-simulation round trip. The operation is out-of-place so the
+    // pre-QAT query remains available to the main attention path.
+    GGML_API struct ggml_tensor * ggml_ds4_indexer_qat(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * input);
+
+    // Compute the official ratio-4 indexer score matrix without materializing
+    // [n_comp,n_head,n_tokens] per-head dots. q is F32
+    // [128,n_head,n_tokens], head_weights is F32 [n_head,n_tokens], and
+    // index_comp is F16 [128,n_comp]. head_weights already includes the
+    // model's 1/sqrt(128*n_head) scale.
+    GGML_API struct ggml_tensor * ggml_ds4_indexer_score(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * q,
+            struct ggml_tensor  * head_weights,
+            struct ggml_tensor  * index_comp,
+            int                   kv_start,
+            int                   ratio);
+
+    // Preserve the raw rows of base_mask and retain only selected compressed
+    // rows. selected is I32 [top_k,n_tokens], indexing the compressed span;
+    // base_mask is F32 [raw_rows+n_comp,n_tokens].
+    GGML_API struct ggml_tensor * ggml_ds4_indexer_mask(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * base_mask,
+            struct ggml_tensor  * selected,
+            int                   raw_rows);
+
     // TODO: needs to be adapted to ggml_flash_attn_ext
     GGML_API struct ggml_tensor * ggml_flash_attn_back(
            struct ggml_context * ctx,