diff --git a/examples/common-ggml.cpp b/examples/common-ggml.cpp index 3f2eded86f7..11a5c0a1a05 100644 --- a/examples/common-ggml.cpp +++ b/examples/common-ggml.cpp @@ -75,6 +75,7 @@ bool ggml_common_quantize_0( case GGML_FTYPE_MOSTLY_MXFP4: case GGML_FTYPE_MOSTLY_NVFP4: case GGML_FTYPE_MOSTLY_Q1_0: + case GGML_FTYPE_MOSTLY_Q2_0: { fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype); return false; @@ -217,6 +218,7 @@ bool ggml_common_quantize_0( case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_COUNT: { fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype)); diff --git a/examples/talk-llama/CMakeLists.txt b/examples/talk-llama/CMakeLists.txt index 13b284ed0e9..59643c39049 100644 --- a/examples/talk-llama/CMakeLists.txt +++ b/examples/talk-llama/CMakeLists.txt @@ -21,6 +21,7 @@ if (WHISPER_SDL2) llama-kv-cache.cpp llama-kv-cache-iswa.cpp llama-kv-cache-dsa.cpp + llama-kv-cache-dsv4.cpp llama-memory-recurrent.cpp llama-memory-hybrid.cpp llama-memory-hybrid-iswa.cpp diff --git a/examples/talk-llama/llama-arch.cpp b/examples/talk-llama/llama-arch.cpp index 4a52d977297..b890e66fcf6 100644 --- a/examples/talk-llama/llama-arch.cpp +++ b/examples/talk-llama/llama-arch.cpp @@ -77,6 +77,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_DEEPSEEK2, "deepseek2" }, { LLM_ARCH_DEEPSEEK2OCR, "deepseek2-ocr" }, { LLM_ARCH_DEEPSEEK32, "deepseek32" }, + { LLM_ARCH_DEEPSEEK4, "deepseek4" }, { LLM_ARCH_CHATGLM, "chatglm" }, { LLM_ARCH_GLM4, "glm4" }, { LLM_ARCH_GLM4_MOE, "glm4moe" }, @@ -129,6 +130,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_PANGU_EMBED, "pangu-embedded" }, { LLM_ARCH_MISTRAL3, "mistral3" }, { LLM_ARCH_EAGLE3, "eagle3" }, + { LLM_ARCH_DFLASH, "dflash" }, { LLM_ARCH_MISTRAL4, "mistral4" }, { LLM_ARCH_PADDLEOCR, "paddleocr" }, { LLM_ARCH_MIMO2, "mimo2" }, @@ -249,9 +251,19 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, "%s.attention.indexer.head_count" }, { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, "%s.attention.indexer.key_length" }, { LLM_KV_ATTENTION_INDEXER_TOP_K, "%s.attention.indexer.top_k" }, + { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, + { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, + { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, + { LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" }, { LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" }, { LLM_KV_ATTENTION_RECURRENT_LAYERS, "%s.attention.recurrent_layers" }, + { LLM_KV_HYPER_CONNECTION_COUNT, "%s.hyper_connection.count" }, + { LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, "%s.hyper_connection.sinkhorn_iterations" }, + { LLM_KV_HYPER_CONNECTION_EPSILON, "%s.hyper_connection.epsilon" }, + + { LLM_KV_HASH_LAYER_COUNT, "%s.hash_layer_count" }, + { LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" }, { LLM_KV_ROPE_DIMENSION_COUNT_SWA, "%s.rope.dimension_count_swa" }, { LLM_KV_ROPE_DIMENSION_SECTIONS, "%s.rope.dimension_sections" }, @@ -439,6 +451,23 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_ATTN_Q_B, "blk.%d.attn_q_b" }, { LLM_TENSOR_ATTN_KV_A_MQA, "blk.%d.attn_kv_a_mqa" }, { LLM_TENSOR_ATTN_KV_B, "blk.%d.attn_kv_b" }, + { LLM_TENSOR_ATTN_KV, "blk.%d.attn_kv" }, + { LLM_TENSOR_ATTN_KV_NORM, "blk.%d.attn_kv_a_norm" }, + { LLM_TENSOR_ATTN_OUT_A, "blk.%d.attn_output_a" }, + { LLM_TENSOR_ATTN_OUT_B, "blk.%d.attn_output_b" }, + { LLM_TENSOR_HC_HEAD_FN, "output_hc_fn" }, + { LLM_TENSOR_HC_HEAD_BASE, "output_hc_base" }, + { LLM_TENSOR_HC_HEAD_SCALE, "output_hc_scale" }, + { LLM_TENSOR_HC_ATTN_FN, "blk.%d.hc_attn_fn" }, + { LLM_TENSOR_HC_ATTN_BASE, "blk.%d.hc_attn_base" }, + { LLM_TENSOR_HC_ATTN_SCALE, "blk.%d.hc_attn_scale" }, + { LLM_TENSOR_HC_FFN_FN, "blk.%d.hc_ffn_fn" }, + { LLM_TENSOR_HC_FFN_BASE, "blk.%d.hc_ffn_base" }, + { LLM_TENSOR_HC_FFN_SCALE, "blk.%d.hc_ffn_scale" }, + { LLM_TENSOR_ATTN_COMPRESSOR_WKV, "blk.%d.attn_compressor_kv" }, + { LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "blk.%d.attn_compressor_gate" }, + { LLM_TENSOR_ATTN_COMPRESSOR_APE, "blk.%d.attn_compressor_ape" }, + { LLM_TENSOR_ATTN_COMPRESSOR_NORM, "blk.%d.attn_compressor_norm" }, { LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "per_layer_token_embd" }, { LLM_TENSOR_PER_LAYER_MODEL_PROJ, "per_layer_model_proj" }, { LLM_TENSOR_PER_LAYER_PROJ_NORM, "per_layer_proj_norm" }, @@ -565,6 +594,11 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_INDEXER_PROJ, "blk.%d.indexer.proj" }, { LLM_TENSOR_INDEXER_ATTN_K, "blk.%d.indexer.attn_k" }, { LLM_TENSOR_INDEXER_ATTN_Q_B, "blk.%d.indexer.attn_q_b" }, + { LLM_TENSOR_INDEXER_COMPRESSOR_WKV, "blk.%d.indexer_compressor_kv" }, + { LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "blk.%d.indexer_compressor_gate" }, + { LLM_TENSOR_INDEXER_COMPRESSOR_APE, "blk.%d.indexer_compressor_ape" }, + { LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "blk.%d.indexer_compressor_norm" }, + { LLM_TENSOR_FFN_GATE_TID2EID, "blk.%d.ffn_gate_tid2eid" }, { LLM_TENSOR_MASKED_EMBD_CENTROIDS, "masked_embd_centroids" }, { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, { LLM_TENSOR_FC, "fc" }, @@ -615,6 +649,23 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_ATTN_Q_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ATTN_KV_A_MQA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ATTN_KV_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_KV_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ATTN_OUT_A, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_OUT_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_HEAD_FN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_HEAD_BASE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}}, + {LLM_TENSOR_HC_HEAD_SCALE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_HC_ATTN_FN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_HC_ATTN_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_HC_FFN_FN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_HC_FFN_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_ATTN_COMPRESSOR_WKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_COMPRESSOR_WGATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_COMPRESSOR_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_ATTN_COMPRESSOR_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ATTN_K_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ATTN_V_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ATTN_SINKS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SCALE}}, @@ -778,6 +829,11 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_INDEXER_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_INDEXER_ATTN_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_INDEXER_ATTN_Q_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_INDEXER_COMPRESSOR_WKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_INDEXER_COMPRESSOR_APE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}}, + {LLM_TENSOR_INDEXER_COMPRESSOR_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_FFN_GATE_TID2EID, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // NextN/MTP tensors are stored per-block (blk.%d.nextn.*) even though only the @@ -932,6 +988,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_OLMOE: case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK32: + case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_GLM_DSA: case LLM_ARCH_BITNET: case LLM_ARCH_T5: diff --git a/examples/talk-llama/llama-arch.h b/examples/talk-llama/llama-arch.h index 989da06d8d5..a4f5091e717 100644 --- a/examples/talk-llama/llama-arch.h +++ b/examples/talk-llama/llama-arch.h @@ -82,6 +82,7 @@ enum llm_arch { LLM_ARCH_DEEPSEEK2, LLM_ARCH_DEEPSEEK2OCR, LLM_ARCH_DEEPSEEK32, + LLM_ARCH_DEEPSEEK4, LLM_ARCH_CHATGLM, LLM_ARCH_GLM4, LLM_ARCH_GLM4_MOE, @@ -143,6 +144,7 @@ enum llm_arch { LLM_ARCH_TALKIE, LLM_ARCH_MELLUM, LLM_ARCH_EAGLE3, + LLM_ARCH_DFLASH, LLM_ARCH_UNKNOWN, }; @@ -254,9 +256,19 @@ enum llm_kv { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, LLM_KV_ATTENTION_INDEXER_TOP_K, + LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, + LLM_KV_ATTENTION_OUTPUT_LORA_RANK, + LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, + LLM_KV_ATTENTION_COMPRESS_RATIOS, LLM_KV_ATTENTION_SHARED_KV_LAYERS, LLM_KV_ATTENTION_RECURRENT_LAYERS, + LLM_KV_HYPER_CONNECTION_COUNT, + LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, + LLM_KV_HYPER_CONNECTION_EPSILON, + + LLM_KV_HASH_LAYER_COUNT, + LLM_KV_ROPE_DIMENSION_COUNT, LLM_KV_ROPE_DIMENSION_COUNT_SWA, LLM_KV_ROPE_DIMENSION_SECTIONS, @@ -500,10 +512,27 @@ enum llm_tensor { LLM_TENSOR_ATTN_Q_B, LLM_TENSOR_ATTN_KV_A_MQA, LLM_TENSOR_ATTN_KV_B, + LLM_TENSOR_ATTN_KV, + LLM_TENSOR_ATTN_KV_NORM, + LLM_TENSOR_ATTN_OUT_A, + LLM_TENSOR_ATTN_OUT_B, LLM_TENSOR_ATTN_K_B, LLM_TENSOR_ATTN_V_B, LLM_TENSOR_ATTN_Q_A_NORM, LLM_TENSOR_ATTN_KV_A_NORM, + LLM_TENSOR_HC_HEAD_FN, + LLM_TENSOR_HC_HEAD_BASE, + LLM_TENSOR_HC_HEAD_SCALE, + LLM_TENSOR_HC_ATTN_FN, + LLM_TENSOR_HC_ATTN_BASE, + LLM_TENSOR_HC_ATTN_SCALE, + LLM_TENSOR_HC_FFN_FN, + LLM_TENSOR_HC_FFN_BASE, + LLM_TENSOR_HC_FFN_SCALE, + LLM_TENSOR_ATTN_COMPRESSOR_WKV, + LLM_TENSOR_ATTN_COMPRESSOR_WGATE, + LLM_TENSOR_ATTN_COMPRESSOR_APE, + LLM_TENSOR_ATTN_COMPRESSOR_NORM, LLM_TENSOR_ATTN_SUB_NORM, LLM_TENSOR_FFN_SUB_NORM, LLM_TENSOR_DEC_ATTN_NORM, @@ -565,6 +594,11 @@ enum llm_tensor { LLM_TENSOR_INDEXER_PROJ, LLM_TENSOR_INDEXER_ATTN_K, LLM_TENSOR_INDEXER_ATTN_Q_B, + LLM_TENSOR_INDEXER_COMPRESSOR_WKV, + LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, + LLM_TENSOR_INDEXER_COMPRESSOR_APE, + LLM_TENSOR_INDEXER_COMPRESSOR_NORM, + LLM_TENSOR_FFN_GATE_TID2EID, LLM_TENSOR_NEXTN_PROJ_PRE, LLM_TENSOR_NEXTN_PROJ_POST, LLM_TENSOR_NEXTN_EH_PROJ, diff --git a/examples/talk-llama/llama-batch.cpp b/examples/talk-llama/llama-batch.cpp index 6bf76939cdd..2b98a552f48 100644 --- a/examples/talk-llama/llama-batch.cpp +++ b/examples/talk-llama/llama-batch.cpp @@ -379,6 +379,8 @@ bool llama_batch_allocr::init( LLAMA_LOG_ERROR("%s: sequence %d positions are decreasing (not allowed)\n", __func__, seq_id); return false; } + + cur_seq_pos[seq_id] = pos; } } } @@ -505,7 +507,7 @@ llama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) { return ubatch_add(idxs, idxs.size(), false); } -llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) { +llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail) { if (sequential && has_cpl) { LLAMA_LOG_ERROR("%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\n", __func__); @@ -548,7 +550,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) } } - const uint32_t n_seqs = cur_seq_set.size(); + uint32_t n_seqs = cur_seq_set.size(); // we are done if (n_seqs == 0) { @@ -569,7 +571,7 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) std::vector idxs_per_seq(n_seqs); while (true) { - // we can only add new n_seq_tokens tokens if all the sequence sets have at least one more unused token and + // we can only add new n_seq_tokens tokens if all the sequence sets have at least 1 more unused tokens and // if we haven't reached n_ubatch bool can_expand = true; @@ -600,6 +602,72 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) } } + // if n_keep_tail > 0, keep only the seqs that either finish in this ubatch or have at least + // n_keep_tail tokens remaining for a future ubatch, so that the trailing n_keep_tail tokens + // of each seq are never split across ubatches + if (n_keep_tail > 0) { + GGML_ASSERT(n_ubatch > n_keep_tail); + + auto n_remaining = [&](uint32_t s) { + return (uint32_t) (seq_set_map[cur_seq_set[s]].size() - cur_idx[s]); + }; + + // keep the longest prefix of seqs that satisfy the constraint, to preserve sequential seq ids + uint32_t n_keep = 0; + while (n_keep < n_seqs) { + const uint32_t remaining = n_remaining(n_keep); + + if (remaining != 0 && remaining < n_keep_tail) { + break; + } + + n_keep++; + } + + // all seqs violate the constraint - resolve the first one directly and emit it alone + if (n_keep == 0) { + auto & idxs = idxs_per_seq[0]; + + const auto & seq_idxs = seq_set_map[cur_seq_set[0]]; + + if (idxs.size() + n_remaining(0) <= n_ubatch) { + // extend the seq to completion + while (n_remaining(0) > 0) { + const int32_t idx = seq_idxs[cur_idx[0]]; + + idxs.push_back(idx); + + used[idx] = true; + ++n_used; + + ++cur_idx[0]; + } + } else { + // truncate the seq so that at least n_keep_tail tokens remain + while (n_remaining(0) < n_keep_tail) { + used[idxs.back()] = false; + --n_used; + + idxs.pop_back(); + + --cur_idx[0]; + } + } + + n_keep = 1; + } + + // return the tokens of the deferred seqs back to the pool + for (uint32_t s = n_keep; s < n_seqs; ++s) { + for (const int32_t idx : idxs_per_seq[s]) { + used[idx] = false; + --n_used; + } + } + + n_seqs = n_keep; + } + // concat the per-sequence-set lists std::vector idxs; @@ -814,7 +882,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) { LLAMA_LOG_DEBUG("%s: output = %p\n", __func__, (void *) ubatch.output); LLAMA_LOG_DEBUG("%s: n_outputs = %d\n", __func__, n_outputs); - if (debug > 1) { + if (debug > 0) { int seq_id_max = 0; for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { for (int s = 0; s < ubatch.n_seq_id[i]; ++s) { diff --git a/examples/talk-llama/llama-batch.h b/examples/talk-llama/llama-batch.h index f77520e86c3..a3d1889d4a0 100644 --- a/examples/talk-llama/llama-batch.h +++ b/examples/talk-llama/llama-batch.h @@ -104,7 +104,8 @@ class llama_batch_allocr { // make ubatches of equal-length sequences sets // if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids - llama_ubatch split_equal(uint32_t n_ubatch, bool sequential); + // n_keep_tail = minimum trailing tokens of a seq that must land in the same ubatch + llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail); // sequence-set-wise split - each ubatch contains a single sequence-set llama_ubatch split_seq(uint32_t n_ubatch); diff --git a/examples/talk-llama/llama-context.cpp b/examples/talk-llama/llama-context.cpp index 220240ea952..5edfc85abfd 100644 --- a/examples/talk-llama/llama-context.cpp +++ b/examples/talk-llama/llama-context.cpp @@ -17,6 +17,7 @@ #include #include #include +#include // // llama_context @@ -30,6 +31,30 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { throw std::runtime_error("Unsupported ctx type"); } +struct llm_fused_op_probe { + llm_fused_op op; + const char * name; + uint32_t n_tokens_per_seq; +}; + +static const llm_fused_op_probe llm_fused_op_flash_attn_probe = { + /*.op =*/ LLM_FUSED_OP_FLASH_ATTN, + /*.name =*/ "Flash Attention", + /*.n_tokens_per_seq =*/ 1, +}; + +static const llm_fused_op_probe llm_fused_op_gdn_ar_probe = { + /*.op =*/ LLM_FUSED_OP_GDN_AR, + /*.name =*/ "fused Gated Delta Net (autoregressive)", + /*.n_tokens_per_seq =*/ 1, +}; + +static const llm_fused_op_probe llm_fused_op_gdn_ch_probe = { + /*.op =*/ LLM_FUSED_OP_GDN_CH, + /*.name =*/ "fused Gated Delta Net (chunked)", + /*.n_tokens_per_seq =*/ 16, +}; + llama_context::llama_context( const llama_model & model, llama_context_params params) : @@ -100,10 +125,10 @@ llama_context::llama_context( cparams.ctx_other = params.ctx_other; } - if (model.arch == LLM_ARCH_EAGLE3) { + if (model.arch == LLM_ARCH_EAGLE3 || model.arch == LLM_ARCH_DFLASH) { if (model.tok_embd == nullptr || model.output == nullptr) { if (params.ctx_other == nullptr) { - throw std::runtime_error("EAGLE3 requires ctx_other to be set (this warning is normal during memory fitting)"); + throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)"); } cparams.ctx_other = params.ctx_other; } @@ -256,7 +281,7 @@ llama_context::llama_context( LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max); if (cparams.n_ctx_seq < hparams.n_ctx_train) { - LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", + LLAMA_LOG_INFO("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", __func__, cparams.n_ctx_seq, hparams.n_ctx_train); } @@ -436,6 +461,69 @@ llama_context::~llama_context() { ggml_opt_free(opt_ctx); } +void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs) { + const char * func = __func__; + auto resolve = [&](const llm_fused_op_probe & probe, bool & enabled) { + if (!enabled) { + return; + } + + const uint32_t n_tokens_probe = probe.n_tokens_per_seq*n_seqs; + + auto * gf = graph_reserve(n_tokens_probe, n_seqs, n_tokens_probe, mctx, true); + if (!gf) { + throw std::runtime_error(std::string("failed to reserve graph for ") + probe.name + " check"); + } + + bool device_mismatch = false; + for (const auto & node : get_gf_res_reserve()->get_fused_nodes()) { + if (node.op != probe.op) { + continue; + } + + GGML_ASSERT(node.il >= 0); + + ggml_backend_t backend_fused = ggml_backend_sched_get_tensor_backend(sched.get(), node.tensor); + ggml_backend_dev_t device_fused = backend_fused ? ggml_backend_get_device(backend_fused) : nullptr; + + // TODO: make this descriptor-specific; model.dev_layer() preserves the current behavior, + // but is still wrong for cases like --no-kv-offload. + ggml_backend_dev_t device_layer = model.dev_layer(node.il); + + if (device_fused != device_layer) { + LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but %s " + "is assigned to device %s (usually due to missing support)\n", + func, node.il, + device_layer ? ggml_backend_dev_name(device_layer) : "none", + probe.name, + device_fused ? ggml_backend_dev_name(device_fused) : "none"); + device_mismatch = true; + break; + } + } + + if (device_mismatch) { + enabled = false; + LLAMA_LOG_WARN("%s: %s not supported, set to disabled\n", func, probe.name); + } else { + enabled = true; + LLAMA_LOG_INFO("%s: %s enabled\n", func, probe.name); + } + }; + + if (cparams.auto_fa) { + resolve(llm_fused_op_flash_attn_probe, cparams.flash_attn); + cparams.auto_fa = false; + } + + if (cparams.auto_fgdn) { + LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func); + resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar); + resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch); + cparams.auto_fgdn = false; + } +} + void llama_context::sched_reserve() { if (!sched_need_reserve) { return; @@ -475,128 +563,7 @@ void llama_context::sched_reserve() { LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs); - // resolve automatic Flash Attention use - if (cparams.auto_fa) { - auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true); - if (!gf) { - throw std::runtime_error("failed to reserve graph for Flash Attention check"); - } - - const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN) + 1; - bool fa_device_mismatch = false; - for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { - ggml_tensor * n = ggml_graph_node(gf, i); - if (n->op != GGML_OP_FLASH_ATTN_EXT) { - continue; - } - ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); - - // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer - GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0); - const int il = std::stoi(n->name + prefix_len); - ggml_backend_dev_t device_kv = model.dev_layer(il); - if (device_fa != device_kv) { - LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor " - "is assigned to device %s (usually due to missing support)\n", - __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa)); - // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways - fa_device_mismatch = true; - break; - } - } - - if (fa_device_mismatch) { - cparams.flash_attn = false; - LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__); - } else { - cparams.flash_attn = true; - LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__); - } - - cparams.auto_fa = false; - } - - if (cparams.auto_fgdn) { - LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__); - - if (cparams.fused_gdn_ar) { - auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true); - if (!gf) { - throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)"); - } - - const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR) + 1; - bool gdn_device_mismatch = false; - for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { - ggml_tensor * n = ggml_graph_node(gf, i); - if (n->op != GGML_OP_GATED_DELTA_NET) { - continue; - } - ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); - - GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0); - const int il = std::stoi(n->name + prefix_len); - ggml_backend_dev_t device_kv = model.dev_layer(il); - if (device_gdn != device_kv) { - LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " - "is assigned to device %s (usually due to missing support)\n", - __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn)); - gdn_device_mismatch = true; - break; - } - } - - if (gdn_device_mismatch) { - cparams.fused_gdn_ar = false; - LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__); - } else { - LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__); - } - } - - if (cparams.fused_gdn_ch) { - // more than one token in the batch per sequence in order to take the chunked path - // note: n_outputs must match n_tokens for embedding models with mean/rank pooling, - // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies - // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens, - // the ggml_mul_mat assertion fails. - const uint32_t n_tokens_ch = 16*n_seqs; - auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true); - if (!gf) { - throw std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)"); - } - - const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH) + 1; - bool gdn_device_mismatch = false; - for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { - ggml_tensor * n = ggml_graph_node(gf, i); - if (n->op != GGML_OP_GATED_DELTA_NET) { - continue; - } - ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); - - GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0); - const int il = std::stoi(n->name + prefix_len); - ggml_backend_dev_t device_kv = model.dev_layer(il); - if (device_gdn != device_kv) { - LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " - "is assigned to device %s (usually due to missing support)\n", - __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn)); - gdn_device_mismatch = true; - break; - } - } - - if (gdn_device_mismatch) { - cparams.fused_gdn_ch = false; - LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__); - } else { - LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__); - } - } - - cparams.auto_fgdn = false; - } + resolve_fused_ops(mctx.get(), n_seqs); // reserve worst-case graph int n_splits_pp = -1; @@ -2321,7 +2288,11 @@ void llama_context::output_reorder() { // uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { - if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) { + if (model.arch == LLM_ARCH_QWEN3NEXT || + model.arch == LLM_ARCH_KIMI_LINEAR || + model.arch == LLM_ARCH_QWEN35 || + model.arch == LLM_ARCH_QWEN35MOE || + model.arch == LLM_ARCH_DEEPSEEK4) { return std::max(n_tokens * 40, 32u * model.n_tensors()); } uint32_t res = std::max(1024u, 8u*model.n_tensors()); diff --git a/examples/talk-llama/llama-context.h b/examples/talk-llama/llama-context.h index f8b7805871e..bf91daa8b56 100644 --- a/examples/talk-llama/llama-context.h +++ b/examples/talk-llama/llama-context.h @@ -262,6 +262,10 @@ struct llama_context { llm_graph_cb graph_get_cb() const; + // disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device + // that differs from the layer it belongs to (usually due to missing backend support) + void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs); + // TODO: read/write lora adapters and cvec size_t state_write_data(llama_io_write_i & io); size_t state_read_data (llama_io_read_i & io); diff --git a/examples/talk-llama/llama-graph.cpp b/examples/talk-llama/llama-graph.cpp index 68c9e606c3e..a8fd11ebc38 100644 --- a/examples/talk-llama/llama-graph.cpp +++ b/examples/talk-llama/llama-graph.cpp @@ -8,6 +8,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" #include "llama-memory-recurrent.h" @@ -17,6 +18,7 @@ #include #include #include +#include #include // dedup helpers @@ -61,26 +63,6 @@ static bool can_reuse_kq_mask( // impl -static ggml_tensor * ggml_mul_mat_aux( - ggml_context * ctx, - ggml_tensor * cur, - ggml_tensor * rot) { - const auto n = rot->ne[0]; - - ggml_tensor * res; - - if (!ggml_is_contiguous(cur)) { - res = ggml_cont_2d (ctx, cur, n, ggml_nelements(cur)/n); - } else { - res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n); - } - res = ggml_mul_mat (ctx, rot, res); - ggml_mul_mat_set_hint(res, GGML_HINT_SRC0_IS_HADAMARD); - res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]); - - return res; -} - void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { if (ubatch->token) { const int64_t n_tokens = ubatch->n_tokens; @@ -486,13 +468,17 @@ void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) { mctx->set_input_k_idxs(self_k_idxs, ubatch); mctx->set_input_v_idxs(self_v_idxs, ubatch); - mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + // the mask is left unallocated when the graph only stores K/V without attending + // (e.g. DFlash's KV-injection pass) + if (self_kq_mask && self_kq_mask->buffer) { + mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + } - if (self_k_rot) { + if (self_k_rot && self_k_rot->buffer) { mctx->set_input_k_rot(self_k_rot); } - if (self_v_rot) { + if (self_v_rot && self_v_rot->buffer) { mctx->set_input_v_rot(self_v_rot); } } @@ -564,7 +550,9 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { // base tensors may not be allocated if there are no non-SWA attention layers if (self_k_idxs && self_k_idxs->buffer) { mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch); - mctx->get_base()->set_input_v_idxs(self_v_idxs, ubatch); + if (self_v_idxs) { + mctx->get_base()->set_input_v_idxs(self_v_idxs, ubatch); + } } // the kq mask guards on its own buffer: shared cells leave idxs unbacked while the mask stays live @@ -575,26 +563,28 @@ void llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) { // swa tensors may not be allocated if there are no SWA attention layers if (self_k_idxs_swa && self_k_idxs_swa->buffer) { mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch); - mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch); + if (self_v_idxs_swa) { + mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch); + } } if (self_kq_mask_swa && self_kq_mask_swa->buffer) { mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn); } - if (self_k_rot) { + if (self_k_rot && self_k_rot->buffer) { mctx->get_base()->set_input_k_rot(self_k_rot); } - if (self_v_rot) { + if (self_v_rot && self_v_rot->buffer) { mctx->get_base()->set_input_v_rot(self_v_rot); } - if (self_k_rot_swa) { + if (self_k_rot_swa && self_k_rot_swa->buffer) { mctx->get_swa()->set_input_k_rot(self_k_rot_swa); } - if (self_v_rot_swa) { + if (self_v_rot_swa && self_v_rot_swa->buffer) { mctx->get_swa()->set_input_v_rot(self_v_rot_swa); } } @@ -629,6 +619,305 @@ bool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) { return res; } +static void dsv4_set_i64(ggml_tensor * dst, const std::vector & src) { + if (!dst || !dst->buffer) { + return; + } + + GGML_ASSERT(dst->ne[0] == (int64_t) src.size()); + ggml_backend_tensor_set(dst, src.data(), 0, src.size()*ggml_element_size(dst)); +} + +static void dsv4_set_i32(ggml_tensor * dst, const std::vector & src) { + if (!dst || !dst->buffer) { + return; + } + + GGML_ASSERT(dst->ne[0] == (int64_t) src.size()); + ggml_backend_tensor_set(dst, src.data(), 0, src.size()*ggml_element_size(dst)); +} + +static void dsv4_set_kq_mask( + ggml_tensor * dst, + const llama_kv_cache_dsv4_context::comp_plan & plan, + uint32_t n_tokens, + int64_t n_stream) { + if (!dst || !dst->buffer) { + return; + } + + GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(n_stream > 0); + GGML_ASSERT(n_tokens%n_stream == 0); + GGML_ASSERT(dst->ne[0] == plan.n_kv); + GGML_ASSERT(dst->ne[1] == (int64_t) n_tokens/n_stream); + GGML_ASSERT(dst->ne[2] == 1); + GGML_ASSERT(dst->ne[3] == n_stream); + GGML_ASSERT((int64_t) plan.n_visible.size() == (int64_t) n_tokens); + GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); + + if (dst->type == GGML_TYPE_F32) { + float * data = (float *) dst->data; + + for (int64_t i = 0; i < (int64_t) n_tokens; ++i) { + const int32_t n_visible = plan.n_visible[i]; + + for (int64_t j = 0; j < dst->ne[0]; ++j) { + data[i*dst->ne[0] + j] = j < n_visible ? 0.0f : -INFINITY; + } + } + } else if (dst->type == GGML_TYPE_F16) { + ggml_fp16_t * data = (ggml_fp16_t *) dst->data; + const ggml_fp16_t fp16_ninf = llama_cast(-INFINITY); + const ggml_fp16_t fp16_zero = llama_cast(0.0f); + + for (int64_t i = 0; i < (int64_t) n_tokens; ++i) { + const int32_t n_visible = plan.n_visible[i]; + + for (int64_t j = 0; j < dst->ne[0]; ++j) { + data[i*dst->ne[0] + j] = j < n_visible ? fp16_zero : fp16_ninf; + } + } + } +} + +static ggml_tensor * dsv4_build_raw_kq_mask( + ggml_context * ctx, + const llama_kv_cache_dsv4_raw_context * mctx, + const llama_ubatch & ubatch, + const llama_cparams & cparams, + int64_t n_stream) { + const auto n_kv = mctx->get_n_kv(); + const auto n_tokens = ubatch.n_tokens; + + GGML_ASSERT(n_stream > 0); + GGML_ASSERT(n_tokens%n_stream == 0); + + const auto type = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32; + + ggml_tensor * res = ggml_new_tensor_4d(ctx, type, n_kv, n_tokens/n_stream, 1, n_stream); + ggml_set_input(res); + ggml_set_name(res, "attn_inp_kq_mask"); + + return res; +} + +static bool dsv4_can_reuse_raw_kq_mask( + ggml_tensor * kq_mask, + const llama_kv_cache_dsv4_raw_context * mctx, + const llama_ubatch & ubatch, + int64_t n_stream) { + const auto n_kv = mctx->get_n_kv(); + const auto n_tokens = ubatch.n_tokens; + + GGML_ASSERT(n_stream > 0); + + bool res = true; + + res &= (kq_mask->ne[0] == n_kv); + res &= (kq_mask->ne[1] == n_tokens/n_stream); + res &= (kq_mask->ne[2] == 1); + res &= (kq_mask->ne[3] == n_stream); + + return res; +} + +static std::string dsv4_plan_positions(const std::vector & values) { + std::ostringstream ss; + ss << "["; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + ss << ", "; + } + ss << values[i]; + } + ss << "]"; + return ss.str(); +} + +static bool dsv4_compress_debug() { + static const bool debug = []() { + const char * env = getenv("LLAMA_DSV4_COMPRESS_DEBUG"); + return env && atoi(env) > 0; + }(); + + return debug; +} + +static void dsv4_set_comp_inputs( + const llm_graph_input_dsv4::comp_input & inp, + const llama_kv_cache_dsv4_context::comp_plan & plan, + const char * name, + bool debug, + uint32_t n_tokens, + int64_t n_stream) { + dsv4_set_i32(inp.state_pos, plan.state_pos); + dsv4_set_i32(inp.state_persist_src_idxs, plan.state_persist_src_idxs); + dsv4_set_i32(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs); + dsv4_set_i32(inp.state_read_idxs, plan.state_read_idxs); + dsv4_set_i64(inp.state_write_idxs, plan.state_write_idxs); + dsv4_set_i32(inp.state_write_pos, plan.state_write_pos); + dsv4_set_kq_mask(inp.kq_mask, plan, n_tokens, n_stream); + + if (debug || dsv4_compress_debug()) { + LLAMA_LOG_INFO("%s: %s n_tokens=%u, n_stream=%d, state_persist_dst=%s, state_write_pos=%s\n", + __func__, name, n_tokens, (int) n_stream, + dsv4_plan_positions(plan.state_persist_dst_idxs).c_str(), + dsv4_plan_positions(plan.state_write_pos).c_str()); + } +} + +static bool dsv4_can_reuse_tensor_1d(ggml_tensor * t, int64_t ne0) { + return (t == nullptr && ne0 == 0) || (t != nullptr && t->ne[0] == ne0); +} + +static bool dsv4_can_reuse_kq_mask( + ggml_tensor * t, + const llama_kv_cache_dsv4_context::comp_plan & plan, + uint32_t n_tokens, + int64_t n_stream) { + if (plan.n_kv == 0) { + return t == nullptr; + } + + GGML_ASSERT(n_stream > 0); + + return t != nullptr && + t->ne[0] == plan.n_kv && + t->ne[1] == (int64_t) n_tokens/n_stream && + t->ne[2] == 1 && + t->ne[3] == n_stream; +} + +static bool dsv4_can_reuse_comp_input( + const llm_graph_input_dsv4::comp_input & inp, + const llama_kv_cache_dsv4_context::comp_plan & plan, + uint32_t n_tokens, + int64_t n_stream) { + bool res = true; + res &= dsv4_can_reuse_tensor_1d(inp.state_pos, plan.state_pos.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_persist_src_idxs, plan.state_persist_src_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_persist_dst_idxs, plan.state_persist_dst_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_read_idxs, plan.state_read_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_write_idxs, plan.state_write_idxs.size()); + res &= dsv4_can_reuse_tensor_1d(inp.state_write_pos, plan.state_write_pos.size()); + res &= dsv4_can_reuse_kq_mask(inp.kq_mask, plan, n_tokens, n_stream); + + return res; +} + +static ggml_tensor * dsv4_build_input_1d( + ggml_context * ctx, + ggml_type type, + int64_t ne0, + const std::string & name) { + if (ne0 == 0) { + return nullptr; + } + + ggml_tensor * res = ggml_new_tensor_1d(ctx, type, ne0); + ggml_set_input(res); + ggml_set_name(res, name.c_str()); + + return res; +} + +static void dsv4_build_comp_inputs( + ggml_context * ctx, + llm_graph_input_dsv4::comp_input & inp, + const llama_kv_cache_dsv4_context::comp_plan & plan, + const char * name, + const llama_cparams & cparams, + int64_t n_stream) { + inp.state_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_pos.size(), std::string("dsv4_") + name + "_state_pos"); + inp.state_persist_src_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_src_idxs.size(), std::string("dsv4_") + name + "_state_persist_src_idxs"); + inp.state_persist_dst_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_persist_dst_idxs.size(), std::string("dsv4_") + name + "_state_persist_dst_idxs"); + inp.state_read_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_read_idxs.size(), std::string("dsv4_") + name + "_state_read_idxs"); + inp.state_write_idxs = dsv4_build_input_1d(ctx, GGML_TYPE_I64, plan.state_write_idxs.size(), std::string("dsv4_") + name + "_state_write_idxs"); + inp.state_write_pos = dsv4_build_input_1d(ctx, GGML_TYPE_I32, plan.state_write_pos.size(), std::string("dsv4_") + name + "_state_write_pos"); + + if (plan.n_kv > 0) { + const int64_t n_tokens = (int64_t) plan.n_visible.size(); + + GGML_ASSERT(n_stream > 0); + GGML_ASSERT(n_tokens%n_stream == 0); + + inp.kq_mask = ggml_new_tensor_4d(ctx, cparams.flash_attn && strcmp(name, "lid") != 0 ? GGML_TYPE_F16 : GGML_TYPE_F32, plan.n_kv, n_tokens/n_stream, 1, n_stream); + ggml_set_input(inp.kq_mask); + ggml_set_name(inp.kq_mask, (std::string("dsv4_") + name + "_kq_mask").c_str()); + } +} + +void llm_graph_input_dsv4_raw::set_input(const llama_ubatch * ubatch) { + if (self_k_idxs && self_k_idxs->buffer) { + mctx->set_input_k_idxs(self_k_idxs); + } + + if (self_kq_mask && self_kq_mask->buffer) { + mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn); + } + + if (self_k_rot) { + mctx->set_input_k_rot(self_k_rot); + } +} + +void llm_graph_input_dsv4::set_input(const llama_ubatch * ubatch) { + const auto & plan_csa = mctx->get_csa_plan(*ubatch); + const auto & plan_hca = mctx->get_hca_plan(*ubatch); + const auto & plan_lid = mctx->get_lid_plan(*ubatch); + const int64_t n_stream = plan_csa.n_stream; + + inp_raw->mctx = mctx->get_raw(); + inp_raw->set_input(ubatch); + + dsv4_set_comp_inputs(inp_csa, plan_csa, "csa", debug > 0, ubatch->n_tokens, n_stream); + dsv4_set_comp_inputs(inp_hca, plan_hca, "hca", debug > 0, ubatch->n_tokens, n_stream); + dsv4_set_comp_inputs(inp_lid, plan_lid, "lid", debug > 0, ubatch->n_tokens, n_stream); + + if (inp_csa.k_rot && inp_csa.k_rot->buffer) { + mctx->get_csa()->set_input_k_rot(inp_csa.k_rot); + } + + if (inp_hca.k_rot && inp_hca.k_rot->buffer) { + mctx->get_hca()->set_input_k_rot(inp_hca.k_rot); + } + + if (inp_lid.k_rot && inp_lid.k_rot->buffer) { + mctx->get_lid()->set_input_k_rot(inp_lid.k_rot); + } +} + +bool llm_graph_input_dsv4::can_reuse(const llm_graph_params & params) { + const auto * mctx = static_cast(params.mctx); + + this->mctx = mctx; + inp_raw->mctx = mctx->get_raw(); + + bool res = true; + + const auto & plan_csa = mctx->get_csa_plan(params.ubatch); + const auto & plan_hca = mctx->get_hca_plan(params.ubatch); + const auto & plan_lid = mctx->get_lid_plan(params.ubatch); + const int64_t n_stream = plan_csa.n_stream; + + const auto * raw_ctx = mctx->get_raw(); + inp_raw->mctx = raw_ctx; + + if (inp_raw->self_k_idxs && inp_raw->self_k_idxs->buffer) { + res &= inp_raw->self_k_idxs->ne[0] == raw_ctx->get_n_write(); + } + if (inp_raw->self_kq_mask && inp_raw->self_kq_mask->buffer) { + res &= dsv4_can_reuse_raw_kq_mask(inp_raw->self_kq_mask, raw_ctx, params.ubatch, n_stream); + } + + res &= dsv4_can_reuse_comp_input(inp_csa, plan_csa, params.ubatch.n_tokens, n_stream); + res &= dsv4_can_reuse_comp_input(inp_hca, plan_hca, params.ubatch.n_tokens, n_stream); + res &= dsv4_can_reuse_comp_input(inp_lid, plan_lid, params.ubatch.n_tokens, n_stream); + + return res; +} + void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { GGML_ASSERT(cross_kq_mask); @@ -904,6 +1193,7 @@ void llm_graph_result::reset() { t_logits = nullptr; t_embd = nullptr; t_embd_pooled = nullptr; + t_h_nextn = nullptr; t_layer_inp.resize(LLAMA_MAX_LAYERS); std::fill(t_layer_inp.begin(), t_layer_inp.end(), nullptr); @@ -916,6 +1206,7 @@ void llm_graph_result::reset() { params = {}; inputs.clear(); + fused_nodes.clear(); buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false)); @@ -1017,6 +1308,10 @@ llm_graph_input_i * llm_graph_result::add_input(llm_graph_input_ptr input) { return inputs.back().get(); } +void llm_graph_result::add_fused_node(llm_graph_fused_node result) { + fused_nodes.push_back(result); +} + void llm_graph_result::set_params(const llm_graph_params & params) { this->params = params; } @@ -1076,6 +1371,8 @@ void llm_graph_context::cb(ggml_tensor * cur, const char * name, int il) const { } } + + ggml_tensor * llm_graph_context::build_cvec( ggml_tensor * cur, int il) const { @@ -1346,20 +1643,24 @@ ggml_tensor * llm_graph_context::build_ffn( switch (type_op) { case LLM_FFN_SILU: if (gate && type_gate == LLM_FFN_PAR) { - // Step35: HF clamps gate (after SiLU) and up before multiplication - if (arch == LLM_ARCH_STEP35 && il >= 0) { + if (il >= 0) { const float limit = hparams.swiglu_clamp_shexp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - ggml_tensor * gate_act = ggml_silu(ctx0, cur); - cb(gate_act, "ffn_silu", il); - gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); - cb(gate_act, "ffn_silu_clamped", il); - tmp = ggml_clamp(ctx0, tmp, -limit, limit); cb(tmp, "ffn_up_clamped", il); - cur = ggml_mul(ctx0, gate_act, tmp); + if (arch == LLM_ARCH_DEEPSEEK4) { + cur = ggml_clamp(ctx0, cur, -INFINITY, limit); + cb(cur, "ffn_gate_clamped", il); + cur = ggml_swiglu_split(ctx0, cur, tmp); + } else { + ggml_tensor * gate_act = ggml_silu(ctx0, cur); + cb(gate_act, "ffn_silu", il); + gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); + cb(gate_act, "ffn_silu_clamped", il); + cur = ggml_mul(ctx0, gate_act, tmp); + } cb(cur, "ffn_swiglu_limited", il); type_gate = LLM_FFN_SEQ; break; @@ -1469,7 +1770,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * gate_up_exps, ggml_tensor * up_exps_s, ggml_tensor * gate_exps_s, - ggml_tensor * down_exps_s) const { + ggml_tensor * down_exps_s, + ggml_tensor * selected_experts_in) const { return build_moe_ffn( cur, gate_inp, /* gate_inp_b */ nullptr, @@ -1489,7 +1791,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn( /* gate_up_exps_b */ nullptr, up_exps_s, gate_exps_s, - down_exps_s + down_exps_s, + selected_experts_in ); } @@ -1516,7 +1819,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * gate_up_exps_b, ggml_tensor * up_exps_s, ggml_tensor * gate_exps_s, - ggml_tensor * down_exps_s) const { + ggml_tensor * down_exps_s, + ggml_tensor * selected_experts_in) const { const int64_t n_embd = cur->ne[0]; const int64_t n_tokens = cur->ne[1]; const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN @@ -1525,6 +1829,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (probs_in == nullptr) { logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens] + if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { + ggml_mul_mat_set_prec(logits, GGML_PREC_F32); + } cb(logits, "ffn_moe_logits", il); } else { logits = probs_in; @@ -1549,6 +1856,10 @@ ggml_tensor * llm_graph_context::build_moe_ffn( { probs = logits; // [n_expert, n_tokens] } break; + case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: + { + probs = ggml_sqrt(ctx0, ggml_softplus(ctx0, logits)); // [n_expert, n_tokens] + } break; default: GGML_ABORT("fatal error"); } @@ -1599,8 +1910,11 @@ ggml_tensor * llm_graph_context::build_moe_ffn( } // select experts - ggml_tensor * selected_experts = ggml_argsort_top_k(ctx0, selection_probs, n_expert_used); // [n_expert_used, n_tokens] - cb(selected_experts->src[0], "ffn_moe_argsort", il); + ggml_tensor * selected_experts = selected_experts_in; + if (selected_experts == nullptr) { + selected_experts = ggml_argsort_top_k(ctx0, selection_probs, n_expert_used); // [n_expert_used, n_tokens] + cb(selected_experts->src[0], "ffn_moe_argsort", il); + } cb(selected_experts, "ffn_moe_topk", il); if (arch == LLM_ARCH_GROVEMOE && n_expert != hparams.n_expert) { @@ -1713,20 +2027,24 @@ ggml_tensor * llm_graph_context::build_moe_ffn( switch (type_op) { case LLM_FFN_SILU: if (gate_exps) { - // Step35: per-layer clamp for routed experts - if (arch == LLM_ARCH_STEP35 && il >= 0) { + if (il >= 0) { const float limit = hparams.swiglu_clamp_exp[il]; constexpr float eps = 1e-6f; if (limit > eps) { - ggml_tensor * gate_act = ggml_silu(ctx0, cur); - cb(gate_act, "ffn_moe_silu", il); - gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); - cb(gate_act, "ffn_moe_silu_clamped", il); - up = ggml_clamp(ctx0, up, -limit, limit); cb(up, "ffn_moe_up_clamped", il); - cur = ggml_mul(ctx0, gate_act, up); + if (arch == LLM_ARCH_DEEPSEEK4) { + cur = ggml_clamp(ctx0, cur, -INFINITY, limit); + cb(cur, "ffn_moe_gate_clamped", il); + cur = ggml_swiglu_split(ctx0, cur, up); + } else { + ggml_tensor * gate_act = ggml_silu(ctx0, cur); + cb(gate_act, "ffn_moe_silu", il); + gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); + cb(gate_act, "ffn_moe_silu_clamped", il); + cur = ggml_mul(ctx0, gate_act, up); + } cb(cur, "ffn_moe_swiglu_limited", il); break; } @@ -2105,7 +2423,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias, hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f); - cb(cur, LLAMA_TENSOR_NAME_FATTN, il); + res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il}); ggml_flash_attn_ext_add_sinks(cur, sinks); ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32); @@ -2324,12 +2642,12 @@ ggml_tensor * llm_graph_context::build_attn( GGML_ASSERT(v_mla == nullptr); if (inp->self_k_rot) { - q_cur = ggml_mul_mat_aux(ctx0, q_cur, inp->self_k_rot); - k_cur = ggml_mul_mat_aux(ctx0, k_cur, inp->self_k_rot); + q_cur = llama_mul_mat_hadamard(ctx0, q_cur, inp->self_k_rot); + k_cur = llama_mul_mat_hadamard(ctx0, k_cur, inp->self_k_rot); } if (inp->self_v_rot) { - v_cur = ggml_mul_mat_aux(ctx0, v_cur, inp->self_v_rot); + v_cur = llama_mul_mat_hadamard(ctx0, v_cur, inp->self_v_rot); } // these nodes are added to the graph together so that they are not reordered @@ -2360,7 +2678,7 @@ ggml_tensor * llm_graph_context::build_attn( cb(cur, "kqv_out", il); if (inp->self_v_rot) { - cur = ggml_mul_mat_aux(ctx0, cur, inp->self_v_rot); + cur = llama_mul_mat_hadamard(ctx0, cur, inp->self_v_rot); } if (wo) { @@ -2565,14 +2883,14 @@ ggml_tensor * llm_graph_context::build_attn( auto * v_rot = is_swa ? inp->self_v_rot_swa : inp->self_v_rot; if (k_rot) { - q_cur = ggml_mul_mat_aux(ctx0, q_cur, k_rot); + q_cur = llama_mul_mat_hadamard(ctx0, q_cur, k_rot); if (k_cur) { - k_cur = ggml_mul_mat_aux(ctx0, k_cur, k_rot); + k_cur = llama_mul_mat_hadamard(ctx0, k_cur, k_rot); } } if (v_rot) { if (v_cur) { - v_cur = ggml_mul_mat_aux(ctx0, v_cur, v_rot); + v_cur = llama_mul_mat_hadamard(ctx0, v_cur, v_rot); } } @@ -2615,7 +2933,7 @@ ggml_tensor * llm_graph_context::build_attn( cb(cur, "kqv_out", il); if (v_rot) { - cur = ggml_mul_mat_aux(ctx0, cur, v_rot); + cur = llama_mul_mat_hadamard(ctx0, cur, v_rot); } if (wo) { @@ -2755,6 +3073,33 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp)); } +llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const { + const auto * mctx_cur = static_cast(mctx); + const auto * raw_ctx = mctx_cur->get_raw(); + + auto inp_raw = std::make_unique(cparams, raw_ctx); + + const int64_t n_stream = mctx_cur->get_csa_plan(ubatch).n_stream; + + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE && "DSV4 expects SWA raw cache"); + + inp_raw->self_k_idxs = raw_ctx->build_input_k_idxs(ctx0, ubatch); + inp_raw->self_kq_mask = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream); + inp_raw->self_kq_mask_cnv = inp_raw->self_kq_mask; + + inp_raw->self_k_rot = raw_ctx->build_input_k_rot(ctx0); + auto inp = std::make_unique(cparams, std::move(inp_raw), mctx_cur); + + dsv4_build_comp_inputs(ctx0, inp->inp_csa, mctx_cur->get_csa_plan(ubatch), "csa", cparams, n_stream); + dsv4_build_comp_inputs(ctx0, inp->inp_hca, mctx_cur->get_hca_plan(ubatch), "hca", cparams, n_stream); + dsv4_build_comp_inputs(ctx0, inp->inp_lid, mctx_cur->get_lid_plan(ubatch), "lid", cparams, n_stream); + inp->inp_csa.k_rot = mctx_cur->get_csa()->build_input_k_rot(ctx0); + inp->inp_hca.k_rot = mctx_cur->get_hca()->build_input_k_rot(ctx0); + inp->inp_lid.k_rot = mctx_cur->get_lid()->build_input_k_rot(ctx0); + + return (llm_graph_input_dsv4 *) res->add_input(std::move(inp)); +} + ggml_tensor * llm_graph_context::build_rs( ggml_tensor * s, ggml_tensor * state_copy_main, diff --git a/examples/talk-llama/llama-graph.h b/examples/talk-llama/llama-graph.h index a6e8c3985ba..97141ef93be 100644 --- a/examples/talk-llama/llama-graph.h +++ b/examples/talk-llama/llama-graph.h @@ -23,6 +23,8 @@ struct llama_memory_context_i; class llama_kv_cache_context; class llama_kv_cache_dsa_context; +class llama_kv_cache_dsv4_raw_context; +class llama_kv_cache_dsv4_context; class llama_kv_cache_iswa_context; class llama_memory_recurrent_context; class llama_memory_hybrid_context; @@ -36,6 +38,12 @@ enum llm_graph_type { LLM_GRAPH_TYPE_DECODER_MTP, }; +enum llm_fused_op { + LLM_FUSED_OP_FLASH_ATTN, + LLM_FUSED_OP_GDN_AR, + LLM_FUSED_OP_GDN_CH, +}; + enum llm_ffn_op_type : int { LLM_FFN_NONE = 0, // sentinel: unset; archs must assign before use LLM_FFN_SILU, @@ -459,6 +467,79 @@ class llm_graph_input_attn_kv_iswa : public llm_graph_input_i { const llama_kv_cache_iswa_context * mctx; }; +// DSV4 raw graph inputs are SWA-only, but their mask may be stream-shaped +// so raw K can be concatenated with DSV4 compressed K in one attention op. +class llm_graph_input_dsv4_raw { +public: + llm_graph_input_dsv4_raw( + const llama_cparams & cparams, + const llama_kv_cache_dsv4_raw_context * mctx) : + cparams(cparams), + mctx(mctx) { + } + + void set_input(const llama_ubatch * ubatch); + + ggml_tensor * get_k_idxs() const { return self_k_idxs; } + ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; } + + ggml_tensor * self_k_idxs = nullptr; // I64 [n_batch] + + ggml_tensor * self_kq_mask = nullptr; // F32/F16 [n_kv, n_batch/n_stream, 1, n_stream] + ggml_tensor * self_kq_mask_cnv = nullptr; // [n_kv, n_batch/n_stream, 1, n_stream] + + ggml_tensor * self_k_rot = nullptr; + + const llama_cparams cparams; + + const llama_kv_cache_dsv4_raw_context * mctx; +}; + +class llm_graph_input_dsv4 : public llm_graph_input_i { +public: + struct comp_input { + ggml_tensor * state_pos = nullptr; // I32 [n_state] + ggml_tensor * state_persist_src_idxs = nullptr; // I32 [n_state_persist] + ggml_tensor * state_persist_dst_idxs = nullptr; // I32 [n_state_persist] + ggml_tensor * state_read_idxs = nullptr; // I32 [ratio*n_state_write] + ggml_tensor * state_write_idxs = nullptr; // I64 [n_state_write] + ggml_tensor * state_write_pos = nullptr; // I32 [n_state_write] + + ggml_tensor * kq_mask = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream] + + ggml_tensor * k_rot = nullptr; + }; + + llm_graph_input_dsv4( + const llama_cparams & cparams, + std::unique_ptr inp_raw, + const llama_kv_cache_dsv4_context * mctx) : + inp_raw(std::move(inp_raw)), + cparams(cparams), + mctx(mctx) { + } + ~llm_graph_input_dsv4() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + llm_graph_input_dsv4_raw * get_raw() const { return inp_raw.get(); } + const comp_input & get_csa() const { return inp_csa; } + const comp_input & get_hca() const { return inp_hca; } + const comp_input & get_lid() const { return inp_lid; } + + std::unique_ptr inp_raw; + + comp_input inp_csa; + comp_input inp_hca; + comp_input inp_lid; + + const llama_cparams cparams; + + const llama_kv_cache_dsv4_context * mctx; +}; + class llm_graph_input_attn_cross : public llm_graph_input_i { public: llm_graph_input_attn_cross(const llama_cross * cross) : cross(cross) {} @@ -700,6 +781,12 @@ struct llm_graph_params { } }; +struct llm_graph_fused_node { + llm_fused_op op; + ggml_tensor * tensor; + int il; +}; + class llm_graph_result { public: llm_graph_result(int64_t max_nodes); @@ -733,6 +820,10 @@ class llm_graph_result { llm_graph_input_i * add_input(llm_graph_input_ptr input); + void add_fused_node(llm_graph_fused_node result); + + const std::vector & get_fused_nodes() const { return fused_nodes; } + void set_params(const llm_graph_params & params); // important graph nodes @@ -751,6 +842,7 @@ class llm_graph_result { std::map t_sampled_probs; std::vector inputs; + std::vector fused_nodes; ggml_context_ptr ctx_compute; @@ -920,7 +1012,8 @@ struct llm_graph_context { ggml_tensor * gate_up_exps = nullptr, ggml_tensor * up_exps_s = nullptr, ggml_tensor * gate_exps_s = nullptr, - ggml_tensor * down_exps_s = nullptr) const; + ggml_tensor * down_exps_s = nullptr, + ggml_tensor * selected_experts_in = nullptr) const; ggml_tensor * build_moe_ffn( ggml_tensor * cur, @@ -945,7 +1038,8 @@ struct llm_graph_context { ggml_tensor * gate_up_exps_b = nullptr, ggml_tensor * up_exps_s = nullptr, ggml_tensor * gate_exps_s = nullptr, - ggml_tensor * down_exps_s = nullptr) const; + ggml_tensor * down_exps_s = nullptr, + ggml_tensor * selected_experts_in = nullptr) const; // // inputs @@ -1045,6 +1139,8 @@ struct llm_graph_context { llm_graph_input_attn_kv_iswa * build_attn_inp_kv_iswa() const; + llm_graph_input_dsv4 * build_inp_dsv4() const; + // note: if k_cur or v_cur are not provided, they will not be stored in the memory ggml_tensor * build_attn( llm_graph_input_attn_kv_iswa * inp, diff --git a/examples/talk-llama/llama-hparams.h b/examples/talk-llama/llama-hparams.h index 2eadeb21481..8be5f28f39e 100644 --- a/examples/talk-llama/llama-hparams.h +++ b/examples/talk-llama/llama-hparams.h @@ -14,6 +14,7 @@ enum llama_expert_gating_func_type { LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX = 1, LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID = 2, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT = 3, // applied to the router weights instead of the logits + LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS = 4, }; enum llama_swa_type { @@ -226,6 +227,16 @@ struct llama_hparams { uint32_t indexer_head_size = 0; uint32_t indexer_top_k = 0; + // DeepSeek-V4 + uint32_t dsv4_o_group_count = 0; + uint32_t dsv4_o_lora_rank = 0; + uint32_t dsv4_hc_mult = 0; + uint32_t dsv4_hc_sinkhorn_iters = 0; + uint32_t dsv4_hash_layer_count = 0; + float dsv4_compress_rope_base = 0.0f; + float dsv4_hc_eps = 0.0f; + std::array dsv4_compress_ratios; + // qwen3vl deepstack // When parsed from GGUF, this implies the first N layers consume the first // N deepstack embeddings. Use deepstack_mapping_arr if you need a more diff --git a/examples/talk-llama/llama-impl.h b/examples/talk-llama/llama-impl.h index 7923c3f7ed5..4988b06d2ca 100644 --- a/examples/talk-llama/llama-impl.h +++ b/examples/talk-llama/llama-impl.h @@ -54,6 +54,26 @@ static inline dst_t llama_cast(src_t v) { } } +static inline ggml_tensor * llama_mul_mat_hadamard( + ggml_context * ctx, + ggml_tensor * cur, + ggml_tensor * rot) { + const auto n = rot->ne[0]; + + ggml_tensor * res; + + if (!ggml_is_contiguous(cur)) { + res = ggml_cont_2d(ctx, cur, n, ggml_nelements(cur)/n); + } else { + res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n); + } + res = ggml_mul_mat(ctx, rot, res); + ggml_mul_mat_set_hint(res, GGML_HINT_SRC0_IS_HADAMARD); + res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]); + + return res; +} + struct time_meas { time_meas(int64_t & t_acc, bool disable = false); ~time_meas(); @@ -83,7 +103,3 @@ std::string llama_format_tensor_shape(const std::vector & ne); std::string llama_format_tensor_shape(const struct ggml_tensor * t); std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i); - -#define LLAMA_TENSOR_NAME_FATTN "__fattn__" -#define LLAMA_TENSOR_NAME_FGDN_AR "__fgdn_ar__" -#define LLAMA_TENSOR_NAME_FGDN_CH "__fgdn_ch__" diff --git a/examples/talk-llama/llama-kv-cache-dsa.cpp b/examples/talk-llama/llama-kv-cache-dsa.cpp index 916ab653756..241c50365a1 100644 --- a/examples/talk-llama/llama-kv-cache-dsa.cpp +++ b/examples/talk-llama/llama-kv-cache-dsa.cpp @@ -113,7 +113,7 @@ llama_memory_context_ptr llama_kv_cache_dsa::init_batch( std::vector ubatches; while (true) { - auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true); + auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0); if (ubatch.n_tokens == 0) { break; diff --git a/examples/talk-llama/llama-kv-cache-dsv4.cpp b/examples/talk-llama/llama-kv-cache-dsv4.cpp new file mode 100644 index 00000000000..9fccf347ed6 --- /dev/null +++ b/examples/talk-llama/llama-kv-cache-dsv4.cpp @@ -0,0 +1,1807 @@ +#include "llama-kv-cache-dsv4.h" + +#include "ggml-backend.h" +#include "llama-impl.h" +#include "llama-batch.h" +#include "llama-io.h" +#include "llama-model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +static constexpr uint32_t DSV4_CSA_RATIO = 4; +static constexpr uint32_t DSV4_HCA_RATIO = 128; + +static constexpr uint32_t DSV4_STATE_MAGIC = 0x34565344; // DSV4 +static constexpr uint32_t DSV4_STATE_VERSION = 1; +static constexpr uint32_t DSV4_STATE_MODE_FULL = 0; +static constexpr uint32_t DSV4_STATE_MODE_PARTIAL = 1; +static constexpr uint32_t DSV4_K_CACHE_STATE_VER = 1; +static constexpr uint32_t DSV4_COMP_STATE_VER = 1; + +static uint32_t dsv4_comp_size(uint32_t kv_size, uint32_t ratio) { + return std::max(1, (kv_size + ratio - 1)/ratio); +} + +static int64_t dsv4_stream_offset(uint32_t n_stream, llama_seq_id seq_id, uint32_t size) { + if (n_stream <= 1) { + return 0; + } + if (seq_id < 0 || (uint32_t) seq_id >= n_stream) { + throw std::runtime_error("DSV4 sequence id out of stream range"); + } + + return (int64_t) seq_id*size; +} + +static bool dsv4_ubatch_has_coupled(const llama_ubatch & ubatch) { + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + if (ubatch.n_seq_id[i] > 1) { + return true; + } + } + + return false; +} + +static bool dsv4_token_has_seq(const llama_ubatch & ubatch, uint32_t i, llama_seq_id seq_id) { + for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) { + if (ubatch.seq_id[i][s] == seq_id) { + return true; + } + } + + return false; +} + +static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) { + if (!dsv4_ubatch_has_coupled(ubatch)) { + return ubatch; + } + if (ubatch.embd) { + throw std::runtime_error("DSV4 coupled embedding ubatches are not supported"); + } + + std::vector counts(ubatch.n_seqs_unq, 0); + uint32_t n_tokens = 0; + for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { + const llama_seq_id seq_id = ubatch.seq_id_unq[s]; + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + if (dsv4_token_has_seq(ubatch, i, seq_id)) { + ++counts[s]; + ++n_tokens; + } + } + } + + if (n_tokens == 0) { + return ubatch; + } + + const uint32_t n_seq_tokens = counts[0]; + for (uint32_t s = 1; s < counts.size(); ++s) { + if (counts[s] != n_seq_tokens) { + throw std::runtime_error("DSV4 coupled raw writes require equal sequence lengths"); + } + } + + auto data = std::make_shared(); + data->pos.resize((size_t) n_tokens*ubatch.n_pos); + data->n_seq_id.reserve(n_tokens); + data->seq_id.reserve(n_tokens); + data->seq_id_data.reserve(n_tokens); + data->seq_id_unq.assign(ubatch.seq_id_unq, ubatch.seq_id_unq + ubatch.n_seqs_unq); + data->seq_idx.assign(LLAMA_MAX_SEQ, -1); + data->output.assign(n_tokens, 0); + if (ubatch.token) { + data->token.reserve(n_tokens); + } + + for (uint32_t s = 0; s < data->seq_id_unq.size(); ++s) { + data->seq_idx[data->seq_id_unq[s]] = s; + } + + for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { + const llama_seq_id seq_id = ubatch.seq_id_unq[s]; + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + if (!dsv4_token_has_seq(ubatch, i, seq_id)) { + continue; + } + + const uint32_t dst = data->n_seq_id.size(); + if (ubatch.token) { + data->token.push_back(ubatch.token[i]); + } + for (uint32_t p = 0; p < ubatch.n_pos; ++p) { + data->pos[(size_t) p*n_tokens + dst] = ubatch.pos[(size_t) p*ubatch.n_tokens + i]; + } + data->n_seq_id.push_back(1); + data->seq_id_data.push_back(seq_id); + } + } + + for (uint32_t i = 0; i < n_tokens; ++i) { + data->seq_id.push_back(&data->seq_id_data[i]); + } + + llama_ubatch res { + /*.b_equal_seqs =*/ true, + /*.n_tokens =*/ n_tokens, + /*.n_seq_tokens =*/ n_seq_tokens, + /*.n_seqs =*/ ubatch.n_seqs_unq, + /*.n_seqs_unq =*/ ubatch.n_seqs_unq, + /*.n_pos =*/ ubatch.n_pos, + /*.token =*/ data->token.empty() ? nullptr : data->token.data(), + /*.embd =*/ nullptr, + /*.pos =*/ data->pos.data(), + /*.n_seq_id =*/ data->n_seq_id.data(), + /*.seq_id =*/ data->seq_id.data(), + /*.seq_id_unq =*/ data->seq_id_unq.data(), + /*.seq_idx =*/ data->seq_idx.data(), + /*.output =*/ data->output.data(), + /*.data =*/ data, + }; + + return res; +} + +static std::vector dsv4_build_raw_write_ubatches(const std::vector & ubatches) { + std::vector res; + res.reserve(ubatches.size()); + for (const llama_ubatch & ubatch : ubatches) { + res.push_back(dsv4_build_raw_write_ubatch(ubatch)); + } + return res; +} + +static bool dsv4_batch_has_coupled(const llama_batch & batch) { + if (!batch.n_seq_id) { + return false; + } + + for (int32_t i = 0; i < batch.n_tokens; ++i) { + if (batch.n_seq_id[i] > 1) { + return true; + } + } + + return false; +} + +static int64_t dsv4_comp_graph_n_stream(const llama_ubatch & ubatch, uint32_t n_stream) { + // Coupled sequence sets must stay in one graph stream because their + // compressed state is shared. Independent per-seq state can fan out. + if (n_stream <= 1 || ubatch.n_seqs_unq <= 1 || dsv4_ubatch_has_coupled(ubatch)) { + return 1; + } + + return ubatch.n_seqs_unq; +} + +static void dsv4_state_src_stream_range( + uint32_t n_stream, + llama_seq_id seq_id, + uint32_t & s0, + uint32_t & ns) { + if (seq_id >= 0 && n_stream > 1) { + if ((uint32_t) seq_id >= n_stream) { + throw std::runtime_error("DSV4 state sequence id out of stream range"); + } + + s0 = (uint32_t) seq_id; + ns = 1; + return; + } + + s0 = 0; + ns = seq_id >= 0 ? 1 : n_stream; +} + +static void dsv4_state_dst_stream_range( + uint32_t n_stream, + llama_seq_id seq_id, + uint32_t ns, + uint32_t & s0) { + if (seq_id >= 0) { + if (ns != 1) { + throw std::runtime_error("DSV4 sequence state stream count mismatch"); + } + if (n_stream > 1 && (uint32_t) seq_id >= n_stream) { + throw std::runtime_error("DSV4 state sequence id out of stream range"); + } + + s0 = n_stream > 1 ? (uint32_t) seq_id : 0; + return; + } + + if (ns != n_stream) { + throw std::runtime_error("DSV4 full state stream count mismatch"); + } + + s0 = 0; +} + +static void dsv4_state_write_tensor_streams( + llama_io_write_i & io, + ggml_tensor * tensor, + uint32_t n_rows, + uint32_t s0, + uint32_t ns) { + const int32_t type_i = (int32_t) tensor->type; + const uint64_t ne0 = tensor->ne[0]; + const uint64_t rows = n_rows; + const uint64_t row_size = ggml_row_size(tensor->type, tensor->ne[0]); + + io.write(&type_i, sizeof(type_i)); + io.write(&ne0, sizeof(ne0)); + io.write(&rows, sizeof(rows)); + io.write(&row_size, sizeof(row_size)); + + const size_t offset = (size_t) s0*n_rows*row_size; + const size_t size = (size_t) ns*n_rows*row_size; + + io.write_tensor(tensor, offset, size); +} + +static void dsv4_state_read_tensor_streams( + llama_io_read_i & io, + ggml_tensor * tensor, + uint32_t n_rows, + uint32_t s0, + uint32_t ns) { + int32_t type_i_ref; + uint64_t ne0_ref; + uint64_t rows_ref; + uint64_t row_size_ref; + + io.read(&type_i_ref, sizeof(type_i_ref)); + io.read(&ne0_ref, sizeof(ne0_ref)); + io.read(&rows_ref, sizeof(rows_ref)); + io.read(&row_size_ref, sizeof(row_size_ref)); + + const int32_t type_i = (int32_t) tensor->type; + const uint64_t ne0 = tensor->ne[0]; + const uint64_t rows = n_rows; + const uint64_t row_size = ggml_row_size(tensor->type, tensor->ne[0]); + + if (type_i != type_i_ref || ne0 != ne0_ref || rows != rows_ref || row_size != row_size_ref) { + throw std::runtime_error("DSV4 state tensor metadata mismatch"); + } + + const size_t offset = (size_t) s0*n_rows*row_size; + const size_t size = (size_t) ns*n_rows*row_size; + + io.read_tensor(tensor, offset, size); +} + +static void dsv4_state_write_k_cache( + llama_io_write_i & io, + const llama_kv_cache * kv, + llama_seq_id seq_id, + llama_state_seq_flags flags) { + GGML_UNUSED(flags); + + uint32_t s0; + uint32_t ns; + dsv4_state_src_stream_range(kv->get_n_stream(), seq_id, s0, ns); + + const uint32_t version = DSV4_K_CACHE_STATE_VER; + const uint32_t kv_size = kv->get_size(); + const auto layer_ids = kv->get_layer_ids(); + const uint32_t n_layer = layer_ids.size(); + + io.write(&version, sizeof(version)); + io.write(&kv_size, sizeof(kv_size)); + io.write(&ns, sizeof(ns)); + io.write(&n_layer, sizeof(n_layer)); + + for (uint32_t il : layer_ids) { + io.write(&il, sizeof(il)); + dsv4_state_write_tensor_streams(io, kv->get_k_storage(il), kv_size, s0, ns); + } +} + +static void dsv4_state_read_k_cache( + llama_io_read_i & io, + llama_kv_cache * kv, + llama_seq_id seq_id, + llama_state_seq_flags flags) { + GGML_UNUSED(flags); + + uint32_t version; + uint32_t kv_size_ref; + uint32_t ns; + uint32_t n_layer_ref; + + io.read(&version, sizeof(version)); + io.read(&kv_size_ref, sizeof(kv_size_ref)); + io.read(&ns, sizeof(ns)); + io.read(&n_layer_ref, sizeof(n_layer_ref)); + + if (version != DSV4_K_CACHE_STATE_VER) { + throw std::runtime_error("DSV4 K-cache state version mismatch"); + } + if (kv_size_ref != kv->get_size()) { + throw std::runtime_error("DSV4 K-cache state size mismatch"); + } + + uint32_t s0; + dsv4_state_dst_stream_range(kv->get_n_stream(), seq_id, ns, s0); + + const auto layer_ids = kv->get_layer_ids(); + if (n_layer_ref != layer_ids.size()) { + throw std::runtime_error("DSV4 K-cache layer count mismatch"); + } + + for (uint32_t il : layer_ids) { + uint32_t il_ref; + io.read(&il_ref, sizeof(il_ref)); + if (il_ref != il) { + throw std::runtime_error("DSV4 K-cache layer id mismatch"); + } + + dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv->get_size(), s0, ns); + } +} + +static std::string dsv4_plan_positions(const std::vector & values) { + std::ostringstream ss; + ss << "["; + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + ss << ", "; + } + ss << values[i]; + } + ss << "]"; + return ss.str(); +} + +static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan( + const llama_ubatch & ubatch, + uint32_t ratio, + bool overlap, + uint32_t state_size, + uint32_t kv_size, + uint32_t n_stream) { + llama_kv_cache_dsv4_context::comp_plan plan; + plan.n_visible.resize(ubatch.n_tokens); + plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream); + + // n_stream is the persistent cache/state layout; plan.n_stream is the + // graph view for this ubatch and can be a subset of those streams. + if (n_stream <= 1 && ubatch.n_seqs_unq > 1) { + throw std::runtime_error("DSV4 single compressed stream cannot serve multiple sequences"); + } + + const int64_t state_rows = (int64_t) state_size*n_stream; + + struct persist_row { + int32_t dst; + int32_t src; + llama_pos pos; + }; + + std::vector persist_rows; + + // For the overlap compressor, build_overlap_compressed_kv_from_state() consumes + // state_read_idxs as two contiguous halves: the first ratio*n_blocks entries are + // the "previous-window" gather indices for every block, followed by the + // "current-window" indices for every block. Collect them separately here and + // append cur after prev once the loop has visited all completed blocks + std::vector overlap_prev_reads; + std::vector overlap_cur_reads; + + std::map, int64_t> curr_token_idx_map; + + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) { + curr_token_idx_map[std::make_pair(ubatch.seq_id[i][s], ubatch.pos[i])] = i; + } + } + + const auto state_source_idx = [&](llama_seq_id seq_id, llama_pos pos) -> int32_t { + if (pos < 0) { + // The overlap compressor needs a zero/-inf source for the first + // block's previous half. The graph appends that row after the + // current-ubatch scratch rows. + return (int32_t) (state_rows + ubatch.n_tokens); + } + + const auto key = std::make_pair(seq_id, pos); + if (curr_token_idx_map.find(key) != curr_token_idx_map.end()) { + return (int32_t) (state_rows + curr_token_idx_map.at(key)); + } + + const int64_t stream_off = dsv4_stream_offset(n_stream, seq_id, state_size); + return (int32_t) (stream_off + pos%state_size); + }; + + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + const llama_pos pos = ubatch.pos[i]; + + if (pos < 0) { + continue; + } + + plan.state_pos.push_back((int32_t) (pos%ratio)); + + const int64_t n_visible = (int64_t) (pos + 1)/ratio; + plan.n_visible[i] = (int32_t) n_visible; + plan.n_kv = std::max(plan.n_kv, n_visible); + + for (int32_t s = 0; s < ubatch.n_seq_id[i]; ++s) { + const llama_seq_id seq_id = ubatch.seq_id[i][s]; + const int64_t stream_off = dsv4_stream_offset(n_stream, seq_id, state_size); + const int32_t state_idx = (int32_t) (stream_off + pos%state_size); + + const auto it = std::find_if(persist_rows.begin(), persist_rows.end(), + [state_idx](const persist_row & row) { + return row.dst == state_idx; + }); + if (it == persist_rows.end()) { + persist_rows.push_back({ state_idx, (int32_t) i, pos }); + } else if (pos > it->pos) { + it->src = (int32_t) i; + it->pos = pos; + } + + if ((pos + 1) % ratio != 0) { + continue; + } + + const llama_pos source_start = pos + 1 - ratio; + const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size); + + plan.state_write_idxs.push_back(cache_off + pos/ratio); + plan.state_write_pos.push_back((int32_t) source_start); + + if (overlap) { + const llama_pos prev_start = source_start - ratio; + + for (uint32_t j = 0; j < ratio; ++j) { + overlap_prev_reads.push_back(state_source_idx(seq_id, prev_start + j)); + } + for (uint32_t j = 0; j < ratio; ++j) { + overlap_cur_reads.push_back(state_source_idx(seq_id, source_start + j)); + } + } else { + for (uint32_t j = 0; j < ratio; ++j) { + plan.state_read_idxs.push_back(state_source_idx(seq_id, source_start + j)); + } + } + } + } + + if (ratio == DSV4_CSA_RATIO && plan.state_write_idxs.empty() && !plan.state_pos.empty()) { + // Non-boundary CSA steps still need a write op so their graph matches + // boundary steps. Use a padded scratch row that is masked from attention. + assert(kv_size > 0); + + uint32_t i = 0; + while (i < ubatch.n_tokens && ubatch.pos[i] < 0) { + ++i; + } + assert(i < ubatch.n_tokens); + + const llama_pos pos = ubatch.pos[i]; + const llama_seq_id seq_id = ubatch.seq_id[i][0]; + const int64_t cache_off = dsv4_stream_offset(n_stream, seq_id, kv_size); + const int32_t source_idx = state_source_idx(seq_id, pos); + + plan.state_write_idxs.push_back(cache_off + kv_size - 1); + plan.state_write_pos .push_back(0); + + if (overlap) { + for (uint32_t j = 0; j < ratio; ++j) { + overlap_prev_reads.push_back(source_idx); + overlap_cur_reads .push_back(source_idx); + } + } else { + for (uint32_t j = 0; j < ratio; ++j) { + plan.state_read_idxs.push_back(source_idx); + } + } + } + + if (overlap) { + // [ all blocks' prev-window indices | all blocks' cur-window indices ] + plan.state_read_idxs.reserve(overlap_prev_reads.size() + overlap_cur_reads.size()); + plan.state_read_idxs.insert(plan.state_read_idxs.end(), + overlap_prev_reads.begin(), overlap_prev_reads.end()); + plan.state_read_idxs.insert(plan.state_read_idxs.end(), + overlap_cur_reads.begin(), overlap_cur_reads.end()); + } + + plan.n_kv = GGML_PAD(plan.n_kv, 256u); + + std::sort(persist_rows.begin(), persist_rows.end(), + [](const persist_row & a, const persist_row & b) { + return a.dst < b.dst; + }); + + for (const persist_row & row : persist_rows) { + plan.state_persist_src_idxs.push_back(row.src); + plan.state_persist_dst_idxs.push_back(row.dst); + } + + static const bool debug = []() { + const char * env = getenv("LLAMA_DSV4_COMPRESS_DEBUG"); + return env && atoi(env) > 0; + }(); + + if (debug) { + LLAMA_LOG_INFO("%s: ratio=%u, n_tokens=%u, state_persist_dst=%s, state_write_pos=%s\n", + __func__, ratio, ubatch.n_tokens, + dsv4_plan_positions(plan.state_persist_dst_idxs).c_str(), + dsv4_plan_positions(plan.state_write_pos).c_str()); + } + + return plan; +} + +static std::vector dsv4_build_comp_plans( + const std::vector & ubatches, + uint32_t ratio, + bool overlap, + uint32_t state_size, + uint32_t kv_size, + uint32_t n_stream) { + std::vector plans; + plans.reserve(ubatches.size()); + + for (const llama_ubatch & ubatch : ubatches) { + plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream)); + } + + return plans; +} + +static llama_kv_cache::slot_info_vec_t dsv4_build_comp_sinfos( + const std::vector & ubatches, + uint32_t n_stream) { + llama_kv_cache::slot_info_vec_t sinfos; + sinfos.reserve(ubatches.size()); + + for (const llama_ubatch & ubatch : ubatches) { + if (n_stream <= 1 && ubatch.n_seqs_unq > 1) { + throw std::runtime_error("DSV4 single compressed stream cannot serve multiple sequences"); + } + + const uint32_t ns = (uint32_t) dsv4_comp_graph_n_stream(ubatch, n_stream); + llama_kv_cache::slot_info sinfo; + sinfo.s0 = n_stream > 1 ? LLAMA_MAX_SEQ : 0; + sinfo.s1 = 0; + sinfo.resize(ns); + + for (uint32_t s = 0; s < ns; ++s) { + const llama_seq_id seq_id = n_stream > 1 ? ubatch.seq_id_unq[s] : 0; + const uint32_t strm = (uint32_t) dsv4_stream_offset(n_stream, seq_id, 1); + + sinfo.s0 = std::min(sinfo.s0, strm); + sinfo.s1 = std::max(sinfo.s1, strm); + sinfo.strm[s] = strm; + sinfo.idxs[s].resize(1, 0); + } + + if (n_stream > 1 && sinfo.s1 - sinfo.s0 + 1 != ns) { + throw std::runtime_error("DSV4 compressed streams are not contiguous in ubatch"); + } + + sinfos.push_back(std::move(sinfo)); + } + + return sinfos; +} + +static llama_kv_cache::slot_info_vec_t dsv4_build_raw_read_sinfos( + const llama_kv_cache::slot_info_vec_t & sinfos_write, + const std::vector & ubatches) { + llama_kv_cache::slot_info_vec_t sinfos; + sinfos.reserve(ubatches.size()); + + for (size_t i = 0; i < ubatches.size(); ++i) { + const llama_ubatch & ubatch = ubatches[i]; + const auto & sinfo_write = sinfos_write[i]; + + if (!dsv4_ubatch_has_coupled(ubatch)) { + sinfos.push_back(sinfo_write); + continue; + } + + const llama_seq_id seq_id = ubatch.seq_id[0][0]; + uint32_t i_stream = 0; + for (; i_stream < sinfo_write.n_stream(); ++i_stream) { + if (sinfo_write.strm[i_stream] == seq_id) { + break; + } + } + if (i_stream == sinfo_write.n_stream()) { + throw std::runtime_error("DSV4 raw write stream not found for coupled read"); + } + + llama_kv_cache::slot_info sinfo; + sinfo.s0 = sinfo_write.strm[i_stream]; + sinfo.s1 = sinfo_write.strm[i_stream]; + sinfo.resize(1); + sinfo.strm[0] = sinfo_write.strm[i_stream]; + sinfo.idxs[0] = sinfo_write.idxs[i_stream]; + sinfos.push_back(std::move(sinfo)); + } + + return sinfos; +} + +static llama_kv_cache_dsv4_context::comp_plan dsv4_build_reserve_comp_plan( + const llama_ubatch & ubatch, + uint32_t ratio, + bool overlap, + uint32_t state_size, + uint32_t kv_size, + uint32_t n_stream) { + llama_kv_cache_dsv4_context::comp_plan plan; + plan.n_visible.resize(ubatch.n_tokens); + plan.n_stream = dsv4_comp_graph_n_stream(ubatch, n_stream); + plan.n_kv = kv_size; + + if (ubatch.n_tokens == 0) { + return plan; + } + + const uint32_t n_seqs = std::max(1, ubatch.n_seqs); + const uint32_t n_seq_tokens = std::max(1, ubatch.n_seq_tokens); + const uint64_t n_blocks_u64 = (uint64_t) n_seqs*((n_seq_tokens + ratio - 1)/ratio); + const size_t n_blocks = (size_t) std::max(1, n_blocks_u64); + GGML_ASSERT((uint64_t) n_blocks == std::max(1, n_blocks_u64)); + + const uint64_t state_rows = (uint64_t) state_size*n_stream; + const size_t n_persist = (size_t) std::min(ubatch.n_tokens, state_rows); + + plan.state_pos .resize(ubatch.n_tokens); + plan.state_persist_src_idxs.resize(n_persist); + plan.state_persist_dst_idxs.resize(n_persist); + plan.state_read_idxs .resize((overlap ? 2u : 1u)*ratio*n_blocks); + plan.state_write_idxs.resize(n_blocks); + plan.state_write_pos .resize(n_blocks); + + return plan; +} + +static void dsv4_make_k_only(llama_hparams & hparams) { + // llama_kv_cache uses hparams.is_mla() to allocate K-only storage. + hparams.n_embd_head_k_mla_impl = hparams.n_embd_head_k(); + hparams.n_embd_head_v_mla_impl = hparams.n_embd_head_k(); +} + +// +// llama_dsv4_comp_state +// + +llama_dsv4_comp_state::llama_dsv4_comp_state( + const llama_model & model, + bool offload, + bool unified, + uint32_t n_seq_max, + uint32_t ratio, + uint32_t state_size, + uint32_t n_embd_state, + const char * name, + const llama_memory_i::layer_filter_cb & filter) : + ratio(ratio), + state_size(state_size), + n_embd_state(n_embd_state), + n_stream(unified ? 1 : n_seq_max) { + const llama_hparams & hparams = model.hparams; + + struct ggml_backend_buft_comparator { + bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const { + return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0; + } + }; + + std::map ctx_map; + + auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * { + auto it = ctx_map.find(buft); + if (it == ctx_map.end()) { + ggml_init_params params = { + /*.mem_size =*/ size_t(2u*hparams.n_layer()*ggml_tensor_overhead()), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ggml_context * ctx = ggml_init(params); + if (!ctx) { + return nullptr; + } + + ctx_map.emplace(buft, ctx); + + return ctx; + } + + return it->second.get(); + }; + + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (filter && !filter(il)) { + continue; + } + + const char * dev_name = "CPU"; + + ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type(); + + if (offload) { + auto * dev = model.dev_layer(il); + buft = ggml_backend_dev_buffer_type(dev); + + dev_name = ggml_backend_dev_name(dev); + } + + LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name); + + ggml_context * ctx = ctx_for_buft(buft); + if (!ctx) { + throw std::runtime_error("failed to create ggml context for DSV4 compressor state"); + } + + ggml_tensor * kv = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream); + ggml_tensor * score = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_state, state_size, n_stream); + + ggml_format_name(kv, "dsv4_%s_state_kv_l%d", name, il); + ggml_format_name(score, "dsv4_%s_state_score_l%d", name, il); + + map_layer_ids[il] = layers.size(); + + layers.push_back({ il, kv, score }); + } + + for (auto & [buft, ctx] : ctx_map) { + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); + if (!buf) { + throw std::runtime_error("failed to allocate buffer for DSV4 compressor state"); + } + + ggml_backend_buffer_clear(buf, 0); + + LLAMA_LOG_INFO("%s: %10s DSV4 %s state buffer size = %8.2f MiB\n", + __func__, ggml_backend_buffer_name(buf), name, ggml_backend_buffer_get_size(buf)/1024.0/1024.0); + + ctxs_bufs.emplace_back(std::move(ctx), buf); + } + + LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, layers = %zu, size = %7.2f MiB\n", + __func__, name, ratio, state_size, n_embd_state, n_stream, layers.size(), total_size()/1024.0/1024.0); +} + +void llama_dsv4_comp_state::clear(bool data) { + if (!data) { + return; + } + + for (auto & [_, buf] : ctxs_bufs) { + ggml_backend_buffer_clear(buf.get(), 0); + } +} + +uint32_t llama_dsv4_comp_state::get_ratio() const { + return ratio; +} + +uint32_t llama_dsv4_comp_state::get_state_size() const { + return state_size; +} + +uint32_t llama_dsv4_comp_state::get_n_stream() const { + return n_stream; +} + +std::map llama_dsv4_comp_state::memory_breakdown() const { + std::map ret; + for (const auto & [_, buf] : ctxs_bufs) { + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); + ret[buft] += ggml_backend_buffer_get_size(buf.get()); + } + return ret; +} + +void llama_dsv4_comp_state::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + GGML_UNUSED(flags); + + uint32_t s0; + uint32_t ns; + dsv4_state_src_stream_range(n_stream, seq_id, s0, ns); + + const uint32_t version = DSV4_COMP_STATE_VER; + const uint32_t n_layer = layers.size(); + + io.write(&version, sizeof(version)); + io.write(&ratio, sizeof(ratio)); + io.write(&state_size, sizeof(state_size)); + io.write(&n_embd_state, sizeof(n_embd_state)); + io.write(&ns, sizeof(ns)); + io.write(&n_layer, sizeof(n_layer)); + + for (const auto & layer : layers) { + io.write(&layer.il, sizeof(layer.il)); + + dsv4_state_write_tensor_streams(io, layer.kv, state_size, s0, ns); + dsv4_state_write_tensor_streams(io, layer.score, state_size, s0, ns); + } +} + +void llama_dsv4_comp_state::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + GGML_UNUSED(flags); + + uint32_t version; + uint32_t ratio_ref; + uint32_t state_size_ref; + uint32_t n_embd_state_ref; + uint32_t ns; + uint32_t n_layer_ref; + + io.read(&version, sizeof(version)); + io.read(&ratio_ref, sizeof(ratio_ref)); + io.read(&state_size_ref, sizeof(state_size_ref)); + io.read(&n_embd_state_ref, sizeof(n_embd_state_ref)); + io.read(&ns, sizeof(ns)); + io.read(&n_layer_ref, sizeof(n_layer_ref)); + + if (version != DSV4_COMP_STATE_VER) { + throw std::runtime_error("DSV4 compressor state version mismatch"); + } + if (ratio_ref != ratio || state_size_ref != state_size || n_embd_state_ref != n_embd_state) { + throw std::runtime_error("DSV4 compressor state metadata mismatch"); + } + if (n_layer_ref != layers.size()) { + throw std::runtime_error("DSV4 compressor state layer count mismatch"); + } + + uint32_t s0; + dsv4_state_dst_stream_range(n_stream, seq_id, ns, s0); + + for (const auto & layer : layers) { + uint32_t il_ref; + io.read(&il_ref, sizeof(il_ref)); + if (il_ref != layer.il) { + throw std::runtime_error("DSV4 compressor state layer id mismatch"); + } + + dsv4_state_read_tensor_streams(io, layer.kv, state_size, s0, ns); + dsv4_state_read_tensor_streams(io, layer.score, state_size, s0, ns); + } +} + +ggml_tensor * llama_dsv4_comp_state::get_kv(ggml_context * ctx, int32_t il) const { + const int32_t ids = map_layer_ids.at(il); + + ggml_tensor * state = layers[ids].kv; + + return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]); +} + +ggml_tensor * llama_dsv4_comp_state::get_score(ggml_context * ctx, int32_t il) const { + const int32_t ids = map_layer_ids.at(il); + + ggml_tensor * state = layers[ids].score; + + return ggml_reshape_2d(ctx, state, state->ne[0], state->ne[1]*state->ne[2]); +} + +ggml_tensor * llama_dsv4_comp_state::cpy_kv(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const { + return ggml_set_rows(ctx, get_kv(ctx, il), cur, idxs); +} + +ggml_tensor * llama_dsv4_comp_state::cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const { + return ggml_set_rows(ctx, get_score(ctx, il), cur, idxs); +} + +size_t llama_dsv4_comp_state::total_size() const { + size_t size = 0; + + for (const auto & [_, buf] : ctxs_bufs) { + size += ggml_backend_buffer_get_size(buf.get()); + } + + return size; +} + +// +// llama_kv_cache_dsv4 +// + +llama_kv_cache_dsv4::llama_kv_cache_dsv4( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse) : + hparams_raw(model.hparams), + hparams_csa(model.hparams), + hparams_hca(model.hparams), + hparams_lid(model.hparams), + n_seq_max(n_seq_max) { + + const layer_filter_cb filter_raw = [&](int32_t il) { + if (filter && !filter(il)) { + return false; + } + + return true; + }; + + GGML_UNUSED(unified); + + // Keep DSV4 KV/state streams per sequence even when public KV mode is unified. + const bool unified_raw = false; + + LLAMA_LOG_INFO("%s: creating DSV4 raw KV cache\n", __func__); + + dsv4_make_k_only(hparams_raw); + + kv_raw = std::make_unique( + model, hparams_raw, type_k, type_v, + v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad, + nullptr, filter_raw, reuse, nullptr); + + dsv4_make_k_only(hparams_csa); + dsv4_make_k_only(hparams_hca); + + std::fill(hparams_lid.n_head_kv_arr.begin(), hparams_lid.n_head_kv_arr.end(), 1); + hparams_lid.n_embd_head_k_full = model.hparams.indexer_head_size; + hparams_lid.n_embd_head_v_full = model.hparams.indexer_head_size; + hparams_lid.n_embd_head_k_swa = model.hparams.indexer_head_size; + hparams_lid.n_embd_head_v_swa = model.hparams.indexer_head_size; + hparams_lid.rope_type = LLAMA_ROPE_TYPE_NEOX; + dsv4_make_k_only(hparams_lid); + + const layer_filter_cb filter_csa = [&](int32_t il) { + if (filter && !filter(il)) { + return false; + } + + return model.hparams.dsv4_compress_ratios[il] == DSV4_CSA_RATIO; + }; + + const layer_filter_cb filter_hca = [&](int32_t il) { + if (filter && !filter(il)) { + return false; + } + + return model.hparams.dsv4_compress_ratios[il] == DSV4_HCA_RATIO; + }; + + const bool unified_compressed = false; + + LLAMA_LOG_INFO("%s: creating DSV4 CSA compressed KV cache, size = %u cells\n", + __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO)); + + kv_csa = std::make_unique( + model, hparams_csa, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr); + + LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n", + __func__, dsv4_comp_size(kv_size, DSV4_HCA_RATIO)); + + kv_hca = std::make_unique( + model, hparams_hca, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_HCA_RATIO), 256u), n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr); + + LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n", + __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO)); + + kv_lid = std::make_unique( + model, hparams_lid, type_k, type_v, + v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad, + 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr); + + LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__); + + csa_state = std::make_unique( + model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, + 2*model.hparams.n_embd_head_k(), "csa", filter_csa); + + LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__); + + hca_state = std::make_unique( + model, offload, unified_compressed, n_seq_max, DSV4_HCA_RATIO, DSV4_HCA_RATIO, + model.hparams.n_embd_head_k(), "hca", filter_hca); + + LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__); + + lid_state = std::make_unique( + model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO, + 2*model.hparams.indexer_head_size, "lid", filter_csa); + + // DSV4 attention reads compressed-K / compressor-state rows that the current + // graph does not necessarily overwrite; uninitialized buffer contents would + // otherwise leak in (instance-specific garbage) and corrupt recall. Zero all + // compressed buffers up front so reads of un-written rows are deterministic. + clear_compressed(true); +} + +llama_memory_context_ptr llama_kv_cache_dsv4::init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) { + GGML_UNUSED(embd_all); + + const bool raw_per_seq = kv_raw->get_base()->get_n_stream() != 1; + const bool comp_per_seq = csa_state->get_n_stream() > 1; + const bool has_coupled = dsv4_batch_has_coupled(balloc.get_batch()); + + const auto make_context = [&](std::vector ubatches) -> llama_memory_context_ptr { + auto ubatches_raw = dsv4_build_raw_write_ubatches(ubatches); + + auto sinfos_raw_base_write = kv_raw->get_base()->prepare(ubatches_raw); + if (sinfos_raw_base_write.empty()) { + return nullptr; + } + + auto sinfos_raw_swa_write = kv_raw->get_swa()->prepare(ubatches_raw); + if (sinfos_raw_swa_write.empty()) { + return nullptr; + } + + auto sinfos_raw_swa_read = dsv4_build_raw_read_sinfos(sinfos_raw_swa_write, ubatches); + + return std::make_unique( + this, + std::move(sinfos_raw_base_write), + std::move(sinfos_raw_swa_write), + std::move(sinfos_raw_swa_read), + std::move(ubatches), + std::move(ubatches_raw)); + }; + + // Match llama_kv_cache_iswa splitting when DSV4 compressed state does not + // require per-sequence graph layout. + do { + if (raw_per_seq || comp_per_seq) { + break; + } + + balloc.split_reset(); + + std::vector ubatches; + while (true) { + auto ubatch = balloc.split_simple(n_ubatch); + if (ubatch.n_tokens == 0) { + break; + } + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + break; + } + + if (auto ctx = make_context(std::move(ubatches))) { + return ctx; + } + } while (false); + + // When raw or compressed state is per-sequence, independent sequences can + // share an equal-length ubatch. Coupled sequence sets still serialize until + // DSV4 has explicit shared-state handling for compressed streams. + do { + balloc.split_reset(); + + std::vector ubatches; + while (true) { + llama_ubatch ubatch; + if (has_coupled) { + ubatch = balloc.split_seq(n_ubatch); + } else { + ubatch = balloc.split_equal(n_ubatch, raw_per_seq || comp_per_seq, 0); + } + + if (ubatch.n_tokens == 0) { + break; + } + ubatches.push_back(std::move(ubatch)); // NOLINT + } + + if (balloc.get_n_used() < balloc.get_n_tokens()) { + break; + } + + if (auto ctx = make_context(std::move(ubatches))) { + return ctx; + } + } while (false); + + return std::make_unique(LLAMA_MEMORY_STATUS_FAILED_PREPARE); +} + +llama_memory_context_ptr llama_kv_cache_dsv4::init_full() { + return std::make_unique(this); +} + +llama_memory_context_ptr llama_kv_cache_dsv4::init_update(llama_context * lctx, bool optimize) { + return std::make_unique(this, lctx, optimize); +} + +bool llama_kv_cache_dsv4::get_can_shift() const { + // Compressed row metadata uses block-derived positions. Keep shifting + // disabled until DSV4 compressed-cache shift semantics are wired. + return false; +} + +void llama_kv_cache_dsv4::clear(bool data) { + kv_raw->clear(data); + clear_compressed(true); // DSV4 compressed buffers must never expose stale/uninit rows +} + +bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + if (p1 >= 0) { + return false; + } + + if (p0 > 0) { + // DSV4 compressed cache rows are derived from running compressor state, + // so arbitrary rollback is not reconstructible from the raw cache alone. + // Allow the common prompt-cache cleanup no-op: remove [end, infinity). + if (seq_id >= 0 && p0 > kv_raw->seq_pos_max(seq_id)) { + return true; + } + + return false; + } + + const bool res = kv_raw->seq_rm(seq_id, p0, p1); + + if (res) { + clear_compressed(true); + } + + return res; +} + +void llama_kv_cache_dsv4::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + kv_raw->seq_cp(seq_id_src, seq_id_dst, p0, p1); + clear_compressed(true); +} + +void llama_kv_cache_dsv4::seq_keep(llama_seq_id seq_id) { + kv_raw->seq_keep(seq_id); + clear_compressed(true); +} + +void llama_kv_cache_dsv4::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + kv_raw->seq_add(seq_id, p0, p1, shift); + clear_compressed(true); +} + +void llama_kv_cache_dsv4::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + kv_raw->seq_div(seq_id, p0, p1, d); + clear_compressed(true); +} + +llama_pos llama_kv_cache_dsv4::seq_pos_min(llama_seq_id seq_id) const { + if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) { + return -1; + } + + // The raw SWA cache may contain a wider window, but the compressed DSV4 + // state cannot be rolled back within that window. Report only the current + // boundary so server-context uses checkpoints for rollback. + return kv_raw->seq_pos_max(seq_id); +} + +llama_pos llama_kv_cache_dsv4::seq_pos_max(llama_seq_id seq_id) const { + if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) { + return -1; + } + + return kv_raw->seq_pos_max(seq_id); +} + +std::map llama_kv_cache_dsv4::memory_breakdown() const { + std::map mb = kv_raw->memory_breakdown(); + for (const auto & buft_size : kv_csa->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + for (const auto & buft_size : kv_hca->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + for (const auto & buft_size : kv_lid->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + for (const auto & buft_size : csa_state->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + for (const auto & buft_size : hca_state->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + for (const auto & buft_size : lid_state->memory_breakdown()) { + mb[buft_size.first] += buft_size.second; + } + return mb; +} + +void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const { + const bool partial_only = flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY; + + const uint32_t magic = DSV4_STATE_MAGIC; + const uint32_t version = DSV4_STATE_VERSION; + const uint32_t mode = partial_only ? DSV4_STATE_MODE_PARTIAL : DSV4_STATE_MODE_FULL; + + io.write(&magic, sizeof(magic)); + io.write(&version, sizeof(version)); + io.write(&mode, sizeof(mode)); + + kv_raw->state_write(io, seq_id, flags); + + if (!partial_only) { + dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags); + dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags); + dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags); + } + + csa_state->state_write(io, seq_id, flags); + hca_state->state_write(io, seq_id, flags); + lid_state->state_write(io, seq_id, flags); +} + +void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { + uint32_t magic; + uint32_t version; + uint32_t mode = DSV4_STATE_MODE_FULL; + + io.read(&magic, sizeof(magic)); + io.read(&version, sizeof(version)); + + if (magic != DSV4_STATE_MAGIC) { + throw std::runtime_error("DSV4 state magic mismatch"); + } + if (version != DSV4_STATE_VERSION) { + throw std::runtime_error("DSV4 state version mismatch"); + } + + io.read(&mode, sizeof(mode)); + if (mode != DSV4_STATE_MODE_FULL && mode != DSV4_STATE_MODE_PARTIAL) { + throw std::runtime_error("DSV4 state mode mismatch"); + } + + const bool partial_only = mode == DSV4_STATE_MODE_PARTIAL; + if (partial_only != !!(flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY)) { + throw std::runtime_error("DSV4 state flags mismatch"); + } + + kv_raw->state_read(io, seq_id, flags); + + if (!partial_only) { + dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags); + dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags); + dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags); + } + + csa_state->state_read(io, seq_id, flags); + hca_state->state_read(io, seq_id, flags); + lid_state->state_read(io, seq_id, flags); + +} + +llama_kv_cache_iswa * llama_kv_cache_dsv4::get_raw() const { + return kv_raw.get(); +} + +llama_kv_cache * llama_kv_cache_dsv4::get_csa() const { + return kv_csa.get(); +} + +llama_kv_cache * llama_kv_cache_dsv4::get_hca() const { + return kv_hca.get(); +} + +llama_kv_cache * llama_kv_cache_dsv4::get_lid() const { + return kv_lid.get(); +} + +llama_dsv4_comp_state * llama_kv_cache_dsv4::get_csa_state() const { + return csa_state.get(); +} + +llama_dsv4_comp_state * llama_kv_cache_dsv4::get_hca_state() const { + return hca_state.get(); +} + +llama_dsv4_comp_state * llama_kv_cache_dsv4::get_lid_state() const { + return lid_state.get(); +} + +void llama_kv_cache_dsv4::clear_compressed(bool data) { + kv_csa->clear(data); + kv_hca->clear(data); + kv_lid->clear(data); + csa_state->clear(data); + hca_state->clear(data); + lid_state->clear(data); +} + +// +// llama_kv_cache_dsv4_raw_context +// + +static llama_kv_cache::slot_info dsv4_build_full_sinfo(const llama_kv_cache * kv) { + const uint32_t n_stream = kv->get_n_stream(); + + llama_kv_cache::slot_info sinfo; + sinfo.s0 = 0; + sinfo.s1 = n_stream - 1; + sinfo.resize(n_stream); + for (uint32_t s = 0; s < n_stream; ++s) { + sinfo.strm[s] = s; + sinfo.idxs[s].resize(1, 0); + } + + return sinfo; +} + +llama_kv_cache_dsv4_raw_context::llama_kv_cache_dsv4_raw_context(llama_kv_cache_iswa * kv) : + kv_swa(kv->get_swa()), + ctx_base_mem(nullptr), + ctx_swa_mem(nullptr), + n_kv(kv_swa->get_size()), + status(LLAMA_MEMORY_STATUS_SUCCESS) { + sinfos_read.push_back(dsv4_build_full_sinfo(kv_swa)); + sinfos_write = sinfos_read; +} + +llama_kv_cache_dsv4_raw_context::llama_kv_cache_dsv4_raw_context( + llama_kv_cache_iswa * kv, + llama_context * lctx, + bool optimize) : + kv_swa(kv->get_swa()), + ctx_base_mem(kv->get_base()->init_update(lctx, optimize)), + ctx_swa_mem(kv->get_swa()->init_update(lctx, optimize)), + n_kv(kv_swa->get_size()), + status(llama_memory_status_combine(ctx_base_mem->get_status(), ctx_swa_mem->get_status())) { +} + +llama_kv_cache_dsv4_raw_context::llama_kv_cache_dsv4_raw_context( + llama_kv_cache_iswa * kv, + slot_info_vec_t sinfos_base_write, + slot_info_vec_t sinfos_swa_write, + slot_info_vec_t sinfos_swa_read, + std::vector ubatches, + std::vector ubatches_write) : + kv_swa(kv->get_swa()), + sinfos_write(std::move(sinfos_swa_write)), + sinfos_read(std::move(sinfos_swa_read)), + ubatches(std::move(ubatches)), + ubatches_write(std::move(ubatches_write)), + ctx_base_mem(std::make_unique( + kv->get_base(), std::move(sinfos_base_write), this->ubatches_write)), + ctx_swa_mem(nullptr), + n_kv(kv_swa->get_size()), + status(LLAMA_MEMORY_STATUS_SUCCESS) { +} + +bool llama_kv_cache_dsv4_raw_context::next() { + if (ubatches.empty()) { + return true; + } + + if (ctx_base_mem) { + ctx_base_mem->next(); + } + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_kv_cache_dsv4_raw_context::apply() { + bool res = true; + + if (ctx_base_mem) { + res = res & ctx_base_mem->apply(); + } + if (ctx_swa_mem) { + res = res & ctx_swa_mem->apply(); + } + if (!ubatches_write.empty()) { + kv_swa->apply_ubatch(sinfos_write[i_next], ubatches_write[i_next]); + n_kv = kv_swa->get_n_kv(sinfos_read[i_next]); + } + + return res; +} + +llama_memory_status llama_kv_cache_dsv4_raw_context::get_status() const { + return status; +} + +const llama_ubatch & llama_kv_cache_dsv4_raw_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +uint32_t llama_kv_cache_dsv4_raw_context::get_n_kv() const { + return n_kv; +} + +uint32_t llama_kv_cache_dsv4_raw_context::get_n_write() const { + if (ubatches_write.empty()) { + return 0; + } + + return ubatches_write[i_next].n_tokens; +} + +ggml_tensor * llama_kv_cache_dsv4_raw_context::get_k(ggml_context * ctx, int32_t il) const { + return kv_swa->get_k(ctx, il, n_kv, sinfos_read[i_next]); +} + +ggml_tensor * llama_kv_cache_dsv4_raw_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { + const auto & sinfo = sinfos_write[i_next]; + + if (k_cur->ne[2] == k_idxs->ne[0]) { + return kv_swa->cpy_k(ctx, k_cur, k_idxs, il, sinfo); + } + + // k_idxs may be expanded to one block per stream while k_cur is only + // the token block. Keep zero deps on all copies so each write executes. + const int64_t n_fanout = (int64_t) sinfo.size()*sinfo.n_stream(); + + GGML_ASSERT(sinfo.n_stream() > 1); + GGML_ASSERT(k_cur->ne[2] == (int64_t) sinfo.size()); + GGML_ASSERT(k_idxs->ne[0] == n_fanout); + + ggml_tensor * res = nullptr; + for (uint32_t s = 0; s < sinfo.n_stream(); ++s) { + ggml_tensor * k_idxs_s = ggml_view_1d(ctx, k_idxs, sinfo.size(), s*sinfo.size()*ggml_element_size(k_idxs)); + ggml_tensor * cur = kv_swa->cpy_k(ctx, k_cur, k_idxs_s, il, sinfo); + if (res == nullptr) { + res = cur; + } else { + res = ggml_add(ctx, res, ggml_sub(ctx, cur, cur)); + } + } + + return res; +} + +ggml_tensor * llama_kv_cache_dsv4_raw_context::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const { + const uint32_t n_tokens = ubatches_write.empty() ? ubatch.n_tokens : ubatches_write[i_next].n_tokens; + + ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens); + ggml_set_input(k_idxs); + + return k_idxs; +} + +ggml_tensor * llama_kv_cache_dsv4_raw_context::build_input_k_rot(ggml_context * ctx) const { + return kv_swa->build_input_k_rot(ctx); +} + +void llama_kv_cache_dsv4_raw_context::set_input_k_idxs(ggml_tensor * dst) const { + kv_swa->set_input_k_idxs(dst, &ubatches_write[i_next], sinfos_write[i_next]); +} + +void llama_kv_cache_dsv4_raw_context::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const { + kv_swa->set_input_kq_mask(dst, ubatch, causal_attn); +} + +void llama_kv_cache_dsv4_raw_context::set_input_k_rot(ggml_tensor * dst) const { + kv_swa->set_input_k_rot(dst); +} + +// +// llama_kv_cache_dsv4_comp_context +// + +llama_kv_cache_dsv4_comp_context::llama_kv_cache_dsv4_comp_context(llama_kv_cache * kv) : kv(kv), n_kv(kv->get_size()) { + const uint32_t n_stream = kv->get_n_stream(); + + sinfos.resize(1); + sinfos[0].s0 = 0; + sinfos[0].s1 = n_stream - 1; + sinfos[0].idxs.resize(n_stream); + for (uint32_t s = 0; s < n_stream; ++s) { + sinfos[0].strm.push_back(s); + sinfos[0].idxs[s].resize(1, 0); + } +} + +llama_kv_cache_dsv4_comp_context::llama_kv_cache_dsv4_comp_context( + llama_kv_cache * kv, + slot_info_vec_t sinfos, + std::vector ubatches) : + kv(kv), + sinfos(std::move(sinfos)), + ubatches(std::move(ubatches)), + n_kv(kv->get_size()) { +} + +bool llama_kv_cache_dsv4_comp_context::next() { + if (ubatches.empty()) { + return true; + } + + if (++i_cur >= ubatches.size()) { + return false; + } + + return true; +} + +uint32_t llama_kv_cache_dsv4_comp_context::get_n_kv() const { + return n_kv; +} + +ggml_tensor * llama_kv_cache_dsv4_comp_context::get_k(ggml_context * ctx, int32_t il) const { + return kv->get_k(ctx, il, n_kv, sinfos[i_cur]); +} + +ggml_tensor * llama_kv_cache_dsv4_comp_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const { + return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]); +} + +ggml_tensor * llama_kv_cache_dsv4_comp_context::build_input_k_rot(ggml_context * ctx) const { + return kv->build_input_k_rot(ctx); +} + +void llama_kv_cache_dsv4_comp_context::set_input_k_rot(ggml_tensor * dst) const { + kv->set_input_k_rot(dst); +} + +// +// llama_kv_cache_dsv4_context +// + +llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(llama_memory_status status) : status(status) {} + +llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv) : + ctx_raw(std::make_unique(kv->get_raw())), + ctx_csa_mem(kv->get_csa()->init_full()), + ctx_hca_mem(kv->get_hca()->init_full()), + ctx_lid_mem(kv->get_lid()->init_full()), + ctx_csa(std::make_unique(kv->get_csa())), + ctx_hca(std::make_unique(kv->get_hca())), + ctx_lid(std::make_unique(kv->get_lid())), + csa_state(kv->get_csa_state()), + hca_state(kv->get_hca_state()), + lid_state(kv->get_lid_state()), + reserve_plans(true), + status(llama_memory_status_combine( + llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()), + llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status()))) { +} + +llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv, + llama_context * lctx, + bool optimize) : + ctx_raw(std::make_unique(kv->get_raw(), lctx, optimize)), + ctx_csa_mem(kv->get_csa()->init_update(lctx, optimize)), + ctx_hca_mem(kv->get_hca()->init_update(lctx, optimize)), + ctx_lid_mem(kv->get_lid()->init_update(lctx, optimize)), + ctx_csa(std::make_unique(kv->get_csa())), + ctx_hca(std::make_unique(kv->get_hca())), + ctx_lid(std::make_unique(kv->get_lid())), + csa_state(kv->get_csa_state()), + hca_state(kv->get_hca_state()), + lid_state(kv->get_lid_state()), + status(llama_memory_status_combine( + llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()), + llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status()))) { +} + +llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv, + slot_info_vec_t sinfos_raw_base_write, + slot_info_vec_t sinfos_raw_swa_write, + slot_info_vec_t sinfos_raw_swa_read, + std::vector ubatches, + std::vector ubatches_raw) : + ubatches(std::move(ubatches)), + plans_csa(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true, + kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream())), + plans_hca(dsv4_build_comp_plans(this->ubatches, DSV4_HCA_RATIO, false, + kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream())), + plans_lid(plans_csa), + ctx_raw(std::make_unique( + kv->get_raw(), + std::move(sinfos_raw_base_write), + std::move(sinfos_raw_swa_write), + std::move(sinfos_raw_swa_read), + this->ubatches, + std::move(ubatches_raw))), + ctx_csa_mem(nullptr), + ctx_hca_mem(nullptr), + ctx_lid_mem(nullptr), + ctx_csa(std::make_unique( + kv->get_csa(), + dsv4_build_comp_sinfos(this->ubatches, kv->get_csa()->get_n_stream()), + this->ubatches)), + ctx_hca(std::make_unique( + kv->get_hca(), + dsv4_build_comp_sinfos(this->ubatches, kv->get_hca()->get_n_stream()), + this->ubatches)), + ctx_lid(std::make_unique( + kv->get_lid(), + dsv4_build_comp_sinfos(this->ubatches, kv->get_lid()->get_n_stream()), + this->ubatches)), + csa_state(kv->get_csa_state()), + hca_state(kv->get_hca_state()), + lid_state(kv->get_lid_state()), + status(ctx_raw->get_status()) { +} + +llama_kv_cache_dsv4_context::~llama_kv_cache_dsv4_context() = default; + +bool llama_kv_cache_dsv4_context::next() { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + ctx_raw->next(); + ctx_csa->next(); + ctx_hca->next(); + ctx_lid->next(); + + if (++i_next >= ubatches.size()) { + return false; + } + + return true; +} + +bool llama_kv_cache_dsv4_context::apply() { + assert(!llama_memory_status_is_fail(status)); + + bool res = true; + + res = res & ctx_raw->apply(); + + return res; +} + +llama_memory_status llama_kv_cache_dsv4_context::get_status() const { + return status; +} + +const llama_ubatch & llama_kv_cache_dsv4_context::get_ubatch() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ubatches[i_next]; +} + +const llama_kv_cache_dsv4_raw_context * llama_kv_cache_dsv4_context::get_raw() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ctx_raw.get(); +} + +const llama_kv_cache_dsv4_comp_context * llama_kv_cache_dsv4_context::get_csa() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ctx_csa.get(); +} + +const llama_kv_cache_dsv4_comp_context * llama_kv_cache_dsv4_context::get_hca() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ctx_hca.get(); +} + +const llama_kv_cache_dsv4_comp_context * llama_kv_cache_dsv4_context::get_lid() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return ctx_lid.get(); +} + +const llama_dsv4_comp_state * llama_kv_cache_dsv4_context::get_csa_state() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return csa_state; +} + +const llama_dsv4_comp_state * llama_kv_cache_dsv4_context::get_hca_state() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return hca_state; +} + +const llama_dsv4_comp_state * llama_kv_cache_dsv4_context::get_lid_state() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + return lid_state; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_csa_plan() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + static const comp_plan empty; + if (plans_csa.empty()) { + return empty; + } + + return plans_csa[i_next]; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_hca_plan() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + static const comp_plan empty; + if (plans_hca.empty()) { + return empty; + } + + return plans_hca[i_next]; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_lid_plan() const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + static const comp_plan empty; + if (plans_lid.empty()) { + return empty; + } + + return plans_lid[i_next]; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_csa_plan(const llama_ubatch & ubatch) const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + if (!reserve_plans) { + return get_csa_plan(); + } + + reserve_plan_csa = dsv4_build_reserve_comp_plan( + ubatch, DSV4_CSA_RATIO, true, + csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream()); + + return reserve_plan_csa; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_hca_plan(const llama_ubatch & ubatch) const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + if (!reserve_plans) { + return get_hca_plan(); + } + + reserve_plan_hca = dsv4_build_reserve_comp_plan( + ubatch, DSV4_HCA_RATIO, false, + hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream()); + + return reserve_plan_hca; +} + +const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_lid_plan(const llama_ubatch & ubatch) const { + assert(status == LLAMA_MEMORY_STATUS_SUCCESS); + + if (!reserve_plans) { + return get_lid_plan(); + } + + reserve_plan_lid = dsv4_build_reserve_comp_plan( + ubatch, DSV4_CSA_RATIO, true, + lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream()); + + return reserve_plan_lid; +} diff --git a/examples/talk-llama/llama-kv-cache-dsv4.h b/examples/talk-llama/llama-kv-cache-dsv4.h new file mode 100644 index 00000000000..772b428cd79 --- /dev/null +++ b/examples/talk-llama/llama-kv-cache-dsv4.h @@ -0,0 +1,362 @@ +#pragma once + +#include "llama-kv-cache.h" +#include "llama-kv-cache-iswa.h" + +#include +#include +#include +#include + +class llama_dsv4_comp_state { +public: + llama_dsv4_comp_state( + const llama_model & model, + bool offload, + bool unified, + uint32_t n_seq_max, + uint32_t ratio, + uint32_t state_size, + uint32_t n_embd_state, + const char * name, + const llama_memory_i::layer_filter_cb & filter); + + void clear(bool data); + + uint32_t get_ratio() const; + uint32_t get_state_size() const; + uint32_t get_n_stream() const; + + std::map memory_breakdown() const; + + void state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const; + void state_read (llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags); + + ggml_tensor * get_kv (ggml_context * ctx, int32_t il) const; + ggml_tensor * get_score(ggml_context * ctx, int32_t il) const; + + ggml_tensor * cpy_kv (ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const; + ggml_tensor * cpy_score(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * idxs, int32_t il) const; + +private: + struct layer { + uint32_t il; + + ggml_tensor * kv; + ggml_tensor * score; + }; + + const uint32_t ratio; + const uint32_t state_size; + const uint32_t n_embd_state; + const uint32_t n_stream; + + std::vector> ctxs_bufs; + + std::vector layers; + + std::unordered_map map_layer_ids; + + size_t total_size() const; +}; + +// +// llama_kv_cache_dsv4 +// + +// DSV4 uses a normal raw/SWA token cache plus compressed K-only block caches. +// The compressed caches are storage only; DSV4-specific visibility and block +// planning are handled by llama_kv_cache_dsv4_context / llm_graph_input_dsv4. + +class llama_kv_cache_dsv4 : public llama_memory_i { +public: + llama_kv_cache_dsv4( + const llama_model & model, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse); + + ~llama_kv_cache_dsv4() = default; + + // + // llama_memory_i + // + + llama_memory_context_ptr init_batch( + llama_batch_allocr & balloc, + uint32_t n_ubatch, + bool embd_all) override; + + llama_memory_context_ptr init_full() override; + + llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override; + + bool get_can_shift() const override; + + void clear(bool data) override; + + bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; + void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override; + void seq_keep(llama_seq_id seq_id) override; + void seq_add (llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) override; + void seq_div (llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) override; + + llama_pos seq_pos_min(llama_seq_id seq_id) const override; + llama_pos seq_pos_max(llama_seq_id seq_id) const override; + + std::map memory_breakdown() const override; + + void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; + void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; + + // + // llama_kv_cache_dsv4 specific API + // + + llama_kv_cache_iswa * get_raw() const; + llama_kv_cache * get_csa() const; + llama_kv_cache * get_hca() const; + llama_kv_cache * get_lid() const; + llama_dsv4_comp_state * get_csa_state() const; + llama_dsv4_comp_state * get_hca_state() const; + llama_dsv4_comp_state * get_lid_state() const; + +private: + llama_hparams hparams_raw; + llama_hparams hparams_csa; + llama_hparams hparams_hca; + llama_hparams hparams_lid; + + const uint32_t n_seq_max; + + std::unique_ptr kv_raw; + std::unique_ptr kv_csa; + std::unique_ptr kv_hca; + std::unique_ptr kv_lid; + std::unique_ptr csa_state; + std::unique_ptr hca_state; + std::unique_ptr lid_state; + + void clear_compressed(bool data); +}; + +// DSV4 raw attention only uses the SWA half of kv_raw. The base half is kept +// for generic ISWA bookkeeping, but it has no DSV4 layers to expose here. +class llama_kv_cache_dsv4_raw_context : public llama_memory_context_i { +public: + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + llama_kv_cache_dsv4_raw_context(llama_kv_cache_iswa * kv); + + llama_kv_cache_dsv4_raw_context( + llama_kv_cache_iswa * kv, + llama_context * lctx, + bool optimize); + + llama_kv_cache_dsv4_raw_context( + llama_kv_cache_iswa * kv, + slot_info_vec_t sinfos_base_write, + slot_info_vec_t sinfos_swa_write, + slot_info_vec_t sinfos_swa_read, + std::vector ubatches, + std::vector ubatches_write); + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + uint32_t get_n_kv() const; + uint32_t get_n_write() const; + + ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; + ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const; + + ggml_tensor * build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const; + ggml_tensor * build_input_k_rot(ggml_context * ctx) const; + + void set_input_k_idxs(ggml_tensor * dst) const; + void set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const; + void set_input_k_rot(ggml_tensor * dst) const; + +private: + size_t i_next = 0; + + llama_kv_cache * kv_swa = nullptr; + + slot_info_vec_t sinfos_write; + slot_info_vec_t sinfos_read; + std::vector ubatches; + std::vector ubatches_write; + + const llama_memory_context_ptr ctx_base_mem; + const llama_memory_context_ptr ctx_swa_mem; + + uint32_t n_kv = 0; + + const llama_memory_status status; +}; + +// DSV4 compressed KV rows are graph outputs, not normal token KV writes. +// Keep a small context that exposes K tensors without generic apply() semantics. +class llama_kv_cache_dsv4_comp_context { +public: + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + llama_kv_cache_dsv4_comp_context(llama_kv_cache * kv); + + llama_kv_cache_dsv4_comp_context( + llama_kv_cache * kv, + slot_info_vec_t sinfos, + std::vector ubatches); + + bool next(); + + uint32_t get_n_kv() const; + + ggml_tensor * get_k(ggml_context * ctx, int32_t il) const; + ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const; + + ggml_tensor * build_input_k_rot(ggml_context * ctx) const; + void set_input_k_rot(ggml_tensor * dst) const; + +private: + llama_kv_cache * kv; + + size_t i_cur = 0; + slot_info_vec_t sinfos; + std::vector ubatches; + + uint32_t n_kv; +}; + +class llama_kv_cache_dsv4_context : public llama_memory_context_i { +public: + using slot_info_vec_t = llama_kv_cache::slot_info_vec_t; + + struct comp_plan { + // Per-ubatch recipe for updating compressor state, committing completed + // compressed rows, and masking the compressed attention source. + + // APE row ids, i.e. pos % ratio, for the compressor-state updates. + std::vector state_pos; + + // Current-ubatch source row ids and unique persistent-state + // destination row ids for deterministic ring-state updates. + std::vector state_persist_src_idxs; + std::vector state_persist_dst_idxs; + + // Flattened source row ids used for state-backed commits. Source rows + // index the graph-local [persistent_state | current_ubatch_scratch] + // tensor. For overlapped compression the first half is previous rows + // and the second half is current rows; a final synthetic zero/-inf row + // may be addressed for the first block's previous half. + std::vector state_read_idxs; + + // Final compressed-cache row ids written by state-backed commits. + // A non-boundary CSA/LID decode step can target a masked scratch row. + std::vector state_write_idxs; + + // RoPE positions for state-backed commits. + std::vector state_write_pos; + + // Number of completed compressed rows visible for each query token. + std::vector n_visible; + + // Number of streams used by the attention graph for this ubatch. + int64_t n_stream = 1; + + // Graph-width for compressed rows. This can be larger than n_visible + // so masked padding rows do not force a new graph at every CSA block. + int64_t n_kv = 0; + }; + + llama_kv_cache_dsv4_context(llama_memory_status status); + + llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv); + + llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv, + llama_context * lctx, + bool optimize); + + llama_kv_cache_dsv4_context( + llama_kv_cache_dsv4 * kv, + slot_info_vec_t sinfos_raw_base_write, + slot_info_vec_t sinfos_raw_swa_write, + slot_info_vec_t sinfos_raw_swa_read, + std::vector ubatches, + std::vector ubatches_raw); + + virtual ~llama_kv_cache_dsv4_context(); + + // + // llama_memory_context_i + // + + bool next() override; + bool apply() override; + + llama_memory_status get_status() const override; + const llama_ubatch & get_ubatch() const override; + + // + // llama_kv_cache_dsv4_context specific API + // + + const llama_kv_cache_dsv4_raw_context * get_raw() const; + const llama_kv_cache_dsv4_comp_context * get_csa() const; + const llama_kv_cache_dsv4_comp_context * get_hca() const; + const llama_kv_cache_dsv4_comp_context * get_lid() const; + const llama_dsv4_comp_state * get_csa_state() const; + const llama_dsv4_comp_state * get_hca_state() const; + const llama_dsv4_comp_state * get_lid_state() const; + + const comp_plan & get_csa_plan() const; + const comp_plan & get_hca_plan() const; + const comp_plan & get_lid_plan() const; + + const comp_plan & get_csa_plan(const llama_ubatch & ubatch) const; + const comp_plan & get_hca_plan(const llama_ubatch & ubatch) const; + const comp_plan & get_lid_plan(const llama_ubatch & ubatch) const; + +private: + size_t i_next = 0; + + std::vector ubatches; + + std::vector plans_csa; + std::vector plans_hca; + std::vector plans_lid; + + const std::unique_ptr ctx_raw; + const llama_memory_context_ptr ctx_csa_mem; + const llama_memory_context_ptr ctx_hca_mem; + const llama_memory_context_ptr ctx_lid_mem; + + const std::unique_ptr ctx_csa; + const std::unique_ptr ctx_hca; + const std::unique_ptr ctx_lid; + + const llama_dsv4_comp_state * csa_state = nullptr; + const llama_dsv4_comp_state * hca_state = nullptr; + const llama_dsv4_comp_state * lid_state = nullptr; + + bool reserve_plans = false; + mutable comp_plan reserve_plan_csa; + mutable comp_plan reserve_plan_hca; + mutable comp_plan reserve_plan_lid; + + const llama_memory_status status; +}; diff --git a/examples/talk-llama/llama-kv-cache-iswa.cpp b/examples/talk-llama/llama-kv-cache-iswa.cpp index aa1b1b72ebe..e91866469ad 100644 --- a/examples/talk-llama/llama-kv-cache-iswa.cpp +++ b/examples/talk-llama/llama-kv-cache-iswa.cpp @@ -26,7 +26,28 @@ llama_kv_cache_iswa::llama_kv_cache_iswa( llama_memory_t mem_other, const layer_filter_cb & filter, const layer_reuse_cb & reuse, - const layer_share_cb & share) : hparams(model.hparams), unified(unified) { + const layer_share_cb & share) : + llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified, + kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) { +} + +llama_kv_cache_iswa::llama_kv_cache_iswa( + const llama_model & model, + const llama_hparams & hparams, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + llama_memory_t mem_other, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, + const layer_share_cb & share) : unified(unified) { // chain filters const layer_filter_cb filter_base = [&](int32_t il) { @@ -185,7 +206,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & ba std::vector ubatches; while (true) { - auto ubatch = balloc.split_equal(n_ubatch, !unified); + auto ubatch = balloc.split_equal(n_ubatch, !unified, 0); if (ubatch.n_tokens == 0) { break; diff --git a/examples/talk-llama/llama-kv-cache-iswa.h b/examples/talk-llama/llama-kv-cache-iswa.h index dfafc1ef510..7dab6eaa82c 100644 --- a/examples/talk-llama/llama-kv-cache-iswa.h +++ b/examples/talk-llama/llama-kv-cache-iswa.h @@ -30,6 +30,24 @@ class llama_kv_cache_iswa : public llama_memory_i { const layer_reuse_cb & reuse, const layer_share_cb & share); + llama_kv_cache_iswa( + const llama_model & model, + const llama_hparams & hparams, + ggml_type type_k, + ggml_type type_v, + bool v_trans, + bool offload, + bool swa_full, + bool unified, + uint32_t kv_size, + uint32_t n_seq_max, + uint32_t n_ubatch, + uint32_t n_pad, + llama_memory_t mem_other, + const layer_filter_cb & filter, + const layer_reuse_cb & reuse, + const layer_share_cb & share); + ~llama_kv_cache_iswa() = default; // @@ -73,8 +91,6 @@ class llama_kv_cache_iswa : public llama_memory_i { llama_kv_cache * get_swa () const; private: - const llama_hparams & hparams; - const bool unified; std::unique_ptr kv_base; diff --git a/examples/talk-llama/llama-kv-cache.cpp b/examples/talk-llama/llama-kv-cache.cpp index 2802103bdd8..e70583e6415 100644 --- a/examples/talk-llama/llama-kv-cache.cpp +++ b/examples/talk-llama/llama-kv-cache.cpp @@ -57,22 +57,6 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) { } } -static ggml_tensor * ggml_mul_mat_aux( - ggml_context * ctx, - ggml_tensor * cur, - ggml_tensor * rot) { - const auto n = rot->ne[0]; - - ggml_tensor * res; - - res = ggml_reshape_2d(ctx, cur, n, ggml_nelements(cur)/n); - res = ggml_mul_mat (ctx, rot, res); - ggml_mul_mat_set_hint(res, GGML_HINT_SRC0_IS_HADAMARD); - res = ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]); - - return res; -} - // // llama_kv_cache // @@ -211,10 +195,12 @@ llama_kv_cache::llama_kv_cache( n_embd_head_k_all = -1; } - if (n_embd_head_v_all == 0) { - n_embd_head_v_all = (int32_t) hparams.n_embd_head_v(il); - } else if (n_embd_head_v_all > 0 && n_embd_head_v_all != (int32_t) hparams.n_embd_head_v(il)) { - n_embd_head_v_all = -1; + if (!is_mla) { + if (n_embd_head_v_all == 0) { + n_embd_head_v_all = (int32_t) hparams.n_embd_head_v(il); + } else if (n_embd_head_v_all > 0 && n_embd_head_v_all != (int32_t) hparams.n_embd_head_v(il)) { + n_embd_head_v_all = -1; + } } // [TAG_V_CACHE_VARIABLE] @@ -336,8 +322,9 @@ llama_kv_cache::llama_kv_cache( ggml_is_quantized(type_k) && hparams.n_embd_head_k() % 64 == 0; - // always create Hadamard rotation tensors for DeepSeek V3.2 DSA lightning indexer - if (model.arch == LLM_ARCH_DEEPSEEK32 && hparams.n_embd_head_k_full == hparams.indexer_head_size) { + // always create Hadamard rotation tensors for DeepSeek lightning indexers + if ((model.arch == LLM_ARCH_DEEPSEEK32 || model.arch == LLM_ARCH_DEEPSEEK4) && + hparams.n_embd_head_k_full == hparams.indexer_head_size) { attn_rot_k = true; } @@ -719,7 +706,7 @@ llama_memory_context_ptr llama_kv_cache::init_batch( std::vector ubatches; while (true) { - auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true); + auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0); if (ubatch.n_tokens == 0) { break; @@ -1220,6 +1207,23 @@ ggml_type llama_kv_cache::type_v() const { return layers[0].v->type; } +std::vector llama_kv_cache::get_layer_ids() const { + std::vector res; + res.reserve(layers.size()); + + for (const auto & layer : layers) { + res.push_back(layer.il); + } + + return res; +} + +ggml_tensor * llama_kv_cache::get_k_storage(int32_t il) const { + const int32_t ikv = map_layer_ids.at(il); + + return layers[ikv].k; +} + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { uint32_t result = 0; @@ -1855,14 +1859,14 @@ ggml_tensor * llama_kv_cache::build_rope_shift( tmp = ggml_cast(ctx, cur, GGML_TYPE_F32); // rotate back - tmp = ggml_mul_mat_aux(ctx, tmp, rot); + tmp = llama_mul_mat_hadamard(ctx, tmp, rot); tmp = ggml_rope_ext(ctx, tmp, shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow); // rotate fwd - tmp = ggml_mul_mat_aux(ctx, tmp, rot); + tmp = llama_mul_mat_hadamard(ctx, tmp, rot); tmp = ggml_cpy(ctx, tmp, cur); } else { diff --git a/examples/talk-llama/llama-kv-cache.h b/examples/talk-llama/llama-kv-cache.h index 3d68f98c142..531d99dbdec 100644 --- a/examples/talk-llama/llama-kv-cache.h +++ b/examples/talk-llama/llama-kv-cache.h @@ -161,6 +161,9 @@ class llama_kv_cache : public llama_memory_i { ggml_type type_k() const; ggml_type type_v() const; + std::vector get_layer_ids() const; + ggml_tensor * get_k_storage(int32_t il) const; + // // graph_build API // diff --git a/examples/talk-llama/llama-memory-hybrid-iswa.cpp b/examples/talk-llama/llama-memory-hybrid-iswa.cpp index c7d4bcd413e..06f7fd5428c 100644 --- a/examples/talk-llama/llama-memory-hybrid-iswa.cpp +++ b/examples/talk-llama/llama-memory-hybrid-iswa.cpp @@ -77,15 +77,15 @@ llama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr // if all tokens are output, split by sequence ubatch = balloc.split_seq(n_ubatch); } else { - if (mem_recr->n_rs_seq > 0) { - // [TAG_RECURRENT_ROLLBACK_SPLITS] - // TODO: recurrent state rollback does not support equal splits - ubatch = balloc.split_seq(n_ubatch); - } else { - // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) - const bool unified = (mem_attn->get_base()->get_n_stream() == 1); - ubatch = balloc.split_equal(n_ubatch, !unified); - } + // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) + const bool unified = (mem_attn->get_base()->get_n_stream() == 1); + + // [TAG_RECURRENT_ROLLBACK_SPLITS] + // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch + // so that the rollback snapshots remain valid + const uint32_t n_rs_seq = mem_recr->n_rs_seq; + + ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0); } if (ubatch.n_tokens == 0) { diff --git a/examples/talk-llama/llama-memory-hybrid.cpp b/examples/talk-llama/llama-memory-hybrid.cpp index f2d49cbce54..42c7381a9e6 100644 --- a/examples/talk-llama/llama-memory-hybrid.cpp +++ b/examples/talk-llama/llama-memory-hybrid.cpp @@ -78,15 +78,15 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba // if all tokens are output, split by sequence ubatch = balloc.split_seq(n_ubatch); } else { - if (mem_recr->n_rs_seq > 0) { - // [TAG_RECURRENT_ROLLBACK_SPLITS] - // TODO: recurrent state rollback does not support equal splits - ubatch = balloc.split_seq(n_ubatch); - } else { - // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) - const bool unified = (mem_attn->get_n_stream() == 1); - ubatch = balloc.split_equal(n_ubatch, !unified); - } + // Use non-sequential split when KV cache is unified (needed for hellaswag/winogrande/multiple-choice) + const bool unified = (mem_attn->get_n_stream() == 1); + + // [TAG_RECURRENT_ROLLBACK_SPLITS] + // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch + // so that the rollback snapshots remain valid + const uint32_t n_rs_seq = mem_recr->n_rs_seq; + + ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0); } if (ubatch.n_tokens == 0) { diff --git a/examples/talk-llama/llama-memory-recurrent.cpp b/examples/talk-llama/llama-memory-recurrent.cpp index 6a4892fb471..3d6c6db876b 100644 --- a/examples/talk-llama/llama-memory-recurrent.cpp +++ b/examples/talk-llama/llama-memory-recurrent.cpp @@ -416,15 +416,12 @@ llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & // if all tokens are output, split by sequence ubatch = balloc.split_seq(n_ubatch); } else { - if (n_rs_seq > 0) { - // [TAG_RECURRENT_ROLLBACK_SPLITS] - // TODO: recurrent state rollback does not support equal splits - ubatch = balloc.split_seq(n_ubatch); - } else { - // TODO: non-sequential equal split can be done if using unified KV cache - // for simplicity, we always use sequential equal split for now - ubatch = balloc.split_equal(n_ubatch, true); - } + // TODO: non-sequential equal split can be done if using unified KV cache + // for simplicity, we always use sequential equal split for now + // [TAG_RECURRENT_ROLLBACK_SPLITS] + // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch + // so that the rollback snapshots remain valid + ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0); } if (ubatch.n_tokens == 0) { diff --git a/examples/talk-llama/llama-model-loader.cpp b/examples/talk-llama/llama-model-loader.cpp index 474cabdfc09..28f8bb7934b 100644 --- a/examples/talk-llama/llama-model-loader.cpp +++ b/examples/talk-llama/llama-model-loader.cpp @@ -27,52 +27,55 @@ const char * llama_file_version_name(llama_fver version) { return "unknown"; } -static std::string llama_model_ftype_name(llama_ftype ftype) { - if (ftype & LLAMA_FTYPE_GUESSED) { - return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)"; - } - - switch (ftype) { - case LLAMA_FTYPE_ALL_F32: return "all F32"; - case LLAMA_FTYPE_MOSTLY_F16: return "F16"; - case LLAMA_FTYPE_MOSTLY_BF16: return "BF16"; - case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0"; - case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0"; - case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1"; - case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0"; - case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1"; - case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0"; - case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE"; - case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4"; - case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large"; - case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K"; - case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary"; - case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary"; - case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw"; - case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw"; - - default: return "unknown, may not work"; +#define LLAMA_FTYPE_PREFIX "(guessed) " + +const char * llama_ftype_name(llama_ftype ftype) { + static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1; + const char * name; + switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) { + case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break; + case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break; + case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break; + case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break; + case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break; + case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break; + case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break; + case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break; + case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break; + case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break; + case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break; + case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break; + case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break; + case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break; + case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break; + default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break; } + return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len; } +#undef LLAMA_FTYPE_PREFIX + // return a list of splits for a given path // for example, given "-00002-of-00004.gguf", returns list of all 4 splits static std::vector llama_get_list_splits(const std::string & path, const int idx, const int n_split) { @@ -294,6 +297,8 @@ namespace GGUFMeta { } template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required); + template std::enable_if::value, bool>::type + llama_model_loader::get_arr_n(const std::string & key, uint32_t & result, bool required); template bool llama_model_loader::get_arr(const std::string & key, std::vector & result, bool required) { @@ -395,6 +400,7 @@ namespace GGUFMeta { template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); + template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_key(const std::string & key, T & result, bool required) { @@ -762,6 +768,7 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; + case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); @@ -1690,12 +1697,12 @@ bool llama_model_loader::load_all_data( } std::string llama_model_loader::ftype_name() const { - return llama_model_ftype_name(ftype); + return llama_ftype_name(ftype); } void llama_model_loader::print_info() const { LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver)); - LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str()); + LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_ftype_name(ftype)); if (n_bytes < GiB) { LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements); } else { diff --git a/examples/talk-llama/llama-model.cpp b/examples/talk-llama/llama-model.cpp index 6cb0ec3791c..adacf702d05 100644 --- a/examples/talk-llama/llama-model.cpp +++ b/examples/talk-llama/llama-model.cpp @@ -11,6 +11,7 @@ #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" +#include "llama-kv-cache-dsv4.h" #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" #include "llama-memory-recurrent.h" @@ -181,6 +182,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_deepseek2ocr(params); case LLM_ARCH_DEEPSEEK32: return new llama_model_deepseek32(params); + case LLM_ARCH_DEEPSEEK4: + return new llama_model_deepseek4(params); case LLM_ARCH_GLM_DSA: return new llama_model_glm_dsa(params); case LLM_ARCH_MISTRAL4: @@ -291,6 +294,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_mistral3(params); case LLM_ARCH_EAGLE3: return new llama_model_eagle3(params); + case LLM_ARCH_DFLASH: + return new llama_model_dflash(params); case LLM_ARCH_MIMO2: return new llama_model_mimo2(params); case LLM_ARCH_KIMI_LINEAR: @@ -815,6 +820,7 @@ static const char * llama_expert_gating_func_name(llama_expert_gating_func_type switch (type) { case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax"; case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid"; + case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: return "sqrtsoftplus"; default: return "unknown"; } } @@ -947,6 +953,8 @@ static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode s if (buft != nullptr) { buft_list.emplace_back(dev, buft); } + } else { + throw std::runtime_error(format("device %s does not support split buffers", ggml_backend_dev_name(dev))); } } @@ -981,6 +989,8 @@ struct llama_model::impl { std::string desc_str; + llama_ftype ftype = LLAMA_FTYPE_ALL_F32; + // model memory mapped files llama_mmaps mappings; @@ -1004,9 +1014,17 @@ struct llama_model::impl { std::vector dev_layer; bool has_tensor_overrides; + + std::vector tensor_split_owned; }; llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique()) { + if (params.tensor_split != nullptr) { + // llama_model_params stores tensor_split as a borrowed pointer, but the model + // may need it later for tensor-parallel KV-cache split metadata. + pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices()); + this->params.tensor_split = pimpl->tensor_split_owned.data(); + } pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern; } @@ -1194,6 +1212,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name(); + pimpl->ftype = ml.ftype; + if (hparams.f_max_alibi_bias > 0.0f) { hparams.use_alibi = true; } @@ -1640,6 +1660,10 @@ std::string llama_model::desc() const { return pimpl->desc_str; } +llama_ftype llama_model::ftype() const { + return pimpl->ftype; +} + size_t llama_model::size() const { return pimpl->n_bytes; } @@ -2154,7 +2178,24 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, } } - if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { + if (arch == LLM_ARCH_DEEPSEEK4) { + GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE); + + res = new llama_kv_cache_dsv4( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + params.swa_full, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + cparams.n_ubatch, + 1, + filter, + reuse); + } else if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { GGML_ASSERT(hparams.is_swa_any()); if (arch == LLM_ARCH_GEMMA4_ASSISTANT) { @@ -2326,6 +2367,11 @@ int32_t llama_model_n_head_kv(const llama_model * model) { } int32_t llama_model_n_swa(const llama_model * model) { + // dsv4 kv-cache has SWA but it cannot be used as a rollback because of + // other compression ratios, so we return 0 here + if (model->arch == LLM_ARCH_DEEPSEEK4) { + return 0; + } return model->hparams.n_swa; } @@ -2407,6 +2453,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DEEPSEEK2: case LLM_ARCH_DEEPSEEK2OCR: case LLM_ARCH_DEEPSEEK32: + case LLM_ARCH_DEEPSEEK4: case LLM_ARCH_PLM: case LLM_ARCH_CHATGLM: case LLM_ARCH_GRANITE: @@ -2494,6 +2541,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_STEP35: case LLM_ARCH_TALKIE: case LLM_ARCH_MELLUM: + case LLM_ARCH_DFLASH: return LLAMA_ROPE_TYPE_NEOX; case LLM_ARCH_QWEN2VL: @@ -2586,6 +2634,10 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) return snprintf(buf, buf_size, "%s", model->desc().c_str()); } +llama_ftype llama_model_ftype(const llama_model * model) { + return model->ftype(); +} + uint64_t llama_model_size(const llama_model * model) { return model->size(); } @@ -2617,7 +2669,8 @@ bool llama_model_has_encoder(const llama_model * model) { switch (model->arch) { case LLM_ARCH_T5: case LLM_ARCH_T5ENCODER: - case LLM_ARCH_EAGLE3: return true; + case LLM_ARCH_EAGLE3: + case LLM_ARCH_DFLASH: return true; default: return false; } } diff --git a/examples/talk-llama/llama-model.h b/examples/talk-llama/llama-model.h index 77d8d3b6258..45b054cedf1 100644 --- a/examples/talk-llama/llama-model.h +++ b/examples/talk-llama/llama-model.h @@ -255,9 +255,11 @@ struct llama_layer { struct ggml_tensor * wq_b = nullptr; struct ggml_tensor * wkv_a_mqa = nullptr; struct ggml_tensor * wkv_b = nullptr; + struct ggml_tensor * wkv = nullptr; struct ggml_tensor * wk_b = nullptr; struct ggml_tensor * wv_b = nullptr; struct ggml_tensor * wqkv_b = nullptr; + struct ggml_tensor * wo_a = nullptr; struct ggml_tensor * wo_b = nullptr; struct ggml_tensor * wq_cross = nullptr; struct ggml_tensor * wk_cross = nullptr; @@ -333,6 +335,7 @@ struct llama_layer { struct ggml_tensor * ffn_up_b = nullptr; // b3 struct ggml_tensor * ffn_act = nullptr; struct ggml_tensor * ffn_exp_probs_b = nullptr; + struct ggml_tensor * ffn_gate_tid2eid = nullptr; // mamba proj struct ggml_tensor * ssm_in = nullptr; @@ -463,6 +466,23 @@ struct llama_layer { // openai-moe struct ggml_tensor * attn_sinks = nullptr; + // DeepSeek-V4 + struct ggml_tensor * attn_kv_norm = nullptr; + struct ggml_tensor * hc_attn_fn = nullptr; + struct ggml_tensor * hc_attn_base = nullptr; + struct ggml_tensor * hc_attn_scale = nullptr; + struct ggml_tensor * hc_ffn_fn = nullptr; + struct ggml_tensor * hc_ffn_base = nullptr; + struct ggml_tensor * hc_ffn_scale = nullptr; + struct ggml_tensor * attn_comp_wkv = nullptr; + struct ggml_tensor * attn_comp_wgate = nullptr; + struct ggml_tensor * attn_comp_ape = nullptr; + struct ggml_tensor * attn_comp_norm = nullptr; + struct ggml_tensor * indexer_comp_wkv = nullptr; + struct ggml_tensor * indexer_comp_wgate = nullptr; + struct ggml_tensor * indexer_comp_ape = nullptr; + struct ggml_tensor * indexer_comp_norm = nullptr; + // cogvlm struct ggml_tensor * visexp_attn_wqkv = nullptr; struct ggml_tensor * visexp_attn_wo = nullptr; @@ -553,6 +573,11 @@ struct llama_model { struct ggml_tensor * nextn_proj_pre = nullptr; struct ggml_tensor * nextn_proj_post = nullptr; + // DeepSeek-V4 + struct ggml_tensor * hc_head_fn = nullptr; + struct ggml_tensor * hc_head_base = nullptr; + struct ggml_tensor * hc_head_scale = nullptr; + // classifier struct ggml_tensor * cls = nullptr; struct ggml_tensor * cls_b = nullptr; @@ -612,6 +637,8 @@ struct llama_model { std::string desc() const; + llama_ftype ftype() const; + size_t size() const; // file size size_t n_tensors() const; size_t n_devices() const; diff --git a/examples/talk-llama/llama-quant.cpp b/examples/talk-llama/llama-quant.cpp index 847e79f4655..aebbc1ffb6f 100644 --- a/examples/talk-llama/llama-quant.cpp +++ b/examples/talk-llama/llama-quant.cpp @@ -380,6 +380,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: // types on the right: block size 32 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break; + case GGML_TYPE_Q2_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_TQ1_0: @@ -480,7 +481,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_IQ3_S; } - else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0) { + else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) { new_type = GGML_TYPE_Q4_K; } } @@ -800,6 +801,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16; case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32; case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0; + case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0; case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4; diff --git a/examples/talk-llama/llama-vocab.cpp b/examples/talk-llama/llama-vocab.cpp index 6e78a3f6c0e..fdd447147d4 100644 --- a/examples/talk-llama/llama-vocab.cpp +++ b/examples/talk-llama/llama-vocab.cpp @@ -887,9 +887,6 @@ struct llm_tokenizer_ugm : llm_tokenizer { // blob containing XOR-compressed compact double array (XCDA) entries uint32_t xcda_blob_size = *(const uint32_t *) &precompiled_charsmap[0]; charsmap_offset += sizeof(xcda_blob_size); - if (xcda_blob_size + charsmap_offset >= precompiled_charsmap.size()) { - throw std::runtime_error("Index out of array bounds in precompiled charsmap!"); - } // Next xcda_blob_size bytes contain entries of XOR-compressed compact // double array (XCDA). Each entry is bit-packed into a 32-bit integer. @@ -1205,7 +1202,15 @@ struct llm_tokenizer_ugm_session { throw std::runtime_error("Index out of array bounds in precompiled charsmap!"); } const char * prefix_replacement = &(tokenizer.prefix_replacements)[longest_prefix_offset]; - return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length }; + size_t max_len = tokenizer.prefix_replacements_size - longest_prefix_offset; + size_t repl_len = 0; + while (repl_len < max_len && prefix_replacement[repl_len] != '\0') { + repl_len++; + } + if (repl_len == max_len) { + throw std::runtime_error("Unterminated string in precompiled charsmap!"); + } + return { prefix_replacement, repl_len, longest_prefix_length }; } // check if the input prefix contains a valid sequence of UTF-8 code units @@ -2018,11 +2023,18 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx); const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx); precompiled_charsmap.assign(pc, pc + n_precompiled_charsmap); -#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - // correct endianness of data in precompiled_charsmap binary blob + if (precompiled_charsmap.size() < sizeof(uint32_t)) { + throw std::runtime_error("precompiled_charsmap too small for xcda_blob_size header!"); + } uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0]; +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ *xcda_blob_size = __builtin_bswap32(*xcda_blob_size); - assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap); +#endif + if (*xcda_blob_size + sizeof(uint32_t) >= precompiled_charsmap.size()) { + throw std::runtime_error("Index out of array bounds in precompiled charsmap!"); + } +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + // correct endianness of data in precompiled_charsmap binary blob size_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t); uint32_t * xcda_array = (uint32_t *) &precompiled_charsmap[sizeof(uint32_t)]; for (size_t i = 0; i < xcda_array_size; ++i) { diff --git a/examples/talk-llama/llama.h b/examples/talk-llama/llama.h index f723c9f60cf..a311ac20235 100644 --- a/examples/talk-llama/llama.h +++ b/examples/talk-llama/llama.h @@ -155,10 +155,14 @@ extern "C" { LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; + // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); + enum llama_rope_scaling_type { LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, LLAMA_ROPE_SCALING_TYPE_NONE = 0, @@ -606,6 +610,9 @@ extern "C" { // Get a string describing the model type LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size); + // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 + LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); + // Returns the total size of all the tensors in the model in bytes LLAMA_API uint64_t llama_model_size(const struct llama_model * model); diff --git a/examples/talk-llama/models/deepseek4.cpp b/examples/talk-llama/models/deepseek4.cpp new file mode 100644 index 00000000000..07aa477e1e4 --- /dev/null +++ b/examples/talk-llama/models/deepseek4.cpp @@ -0,0 +1,1165 @@ +#include "models.h" + +#include "llama-kv-cache-dsv4.h" + +#include +#include +#include +#include + +static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) { + if (ext_factor == 0.0f) { + return 1.0f; + } + + return 1.0f / (1.0f + 0.1f*logf(1.0f/freq_scale)); +} + +void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer()); + if (!ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer(), 0)) { + hparams.swiglu_clamp_shexp = hparams.swiglu_clamp_exp; + } + + ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head); + ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size); + ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k); + + ml.get_key(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); + ml.get_key(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + ml.get_key(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base); + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters); + ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps); + ml.get_key(LLM_KV_HASH_LAYER_COUNT, hparams.dsv4_hash_layer_count); + + uint32_t n_compress_ratios = 0; + ml.get_arr_n(LLM_KV_ATTENTION_COMPRESS_RATIOS, n_compress_ratios); + if (n_compress_ratios < hparams.n_layer()) { + throw std::runtime_error("DeepSeek-V4 compress_ratios is shorter than block_count"); + } + ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios); + + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + if (hparams.expert_gating_func != LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) { + throw std::runtime_error("DeepSeek-V4 loader currently expects sqrtsoftplus MoE scoring"); + } + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + hparams.set_swa_pattern(0); + + switch (hparams.n_layer()) { + case 43: type = LLM_TYPE_UNKNOWN; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_deepseek4::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t q_lora_rank = hparams.n_lora_q; + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_expert_shared = hparams.n_expert_shared; + + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t o_groups = hparams.dsv4_o_group_count; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t hc_mult = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc_mult * n_embd; + const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0); + hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0); + hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0); + layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0); + layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0); + layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0); + layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0); + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, 0); + layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0); + + layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); + layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0); + layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0); + layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0); + layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0); + layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0); + + const int64_t ratio = hparams.dsv4_compress_ratios[i]; + if (ratio != 0) { + const int64_t coff = ratio == 4 ? 2 : 1; + + layer.attn_comp_wkv = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV, "weight", i), {n_embd, coff * n_embd_head}, 0); + layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, coff * n_embd_head}, 0); + layer.attn_comp_ape = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_APE, "weight", i), {coff * n_embd_head, ratio}, 0); + layer.attn_comp_norm = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM, "weight", i), {n_embd_head}, 0); + + if (ratio == 4) { + const int64_t n_embd_indexer = hparams.indexer_head_size; + + layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, hparams.indexer_n_head}, 0); + layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, hparams.indexer_n_head * n_embd_indexer}, 0); + + layer.indexer_comp_wkv = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WKV, "weight", i), {n_embd, 2 * n_embd_indexer}, 0); + layer.indexer_comp_wgate = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_WGATE, "weight", i), {n_embd, 2 * n_embd_indexer}, 0); + layer.indexer_comp_ape = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_APE, "weight", i), {2 * n_embd_indexer, ratio}, 0); + layer.indexer_comp_norm = create_tensor(tn(LLM_TENSOR_INDEXER_COMPRESSOR_NORM, "weight", i), {n_embd_indexer}, 0); + } else if (ratio != 128) { + throw std::runtime_error("DeepSeek-V4 loader only supports compression ratios 0, 4, and 128"); + } + } + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + if ((uint32_t) i < hparams.dsv4_hash_layer_count) { + layer.ffn_gate_tid2eid = create_tensor(tn(LLM_TENSOR_FFN_GATE_TID2EID, "weight", i), {n_expert_used, n_vocab}, 0); + } else { + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + } + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd }, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0); + } +} + +std::unique_ptr llama_model_deepseek4::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +static size_t dsv4_elem_offset(const ggml_tensor * t, int64_t i) { + return ggml_row_size(t->type, i); +} + +static ggml_tensor * dsv4_view_1d(ggml_context * ctx, ggml_tensor * t, int64_t ne0, int64_t i0) { + return ggml_view_1d(ctx, t, ne0, dsv4_elem_offset(t, i0)); +} + +static ggml_tensor * dsv4_view_2d( + ggml_context * ctx, + ggml_tensor * t, + int64_t ne0, + int64_t ne1, + int64_t i0) { + return ggml_view_2d(ctx, t, ne0, ne1, t->nb[1], dsv4_elem_offset(t, i0)); +} + +static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, bool neg_inf) { + ggml_tensor * row = ggml_view_1d(ctx, t, t->ne[0], 0); + row = neg_inf ? ggml_scale_bias(ctx, row, 0.0f, -INFINITY) : ggml_scale(ctx, row, 0.0f); + row = ggml_reshape_2d(ctx, row, t->ne[0], 1); + + return ggml_concat(ctx, t, row, 1); +} + +static ggml_tensor * dsv4_with_zero_dep(ggml_context * ctx, ggml_tensor * t, ggml_tensor * dep) { + if (dep == nullptr) { + return t; + } + + ggml_tensor * zero = ggml_scale(ctx, ggml_sum(ctx, dep), 0.0f); + return ggml_add(ctx, t, zero); +} + +static constexpr int64_t DSV4_CSA_RATIO = 4; +static constexpr int64_t DSV4_HCA_RATIO = 128; + +static ggml_tensor * dsv4_hc_affine( + ggml_context * ctx, + ggml_tensor * x, + ggml_tensor * scale, + ggml_tensor * base) { + x = ggml_mul(ctx, x, scale); + x = ggml_add(ctx, x, base); + return x; +} + +ggml_tensor * llama_model_deepseek4::graph::build_hc_weighted_sum( + ggml_tensor * x, + ggml_tensor * weights) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[2]; + + ggml_tensor * acc = nullptr; + for (int64_t ih = 0; ih < hc; ++ih) { + ggml_tensor * xh = ggml_view_2d(ctx0, x, n_embd, nt, x->nb[2], ih*x->nb[1]); + ggml_tensor * wh = ggml_view_2d(ctx0, weights, 1, nt, weights->nb[1], ih*weights->nb[0]); + + ggml_tensor * cur = ggml_mul(ctx0, xh, wh); + acc = acc ? ggml_add(ctx0, acc, cur) : cur; + } + + return acc; +} + +ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn( + ggml_tensor * comb, + int il) const { + GGML_UNUSED(il); + + // comb is [dst_hc, src_hc, n_tokens]. Sinkhorn follows the reference: + // row softmax over dst, one column normalization, then repeated row/column normalization. + comb = ggml_soft_max(ctx0, comb); + + ggml_tensor * eps = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + eps = ggml_fill(ctx0, eps, hparams.dsv4_hc_eps); + + comb = ggml_add(ctx0, comb, eps); + + auto norm_cols = [&]() { + ggml_tensor * comb_src_dst = ggml_cont(ctx0, ggml_permute(ctx0, comb, 1, 0, 2, 3)); + ggml_tensor * col_sum = ggml_sum_rows(ctx0, comb_src_dst); + col_sum = ggml_add(ctx0, col_sum, eps); + col_sum = ggml_permute(ctx0, col_sum, 1, 0, 2, 3); + comb = ggml_div(ctx0, comb, col_sum); + }; + + auto norm_rows = [&]() { + ggml_tensor * row_sum = ggml_sum_rows(ctx0, comb); + row_sum = ggml_add(ctx0, row_sum, eps); + comb = ggml_div(ctx0, comb, row_sum); + }; + + norm_cols(); + for (uint32_t i = 1; i < hparams.dsv4_hc_sinkhorn_iters; ++i) { + norm_rows(); + norm_cols(); + } + + return comb; +} + +ggml_tensor * llama_model_deepseek4::graph::build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc*n_embd; + const int64_t hc_mix_dim = (2 + hc)*hc; + const int64_t nt = x->ne[2]; + + GGML_ASSERT(hc == 4); + GGML_ASSERT(hc_fn->ne[1] == hc_mix_dim); + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_mixes", il); + + ggml_tensor * scale_pre = dsv4_view_1d(ctx0, hc_scale, 1, 0); + ggml_tensor * scale_post = dsv4_view_1d(ctx0, hc_scale, 1, 1); + ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2); + + ggml_tensor * base_pre = dsv4_view_1d(ctx0, hc_base, hc, 0); + ggml_tensor * base_post = dsv4_view_1d(ctx0, hc_base, hc, hc); + ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc); + + ggml_tensor * pre = dsv4_view_2d(ctx0, mixes, hc, nt, 0); + pre = dsv4_hc_affine(ctx0, pre, scale_pre, base_pre); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_pre", il); + + *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc); + *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post); + *post = ggml_sigmoid(ctx0, *post); + *post = ggml_scale(ctx0, *post, 2.0f); + cb(*post, "hc_post", il); + + *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc); + *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb); + *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt); + *comb = build_hc_sinkhorn(*comb, il); + cb(*comb, "hc_comb", il); + + return build_hc_weighted_sum(x, pre); +} + +ggml_tensor * llama_model_deepseek4::graph::build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + ggml_tensor * comb, + int il) const { + GGML_UNUSED(il); + + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t nt = x->ne[1]; + + ggml_tensor * out = nullptr; + for (int64_t dst = 0; dst < hc; ++dst) { + ggml_tensor * post_dst = ggml_view_2d(ctx0, post, 1, nt, post->nb[1], dst*post->nb[0]); + ggml_tensor * cur = ggml_mul(ctx0, x, post_dst); + + for (int64_t src = 0; src < hc; ++src) { + ggml_tensor * res_src = ggml_view_2d(ctx0, residual, n_embd, nt, residual->nb[2], src*residual->nb[1]); + ggml_tensor * comb_src_dst = ggml_view_2d(ctx0, comb, 1, nt, comb->nb[2], dst*comb->nb[0] + src*comb->nb[1]); + cur = ggml_add(ctx0, cur, ggml_mul(ctx0, res_src, comb_src_dst)); + } + + cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, nt); + out = out ? ggml_concat(ctx0, out, cur, 1) : cur; + } + + return out; +} + +ggml_tensor * llama_model_deepseek4::graph::build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const { + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc*n_embd; + const int64_t nt = x->ne[2]; + + ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt); + ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps); + ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm); + cb(mixes, "hc_head_mixes", -1); + + ggml_tensor * pre = dsv4_hc_affine(ctx0, mixes, hc_scale, hc_base); + pre = ggml_sigmoid(ctx0, pre); + pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps); + cb(pre, "hc_head_pre", -1); + + return build_hc_weighted_sum(x, pre); +} + +ggml_tensor * llama_model_deepseek4::graph::build_hca_compressed_kv_from_state( + ggml_tensor * kv_state, + ggml_tensor * score_state, + ggml_tensor * state_read_idxs, + ggml_tensor * comp_pos, + ggml_tensor * norm, + int64_t n_embd_head, + const char * name, + int il) const { + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + const int64_t n_blocks = comp_pos ? comp_pos->ne[0] : 0; + + GGML_ASSERT(n_blocks > 0); + GGML_ASSERT(state_read_idxs); + GGML_ASSERT(state_read_idxs->ne[0] == DSV4_HCA_RATIO*n_blocks); + GGML_ASSERT(n_embd_head >= n_embd_head_rope); + + ggml_tensor * kv = ggml_get_rows(ctx0, kv_state, state_read_idxs); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, DSV4_HCA_RATIO, n_blocks); + cb(kv, name, il); + + ggml_tensor * score = ggml_get_rows(ctx0, score_state, state_read_idxs); + score = ggml_reshape_3d(ctx0, score, n_embd_head, DSV4_HCA_RATIO, n_blocks); + cb(score, name, il); + + ggml_tensor * values = ggml_cont(ctx0, ggml_permute(ctx0, kv, 1, 0, 2, 3)); + ggml_tensor * scores = ggml_cont(ctx0, ggml_permute(ctx0, score, 1, 0, 2, 3)); + + ggml_tensor * weights = ggml_soft_max(ctx0, scores); + ggml_tensor * comp = ggml_mul(ctx0, values, weights); + comp = ggml_sum_rows(ctx0, comp); + comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3)); + cb(comp, name, il); + + comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il); + cb(comp, name, il); + + ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks, + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head), + 0); + ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks, + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head_nope)); + + comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, + hparams.dsv4_compress_rope_base, freq_scale, ext_factor, + dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); + cb(comp_pe, name, il); + + comp = ggml_concat(ctx0, comp_nope, comp_pe, 0); + cb(comp, name, il); + + return comp; +} + +ggml_tensor * llama_model_deepseek4::graph::build_overlap_compressed_kv_from_state( + ggml_tensor * kv_state, + ggml_tensor * score_state, + ggml_tensor * state_read_idxs, + ggml_tensor * comp_pos, + ggml_tensor * norm, + int64_t ratio, + int64_t n_embd_head, + const char * name, + int il) const { + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + const int64_t n_blocks = comp_pos ? comp_pos->ne[0] : 0; + + GGML_ASSERT(n_blocks > 0); + GGML_ASSERT(state_read_idxs); + GGML_ASSERT(state_read_idxs->ne[0] == 2*ratio*n_blocks); + GGML_ASSERT(kv_state->ne[0] == 2*n_embd_head); + GGML_ASSERT(score_state->ne[0] == 2*n_embd_head); + GGML_ASSERT(n_embd_head >= n_embd_head_rope); + + kv_state = dsv4_append_zero_row(ctx0, kv_state, false); + score_state = dsv4_append_zero_row(ctx0, score_state, true); + + ggml_tensor * prev_idxs = dsv4_view_1d(ctx0, state_read_idxs, ratio*n_blocks, 0); + ggml_tensor * cur_idxs = dsv4_view_1d(ctx0, state_read_idxs, ratio*n_blocks, ratio*n_blocks); + + ggml_tensor * kv_prev = ggml_get_rows(ctx0, kv_state, prev_idxs); + kv_prev = ggml_cont(ctx0, ggml_view_2d(ctx0, kv_prev, n_embd_head, ratio*n_blocks, kv_prev->nb[1], 0)); + kv_prev = ggml_reshape_3d(ctx0, kv_prev, n_embd_head, ratio, n_blocks); + cb(kv_prev, name, il); + + ggml_tensor * score_prev = ggml_get_rows(ctx0, score_state, prev_idxs); + score_prev = ggml_cont(ctx0, ggml_view_2d(ctx0, score_prev, n_embd_head, ratio*n_blocks, score_prev->nb[1], 0)); + score_prev = ggml_reshape_3d(ctx0, score_prev, n_embd_head, ratio, n_blocks); + cb(score_prev, name, il); + + ggml_tensor * kv_cur = ggml_get_rows(ctx0, kv_state, cur_idxs); + kv_cur = ggml_cont(ctx0, ggml_view_2d(ctx0, kv_cur, n_embd_head, ratio*n_blocks, kv_cur->nb[1], + ggml_row_size(kv_cur->type, n_embd_head))); + kv_cur = ggml_reshape_3d(ctx0, kv_cur, n_embd_head, ratio, n_blocks); + + ggml_tensor * score_cur = ggml_get_rows(ctx0, score_state, cur_idxs); + score_cur = ggml_cont(ctx0, ggml_view_2d(ctx0, score_cur, n_embd_head, ratio*n_blocks, score_cur->nb[1], + ggml_row_size(score_cur->type, n_embd_head))); + score_cur = ggml_reshape_3d(ctx0, score_cur, n_embd_head, ratio, n_blocks); + + ggml_tensor * values = ggml_concat(ctx0, kv_prev, kv_cur, 1); + ggml_tensor * scores = ggml_concat(ctx0, score_prev, score_cur, 1); + + values = ggml_cont(ctx0, ggml_permute(ctx0, values, 1, 0, 2, 3)); + scores = ggml_cont(ctx0, ggml_permute(ctx0, scores, 1, 0, 2, 3)); + + ggml_tensor * weights = ggml_soft_max(ctx0, scores); + ggml_tensor * comp = ggml_mul(ctx0, values, weights); + comp = ggml_sum_rows(ctx0, comp); + comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3)); + cb(comp, name, il); + + comp = build_norm(comp, norm, nullptr, LLM_NORM_RMS, il); + cb(comp, name, il); + + ggml_tensor * comp_nope = ggml_view_3d(ctx0, comp, n_embd_head_nope, 1, n_blocks, + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head), + 0); + ggml_tensor * comp_pe = ggml_view_3d(ctx0, comp, n_embd_head_rope, 1, n_blocks, + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head), + ggml_row_size(comp->type, n_embd_head_nope)); + + comp_pe = ggml_rope_ext(ctx0, comp_pe, comp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig, + hparams.dsv4_compress_rope_base, freq_scale, ext_factor, + dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); + cb(comp_pe, name, il); + + comp = ggml_concat(ctx0, comp_nope, comp_pe, 0); + cb(comp, name, il); + + return comp; +} + +ggml_tensor * llama_model_deepseek4::graph::build_lid_top_k( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * qr, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const { + const auto & layer = model.layers[il]; + const auto & inp_lid = inp_dsv4->get_lid(); + const int64_t n_embd_indexer_head = hparams.indexer_head_size; + const int64_t n_embd_indexer_head_rope = hparams.n_rot(); + const int64_t n_embd_indexer_head_nope = n_embd_indexer_head - n_embd_indexer_head_rope; + const int64_t n_indexer_head = hparams.indexer_n_head; + const int64_t nt = cur->ne[1]; + + GGML_ASSERT(inp_lid.kq_mask); + GGML_ASSERT(inp_lid.k_rot); + GGML_ASSERT(n_embd_indexer_head >= n_embd_indexer_head_rope); + + ggml_tensor * indexer_q = build_lora_mm(layer.indexer_attn_q_b, qr); + indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, nt); + cb(indexer_q, "lid_q", il); + + ggml_tensor * indexer_q_nope = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_nope, n_indexer_head, nt, + ggml_row_size(indexer_q->type, n_embd_indexer_head), + ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head, + 0); + ggml_tensor * indexer_q_pe = ggml_view_3d(ctx0, indexer_q, n_embd_indexer_head_rope, n_indexer_head, nt, + ggml_row_size(indexer_q->type, n_embd_indexer_head), + ggml_row_size(indexer_q->type, n_embd_indexer_head)*n_indexer_head, + ggml_row_size(indexer_q->type, n_embd_indexer_head_nope)); + + indexer_q_pe = ggml_rope_ext(ctx0, indexer_q_pe, inp_pos, nullptr, n_embd_indexer_head_rope, + rope_type, n_ctx_orig, hparams.dsv4_compress_rope_base, freq_scale, + ext_factor, dsv4_rope_attn_factor(freq_scale, ext_factor), beta_fast, beta_slow); + cb(indexer_q_pe, "lid_q_pe", il); + + indexer_q = ggml_concat(ctx0, indexer_q_nope, indexer_q_pe, 0); + indexer_q = llama_mul_mat_hadamard(ctx0, indexer_q, inp_lid.k_rot); + cb(indexer_q, "lid_q_rot", il); + + ggml_tensor * indexer_weights = build_lora_mm(layer.indexer_proj, cur); + indexer_weights = ggml_scale(ctx0, indexer_weights, 1.0f/sqrtf(float(n_embd_indexer_head*n_indexer_head))); + cb(indexer_weights, "lid_weights", il); + + ggml_tensor * indexer_k = inp_dsv4->mctx->get_lid()->get_k(ctx0, il); + const int64_t n_lid = inp_lid.kq_mask->ne[0]; + GGML_ASSERT(n_lid > 0); + GGML_ASSERT(n_lid <= indexer_k->ne[2]); + + indexer_k = ggml_view_4d(ctx0, indexer_k, + indexer_k->ne[0], indexer_k->ne[1], n_lid, indexer_k->ne[3], + indexer_k->nb[1], indexer_k->nb[2], indexer_k->nb[3], 0); + cb(indexer_k, "lid_k", il); + + const int64_t n_stream = indexer_k->ne[3]; + indexer_q = ggml_view_4d(ctx0, indexer_q, + indexer_q->ne[0], indexer_q->ne[1], indexer_q->ne[2]/n_stream, n_stream, + indexer_q->nb[1], indexer_q->nb[2], indexer_q->nb[3]/n_stream, 0); + indexer_weights = ggml_view_4d(ctx0, indexer_weights, + indexer_weights->ne[0], indexer_weights->ne[1]/n_stream, indexer_weights->ne[2], n_stream, + indexer_weights->nb[1], indexer_weights->nb[2]/n_stream, indexer_weights->nb[3]/n_stream, 0); + + indexer_q = ggml_permute(ctx0, indexer_q, 0, 2, 1, 3); + cb(indexer_q, "lid_q", il); + indexer_k = ggml_permute(ctx0, indexer_k, 0, 2, 1, 3); + cb(indexer_k, "lid_k", il); + + ggml_tensor * indexer_kq = ggml_mul_mat(ctx0, indexer_k, indexer_q); + cb(indexer_kq, "lid_kq", il); + + indexer_kq = ggml_cont(ctx0, ggml_permute(ctx0, indexer_kq, 2, 1, 0, 3)); + cb(indexer_kq, "lid_kq", il); + + ggml_tensor * indexer_score = ggml_relu(ctx0, indexer_kq); + indexer_score = ggml_mul(ctx0, indexer_score, indexer_weights); + indexer_score = ggml_sum_rows(ctx0, indexer_score); + indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); + cb(indexer_score, "lid_score", il); + + indexer_score = ggml_add(ctx0, indexer_score, inp_lid.kq_mask); + cb(indexer_score, "lid_score_masked", il); + + const uint32_t n_top_k = indexer_score->ne[0] < hparams.indexer_top_k ? indexer_score->ne[0] : hparams.indexer_top_k; + ggml_tensor * top_k = ggml_cont(ctx0, ggml_top_k(ctx0, indexer_score, n_top_k)); + cb(top_k, "lid_top_k", il); + + return top_k; +} + +ggml_tensor * llama_model_deepseek4::graph::build_top_k_mask( + ggml_tensor * kq_mask, + ggml_tensor * top_k, + const char * name, + int il) const { + GGML_ASSERT(kq_mask); + GGML_ASSERT(top_k); + + ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY); + kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], + kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0); + + ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, + top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0); + + ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]); + zeros = ggml_fill(ctx0, zeros, 0.0f); + + ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d); + kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, + kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], + kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0); + + kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask); + cb(kq_mask_top_k, name, il); + + return kq_mask_top_k; +} + +ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * qr, + ggml_tensor * cur, + ggml_tensor * inp_pos, + ggml_tensor * sinks, + float kq_scale, + int il) const { + const auto & inp_csa = inp_dsv4->get_csa(); + GGML_ASSERT(inp_csa.kq_mask); + + ggml_tensor * top_k = build_lid_top_k(model, inp_dsv4, qr, cur, inp_pos, il); + + ggml_tensor * k_rot = inp_attn->self_k_rot; + if (k_rot) { + q = llama_mul_mat_hadamard(ctx0, q, k_rot); + kv = llama_mul_mat_hadamard(ctx0, kv, k_rot); + } + + ggml_build_forward_expand(gf, q); + ggml_build_forward_expand(gf, kv); + + const llama_kv_cache_dsv4_raw_context * mctx_raw = inp_attn->mctx; + + ggml_build_forward_expand(gf, mctx_raw->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); + + ggml_tensor * raw_k = mctx_raw->get_k(ctx0, il); + cb(raw_k, "csa_raw_k", il); + + ggml_tensor * csa_k = inp_dsv4->mctx->get_csa()->get_k(ctx0, il); + const int64_t n_csa = inp_csa.kq_mask->ne[0]; + GGML_ASSERT(n_csa > 0); + GGML_ASSERT(n_csa <= csa_k->ne[2]); + + csa_k = ggml_view_4d(ctx0, csa_k, + csa_k->ne[0], csa_k->ne[1], n_csa, csa_k->ne[3], + csa_k->nb[1], csa_k->nb[2], csa_k->nb[3], 0); + cb(csa_k, "csa_comp_k", il); + + ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2); + cb(k_all, "csa_k_all", il); + + ggml_tensor * raw_mask = inp_attn->get_kq_mask(); + ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il); + + ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0); + cb(kq_mask, "csa_lid_kq_mask", il); + + ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il); + if (k_rot) { + out = llama_mul_mat_hadamard(ctx0, out, k_rot); + } + cb(out, "attn_csa_lid", il); + + return out; +} + +ggml_tensor * llama_model_deepseek4::graph::build_hca_attention( + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * sinks, + float kq_scale, + int il) const { + const auto & inp_hca = inp_dsv4->get_hca(); + GGML_ASSERT(inp_hca.kq_mask); + + ggml_tensor * k_rot = inp_attn->self_k_rot; + if (k_rot) { + q = llama_mul_mat_hadamard(ctx0, q, k_rot); + kv = llama_mul_mat_hadamard(ctx0, kv, k_rot); + } + + ggml_build_forward_expand(gf, q); + ggml_build_forward_expand(gf, kv); + + const llama_kv_cache_dsv4_raw_context * mctx_raw = inp_attn->mctx; + + ggml_build_forward_expand(gf, mctx_raw->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); + + ggml_tensor * raw_k = mctx_raw->get_k(ctx0, il); + cb(raw_k, "hca_raw_k", il); + + ggml_tensor * hca_k = inp_dsv4->mctx->get_hca()->get_k(ctx0, il); + const int64_t n_hca = inp_hca.kq_mask->ne[0]; + GGML_ASSERT(n_hca > 0); + GGML_ASSERT(n_hca <= hca_k->ne[2]); + + hca_k = ggml_view_4d(ctx0, hca_k, + hca_k->ne[0], hca_k->ne[1], n_hca, hca_k->ne[3], + hca_k->nb[1], hca_k->nb[2], hca_k->nb[3], 0); + cb(hca_k, "hca_comp_k", il); + + ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2); + cb(k_all, "hca_k_all", il); + + ggml_tensor * raw_mask = inp_attn->get_kq_mask(); + ggml_tensor * hca_mask = inp_hca.kq_mask; + + ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0); + cb(kq_mask, "hca_kq_mask", il); + + ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il); + if (k_rot) { + out = llama_mul_mat_hadamard(ctx0, out, k_rot); + } + cb(out, "attn_hca", il); + + return out; +} + +ggml_tensor * llama_model_deepseek4::graph::build_raw_attention( + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * sinks, + float kq_scale, + int il) const { + GGML_ASSERT(hparams.is_swa(il)); + + ggml_tensor * k_rot = inp_attn->self_k_rot; + + if (k_rot) { + q = llama_mul_mat_hadamard(ctx0, q, k_rot); + kv = llama_mul_mat_hadamard(ctx0, kv, k_rot); + } + + ggml_build_forward_expand(gf, q); + ggml_build_forward_expand(gf, kv); + + const llama_kv_cache_dsv4_raw_context * mctx_cur = inp_attn->mctx; + + ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); + + ggml_tensor * kq_mask = inp_attn->get_kq_mask(); + + ggml_tensor * k = mctx_cur->get_k(ctx0, il); + + ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, kq_scale, il); + if (k_rot) { + out = llama_mul_mat_hadamard(ctx0, out, k_rot); + } + cb(out, "attn_raw", il); + + return out; +} + +ggml_tensor * llama_model_deepseek4::graph::build_attention( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const { + const auto & layer = model.layers[il]; + llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw(); + + const int64_t n_embd_head = hparams.n_embd_head_k(); + const int64_t n_embd_head_rope = hparams.n_rot(); + const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope; + const int64_t n_groups = hparams.dsv4_o_group_count; + const int64_t n_heads_group = n_head / n_groups; + const int64_t o_lora_rank = hparams.dsv4_o_lora_rank; + const int64_t o_group_dim = n_heads_group*n_embd_head; + const int64_t nt = cur->ne[1]; + + GGML_ASSERT(n_embd_head == n_embd_head_v); + GGML_ASSERT(n_head % n_groups == 0); + + const bool use_compress_rope = hparams.dsv4_compress_ratios[il] != 0; + const float freq_base_l = use_compress_rope ? hparams.dsv4_compress_rope_base : freq_base; + const float freq_scale_l = use_compress_rope ? freq_scale : 1.0f; + const float ext_factor_l = use_compress_rope ? ext_factor : 0.0f; + const float attn_factor_l = dsv4_rope_attn_factor(freq_scale_l, ext_factor_l); + const float beta_fast_l = use_compress_rope ? beta_fast : 0.0f; + const float beta_slow_l = use_compress_rope ? beta_slow : 0.0f; + const int32_t n_ctx_orig_l = use_compress_rope ? n_ctx_orig : 0; + + ggml_tensor * qr = build_lora_mm(layer.wq_a, cur); + cb(qr, "qr", il); + + qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(qr, "qr_norm", il); + + ggml_tensor * q = build_lora_mm(layer.wq_b, qr); + q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, nt); + q = ggml_rms_norm(ctx0, q, norm_rms_eps); + cb(q, "q_norm", il); + + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_nope, n_head, nt, + ggml_row_size(q->type, n_embd_head), + ggml_row_size(q->type, n_embd_head)*n_head, + 0); + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_rope, n_head, nt, + ggml_row_size(q->type, n_embd_head), + ggml_row_size(q->type, n_embd_head)*n_head, + ggml_row_size(q->type, n_embd_head_nope)); + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + cb(q_pe, "q_pe", il); + q = ggml_concat(ctx0, q_nope, q_pe, 0); + cb(q, "q", il); + + ggml_tensor * kv = build_lora_mm(layer.wkv, cur); + kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il); + kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt); + cb(kv, "kv_norm", il); + + ggml_tensor * kv_nope = ggml_view_3d(ctx0, kv, n_embd_head_nope, 1, nt, + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head), + 0); + ggml_tensor * kv_pe = ggml_view_3d(ctx0, kv, n_embd_head_rope, 1, nt, + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head), + ggml_row_size(kv->type, n_embd_head_nope)); + kv_pe = ggml_rope_ext(ctx0, kv_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + cb(kv_pe, "kv_pe", il); + kv = ggml_concat(ctx0, kv_nope, kv_pe, 0); + cb(kv, "kv", il); + + const int64_t ratio = hparams.dsv4_compress_ratios[il]; + + ggml_tensor * hca_state_kv = nullptr; + ggml_tensor * hca_state_score = nullptr; + if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) { + hca_state_kv = build_lora_mm(layer.attn_comp_wkv, cur); + cb(hca_state_kv, "hca_state_kv", il); + + hca_state_score = build_lora_mm(layer.attn_comp_wgate, cur); + cb(hca_state_score, "hca_state_score", il); + + ggml_tensor * ape = layer.attn_comp_ape; + + ggml_tensor * ape_rows = ggml_get_rows(ctx0, ape, inp_dsv4->get_hca().state_pos); + hca_state_score = ggml_add(ctx0, hca_state_score, ape_rows); + cb(hca_state_score, "hca_state_score_ape", il); + + } + + if (ratio == DSV4_CSA_RATIO && inp_dsv4->get_csa().state_pos) { + ggml_tensor * csa_state_kv = build_lora_mm(layer.attn_comp_wkv, cur); + cb(csa_state_kv, "csa_state_kv", il); + + ggml_tensor * csa_state_score = build_lora_mm(layer.attn_comp_wgate, cur); + cb(csa_state_score, "csa_state_score", il); + + ggml_tensor * csa_ape = layer.attn_comp_ape; + + ggml_tensor * csa_ape_rows = ggml_get_rows(ctx0, csa_ape, inp_dsv4->get_csa().state_pos); + csa_state_score = ggml_add(ctx0, csa_state_score, csa_ape_rows); + cb(csa_state_score, "csa_state_score_ape", il); + + GGML_ASSERT(inp_dsv4->get_csa().state_write_idxs); + + ggml_tensor * csa_source_kv = ggml_concat(ctx0, + inp_dsv4->mctx->get_csa_state()->get_kv(ctx0, il), csa_state_kv, 1); + ggml_tensor * csa_source_score = ggml_concat(ctx0, + inp_dsv4->mctx->get_csa_state()->get_score(ctx0, il), csa_state_score, 1); + + ggml_tensor * kv_comp_csa_state = build_overlap_compressed_kv_from_state( + csa_source_kv, + csa_source_score, + inp_dsv4->get_csa().state_read_idxs, + inp_dsv4->get_csa().state_write_pos, + layer.attn_comp_norm, + DSV4_CSA_RATIO, + n_embd_head, + "csa_state_compress", + il); + + if (inp_dsv4->get_csa().k_rot) { + kv_comp_csa_state = llama_mul_mat_hadamard(ctx0, kv_comp_csa_state, inp_dsv4->get_csa().k_rot); + cb(kv_comp_csa_state, "csa_state_compress_rot", il); + } + + ggml_build_forward_expand(gf, inp_dsv4->mctx->get_csa()->cpy_k(ctx0, + kv_comp_csa_state, inp_dsv4->get_csa().state_write_idxs, il)); + + csa_state_kv = dsv4_with_zero_dep(ctx0, csa_state_kv, kv_comp_csa_state); + csa_state_score = dsv4_with_zero_dep(ctx0, csa_state_score, kv_comp_csa_state); + + ggml_tensor * csa_persist_kv = ggml_get_rows(ctx0, csa_state_kv, inp_dsv4->get_csa().state_persist_src_idxs); + ggml_tensor * csa_persist_score = ggml_get_rows(ctx0, csa_state_score, inp_dsv4->get_csa().state_persist_src_idxs); + + csa_state_kv = inp_dsv4->mctx->get_csa_state()->cpy_kv(ctx0, + csa_persist_kv, inp_dsv4->get_csa().state_persist_dst_idxs, il); + csa_state_score = inp_dsv4->mctx->get_csa_state()->cpy_score(ctx0, + csa_persist_score, inp_dsv4->get_csa().state_persist_dst_idxs, il); + + ggml_build_forward_expand(gf, csa_state_kv); + ggml_build_forward_expand(gf, csa_state_score); + + ggml_tensor * lid_state_kv = build_lora_mm(layer.indexer_comp_wkv, cur); + cb(lid_state_kv, "lid_state_kv", il); + + ggml_tensor * lid_state_score = build_lora_mm(layer.indexer_comp_wgate, cur); + cb(lid_state_score, "lid_state_score", il); + + ggml_tensor * lid_ape = layer.indexer_comp_ape; + + ggml_tensor * lid_ape_rows = ggml_get_rows(ctx0, lid_ape, inp_dsv4->get_lid().state_pos); + lid_state_score = ggml_add(ctx0, lid_state_score, lid_ape_rows); + cb(lid_state_score, "lid_state_score_ape", il); + + GGML_ASSERT(inp_dsv4->get_lid().state_write_idxs); + + ggml_tensor * lid_source_kv = ggml_concat(ctx0, + inp_dsv4->mctx->get_lid_state()->get_kv(ctx0, il), lid_state_kv, 1); + ggml_tensor * lid_source_score = ggml_concat(ctx0, + inp_dsv4->mctx->get_lid_state()->get_score(ctx0, il), lid_state_score, 1); + + ggml_tensor * kv_comp_lid_state = build_overlap_compressed_kv_from_state( + lid_source_kv, + lid_source_score, + inp_dsv4->get_lid().state_read_idxs, + inp_dsv4->get_lid().state_write_pos, + layer.indexer_comp_norm, + DSV4_CSA_RATIO, + hparams.indexer_head_size, + "lid_state_compress", + il); + + if (inp_dsv4->get_lid().k_rot) { + kv_comp_lid_state = llama_mul_mat_hadamard(ctx0, kv_comp_lid_state, inp_dsv4->get_lid().k_rot); + cb(kv_comp_lid_state, "lid_state_compress_rot", il); + } + + ggml_build_forward_expand(gf, inp_dsv4->mctx->get_lid()->cpy_k(ctx0, + kv_comp_lid_state, inp_dsv4->get_lid().state_write_idxs, il)); + + lid_state_kv = dsv4_with_zero_dep(ctx0, lid_state_kv, kv_comp_lid_state); + lid_state_score = dsv4_with_zero_dep(ctx0, lid_state_score, kv_comp_lid_state); + + ggml_tensor * lid_persist_kv = ggml_get_rows(ctx0, lid_state_kv, inp_dsv4->get_lid().state_persist_src_idxs); + ggml_tensor * lid_persist_score = ggml_get_rows(ctx0, lid_state_score, inp_dsv4->get_lid().state_persist_src_idxs); + + lid_state_kv = inp_dsv4->mctx->get_lid_state()->cpy_kv(ctx0, + lid_persist_kv, inp_dsv4->get_lid().state_persist_dst_idxs, il); + lid_state_score = inp_dsv4->mctx->get_lid_state()->cpy_score(ctx0, + lid_persist_score, inp_dsv4->get_lid().state_persist_dst_idxs, il); + + ggml_build_forward_expand(gf, lid_state_kv); + ggml_build_forward_expand(gf, lid_state_score); + } + + ggml_tensor * hca_state_dep = nullptr; + if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_write_idxs) { + GGML_ASSERT(hca_state_kv); + GGML_ASSERT(hca_state_score); + + ggml_tensor * hca_source_kv = ggml_concat(ctx0, + inp_dsv4->mctx->get_hca_state()->get_kv(ctx0, il), hca_state_kv, 1); + ggml_tensor * hca_source_score = ggml_concat(ctx0, + inp_dsv4->mctx->get_hca_state()->get_score(ctx0, il), hca_state_score, 1); + + ggml_tensor * kv_comp_hca = build_hca_compressed_kv_from_state( + hca_source_kv, + hca_source_score, + inp_dsv4->get_hca().state_read_idxs, + inp_dsv4->get_hca().state_write_pos, + layer.attn_comp_norm, + n_embd_head, + "hca_state_compress", + il); + + if (inp_dsv4->get_hca().k_rot) { + kv_comp_hca = llama_mul_mat_hadamard(ctx0, kv_comp_hca, inp_dsv4->get_hca().k_rot); + cb(kv_comp_hca, "hca_state_compress_rot", il); + } + + ggml_build_forward_expand(gf, inp_dsv4->mctx->get_hca()->cpy_k(ctx0, + kv_comp_hca, inp_dsv4->get_hca().state_write_idxs, il)); + hca_state_dep = kv_comp_hca; + } + + if (ratio == DSV4_HCA_RATIO && inp_dsv4->get_hca().state_pos) { + GGML_ASSERT(hca_state_kv); + GGML_ASSERT(hca_state_score); + + hca_state_kv = dsv4_with_zero_dep(ctx0, hca_state_kv, hca_state_dep); + hca_state_score = dsv4_with_zero_dep(ctx0, hca_state_score, hca_state_dep); + + ggml_tensor * hca_persist_kv = ggml_get_rows(ctx0, hca_state_kv, inp_dsv4->get_hca().state_persist_src_idxs); + ggml_tensor * hca_persist_score = ggml_get_rows(ctx0, hca_state_score, inp_dsv4->get_hca().state_persist_src_idxs); + + hca_state_kv = inp_dsv4->mctx->get_hca_state()->cpy_kv(ctx0, + hca_persist_kv, inp_dsv4->get_hca().state_persist_dst_idxs, il); + hca_state_score = inp_dsv4->mctx->get_hca_state()->cpy_score(ctx0, + hca_persist_score, inp_dsv4->get_hca().state_persist_dst_idxs, il); + + ggml_build_forward_expand(gf, hca_state_kv); + ggml_build_forward_expand(gf, hca_state_score); + } + + ggml_tensor * out = nullptr; + if (ratio == DSV4_CSA_RATIO && + inp_dsv4->get_csa().kq_mask && + inp_dsv4->get_lid().kq_mask && + inp_dsv4->get_lid().k_rot) { + out = build_csa_lid_attention(model, inp_dsv4, inp_attn, q, kv, qr, cur, inp_pos, layer.attn_sinks, + 1.0f/sqrtf(float(n_embd_head)), il); + } else if (ratio == DSV4_HCA_RATIO && + inp_dsv4->get_hca().kq_mask) { + out = build_hca_attention(inp_dsv4, inp_attn, q, kv, layer.attn_sinks, + 1.0f/sqrtf(float(n_embd_head)), il); + } else { + out = build_raw_attention(inp_attn, q, kv, layer.attn_sinks, + 1.0f/sqrtf(float(n_embd_head)), il); + } + + out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt); + ggml_tensor * out_nope = ggml_view_3d(ctx0, out, n_embd_head_nope, n_head, nt, + ggml_row_size(out->type, n_embd_head), + ggml_row_size(out->type, n_embd_head)*n_head, + 0); + ggml_tensor * out_pe = ggml_view_3d(ctx0, out, n_embd_head_rope, n_head, nt, + ggml_row_size(out->type, n_embd_head), + ggml_row_size(out->type, n_embd_head)*n_head, + ggml_row_size(out->type, n_embd_head_nope)); + out_pe = ggml_rope_ext_back(ctx0, out_pe, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l, + freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + out = ggml_concat(ctx0, out_nope, out_pe, 0); + cb(out, "attn_derope", il); + + out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt); + out = ggml_permute(ctx0, out, 0, 2, 1, 3); + ggml_tensor * oa = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, layer.wo_a, layer.wo_a->ne[0], o_lora_rank, n_groups), out); + cb(oa, "attn_wo_a", il); + oa = ggml_permute(ctx0, oa, 0, 2, 1, 3); + oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, nt); + + out = build_lora_mm(layer.wo_b, oa); + cb(out, "attn_out", il); + + return out; +} + +llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + ggml_tensor * cur; + + ggml_tensor * inp = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + llm_graph_input_dsv4 * inp_dsv4 = build_inp_dsv4(); + llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw(); + ggml_build_forward_expand(gf, inp_attn->self_kq_mask); + + const int64_t hc = hparams.dsv4_hc_mult; + ggml_tensor * inpL = ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens); + inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); + cb(inpL, "hc_init", -1); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * residual = inpL; + ggml_tensor * post = nullptr; + ggml_tensor * comb = nullptr; + + cur = build_hc_pre(inpL, + model.layers[il].hc_attn_fn, + model.layers[il].hc_attn_scale, + model.layers[il].hc_attn_base, + &post, &comb, il); + cb(cur, "hc_attn_pre", il); + + cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + cur = build_attention(model, inp_dsv4, cur, inp_pos, il); + + inpL = build_hc_post(cur, residual, post, comb, il); + cb(inpL, "hc_attn_post", il); + + residual = inpL; + cur = build_hc_pre(inpL, + model.layers[il].hc_ffn_fn, + model.layers[il].hc_ffn_scale, + model.layers[il].hc_ffn_base, + &post, &comb, il); + cb(cur, "hc_ffn_pre", il); + + cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + const auto & layer = model.layers[il]; + ggml_tensor * selected_experts = nullptr; + ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b; + if ((uint32_t) il < hparams.dsv4_hash_layer_count) { + selected_experts = ggml_get_rows(ctx0, layer.ffn_gate_tid2eid, res->t_inp_tokens); + exp_probs_b = nullptr; + } + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + exp_probs_b, + n_expert, hparams.n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + selected_experts); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + + inpL = build_hc_post(cur, residual, post, comb, il); + inpL = build_cvec(inpL, il); + cb(inpL, "l_out", il); + } + + if (inp_out_ids) { + ggml_tensor * flat = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens); + flat = ggml_get_rows(ctx0, flat, inp_out_ids); + inpL = ggml_reshape_3d(ctx0, flat, n_embd, hc, n_outputs); + } + + cur = build_hc_head(inpL, model.hc_head_fn, model.hc_head_scale, model.hc_head_base); + cb(cur, "hc_head", -1); + + cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/examples/talk-llama/models/delta-net-base.cpp b/examples/talk-llama/models/delta-net-base.cpp index ad9ce771408..ad661264773 100644 --- a/examples/talk-llama/models/delta-net-base.cpp +++ b/examples/talk-llama/models/delta-net-base.cpp @@ -401,9 +401,9 @@ std::pair llm_build_delta_net_base::build_delta_ne // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs]. ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1); if (n_tokens == 1) { - cb(result, LLAMA_TENSOR_NAME_FGDN_AR, il); + res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il}); } else { - cb(result, LLAMA_TENSOR_NAME_FGDN_CH, il); + res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il}); } ggml_tensor * output = ggml_view_4d(ctx0, result, @@ -496,8 +496,8 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update)); } else { // [TAG_RECURRENT_ROLLBACK_SPLITS] - // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are - // inside the same ubatch. currently with `split_equal()` this is not correct + // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside + // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument const int64_t K = (int64_t) cparams.n_rs_seq + 1; @@ -566,9 +566,9 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output. ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K); if (n_seq_tokens > 1) { - cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH, il); + res->add_fused_node({LLM_FUSED_OP_GDN_CH, gdn_out, il}); } else { - cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_AR, il); + res->add_fused_node({LLM_FUSED_OP_GDN_AR, gdn_out, il}); } const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs; diff --git a/examples/talk-llama/models/dflash.cpp b/examples/talk-llama/models/dflash.cpp new file mode 100644 index 00000000000..a7b4f4435a8 --- /dev/null +++ b/examples/talk-llama/models/dflash.cpp @@ -0,0 +1,276 @@ +#include "models.h" + +#include "llama-kv-cache.h" +#include "llama-kv-cache-iswa.h" + +void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) { + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + if (!ml.get_arr(LLM_KV_TARGET_LAYERS, target_layer_ids, false)) { + throw std::runtime_error("DFlash model requires 'target_layers' in GGUF metadata"); + } + + hparams.n_embd_inp_enc_impl = (uint32_t) target_layer_ids.size() * hparams.n_embd; + + LLAMA_LOG_INFO("%s: DFlash extract_layers = [", __func__); + for (size_t i = 0; i < target_layer_ids.size(); ++i) { + LLAMA_LOG_INFO("%d%s", target_layer_ids[i], i + 1 < target_layer_ids.size() ? ", " : ""); + } + LLAMA_LOG_INFO("]\n"); + + // optional interleaved sliding-window attention with per-layer pattern array. + // DFlash has a single rope, so the SWA rope == main rope. + if (ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false) && hparams.n_swa > 0) { + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer()); + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + } + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_dflash::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_embd_inp = hparams.n_embd_inp_enc(); + + fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0); + output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc) + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); + + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head }, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), { n_embd_head_k }, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), { n_embd_head_k }, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), { n_embd, n_ff }, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, n_ff }, 0); + } +} + +std::unique_ptr llama_model_dflash::build_arch_graph(const llm_graph_params & params) const { + switch (params.gtype) { + case LLM_GRAPH_TYPE_ENCODER: + return std::make_unique>(*this, params); + case LLM_GRAPH_TYPE_DEFAULT: + case LLM_GRAPH_TYPE_DECODER: + return std::make_unique>(*this, params); + default: + GGML_ABORT("invalid graph type"); + }; +} + +template <> +ggml_tensor * llama_model_dflash::graph::build_inp_embd_enc() const { + auto inp_target = std::make_unique(hparams.n_embd_inp_enc()); + + inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp_enc(), n_tokens); + ggml_set_input(inp_target->embd); + + ggml_tensor * cur = inp_target->embd; + cb(cur, "inp_embd", -1); + + res->add_input(std::move(inp_target)); + + return cur; +} + +// DFlash Encoder: processes target model features through feature fusion layer +template <> +llama_model_dflash::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + ggml_tensor * cur = build_inp_embd_enc(); + + cur = build_lora_mm(model.fc, cur); + cb(cur, "fc_out", -1); + + cur = build_norm(cur, model.output_norm_enc, NULL, LLM_NORM_RMS, -1); + cb(cur, "enc_norm_out", -1); + + ggml_set_output(cur); + res->t_h_nextn = cur; + + ggml_build_forward_expand(gf, cur); +} + +// DFlash decoder, dual-mode by batch type: +// * embd batch -> fused target features: project + inject K/V into the cache. +// * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens +template <> +llama_model_dflash::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * inp_pos = build_inp_pos(); + + // optional iSWA: pick the matching attention input + const bool use_iswa = hparams.swa_type != LLAMA_SWA_TYPE_NONE; + + llm_graph_input_attn_kv * inp_attn = nullptr; + llm_graph_input_attn_kv_iswa * inp_attn_iswa = nullptr; + if (use_iswa) { + inp_attn_iswa = build_attn_inp_kv_iswa(); + } else { + inp_attn = build_attn_inp_kv(); + } + + const float kq_scale = 1.0f/sqrtf(float(n_embd_head)); + + // KV cache injection + if (ubatch.embd) { + auto inp = std::make_unique(n_embd); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * inp_g = inp->embd; + cb(inp_g, "inp_g_embeddings", -1); + + res->add_input(std::move(inp)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * Kcur = build_lora_mm(layer.wk, inp_g); + ggml_tensor * Vcur = build_lora_mm(layer.wv, inp_g); + + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur_injected", il); + cb(Vcur, "Vcur_injected", il); + + if (use_iswa) { + // route each layer's K/V to its sub-cache: SWA layers -> sliding cache, full -> dense + const bool is_swa = hparams.is_swa(il); + const auto * kv = is_swa ? inp_attn_iswa->mctx->get_swa() : inp_attn_iswa->mctx->get_base(); + ggml_tensor * k_idxs = is_swa ? inp_attn_iswa->get_k_idxs_swa() : inp_attn_iswa->get_k_idxs(); + ggml_tensor * v_idxs = is_swa ? inp_attn_iswa->get_v_idxs_swa() : inp_attn_iswa->get_v_idxs(); + ggml_build_forward_expand(gf, kv->cpy_k(ctx0, Kcur, k_idxs, il)); + ggml_build_forward_expand(gf, kv->cpy_v(ctx0, Vcur, v_idxs, il)); + } else { + ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il)); + ggml_build_forward_expand(gf, inp_attn->mctx->cpy_v(ctx0, Vcur, inp_attn->get_v_idxs(), il)); + } + } + + res->t_embd = inp_g; + + ggml_build_forward_expand(gf, inp_g); + return; + } + + // tok_embd from the target model (shared via ctx_other) + auto * tok_embd = model.tok_embd; + if (tok_embd == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + + GGML_ASSERT(model_other->tok_embd != nullptr && "DFlash decoder requires the target model's token embeddings"); + tok_embd = model_other->tok_embd; + } + + auto inp = std::make_unique(n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens); + cb(inpL, "inp_noise_embd", -1); + + res->add_input(std::move(inp)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * noise_norm = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(noise_norm, "noise_norm", il); + + ggml_tensor * Qcur = build_lora_mm(layer.wq, noise_norm); + ggml_tensor * Kcur = build_lora_mm(layer.wk, noise_norm); + ggml_tensor * Vcur = build_lora_mm(layer.wv, noise_norm); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + + Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + // cache-aware, non-causal attention + ggml_tensor * cur = use_iswa + ? build_attn(inp_attn_iswa, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il) + : build_attn(inp_attn, layer.wo, NULL, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + layer.ffn_up, NULL, NULL, + layer.ffn_gate, NULL, NULL, + layer.ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "l_out", il); + + inpL = cur; + } + + ggml_tensor * cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + + res->t_embd = cur; + + // lm_head from the target model (shared via ctx_other) + auto * output = model.output; + if (output == nullptr) { + GGML_ASSERT(cparams.ctx_other != nullptr); + const auto * model_other = llama_get_model(cparams.ctx_other); + GGML_ASSERT(model_other->output != nullptr && "DFlash decoder requires the target model's output projection"); + output = model_other->output; + } + + cur = build_lora_mm(output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/examples/talk-llama/models/models.h b/examples/talk-llama/models/models.h index 2ac8415a363..7a52e7bc1ab 100644 --- a/examples/talk-llama/models/models.h +++ b/examples/talk-llama/models/models.h @@ -1085,6 +1085,121 @@ struct llama_model_deepseek32 : public llama_model_base { }; +struct llama_model_deepseek4 : public llama_model_base { + llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + ggml_tensor * build_hc_pre( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base, + ggml_tensor ** post, + ggml_tensor ** comb, + int il) const; + + ggml_tensor * build_hc_post( + ggml_tensor * x, + ggml_tensor * residual, + ggml_tensor * post, + ggml_tensor * comb, + int il) const; + + ggml_tensor * build_hc_head( + ggml_tensor * x, + ggml_tensor * hc_fn, + ggml_tensor * hc_scale, + ggml_tensor * hc_base) const; + + ggml_tensor * build_attention( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const; + + ggml_tensor * build_hca_compressed_kv_from_state( + ggml_tensor * kv_state, + ggml_tensor * score_state, + ggml_tensor * state_read_idxs, + ggml_tensor * comp_pos, + ggml_tensor * norm, + int64_t n_embd_head, + const char * name, + int il) const; + + ggml_tensor * build_overlap_compressed_kv_from_state( + ggml_tensor * kv_state, + ggml_tensor * score_state, + ggml_tensor * state_read_idxs, + ggml_tensor * comp_pos, + ggml_tensor * norm, + int64_t ratio, + int64_t n_embd_head, + const char * name, + int il) const; + + ggml_tensor * build_lid_top_k( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + ggml_tensor * qr, + ggml_tensor * cur, + ggml_tensor * inp_pos, + int il) const; + + ggml_tensor * build_top_k_mask( + ggml_tensor * kq_mask, + ggml_tensor * top_k, + const char * name, + int il) const; + + ggml_tensor * build_csa_lid_attention( + const llama_model & model, + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * qr, + ggml_tensor * cur, + ggml_tensor * inp_pos, + ggml_tensor * sinks, + float kq_scale, + int il) const; + + ggml_tensor * build_hca_attention( + llm_graph_input_dsv4 * inp_dsv4, + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * sinks, + float kq_scale, + int il) const; + + ggml_tensor * build_raw_attention( + llm_graph_input_dsv4_raw * inp_attn, + ggml_tensor * q, + ggml_tensor * kv, + ggml_tensor * sinks, + float kq_scale, + int il) const; + + ggml_tensor * build_hc_weighted_sum( + ggml_tensor * x, + ggml_tensor * weights) const; + + ggml_tensor * build_hc_sinkhorn( + ggml_tensor * comb, + int il) const; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_deepseek2ocr : public llama_model_base { llama_model_deepseek2ocr(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1122,6 +1237,22 @@ struct llama_model_eagle3 : public llama_model_base { }; +struct llama_model_dflash : public llama_model_base { + llama_model_dflash(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + template + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + + ggml_tensor * build_inp_embd_enc() const; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_mistral4 : public llama_model_deepseek2 { llama_model_mistral4(const struct llama_model_params & params) : llama_model_deepseek2(params) {} // reuse load_arch_hparams and load_arch_tensors from llama_model_deepseek2 diff --git a/examples/talk-llama/models/qwen3next.cpp b/examples/talk-llama/models/qwen3next.cpp index 97200a44072..09b66423d5a 100644 --- a/examples/talk-llama/models/qwen3next.cpp +++ b/examples/talk-llama/models/qwen3next.cpp @@ -121,6 +121,8 @@ llama_model_qwen3next::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 0ec62a3773d..0470e3e2f7f 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -4,8 +4,8 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 15) -set(GGML_VERSION_PATCH 3) +set(GGML_VERSION_MINOR 16) +set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h index 5436c7ef579..1cd81eeaebc 100644 --- a/ggml/include/ggml-cuda.h +++ b/ggml/include/ggml-cuda.h @@ -30,9 +30,6 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int de // conduct allreduce operation between devices GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends); -// split tensor buffer that splits matrices by rows across multiple devices -GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split); - // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d6807b6dd47..ac133665d97 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -429,7 +429,8 @@ extern "C" { GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, - GGML_TYPE_COUNT = 42, + GGML_TYPE_Q2_0 = 42, + GGML_TYPE_COUNT = 43, }; // precision @@ -473,6 +474,7 @@ extern "C" { GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors }; // available tensor operations: diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 0a36f099000..1f29ec86712 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -1144,6 +1144,11 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m ggml_context * simple_ctx = stc.ctxs[j].get(); ggml_backend_buffer_t simple_buf = buf_ctx->bufs[j].get(); + if ((simple_buf != nullptr) && ggml_backend_buffer_is_multi_buffer(simple_buf)) { + // see https://github.com/ggml-org/llama.cpp/issues/22197 + GGML_ABORT("multi buffers are not supported by the meta backend"); + } + if (split_dim >= 0 && split_dim < GGML_MAX_DIMS) { // TODO: the following assert fails for llama-parallel even though the results are correct: // GGML_ASSERT(ggml_is_contiguously_allocated(tensor)); @@ -1245,9 +1250,8 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); - GGML_ASSERT(ggml_is_contiguous(tensor)); - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); + GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); if (split_state.n_segments != 1 || split_state.nr[0] != 1) { GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS); @@ -1360,9 +1364,8 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); - GGML_ASSERT(ggml_is_contiguous(tensor)); - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); + GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); if (split_state.n_segments != 1 || split_state.nr[0] != 1) { GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS); diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index f05683b44cd..83f9118da84 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -96,6 +96,9 @@ typedef sycl::half2 ggml_half2; #define QI1_0 (QK1_0 / 32) #define QR1_0 1 +#define QI2_0 (QK2_0 / 32) +#define QR2_0 1 + #define QI4_0 (QK4_0 / (4 * QR4_0)) #define QR4_0 2 @@ -181,6 +184,13 @@ typedef struct { } block_q1_0; static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding"); +#define QK2_0 64 +typedef struct { + ggml_half d; // delta (scale) + uint8_t qs[QK2_0 / 4]; // 2 bits per element +} block_q2_0; +static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); + #define QK4_0 32 typedef struct { ggml_half d; // delta @@ -1111,11 +1121,12 @@ GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16) -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, GGML_TABLE_END() -// e2m1 values (doubled) +// e2m1 values (doubled), shared by MXFP4 and NVFP4 // ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf -GGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16) +GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16) 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, GGML_TABLE_END() +#define kvalues_mxfp4 kvalues_fp4 #define NGRID_IQ1S 2048 #define IQ1S_DELTA 0.125f diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 1fc2b4b71bd..152e0bac99b 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -17,6 +17,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -82,7 +83,7 @@ #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // quants.c -#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 @@ -114,6 +115,7 @@ #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K @@ -163,6 +165,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -203,6 +206,7 @@ #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 @@ -244,6 +248,7 @@ #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -307,6 +312,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index fe621332970..b988abf9963 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -219,6 +219,80 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + // Replicate pattern: each byte repeated 4 times + static const uint8_t tbl_idx_lo[16] = {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3}; + static const uint8_t tbl_idx_hi[16] = {4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7}; + // Right-shift amounts: 0,2,4,6 repeated for each group of 4 + static const int8_t shift_vals[16] = {0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6}; + + const uint8x16_t idx_lo = vld1q_u8(tbl_idx_lo); + const uint8x16_t idx_hi = vld1q_u8(tbl_idx_hi); + const int8x16_t shifts = vld1q_s8(shift_vals); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + const int8x16_t one = vdupq_n_s8(1); + + float32x4_t sumv = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64) + for (int k = 0; k < 2; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + + // Load 8 bytes of packed 2-bit values + const uint8x8_t raw = vld1_u8(&x[i].qs[k * 8]); + const uint8x16_t raw16 = vcombine_u8(raw, raw); + + // First 16 elements: replicate bytes 0-3, shift, mask, subtract 1 + uint8x16_t bytes0 = ggml_vqtbl1q_u8(raw16, idx_lo); + int8x16_t qv0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)), + one); + + // Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1 + uint8x16_t bytes1 = ggml_vqtbl1q_u8(raw16, idx_hi); + int8x16_t qv1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)), + one); + + // Load Q8_0 values and dot product + const int8x16_t y0 = vld1q_s8(yb->qs); + const int8x16_t y1 = vld1q_s8(yb->qs + 16); + + int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), qv0, y0); + int32x4_t p1 = ggml_vdotq_s32(p0, qv1, y1); + + sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1); + } + } + + sumf = vaddvq_f32(sumv); +#else + ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + return; +#endif + + *s = sumf; +} void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; @@ -812,10 +886,10 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); const float32x4_t nvsc = { - ggml_ue4m3_to_fp32(x[ib].d[0]), - ggml_ue4m3_to_fp32(x[ib].d[1]), - ggml_ue4m3_to_fp32(x[ib].d[2]), - ggml_ue4m3_to_fp32(x[ib].d[3]) + GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) }; const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1}); diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 94b19b82bbc..ea54cfe44ce 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -934,7 +934,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo #if defined __AVX2__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); const __m256i mone = _mm256_set1_epi16(1); @@ -963,7 +963,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo sumf = hsum_float_8(_mm256_add_ps(accum1, accum2)); #elif defined __AVX__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); __m256 accum = _mm256_setzero_ps(); @@ -993,14 +993,152 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo int sumi1 = 0; int sumi2 = 0; for (int j = 0; j < QK_MXFP4/2; ++j) { - sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf]; - sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4]; + sumi1 += y[ib].qs[j + 0] * kvalues_fp4[x[ib].qs[j] & 0xf]; + sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_fp4[x[ib].qs[j] >> 4]; } sumf += d * (sumi1 + sumi2); } *s = sumf; } +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_NVFP4 == 0); + + const block_nvfp4 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK_NVFP4; + int ib = 0; + float sumf = 0; + +#if defined(__AVX2__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + const __m256i mone = _mm256_set1_epi16(1); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m256i q8_01 = _mm256_loadu_si256((const __m256i *)y[2*ib + 0].qs); + const __m256i q8_23 = _mm256_loadu_si256((const __m256i *)y[2*ib + 1].qs); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + //reordering + const __m256i q4_01 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_01_lo,q4_01_hi), _mm_unpacklo_epi64(q4_01_lo,q4_01_hi)); + const __m256i q4_23 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_23_lo,q4_23_hi),_mm_unpacklo_epi64(q4_23_lo,q4_23_hi)); + + const __m256i p01 = mul_add_epi8(q4_01,q8_01); + const __m256i p_1 = _mm256_madd_epi16(p01, mone); + + const __m256i p23 = mul_add_epi8(q4_23,q8_23); + const __m256i p_2 = _mm256_madd_epi16(p23, mone); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_fmadd_ps(scales01, _mm256_cvtepi32_ps(p_1), accum); + accum = _mm256_fmadd_ps(scales23, _mm256_cvtepi32_ps(p_2), accum); + } + sumf = hsum_float_8(accum); + +#elif defined(__AVX__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m128i q8_0 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 0)); + const __m128i q8_1 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 16)); + const __m128i q8_2 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 0)); + const __m128i q8_3 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 16)); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + const __m128i q4_0 = _mm_unpacklo_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_1 = _mm_unpackhi_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_2 = _mm_unpacklo_epi64(q4_23_lo, q4_23_hi); + const __m128i q4_3 = _mm_unpackhi_epi64(q4_23_lo, q4_23_hi); + + const __m128i p0_i32 = mul_sum_i8_pairs(q4_0, q8_0); + const __m128i p1_i32 = mul_sum_i8_pairs(q4_1, q8_1); + const __m128i p2_i32 = mul_sum_i8_pairs(q4_2, q8_2); + const __m128i p3_i32 = mul_sum_i8_pairs(q4_3, q8_3); + + const __m128 p0 = _mm_cvtepi32_ps(p0_i32); + const __m128 p1 = _mm_cvtepi32_ps(p1_i32); + const __m128 p2 = _mm_cvtepi32_ps(p2_i32); + const __m128 p3 = _mm_cvtepi32_ps(p3_i32); + + const __m256 p01 = _mm256_set_m128(p1, p0); + const __m256 p23 = _mm256_set_m128(p3, p2); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_add_ps(accum, _mm256_mul_ps(p01, scales01)); + accum = _mm256_add_ps(accum, _mm256_mul_ps(p23, scales23)); + } + sumf = hsum_float_8(accum); + +#endif + + for (;ib < nb; ++ib) { + for (int s_idx = 0; s_idx < 4; ++s_idx) { + const float d = GGML_CPU_UE4M3_TO_FP32(x[ib].d[s_idx]); + const int q8_block = s_idx / 2; + const int q8_off = (s_idx % 2) * QK_NVFP4_SUB; + const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d); + + int sumi_lo = 0, sumi_hi = 0; + for (int j = 0; j < QK_NVFP4_SUB/2; ++j) { + const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j]; + sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_fp4[qv & 0xf]; + sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_fp4[qv >> 4]; + } + + sumf += dy * d * (sumi_lo + sumi_hi); + } + } + *s = sumf; +} + void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index eb8341c9aec..a82842fcffc 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -82,6 +82,9 @@ float ggml_table_f32_f16[1 << 16]; // precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h) float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h) +float ggml_table_f32_ue4m3[1 << 8]; + #if defined(__ARM_ARCH) struct ggml_arm_arch_features_type { int sve_cnt; @@ -227,6 +230,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_Q2_0] = { + .from_float = quantize_row_q2_0, + .vec_dot = ggml_vec_dot_q2_0_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q4_0] = { .from_float = quantize_row_q4_0, .vec_dot = ggml_vec_dot_q4_0_q8_0, @@ -3798,6 +3807,11 @@ void ggml_cpu_init(void) { ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i); } + // initialize UE4M3 table (256 entries) + for (int i = 0; i < (1 << 8); ++i) { + ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i); + } + const uint64_t t_end = ggml_time_us(); UNUSED(t_end); GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 0b8323e60ce..5efaaa5b2a0 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -2321,24 +2321,28 @@ class tinyBLAS_Q0_PPC { } void matmul(int64_t m, int64_t n) { - #if defined(_AIX) || defined(__BIG_ENDIAN__) - mnpack(0, m, 0, n); - #else - const int64_t mc = 64; - const int64_t kc = 64; + int64_t mc = 64; int64_t nc = 64; + int64_t kc = 64; + int64_t n_chunk = 64; + #if defined(_AIX) || defined(__BIG_ENDIAN__) + mc = 32; + nc = 32; + kc = 32; + n_chunk = 32 + #endif int64_t n_aligned = 0; - if (n % 64 == 0) { + if (n % n_chunk == 0) { n_aligned = n; } else if (n == 4) { n_aligned = 4; - } else if (n < 64) { + } else if (n < n_chunk) { n_aligned = (n / 8) * 8; } else { - n_aligned = (n / 64) * 64; + n_aligned = (n / n_chunk) * n_chunk; } if (n_aligned > 0) { - if (n_aligned % 64 == 0) nc = 64; + if (n_aligned % n_chunk == 0) nc = n_chunk; else if (n_aligned == n) nc = n; else if (n_aligned % 32 == 0) nc = 32; else if (n_aligned % 24 == 0) nc = 24; @@ -2354,7 +2358,6 @@ class tinyBLAS_Q0_PPC { } else { mnpack(0, m, 0, n); } - #endif } private: @@ -3195,16 +3198,19 @@ class tinyBLAS_PPC { } void matmul(int64_t m, int64_t n) { + int64_t mc = 256; + int64_t nc = 256; + int64_t kc = 256; #if defined(_AIX) || defined(__BIG_ENDIAN__) - mnpack(0, m, 0, n); - #else - int64_t mc = 256; int64_t nc = 256; int64_t kc = 256; + mc = 128; + nc = 128; + kc = 128; + #endif if (m % mc == 0 && n % nc == 0 && k % kc == 0) { matmul_tiled(m, n, mc, nc, kc); } else { mnpack(0, m, 0, n); } - #endif } private: diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 6724686b8ae..df0028cf15e 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -665,6 +665,7 @@ void ggml_compute_forward_add( ggml_compute_forward_add_non_quantized(params, dst); } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1115,6 +1116,7 @@ void ggml_compute_forward_add1( } } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1245,6 +1247,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1913,7 +1916,11 @@ static void ggml_compute_forward_concat_any( GGML_ASSERT(dim >= 0 && dim < 4); int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; + if (dim == 0) { + o[dim] = src0->ne[dim]/ggml_blck_size(src0->type); + } else { + o[dim] = src0->ne[dim]; + } const char * x; @@ -1921,8 +1928,8 @@ static void ggml_compute_forward_concat_any( for (int i3 = 0; i3 < ne3; i3++) { for (int i2 = ith; i2 < ne2; i2 += nth) { for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + for (int i0 = 0; i0 < ne0/ggml_blck_size(dst->type); i0++) { + if (i0 < ne00/ggml_blck_size(src0->type) && i1 < ne01 && i2 < ne02 && i3 < ne03) { x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; } else { x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; @@ -2071,6 +2078,14 @@ void ggml_compute_forward_concat( ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + if (ggml_is_quantized(src0->type)) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); + } switch (src0->type) { case GGML_TYPE_F16: @@ -4442,6 +4457,7 @@ void ggml_compute_forward_out_prod( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4718,6 +4734,7 @@ void ggml_compute_forward_set( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4942,6 +4959,7 @@ void ggml_compute_forward_get_rows( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5007,8 +5025,8 @@ void ggml_compute_forward_get_rows( //} } -template -static void ggml_compute_forward_set_rows_f32( +template +static void ggml_compute_forward_set_rows_impl( const ggml_compute_params * params, ggml_tensor * dst) { @@ -5023,7 +5041,7 @@ static void ggml_compute_forward_set_rows_f32( assert(ne0 == nc); assert(ne2 == ne02); assert(ne3 == ne03); - assert(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); assert(ne02 % ne11 == 0); assert(ne03 % ne12 == 0); @@ -5037,6 +5055,8 @@ static void ggml_compute_forward_set_rows_f32( const int64_t ir0 = dr*ith; const int64_t ir1 = std::min(ir0 + dr, nr); + const size_t rs = ggml_row_size(src0->type, nc); + ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float; for (int64_t i03 = 0; i03 < ne03; ++i03) { @@ -5050,9 +5070,18 @@ static void ggml_compute_forward_set_rows_f32( GGML_ASSERT(i1 >= 0 && i1 < ne1); - from_float( - (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), - ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + if constexpr (std::is_same_v) { + from_float( + (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } else if constexpr (std::is_same_v) { + memcpy( + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), + ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + rs); + } else { + GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); + } } } } @@ -5069,13 +5098,27 @@ void ggml_compute_forward_set_rows( case GGML_TYPE_F32: { if (src1->type == GGML_TYPE_I64) { - ggml_compute_forward_set_rows_f32(params, dst); + ggml_compute_forward_set_rows_impl(params, dst); } else if (src1->type == GGML_TYPE_I32) { - ggml_compute_forward_set_rows_f32(params, dst); + ggml_compute_forward_set_rows_impl(params, dst); } else { GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); } } break; + case GGML_TYPE_F16: + { + if (dst->type == GGML_TYPE_F16) { + if (src1->type == GGML_TYPE_I64) { + ggml_compute_forward_set_rows_impl(params, dst); + } else if (src1->type == GGML_TYPE_I32) { + ggml_compute_forward_set_rows_impl(params, dst); + } else { + GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); + } + } else { + GGML_ABORT("dst->type = %d (%s) not supported with src0->type = %d (%s)", dst->type, ggml_type_name(dst->type), src0->type, ggml_type_name(src0->type)); + } + } break; default: { GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); @@ -5668,6 +5711,7 @@ void ggml_compute_forward_clamp( } break; case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -7255,6 +7299,13 @@ struct ggml_conv_2d_dw_params { int dilation_y; }; +static inline float ggml_conv_2d_dw_knl_f32(const char * data, int64_t i, ggml_type type) { + if (type == GGML_TYPE_F16) { + return GGML_FP16_TO_FP32(((const ggml_fp16_t *)data)[i]); + } + return ((const float *)data)[i]; +} + static void ggml_compute_forward_conv_2d_dw_cwhn( const ggml_compute_params * params, const ggml_tensor * src, @@ -7263,7 +7314,8 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const ggml_conv_2d_dw_params & p) { const int64_t c = p.channels; - const float * knl_data = (const float *)kernel->data; + const char * knl_data = (const char *)kernel->data; + const ggml_type knl_type = kernel->type; const int64_t rows_total = p.dst_h * p.batch; const int64_t rows_per_thread = (rows_total + params->nth - 1) / params->nth; @@ -7271,13 +7323,16 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const int64_t row_end = MIN(row_start + rows_per_thread, rows_total); #ifdef GGML_SIMD + int64_t c_pkg_end = 0; + int64_t pkg_size = GGML_F32_EPR; + if (knl_type == GGML_TYPE_F32) { #if defined(__ARM_FEATURE_SVE) - const int64_t pkg_size = svcntw(); + pkg_size = svcntw(); #else - const int64_t pkg_size = GGML_F32_EPR; + pkg_size = GGML_F32_EPR; #endif - const int64_t pkg_count = c / pkg_size; - const int64_t c_pkg_end = pkg_count * pkg_size; + c_pkg_end = (c / pkg_size) * pkg_size; + } #else const int64_t c_pkg_end = 0; #endif @@ -7291,7 +7346,6 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const int64_t src_x_base = dst_x * p.stride_x - p.pad_x; #ifdef GGML_SIMD - // Vectorized loop for (int64_t c_i = 0; c_i < c_pkg_end; c_i += pkg_size) { GGML_F32_VEC sum = GGML_F32_VEC_ZERO; for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) { @@ -7304,7 +7358,8 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( if (src_x < 0 || src_x >= p.src_w) { continue; } - GGML_F32_VEC k = GGML_F32_VEC_LOAD(knl_data + (knl_y * p.knl_w + knl_x) * c + c_i); + const float * kp = (const float *)knl_data + (knl_y * p.knl_w + knl_x) * c + c_i; + GGML_F32_VEC k = GGML_F32_VEC_LOAD(kp); GGML_F32_VEC s = GGML_F32_VEC_LOAD(src_data + (src_y * p.src_w + src_x) * c + c_i); sum = GGML_F32_VEC_FMA(sum, k, s); } @@ -7312,7 +7367,6 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( GGML_F32_VEC_STORE(dst_data + c_i, sum); } #endif - // Scalar loop for (int64_t c_i = c_pkg_end; c_i < c; ++c_i) { float sum = 0.0f; for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) { @@ -7325,7 +7379,7 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( if (src_x < 0 || src_x >= p.src_w) { continue; } - sum += knl_data[(knl_y * p.knl_w + knl_x) * c + c_i] + sum += ggml_conv_2d_dw_knl_f32(knl_data, (knl_y * p.knl_w + knl_x) * c + c_i, knl_type) * src_data[(src_y * p.src_w + src_x) * c + c_i]; } } @@ -7346,9 +7400,11 @@ static void ggml_compute_forward_conv_2d_dw_whcn( const int64_t per_thread = (n + params->nth - 1) / params->nth; const int64_t start = params->ith * per_thread; const int64_t end = MIN(start + per_thread, n); + const char * knl_base = (const char *)kernel->data; + const ggml_type knl_type = kernel->type; for (int64_t i = start; i < end; ++i) { - const float * knl_data = (const float *)kernel->data + (i % p.channels) * p.knl_w * p.knl_h; + const int64_t knl_offset = (i % p.channels) * p.knl_w * p.knl_h; const float * src_data = (const float *)src->data + i * p.src_w * p.src_h; float * dst_data = (float *)dst->data + i * p.dst_w * p.dst_h; @@ -7366,7 +7422,7 @@ static void ggml_compute_forward_conv_2d_dw_whcn( if (src_x < 0 || src_x >= p.src_w) { continue; } - sum += knl_data[knl_y * p.knl_w + knl_x] + sum += ggml_conv_2d_dw_knl_f32(knl_base, knl_offset + knl_y * p.knl_w + knl_x, knl_type) * src_data[src_y * p.src_w + src_x]; } } @@ -7398,13 +7454,13 @@ void ggml_compute_forward_conv_2d_dw( p.dilation_x = dst->op_params[4]; p.dilation_y = dst->op_params[5]; + GGML_ASSERT(kernel->type == GGML_TYPE_F32 || kernel->type == GGML_TYPE_F16); GGML_ASSERT(kernel->ne[3] == p.channels); GGML_ASSERT(dst->ne[3] == p.batch); if (ggml_is_contiguous(src)) { ggml_compute_forward_conv_2d_dw_whcn(params, src, kernel, dst, p); } else if (ggml_is_contiguous_channels(src)) { - // kernel should also have channels most contiguous in memory GGML_ASSERT(kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0]); ggml_compute_forward_conv_2d_dw_cwhn(params, src, kernel, dst, p); } else { diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index e5f9a4083f9..5e36459f8cb 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -26,6 +26,10 @@ void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in quantize_row_q1_0_ref(x, y, k); } +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_q2_0_ref(x, y, k); +} + void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_q4_0_ref(x, y, k); } @@ -170,6 +174,53 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + float sumi = 0.0f; + + // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64) + for (int k = 0; k < 2; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + + const uint8_t * GGML_RESTRICT qs = &x[i].qs[k * 8]; + const int8_t * GGML_RESTRICT qy = yb->qs; + + for (int b = 0; b < 8; ++b) { + const uint8_t byte = qs[b]; + // Extract 4 two-bit values, map {0,1,2,3} -> {-1,0,1,2} + sumi_block += ((int)((byte >> 0) & 3) - 1) * qy[b*4 + 0]; + sumi_block += ((int)((byte >> 2) & 3) - 1) * qy[b*4 + 1]; + sumi_block += ((int)((byte >> 4) & 3) - 1) * qy[b*4 + 2]; + sumi_block += ((int)((byte >> 6) & 3) - 1) * qy[b*4 + 3]; + } + + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +} void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index d4bc87a1c05..93ea7eeffe5 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -13,6 +13,7 @@ extern "C" { // Quantization void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -38,6 +39,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, // Dot product void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -71,6 +73,7 @@ void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRI void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h index 4119d04f895..2ebd10051ed 100644 --- a/ggml/src/ggml-cpu/simd-gemm.h +++ b/ggml/src/ggml-cpu/simd-gemm.h @@ -78,7 +78,7 @@ static void simd_gemm( for (int64_t i = 0; i < GEMM_RM; i++) { float a = C[i * N + jj]; for (int64_t kk = 0; kk < K; kk++) { - a += A[i + kk] * B[kk * N + jj]; + a += A[i * K + kk] * B[kk * N + jj]; } C[i * N + jj] = a; } diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 62e687201ef..fca5119e1a1 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -120,6 +120,10 @@ extern float ggml_table_f32_f16[1 << 16]; // defined in ggml-cpu.c, initialized in ggml_cpu_init() extern float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) +// defined in ggml-cpu.c, initialized in ggml_cpu_init() +extern float ggml_table_f32_ue4m3[1 << 8]; + // Use lookup table for E8M0 on x86 (faster than bit manipulation) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)] @@ -127,6 +131,13 @@ extern float ggml_table_f32_e8m0_half[1 << 8]; #define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x) #endif +// Use lookup table for UE4M3 on x86 and ARM (faster than bit manipulation) +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__ARM_NEON) +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_table_f32_ue4m3[(uint8_t)(x)] +#else +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x) +#endif + // On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, // so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. // This is also true for POWER9. diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index e6e50e04119..290dc4aff25 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1505,12 +1505,16 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * x_bias = nullptr; const ggml_tensor * gate = nullptr; const ggml_tensor * gate_bias = nullptr; + const ggml_tensor * x_scale = nullptr; + const ggml_tensor * gate_scale = nullptr; ggml_glu_op glu_op; }; struct ggml_cuda_mm_fusion_args_device { const void * x_bias = nullptr; const void * gate = nullptr; const void * gate_bias = nullptr; + const void * x_scale = nullptr; + const void * gate_scale = nullptr; ggml_glu_op glu_op; }; diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu index 8d557092b2b..276ee64e8c0 100644 --- a/ggml/src/ggml-cuda/concat.cu +++ b/ggml/src/ggml-cuda/concat.cu @@ -152,8 +152,8 @@ static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml src0_d + i3*(src0->nb[3] / sizeof(T)), src1_d + i3*(src1->nb[3] / sizeof(T)), dst_d + i3*( dst->nb[3] / sizeof(T)), - src0->ne[0], src0->ne[1], src0->ne[2], - dst->ne[0], dst->ne[1], dst->ne[2], dim, stream); + ggml_row_size(src0->type, src0->ne[0])/sizeof(T), src0->ne[1], src0->ne[2], + ggml_row_size(dst->type, dst->ne[0])/sizeof(T), dst->ne[1], dst->ne[2], dim, stream); } } else { const size_t size0 = ggml_nbytes(src0); @@ -163,6 +163,8 @@ static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml CUDA_CHECK(cudaMemcpyAsync((char *) dst->data + size0, src1->data, size1, cudaMemcpyDeviceToDevice, stream)); } } else { + GGML_ASSERT(!ggml_is_quantized(src0->type)); + dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]); auto launch_kernel = [&](auto dim) { concat_non_cont<<>>( @@ -204,24 +206,34 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == src1->type); GGML_ASSERT(dst->type == src0->type); - GGML_ASSERT(!ggml_is_quantized(src0->type)); - GGML_ASSERT(ggml_blck_size(src0->type) == 1); - - switch (ggml_type_size(src0->type)) { - case 1: - concat_cuda(src0, src1, dst, dim, stream); - break; - case 2: - concat_cuda(src0, src1, dst, dim, stream); - break; - case 4: - concat_cuda(src0, src1, dst, dim, stream); - break; - case 8: - concat_cuda(src0, src1, dst, dim, stream); - break; - default: - GGML_ABORT("Unsupported type size: %zu", ggml_type_size(src0->type)); - break; + + if (ggml_is_quantized(src0->type)) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); + + // if tensors are contiguous and ne[0] is multiple of the block size we can concat both tensors as byte tensors + concat_cuda(src0, src1, dst, dim, stream); + } else { + GGML_ASSERT(ggml_blck_size(src0->type) == 1); + + switch (ggml_type_size(src0->type)) { + case 1: + concat_cuda(src0, src1, dst, dim, stream); + break; + case 2: + concat_cuda(src0, src1, dst, dim, stream); + break; + case 4: + concat_cuda(src0, src1, dst, dim, stream); + break; + case 8: + concat_cuda(src0, src1, dst, dim, stream); + break; + default: + GGML_ABORT("Unsupported type size: %zu", ggml_type_size(src0->type)); + break; + } } } diff --git a/ggml/src/ggml-cuda/conv-transpose-1d.cu b/ggml/src/ggml-cuda/conv-transpose-1d.cu index 8418ba66731..ebf2aa8045e 100644 --- a/ggml/src/ggml-cuda/conv-transpose-1d.cu +++ b/ggml/src/ggml-cuda/conv-transpose-1d.cu @@ -11,30 +11,32 @@ static __global__ void conv_transpose_1d_kernel( return; } - int out_index = global_index / dst_ne0; + int out_t = global_index % dst_ne0; + int out_ch = (global_index / dst_ne0) % dst_ne1; + int plane = global_index / (dst_ne0 * dst_ne1); float accumulator = 0; for (int c = 0; c < src0_ne2; c++) { - int idx = global_index % dst_ne0; + int kernel_offset = src0_ne0 * (out_ch + src0_ne1 * c); + int input_offset = src1_ne0 * (c + src1_ne1 * plane); - int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0); - int input_offset = src1_ne0 * c; - - for (int i = 0; i < src1_ne0; i++) { - if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) { + for (int k = 0; k < src0_ne0; k++) { + int input_numer = out_t + p0 - k*d0; + if (input_numer < 0 || input_numer % s0 != 0) { continue; } - int weight_idx = idx - i*s0; - float kernel_weight = src0[kernel_offset + weight_idx]; - float input_value = src1[input_offset+i]; + int input_t = input_numer / s0; + if (input_t >= src1_ne0) { + continue; + } - accumulator += kernel_weight * input_value; + accumulator += src0[kernel_offset + k] * src1[input_offset + input_t]; } } dst[global_index] = accumulator; - GGML_UNUSED_VARS(p0, d0, src0_ne3, src1_ne3, dst_ne3, src1_ne1, dst_ne1, src1_ne2, dst_ne2); + GGML_UNUSED_VARS(src0_ne3, src1_ne2, src1_ne3, dst_ne2, dst_ne3); } static void conv_transpose_1d_f32_f32_cuda( diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 61630a35a29..f04a2d5a2cc 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -104,8 +104,8 @@ static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t const uint8_t * q = x->qs + 4*il; for (int l = 0; l < 4; ++l) { - y[l+ 0] = d * (q[l] & 0xF) + dm; - y[l+16] = d * (q[l] >> 4) + dm; + y[l+ 0] = ggml_cuda_cast(d * (q[l] & 0xF) + dm); + y[l+16] = ggml_cuda_cast(d * (q[l] >> 4) + dm); } } @@ -131,8 +131,8 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t const uint8_t * q = x->qs + 4*il; for (int l = 0; l < 4; ++l) { - y[l+ 0] = d.x * (q[l] & 0xF) + d.y; - y[l+16] = d.x * (q[l] >> 4) + d.y; + y[l+ 0] = ggml_cuda_cast(d.x * (q[l] & 0xF) + d.y); + y[l+16] = ggml_cuda_cast(d.x * (q[l] >> 4) + d.y); } } @@ -154,10 +154,10 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t float dall = __low2half(x[i].dm); float dmin = __high2half(x[i].dm); - y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4); - y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4); - y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4); - y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4); + y[l+ 0] = ggml_cuda_cast(dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4)); + y[l+32] = ggml_cuda_cast(dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4)); + y[l+64] = ggml_cuda_cast(dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4)); + y[l+96] = ggml_cuda_cast(dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4)); } template @@ -188,7 +188,9 @@ static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t const uint8_t * q = x[i].qs + 32*n; const uint8_t * hm = x[i].hmask; - for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); + for (int l = l0; l < l0+4; ++l) { + y[l] = ggml_cuda_cast(dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))); + } } static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { @@ -226,8 +228,8 @@ static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t get_scale_min_k4(is + 1, x[i].scales, sc, m); const float d2 = dall * sc; const float m2 = dmin * m; for (int l = 0; l < n; ++l) { - y[l + 0] = d1 * (q[l] & 0xF) - m1; - y[l +32] = d2 * (q[l] >> 4) - m2; + y[l + 0] = ggml_cuda_cast(d1 * (q[l] & 0xF) - m1); + y[l +32] = ggml_cuda_cast(d2 * (q[l] >> 4) - m2); } } @@ -258,11 +260,11 @@ static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t const float d2 = dall * sc; const float m2 = dmin * m; uint8_t hm = 1 << (2*il); - y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1; - y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1; + y[ 0] = ggml_cuda_cast(d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1); + y[ 1] = ggml_cuda_cast(d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1); hm <<= 1; - y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; - y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; + y[32] = ggml_cuda_cast(d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2); + y[33] = ggml_cuda_cast(d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2); } template @@ -285,10 +287,10 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t const uint8_t qh = x[i].qh[32*ip + il]; const int8_t * sc = x[i].scales + is; - y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32); - y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32); - y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32); - y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32); + y[ 0] = ggml_cuda_cast(d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32)); + y[32] = ggml_cuda_cast(d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32)); + y[64] = ggml_cuda_cast(d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32)); + y[96] = ggml_cuda_cast(d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32)); } template @@ -307,7 +309,9 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds const uint32_t aux32 = q2[2] | (q2[3] << 16); const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } } template @@ -324,7 +328,9 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511)); const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } } template @@ -340,7 +346,9 @@ static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300))); const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = x[i].qs[QK_K/8+4*ib+il]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } } template @@ -361,8 +369,8 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f; const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; for (int j = 0; j < 4; ++j) { - y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); - y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + y[j+0] = ggml_cuda_cast(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f)); + y[j+4] = ggml_cuda_cast(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f)); } } @@ -382,8 +390,8 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_ const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)); const uint8_t signs = x[i].signs[4*ib + il]; for (int j = 0; j < 4; ++j) { - y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); - y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + y[j+0] = ggml_cuda_cast(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f)); + y[j+4] = ggml_cuda_cast(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f)); } } @@ -404,7 +412,7 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; grid32[0] &= 0x0f0f0f0f; for (int j = 0; j < 8; ++j) { - y[j] = d * (q[j] + delta); + y[j] = ggml_cuda_cast(d * (q[j] + delta)); } } @@ -429,7 +437,7 @@ static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; grid32[0] &= 0x0f0f0f0f; for (int j = 0; j < 8; ++j) { - y[j] = d * (q[j] + delta); + y[j] = ggml_cuda_cast(d * (q[j] + delta)); } } @@ -446,8 +454,8 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst const uint8_t * q4 = x[ib].qs + 4*il; const float d = (float)x[ib].d; for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; - y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; + y[j+ 0] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] & 0xf]); + y[j+16] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] >> 4]); } } @@ -463,8 +471,8 @@ static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst const uint8_t * q4 = x[i].qs + 16*ib + 4*il; const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32); for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; - y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; + y[j+ 0] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] & 0xf]); + y[j+16] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] >> 4]); } } @@ -481,8 +489,8 @@ static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_ const uint8_t * q4 = x[ib].qs + 4*il; const float d = ggml_cuda_e8m0_to_fp32(x[ib].e); for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_mxfp4[q4[j] & 0xf]*0.5f; - y[j+16] = d * kvalues_mxfp4[q4[j] >> 4]*0.5f; + y[j+ 0] = ggml_cuda_cast(d * kvalues_mxfp4[q4[j] & 0xf]*0.5f); + y[j+16] = ggml_cuda_cast(d * kvalues_mxfp4[q4[j] >> 4]*0.5f); } } @@ -700,6 +708,50 @@ static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q4_0: + return dequantize_row_q4_0_cuda; + case GGML_TYPE_Q4_1: + return dequantize_row_q4_1_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_K: + return dequantize_row_q2_K_cuda; + case GGML_TYPE_Q3_K: + return dequantize_row_q3_K_cuda; + case GGML_TYPE_Q4_K: + return dequantize_row_q4_K_cuda; + case GGML_TYPE_Q5_K: + return dequantize_row_q5_K_cuda; + case GGML_TYPE_Q6_K: + return dequantize_row_q6_K_cuda; + case GGML_TYPE_IQ2_XXS: + return dequantize_row_iq2_xxs_cuda; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_cuda; + case GGML_TYPE_IQ2_S: + return dequantize_row_iq2_s_cuda; + case GGML_TYPE_IQ3_XXS: + return dequantize_row_iq3_xxs_cuda; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ1_M: + return dequantize_row_iq1_m_cuda; + case GGML_TYPE_IQ4_NL: + return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ4_XS: + return dequantize_row_iq4_xs_cuda; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_cuda; + case GGML_TYPE_MXFP4: + return dequantize_row_mxfp4_cuda; + case GGML_TYPE_NVFP4: + return dequantize_row_nvfp4_cuda; case GGML_TYPE_F32: return convert_unary_cont_cuda; case GGML_TYPE_F16: diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 1e625cc1cbe..eb5eb0eb4eb 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -386,6 +386,46 @@ static void ggml_cpy_f32_iq4_nl_cuda( (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +// check if a same-type copy reduces to a 2D strided copy (height rows of width +// contiguous bytes), so it can use cudaMemcpy2DAsync instead of the scalar kernel +static bool ggml_cuda_cpy_as_memcpy_2d(const ggml_tensor * src0, const ggml_tensor * src1, + size_t & width, size_t & height, size_t & spitch, size_t & dpitch) { + // require matching shape: a reshaped copy maps elements by flat order, which the + // prefix walk below does not handle + if (src0->type != src1->type || !ggml_are_same_shape(src0, src1)) { + return false; + } + + // grow the contiguous prefix block shared by both tensors + size_t block_nb = ggml_element_size(src0); + int d = 0; + for (; d < GGML_MAX_DIMS; ++d) { + if (src0->nb[d] != block_nb || src1->nb[d] != block_nb) { + break; + } + block_nb *= src0->ne[d]; + } + + // d == 0: nothing contiguous; d == GGML_MAX_DIMS: fully contiguous (handled by memcpy) + if (d == 0 || d == GGML_MAX_DIMS) { + return false; + } + + // dim d carries the rows; everything above it must be a single element + for (int i = d + 1; i < GGML_MAX_DIMS; ++i) { + if (src0->ne[i] != 1) { + return false; + } + } + + width = block_nb; + height = src0->ne[d]; + spitch = src0->nb[d]; + dpitch = src1->nb[d]; + + return spitch >= width && dpitch >= width; +} + void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) { const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); @@ -421,6 +461,8 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) && src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0); + size_t mc_width = 0, mc_height = 0, mc_spitch = 0, mc_dpitch = 0; + if (src0->type == src1->type && contiguous_srcs) { GGML_ASSERT(ggml_nbytes(src0) == ggml_nbytes(src1)); #if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY) @@ -431,6 +473,9 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg { CUDA_CHECK(cudaMemcpyAsync(src1_ddc, src0_ddc, ggml_nbytes(src0), cudaMemcpyDeviceToDevice, main_stream)); } + } else if (ggml_cuda_cpy_as_memcpy_2d(src0, src1, mc_width, mc_height, mc_spitch, mc_dpitch)) { + CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch, + mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream)); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { if (can_be_transposed) { ggml_cpy_scalar_cuda diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 8dfa51ad1e8..e67cc7fdf78 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -664,7 +664,10 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { template __launch_bounds__(FATTN_KQ_STRIDE/2, 1) static __global__ void flash_attn_mask_to_KV_max( - const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) { + const half2 * mask_ptr, int * KV_max_ptr, const int ne30, const int64_t s31, const int64_t s33) { + const half2 * GGML_CUDA_RESTRICT mask = mask_ptr; + int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + const int ne31 = gridDim.x; const int tid = threadIdx.x; const int sequence = blockIdx.y; @@ -1089,8 +1092,8 @@ void launch_fattn( // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or // multiple sequences of possibly different lengths. if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) { - const int s31 = mask->nb[1] / sizeof(half2); - const int s33 = mask->nb[3] / sizeof(half2); + const int64_t s31 = mask->nb[1] / sizeof(half2); + const int64_t s33 = mask->nb[3] / sizeof(half2); const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1); const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1); @@ -1099,8 +1102,9 @@ void launch_fattn( const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; KV_max.alloc(ne_KV_max); - flash_attn_mask_to_KV_max<<>> - ((const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); + ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_mask_to_KV_max, launch_params, + (const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); CUDA_CHECK(cudaGetLastError()); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 83478a02cb6..7f4cfd5511f 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -2003,6 +2003,10 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 64) +extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); extern DECL_FATTN_MMA_F16_CASE(512, 512, 2, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 4); diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index 0a099810e14..3e07a9f7e04 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -76,6 +76,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -144,6 +145,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 32, 64) @@ -219,6 +221,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 512, 1, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -296,6 +299,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 256, 2, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 4, 64, 64) @@ -1308,12 +1312,12 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm return; } - if constexpr (DV <= 256) { - if (use_gqa_opt && gqa_ratio % 2 == 0) { - launch_fattn_tile_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio % 2 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } + if constexpr (DV <= 256) { launch_fattn_tile_switch_ncols1(ctx, dst); return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index d6c501b1d7e..00ffacf2992 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -99,12 +99,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con return; } - if constexpr (DKQ <= 256) { - if (use_gqa_opt && gqa_ratio > 1) { - ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio > 1) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + if constexpr (DKQ <= 256) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); } else { GGML_ABORT("fatal error"); @@ -337,6 +337,26 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_MMA_F16 = 400, }; +static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { + switch (type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + return true; + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: +#ifndef GGML_CUDA_FA_ALL_QUANTS + return false; +#endif // GGML_CUDA_FA_ALL_QUANTS + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_BF16: + return true; + default: + return false; + } +} + static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { #ifndef FLASH_ATTN_AVAILABLE GGML_UNUSED(device); GGML_UNUSED(dst); @@ -427,22 +447,8 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } #endif // GGML_CUDA_FA_ALL_QUANTS - switch (K->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - break; - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: -#ifndef GGML_CUDA_FA_ALL_QUANTS - return BEST_FATTN_KERNEL_NONE; -#endif // GGML_CUDA_FA_ALL_QUANTS - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q8_0: - case GGML_TYPE_BF16: - break; - default: - return BEST_FATTN_KERNEL_NONE; + if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) { + return BEST_FATTN_KERNEL_NONE; } if (mask && mask->ne[2] != 1) { diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index a547360eb06..1b431a724d7 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -10,6 +10,7 @@ gated_delta_net_cuda(const float * q, const float * beta, const float * curr_state, float * dst, + float * state, int64_t H, int64_t n_tokens, int64_t n_seqs, @@ -25,6 +26,7 @@ gated_delta_net_cuda(const float * q, const uint3 neqk1_magic, const uint3 rq3_magic, float scale, + int64_t state_slot_stride, int K) { const uint32_t h_idx = blockIdx.x; const uint32_t sequence = blockIdx.y; @@ -35,9 +37,7 @@ gated_delta_net_cuda(const float * q, const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); const uint32_t iq3 = fastdiv(sequence, rq3_magic); - const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; float * attn_data = dst; - float * state = dst + attn_score_elems; // input state holds s0 only: [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. @@ -145,10 +145,9 @@ gated_delta_net_cuda(const float * q, if constexpr (keep_rs_t) { // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. - const int64_t state_size_per_token = S_v * S_v * H * n_seqs; // per-slot stride in output const int target_slot = (int) n_tokens - 1 - t; if (target_slot >= 0 && target_slot < K) { - float * curr_state = (dst + attn_score_elems) + target_slot * state_size_per_token + state_out_offset; + float * curr_state = state + target_slot * state_slot_stride; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { const int i = r * warp_size + lane; @@ -171,13 +170,13 @@ template static void launch_gated_delta_net( const float * q_d, const float * k_d, const float * v_d, const float * g_d, const float * b_d, const float * s_d, - float * dst_d, + float * dst_d, float * state_d, int64_t S_v, int64_t H, int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, - float scale, int K, cudaStream_t stream) { + float scale, int64_t state_slot_stride, int K, cudaStream_t stream) { //TODO: Add chunked kernel for even faster pre-fill const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; const int num_warps = 4; @@ -187,34 +186,32 @@ static void launch_gated_delta_net( const uint3 neqk1_magic = init_fastdiv_values(neqk1); const uint3 rq3_magic = init_fastdiv_values(rq3); - int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); switch (S_v) { case 16: ggml_cuda_kernel_launch(gated_delta_net_cuda<16, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 32: ggml_cuda_kernel_launch(gated_delta_net_cuda<32, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 64: { ggml_cuda_kernel_launch(gated_delta_net_cuda<64, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } case 128: { ggml_cuda_kernel_launch(gated_delta_net_cuda<128, KDA, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, K); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } default: @@ -223,7 +220,8 @@ static void launch_gated_delta_net( } } -void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +static void ggml_cuda_op_gated_delta_net_impl( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_cuda_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; ggml_tensor * src_k = dst->src[1]; ggml_tensor * src_v = dst->src[2]; @@ -288,25 +286,42 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * const int K = ggml_get_op_params_i32(dst, 0); const bool keep_rs = K > 1; + // recurrent state -> gdn_out tail (after attention scores), or the cache when fusing + float * state_d = dst_d + S_v * H * n_tokens * n_seqs; + int64_t state_slot_stride = S_v * S_v * H * n_seqs; + if (cache != nullptr) { + state_d = cache->data; + state_slot_stride = cache->slot_stride; + } + if (kda) { if (keep_rs) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } } else { if (keep_rs) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); } } } + +void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, nullptr); +} + +void ggml_cuda_op_gated_delta_net_fused_cache( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, &cache); +} diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh index 7375e81c0c3..f9bf4370678 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cuh +++ b/ggml/src/ggml-cuda/gated_delta_net.cuh @@ -1,4 +1,14 @@ #include "common.cuh" #include "ggml.h" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always D, set in-kernel) +struct ggml_cuda_gated_delta_net_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots (0 when K==1) +}; + void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion) +void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_gated_delta_net_fused_cache cache); diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index eb157b8baf2..0e15707093f 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -78,26 +78,29 @@ static __global__ void k_get_rows_float( template static __global__ void k_get_rows_back_float( - const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) { + const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, + const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) { const int col = blockIdx.x*blockDim.x + threadIdx.x; if (col >= ncols) { return; } - const int dst_row = blockIdx.y*blockDim.y + threadIdx.y; + ggml_cuda_pdl_sync(); - float sum = 0.0f; + // grid.y is clamped to the CUDA grid limit, so stride over the destination rows + for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) { + float sum = 0.0f; - ggml_cuda_pdl_sync(); - for (int64_t i = 0; i < nrows_grad; ++i) { - if (rows[i] != dst_row) { - continue; + for (int64_t i = 0; i < nrows_grad; ++i) { + if (rows[i] != dst_row) { + continue; + } + sum += grad[i*ncols + col]; } - sum += grad[i*ncols + col]; - } - dst[dst_row*ncols + col] = sum; + dst[dst_row*ncols + col] = sum; + } } template @@ -302,7 +305,7 @@ void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * d const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1); const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE; - const dim3 block_nums(block_num_x, ne1, 1); + const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1); - k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10); + k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10, ne1); } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index cca70592f80..98816f885cf 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -543,12 +543,42 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { // the memory allocation handle is no longer needed after mapping CU_CHECK(cuMemRelease(handle)); - // set access - CUmemAccessDesc access = {}; - access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - access.location.id = device; - access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; - CU_CHECK(cuMemSetAccess((CUdeviceptr)((char *)(pool_addr) + pool_size), reserve_size, &access, 1)); + // VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build + bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr; +#if defined(GGML_USE_NCCL) + use_peer_access = true; +#endif // defined(GGML_USE_NCCL) + + if (use_peer_access) { + // NCCL implicitly enables peer access (cudaDeviceEnablePeerAccess), and + // GGML_CUDA_P2P enables it explicitly. Unlike cudaMalloc buffers, VMM + // allocations do not become peer-accessible from that alone, so access + // must be granted explicitly here. + std::vector access_descs; + const int device_count = ggml_cuda_info().device_count; + for (int id = 0; id < device_count; ++id) { + if (id != device) { + int can_access_peer = 0; + CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, device)); + if (!can_access_peer) { + continue; + } + } + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = id; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + access_descs.push_back(access); + } + CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, access_descs.data(), access_descs.size())); + } else { + // set access for non P2P + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = device; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, &access, 1)); + } // add to the pool pool_size += reserve_size; @@ -854,299 +884,6 @@ ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { return &ggml_backend_cuda_buffer_types[device]; } -// cuda split buffer - -static int64_t get_row_rounding(const std::array & tensor_split) { - int64_t row_rounding = 0; - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { - continue; - } - - const int cc = ggml_cuda_info().devices[id].cc; - row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc)); - } - return row_rounding; -} - -static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { - const int64_t nrows = ggml_nrows(tensor); - const int64_t rounding = get_row_rounding(tensor_split); - - *row_low = id == 0 ? 0 : nrows*tensor_split[id]; - *row_low -= *row_low % rounding; - - if (id == ggml_backend_cuda_get_device_count() - 1) { - *row_high = nrows; - } else { - *row_high = nrows*tensor_split[id + 1]; - *row_high -= *row_high % rounding; - } -} - -static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) { - static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - - return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); -} - -struct ggml_backend_cuda_split_buffer_type_context { - int main_device; - std::array tensor_split; - std::string name; -}; - -struct ggml_backend_cuda_split_buffer_context { - ~ggml_backend_cuda_split_buffer_context() { - for (ggml_tensor_extra_gpu * extra : tensor_extras) { - for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) { - for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { - if (extra->events[id][is] != nullptr) { - CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); - } - } - if (extra->data_device[id] != nullptr) { - CUDA_CHECK(cudaFree(extra->data_device[id])); - } - } - delete extra; - } - } - - std::vector tensor_extras; -}; - - -static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - delete ctx; -} - -static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { - // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced - return (void *)0x1000; - - GGML_UNUSED(buffer); -} - -static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - - ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; - ctx->tensor_extras.push_back(extra); - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - // FIXME: do not crash if cudaMalloc fails - // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first - ggml_cuda_set_device(id); - char * buf; - CUDA_CHECK(ggml_cuda_device_malloc((void**)&buf, size, id)); - - // set padding to 0 to avoid possible NaN values - if (size > original_size) { - CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); - } - - extra->data_device[id] = buf; - - for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { - CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); - } - } - tensor->extra = extra; - return GGML_STATUS_SUCCESS; -} - -static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - const char * buf_host = (const char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread)); - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - char * buf_host = (char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - GGML_UNUSED(buffer); - GGML_UNUSED(value); -} - -static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { - /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer, - /* .get_base = */ ggml_backend_cuda_split_buffer_get_base, - /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ NULL, - /* .clear = */ ggml_backend_cuda_split_buffer_clear, - /* .reset = */ NULL, -}; - -// cuda split buffer type - -static const char * ggml_backend_cuda_split_buffer_type_get_name(ggml_backend_buffer_type_t buft) { - ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; - - return ctx->name.c_str(); -} - -static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) { - return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name; -} - -static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point - // instead, we allocate them for each tensor separately in init_tensor - // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, - // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. - ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context(); - - return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); -} - -static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; - - GGML_UNUSED(buft); -} - -static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - size_t total_size = 0; - - const int64_t ne0 = tensor->ne[0]; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - total_size += ggml_nbytes_split(tensor, nrows_split); - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - } - - return total_size; -} - -static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; - - GGML_UNUSED(buft); -} - -static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = { - /* .get_name = */ ggml_backend_cuda_split_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, -}; - // Communication context for multi-GPU AllReduce during tensor parallelism. // // Created once per meta backend instance. Resources for the selected mode @@ -1436,48 +1173,6 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg return comm_ctx->try_allreduce(comm_ctx, tensors); } -ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) { - static std::mutex mutex; - std::lock_guard lock(mutex); - - static std::map>, struct ggml_backend_buffer_type> buft_map; - - std::array tensor_split_arr = {}; - - bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); - if (all_zero) { - tensor_split_arr = ggml_cuda_info().default_tensor_split; - } else { - float split_sum = 0.0f; - for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { - tensor_split_arr[i] = split_sum; - split_sum += tensor_split[i]; - } - for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { - tensor_split_arr[i] /= split_sum; - } - } - - auto it = buft_map.find({main_device, tensor_split_arr}); - if (it != buft_map.end()) { - return &it->second; - } - auto * ctx = new ggml_backend_cuda_split_buffer_type_context{ - main_device, - tensor_split_arr, - GGML_CUDA_NAME + std::to_string(main_device) + "_Split", - }; - - struct ggml_backend_buffer_type buft { - /* .iface = */ ggml_backend_cuda_split_buffer_type_interface, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), main_device), - /* .context = */ ctx, - }; - - auto result = buft_map.emplace(std::make_pair(main_device, tensor_split_arr), buft); - return &result.first->second; -} - // host buffer type static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) { @@ -1553,574 +1248,8 @@ ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { typedef void (*ggml_cuda_op_mul_mat_t)( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, - const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, - const int64_t src1_padded_row_size, cudaStream_t stream); - -#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE -#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128 -#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE - -#define MUL_MAT_SRC1_COL_STRIDE 128 - -static cudaError_t ggml_cuda_cpy_tensor_2d( - void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) { - - const char * src_ptr = (const char *) src->data; - char * dst_ptr = (char *) dst; - - const int64_t ne0 = src->ne[0]; - const int64_t nb0 = src->nb[0]; - const int64_t nb1 = src->nb[1]; - const int64_t nb2 = src->nb[2]; - const int64_t nb3 = src->nb[3]; - const enum ggml_type type = src->type; - const int64_t ts = ggml_type_size(type); - const int64_t bs = ggml_blck_size(type); - const int64_t i1_diff = i1_high - i1_low; - - const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3; - if (nb0 == ts && nb1 == ts*ne0/bs) { - return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream); - } else if (nb0 == ts) { - return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream); - } else { - for (int64_t i1 = 0; i1 < i1_diff; i1++) { - const void * rx = (const void *) ((const char *) x + i1*nb1); - void * rd = (void *) (dst_ptr + i1*ts*ne0/bs); - // pretend the row is a matrix with cols=1 - cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream); - if (r != cudaSuccess) { - return r; - } - } - return cudaSuccess; - } -} - -struct cublas_force_compute_type { - bool fp32 = false; - bool fp16 = false; -}; - -static const cublas_force_compute_type & ggml_cuda_cublas_get_force_compute_type() { - static const cublas_force_compute_type compute_type = [] { - cublas_force_compute_type result; - - const bool ggml_cuda_force_cublas_compute_32f_env = getenv("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F") != nullptr; - const bool ggml_cuda_force_cublas_compute_16f_env = getenv("GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F") != nullptr; - - GGML_ASSERT(ggml_cuda_force_cublas_compute_16f_env == false || ggml_cuda_force_cublas_compute_32f_env == false); - - if (ggml_cuda_force_cublas_compute_32f_env) { - GGML_LOG_INFO("Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F\n"); - result.fp32 = true; - } else if (ggml_cuda_force_cublas_compute_16f_env) { - GGML_LOG_INFO("Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F\n"); - result.fp16 = true; - } - - return result; - }(); - - return compute_type; -} - -static void ggml_cuda_op_mul_mat_cublas( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, - const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, - const int64_t src1_padded_row_size, cudaStream_t stream) { - - GGML_ASSERT(src0_dd_i != nullptr); - GGML_ASSERT(src1_ddf_i != nullptr); - GGML_ASSERT(dst_dd_i != nullptr); - - const int64_t ne00 = src0->ne[0]; - const int64_t ne10 = src1->ne[0]; - - const int64_t ne0 = dst->ne[0]; - - const int64_t row_diff = row_high - row_low; - - int id = ggml_cuda_get_device(); - - // the main device has a larger memory buffer to hold the results from all GPUs - // ldc == nrows of the matrix that cuBLAS writes into - int64_t ldc = id == ctx.device ? ne0 : row_diff; - - const int cc = ggml_cuda_info().devices[id].cc; - - const bool supports_bf16 = GGML_CUDA_CC_IS_NVIDIA(cc) || GGML_CUDA_CC_IS_AMD(cc) || - (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); - - const bool use_fp16 = - src0->type != GGML_TYPE_NVFP4 && - (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - ggml_is_contiguous(src0) && - row_diff == src0->ne[1] && - dst->op_params[0] == GGML_PREC_DEFAULT; - - if (supports_bf16 && src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) { - ggml_cuda_pool_alloc src1_as_bf16(ctx.pool(id)); - if (src1->type != GGML_TYPE_BF16) { - const to_bf16_cuda_t to_bf16_cuda = ggml_get_to_bf16_cuda(src1->type); - GGML_ASSERT(to_bf16_cuda != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_bf16.alloc(ne); - to_bf16_cuda(src1_ddf_i, src1_as_bf16.get(), ne, stream); - } - const nv_bfloat16 * src1_ptr = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1_ddf_i : src1_as_bf16.get(); - const nv_bfloat16 * src0_ptr = (const nv_bfloat16 *)src0_dd_i; - ggml_cuda_pool_alloc dst_bf16(ctx.pool(id), row_diff*src1_ncols); - - const float alpha_f32 = 1.0f; - const float beta_f32 = 0.0f; - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha_f32, src0_ptr, CUDA_R_16BF, ne00, - src1_ptr, CUDA_R_16BF, ne10, - &beta_f32, dst_bf16.get(), CUDA_R_16BF, ldc, - CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_BF16); - to_fp32_cuda(dst_bf16.get(), dst_dd_i, row_diff*src1_ncols, stream); - } else if (fast_fp16_hardware_available(cc) && use_fp16) { - // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32 - ggml_cuda_pool_alloc src0_as_f16(ctx.pool(id)); - if (src0->type != GGML_TYPE_F16) { - const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type); - GGML_ASSERT(to_fp16_cuda != nullptr); - size_t ne = row_diff*ne00; - src0_as_f16.alloc(ne); - to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream); - } - const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get(); - - ggml_cuda_pool_alloc src1_as_f16(ctx.pool(id)); - if (src1->type != GGML_TYPE_F16) { - const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type); - GGML_ASSERT(to_fp16_cuda != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_f16.alloc(ne); - to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream); - } - const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get(); - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - - const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type(); - - if (!force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc) - || GGML_CUDA_CC_IS_RDNA4(cc) - || cc == GGML_CUDA_CC_VOLTA - || force_compute_type.fp32)) - { - const float alpha = 1.0f; - const float beta = 0.0f; - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha, src0_ptr, CUDA_R_16F, ne00, - src1_ptr, CUDA_R_16F, ne10, - &beta, dst_dd_i, CUDA_R_32F, ldc, - CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - } else { - ggml_cuda_pool_alloc dst_f16(ctx.pool(id), row_diff*src1_ncols); - - const half alpha_f16 = 1.0f; - const half beta_f16 = 0.0f; - - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha_f16, src0_ptr, CUDA_R_16F, ne00, - src1_ptr, CUDA_R_16F, ne10, - &beta_f16, dst_f16.get(), CUDA_R_16F, ldc, - CUBLAS_COMPUTE_16F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16); - to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); - } - } else { - ggml_cuda_pool_alloc src0_ddq_as_f32(ctx.pool(id)); - ggml_cuda_pool_alloc src1_ddq_as_f32(ctx.pool(id)); - - if (src0->type != GGML_TYPE_F32) { - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type); - GGML_ASSERT(to_fp32_cuda != nullptr); - src0_ddq_as_f32.alloc(row_diff*ne00); - to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream); - } - if (src1->type != GGML_TYPE_F32) { - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type); - GGML_ASSERT(to_fp32_cuda != nullptr); - src1_ddq_as_f32.alloc(src1_ncols*ne10); - to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream); - } - - const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get(); - const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get(); - - const float alpha = 1.0f; - const float beta = 0.0f; - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - CUBLAS_CHECK( - cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha, src0_ddf_i, ne00, - src1_ddf1_i, ne10, - &beta, dst_dd_i, ldc)); - } - - GGML_UNUSED_VARS(dst, src1_ddq_i, src1_padded_row_size); -} - -static cudaError_t ggml_cuda_Memcpy2DPeerAsync( - void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) { - -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices - cudaMemcpy3DPeerParms p = {}; - p.dstDevice = dstDevice; - p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height); - p.srcDevice = srcDevice; - p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height); - p.extent = make_cudaExtent(width, height, 1); - return cudaMemcpy3DPeerAsync(&p, stream); -#else - // HIP does not support cudaMemcpy3DPeerAsync or vmm pools - GGML_UNUSED(dstDevice); - GGML_UNUSED(srcDevice); - return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream); -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -} - -static void ggml_cuda_op_mul_mat( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op, - quantize_cuda_t quantize_src1) { - - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; - const int64_t ne03 = src0->ne[3]; - - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - const int64_t ne13 = src1->ne[3]; - const int64_t nrows1 = ggml_nrows(src1); - - const int64_t ne0 = dst->ne[0]; - const int64_t ne1 = dst->ne[1]; - - // const int64_t nb10 = src1->nb[0]; - const int64_t nb11 = src1->nb[1]; - const int64_t nb12 = src1->nb[2]; - const int64_t nb13 = src1->nb[3]; - - const int64_t nb2 = dst->nb[2]; - const int64_t nb3 = dst->nb[3]; - - ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context; - ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context; - - GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); - - GGML_ASSERT(ne12 % ne02 == 0); - GGML_ASSERT(ne13 % ne03 == 0); - - const int64_t i02_divisor = ne12 / ne02; - const int64_t i03_divisor = ne13 / ne03; - - const size_t src0_ts = ggml_type_size(src0->type); - const size_t src0_bs = ggml_blck_size(src0->type); - const size_t q8_1_ts = sizeof(block_q8_1); - const size_t q8_1_bs = QK8_1; - - const bool src0_is_contiguous = ggml_is_contiguous(src0); - const bool src1_is_contiguous = ggml_is_contiguous(src1); - - const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING); - - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft); - GGML_ASSERT(!(split && ne02 > 1)); - GGML_ASSERT(!(split && ne03 > 1)); - GGML_ASSERT(!(split && ne02 < ne12)); - GGML_ASSERT(!(split && ne03 < ne13)); - - ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr; - - - std::array tensor_split; - if (split) { - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; - tensor_split = buft_ctx->tensor_split; - } - - struct dev_data { - int cc; - - ggml_cuda_pool_alloc src0_dd_alloc; - ggml_cuda_pool_alloc src1_ddf_alloc; - ggml_cuda_pool_alloc src1_ddq_alloc; - ggml_cuda_pool_alloc dst_dd_alloc; - - char * src0_dd = nullptr; - float * src1_ddf = nullptr; // float - char * src1_ddq = nullptr; // q8_1 - float * dst_dd = nullptr; - - int64_t row_low; - int64_t row_high; - }; - - dev_data dev[GGML_CUDA_MAX_DEVICES]; - - int used_devices = 0; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - dev[id].cc = ggml_cuda_info().devices[id].cc; - - // by default, use all rows - dev[id].row_low = 0; - dev[id].row_high = ne01; - - // for multi GPU, get the row boundaries from tensor split - // and round to mul_mat_q tile sizes - if (split) { - const int64_t rounding = get_row_rounding(tensor_split); - - if (id != 0) { - dev[id].row_low = ne01*tensor_split[id]; - if (dev[id].row_low < ne01) { - dev[id].row_low -= dev[id].row_low % rounding; - } - } - - if (id != ggml_backend_cuda_get_device_count() - 1) { - dev[id].row_high = ne01*tensor_split[id + 1]; - if (dev[id].row_high < ne01) { - dev[id].row_high -= dev[id].row_high % rounding; - } - } - } - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { - continue; - } - - used_devices++; - - const bool src1_on_device = id == src1_ctx->device; - const bool dst_on_device = id == dst_ctx->device; - - ggml_cuda_set_device(id); - cudaStream_t stream = ctx.stream(id, 0); - - if (src0_is_contiguous) { - dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data; - } else { - // If src0 is not contiguous it will be copied to a temporary buffer. - // This buffer needs to be cleared entirely because multiple regions will function as padding. - const size_t nbytes_data = ggml_nbytes(src0); - const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING); - dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), nbytes_data + nbytes_padding); - CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd, 0, nbytes_data + nbytes_padding, stream)); - } - - // If src0 is on a temporary compute buffer (partial offloading) there may be some padding that needs to be cleared: - if (ne00 % MATRIX_ROW_PADDING != 0 && ggml_is_quantized(src0->type) && ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && src0->view_src == nullptr) { - GGML_ASSERT(ggml_is_contiguously_allocated(src0)); - GGML_ASSERT(!src0->view_src); - const size_t nbytes_data = ggml_row_size(src0->type, (dev[id].row_high - dev[id].row_low)*ne00); - const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING); - CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd + nbytes_data, 0, nbytes_padding, stream)); - } - - if (src1_on_device && src1_is_contiguous) { - dev[id].src1_ddf = (float *) src1->data; - } else { - dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1)); - } - - if (quantize_src1) { - size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq); - } - dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size); - - if (src1_on_device && src1_is_contiguous) { - quantize_src1( - dev[id].src1_ddf, nullptr, dev[id].src1_ddq, src0->type, ne10, - nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float), - src1_padded_col_size, ne11, ne12, ne13, stream); - CUDA_CHECK(cudaGetLastError()); - } - } - - if (dst_on_device) { - dev[id].dst_dd = (float *) dst->data; - } else { - const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst); - dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf); - } - } - - // if multiple devices are used they need to wait for the main device - // here an event is recorded that signals that the main device has finished calculating the input data - if (split && used_devices > 1) { - ggml_cuda_set_device(ctx.device); - CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream())); - } - - const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11; - for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) { - const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0; - const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { - continue; - } - - const bool src1_on_device = id == src1_ctx->device; - const bool dst_on_device = id == dst_ctx->device; - const int64_t row_diff = dev[id].row_high - dev[id].row_low; - - ggml_cuda_set_device(id); - cudaStream_t stream = ctx.stream(id, is); - - // wait for main GPU data if necessary - if (split && (id != ctx.device || is != 0)) { - CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0)); - } - - for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) { - const int64_t i03 = i0 / ne12; - const int64_t i02 = i0 % ne12; - - size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq); - } else { - src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs; - } - - // for split tensors the data begins at i0 == i0_offset_low - const size_t nbytes_src0_matrix = ne01*ne00*src0_ts / src0_bs; - char * src0_dd_i = dev[id].src0_dd + ((i03/i03_divisor)*ne02 + (i02/i02_divisor)) * nbytes_src0_matrix; - float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10; - char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset; - float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); - - // the main device memory buffer can be on VRAM scratch, with space for all partial results - // in that case an offset on dst_ddf_i is needed - if (id == ctx.device) { - dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split - } - - // copy src0, src1 to device if necessary - if (src1_is_contiguous) { - if (id != ctx.device) { - if (quantize_src1) { - char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - const size_t pitch = ne11*sizeof(block_q8_1_mmq); - const size_t width = src1_ncols*sizeof(block_q8_1_mmq); - const size_t height = src1_padded_col_size/(4*QK8_1); - CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream)); - } else { - CUDA_CHECK(cudaMemcpyPeerAsync( - src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream)); - } - } else { - float * src1_ddf_i_source = (float *) src1->data; - src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10; - CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device, - src1_ncols*ne10*sizeof(float), stream)); - } - } - } else if (src1_on_device && !src1_is_contiguous) { - CUDA_CHECK(ggml_cuda_cpy_tensor_2d( - src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream)); - } else { - GGML_ABORT("fatal error"); - } - - if (quantize_src1 && !src1_is_contiguous) { - quantize_src1( - src1_ddf_i, nullptr, src1_ddq_i, src0->type, ne10, ne10, ne11*ne10, ne12*ne11*ne10, - src1_padded_col_size, src1_ncols, 1, 1, stream); - CUDA_CHECK(cudaGetLastError()); - } - - if (src1_col_0 == 0 && !src0_is_contiguous && i03 % i03_divisor == 0 && i02 % i02_divisor == 0) { - CUDA_CHECK(ggml_cuda_cpy_tensor_2d( - src0_dd_i, src0, i03/i03_divisor, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream)); - } - - // do the computation - op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i, - dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream); - CUDA_CHECK(cudaGetLastError()); - - // copy dst to host or other device if necessary - if (!dst_on_device) { - void * dst_off_device = dst->data; - if (split) { - // src0 = weight matrix is saved as a transposed matrix for better memory layout. - // dst is NOT transposed. - // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU. - // Instead they need to be copied to the correct slice in ne0 = dst row index. - // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results. - float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); - GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); - dhf_dst_i += src1_col_0*ne0 + dev[id].row_low; - CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync( - dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream)); - } else { - float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); - GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); - dhf_dst_i += src1_col_0*ne0; - CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream)); - } - } - - // add event for the main device to wait on until other device is done - if (split && (id != ctx.device || is != 0)) { - CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream)); - } - } - } - } - - // main device waits for all other devices to be finished - if (split && ggml_backend_cuda_get_device_count() > 1) { - int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE; - is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS; - - ggml_cuda_set_device(ctx.device); - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if (dev[id].row_low == dev[id].row_high) { - continue; - } - for (int64_t is = 0; is < is_max; ++is) { - CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0)); - } - } - } -} + const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, + const int64_t src1_padded_row_size, cudaStream_t stream); static __global__ void k_compute_batched_ptrs( const void * src0_as_f16, const void * src1_as_f16, char * dst, @@ -2160,7 +1289,8 @@ struct batched_mul_mat_traits { static inline const float beta = 0.0f; static inline const void* get_alpha() { static const float val = alpha; return &val; } static inline const void* get_beta() { static const float val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp32_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_fp32_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp32_nc_cuda(src_type); } }; template<> @@ -2173,7 +1303,8 @@ struct batched_mul_mat_traits { static inline const float beta = 0.0f; static inline const void* get_alpha() { static const float val = alpha; return &val; } static inline const void* get_beta() { static const float val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_bf16_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_bf16_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_bf16_nc_cuda(src_type); } }; template<> @@ -2186,18 +1317,15 @@ struct batched_mul_mat_traits { static inline const half beta = 0.0; static inline const void* get_alpha() { static const half val = alpha; return &val; } static inline const void* get_beta() { static const half val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_fp16_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } }; -template -static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - using traits = batched_mul_mat_traits; +template +static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + using traits = batched_mul_mat_traits; using cuda_t = typename traits::cuda_type; - GGML_ASSERT(!ggml_is_transposed(src0)); - GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft)); - GGML_ASSERT(src0->type == src0_type); GGML_ASSERT(ggml_is_contiguous(dst)); // Byte offsets and tensor dimensions are currently used in an inconsistent way for dst. @@ -2209,12 +1337,19 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct cudaStream_t main_stream = ctx.stream(); CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); + const size_t src0_ts = ggml_type_size(src0->type); + GGML_ASSERT(nb00 == src0_ts); + int64_t s01 = nb01 / src0_ts; + int64_t s02 = nb02 / src0_ts; + int64_t s03 = nb03 / src0_ts; + + const size_t src1_ts = ggml_type_size(src1->type); + GGML_ASSERT(nb10 == src1_ts); + int64_t s11 = nb11 / src1_ts; + int64_t s12 = nb12 / src1_ts; + int64_t s13 = nb13 / src1_ts; + float * dst_ddf = (float *) dst->data; - const size_t ts_src1 = ggml_type_size(src1->type); - GGML_ASSERT(nb10 == ts_src1); - int64_t s11 = nb11 / ts_src1; - int64_t s12 = nb12 / ts_src1; - int64_t s13 = nb13 / ts_src1; const cuda_t * src0_ptr = nullptr; const cuda_t * src1_ptr = nullptr; @@ -2225,31 +1360,58 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct bool is_src0_cont_2 = ggml_is_contiguous_2(src0); bool is_src1_cont_2 = ggml_is_contiguous_2(src1); - // Handle src0 - src0_ptr = (const cuda_t *) src0->data; + if (src0->type == compute_type) { + src0_ptr = (const cuda_t *) src0->data; + } else { + src0_alloc.alloc(ggml_nelements(src0)); + + if (ggml_is_contiguously_allocated(src0)) { + const auto convert_func = traits::convert(src0->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src0->data, src0_alloc.get(), ggml_nelements(src0), main_stream); + const size_t src0_bs = ggml_blck_size(src0->type); + s01 *= src0_bs; + s02 *= src0_bs; + s03 *= src0_bs; + } else { + const auto convert_func = traits::convert_nc(src0->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src0->data, src0_alloc.get(), ne00, ne01, ne02, ne03, s01, s02, s03, main_stream); + s01 = ne00; + s02 = ne01*s01; + s03 = ne02*s02; + is_src0_cont_2 = true; + } + src0_ptr = src0_alloc.get(); + } - // Handle src1 - convert if necessary - if (src1->type == src0_type) { + if (src1->type == compute_type) { src1_ptr = (const cuda_t *) src1->data; } else { - // Convert src1 to target type using traits conversion functions - const int64_t ne_src1 = ggml_nelements(src1); - src1_alloc.alloc(ne_src1); - - const auto convert_func = traits::get_nc_converter(src1->type); - GGML_ASSERT(convert_func != nullptr); - convert_func(src1->data, src1_alloc.get(), ne10, ne11, ne12, ne13, s11, s12, s13, main_stream); + src1_alloc.alloc(ggml_nelements(src1)); + + if (ggml_is_contiguously_allocated(src1)) { + const auto convert_func = traits::convert(src1->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src1->data, src1_alloc.get(), ggml_nelements(src1), main_stream); + const size_t src1_bs = ggml_blck_size(src1->type); + s11 *= src1_bs; + s12 *= src1_bs; + s13 *= src1_bs; + } else { + const auto convert_func = traits::convert_nc(src1->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src1->data, src1_alloc.get(), ne10, ne11, ne12, ne13, s11, s12, s13, main_stream); + s11 = ne10; + s12 = ne11*s11; + s13 = ne12*s12; + is_src1_cont_2 = true; + } src1_ptr = src1_alloc.get(); - s11 = ne10; - s12 = ne11*s11; - s13 = ne12*s12; - - is_src1_cont_2 = true; } - // Setup destination buffer ggml_cuda_pool_alloc dst_temp(ctx.pool()); - char * dst_t; + char * dst_ptr; size_t nbd2 = dst->nb[2]; size_t nbd3 = dst->nb[3]; @@ -2260,35 +1422,28 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct const void * alpha = traits::get_alpha(); const void * beta = traits::get_beta(); - const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type(); - - int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - static constexpr bool is_src0_type_f16 = src0_type == GGML_TYPE_F16; - - // bf16 and fp32 are already being computed in fp32 (ensure it using static_assert), - // so checking necessity of forced fp32 only for fp16 src0_type - static_assert(is_src0_type_f16 || traits::compute_type == CUBLAS_COMPUTE_32F); - - const bool need_compute_32f = is_src0_type_f16 && !force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc) - || GGML_CUDA_CC_IS_RDNA4(cc) - || cc == GGML_CUDA_CC_VOLTA - || force_compute_type.fp32); + const int cc = ggml_cuda_info().devices[ctx.device].cc; + bool prefer_f32_output = false; + if (compute_type == GGML_TYPE_F16) { + prefer_f32_output = cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc); + } else if (compute_type == GGML_TYPE_BF16) { + prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc); + } - if (dst->op_params[0] == GGML_PREC_DEFAULT && !need_compute_32f) { - if constexpr (src0_type == GGML_TYPE_F32) { - dst_t = (char *) dst_ddf; // Direct F32 output - } else { - dst_t = (char *) dst_temp.alloc(ne_dst); - nbd2 /= sizeof(float) / sizeof(cuda_t); - nbd3 /= sizeof(float) / sizeof(cuda_t); - } - } else { - dst_t = (char *) dst_ddf; + if (prefer_f32_output) { + dst_ptr = (char *) dst_ddf; cu_compute_type = batched_mul_mat_traits::compute_type; cu_data_type = batched_mul_mat_traits::data_type; alpha = batched_mul_mat_traits::get_alpha(); beta = batched_mul_mat_traits::get_beta(); + } else { + if constexpr (compute_type == GGML_TYPE_F32) { + dst_ptr = (char *) dst_ddf; // Direct F32 output + } else { + dst_ptr = (char *) dst_temp.alloc(ne_dst); + nbd2 /= sizeof(float) / sizeof(cuda_t); + nbd3 /= sizeof(float) / sizeof(cuda_t); + } } GGML_ASSERT(ne12 % ne02 == 0); @@ -2298,19 +1453,38 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct const int64_t r2 = ne12/ne02; const int64_t r3 = ne13/ne03; - if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) { + // Theoretically cublasGemmStridedBatchedEx would always work, even for a single matrix. + // However, for some old NVIDIA and AMD GPUs the strided/Ex GEMM is much slower, + // probably because the internal kernel selection logic is suboptimal. + if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) { + CUBLAS_CHECK( + cublasSgemm(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + ne01, ne11, ne10, + (const float *) alpha, (const float *) src0_ptr, s01, + (const float *) src1_ptr, s11, + (const float *) beta, (float *) dst_ptr, ne0)); + } else if (ne12 == 1 && ne13 == 1) { + CUBLAS_CHECK( + cublasGemmEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + ne01, ne11, ne10, + alpha, src0_ptr, cu_data_type_a, s01, + src1_ptr, cu_data_type_b, s11, + beta, dst_ptr, cu_data_type, ne0, + cu_compute_type, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } else if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) { // with a [0, 2, 1, 3] perm. and ne02==1 the matrix strides need to be determined from dim 3: - const int64_t sma = ne02 == 1 ? nb03/nb00 : nb02/nb00; - const int64_t smb = ne12 == 1 ? s13 : s12; + const int64_t sma = ne02 == 1 ? s03 : s02; + const int64_t smb = ne12 == 1 ? s13 : s12; // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, - alpha, src0_ptr, cu_data_type_a, nb01/nb00, sma, // strideA - src1_ptr, cu_data_type_b, s11, smb, // strideB - beta, dst_t, cu_data_type, ne0, ne1*ne0, // strideC + alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA + src1_ptr, cu_data_type_b, s11, smb, // strideB + beta, dst_ptr, cu_data_type, ne0, ne1*ne0, // strideC ne12*ne13, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); @@ -2321,24 +1495,23 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct ggml_cuda_pool_alloc ptrs_src(ctx.pool(), 2*ne23); ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23); - size_t src1_stride_size = sizeof(cuda_t); + const size_t src_type_size = sizeof(cuda_t); const int threads_x = 16; const int threads_y = 16; - dim3 block_dims(threads_x, threads_y); + const dim3 block_dims(threads_x, threads_y); - dim3 grid_dims( + const dim3 grid_dims( (ne13 + threads_x - 1) / threads_x, (ne12 + threads_y - 1) / threads_y ); k_compute_batched_ptrs<<>>( - src0_ptr, src1_ptr, dst_t, + src0_ptr, src1_ptr, dst_ptr, ptrs_src.get(), ptrs_dst.get(), ne12, ne13, ne23, - nb02, nb03, - (src1->type == src0_type) ? nb12 : s12*src1_stride_size, - (src1->type == src0_type) ? nb13 : s13*src1_stride_size, + s02*src_type_size, s03*src_type_size, + s12*src_type_size, s13*src_type_size, nbd2, nbd3, r2, r3); @@ -2347,7 +1520,7 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct CUBLAS_CHECK( cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, - alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, nb01/nb00, + alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne0, ne23, @@ -2356,27 +1529,52 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct } // Convert output back to F32 if needed - if (dst->op_params[0] == GGML_PREC_DEFAULT && cu_data_type != CUDA_R_32F) { + if (cu_data_type != CUDA_R_32F) { const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val); to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream); } } -static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || src0->type == GGML_TYPE_F32); +static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_type compute_type = src0->type; + if (ggml_is_quantized(compute_type)) { + compute_type = fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; + } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(ggml_cuda_info().devices[ctx.device].cc)) { + compute_type = GGML_TYPE_F32; + } + if (dst->op_params[0] == GGML_PREC_F32) { + compute_type = GGML_TYPE_F32; + } + + const char * env_c = getenv("GGML_CUDA_CUBLAS_COMPUTE_TYPE"); + if (env_c != nullptr) { + std::string env_cpp = env_c; + for (char & c : env_cpp) { + c = std::tolower(c); + } + if (env_cpp == "f32" || env_cpp == "fp32") { + compute_type = GGML_TYPE_F32; + } else if (env_cpp == "f16" || env_cpp == "fp16") { + compute_type = GGML_TYPE_F16; + } else if (env_cpp == "bf16") { + compute_type = GGML_TYPE_BF16; + } else if (env_cpp != "auto") { + GGML_LOG_WARN("%s: unknown value for GGML_CUDA_CUBLAS_COMPUTE_TYPE: %s", __func__, env_cpp.c_str()); + } + } - switch (src0->type) { + switch (compute_type) { case GGML_TYPE_F32: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; case GGML_TYPE_BF16: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; case GGML_TYPE_F16: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; default: - GGML_ABORT("Unsupported type"); + GGML_ABORT("fatal error"); } } @@ -2384,12 +1582,18 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, const ggml_tensor * ffn_gate, const ggml_tensor * glu, const ggml_tensor * ffn_up_bias = nullptr, - const ggml_tensor * ffn_gate_bias = nullptr) { + const ggml_tensor * ffn_gate_bias = nullptr, + const ggml_tensor * ffn_up_scale = nullptr, + const ggml_tensor * ffn_gate_scale = nullptr) { const bool has_bias = ffn_up_bias != nullptr || ffn_gate_bias != nullptr; + const bool has_scale = ffn_up_scale != nullptr || ffn_gate_scale != nullptr; if (has_bias && (!ffn_up_bias || !ffn_gate_bias)) { return false; } + if (has_scale && (!ffn_up_scale || !ffn_gate_scale)) { + return false; + } const bool is_mul_mat = ffn_up->op == GGML_OP_MUL_MAT && ffn_gate->op == GGML_OP_MUL_MAT && glu->op == GGML_OP_GLU; const bool is_mul_mat_id = ffn_up->op == GGML_OP_MUL_MAT_ID && ffn_gate->op == GGML_OP_MUL_MAT_ID && glu->op == GGML_OP_GLU; @@ -2401,34 +1605,45 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, } const ggml_op expected_bias_op = is_mul_mat ? GGML_OP_ADD : GGML_OP_ADD_ID; + const ggml_tensor * ffn_up_bias_src = has_scale ? ffn_up_scale : ffn_up; + const ggml_tensor * ffn_gate_bias_src = has_scale ? ffn_gate_scale : ffn_gate; + const ggml_tensor * ffn_up_out = has_bias ? ffn_up_bias : ffn_up_bias_src; + const ggml_tensor * ffn_gate_out = has_bias ? ffn_gate_bias : ffn_gate_bias_src; - if (has_bias) { - if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) { + if (glu->src[0] != ffn_gate_out || glu->src[1] != ffn_up_out) { + return false; + } + + if (has_scale) { + if (ffn_up_scale->op != GGML_OP_MUL || ffn_gate_scale->op != GGML_OP_MUL) { + return false; + } + const bool up_has_mm = ffn_up_scale->src[0] == ffn_up || ffn_up_scale->src[1] == ffn_up; + const bool gate_has_mm = ffn_gate_scale->src[0] == ffn_gate || ffn_gate_scale->src[1] == ffn_gate; + if (!up_has_mm || !gate_has_mm) { return false; } + } - if (glu->src[0] != ffn_gate_bias || glu->src[1] != ffn_up_bias) { + if (has_bias) { + if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) { return false; } if (expected_bias_op == GGML_OP_ADD) { - const bool up_has_mul = ffn_up_bias->src[0] == ffn_up || ffn_up_bias->src[1] == ffn_up; - const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate || ffn_gate_bias->src[1] == ffn_gate; + const bool up_has_mul = ffn_up_bias->src[0] == ffn_up_bias_src || ffn_up_bias->src[1] == ffn_up_bias_src; + const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate_bias_src || ffn_gate_bias->src[1] == ffn_gate_bias_src; if (!up_has_mul || !gate_has_mul) { return false; } } else { // GGML_OP_ADD_ID - if (ffn_up_bias->src[0] != ffn_up || ffn_gate_bias->src[0] != ffn_gate) { + if (ffn_up_bias->src[0] != ffn_up_bias_src || ffn_gate_bias->src[0] != ffn_gate_bias_src) { return false; } if (ffn_up_bias->src[2] != ffn_up->src[2] || ffn_gate_bias->src[2] != ffn_gate->src[2]) { return false; } } - } else { - if (glu->src[0] != ffn_gate && glu->src[1] != ffn_up) { - return false; - } } if (ffn_up->src[0]->type != ffn_gate->src[0]->type || !ggml_are_same_shape(ffn_up->src[0], ffn_gate->src[0]) || @@ -2440,7 +1655,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return false; } - if (ffn_up->src[2] && (ffn_up->src[2] != ffn_gate->src[2])) { + if (is_mul_mat_id && ffn_up->src[2] != ffn_gate->src[2]) { return false; } @@ -2454,14 +1669,6 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return false; } - const bool split = ggml_backend_buft_is_cuda_split(ffn_up->src[0]->buffer->buft) || - ggml_backend_buft_is_cuda_split(ffn_gate->src[0]->buffer->buft); - - //TODO: add support for fusion for split buffers - if (split) { - return false; - } - return true; } @@ -2479,14 +1686,6 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) || - ggml_backend_buft_is_cuda_split(src1->buffer->buft); - - //TODO: add support for fusion for split buffers - if (split) { - return false; - } - //we only support fusion for ncols_dst = 1 if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) { return false; @@ -2526,108 +1725,49 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { return false; } - - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) || - ggml_backend_buft_is_cuda_split(src1->buffer->buft); - - //TODO: add support for fusion for split buffers - if (split) { - return false; - } - return use_mul_mat_vec_q; } static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft); + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t hint = ggml_get_op_params_i32(dst, 1); + if (hint == GGML_HINT_SRC0_IS_HADAMARD && ggml_cuda_op_fwht(ctx, src1, dst)) { + return; + } // If src0 is a temporary compute buffer it may have some padding that needs to be cleared for mul_mat_vec_q or mul_mat_q. // But if src0 is also a view of another tensor then this cannot be done safely because it may overwrite valid tensor data. // Therefore, in such cases use cuBLAS. const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; - - bool use_mul_mat_vec_f = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - bool use_mul_mat_f = !ggml_is_quantized(src0->type) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 - && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; - bool use_mul_mat_q = ggml_is_quantized(src0->type) && !bad_padding_clear - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - - bool any_gpus_with_slow_fp16 = false; - - if (split) { - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; - auto & tensor_split = buft_ctx->tensor_split; - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - // skip devices that are not going to do any work: - if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { - continue; - } - - const int cc = ggml_cuda_info().devices[id].cc; - const int warp_size = ggml_cuda_info().devices[id].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); - use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); - use_mul_mat_vec_q = use_mul_mat_vec_q && ggml_cuda_should_use_mmvq(src0->type, cc, src1->ne[1]); - any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); - } - } else { - const int cc = ggml_cuda_info().devices[ctx.device].cc; - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); - use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); - use_mul_mat_vec_q = use_mul_mat_vec_q && ggml_cuda_should_use_mmvq(src0->type, cc, src1->ne[1]); - any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); - } - - // debug helpers - //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - //printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); - //printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]); - //printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); - //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name); - //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name); - - //TODO update for generic tensor parallelism - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - bool use_batched_cublas_f16 = src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || !any_gpus_with_slow_fp16); - bool use_batched_cublas_bf16 = src0->type == GGML_TYPE_BF16 && bf16_mma_hardware_available(cc); - bool use_batched_cublas_f32 = src0->type == GGML_TYPE_F32; - - const int32_t hint = ggml_get_op_params_i32(dst, 1); - if (hint == GGML_HINT_SRC0_IS_HADAMARD && !split && ggml_cuda_op_fwht(ctx, src1, dst)) { + if (bad_padding_clear || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); return; } - if (!split && use_mul_mat_vec_f) { - // the custom F16 vector kernel can be used over batched cuBLAS GEMM - // but this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; + + if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) { + // The custom F16 vector kernel can be used over batched cuBLAS GEMM. + // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_f) { + return; + } + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) { ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_vec_q) { + return; + } + if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_q) { + return; + } + if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); - } else if (!split && (use_batched_cublas_f16 || use_batched_cublas_bf16 || use_batched_cublas_f32) - && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { - // general KQ + KQV multi-batch without FlashAttention - ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst); - } else if (use_mul_mat_vec_f) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_f, nullptr); - } else if (use_mul_mat_vec_q) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda); - } else if (use_mul_mat_q) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda); - } else { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr); + return; } + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); } static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -2637,7 +1777,6 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft) && "mul_mat_id does not support split buffers"); GGML_TENSOR_BINARY_OP_LOCALS @@ -3251,6 +2390,11 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { GGML_UNUSED(backend); } +static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) { + return ggml_is_empty(t) || t->op == GGML_OP_RESHAPE || t->op == GGML_OP_TRANSPOSE || + t->op == GGML_OP_VIEW || t->op == GGML_OP_PERMUTE || t->op == GGML_OP_NONE; +} + #ifdef USE_CUDA_GRAPH static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { @@ -3260,17 +2404,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } - if (node->src[0] && node->src[0]->buffer && ggml_backend_buft_is_cuda_split(node->src[0]->buffer->buft)) { - use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to split buffer\n", __func__); -#endif - } - // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (node->op == GGML_OP_MUL_MAT_ID) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; @@ -3403,6 +2540,70 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, return true; } +// match gated_delta_net + the strided cpy that scatters its state snapshots into the cache +// (slot i -> rollback group i, slot 0 newest), so the kernel can write them and skip the cpy. +static int ggml_cuda_try_gdn_cache_fusion( + const ggml_cgraph * cgraph, int node_idx, ggml_cuda_gated_delta_net_fused_cache & fused_state_cpy) { + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->op != GGML_OP_GATED_DELTA_NET || gdn->type != GGML_TYPE_F32 || + (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t D = S_v * S_v * H; + const int64_t K = ggml_get_op_params_i32(gdn, 0); // snapshot slot count + const int64_t n_written = std::min(n_tokens, K); // newest n_written slots are written + + // snapshot tail starts right after the attention scores + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + // snapshot cpy is the first real node after the gdn (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(n)) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } + + const ggml_tensor * src = cpy->src[0]; // view of the gdn snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } + + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D (the per-seq stride the kernel + // assumes). ggml_cpy pins src to the same element count. + const std::array expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } + + fused_state_cpy.data = (float *) dst->data; // rollback group 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} + static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { args.sigmoid = false; args.softmax = false; @@ -3844,6 +3045,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_GATED_DELTA_NET) { + ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; + const int nodes_to_skip = ggml_cuda_try_gdn_cache_fusion(cgraph, i, fused_state_cpy); + if (nodes_to_skip > 0) { +#ifdef GGML_CUDA_DEBUG + GGML_LOG_INFO("%s: fused gated_delta_net snapshot copies for %s (skipped %d nodes)\n", + __func__, node->name, nodes_to_skip); +#endif + ggml_cuda_op_gated_delta_net_fused_cache(*cuda_ctx, node, fused_state_cpy); + return nodes_to_skip; + } + } + //topk-moe if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || cgraph->nodes[i]->op == GGML_OP_ARGSORT) { @@ -3950,18 +3165,21 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph (a->ne[2] == 1 && a->ne[3] == 1); const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; - // x must be in the supported whitelist and every operand / intermediate - // result must share x's type, since launch_snake casts a / inv_b as - // float and templates the kernel on a single T. Mixed precision chains - // fall back to the naive path. + // x is in the supported whitelist and every chain intermediate shares + // x's type. launch_snake reads a and inv_b as const float *, so they + // stay F32. const ggml_tensor * sin1 = cgraph->nodes[i + 1]; const bool types_ok = (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && - (a->type == x->type) && (inv_b->type == x->type) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && (mul0->type == x->type) && (sin1->type == x->type) && (sqr->type == x->type) && (mul1->type == x->type) && (add->type == x->type); - if (types_ok && shape_ok && dim_ok && x_in_add == x) { + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + if (types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x) { ggml_cuda_op_snake_fused(*cuda_ctx, x, a, inv_b, add); return 4; } @@ -4006,10 +3224,240 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph bool fused_mul_mat_vec = false; int fused_node_count = 0; - // gate + glu + up + auto get_mul_mat_scale = [](const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { + const bool scale_lhs_mm = scale_node->src[0] == mm_node; + const bool scale_rhs_mm = scale_node->src[1] == mm_node; + if (!scale_lhs_mm && !scale_rhs_mm) { + return nullptr; + } + + const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; + if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; + }; + + auto get_mul_mat_id_scale = [](const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, + const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { + if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { + return nullptr; + } + if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || + (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { + return nullptr; + } + + const ggml_tensor * scale = reshape->src[0]; + if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; + }; + + auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) -> const ggml_tensor * { + if (op_bias == GGML_OP_ADD) { + if (bias_node->src[0] == mul_node) { + return bias_node->src[1]; + } + if (bias_node->src[1] == mul_node) { + return bias_node->src[0]; + } + return nullptr; + } + GGML_ASSERT(op_bias == GGML_OP_ADD_ID); + GGML_ASSERT(bias_node->src[0] == mul_node); + return bias_node->src[1]; + }; + + // gate + glu + up, with optional scale/bias on both lanes. for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; + if (op == GGML_OP_MUL_MAT) { + for (const bool with_bias : { false, true }) { + const int gate_idx = i; + const int gate_scale_idx = i + 1; + const int gate_bias_idx = with_bias ? i + 2 : -1; + const int up_idx = with_bias ? i + 3 : i + 2; + const int up_scale_idx = up_idx + 1; + const int up_bias_idx = with_bias ? up_idx + 2 : -1; + const int glu_idx = with_bias ? up_idx + 3 : up_idx + 2; + + const int out_nodes[] = { glu_idx }; + ggml_op ops[7]; + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = bias_op; + ops[3] = op; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + ops[6] = GGML_OP_GLU; + } else { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = op; + ops[3] = GGML_OP_MUL; + ops[4] = GGML_OP_GLU; + } + const int n_ops = with_bias ? 7 : 5; + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * gate_n = cgraph->nodes[gate_idx]; + ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx]; + ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n; + ggml_tensor * up_n = cgraph->nodes[up_idx]; + ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx]; + ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n; + const ggml_tensor * glu = cgraph->nodes[glu_idx]; + + if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu, + with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) { + continue; + } + + const ggml_tensor * gate_scale = get_mul_mat_scale(gate_scale_n, gate_n); + const ggml_tensor * up_scale = get_mul_mat_scale(up_scale_n, up_n); + if (!gate_scale || !up_scale) { + continue; + } + + const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr; + const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr; + if (with_bias && (!ggml_are_same_shape(gate_out_n->src[0], gate_out_n->src[1]) || + !ggml_are_same_shape(up_out_n->src[0], up_out_n->src[1]))) { + continue; + } + + const ggml_tensor * src0 = up_n->src[0]; + const ggml_tensor * src1 = up_n->src[1]; + const ggml_tensor * ids = up_n->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias; + fusion_data.gate_bias = gate_bias; + fusion_data.x_scale = up_scale; + fusion_data.gate_scale = gate_scale; + fusion_data.glu_op = ggml_get_glu_op(glu); + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + + if (fused_mul_mat_vec) { + break; + } + } else { + for (const bool with_bias : { false, true }) { + const int gate_idx = i; + const int gate_scale_idx = i + 4; + const int gate_bias_idx = with_bias ? i + 5 : -1; + const int up_idx = with_bias ? i + 6 : i + 5; + const int up_scale_idx = up_idx + 4; + const int up_bias_idx = with_bias ? up_idx + 5 : -1; + const int glu_idx = with_bias ? up_idx + 6 : up_idx + 5; + + const int out_nodes[] = { glu_idx }; + ggml_op ops[13]; + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + ops[6] = op; + ops[7] = GGML_OP_RESHAPE; + ops[8] = GGML_OP_REPEAT; + ops[9] = GGML_OP_GET_ROWS; + ops[10] = GGML_OP_MUL; + ops[11] = bias_op; + ops[12] = GGML_OP_GLU; + } else { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = op; + ops[6] = GGML_OP_RESHAPE; + ops[7] = GGML_OP_REPEAT; + ops[8] = GGML_OP_GET_ROWS; + ops[9] = GGML_OP_MUL; + ops[10] = GGML_OP_GLU; + } + const int n_ops = with_bias ? 13 : 11; + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * gate_n = cgraph->nodes[gate_idx]; + ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx]; + ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n; + ggml_tensor * up_n = cgraph->nodes[up_idx]; + ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx]; + ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n; + const ggml_tensor * glu = cgraph->nodes[glu_idx]; + + if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu, + with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) { + continue; + } + + const ggml_tensor * gate_scale = get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], + cgraph->nodes[gate_idx + 3], gate_scale_n, gate_n); + const ggml_tensor * up_scale = get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], + cgraph->nodes[up_idx + 3], up_scale_n, up_n); + if (!gate_scale || !up_scale) { + continue; + } + + const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr; + const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr; + + const ggml_tensor * src0 = up_n->src[0]; + const ggml_tensor * src1 = up_n->src[1]; + const ggml_tensor * ids = up_n->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias; + fusion_data.gate_bias = gate_bias; + fusion_data.x_scale = up_scale; + fusion_data.gate_scale = gate_scale; + fusion_data.glu_op = ggml_get_glu_op(glu); + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + + if (fused_mul_mat_vec) { + break; + } + } + if (ggml_cuda_can_fuse(cgraph, i, { op, bias_op, op, bias_op, GGML_OP_GLU }, {})) { ggml_tensor * glu = cgraph->nodes[i + 4]; ggml_tensor * gate_bias_n = glu->src[0]; @@ -4029,23 +3477,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph continue; } - auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) { - if (op_bias == GGML_OP_ADD) { - if (bias_node->src[0] == mul_node) { - return bias_node->src[1]; - } - if (bias_node->src[1] == mul_node) { - return bias_node->src[0]; - } - return (ggml_tensor *) nullptr; - } - GGML_ASSERT(op_bias == GGML_OP_ADD_ID); - GGML_ASSERT(bias_node->src[0] == mul_node); - return bias_node->src[1]; - }; - - ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op); - ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op); + const ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op); + const ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op); if (!up_bias_tensor || !gate_bias_tensor) { continue; @@ -4133,7 +3566,95 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph fused_mul_mat_vec = false; fused_node_count = 0; - // gate + add + glu + up + add + // mul_mat + scale + optional bias + for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { + const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; + + for (const bool with_bias : { false, true }) { + const int n_ops = op == GGML_OP_MUL_MAT ? (with_bias ? 3 : 2) : (with_bias ? 6 : 5); + const int out_nodes[] = { i + n_ops - 1 }; + ggml_op ops[6]; + if (op == GGML_OP_MUL_MAT) { + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = bias_op; + } else { + ops[0] = op; + ops[1] = GGML_OP_MUL; + } + } else { + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + } else { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + } + } + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * mm_node = cgraph->nodes[i]; + ggml_tensor * scale_node = op == GGML_OP_MUL_MAT ? cgraph->nodes[i + 1] : cgraph->nodes[i + 4]; + ggml_tensor * out_node = with_bias ? cgraph->nodes[i + n_ops - 1] : scale_node; + + const ggml_tensor * scale = nullptr; + if (op == GGML_OP_MUL_MAT) { + scale = get_mul_mat_scale(scale_node, mm_node); + } else { + scale = get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); + } + if (!scale) { + continue; + } + + const ggml_tensor * bias = with_bias ? get_bias_tensor(out_node, scale_node, bias_op) : nullptr; + if (with_bias && !bias) { + continue; + } + if (with_bias && bias_op == GGML_OP_ADD && !ggml_are_same_shape(out_node->src[0], out_node->src[1])) { + continue; + } + if (with_bias && bias_op == GGML_OP_ADD_ID && out_node->src[2] != mm_node->src[2]) { + continue; + } + + const ggml_tensor * src0 = mm_node->src[0]; + const ggml_tensor * src1 = mm_node->src[1]; + const ggml_tensor * ids = mm_node->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.x_bias = bias; + fusion_data.x_scale = scale; + + if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + if (fused_mul_mat_vec) { + break; + } + } + + if (fused_mul_mat_vec) { + return fused_node_count - 1; + } + + // mul_mat + add for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; @@ -4364,15 +3885,9 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } -#ifdef GGML_CUDA_DEBUG - const int nodes_fused = i - prev_i - 1; - if (nodes_fused > 0) { - GGML_LOG_INFO("nodes_fused: %d\n", nodes_fused); - } -#endif prev_i = i; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } @@ -4383,6 +3898,12 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud int nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, cgraph, i); if (nodes_to_skip != 0) { +#ifdef GGML_CUDA_DEBUG + const int last_fused = i + nodes_to_skip; + GGML_LOG_INFO("nodes_fused: %d, first: %s (%s), last: %s (%s)\n", + nodes_to_skip + 1, ggml_op_name(node->op), node->name, + ggml_op_name(cgraph->nodes[last_fused]->op), cgraph->nodes[last_fused]->name); +#endif i += nodes_to_skip; continue; } @@ -4392,7 +3913,7 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud if (node->src[j] != nullptr) { assert(node->src[j]->buffer); assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || - ggml_backend_buft_is_cuda_split(node->src[j]->buffer->buft) || (integrated && ggml_backend_buft_is_cuda_host(node->src[j]->buffer->buft))); + (integrated && ggml_backend_buft_is_cuda_host(node->src[j]->buffer->buft))); } } #else @@ -5054,15 +4575,6 @@ static ggml_backend_buffer_type_t ggml_backend_cuda_device_get_host_buffer_type( static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context; - // split buffers can only be used with GGML_OP_MUL_MAT - if (op->op != GGML_OP_MUL_MAT) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda_split(op->src[i]->buffer->buft)) { - return false; - } - } - } - // check if all the sources are allocated on this device for (int i = 0; i < GGML_MAX_SRC; i++) { if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda(op->src[i]->buffer->buft)) { @@ -5123,19 +4635,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g { struct ggml_tensor * a = op->src[0]; struct ggml_tensor * b = op->src[1]; - if (a->buffer && ggml_backend_buft_is_cuda_split(a->buffer->buft)) { - if (a->ne[2] > 1 || a->ne[3] > 1) { - return false; - } - // for small weight matrices the active device can end up without any rows, don't use row split in those cases - // this avoids some edge cases (and the performance would not be good anyways) - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) a->buffer->buft->context; - int64_t row_low; - int64_t row_high; - get_row_split(&row_low, &row_high, a, buft_ctx->tensor_split, dev_ctx->device); - if (row_low == row_high) { - return false; - } + if (a->nb[0] != ggml_element_size(a) || b->nb[0] != ggml_element_size(b)) { + return false; // TODO this could in principle be implemented though currently there is no use case. } if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) { return false; @@ -5211,10 +4712,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g } break; case GGML_OP_SET_ROWS: { - return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || - op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || - op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && - op->src[0]->type == GGML_TYPE_F32 && + return ( + ( + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || + op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && + op->src[0]->type == GGML_TYPE_F32 + ) || ( + op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16 + ) + ) && (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); } break; case GGML_OP_SET: @@ -5304,12 +4811,24 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g ggml_type src1_type = op->src[1]->type; return src0_type == src1_type && src0_type == op->type && - !ggml_is_quantized(src0_type) && - ggml_blck_size(src0_type) == 1 && - (ggml_type_size(src0_type) == 1 || - ggml_type_size(src0_type) == 2 || - ggml_type_size(src0_type) == 4 || - ggml_type_size(src0_type) == 8); + ( + ( + ggml_is_quantized(src0_type) && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + op->src[0]->ne[0] % ggml_blck_size(src0_type) == 0 && + op->src[1]->ne[0] % ggml_blck_size(src0_type) == 0 + ) || ( + !ggml_is_quantized(src0_type) && + ggml_blck_size(src0_type) == 1 && + ( + ggml_type_size(src0_type) == 1 || + ggml_type_size(src0_type) == 2 || + ggml_type_size(src0_type) == 4 || + ggml_type_size(src0_type) == 8 + ) + ) + ); } break; case GGML_OP_CONV_TRANSPOSE_1D: { @@ -5398,7 +4917,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: case GGML_OP_CONV_2D: + return true; case GGML_OP_CONV_2D_DW: + return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CONV_TRANSPOSE_2D: case GGML_OP_POOL_2D: return true; @@ -5458,7 +4979,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g static bool ggml_backend_cuda_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context; const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated; - return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft))); + return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)); } static int64_t get_op_batch_size(const ggml_tensor * op) { @@ -5582,10 +5103,6 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t features.push_back({ "USE_GRAPHS", "1" }); #endif - #ifdef GGML_CUDA_PEER_MAX_BATCH_SIZE - features.push_back({ "PEER_MAX_BATCH_SIZE", STRINGIFY(GGML_CUDA_PEER_MAX_BATCH_SIZE) }); - #endif - #ifdef GGML_CUDA_FA_ALL_QUANTS features.push_back({ "FA_ALL_QUANTS", "1" }); #endif @@ -5624,9 +5141,6 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *)ggml_backend_cuda_comm_allreduce_tensor; } - if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { - return (void *)ggml_backend_cuda_split_buffer_type; - } if (strcmp(name, "ggml_backend_register_host_buffer") == 0) { return (void *)ggml_backend_cuda_register_host_buffer; } diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index e1add5e0331..6b3b0d064a5 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -368,5 +368,12 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return true; } + // gfx900 (Vega 10) lacks native dp4a, loses to dequant + hipBLAS + // for dense matrices; keep MMQ only for MoE, where the + // hipBLAS path is much slower. + if (cc == GGML_CUDA_CC_VEGA) { + return n_experts > 0; + } + return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index fe44a58da91..e18ada5377d 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -278,6 +278,9 @@ int get_mmvq_mmid_max_batch(ggml_type type, int cc) { } bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { + if (!ggml_is_quantized(type)) { + return false; + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) { @@ -518,9 +521,13 @@ static __global__ void mul_mat_vec_q( bool use_gate = false; bool use_bias = false; bool use_gate_bias = false; + bool use_scale = false; + bool use_gate_scale = false; [[maybe_unused]] const void * vgate = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; + const float * x_scale = nullptr; + const float * gate_scale = nullptr; ggml_glu_op active_glu; if constexpr (has_fusion) { @@ -531,34 +538,47 @@ static __global__ void mul_mat_vec_q( x_bias = (const float *) fusion.x_bias; gate_bias = (const float *) fusion.gate_bias; active_glu = fusion.glu_op; + if constexpr (type == GGML_TYPE_NVFP4) { + use_scale = fusion.x_scale != nullptr; + use_gate_scale = fusion.gate_scale != nullptr && use_gate; + x_scale = (const float *) fusion.x_scale; + gate_scale = (const float *) fusion.gate_scale; + } } [[maybe_unused]] float x_biases[ncols_dst] = { 0.0f }; [[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float x_scales = 1.0f; + [[maybe_unused]] float gate_scales = 1.0f; if constexpr (has_fusion) { + // 1. Hide latency by prefetching bias, gates and scales here + // 2. load only on threads that won't die after partial sum calculation const uint32_t channel_bias = ids ? channel_x : channel_dst; - if (use_bias) { - x_bias = x_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0; - // 1. Hide latency by prefetching bias and gate here - // 2. load only on threads that won't die after partial sum calculation - if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && - (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && + (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (use_bias) { + x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0; #pragma unroll for (int j = 0; j < ncols_dst; ++j) { x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x]; } } - } - if (use_gate_bias) { - gate_bias = gate_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0; - if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && - (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (use_gate_bias) { + gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0; #pragma unroll for (int j = 0; j < ncols_dst; ++j) { gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x]; } } + if constexpr (type == GGML_TYPE_NVFP4) { + if (use_scale) { + x_scales = x_scale[ids ? channel_x : 0]; + } + if (use_gate_scale) { + gate_scales = gate_scale[ids ? channel_x : 0]; + } + } } } @@ -635,42 +655,46 @@ static __global__ void mul_mat_vec_q( tmp_gate[j][i] = warp_reduce_sum(tmp_gate[j][i]); } } - } - if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { - float result = tmp[j][threadIdx.x]; - if constexpr (has_fusion) { - if (use_bias) { + if (threadIdx.x == i && (rows_per_cuda_block == 1 || uint32_t(row0 + i) < stride_col_dst)) { + float result = tmp[j][i]; + if constexpr (has_fusion) { + if constexpr (type == GGML_TYPE_NVFP4) { + result *= x_scales; + } result += x_biases[j]; - } - if (use_gate) { - float gate_value = tmp_gate[j][threadIdx.x]; - if (use_gate_bias) { + if (use_gate) { + float gate_value = tmp_gate[j][i]; + if constexpr (type == GGML_TYPE_NVFP4) { + gate_value *= gate_scales; + } gate_value += gate_biases[j]; - } - switch (active_glu) { - case GGML_GLU_OP_SWIGLU: - result *= ggml_cuda_op_silu_single(gate_value); - break; - case GGML_GLU_OP_GEGLU: - result *= ggml_cuda_op_gelu_single(gate_value); - break; - case GGML_GLU_OP_SWIGLU_OAI: { - result = ggml_cuda_op_swiglu_oai_single(gate_value, result); - break; + switch (active_glu) { + case GGML_GLU_OP_SWIGLU: + result *= ggml_cuda_op_silu_single(gate_value); + break; + case GGML_GLU_OP_GEGLU: + result *= ggml_cuda_op_gelu_single(gate_value); + break; + case GGML_GLU_OP_SWIGLU_OAI: + result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + break; + default: + result = result * gate_value; + break; } - default: - result = result * gate_value; - break; } } + dst[j*stride_col_dst + i] = result; } - dst[j*stride_col_dst + threadIdx.x] = result; } } if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, gate_bias, x_bias, tmp_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, gate_bias, x_bias, x_scale, gate_scale, tmp_gate); + } + if constexpr (type != GGML_TYPE_NVFP4) { + GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales); } } @@ -766,7 +790,8 @@ static void mul_mat_vec_q_switch_fusion( const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared, const uint32_t ids_stride, cudaStream_t stream) { - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr || + fusion.x_scale != nullptr || fusion.gate_scale != nullptr; if constexpr (c_ncols_dst == 1) { if (has_fusion) { const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); @@ -831,7 +856,6 @@ static void mul_mat_vec_q_switch_ncols_dst( const int warp_size = ggml_cuda_info().devices[device].warp_size; const mmvq_parameter_table_id table_id = get_device_table_id(cc); - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; const bool has_ids = ids != nullptr; const auto should_use_small_k = [&](int c_ncols_dst) { @@ -970,8 +994,6 @@ static void mul_mat_vec_q_switch_ncols_dst( GGML_ABORT("fatal error"); break; } - - GGML_UNUSED(has_fusion); } static void mul_mat_vec_q_switch_type( const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, @@ -1151,6 +1173,9 @@ void ggml_cuda_mul_mat_vec_q( if (fusion) { GGML_ASSERT( !ids || dst->ne[2] == 1); GGML_ASSERT( ids || dst->ne[1] == 1); + // Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is + // non-negligible for some models such as gpt-oss-20b + GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4); if (fusion->x_bias) { GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32); @@ -1168,6 +1193,18 @@ void ggml_cuda_mul_mat_vec_q( GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]); fusion_local.gate_bias = fusion->gate_bias->data; } + if (fusion->x_scale) { + GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); + GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.x_scale = fusion->x_scale->data; + } + if (fusion->gate_scale) { + GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); + GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.gate_scale = fusion->gate_scale->data; + } fusion_local.glu_op = fusion->glu_op; } diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu index 3b4f004c946..4659970651e 100644 --- a/ggml/src/ggml-cuda/set-rows.cu +++ b/ggml/src/ggml-cuda/set-rows.cu @@ -322,17 +322,77 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s } } +template<> +void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const half * src0_d = (const half *)src0->data; + const int32_t * src1_d = (const int32_t *)src1->data; + + GGML_TENSOR_BINARY_OP_LOCALS + + cudaStream_t stream = ctx.stream(); + + + if (dst->type == GGML_TYPE_F16) { + set_rows_cuda( + src0_d, src1_d, (half*)dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +} + +template<> +void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const half * src0_d = (const half *)src0->data; + const int64_t * src1_d = (const int64_t *)src1->data; + + GGML_TENSOR_BINARY_OP_LOCALS + + cudaStream_t stream = ctx.stream(); + + + if (dst->type == GGML_TYPE_F16) { + set_rows_cuda( + src0_d, src1_d, (half*)dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +} + void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); - if (src1->type == GGML_TYPE_I64) { - set_rows_cuda(ctx, src0, src1, dst); + if (src0->type == GGML_TYPE_F32) { + if (src1->type == GGML_TYPE_I64) { + set_rows_cuda(ctx, src0, src1, dst); + } else { + set_rows_cuda(ctx, src0, src1, dst); + } + } else if (src0->type == GGML_TYPE_F16) { + if (src1->type == GGML_TYPE_I64) { + set_rows_cuda(ctx, src0, src1, dst); + } else { + set_rows_cuda(ctx, src0, src1, dst); + } } else { - set_rows_cuda(ctx, src0, src1, dst); + GGML_ABORT("unsupported type %s", ggml_type_name(src0->type)); } } diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu index 2e1883af40e..0c1a37fd008 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 16, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 16, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 16, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu index 407b6cf4c70..d5a4d160a49 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 32, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 32, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 32, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu index 5e46685024b..d8e7588cfc3 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 4, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 4, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 4, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu index a6e6f093dcb..e48e28c4e48 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 8, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 8, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 8, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index af05a9eff71..614b1566c7f 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -92,7 +92,7 @@ def get_short_name(long_quant_name): continue if head_size_kq == 320 and ncols2 != 32: # Mistral Small 4 continue - if head_size_kq == 512 and ncols2 not in (4, 8): # Gemma 4 + if head_size_kq == 512 and ncols2 not in (2, 4, 8): # Gemma 4 (+ MTP) continue if head_size_kq == 576 and ncols2 not in (4, 16, 32): # Deepseek, GLM 4.7 Flash continue diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index c4253bfa43b..c80394e31ff 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -312,6 +312,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, ggml_cuda_kernel_launch(topk_moe_cuda<256, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; + case 288: // StepFun 3.7 + ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + break; case 512: ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); @@ -377,8 +381,10 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op, const ggml_tensor * weights, const ggml_tensor * logits, const ggml_tensor * ids) { + // must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512, + // or one of the non-power-of-2 expert counts of supported models const int n_expert = ids->nb[1] / ids->nb[0]; - if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 576) { + if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) { return false; } diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt index c6e49a71d11..845b922bfed 100644 --- a/ggml/src/ggml-hexagon/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/CMakeLists.txt @@ -23,7 +23,6 @@ include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake) include(ExternalProject) option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF) -option(GGML_HEXAGON_FA_EXP2_HF "ggml-hexagon: use FP16 exp2 polynomial in FA softmax instead of F32 exp round-trip" OFF) set(GGML_HEXAGON_HTP_CERT "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate") add_library(htp_iface OBJECT diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 3d41c47b651..0c78879cbb4 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -43,6 +43,7 @@ #include "htp-opnode.h" #include "htp-ops.h" #include "htp/matmul-ops.h" +#include "htp/flash-attn-ops.h" #include "htp_iface.h" #include "htp-drv.h" @@ -62,6 +63,7 @@ static int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) static int opt_hostbuf = 1; // hostbuf ON by default static int opt_mm_select = 3; // 3 = HMX -> Tiled -> Flat -> CPU, 2 = Tiled -> Flat -> CPU, 1 = Flat -> CPU +static int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) // Default PMU events, if profiling with PMU (mode=2) is enabled // See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html @@ -125,6 +127,11 @@ static const char * htp_event_name(uint16_t id) { case HTP_TRACE_EVT_HVX_W_DEQUANT: return "HVX_W_DEQUANT"; case HTP_TRACE_EVT_HVX_W_PREP: return "HVX_W_PREP"; case HTP_TRACE_EVT_HVX_O_PROC: return "HVX_O_PROC"; + case HTP_TRACE_EVT_HVX_FA_QK: return "HVX_QK_FA"; + case HTP_TRACE_EVT_HVX_FA_SFM: return "HVX_SFM_FA"; + case HTP_TRACE_EVT_HVX_FA_Q_PREP: return "HVX_Q_PREP"; + case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP"; + case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP"; case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP"; default: return "UNKNOWN"; } @@ -1879,6 +1886,162 @@ ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { // ** backend interface +static bool ggml_hexagon_flash_attn_is_hmx_eligible( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * v, + const struct ggml_tensor * sinks +) { + if (sess->n_hmx == 0) { + return false; + } + + if (opt_fa_select < 2) { + return false; + } + + if (k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { + return false; + } + + const uint32_t DK = q->ne[0]; + const uint32_t DV = v->ne[0]; + + if (DK % 64 != 0 || DV % 64 != 0) { + return false; + } + + // Fall back to HVX for small token counts if head dimension is small (DK <= 128) + const uint32_t neq1 = q->ne[1]; + if (DK <= 128 && neq1 < 5) { + return false; + } + + return true; +} + +static bool ggml_hexagon_precompute_flash_attn_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_fa_kernel_params * kparams +) { + if (opt_fa_select < 1) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * mask = op->src[3]; + const struct ggml_tensor * dst = op; + + const uint32_t neq0 = q->ne[0]; // head_dim (DK) + const uint32_t neq1 = q->ne[1]; // n_tokens + const uint32_t neq2 = q->ne[2]; // n_heads + + const uint32_t nek1 = k->ne[1]; // kv_len + + const uint32_t nev0 = v->ne[0]; // head_dim (DV) + + const uint32_t DK = neq0; + const uint32_t DV = nev0; + + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t G = neq2 / n_kv_heads; + + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + memcpy(&logit_softcap, &op->op_params[2], sizeof(float)); + + if (logit_softcap != 0.0f) { + scale /= logit_softcap; + } + + kparams->scale = scale; + kparams->max_bias = max_bias; + kparams->logit_softcap = logit_softcap; + + kparams->is_q_fp32 = (q->type == GGML_TYPE_F32) ? 1 : 0; + kparams->is_dst_fp32 = (dst->type == GGML_TYPE_F32) ? 1 : 0; + kparams->G = G; + + const uint32_t n_head = q->ne[2]; + kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head)); + kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2); + kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2); + + // Check HMX eligibility + const struct ggml_tensor * sinks = op->src[4]; + if (ggml_hexagon_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) { + size_t Br = 0, Bc = 0; + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK, DV, neq1, nek1, sess->vtcm_size, sess->n_threads); + if (ret == 0) { + kparams->kernel_type = HTP_FA_KERNEL_HMX; + kparams->Br = Br; + kparams->Bc = Bc; + kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; + kparams->n_threads = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? sess->n_threads : 1; + + kparams->u.hmx.g_br = hex_align_up(G * Br, 32); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && sess->n_threads >= 2) ? 1 : 0; + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK, DV, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0); + + const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256); + kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; // HVX vector is 128 bytes + + const size_t m_line_bytes = hex_align_up(Bc * sizeof(uint16_t), 128); + kparams->u.hmx.mask_buf_row_stride = m_line_bytes / sizeof(uint16_t); + kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; + kparams->u.hmx.div_G = init_fastdiv_values(G); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + + kparams->qrows = 0; + kparams->qrows_per_thread = 0; + return true; + } + } + + // Fallback to HVX + kparams->kernel_type = HTP_FA_KERNEL_HVX; + kparams->Br = 1; + kparams->Bc = 64; // FLASH_ATTN_BLOCK_SIZE + kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; + kparams->n_threads = sess->n_threads; + + const size_t size_q_row_padded = hex_round_up(q->ne[0] * (kparams->is_q_fp32 ? 4 : 2), 128); + const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); + const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); + + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, kparams->is_q_fp32 != 0, mask != nullptr, sess->n_threads); + + kparams->u.hvx.size_q_row_padded = size_q_row_padded; + kparams->u.hvx.size_k_row_padded = size_k_row_padded; + kparams->u.hvx.size_v_row_padded = size_v_row_padded; + kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); + kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]); + kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]); + kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]); + kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]); + kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + + kparams->qrows = q->ne[1] * q->ne[2] * q->ne[3]; + kparams->qrows_per_thread = (kparams->qrows + sess->n_threads - 1) / sess->n_threads; + + return true; +} static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { const struct ggml_tensor * src0 = op->src[0]; @@ -1912,6 +2075,17 @@ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_sess return false; } + struct htp_fa_kernel_params kparams; + if (!ggml_hexagon_precompute_flash_attn_params(sess, op, &kparams)) { + return false; + } + + if ((size_t) kparams.vtcm_size > sess->vtcm_size) { + HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", + kparams.vtcm_size, sess->vtcm_size); + return false; + } + return true; } @@ -2211,31 +2385,30 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->kernel_type = (src1_nrows < (int) sess->n_threads) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - size_t vtcm_src0_size = 0, vtcm_src1_size = 0; + struct htp_mm_hvx_vtcm_layout L; uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; uint32_t best_n_prefetch = 2; - size_t total_size = 0; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - total_size = htp_mm_hvx_id_get_vtcm_sizes( - wtype, ne10, src1_nrows, sess->n_threads, src0->nb[1], d, - &vtcm_src0_size, &vtcm_src1_size + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0->nb[1], 0, d, true, false, false ); - if (total_size <= vtcm_budget) { + if (L.total_bytes <= vtcm_budget) { best_n_prefetch = d; break; } } - if (best_n_prefetch == 2 && total_size > vtcm_budget) { - total_size = htp_mm_hvx_id_get_vtcm_sizes( - wtype, ne10, src1_nrows, sess->n_threads, src0->nb[1], 2, - &vtcm_src0_size, &vtcm_src1_size + if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0->nb[1], 0, 2, true, false, false ); } kparams->n_prefetch = best_n_prefetch; - kparams->vtcm_size = total_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = 0; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; } else { bool try_tiled = (k_align && opt_mm_select >= 2); if (try_tiled) { @@ -2246,37 +2419,36 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; } + struct htp_mm_hvx_vtcm_layout L; uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; uint32_t best_n_prefetch = 2; - size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; - size_t total_size = 0; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - total_size = htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], d, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], d, false, false, false ); - if (total_size <= vtcm_budget) { + if (L.total_bytes <= vtcm_budget) { best_n_prefetch = d; break; } } - if (best_n_prefetch == 2 && total_size > vtcm_budget) { - total_size = htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 2, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + if (best_n_prefetch == 2 && L.total_bytes > vtcm_budget) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 2, false, false, false ); } kparams->n_prefetch = best_n_prefetch; - if (total_size <= vtcm_budget) { - kparams->vtcm_size = total_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + if (L.total_bytes <= vtcm_budget) { + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; goto done_quant; } - HEX_VERBOSE("ggml-hex: %s HVX tiled path VTCM size needed (%zu) > budget (%zu), falling back to HVX flat\n", sess->name.c_str(), total_size, vtcm_budget); + HEX_VERBOSE("ggml-hex: %s HVX tiled path VTCM size needed (%zu) > budget (%zu), falling back to HVX flat\n", sess->name.c_str(), L.total_bytes, vtcm_budget); } // Flat HVX fallback @@ -2284,17 +2456,17 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; - size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; - size_t total_size = htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 16, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, false, false, false ); kparams->n_prefetch = 16; - kparams->vtcm_size = total_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; } } @@ -2304,19 +2476,19 @@ static void ggml_hexagon_precompute_hvx_mm_params( const bool is_batched = (ne02 > 1) || (ne03 > 1); const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); - size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; - size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( - HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 16, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, false, false, false ); - if (!is_batched && !is_permuted && vtcm_size <= vtcm_budget) { + if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM; kparams->src1_row_size = hex_round_up(ne10 * 2, 128); - kparams->vtcm_size = vtcm_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; } else { if (src1->type == GGML_TYPE_F32) { @@ -2325,14 +2497,14 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_DDR; } kparams->src1_row_size = src1->nb[1]; - size_t ddr_size = htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 16, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, false, false, false ); - kparams->vtcm_size = ddr_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; } } else { @@ -2340,31 +2512,31 @@ static void ggml_hexagon_precompute_hvx_mm_params( const bool is_batched = (ne02 > 1) || (ne03 > 1); const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); - size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; - size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( - HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 16, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, false, false, false ); - if (!is_batched && !is_permuted && vtcm_size <= vtcm_budget) { + if (!is_batched && !is_permuted && L.total_bytes <= vtcm_budget) { kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM; kparams->src1_row_size = hex_round_up(ne10 * 4, 128); - kparams->vtcm_size = vtcm_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; } else { kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_DDR; kparams->src1_row_size = src1->nb[1]; - size_t ddr_size = htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, - dst->nb[1], src0->nb[1], src1->nb[1], 16, &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, sess->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, false, false, false ); - kparams->vtcm_size = ddr_size; - kparams->vtcm_src0_size = vtcm_src0_size; - kparams->vtcm_src1_size = vtcm_src1_size; - kparams->vtcm_dst_size = vtcm_dst_size; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_dst_size = L.dst_bytes; kparams->n_prefetch = 16; } } @@ -2434,83 +2606,57 @@ static void ggml_hexagon_precompute_fused_qkv_params( const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); const size_t src0_row_size = src0->nb[1]; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - size_t src0_sz_per_thread = 0; - size_t src2_sz_per_thread = 0; - size_t src3_sz_per_thread = 0; uint32_t best_n_prefetch = 16; if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t src1_row_size_padded = hex_round_up(src1_row_size, QK_Q8_0_TILED * sizeof(float)); - size_t src1_sz_per_thread = hex_round_up(src1_row_size * src1_nrows, 128); - size_t src1_sz = src1_sz_per_thread; - const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - size_t repacked_vtcm_size = hex_round_up(d * tile_row_size, 128); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - size_t src0_sz = repacked_vtcm_size * sess->n_threads; - size_t src2_sz = hex_round_up(d * tile_row_size, 128) * sess->n_threads; - size_t src3_sz = hex_round_up(d * tile_row_size, 128) * sess->n_threads; - size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + src3_sz; - - if (tiled_vtcm_size <= sess->vtcm_size) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, src1_row_size, d, false, true, false + ); + if (L.total_bytes <= sess->vtcm_size) { best_n_prefetch = d; - src0_sz_per_thread = repacked_vtcm_size; - src2_sz_per_thread = hex_round_up(d * tile_row_size, 128); - src3_sz_per_thread = hex_round_up(d * tile_row_size, 128); break; } } - if (best_n_prefetch == 2 && src0_sz_per_thread == 0) { - size_t repacked_vtcm_size = hex_round_up(2 * tile_row_size, 128); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - src0_sz_per_thread = repacked_vtcm_size; - src2_sz_per_thread = hex_round_up(2 * tile_row_size, 128); - src3_sz_per_thread = hex_round_up(2 * tile_row_size, 128); - } - } else { - best_n_prefetch = 16; - src0_sz_per_thread = hex_round_up(best_n_prefetch * src0_row_size_padded, 128); - src2_sz_per_thread = hex_round_up(best_n_prefetch * src0_row_size_padded, 128); - src3_sz_per_thread = hex_round_up(best_n_prefetch * src0_row_size_padded, 128); } - size_t src1_sz_per_thread = hex_round_up(src1_row_size * src1_nrows, 128); + struct htp_mm_hvx_vtcm_layout L; + bool try_tiled = (opt_mm_select >= 2); - size_t src0_sz = src0_sz_per_thread * sess->n_threads; - size_t src1_sz = src1_sz_per_thread; - size_t src2_sz = src2_sz_per_thread * sess->n_threads; - size_t src3_sz = src3_sz_per_thread * sess->n_threads; + // Test tiled first + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, src1_row_size, best_n_prefetch, false, true, false + ); - size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + src3_sz; - bool try_tiled = (opt_mm_select >= 2); - if (try_tiled && tiled_vtcm_size <= sess->vtcm_size) { + if (try_tiled && L.total_bytes <= sess->vtcm_size) { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->vtcm_src0_size = src0_sz; - kparams->vtcm_src1_size = src1_sz; - kparams->vtcm_src2_size = src2_sz; - kparams->vtcm_src3_size = src3_sz; - kparams->vtcm_size = tiled_vtcm_size; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; + kparams->vtcm_src3_size = L.src3_bytes; + kparams->vtcm_dst_size = L.dst_bytes; + kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; } else { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - size_t flat_src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128); - kparams->vtcm_src0_size = src0_sz; - kparams->vtcm_src1_size = flat_src1_sz; - kparams->vtcm_src2_size = src2_sz; - kparams->vtcm_src3_size = src3_sz; - kparams->vtcm_size = src0_sz + flat_src1_sz + src2_sz + src3_sz; + + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, flat_src1_row_size, best_n_prefetch, false, true, false + ); + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; + kparams->vtcm_src3_size = L.src3_bytes; + kparams->vtcm_dst_size = L.dst_bytes; + kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; } } @@ -2530,75 +2676,55 @@ static void ggml_hexagon_precompute_fused_ffn_params( const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); const size_t src0_row_size = src0->nb[1]; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - size_t src0_sz_per_thread = 0; - size_t src2_sz_per_thread = 0; uint32_t best_n_prefetch = 16; if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t src1_row_size_padded = hex_round_up(src1_row_size, QK_Q8_0_TILED * sizeof(float)); - size_t src1_sz_per_thread = hex_round_up(src1_row_size * src1_nrows, 128); - size_t src1_sz = src1_sz_per_thread; - const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; best_n_prefetch = 2; for (uint32_t d = max_prefetch; d >= 2; d /= 2) { - size_t repacked_vtcm_size = hex_round_up(d * tile_row_size, 128); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - size_t src0_sz = repacked_vtcm_size * sess->n_threads; - size_t src2_sz = hex_round_up(d * tile_row_size, 128) * sess->n_threads; - size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz; - - if (tiled_vtcm_size <= sess->vtcm_size) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, src1_row_size, d, false, false, true + ); + if (L.total_bytes <= sess->vtcm_size) { best_n_prefetch = d; - src0_sz_per_thread = repacked_vtcm_size; - src2_sz_per_thread = hex_round_up(d * tile_row_size, 128); break; } } - if (best_n_prefetch == 2 && src0_sz_per_thread == 0) { - size_t repacked_vtcm_size = hex_round_up(2 * tile_row_size, 128); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - src0_sz_per_thread = repacked_vtcm_size; - src2_sz_per_thread = hex_round_up(2 * tile_row_size, 128); - } - } else { - best_n_prefetch = 16; - src0_sz_per_thread = hex_round_up(best_n_prefetch * src0_row_size_padded, 128); - src2_sz_per_thread = hex_round_up(best_n_prefetch * src0_row_size_padded, 128); } - size_t src1_sz_per_thread = hex_round_up(src1_row_size * src1_nrows, 128); + struct htp_mm_hvx_vtcm_layout L; + bool try_tiled = (opt_mm_select >= 2); - size_t src0_sz = src0_sz_per_thread * sess->n_threads; - size_t src1_sz = src1_sz_per_thread; - size_t src2_sz = src2_sz_per_thread * sess->n_threads; + // Test tiled first + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, src1_row_size, best_n_prefetch, false, false, true + ); - size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz; - bool try_tiled = (opt_mm_select >= 2); - if (try_tiled && tiled_vtcm_size <= sess->vtcm_size) { + if (try_tiled && L.total_bytes <= sess->vtcm_size) { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; - kparams->vtcm_src0_size = src0_sz; - kparams->vtcm_src1_size = src1_sz; - kparams->vtcm_src2_size = src2_sz; - kparams->vtcm_size = tiled_vtcm_size; + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; + kparams->vtcm_dst_size = L.dst_bytes; + kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; } else { kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - size_t flat_src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128); - kparams->vtcm_src0_size = src0_sz; - kparams->vtcm_src1_size = flat_src1_sz; - kparams->vtcm_src2_size = src2_sz; - kparams->vtcm_size = src0_sz + flat_src1_sz + src2_sz; + + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads, + 0, src0_row_size, flat_src1_row_size, best_n_prefetch, false, false, true + ); + kparams->vtcm_src0_size = L.src0_bytes; + kparams->vtcm_src1_size = L.src1_bytes; + kparams->vtcm_src2_size = L.src2_bytes; + kparams->vtcm_dst_size = L.dst_bytes; + kparams->vtcm_size = L.total_bytes; kparams->n_prefetch = best_n_prefetch; } } @@ -2979,8 +3105,12 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess int mode = op_params[2]; + // n_dims == ne0/2, so the rotation spans the full row if (mode == GGML_ROPE_TYPE_VISION) { - return false; + const int n_dims = op_params[1]; + if (n_dims != (int) (op->src[0]->ne[0] / 2)) { + return false; + } } if (mode & 1) { return false; @@ -3011,16 +3141,23 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess } if (src2) { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(src2) || - !ggml_is_contiguous(dst)) { + if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) { return false; } } else { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + if (!ggml_is_contiguous(src1)) { return false; } } + // src0/dst elements within a row must be contiguous (nb[0] == sizeof(float)). + // nb[1] may exceed ne[0]*sizeof(float) when the tensor is a strided view of a larger one + if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { + return false; + } return true; } @@ -3243,7 +3380,7 @@ static inline bool op_is_compute(ggml_tensor *node) return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); } -static bool is_hmx_eligible(const ggml_tensor * t) { +static bool mm_is_hmx_eligible(const ggml_tensor * t) { if (opt_nhmx == 0) { return false; } const ggml_tensor * src0 = t->src[0]; @@ -3262,7 +3399,7 @@ static bool is_hmx_eligible(const ggml_tensor * t) { static bool is_mergeable_mul_mat(const ggml_tensor * t) { if (!t || t->op != GGML_OP_MUL_MAT) return false; if (t->src[1]->type != GGML_TYPE_F32) return false; - return ggml_is_quantized(t->src[0]->type) && !is_hmx_eligible(t); + return ggml_is_quantized(t->src[0]->type) && !mm_is_hmx_eligible(t); } static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) { @@ -3357,6 +3494,26 @@ static bool try_fuse_node(const ggml_hexagon_session * sess, const ggml_cgraph * } } + if (n->op == GGML_OP_MUL_MAT && next_node) { + if (next_node->op == GGML_OP_ADD && op_is_compute(next_node) && ggml_can_fuse(graph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) { + if (next_node->src[0] == n || next_node->src[1] == n) { + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_matmul_params(sess, n->src[0], n->src[1], next_node, &kparams); + if ((size_t)kparams.vtcm_size <= sess->vtcm_size) { + htp_opnode node(n, {}, HTP_OP_MUL_MAT_ADD); + node.add_fused(next_node); + memcpy(node.kernel_params, &kparams, sizeof(kparams)); + nodes.push_back(std::move(node)); + i += 1; + return true; + } else { + HEX_VERBOSE("ggml-hex: skip MUL_MAT_ADD fusion because VTCM needed (%d) > budget (%zu)\n", + kparams.vtcm_size, sess->vtcm_size); + } + } + } + } + return false; } @@ -3393,6 +3550,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.node->src[0], node.node->src[1], node.node, (struct htp_mm_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { + ggml_hexagon_precompute_flash_attn_params(sess, + node.node, + (struct htp_fa_kernel_params *)node.kernel_params + ); } computed_nodes.push_back(std::move(node)); } @@ -4079,6 +4241,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_use_hmx = getenv("GGML_HEXAGON_USE_HMX"); const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); + const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); const char * str_arch = getenv("GGML_HEXAGON_ARCH"); const char * str_vmem = getenv("GGML_HEXAGON_VMEM"); @@ -4120,6 +4283,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; opt_nhmx = str_nhmx ? atoi(str_nhmx) : (str_use_hmx ? atoi(str_use_hmx) : opt_nhmx); opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; + opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; opt_ndev = str_ndev ? strtoul(str_ndev, NULL, 0) : opt_ndev; opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf; opt_mbuf = str_mbuf ? strtoul(str_mbuf, NULL, 0) * MiB : opt_mbuf; diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index 6fe23b0d6aa..19a2504c784 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -11,6 +11,7 @@ #include #include "htp-ops.h" #include "htp/matmul-ops.h" +#include "htp/flash-attn-ops.h" struct htp_opnode { ggml_tensor * node = nullptr; @@ -335,7 +336,8 @@ struct htp_opformat { } void format_kernel_params(char * str, size_t max_size, const htp_opnode & node) { if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || - node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN) { + node.opcode == HTP_OP_MUL_MAT_QKV || node.opcode == HTP_OP_MUL_MAT_FFN || + node.opcode == HTP_OP_MUL_MAT_ADD) { const auto * kparams = (const struct htp_mm_kernel_params *) node.kernel_params; const char * path = "unknown"; int32_t type = kparams->kernel_type; @@ -350,6 +352,16 @@ struct htp_opformat { path = "hvx-flat"; } snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { + const auto * kparams = (const struct htp_fa_kernel_params *) node.kernel_params; + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_FA_KERNEL_HMX) { + path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq"; + } else if (type == HTP_FA_KERNEL_HVX) { + path = "hvx"; + } + snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index c48a5b86e3b..be575796d6d 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -20,9 +20,7 @@ add_library(${HTP_LIB} SHARED worker-pool.c hex-dma.c hmx-queue.c - flash-attn-ops.c - hmx-flash-attn-ops.c - matmul-ops.c + gated-delta-net-ops.c binary-ops.c unary-ops.c sum-rows-ops.c @@ -40,18 +38,15 @@ add_library(${HTP_LIB} SHARED concat-ops.c diag-ops.c solve-tri-ops.c - gated-delta-net-ops.c pad-ops.c + flash-attn-ops.c + matmul-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE $,HTP_DEBUG=1,NDEBUG=1> $,FARF_HIGH=1,>) -if (GGML_HEXAGON_FA_EXP2_HF) - message(STATUS "ggml-htp: HMX_FA_USE_EXP2_HF=1 (use FP16 exp2 polynomial in FA softmax)") - target_compile_definitions(${HTP_LIB} PRIVATE HMX_FA_USE_EXP2_HF=1) -endif() build_idl(htp_iface.idl ${HTP_LIB}) diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index f2a381313c5..51d39e8d98f 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -4,7 +4,7 @@ #include "hexagon_protos.h" #include "hvx_hexagon_protos.h" #include "hex-dma.h" -#include "vtcm-utils.h" +#include "htp-vtcm.h" #include "hvx-utils.h" #include "hex-fastdiv.h" #include diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index 65f7844ae33..6f2a643e69d 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -3,229 +3,41 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include +#include #include #include #include +#include +#include +#include +#include #include #include "hex-dma.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "hmx-queue.h" +#include "hmx-utils.h" #include "hvx-utils.h" #include "hvx-dump.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" #include "hvx-flash-attn.h" +#include "htp-vtcm.h" +#include "worker-pool.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" -int hmx_flash_attn_ext(struct htp_ops_context * octx); +#include "flash-attn-ops.h" +#include "hvx-fa-kernels.h" +#include "hmx-fa-kernels.h" // Must be multiple of 32 #define FLASH_ATTN_BLOCK_SIZE (32 * 2) -#if __HVX_ARCH__ < 79 -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) -#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b)) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) -#else -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) -#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) -#endif - -// This is a bit of a hack because the compiler is strugling to properly inline -// the default hvx_vec_f32_to_f16 with output into the local array. -static __attribute__((noinline)) void hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1) -{ - *(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1); -} - -// Dot product of two F16 vectors, accumulating to float -static inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) { - const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16 - const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]); - - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); - } - - HVX_Vector rsum = HVX_OP_ADD_F32(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)); - rsum = HVX_OP_MUL_F32(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum)); - hvx_vec_store_u(r, 4, rsum); -} - -static inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y, - const uint8_t * restrict x, - const size_t stride_x, - const size_t nvec, - const size_t nloe) { - const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16 - const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16 - const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16 - const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16 - const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 - - HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - - uint32_t i = 0; - - for (i = 0; i < nvec; i++) { - HVX_Vector y_hf = vy[i]; - HVX_Vector x0_hf = vx0[i]; - HVX_Vector x1_hf = vx1[i]; - HVX_Vector x2_hf = vx2[i]; - HVX_Vector x3_hf = vx3[i]; - - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); - rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); - } - - if (nloe) { - // Load x (fp16) and zero-out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); - HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]); - HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]); - HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]); - HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]); - - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); - rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); - } - - HVX_Vector rsum0 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)); - HVX_Vector rsum1 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)); - HVX_Vector rsum2 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p)); - HVX_Vector rsum3 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p)); - - HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } }; - return hvx_vec_reduce_sum_f32x4(rsum0123); -} - -static inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y, - const uint8_t * restrict x, - const size_t stride_x, - const size_t n, - float s) { - - const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - const size_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector sums = Q6_V_vzero(); - const size_t stride_x_4 = stride_x * 4; - for (uint32_t j = 0; j < VLEN_FP32; j += 4) { - HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe); - HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32); - sums = Q6_V_vmux_QVV(pred, sums, sums_x4); - x += stride_x_4; - } - - return HVX_OP_MUL_F32(hvx_vec_splat_f32(s), sums); -} - -// MAD: y (F32) += x (F16) * s (F16) -static inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, int n) { - const HVX_Vector * restrict vx0 = (const HVX_Vector *) x; - - HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; - HVX_Vector * restrict vy = (HVX_Vector *) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector S0 = hvx_vec_splat_f16(*s); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; ++i) { - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); - } - - if (nloe) { - HVX_VectorPair xy_p = vy_p[i]; - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); - - HVX_Vector xy = Q6_V_lo_W(xy_p); - i = 2 * i; // index for vy - - if (nloe >= VLEN_FP32) { - vy[i] = xy; - nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); - } - - if (nloe) { - hvx_vec_store_a(&vy[i], nloe * 4, xy); - } - } -} - -// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16) -static inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1, - const __fp16 * restrict s0, const __fp16 * restrict s1, int n) { - const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0; - const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1; - - HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; - HVX_Vector * restrict vy = (HVX_Vector *) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector S0 = hvx_vec_splat_f16(*s0); - HVX_Vector S1 = hvx_vec_splat_f16(*s1); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; ++i) { - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1); - } - - if (nloe) { - HVX_VectorPair xy_p = vy_p[i]; - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1); - - HVX_Vector xy = Q6_V_lo_W(xy_p); - i = 2 * i; // index for vy - - if (nloe >= VLEN_FP32) { - vy[i] = xy; - nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); - } - - if (nloe) { - hvx_vec_store_a(&vy[i], nloe * 4, xy); - } - } -} - struct htp_fa_context { const struct htp_ops_context * octx; @@ -242,12 +54,12 @@ struct htp_fa_context { float scale; float max_bias; - float logit_softcap; + __fp16 logit_softcap; uint32_t n_head_log2; float m0; float m1; - float slopes[512]; + __fp16 slopes[512]; uint32_t n_blocks; @@ -264,28 +76,80 @@ struct htp_fa_context { bool is_q_fp32; - uint64_t t_start; -}; - -static inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, HVX_Vector vs) { - assert((size_t) dst % 128 == 0); - assert((size_t) src % 128 == 0); + size_t size_q_block; + size_t size_vkq_acc; - const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src; - HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst; + uint8_t * spad_q; + uint8_t * spad_k; + uint8_t * spad_v; + uint8_t * spad_m; + uint8_t * spad_a; - const uint32_t nvec = n / VLEN_FP32; - const uint32_t nloe = n % VLEN_FP32; + uint64_t t_start; +}; - uint32_t i = 0; - #pragma unroll(4) - for (; i < nvec; ++i) { - vdst[i] = HVX_OP_MUL_F32(vsrc[i], vs); - } - if (nloe) { - hvx_vec_store_a(&vdst[i], nloe * sizeof(float), HVX_OP_MUL_F32(vsrc[i], vs)); - } -} +struct hmx_fa_context { + const struct htp_ops_context * octx; + const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent + bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 + uint32_t n_threads; + + // Op parameters + __fp16 scale; + float max_bias; + __fp16 logit_softcap; + uint32_t n_head_log2; + float m0, m1; + + // Dimensions + uint32_t DK, DV; + uint32_t n_kv; // kv_len + uint32_t n_kv_heads; // number of KV heads + uint32_t n_heads; // number of Q heads + uint32_t G; // GQA factor = n_heads / n_kv_heads + struct fastdiv_values div_G; + struct fastdiv_values src3_div2; + struct fastdiv_values src3_div3; + uint32_t n_kv_blocks; + uint32_t neq1; // Q token count + + // Types + bool is_q_fp32; + bool is_dst_fp32; + + // Dynamic block sizes + uint32_t Br; // Q tokens per block (before GQA expansion) + uint32_t Bc; + uint32_t g_br; // hex_align_up(G * Br, 32) - actual tile row dim + + // VTCM buffers (allocated by vtcm_seq_alloc) + __fp16 * vtcm_q_tiles; // Q tile format [g_br, D] + __fp16 * vtcm_o_tiles[2]; // O ping-pong [g_br, D] + __fp16 * vtcm_k_fp16[2]; // K DMA double-buffer [Bc, D] + __fp16 * vtcm_v_fp16[2]; // V DMA double-buffer [Bc, D] + __fp16 * vtcm_k_tiles; // K tiles (transposed) + __fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered) + __fp16 * vtcm_s_tiles; // S = QK^T [g_br, Bc] + __fp16 * vtcm_p_tiles; // P = softmax(S) [g_br, Bc] + __fp16 * vtcm_d_tiles; // Diagonal rescale [g_br, g_br] + HVX_Vector * vtcm_m_vec; // Row max [g_br] + HVX_Vector * vtcm_l_vec; // Row sum [g_br] + HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] + HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] + HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] + uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) + uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) + __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block + __fp16 * vtcm_slopes; // ALiBi slopes [g_br] + size_t row_buf_stride; // HVX vectors per row buffer (Bc/64) + size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer + size_t q_tile_bytes; + size_t o_tile_bytes; + size_t col_vec_bytes; + size_t d_tile_bytes; + bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA) + dma_cache m_cache; +}; static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { struct htp_fa_context * factx = (struct htp_fa_context *) data; @@ -341,7 +205,6 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * if (ir0 >= ir1) return; struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); dma_queue * dma = octx->ctx->dma[ith]; @@ -353,16 +216,14 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const size_t size_v_row = DV * sizeof(__fp16); // Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator - uint8_t * spad_q = octx->src0_spad.data + octx->src0_spad.size_per_thread * ith; - uint8_t * spad_k = octx->src1_spad.data + octx->src1_spad.size_per_thread * ith; - uint8_t * spad_v = octx->src2_spad.data + octx->src2_spad.size_per_thread * ith; - uint8_t * spad_m = octx->src3_spad.data + octx->src3_spad.size_per_thread * ith; - uint8_t * spad_a = octx->dst_spad.data + octx->dst_spad.size_per_thread * ith; - - const HVX_Vector logit_cap = hvx_vec_splat_f32(factx->logit_softcap); + uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith; + uint8_t * spad_k = factx->spad_k + factx->size_k_block * 2 * ith; + uint8_t * spad_v = factx->spad_v + factx->size_v_block * 2 * ith; + uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith; + uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith; dma_cache m_cache; - dma_cache_init(&m_cache, spad_m, factx->size_m_block, DMA_CACHE_MAX_SIZE); + dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); for (uint32_t ir = ir0; ir < ir1; ++ir) { const uint32_t iq3 = fastdiv(ir, &factx->src0_div21); @@ -379,9 +240,6 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3); dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); - // FARF(HIGH, "fa %u: prefetch Q: ir %u iq1 %u iq2 %u iq3 %u q_row_ptr %p size %u : usec %u", ith, ir, iq1, iq2, iq3, q_row_ptr, size_q_row, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); - const __fp16 * mp_base = NULL; if (mask) { const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); @@ -410,18 +268,13 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * // Mask is 1D contiguous for this row dma_cache_push(dma, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); } - - // FARF(HIGH, "fa %u: prefetch KVM: ir %u ib %u iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u", - // ith, ir, ib, iq1, iq2, iq3, - // size_k_row, size_v_row, current_block_size, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); } const uint32_t h = iq2; // head index - const float slope = factx->slopes[h]; + const __fp16 slope = factx->slopes[h]; HVX_Vector S_vec = hvx_vec_splat_f32(0.0f); - HVX_Vector M_vec = hvx_vec_splat_f32(-INFINITY); + HVX_Vector M_vec = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL); // Clear accumulator hvx_splat_f32_a(spad_a, 0, DV); @@ -433,6 +286,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * } const HVX_Vector slope_vec = hvx_vec_splat_f16(slope); + const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) { const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); @@ -442,113 +296,101 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * uint8_t * v_base = dma_queue_pop(dma).dst; // V __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M - // FARF(HIGH, "fa %u: process: ir %u ib %u : iq1 %u iq2 %u iq3 %u q_ptr_vtcm %p : usec %u", - // ith, ir, ib, iq1, iq2, iq3, q_ptr_vtcm, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir); // Inner loop processing the block from VTCM - uint32_t ic = 0; - - // Process in sub-blocks of 32 (VLEN_FP32) - HVX_Vector sb_scores[FLASH_ATTN_BLOCK_SIZE / VLEN_FP32]; - HVX_Vector v_max = hvx_vec_splat_f32(-INFINITY); - for (uint32_t iv = 0; ic < current_block_size; ic += VLEN_FP32, ++iv) { - // 1. Compute scores - HVX_Vector scores = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + ic * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale); - - // 2. Softcap - if (factx->logit_softcap != 0.0f) { - scores = hvx_vec_tanh_f32(scores); - scores = HVX_OP_MUL_F32(scores, logit_cap); - } + // 1. Compute scores (64 elements FP16) + HVX_Vector scores_f16 = Q6_V_vzero(); + if (current_block_size > 0) { + HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale); + HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero(); + scores_f16 = hvx_vec_f32_to_f16(scores0, scores1); + } - // 3. Mask - if (mask) { - const __fp16 * mp = m_base + ic; - HVX_Vector m_vals_f16 = *(const HVX_UVector *) mp; - - // Multiplying -INFINITY (0xFC00) by a slope in VhfVhf instructions can incorrectly produce NaN on v79. - // Clamp -INFINITY to the max negative fp16 finite value (-65504.0f). - HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00); - HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF); - HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf); - m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16); - - #if __HVX_ARCH__ >= 79 - HVX_VectorPair m_vals_f32_pair = Q6_Wsf_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(m_vals_f16), slope_vec); - HVX_Vector add_val = Q6_V_lo_W(m_vals_f32_pair); - scores = Q6_Vsf_vadd_VsfVsf(add_val, scores); - #else - HVX_VectorPair m_vals_f32_pair = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(m_vals_f16), slope_vec); - HVX_Vector add_val = Q6_V_lo_W(m_vals_f32_pair); - scores = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(add_val, scores)); - #endif - } + // 2. Softcap (in FP16) + if (factx->logit_softcap != 0.0f) { + const HVX_Vector v_cap = hvx_vec_splat_f16(factx->logit_softcap); + scores_f16 = hvx_vec_tanh_f16(scores_f16); + scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap); + } - // Mask out invalid lanes for leftover handling - uint32_t valid_lanes = current_block_size - ic; - if (valid_lanes < VLEN_FP32) { - HVX_VectorPred valid_pred = Q6_Q_vsetq_R(valid_lanes * 4); // 4 bytes per fp32 lane - scores = Q6_V_vmux_QVV(valid_pred, scores, hvx_vec_splat_f32(-INFINITY)); - } + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16)); - sb_scores[iv] = scores; - v_max = hvx_vec_reduce_max2_f32(scores, v_max); // All lanes have block max + // 3. Mask (in FP16) + if (mask) { + HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base; + HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00); + HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF); + HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf); + m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16); + + HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec); + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf); + } else { + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf); } + // Compute block max in FP16 + HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16); + HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); // splat block max in FP32 + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, ir); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir); { + const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); + // 4. Online Softmax Update HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec); HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec, M_new_vec); - HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); + + HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec); + HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e); + HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2); + HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16)); + M_vec = M_new_vec; hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); - HVX_Vector p_sum_vec = hvx_vec_splat_f32(0.0f); - for (uint32_t ic2 = 0, iv = 0; ic2 < current_block_size; ic2 += VLEN_FP32, ++iv) { - HVX_Vector scores = sb_scores[iv]; - HVX_Vector scores_shifted = HVX_OP_SUB_F32(scores, M_vec); - HVX_Vector P = hvx_vec_exp_f32(scores_shifted); + // Compute P = exp2((S - M) * log2(e)) in FP16 + HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec, M_vec); + HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16); - p_sum_vec = HVX_OP_ADD_F32(p_sum_vec, P); + HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e); - // 5. Accumulate V - __fp16 __attribute__((aligned(VLEN))) p_arr[VLEN_FP16]; - hvx_vec_f32_to_f16_a(p_arr, P, hvx_vec_splat_f32(0)); + HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2); + P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero()); - float __attribute__((aligned(128))) P_arr[VLEN_FP32]; - hvx_vec_store_a(P_arr, 128, P); + // Convert P to FP32 to update the running sum S_vec + HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P); + HVX_Vector P0 = Q6_V_lo_W(P_pair); + HVX_Vector P1 = Q6_V_hi_W(P_pair); + HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1)); - for (uint32_t j = 0; j < VLEN_FP32; j += 2) { - const uint32_t cur_ic = ic2 + j; - if (cur_ic >= current_block_size) { - break; - } + S_vec = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec, ms_vec), p_sum_vec); - if (cur_ic + 1 == current_block_size) { - // Odd leftover, process single row - if (P_arr[j] != 0.0f) { - const uint8_t * v_ptr = v_base + cur_ic * factx->size_v_row_padded; - hvx_mad_f32_f16_aa(VKQ32, v_ptr, (p_arr + j), DV); - } - break; - } + // 5. Accumulate V (F16 * F16 -> F32 accumulator) + __fp16 __attribute__((aligned(128))) p_arr[VLEN_FP16]; + hvx_vec_store_a(p_arr, 128, P); - // Avoid NaN * 0.0 = NaN for uninitialized V cache rows. - // Check the f32 values to safely avoid strict aliasing violations. - if (P_arr[j] == 0.0f && P_arr[j + 1] == 0.0f) { - continue; + for (uint32_t j = 0; j < current_block_size; j += 2) { + if (j + 1 == current_block_size) { + if (p_arr[j] != 0.0f) { + const uint8_t * v_ptr = v_base + j * factx->size_v_row_padded; + hvx_mad_f32_f16_aa(VKQ32, v_ptr, (p_arr + j), DV); } + break; + } - const uint8_t * v_ptr = v_base + cur_ic * factx->size_v_row_padded; - hvx_mad_f32_f16_aa_rx2(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, (p_arr + j), (p_arr + j + 1), DV); + if (p_arr[j] == 0.0f && p_arr[j + 1] == 0.0f) { + continue; } - } - p_sum_vec = hvx_vec_reduce_sum_f32(p_sum_vec); - S_vec = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec, ms_vec), p_sum_vec); + const uint8_t * v_ptr = v_base + j * factx->size_v_row_padded; + hvx_mad_f32_f16_aa_rx2(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, (p_arr + j), (p_arr + j + 1), DV); + } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir); // Issue DMA for next+1 block (if exists) if (ib + 2 < factx->n_blocks) { @@ -569,14 +411,10 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start); dma_cache_push(dma, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); } - - // FARF(HIGH, "fa %u: prefetch KVM: ir %u ib %u : iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u", - // ith, ir, next_ib, iq1, iq2, iq3, - // size_k_row, size_v_row, next_block_size, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); } } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, ir); // sinks float M = hvx_vec_get_f32(M_vec); float S = hvx_vec_get_f32(S_vec); @@ -605,9 +443,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * // Store result // dst indices - const int i1 = iq1; - const int i2 = iq2; - const int i3 = iq3; + const uint32_t i1 = iq1; + const uint32_t i2 = iq2; + const uint32_t i3 = iq3; // dst is permuted: [DV, n_heads, n_tokens, n_seq] // head stride is nb[1], token stride is nb[2], batch stride is nb[3] @@ -618,8 +456,1542 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * } else if (dst->type == HTP_TYPE_F16) { hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, ir); + } +} + +// ============================================================================ +// HMX Phase args and thread logic +// ============================================================================ + +typedef struct { + struct hmx_fa_context * factx; + uint32_t kv_rows; + size_t src_stride; + void * curr_k; + uint32_t kv_start; + uint32_t rows_per_t; +} fa_k_int_args_t; + +static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { + fa_k_int_args_t * args = (fa_k_int_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const uint32_t total_rows = args->kv_rows; + const uint32_t rows_per_t = args->rows_per_t; + const uint32_t start = i * rows_per_t; + const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows); + + if (start >= total_rows) { + return; + } + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); + hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, (const __fp16 *) args->curr_k, total_rows, factx->DK, + args->src_stride, start, end); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); +} + +static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start) { + worker_pool_context_t wp = factx->octx->ctx->worker_pool; + uint32_t n = 1; + if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { + n = factx->n_threads; + } + uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); + fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t }; + if (n > 1) { + worker_pool_run_func(wp, fa_k_interleave_thread, &args, n); + } else { + fa_k_interleave_thread(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + uint32_t kv_rows; + size_t src_stride; + void * v_src; + void * v_tiles_dst; + size_t n_col_tiles; + uint32_t kv_start; + uint32_t rows_per_t; +} fa_v_int_args_t; + +static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { + fa_v_int_args_t * args = (fa_v_int_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const uint32_t total_rows = args->kv_rows; + const uint32_t rows_per_t = args->rows_per_t; + const uint32_t start = i * rows_per_t; + const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows); + + if (start >= total_rows) { + return; + } + + __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); + hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV, + args->src_stride, (uint32_t) args->n_col_tiles, start, end); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); +} + +static void fa_phase_v_interleave(struct hmx_fa_context * factx, + uint32_t kv_rows, + size_t src_stride, + void * v_src, + void * v_tiles_dst, + size_t n_col_tiles, + uint32_t kv_start) { + worker_pool_context_t wp = factx->octx->ctx->worker_pool; + uint32_t n = 1; + if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { + n = factx->n_threads; + } + uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); + fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; + if (n > 1) { + worker_pool_run_func(wp, fa_v_interleave_thread, &args, n); + } else { + fa_v_interleave_thread(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + const struct htp_tensor * q; + uint32_t q_start; + uint32_t kv_head; + uint32_t ib3; + size_t n_rows_g; + size_t rows_per_t; + size_t n_rows_q; + bool q_transposed; + atomic_uint barrier; +} fa_q_load_args_t; + +static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { + fa_q_load_args_t * args = (fa_q_load_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t G = factx->G; + const size_t DK = factx->DK; + + // Partition the padded Q rows (g_br) across threads. + // Keep start/end even so r and r+1 are always in the same thread's range. + const size_t rows_per_t = args->rows_per_t; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, factx->g_br); + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); + + // Parallel initialization of per-block state + { + const uint32_t g_br = factx->g_br; + const uint32_t DV = factx->DV; + + const size_t col_vec_bytes = factx->col_vec_bytes; + const size_t d_tile_bytes = factx->d_tile_bytes; + + // Initialize vtcm_l_vec & vtcm_m_vec + const size_t l_bytes_per_t = hex_align_up(col_vec_bytes / n, 128); + const size_t l_start = i * l_bytes_per_t; + const size_t l_end = hex_smin(l_start + l_bytes_per_t, col_vec_bytes); + + const size_t m_bytes_per_t = hex_align_up(col_vec_bytes / n, 128); + const size_t m_start = i * m_bytes_per_t; + const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes); + + if (factx->sinks) { + const float * sinks_data = (const float *) (uintptr_t) factx->sinks->data; + float * m_vec = (float *) factx->vtcm_m_vec; + const size_t r_start = l_start / sizeof(float); + const size_t r_end = l_end / sizeof(float); + const float scale_factor = EXP_LOG2E_F; + + const HVX_Vector v_scale = hvx_vec_splat_f32(scale_factor); + + for (size_t r = r_start; r < r_end; r += 32) { + HVX_VectorAlias local_m; + for (size_t j = 0; j < 32; ++j) { + size_t curr_r = r + j; + if (curr_r < n_rows_g) { + const size_t h_idx = fastmodulo(curr_r, G, &factx->div_G); + const size_t head = args->kv_head * G + h_idx; + local_m.fp32[j] = sinks_data[head]; + } else { + local_m.fp32[j] = HTP_FA_M_INITIAL_VAL; + } + } + HVX_Vector v_scaled = HVX_OP_MUL_F32(local_m.v, v_scale); + *(HVX_Vector *) (m_vec + r) = v_scaled; + } + if (l_start < col_vec_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); + } + } else { + if (l_start < col_vec_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); + } + if (m_start < col_vec_bytes) { + hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float)); + } + } + + // Initialize vtcm_d_tiles to 0 + const size_t d_bytes_per_t = hex_align_up(d_tile_bytes / n, 128); + const size_t d_start = i * d_bytes_per_t; + const size_t d_end = hex_smin(d_start + d_bytes_per_t, d_tile_bytes); + if (d_start < d_tile_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_d_tiles + d_start, 0, d_end - d_start); + } + } + + if (start < factx->g_br) { + const struct htp_tensor * q = args->q; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + assert(factx->DK == factx->DV); + + const size_t o_tile_bytes = factx->o_tile_bytes; + const bool use_q_dma = (2 * o_tile_bytes >= factx->g_br * DK * (factx->is_q_fp32 ? 4 : 2)); + + __fp16 * q_tiles = factx->vtcm_q_tiles; + if (use_q_dma) { + const size_t g_rows_end = hex_smin(end, n_rows_g); + const uint32_t d_limit = factx->is_q_fp32 ? DK / 32 : DK / 64; + + uint8_t * q_flat = (uint8_t *) factx->vtcm_o_tiles[0]; + if (factx->is_q_fp32) { + switch (d_limit) { + case 2: hmx_fa_q_prep_fp32_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + case 4: hmx_fa_q_prep_fp32_d4(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + default: hmx_fa_q_prep_fp32( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break; + } + } else { + switch (d_limit) { + case 1: hmx_fa_q_prep_fp16_d1(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + case 2: hmx_fa_q_prep_fp16_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + default: hmx_fa_q_prep_fp16( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break; + } + } + } else { + // Fallback: direct-from-DDR/L2 path + hmx_fa_q_prep_fallback(q_tiles, q->data, q->nb[1], q->nb[2], q->nb[3], + q_start, kv_head, ib3, start, end, n_rows_g, G, DK, factx->is_q_fp32, &factx->div_G); + } + } + + // Synchronize threads before zeroing out vtcm_o_tiles[0] to prevent race condition + if (n > 1) { + atomic_fetch_sub(&args->barrier, 1); + while (atomic_load(&args->barrier) > 0) { + // spin wait + } + } + + // Zero out vtcm_o_tiles[0] as it was used as temp_q_vtcm + { + const uint32_t g_br = factx->g_br; + const uint32_t DV = factx->DV; + const size_t o_tile_bytes = factx->o_tile_bytes; + const size_t o_bytes_per_t = hex_align_up(o_tile_bytes / n, 128); + const size_t o_start = i * o_bytes_per_t; + const size_t o_end = hex_smin(o_start + o_bytes_per_t, o_tile_bytes); + if (o_start < o_tile_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_o_tiles[0] + o_start, 0, o_end - o_start); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); +} + +static void fa_phase_q_load(struct hmx_fa_context * factx, + const struct htp_tensor * q, + uint32_t q_start, + uint32_t kv_head, + uint32_t ib3, + size_t n_rows_g) { + worker_pool_context_t wp = factx->octx->ctx->worker_pool; + uint32_t n = 1; + if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { + n = factx->n_threads; + } + size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2); + const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start); + fa_q_load_args_t args; + args.factx = factx; + args.q = q; + args.q_start = q_start; + args.kv_head = kv_head; + args.ib3 = ib3; + args.n_rows_g = n_rows_g; + args.rows_per_t = rows_per_t; + args.n_rows_q = n_rows_q; + args.q_transposed = q->nb[1] < q->nb[2]; + atomic_init(&args.barrier, n); + if (n > 1) { + worker_pool_run_func(wp, fa_q_load_thread, &args, n); + } else { + fa_q_load_thread(1, 0, &args); } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); +} + +typedef struct { + struct hmx_fa_context * factx; + const struct htp_tensor * dst; + const __fp16 * o_tile_src; + uint32_t q_start; + uint32_t kv_head; + uint32_t ib3; + size_t n_rows_g; + size_t rows_per_t; +} fa_o_store_args_t; + +static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { + fa_o_store_args_t * args = (fa_o_store_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t G = factx->G; + const size_t DV = factx->DV; + + const size_t rows_per_t = args->rows_per_t; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, n_rows_g); + + if (start >= n_rows_g) { + return; + } + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); + + const struct htp_tensor * dst = args->dst; + const __fp16 * o_tile_src = args->o_tile_src; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + for (size_t r = start; r < end; ++r) { + const size_t q_idx = fastdiv(r, &factx->div_G); + const size_t h_idx = fastmodulo(r, G, &factx->div_G); + + float * out = (float *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] + + (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]); + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV; + + for (uint32_t d = 0; d < DV / 32; ++d) { + const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS); + HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]); + if (r1 % 2 == 0) { + *(HVX_UVector *) (out + d * 32) = Q6_V_lo_W(vp); + } else { + *(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp); + } + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); +} + +static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { + fa_o_store_args_t * args = (fa_o_store_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t rows_per_t = args->rows_per_t; + const size_t G = factx->G; + const size_t DV = factx->DV; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, n_rows_g); + + if (start >= n_rows_g) { + return; + } + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); + + const struct htp_tensor * dst = args->dst; + const __fp16 * o_tile_src = args->o_tile_src; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + for (size_t r = start; r < end; ++r) { + const size_t q_idx = fastdiv(r, &factx->div_G); + const size_t h_idx = fastmodulo(r, G, &factx->div_G); + + __fp16 * out = (__fp16 *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] + + (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]); + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV; + + for (uint32_t d = 0; d < DV / 64; ++d) { + const __fp16 * in_dtile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_in1 = pv_in0 + 16; + HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2); + if (r1 % 2 == 0) { + *(HVX_UVector *) (out + d * 64) = Q6_V_lo_W(vp); + } else { + *(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp); + } + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); +} + +static void fa_phase_o_store(struct hmx_fa_context * factx, + const struct htp_tensor * dst, + const __fp16 * o_tile_src, + uint32_t q_start, + uint32_t kv_head, + uint32_t ib3, + size_t n_rows_g) { + worker_pool_context_t wp = factx->octx->ctx->worker_pool; + uint32_t n = 1; + if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { + n = factx->n_threads; + } + size_t rows_per_t = hmx_ceil_div(n_rows_g, n); + fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; + worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; + if (n > 1) { + worker_pool_run_func(wp, store_fn, &args, n); + } else { + store_fn(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + size_t kv_rows; + size_t n_rows_g; + size_t n_col_tiles; + size_t n_tiles_per_bc; + size_t n_row_tiles; + size_t n_row_tiles_g_br; + uint32_t Bc; + uint32_t G; + uint32_t kv_head; + uint32_t kv_start; + uint32_t q_start; + uint32_t ib3; + bool has_alibi; // true when max_bias != 0 (need slope * mask + add) + __fp16 * slopes; + const struct htp_tensor * mask; + const __fp16 * mask_vtcm; // VTCM mask buffer base (NULL = DDR fallback) + size_t mask_vtcm_row_stride; // elements (__fp16) per row in VTCM mask buffer + struct fastdiv_values thread_div; +} fa_softmax_args_t; + +static inline void fa_softmax_impl( + unsigned int n, unsigned int i, void * data, + const bool has_mask, + const bool mask_broadcast, + const bool is_g1, + const bool has_alibi, + const bool has_softcap +) { + fa_softmax_args_t * args = (fa_softmax_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t kv_rows = args->kv_rows; + const size_t Bc = args->Bc; + const size_t G = args->G; + const size_t n_tiles_per_bc = args->n_tiles_per_bc; + const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64); + const uint32_t im3 = has_mask ? fastmodulo(args->ib3, args->mask->ne[3], &factx->src3_div3) : 0; + + size_t vec_start = 0; + size_t vec_end = n_row_vec_cnt; + if (n > 1) { + const size_t vecs_per_t = fastdiv(n_row_vec_cnt + n - 1, &args->thread_div); + vec_start = i * vecs_per_t; + vec_end = hex_smin(vec_start + vecs_per_t, n_row_vec_cnt); + } + + if (vec_start >= n_row_vec_cnt) { + return; + } + + struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); + + // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride + const size_t row_buf_stride = factx->row_buf_stride; + HVX_Vector * my_row_buf0 = factx->vtcm_row_bufs + i * 2 * row_buf_stride; + HVX_Vector * my_row_buf1 = my_row_buf0 + row_buf_stride; + + const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); + + for (size_t r_vec_idx = vec_start; r_vec_idx < vec_end; ++r_vec_idx) { + HVX_Vector rowmax_acc_v = v_neg_inf; + HVX_Vector rowsum_acc_v = Q6_V_vzero(); + HVX_Vector m_prev_v0 = factx->vtcm_m_vec[r_vec_idx * 2 + 0]; + HVX_Vector m_prev_v1 = factx->vtcm_m_vec[r_vec_idx * 2 + 1]; + + HVX_Vector v_slopes = Q6_V_vzero(); + if (has_alibi) { + v_slopes = hvx_vmem(args->slopes + r_vec_idx * 64); + } + + for (uint32_t r_vec_off = 0; r_vec_off < 64; r_vec_off += 2) { + uint32_t r = r_vec_idx * 64 + r_vec_off; + if (r >= hex_align_up(n_rows_g, 2)) { + break; + } + + uint32_t r0 = r / HMX_FP16_TILE_N_ROWS; + uint32_t r1 = r % HMX_FP16_TILE_N_ROWS; + + const __fp16 * s_ld_base = factx->vtcm_s_tiles + r0 * HMX_FP16_TILE_N_ROWS * Bc; + __fp16 * p_st_base = factx->vtcm_p_tiles + r0 * HMX_FP16_TILE_N_ROWS * Bc; + + // Decode 2 rows from S tiles into per-thread row buffers + if (has_softcap) { + const HVX_Vector v_cap = hvx_vec_splat_f16(factx->logit_softcap); + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_s_in1 = pv_s_in0 + 16; + + HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2); + HVX_Vector v_s_row0 = Q6_V_lo_W(vp_s_drow); + HVX_Vector v_s_row1 = Q6_V_hi_W(vp_s_drow); + + HVX_Vector t0 = hvx_vec_tanh_f16(v_s_row0); + my_row_buf0[ci] = hvx_vec_mul_f16_f16(t0, v_cap); + + HVX_Vector t1 = hvx_vec_tanh_f16(v_s_row1); + my_row_buf1[ci] = hvx_vec_mul_f16_f16(t1, v_cap); + } + } else { + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_s_in1 = pv_s_in0 + 16; + + HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2); + my_row_buf0[ci] = Q6_V_lo_W(vp_s_drow); + my_row_buf1[ci] = Q6_V_hi_W(vp_s_drow); + } + } + + // Apply mask & compute rowmax(S) + HVX_Vector v_slope0 = Q6_V_vzero(); + HVX_Vector v_slope1 = Q6_V_vzero(); + if (has_alibi) { + v_slope0 = hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, r_vec_off * 2)); + v_slope1 = (r + 1 < n_rows_g) ? hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, (r_vec_off + 1) * 2)) : Q6_V_vzero(); + } + + const HVX_Vector v_threshold = Q6_Vh_vsplat_R(0xcc00); // fp16 -16.0 + + HVX_Vector v_s_rowmax0 = v_neg_inf; + HVX_Vector v_s_rowmax1 = v_neg_inf; + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + const size_t ne = hex_smin(kv_rows - c, 64); + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16)); + + if (has_mask) { + HVX_Vector v_mask0, v_mask1; + + if (mask_broadcast) { + if (is_g1) { + const size_t qi0 = r + 0; + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t qi1 = r + 1; + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c); + } + } else { + const size_t qi0 = fastdiv(r + 0, &factx->div_G); + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t qi1 = fastdiv(r + 1, &factx->div_G); + if (qi1 == qi0) { + v_mask1 = v_mask0; + } else { + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c); + } + } + } + } else { + // Head-dependent mask: pre-interleaved per row r. + const size_t r0 = r + 0; + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + r0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t r1 = r + 1; + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + r1 * args->mask_vtcm_row_stride + c); + } + } + + // Threshold: mask values below -16.0 are treated as -inf (causal mask). + HVX_VectorPred q_keep0 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask0, v_threshold), q_tail_keep); + HVX_VectorPred q_keep1 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask1, v_threshold), q_tail_keep); + + // Scale mask values by log2(e) for base-2 calculations + const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); + HVX_Vector v_mask0_scaled = hvx_vec_mul_f16_f16(v_mask0, v_log2e); + HVX_Vector v_mask1_scaled = hvx_vec_mul_f16_f16(v_mask1, v_log2e); + + if (has_alibi) { + HVX_Vector v_sm0 = hvx_vec_mul_f16_f16(v_mask0_scaled, v_slope0); + HVX_Vector v_sm1 = hvx_vec_mul_f16_f16(v_mask1_scaled, v_slope1); + my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_sm0), v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_sm1), v_neg_inf); + } else { + my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_mask0_scaled), v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_mask1_scaled), v_neg_inf); + } + } else { + if (ne < 64) { + my_row_buf0[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf0[ci], v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf1[ci], v_neg_inf); + } + } + + v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]); + v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]); + } + + v_s_rowmax0 = hvx_vec_reduce_max_f16(v_s_rowmax0); + v_s_rowmax1 = hvx_vec_reduce_max_f16(v_s_rowmax1); + + // Splat m_prev[r], m_prev[r+1] from the float per-row accumulators and convert to fp16 vectors + HVX_Vector v_m_prev0, v_m_prev1; + if (r_vec_off < 32) { + HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, r_vec_off * 4)); + v_m_prev0 = hvx_vec_f32_to_f16(v0, v0); + if (r + 1 < n_rows_g) { + HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, (r_vec_off + 1) * 4)); + v_m_prev1 = hvx_vec_f32_to_f16(v1, v1); + } else { + v_m_prev1 = Q6_V_vzero(); + } + } else { + HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off - 32) * 4)); + v_m_prev0 = hvx_vec_f32_to_f16(v0, v0); + if (r + 1 < n_rows_g) { + HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off + 1 - 32) * 4)); + v_m_prev1 = hvx_vec_f32_to_f16(v1, v1); + } else { + v_m_prev1 = Q6_V_vzero(); + } + } + + HVX_Vector v_dup_m0 = Q6_Vhf_vmax_VhfVhf(v_m_prev0, v_s_rowmax0); + HVX_Vector v_dup_m1 = Q6_Vhf_vmax_VhfVhf(v_m_prev1, v_s_rowmax1); + + // Insert row r, r+1 rowmax into rowmax_acc_v + { + HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); + HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); + HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); + HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); + HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); + rowmax_acc_v = Q6_V_vmux_QVV(p_lane0, v_dup_m0, rowmax_acc_v); + rowmax_acc_v = Q6_V_vmux_QVV(p_lane1, v_dup_m1, rowmax_acc_v); + } + + // Compute P = exp(S - m_new) + const HVX_Vector v_zero = Q6_V_vzero(); + HVX_Vector v_p_rowsum0 = v_zero; + HVX_Vector v_p_rowsum1 = v_zero; + + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0); + HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1); + + HVX_Vector v_p_row0_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0)); + HVX_Vector v_p_row1_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1)); + __fp16 * out_dtile = p_st_base + ci * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_p_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_p_out1 = pv_p_out0 + 16; + + HVX_VectorPair vp_p_dual = Q6_W_vshuff_VVR(v_p_row1_hf, v_p_row0_hf, -2); + *pv_p_out0 = Q6_V_lo_W(vp_p_dual); + *pv_p_out1 = Q6_V_hi_W(vp_p_dual); + + HVX_VectorPair vp_p0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf); + HVX_VectorPair vp_p1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf); + + v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0), Q6_V_hi_W(vp_p0))); + v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1), Q6_V_hi_W(vp_p1))); + } + + HVX_Vector rowsum0_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum0)); + HVX_Vector rowsum1_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum1)); + { + HVX_Vector rv0_v = hvx_vec_f32_to_f16(rowsum0_sf, rowsum0_sf); + HVX_Vector rv1_v = hvx_vec_f32_to_f16(rowsum1_sf, rowsum1_sf); + + HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); + HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); + HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); + HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); + HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); + rowsum_acc_v = Q6_V_vmux_QVV(p_lane0, rv0_v, rowsum_acc_v); + rowsum_acc_v = Q6_V_vmux_QVV(p_lane1, rv1_v, rowsum_acc_v); + } + } + + // Inline fa_ml_update_and_build_d for this vector (lock-free and in parallel) + HVX_VectorPair rowmax_acc_pair = hvx_vec_f16_to_f32(rowmax_acc_v); + HVX_Vector v_rowmax_acc_f32_0 = Q6_V_lo_W(rowmax_acc_pair); + HVX_Vector v_rowmax_acc_f32_1 = Q6_V_hi_W(rowmax_acc_pair); + + HVX_Vector v_m_curr0 = Q6_Vsf_vmax_VsfVsf(m_prev_v0, v_rowmax_acc_f32_0); + HVX_Vector v_m_curr1 = Q6_Vsf_vmax_VsfVsf(m_prev_v1, v_rowmax_acc_f32_1); + + HVX_Vector v_m_diff0 = HVX_OP_SUB_F32(m_prev_v0, v_m_curr0); + HVX_Vector v_m_diff1 = HVX_OP_SUB_F32(m_prev_v1, v_m_curr1); + + HVX_Vector v_m_diff_f16 = hvx_vec_f32_to_f16(v_m_diff0, v_m_diff1); + HVX_Vector exp_m_diff_f16 = hvx_vec_exp2_f16(v_m_diff_f16); + + HVX_VectorPair exp_m_diff_pair = hvx_vec_f16_to_f32(exp_m_diff_f16); + HVX_Vector exp_m_diff0 = Q6_V_lo_W(exp_m_diff_pair); + HVX_Vector exp_m_diff1 = Q6_V_hi_W(exp_m_diff_pair); + + HVX_VectorPair rowsum_acc_pair = hvx_vec_f16_to_f32(rowsum_acc_v); + HVX_Vector v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair); + HVX_Vector v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair); + + HVX_Vector v_l_curr0; + HVX_Vector v_l_curr1; + if (args->kv_start == 0 && factx->sinks != NULL) { + // First KV block with sinks: m_prev holds the seeded sink value (not -inf), + // so exp_m_diff = exp2(sink - m_curr) is the sink's contribution to the + // denominator. l_prev is 0 here, so add exp_m_diff directly instead of + // multiplying the (uninitialized) l_prev term. + v_l_curr0 = HVX_OP_ADD_F32(exp_m_diff0, v_rowsum_acc_f32_0); + v_l_curr1 = HVX_OP_ADD_F32(exp_m_diff1, v_rowsum_acc_f32_1); + } else { + HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0]; + HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1]; + v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0); + v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1); + } + + factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0; + factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1; + factx->vtcm_l_vec[r_vec_idx * 2 + 0] = v_l_curr0; + factx->vtcm_l_vec[r_vec_idx * 2 + 1] = v_l_curr1; + + // Build diagonal tile D = diag(exp(m_diff)) + const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; + const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); + HVX_Vector v_exp_m_diff = exp_m_diff_f16; + + size_t t0 = r_vec_idx * 2; + if (t0 < args->n_row_tiles) { + const HVX_Vector v_content = v_exp_m_diff; + __fp16 * out_base = factx->vtcm_d_tiles + t0 * (args->n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } + + size_t t1 = r_vec_idx * 2 + 1; + if (t1 < args->n_row_tiles) { + const HVX_Vector v_content = Q6_V_vror_VR(v_exp_m_diff, 64); + __fp16 * out_base = factx->vtcm_d_tiles + t1 * (args->n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); +} + +static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/false, + /*mask_broadcast=*/false, + /*is_g1=*/false, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/true, + /*mask_broadcast=*/true, + /*is_g1=*/true, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/true, + /*mask_broadcast=*/true, + /*is_g1=*/false, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { + fa_softmax_args_t * args = (fa_softmax_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const bool has_mask = (args->mask != NULL); + const bool mask_broadcast = factx->mask_broadcast; + const bool is_g1 = (args->G == 1); + const bool has_alibi = args->has_alibi; + const bool has_softcap = (factx->logit_softcap != 0.0f); + + fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); +} + +static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx, + size_t n_row_tiles, + size_t n_row_tiles_g_br) { + const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; + const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + HVX_Vector v_content = Q6_V_vzero(); + for (size_t i = 0; i < n_row_tiles; ++i) { + if ((i % 2) == 0) { + HVX_Vector inv_lo = HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i])); + HVX_Vector inv_hi = (i + 1 < n_row_tiles) ? HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i + 1])) : Q6_V_vzero(); + v_content = hvx_vec_f32_to_f16(inv_lo, inv_hi); + } else { + v_content = Q6_V_vror_VR(v_content, 64); + } + + __fp16 * out_base = factx->vtcm_d_tiles + i * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } +} + +static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, + fa_softmax_args_t * sargs, + size_t n_row_tiles, + size_t n_row_tiles_g_br) { + worker_pool_context_t wp = factx->octx->ctx->worker_pool; + const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); + + worker_callback_t softmax_fn = fa_softmax_thread; + if (sargs->mask == NULL && factx->logit_softcap == 0.0f && !sargs->has_alibi) { + softmax_fn = fa_softmax_thread_nomask; + } else if (sargs->mask != NULL && factx->mask_broadcast && factx->logit_softcap == 0.0f && !sargs->has_alibi) { + if (sargs->G == 1) { + softmax_fn = fa_softmax_thread_mask_broadcast_g1; + } else { + softmax_fn = fa_softmax_thread_mask_broadcast_gn; + } + } + + if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { + uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); + sargs->thread_div = init_fastdiv_values(n_use); + worker_pool_run_func(wp, softmax_fn, sargs, n_use); + } else { + softmax_fn(1, 0, sargs); + } +} + +// ============================================================================ +// HMX job structs and worker functions +// ============================================================================ + +typedef struct { + const __fp16 * q_tiles; + const __fp16 * k_tiles; + __fp16 * s_tiles; + size_t n_row_tiles; + size_t n_col_tiles; + size_t n_dot_tiles; // DK / 32 + size_t n_tiles_per_bc; + uint8_t * hmx_scales; +} hmx_fa_qk_job_t; + +static void hmx_fa_qk_dot_worker(void * data) { + hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_col_tiles = job->n_col_tiles; + const size_t n_dot_tiles = job->n_dot_tiles; + const size_t n_tiles_per_bc = job->n_tiles_per_bc; + const __fp16 * restrict q_tiles = job->q_tiles; + const __fp16 * restrict k_tiles = job->k_tiles; + __fp16 * restrict s_tiles = job->s_tiles; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * row_tiles = q_tiles + r * dot_stride; + const __fp16 * col_tiles = k_tiles; + __fp16 * out_tile = s_tiles + r * n_tiles_per_bc * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < n_col_tiles; ++c) { + hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles); + col_tiles += dot_stride; + out_tile += HMX_FP16_TILE_N_ELMS; + } + } +} + +typedef struct { + __fp16 * o_curr; + const __fp16 * o_prev; + const __fp16 * p_tiles; + const __fp16 * v_tiles; + const __fp16 * d_tiles; + uint8_t * hmx_scales; + size_t n_row_tiles; + size_t n_col_tiles; + size_t n_row_tiles_g_br; + size_t n_tiles_per_bc; + size_t DV; +} hmx_fa_o_update_job_t; + +static void hmx_fa_o_update_worker(void * data) { + hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_col_tiles = job->n_col_tiles; + const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; + const size_t n_tiles_per_bc = job->n_tiles_per_bc; + const size_t DV_tiles = job->DV / 32; + const __fp16 * restrict d_tiles = job->d_tiles; + const __fp16 * restrict p_tiles = job->p_tiles; + const __fp16 * restrict v_tiles = job->v_tiles; + const __fp16 * restrict o_prev = job->o_prev; + __fp16 * restrict o_curr = job->o_curr; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(DV_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; + const size_t v_stride = n_tiles_per_bc * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; + const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; + const __fp16 * v_tile_in = v_tiles; + __fp16 * o_tile_out = o_curr + r * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < DV_tiles; ++c) { + hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles); + o_rc += o_stride; + v_tile_in += v_stride; + o_tile_out += o_stride; + } + } +} + +typedef struct { + __fp16 * o_curr; // output (row-major tile layout) + const __fp16 * o_prev; // input (column-major tile layout) + const __fp16 * d_tiles; // diag(1/l) tiles + uint8_t * hmx_scales; + size_t n_row_tiles; + size_t n_row_tiles_g_br; + size_t DV; +} hmx_fa_o_norm_job_t; + +static void hmx_fa_o_norm_worker(void * data) { + hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; + const size_t DV_tiles = job->DV / 32; + const __fp16 * restrict d_tiles = job->d_tiles; + const __fp16 * restrict o_prev = job->o_prev; + __fp16 * restrict o_curr = job->o_curr; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(DV_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; + const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; + __fp16 * o_out = o_curr + r * DV_tiles * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < DV_tiles; ++c) { + hmx_fa_o_norm_tile(d_diag, o_rc, o_out); + o_rc += o_stride; + o_out += HMX_FP16_TILE_N_ELMS; + } + } +} + +// Populate per-GQA-row ALiBi slopes for a given KV head. +static __attribute__((noinline)) void fa_compute_slopes( + const struct hmx_fa_context * factx, + uint32_t kv_head, + size_t n_rows_g) { + __fp16 * slopes = factx->vtcm_slopes; + if (factx->max_bias == 0.0f) { + hvx_splat_f16_a(slopes, 1.0f, n_rows_g); + return; + } + + const uint32_t G = factx->G; + const uint32_t n_head_log2 = factx->n_head_log2; + const float m0 = factx->m0; + const float m1 = factx->m1; + + __fp16 temp_slopes[512] __attribute__((aligned(128))); + if (G <= 32) { + // Fast path: Compute G unique slope values in vector registers + HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1); + + __fp16 temp_slopes_aligned[64] __attribute__((aligned(128))); + hvx_vmem(temp_slopes_aligned) = hvx_vec_f32_to_f16(v_val, Q6_V_vzero()); + + for (uint32_t i = 0; i < G; ++i) { + temp_slopes[i] = temp_slopes_aligned[i]; + } + } else { + // Fallback path: G > 32 (rare configurations) + for (uint32_t i = 0; i < G; ++i) { + temp_slopes[i] = (__fp16)alibi_slope(kv_head * G + i, n_head_log2, m0, m1); + } + } + + // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses) + __fp16 local_slopes[n_rows_g] __attribute__((aligned(128))); + for (size_t r = 0; r < n_rows_g; ++r) { + local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)]; + } + + // Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes) + hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g); +} + +static void fa_push_mask_dma_gqa( + dma_queue * dma, + const struct htp_tensor * mask, + uint32_t q_start, + uint32_t im3, + uint32_t kv_start, + uint32_t kv_head, + uint32_t G, + uint32_t m_line_bytes, + uint32_t kv_rows, + uint32_t n_rows_q, + struct hmx_fa_context * factx +) { + for (uint32_t g = 0; g < G; ++g) { + const uint32_t h_idx = kv_head * G + g; + const uint32_t im2 = fastmodulo(h_idx, mask->ne[2], &factx->src3_div2); + const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + + im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + uint8_t * ms_dst = (uint8_t *) factx->vtcm_mask_buf + g * m_line_bytes; + dma_queue_push(dma, dma_make_ptr(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + } +} + +static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) { + for (uint32_t g = 0; g < G; ++g) { + dma_queue_pop(dma); + } +} + +// ============================================================================ +// Core HMX flash attention algorithm (GQA-merged) +// ============================================================================ + +int hmx_flash_attn_ext(struct htp_ops_context * octx) { + struct htp_thread_trace * tr_hvx = octx->ctx ? &octx->ctx->trace[0] : NULL; + struct htp_thread_trace * tr_hmx = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * mask = (octx->src[3] && octx->src[3]->data) ? octx->src[3] : NULL; + const struct htp_tensor * dst = octx->dst; + + struct htp_context * const ctx = octx->ctx; + + if (!ctx->hmx_enabled) { + return HTP_STATUS_NO_SUPPORT; + } + + // Dimensions + const uint32_t neq0 = q->ne[0]; // head_dim (DK) + const uint32_t neq1 = q->ne[1]; // n_tokens + const uint32_t neq2 = q->ne[2]; // n_heads + const uint32_t neq3 = q->ne[3]; // n_seqs + + const uint32_t nek0 = k->ne[0]; // head_dim + const uint32_t nek1 = k->ne[1]; // kv_len + + const uint32_t nev0 = v->ne[0]; // head_dim (DV) + + const uint32_t DK = neq0; + const uint32_t DV = nev0; + + // HMX requires head_dim to be multiple of 32 + if (DK % 32 != 0 || DV % 32 != 0) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; + const uint32_t n_kv_heads = k->ne[2]; + + // ======== Build context ======== + struct hmx_fa_context factx; + memset(&factx, 0, sizeof(factx)); + factx.octx = octx; + factx.sinks = octx->src[4]; // NULL if this op has no attention sinks + factx.n_threads = kparams->n_threads; + factx.DK = DK; + factx.DV = DV; + factx.n_kv = nek1; + factx.n_kv_heads = n_kv_heads; + factx.n_heads = neq2; + factx.G = kparams->G; + factx.div_G = kparams->u.hmx.div_G; + factx.neq1 = neq1; + factx.Br = kparams->Br; + factx.Bc = kparams->Bc; + factx.g_br = kparams->u.hmx.g_br; + factx.n_kv_blocks = kparams->n_kv_blocks; + factx.is_q_fp32 = (kparams->is_q_fp32 != 0); + factx.is_dst_fp32 = (kparams->is_dst_fp32 != 0); + factx.pipeline = (kparams->u.hmx.pipeline != 0); + factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0); + if (mask) { + factx.src3_div2 = kparams->src3_div2; + factx.src3_div3 = kparams->src3_div3; + } + + if (kparams->logit_softcap == 0.0f) { + factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e) + } else { + factx.scale = (__fp16) kparams->scale; + } + factx.max_bias = kparams->max_bias; + factx.logit_softcap = (__fp16) (kparams->logit_softcap * EXP_LOG2E_F); + + factx.n_head_log2 = kparams->n_head_log2; + factx.m0 = kparams->m0; + factx.m1 = kparams->m1; + + const uint32_t Br = factx.Br; + const uint32_t Bc = factx.Bc; + const uint32_t g_br = factx.g_br; + const bool pipeline = factx.pipeline; + const uint32_t n_threads = factx.n_threads; + const uint32_t G = factx.G; + + // ======== VTCM allocation (GQA-aware) ======== + // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used + // throughout the KV loop below. + const size_t size_k_row = DK * sizeof(__fp16); + const size_t size_v_row = DV * sizeof(__fp16); + const size_t size_k_row_padded = hex_round_up(size_k_row, 128); + const size_t size_v_row_padded = hex_round_up(size_v_row, 128); + + // Build the VTCM layout once (shared with the host estimator) and place every + // scratch buffer at its computed offset. + struct hmx_fa_vtcm_layout L; + hmx_fa_vtcm_layout_build(&L, G, DK, DV, Br, Bc, n_threads, pipeline); + + if (L.total_bytes > ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + uint8_t * const base = ctx->vtcm_base; + + factx.vtcm_q_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_tiles); + factx.vtcm_o_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[0]); + factx.vtcm_o_tiles[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[1]); + factx.vtcm_k_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[0]); + factx.vtcm_k_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[1]); + factx.vtcm_v_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[0]); + factx.vtcm_v_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[1]); + factx.vtcm_k_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_tiles); + factx.vtcm_v_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_tiles[0]); + factx.vtcm_v_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_v_tiles[1], pipeline); + factx.vtcm_s_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_s_tiles); + factx.vtcm_p_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_p_tiles); + factx.vtcm_d_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles); + factx.vtcm_m_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_m_vec); + factx.vtcm_l_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_l_vec); + factx.vtcm_s_rowmax = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_s_rowmax); + factx.vtcm_p_rowsum = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_p_rowsum); + factx.vtcm_row_bufs = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_row_bufs); + factx.row_buf_stride = L.row_buf_stride; + factx.vtcm_hmx_scales_id = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_id); + factx.vtcm_hmx_scales_qk = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_qk); + factx.vtcm_mask_buf = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf); + factx.mask_buf_row_stride = L.mask_buf_row_stride; + factx.q_tile_bytes = L.q_tile_bytes; + factx.o_tile_bytes = L.o_tile_bytes; + factx.col_vec_bytes = L.col_vec_bytes; + factx.d_tile_bytes = L.d_tile_bytes; + factx.vtcm_slopes = VTCM_LAYOUT_PTR(__fp16, base, L.off_slopes); + + const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop + + dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); + + // ======== Initialize HMX output scales ======== + hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 + hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale)); + + // ======== Skip compute if profiling ======== + if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + return HTP_STATUS_OK; + } + + // ======== DMA setup ======== + dma_queue * const dma = ctx->dma[0]; + + const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS; + const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS; + + const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16); + + // ======== Reusable job descriptors for pipeline ======== + hmx_fa_qk_job_t qk_job; + hmx_fa_o_update_job_t ou_job; + hmx_fa_o_norm_job_t on_job; + + // ======== Main loop ======== + for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { + const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; + for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { + const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start); + const size_t n_rows_g = n_rows_q * G; + const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); + const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS; + + for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) { + const uint32_t ik2 = kv_head; + const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3); + const uint32_t iv2 = kv_head; + const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3); + + // 1. Push Q DMA (if Q DMA is used) + const size_t o_tile_bytes = factx.o_tile_bytes; + const bool use_q_dma = (2 * o_tile_bytes >= factx.g_br * factx.DK * (factx.is_q_fp32 ? 4 : 2)); + if (use_q_dma) { + const bool q_transposed = q->nb[1] < q->nb[2]; + const uint8_t * q_ptr = (const uint8_t *) q->data + q_start * q->nb[1] + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; + const size_t el_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16); + const size_t q_row_bytes = q_transposed ? n_rows_q * factx.DK * el_size : factx.G * factx.DK * el_size; + const size_t src_stride = q_transposed ? q->nb[2] : q->nb[1]; + const size_t n_rows = q_transposed ? factx.G : n_rows_q; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_o_tiles[0], q_ptr), q_row_bytes, hex_smax(src_stride, q_row_bytes), q_row_bytes, n_rows); + } + + // 2. Prefetch first KV block + if (factx.n_kv_blocks > 0) { + const uint32_t kv_rows0 = hex_smin(Bc, nek1); + + const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0); + + const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0); + } + + // 3. Pop Q DMA (blocks until Q is loaded) + if (use_q_dma) { + dma_queue_pop(dma); + } + + // ---- Load Q block & Initialize per-block state ---- + fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g); + + __fp16 * o_tile_prev = factx.vtcm_o_tiles[0]; + __fp16 * o_tile_curr = factx.vtcm_o_tiles[1]; + + // ---- KV block loop with DMA double-buffering ---- + size_t buf_idx = 0; + + htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); + fa_compute_slopes(&factx, kv_head, n_rows_g); + htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); + + const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); + const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); + + struct hmx_queue * hmx_q = ctx->hmx_queue; + + if (factx.pipeline) { + // Pipeline path + for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { + const uint32_t kv_start = kv_blk * Bc; + const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); + const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); + + // Push mask DMA + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); + } + } + + // Prefetch next KV block early + if (kv_blk + 1 < factx.n_kv_blocks) { + const uint32_t prefetch_start = (kv_blk + 1) * Bc; + const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); + const size_t prefetch_buf = 1 - buf_idx; + const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + } + + // ---- Phase 1: K_int ---- + if (kv_blk > 0) { + ou_job.o_curr = o_tile_curr; + ou_job.o_prev = o_tile_prev; + ou_job.p_tiles = factx.vtcm_p_tiles; + ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; + ou_job.d_tiles = factx.vtcm_d_tiles; + ou_job.hmx_scales = factx.vtcm_hmx_scales_id; + ou_job.n_row_tiles = n_row_tiles; + ou_job.n_col_tiles = hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); + ou_job.n_row_tiles_g_br = n_row_tiles_g_br; + ou_job.n_tiles_per_bc = n_tiles_per_bc; + ou_job.DV = DV; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); + } + + // Wait for current K DMA and interleave + void * curr_k = dma_queue_pop(dma).dst; + fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start); + + // ---- Phase 2: qk_dot ---- + qk_job.q_tiles = factx.vtcm_q_tiles; + qk_job.k_tiles = factx.vtcm_k_tiles; + qk_job.s_tiles = factx.vtcm_s_tiles; + qk_job.n_row_tiles = n_row_tiles; + qk_job.n_col_tiles = n_col_tiles; + qk_job.n_dot_tiles = DK / 32; + qk_job.n_tiles_per_bc = n_tiles_per_bc; + qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job)); + + // Wait for current V DMA and interleave + void * curr_v = dma_queue_pop(dma).dst; + fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[buf_idx], n_tiles_per_bc, kv_start); + + if (kv_blk > 0) { + hmx_queue_pop(hmx_q); + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + hmx_queue_pop(hmx_q); + + // ---- Phase 3: softmax + build_D ---- + __fp16 * current_mask_vtcm = NULL; + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; + } else { + fa_pop_mask_dma_gqa(dma, G); + current_mask_vtcm = factx.vtcm_mask_buf; + } + } + + fa_softmax_args_t sargs; + memset(&sargs, 0, sizeof(sargs)); + sargs.factx = &factx; + sargs.kv_rows = kv_rows; + sargs.n_rows_g = n_rows_g; + sargs.n_col_tiles = n_col_tiles; + sargs.n_tiles_per_bc = n_tiles_per_bc; + sargs.n_row_tiles = n_row_tiles; + sargs.n_row_tiles_g_br = n_row_tiles_g_br; + sargs.Bc = Bc; + sargs.G = G; + sargs.kv_head = kv_head; + sargs.kv_start = kv_start; + sargs.q_start = q_start; + sargs.ib3 = ib3; + sargs.has_alibi = (factx.max_bias != 0.0f); + sargs.mask = mask; + sargs.mask_vtcm = current_mask_vtcm; + sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; + sargs.slopes = factx.vtcm_slopes; + fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); + + buf_idx = 1 - buf_idx; + } + + // Epilogue + if (factx.n_kv_blocks > 0) { + const uint32_t last_blk = factx.n_kv_blocks - 1; + const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS); + ou_job.o_curr = o_tile_curr; + ou_job.o_prev = o_tile_prev; + ou_job.p_tiles = factx.vtcm_p_tiles; + ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; + ou_job.d_tiles = factx.vtcm_d_tiles; + ou_job.hmx_scales = factx.vtcm_hmx_scales_id; + ou_job.n_row_tiles = n_row_tiles; + ou_job.n_col_tiles = last_cols; + ou_job.n_row_tiles_g_br = n_row_tiles_g_br; + ou_job.n_tiles_per_bc = n_tiles_per_bc; + ou_job.DV = DV; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); + hmx_queue_pop(hmx_q); + + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + } else { + // Fallback path + for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { + const uint32_t kv_start = kv_blk * Bc; + const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); + const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); + + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); + } + } + + if (kv_blk + 1 < factx.n_kv_blocks) { + const uint32_t prefetch_start = (kv_blk + 1) * Bc; + const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); + const size_t prefetch_buf = 1 - buf_idx; + const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + } + + // Wait for current K DMA and interleave + void * curr_k = dma_queue_pop(dma).dst; + fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start); + + { + qk_job.q_tiles = factx.vtcm_q_tiles; + qk_job.k_tiles = factx.vtcm_k_tiles; + qk_job.s_tiles = factx.vtcm_s_tiles; + qk_job.n_row_tiles = n_row_tiles; + qk_job.n_col_tiles = n_col_tiles; + qk_job.n_dot_tiles = (size_t) (DK / 32); + qk_job.n_tiles_per_bc = n_tiles_per_bc; + qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; + + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job)); + hmx_queue_pop(ctx->hmx_queue); + } + + // Wait for current V DMA and interleave + void * curr_v = dma_queue_pop(dma).dst; + fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[0], n_tiles_per_bc, kv_start); + + // ---- Phase 3: softmax + build_D ---- + __fp16 * current_mask_vtcm = NULL; + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst; + } else { + fa_pop_mask_dma_gqa(dma, G); + current_mask_vtcm = factx.vtcm_mask_buf; + } + } + + fa_softmax_args_t sargs; + memset(&sargs, 0, sizeof(sargs)); + sargs.factx = &factx; + sargs.kv_rows = kv_rows; + sargs.n_rows_g = n_rows_g; + sargs.n_col_tiles = n_col_tiles; + sargs.n_tiles_per_bc = n_tiles_per_bc; + sargs.n_row_tiles = n_row_tiles; + sargs.n_row_tiles_g_br = n_row_tiles_g_br; + sargs.Bc = Bc; + sargs.G = G; + sargs.kv_head = kv_head; + sargs.kv_start = kv_start; + sargs.q_start = q_start; + sargs.ib3 = ib3; + sargs.has_alibi = (factx.max_bias != 0.0f); + sargs.mask = mask; + sargs.mask_vtcm = current_mask_vtcm; + sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; + sargs.slopes = factx.vtcm_slopes; + fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); + + { + ou_job.o_curr = o_tile_curr; + ou_job.o_prev = o_tile_prev; + ou_job.p_tiles = factx.vtcm_p_tiles; + ou_job.v_tiles = factx.vtcm_v_tiles[0]; + ou_job.d_tiles = factx.vtcm_d_tiles; + ou_job.hmx_scales = factx.vtcm_hmx_scales_id; + ou_job.n_row_tiles = n_row_tiles; + ou_job.n_col_tiles = n_col_tiles; + ou_job.n_row_tiles_g_br = n_row_tiles_g_br; + ou_job.n_tiles_per_bc = n_tiles_per_bc; + ou_job.DV = DV; + + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); + hmx_queue_pop(ctx->hmx_queue); + + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + buf_idx = 1 - buf_idx; + } + } + + // ---- Final normalization ---- + { + htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br); + htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + + on_job.o_curr = o_tile_curr; + on_job.o_prev = o_tile_prev; + on_job.d_tiles = factx.vtcm_d_tiles; + on_job.hmx_scales = factx.vtcm_hmx_scales_id; + on_job.n_row_tiles = n_row_tiles; + on_job.n_row_tiles_g_br = n_row_tiles_g_br; + on_job.DV = DV; + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job)); + hmx_queue_pop(ctx->hmx_queue); + } + + // ---- Store O block ---- + fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g); + } + } + } + + return HTP_STATUS_OK; } int op_flash_attn_ext(struct htp_ops_context * octx) { @@ -634,13 +2006,14 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - // HMX path: head_dim multiple of 64, F16 KV, and no sinks - if (k->type == HTP_TYPE_F16 && v->type == HTP_TYPE_F16 && k->ne[0] % 64 == 0 && v->ne[0] % 64 == 0 && octx->src[4] == NULL) { - int ret = hmx_flash_attn_ext(octx); - if (ret == HTP_STATUS_OK) { - return ret; - } - // VTCM too small or other failure -> fall through to HVX path + const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; + + if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) { + return HTP_STATUS_NO_SUPPORT; + } + + if (kparams->kernel_type == HTP_FA_KERNEL_HMX) { + return hmx_flash_attn_ext(octx); } struct htp_fa_context factx; @@ -648,94 +2021,68 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { factx.t_start = HAP_perf_get_qtimer_count(); - factx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); - factx.src0_div1 = init_fastdiv_values(q->ne[1]); + factx.src0_div21 = kparams->u.hvx.src0_div21; + factx.src0_div1 = kparams->u.hvx.src0_div1; - factx.broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]); - factx.broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]); - factx.broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]); - factx.broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]); + factx.broadcast_rk2 = kparams->broadcast_rk2; + factx.broadcast_rk3 = kparams->broadcast_rk3; + factx.broadcast_rv2 = kparams->broadcast_rv2; + factx.broadcast_rv3 = kparams->broadcast_rv3; if (mask) { - factx.src3_div2 = init_fastdiv_values(mask->ne[2]); - factx.src3_div3 = init_fastdiv_values(mask->ne[3]); + factx.src3_div2 = kparams->src3_div2; + factx.src3_div3 = kparams->src3_div3; } - factx.is_q_fp32 = (q->type == HTP_TYPE_F32); - factx.size_q_row_padded = hex_round_up(q->ne[0] * (factx.is_q_fp32 ? 4 : 2), 128); - factx.size_k_row_padded = hex_round_up(k->ne[0] * sizeof(__fp16), 128); - factx.size_v_row_padded = hex_round_up(v->ne[0] * sizeof(__fp16), 128); + factx.is_q_fp32 = (kparams->is_q_fp32 != 0); + factx.size_q_row_padded = kparams->u.hvx.size_q_row_padded; + factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded; + factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded; size_t size_q_block = factx.size_q_row_padded * 1; // single row for now factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE; factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE; factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128); - factx.n_blocks = (k->ne[1] + FLASH_ATTN_BLOCK_SIZE - 1) / FLASH_ATTN_BLOCK_SIZE; - - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; - - memcpy(&scale, (float *) octx->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) octx->op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (float *) octx->op_params + 2, sizeof(float)); + factx.n_blocks = kparams->n_kv_blocks; - if (logit_softcap != 0.0f) { - scale /= logit_softcap; - } - - factx.scale = scale; - factx.max_bias = max_bias; - factx.logit_softcap = logit_softcap; + factx.scale = kparams->scale; + factx.max_bias = kparams->max_bias; + factx.logit_softcap = (__fp16) kparams->logit_softcap; - uint32_t n_head = q->ne[2]; - factx.n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); - factx.m0 = powf(2.0f, -(max_bias ) / factx.n_head_log2); - factx.m1 = powf(2.0f, -(max_bias / 2.0f) / factx.n_head_log2); + factx.n_head_log2 = kparams->n_head_log2; + factx.m0 = kparams->m0; + factx.m1 = kparams->m1; + const uint32_t n_head = q->ne[2]; if (n_head > 512) { return HTP_STATUS_NO_SUPPORT; } for (uint32_t h = 0; h < n_head; ++h) { - factx.slopes[h] = (max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f; + factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f); } // total rows in q - const uint32_t neq0 = q->ne[0]; - const uint32_t neq1 = q->ne[1]; - const uint32_t neq2 = q->ne[2]; - const uint32_t neq3 = q->ne[3]; - - factx.qrows = neq1*neq2*neq3; - factx.qrows_per_thread = (factx.qrows + octx->n_threads - 1) / octx->n_threads; + factx.qrows = kparams->qrows; + factx.qrows_per_thread = kparams->qrows_per_thread; size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32 - octx->src0_spad.size_per_thread = size_q_block * 1; - octx->src1_spad.size_per_thread = factx.size_k_block * 2; - octx->src2_spad.size_per_thread = factx.size_v_block * 2; - octx->src3_spad.size_per_thread = mask ? factx.size_m_block * DMA_CACHE_MAX_SIZE : 0; - octx->dst_spad.size_per_thread = size_vkq_acc; + factx.size_q_block = size_q_block; + factx.size_vkq_acc = size_vkq_acc; - octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * octx->n_threads; - octx->src2_spad.size = octx->src2_spad.size_per_thread * octx->n_threads; - octx->src3_spad.size = octx->src3_spad.size_per_thread * octx->n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads; + uint8_t * vtcm_cur = octx->ctx->vtcm_base; - size_t total_spad = octx->src0_spad.size + octx->src1_spad.size + octx->src2_spad.size + octx->src3_spad.size + octx->dst_spad.size; + factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads); + factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads); + factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads); + factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads); + factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads); - if (octx->ctx->vtcm_size < total_spad) { + if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->src2_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->src2_spad.src = NULL; - octx->src3_spad.data = octx->src2_spad.data + octx->src2_spad.size; octx->src3_spad.src = NULL; - octx->dst_spad.data = octx->src3_spad.data + octx->src3_spad.size; octx->dst_spad.src = NULL; - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads); } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h new file mode 100644 index 00000000000..16822f22bf6 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -0,0 +1,303 @@ +#ifndef HTP_FLASH_ATTN_OPS_H +#define HTP_FLASH_ATTN_OPS_H + +#include +#include +#include + +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Tile constants (mirrored from hmx-utils.h for use on host side if needed) +#define HTP_FA_HMX_TILE_SIZE 2048 +#define HMX_FP16_TILE_SIZE 2048 +#define HMX_FP16_TILE_N_ROWS 32 +#define HMX_FP16_TILE_N_COLS 32 +#define HMX_FP16_TILE_N_ELMS 1024 + +#define HVX_FA_DMA_CACHE_SIZE 128 +#define HMX_FA_DMA_CACHE_SIZE 4 + + +#define HTP_FA_M_INITIAL_VAL -10000.0f + +enum htp_fa_kernel_type { + HTP_FA_KERNEL_UNSUPPORTED = 0, + HTP_FA_KERNEL_HVX, + HTP_FA_KERNEL_HMX +}; + +struct htp_fa_kernel_params { + uint8_t kernel_type; // enum htp_fa_kernel_type + uint8_t is_q_fp32; // 1 = Q type is F32, 0 = F16 + uint8_t is_dst_fp32; // 1 = dst type is F32, 0 = F16 + uint8_t n_threads; // Number of threads to run + + // Common parameters + uint16_t Br; + uint16_t Bc; + uint16_t n_kv_blocks; // also HVX's n_blocks + uint16_t G; // GQA factor (n_heads / n_kv_heads) + + float scale; + float max_bias; + float logit_softcap; + uint32_t vtcm_size; + + uint32_t qrows; + uint32_t qrows_per_thread; + float m0; + float m1; + uint32_t n_head_log2; + + struct fastdiv_values src3_div2; + struct fastdiv_values src3_div3; + + struct fastdiv_values broadcast_rk2; + struct fastdiv_values broadcast_rk3; + struct fastdiv_values broadcast_rv2; + struct fastdiv_values broadcast_rv3; + + union { + struct { + uint32_t g_br; + uint32_t row_buf_stride; + uint32_t mask_buf_row_stride; + int32_t mask_broadcast; + int32_t pipeline; + struct fastdiv_values div_G; + } hmx; + struct { + uint32_t size_q_row_padded; + uint32_t size_k_row_padded; + uint32_t size_v_row_padded; + struct fastdiv_values src0_div21; + struct fastdiv_values src0_div1; + } hvx; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob"); +#endif + +// VTCM region layout for the HMX flash-attention kernel. +// +// Single source of truth for both the host (which needs the total size to pick a +// (Br, Bc) tiling that fits the VTCM budget) and the device (which needs the actual +// byte offsets to place each scratch buffer). Building the layout once and reading +// offsets/total from it makes host estimate and device allocation impossible to +// desync -- previously they were duplicated formulas in two files and drifted. +// +// All fields are byte offsets / byte sizes -- no HVX_Vector type is named here so the +// header stays host-includable. The device casts (base + off_*) to the proper type. +// An offset of 0 marks a region that is not allocated for this configuration (only +// off_v_tiles[1], which exists only when pipelining); the device sets such pointers NULL. +struct hmx_fa_vtcm_layout { + // Byte offsets from vtcm_base for each region. + size_t off_q_tiles; + size_t off_o_tiles[2]; + size_t off_k_fp16[2]; + size_t off_v_fp16[2]; + size_t off_k_tiles; + size_t off_v_tiles[2]; // [1] allocated only when pipeline, else 0 + size_t off_s_tiles; + size_t off_p_tiles; + size_t off_d_tiles; + size_t off_m_vec; + size_t off_l_vec; + size_t off_s_rowmax; + size_t off_p_rowsum; + size_t off_row_bufs; + size_t off_hmx_scales_id; + size_t off_hmx_scales_qk; + size_t off_mask_buf; + size_t off_slopes; + + // Region byte sizes reused by the device at runtime (not just for allocation). + size_t q_tile_bytes; + size_t o_tile_bytes; + size_t s_tile_bytes; // S and P tiles (same size) + size_t d_tile_bytes; + size_t m_line_bytes; // one mask row + size_t m_buf_slot_bytes; // one dma_cache slot = align_up(Br * m_line_bytes, 4096) + size_t col_vec_bytes; + + // Derived strides. + size_t row_buf_stride; // HVX vectors (128B) per row buffer + size_t mask_buf_row_stride; // __fp16 elements per row in the mask buffer + + bool pipeline; + size_t total_bytes; +}; + +// Build the VTCM layout. + +static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, + size_t gqa_factor, size_t DK, size_t DV, + size_t Br, size_t Bc, size_t n_threads, bool pipeline) { + const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); + const size_t q_tile_size = hex_align_up(g_br * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t o_tile_size = hex_align_up(g_br * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t k_tile_size = hex_align_up(Bc * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t v_tile_size = hex_align_up(Bc * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t s_tile_size = hex_align_up(g_br * Bc * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t d_tile_size = hex_align_up(g_br * g_br * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + + const size_t k_dma_size = hex_align_up(Bc * hex_round_up(DK * sizeof(__fp16), 128), 128); + const size_t v_dma_size = hex_align_up(Bc * hex_round_up(DV * sizeof(__fp16), 128), 128); + const size_t col_vec_size = hex_align_up(g_br * sizeof(float), 256); + const size_t row_vec_size = hex_align_up(Bc * sizeof(__fp16), 256); + const size_t m_line_size = hex_align_up(Bc * sizeof(__fp16), 128); + const size_t m_buf_slot = hex_align_up(Br * m_line_size, 256); + const size_t m_buf_size = m_buf_slot * HMX_FA_DMA_CACHE_SIZE; + const size_t slopes_size = hex_align_up(g_br * sizeof(__fp16), 128); + + size_t off = 0; + + // Section 1: HMX Tiled Buffers (FA_HMX_TILE_SIZE = 2KB Aligned) + VTCM_LAYOUT_ALLOC(off, off_q_tiles, q_tile_size); + VTCM_LAYOUT_ALLOC(off, off_o_tiles[0], o_tile_size); + VTCM_LAYOUT_ALLOC(off, off_o_tiles[1], o_tile_size); + VTCM_LAYOUT_ALLOC(off, off_k_tiles, k_tile_size); + VTCM_LAYOUT_ALLOC(off, off_v_tiles[0], v_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_v_tiles[1], v_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off, off_s_tiles, s_tile_size); + VTCM_LAYOUT_ALLOC(off, off_p_tiles, s_tile_size); + VTCM_LAYOUT_ALLOC(off, off_d_tiles, d_tile_size); + + // Section 2: HVX/DMA flat and vector buffers (128B / 256B Aligned) + VTCM_LAYOUT_ALLOC(off, off_k_fp16[0], k_dma_size); + VTCM_LAYOUT_ALLOC(off, off_k_fp16[1], k_dma_size); + VTCM_LAYOUT_ALLOC(off, off_v_fp16[0], v_dma_size); + VTCM_LAYOUT_ALLOC(off, off_v_fp16[1], v_dma_size); + VTCM_LAYOUT_ALLOC(off, off_m_vec, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_l_vec, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_s_rowmax, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_p_rowsum, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_row_bufs, row_vec_size * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off, off_hmx_scales_id, 256); + VTCM_LAYOUT_ALLOC(off, off_hmx_scales_qk, 256); + VTCM_LAYOUT_ALLOC(off, off_mask_buf, m_buf_size); + VTCM_LAYOUT_ALLOC(off, off_slopes, slopes_size); + + L->q_tile_bytes = q_tile_size; + L->o_tile_bytes = o_tile_size; + L->col_vec_bytes = col_vec_size; + L->s_tile_bytes = s_tile_size; + L->d_tile_bytes = d_tile_size; + L->m_line_bytes = m_line_size; + L->m_buf_slot_bytes = m_buf_slot; + L->row_buf_stride = row_vec_size / 128; + L->mask_buf_row_stride = m_line_size / sizeof(__fp16); + L->pipeline = pipeline; + L->total_bytes = off; +} + +// Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline) { + struct hmx_fa_vtcm_layout L; + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline); + return L.total_bytes; +} + +#define FA_HVX_BLOCK_SIZE 64 + +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, size_t n_threads) { + const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); + const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); + const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); + + const size_t size_q_block = size_q_row_padded * 1; + const size_t size_k_block = size_k_row_padded * FA_HVX_BLOCK_SIZE; + const size_t size_v_block = size_v_row_padded * FA_HVX_BLOCK_SIZE; + const size_t size_m_block = hex_round_up(FA_HVX_BLOCK_SIZE * sizeof(__fp16), 128); + const size_t size_vkq_acc = hex_round_up(DV * sizeof(float), 128); + + const size_t size_per_thread = size_q_block * 1 + + size_k_block * 2 + + size_v_block * 2 + + (has_mask ? size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) + + size_vkq_acc; + + return size_per_thread * n_threads; +} + +#define FA_MIN_KV_BLOCKS 3 + +// Cost-based (Br, Bc) search for flash attention with pipeline constraint. +static inline int hmx_fa_find_chunk_size(size_t * Br_out, + size_t * Bc_out, + size_t gqa_factor, + size_t DK, + size_t DV, + size_t qo_len, + size_t kv_len, + size_t vtcm_budget, + size_t n_threads) { + const size_t T = HMX_FP16_TILE_N_ROWS; // 32 + const size_t br_unit = hmx_ceil_div(T, gqa_factor); + const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 + const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); + + // Br_max: largest Br aligned to br_unit that does not exceed qo_len. + const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; + + // Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS. + // Only relax when kv_len is too short to form enough blocks. + const size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) : + (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit); + // Cost coefficients calibrated from profiling + const size_t c_q_fixed = 1400; // per-Q-block: q_load + epilogue o_update + o_norm + o_store + const size_t c_iter_fixed = 200; // per-KV-iter: HMX queue push/pop + DMA pop + barriers + + size_t best_cost = SIZE_MAX, best_mn = 0; + size_t best_Br = 0, best_Bc = 0; + + for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { + // Try all Bc candidates from Bc_limit down to bc_unit + for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline); + if (vtcm_needed <= vtcm_budget) { + // This Bc fits for this Br! + const size_t q_blocks = (qo_len + Br - 1) / Br; + const size_t kv_blocks = (kv_len + Bc - 1) / Bc; + const size_t cost = q_blocks * (c_q_fixed + kv_blocks * c_iter_fixed); + const size_t mn = Br * Bc; + + if (cost < best_cost || (cost == best_cost && mn > best_mn)) { + best_cost = cost; + best_mn = mn; + best_Br = Br; + best_Bc = Bc; + } + // Since we iterate Bc from largest to smallest, this is the largest Bc that fits + // for this Br. We can break to the next Br. + break; + } + } + + if (Br == br_unit) { + break; + } + } + + if (best_Br == 0 || best_Bc == 0) { + return -1; + } + + *Br_out = best_Br; + *Bc_out = best_Bc; + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* HTP_FLASH_ATTN_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.h b/ggml/src/ggml-hexagon/htp/hex-dma.h index 8031a5679c4..98fcc9fda63 100644 --- a/ggml/src/ggml-hexagon/htp/hex-dma.h +++ b/ggml/src/ggml-hexagon/htp/hex-dma.h @@ -138,27 +138,28 @@ static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t } dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx]; - desc->next = NULL; - desc->desc_size = 0; // 1D mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->order = 0; - desc->done = 0; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->size = size; + desc->src = (void *) dptr.src; + desc->dst = (void *) dptr.dst; + desc->size = size; q->dptr[q->push_idx] = dptr; if (size) { + desc->next = NULL; + desc->desc_size = 0; // 1D mode + desc->src_bypass = dma_src_l2_bypass_on; + desc->dst_bypass = dma_dst_l2_bypass_on; + desc->order = 0; + desc->done = 0; + htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx); dmlink(q->tail, desc); q->tail = (dma_descriptor_2d *) desc; } else { - desc->done = 1; + desc->desc_size = 0; + desc->done = 1; } - // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src); q->push_idx = (q->push_idx + 1) & q->idx_mask; return true; } @@ -320,7 +321,7 @@ static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_ return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); } -#define DMA_CACHE_MAX_SIZE 64U +#define DMA_CACHE_MAX_SIZE 256U typedef struct { uint8_t *base; @@ -352,20 +353,19 @@ static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * sr if (c->src[i] == (uint32_t) src) { c->age[i] = 0; dst = c->base + (i * c->line_size); nrows = 0; // dummy dma - // FARF(ERROR, "dma-cache: found %p", src); } else { c->age[i]++; if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } } } if (!dst) { - // FARF(ERROR, "dma-cache: replacing #%u : age %u %p -> %p", o_idx, c->age[o_idx], (void *) c->src[o_idx], src); c->age[o_idx] = 0; c->src[o_idx] = (uint32_t) src; dst = c->base + o_idx * c->line_size; // normal nrows dma + return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); } - return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); + return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0); } #ifdef __cplusplus diff --git a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h new file mode 100644 index 00000000000..d6795bf0b57 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h @@ -0,0 +1,555 @@ +#ifndef HMX_FA_KERNELS_H +#define HMX_FA_KERNELS_H + +#include +#include +#include +#include "hvx-utils.h" +#include "hmx-utils.h" +#include "hex-fastdiv.h" + +// HMX-specific parameters, offsets and inner kernels for Flash Attention + +// Scatter offsets for diagonal tile: entry[2i] = i*136, entry[2i+1] = i*136+6 +// 136 = 4 * 32 + 8 = byte offset to diagonal in a 32x32 fp16 interleaved tile +static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = { + 0 * 136, 0 * 136 + 6, + 1 * 136, 1 * 136 + 6, + 2 * 136, 2 * 136 + 6, + 3 * 136, 3 * 136 + 6, + 4 * 136, 4 * 136 + 6, + 5 * 136, 5 * 136 + 6, + 6 * 136, 6 * 136 + 6, + 7 * 136, 7 * 136 + 6, + 8 * 136, 8 * 136 + 6, + 9 * 136, 9 * 136 + 6, + 10 * 136, 10 * 136 + 6, + 11 * 136, 11 * 136 + 6, + 12 * 136, 12 * 136 + 6, + 13 * 136, 13 * 136 + 6, + 14 * 136, 14 * 136 + 6, + 15 * 136, 15 * 136 + 6, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, +}; +// Inner HMX tile computation kernels + +static void hmx_fa_qk_dot_tile( + const __fp16 * row_tiles, + const __fp16 * col_tiles, + __fp16 * out_tile, + size_t n_dot_tiles +) { + if (n_dot_tiles == 2) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_dot_tiles == 4) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_dot_tiles == 8) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + HMX_LOAD_MPY_F16("%9", "%10", "%0") + HMX_LOAD_MPY_F16("%11", "%12", "%0") + HMX_LOAD_MPY_F16("%13", "%14", "%0") + HMX_LOAD_MPY_F16("%15", "%16", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 4 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 4 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 5 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 5 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 6 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 6 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 7 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 7 * HMX_FP16_TILE_N_ELMS) + ); + } else { + for (size_t k = 0; k < n_dot_tiles; ++k) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(row_tiles), "r"(col_tiles) + ); + row_tiles += HMX_FP16_TILE_N_ELMS; + col_tiles += HMX_FP16_TILE_N_ELMS; + } + } + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(out_tile), "r"(0) + : "memory" + ); +} + +static void hmx_fa_o_update_tile( + const __fp16 * d_diag, + const __fp16 * o_rc, + const __fp16 * p_tile_in, + const __fp16 * v_tile_in, + __fp16 * o_tile_out, + size_t n_col_tiles +) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(d_diag), "r"(o_rc) + ); + if (n_col_tiles == 2) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_col_tiles == 4) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_col_tiles == 8) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + HMX_LOAD_MPY_F16("%9", "%10", "%0") + HMX_LOAD_MPY_F16("%11", "%12", "%0") + HMX_LOAD_MPY_F16("%13", "%14", "%0") + HMX_LOAD_MPY_F16("%15", "%16", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 4 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 4 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 5 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 5 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 6 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 6 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 7 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 7 * HMX_FP16_TILE_N_ELMS) + ); + } else { + for (size_t k = 0; k < n_col_tiles; ++k) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(p_tile_in), "r"(v_tile_in) + ); + p_tile_in += HMX_FP16_TILE_N_ELMS; + v_tile_in += HMX_FP16_TILE_N_ELMS; + } + } + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(o_tile_out), "r"(0) + : "memory" + ); +} + +static inline void hmx_fa_o_norm_tile( + const __fp16 * d_diag, + const __fp16 * o_rc, + __fp16 * o_out +) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(d_diag), "r"(o_rc) + ); + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(o_out), "r"(0) + : "memory" + ); +} + +static inline void hmx_fa_q_prep_fp32_d2( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + ((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + ((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + { + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + { + HVX_Vector v0 = pv_in0[1]; + HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp32_d4( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < 4; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + for (uint32_t d = 0; d < 4; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp32( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < d_limit; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + for (uint32_t d = 0; d < d_limit; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + out_tile[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp16_d1( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } +} + +static inline void hmx_fa_q_prep_fp16_d2( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < 2; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + { + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + { + HVX_Vector v0 = pv_in0[1]; + HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 1 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + +static inline void hmx_fa_q_prep_fp16( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < d_limit; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + for (uint32_t d = 0; d < d_limit; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + + +static inline void hmx_fa_q_prep_fallback( + __fp16 * vtcm_q_tiles, uintptr_t q_data, + size_t q_nb1, size_t q_nb2, size_t q_nb3, + uint32_t q_start, uint32_t kv_head, uint32_t ib3, + size_t start, size_t end, size_t n_rows_g, + size_t G, size_t DK, bool is_q_fp32, + const struct fastdiv_values * div_G +) { + for (size_t r = start; r < end; r += 2) { + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const uint8_t * q_ptr0 = (r + 0 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx0) * q_nb1 + + (kv_head * G + h_idx0) * q_nb2 + ib3 * q_nb3) : + NULL; + const uint8_t * q_ptr1 = (r + 1 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx1) * q_nb1 + + (kv_head * G + h_idx1) * q_nb2 + ib3 * q_nb3) : + NULL; + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (is_q_fp32) { + const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; + const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; + + for (uint32_t d = 0; d < DK / 32; ++d) { + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + out_tile[r1 / 2] = v_hf; + } + } else { + const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; + const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; + + for (uint32_t d = 0; d < DK / 64; ++d) { + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } + } +} + +#endif /* HMX_FA_KERNELS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c deleted file mode 100644 index 996fd597570..00000000000 --- a/ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c +++ /dev/null @@ -1,1838 +0,0 @@ -// HMX-accelerated Flash Attention for prefill (neq1 >= 32). -// Ported from htp-ops-lib/src/dsp/ops/flash_attn.c, adapted to the htp/ codebase. - -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic ignored "-Wunused-function" -#pragma clang diagnostic ignored "-Wunused-but-set-variable" - -#include -#include -#include -#include -#include -#include -#include -#include - -#define GGML_COMMON_DECL_C -#include "ggml-common.h" -#include "hex-dma.h" -#include "hex-fastdiv.h" -#include "hex-profile.h" -#include "hmx-queue.h" -#include "hmx-utils.h" -#include "htp-ctx.h" -#include "htp-ops.h" -#include "hvx-dump.h" -#include "hvx-copy.h" -#include "hvx-reduce.h" -#include "hvx-utils.h" -#include "hvx-flash-attn.h" -#include "vtcm-utils.h" -#include "worker-pool.h" - -// ============================================================================ -// Constants -// ============================================================================ - -// Tile constants from hmx-utils.h -// HMX_FP16_TILE_N_ROWS = 32 -// HMX_FP16_TILE_N_COLS = 32 -// HMX_FP16_TILE_N_ELMS = 1024 -// HMX_FP16_TILE_SIZE = 2048 - -// ============================================================================ -// Dynamic block size computation (GQA-aware) -// ============================================================================ - -// Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. -// g_br = hex_align_up(gqa_factor * Br, 32) replaces Br for all Q/O/S/P/D dimensions. -// Layout: Q + O_ping + O_pong + K_dma*2 + V_dma*2 + K_tile + V_tile + S + P + D + vectors + scales -// Mask is DMA'd into a VTCM buffer (Br rows per KV block) to avoid DDR reads in softmax. -static size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline) { - const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); - const size_t q_tile_size = hex_align_up(g_br * DK * sizeof(__fp16), 4096); // Q: [g_br, DK] - const size_t o_tile_size = hex_align_up(g_br * DV * sizeof(__fp16), 4096); // O: [g_br, DV] x2 ping-pong - const size_t k_dma_size = hex_align_up(Bc * hex_round_up(DK * sizeof(__fp16), 128), 4096); // K DMA: [Bc, DK] x2 double-buf - const size_t v_dma_size = hex_align_up(Bc * hex_round_up(DV * sizeof(__fp16), 128), 4096); // V DMA: [Bc, DV] x2 double-buf - const size_t k_tile_size = hex_align_up(Bc * DK * sizeof(__fp16), 4096); // K tiles: [Bc, DK] interleaved - const size_t v_tile_size = hex_align_up(Bc * DV * sizeof(__fp16), 4096); // V tiles: [Bc, DV] interleaved - const size_t s_tile_size = hex_align_up(g_br * Bc * sizeof(__fp16), 4096); // S/P:[g_br, Bc] - const size_t d_tile_size = hex_align_up(g_br * g_br * sizeof(__fp16), 4096); // D: [g_br, g_br] - const size_t col_vec_size = hex_align_up(g_br * sizeof(__fp16), 256); // m, l, etc. - const size_t row_vec_size = hex_align_up(Bc * sizeof(__fp16), 256); - const size_t m_line_size = hex_align_up(Bc * sizeof(__fp16), 128); - const size_t m_buf_size = hex_align_up(Br * m_line_size, 4096); - const size_t slopes_size = hex_align_up(g_br * sizeof(__fp16), 128); - - return q_tile_size * 1 // Q tiles - + o_tile_size * 2 // O ping-pong - + k_dma_size * 2 // K DMA x2 - + v_dma_size * 2 // V DMA x2 - + k_tile_size * 1 // K tiles - + v_tile_size * (pipeline ? 2 : 1) // V tiles (double-buffered if pipelining) - + s_tile_size * 2 // S + P - + d_tile_size * 1 // D (diagonal matrix) - + col_vec_size * 4 // m_vec, l_vec, s_rowmax, p_rowsum - + row_vec_size * 2 * n_threads // per-thread softmax row scratch - + m_buf_size * 1 // mask VTCM buffer [Br rows] - + slopes_size // Slopes - + 256 * 2; // HMX scales (id + qk) -} - -// ============================================================================ -// FP16 exp2 polynomial (ported from htp-ops-lib/include/dsp/hvx_math.h) -// ============================================================================ -// 5th-order Horner polynomial for exp2(x) in qf16/hf16 domain. Input must be -// ≤ 0 (safe softmax invariant — overflow handling omitted). ~18 ALU ops per -// 64 fp16 lanes, fully parallel across HVX threads (no scatter/gather engine). -// Replaces the F32 round-trip (qf16→f32→exp→f32→f16, ~44 ops for 2×32 lanes). -static inline HVX_Vector hvx_exp2_hf(HVX_Vector x_v) { - const HVX_Vector zero_v = Q6_V_vzero(); - const HVX_Vector half_hf_v = Q6_Vh_vsplat_R(0x3800); // fp16 0.5 - - // k = round_toward_neg_inf(x); f = (float)k; frac = x - f - HVX_Vector x_minus_half = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(x_v, half_hf_v)); - HVX_Vector k_v = Q6_Vh_equals_Vhf(x_minus_half); // truncate to int16 - HVX_Vector f_v = Q6_Vhf_equals_Vh(k_v); // back to fp16 - - HVX_Vector x_qf16 = Q6_Vqf16_vsub_VhfVhf(x_v, f_v); // fractional part in qf16 - - // Horner: y = ((((E5*x + E4)*x + E3)*x + E2)*x + E1)*x + E0 - HVX_Vector y = Q6_Vqf16_vmpy_Vqf16Vqf16(Q6_Vh_vsplat_R(0x5082), x_qf16); // E5*x - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x157d)); // + E4 - y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x20ed)); // + E3 - y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x2b1b)); // + E2 - y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x33b0)); // + E1 - y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x398c)); // + E0 - y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); // y = y * x - y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x3c00)); // + 1.0 - - // Combine polynomial (mantissa) with integer part (exponent): result = y * 2^k - y = Q6_Vhf_equals_Vqf16(y); - HVX_Vector y_exp = Q6_Vuh_vlsr_VuhR(Q6_Vh_vasl_VhR(y, 1), 11); - y_exp = Q6_Vh_vadd_VhVh(k_v, y_exp); - HVX_VectorPred q_underflow = Q6_Q_vcmp_gt_VhVh(zero_v, y_exp); - y = Q6_Vh_vaslacc_VhVhR(y, k_v, 10); - return Q6_V_vmux_QVV(q_underflow, zero_v, y); -} - -#define FA_MIN_KV_BLOCKS 3 - -// Cost-based (Br, Bc) search for flash attention with pipeline constraint. -// -// VTCM model (same as before): -// overhead + g_br * per_gbr + g_br² * per_gbr2 + Bc * per_bc + g_br * Bc * per_gbr_bc -// -// Cost model (minimization objective): -// Q * (c_q_fixed + K * c_iter_fixed), where Q = ceil(qo/Br), K = ceil(kv/Bc) -static int hmx_fa_find_chunk_size(size_t * Br_out, - size_t * Bc_out, - size_t gqa_factor, - size_t DK, - size_t DV, - size_t qo_len, - size_t kv_len, - size_t vtcm_budget, - size_t n_threads) { - const size_t T = HMX_FP16_TILE_N_ROWS; // 32 - const size_t br_unit = hmx_ceil_div(T, gqa_factor); - // Bc must be a multiple of 64 so that n_tiles_per_bc is even. The softmax - // P-tile write uses a dual-tile pattern (vshuff + two stores 16 slots apart) - // that would race across r0 blocks if the last dual-tile is half-occupied. - // See .cursor/todos/hmx-flash-attn-bc-search-space.md for the perf trade-off. - const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 - const size_t fp16 = sizeof(__fp16); - const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); - - // Approximate per-unit VTCM costs (without per-buffer alignment padding). - const size_t per_gbr = (DK + 2 * DV) * fp16 + 4 * fp16; // Q + O×2 + 4 col vectors - const size_t per_gbr2 = fp16; // D diagonal matrix - const size_t per_bc = - 3 * DK * fp16 + (can_pipeline ? 4 : 3) * DV * fp16 + 2 * n_threads * fp16; // K/V DMA x2 + tiles + row bufs - const size_t per_gbr_bc = 2 * fp16; // S + P - - const size_t overhead = 256 * 2 + 13 * 4096; - - if (vtcm_budget <= overhead) { - return -1; - } - const size_t usable = vtcm_budget - overhead; - - // Br_max: largest Br aligned to br_unit that does not exceed qo_len. - const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; - - // Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS. - // Only relax when kv_len is too short to form enough blocks. - const size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) : - (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit); - // Cost coefficients calibrated from profiling - const size_t c_q_fixed = 1400; // per-Q-block: q_load + epilogue o_update + o_norm + o_store - const size_t c_iter_fixed = 200; // per-KV-iter: HMX queue push/pop + DMA pop + barriers - - size_t best_cost = SIZE_MAX, best_mn = 0; - size_t best_Br = 0, best_Bc = 0; - - for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { - const size_t g_br = hex_align_up(gqa_factor * Br, T); - - // g_br-dependent VTCM cost: g_br * per_gbr + g_br² * per_gbr2 - const size_t gbr_cost = g_br * per_gbr + g_br * g_br * per_gbr2; - if (gbr_cost >= usable) { - if (Br == br_unit) { - break; - } - continue; - } - - // Analytically solve for max Bc: - // remain >= Bc * (per_bc + g_br * per_gbr_bc + Br * fp16_mask) - // The Br * fp16 term accounts for the VTCM mask buffer [Br × Bc]. - const size_t remain = usable - gbr_cost; - const size_t bc_denom = per_bc + g_br * per_gbr_bc + Br * fp16; - size_t Bc = hex_smin(hex_align_down(remain / bc_denom, bc_unit), Bc_limit); - if (Bc < bc_unit) { - if (Br == br_unit) { - break; - } - continue; - } - - // Exact VTCM verification (alignment padding may push over budget) - while (Bc >= bc_unit && hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline) > vtcm_budget) { - Bc -= bc_unit; - } - if (Bc < bc_unit) { - if (Br == br_unit) { - break; - } - continue; - } - - const size_t q_blocks = (qo_len + Br - 1) / Br; - const size_t kv_blocks = (kv_len + Bc - 1) / Bc; - const size_t cost = q_blocks * (c_q_fixed + kv_blocks * c_iter_fixed); - const size_t mn = Br * Bc; - - if (cost < best_cost || (cost == best_cost && mn > best_mn)) { - best_cost = cost; - best_mn = mn; - best_Br = Br; - best_Bc = Bc; - } - - if (Br == br_unit) { - break; - } - } - - if (best_Br == 0) { - return -1; - } - - *Br_out = best_Br; - *Bc_out = best_Bc; - return 0; -} - -// ============================================================================ -// Tile interleave / extract helpers -// ============================================================================ - -// transpose scatter offsets moved to hmx-utils.h as hmx_transpose_scatter_offsets - -// Scatter offsets for diagonal tile: entry[2i] = i*136, entry[2i+1] = i*136+6 -// 136 = 4 * 32 + 8 = byte offset to diagonal in a 32x32 fp16 interleaved tile -static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = { - 0 * 136, 0 * 136 + 6, - 1 * 136, 1 * 136 + 6, - 2 * 136, 2 * 136 + 6, - 3 * 136, 3 * 136 + 6, - 4 * 136, 4 * 136 + 6, - 5 * 136, 5 * 136 + 6, - 6 * 136, 6 * 136 + 6, - 7 * 136, 7 * 136 + 6, - 8 * 136, 8 * 136 + 6, - 9 * 136, 9 * 136 + 6, - 10 * 136, 10 * 136 + 6, - 11 * 136, 11 * 136 + 6, - 12 * 136, 12 * 136 + 6, - 13 * 136, 13 * 136 + 6, - 14 * 136, 14 * 136 + 6, - 15 * 136, 15 * 136 + 6, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, -}; - -// hmx_interleave_rows_to_tiles and hmx_interleave_cols_to_tiles are in hmx-utils.h - -// ============================================================================ -// HMX Flash Attention context (GQA-merged) -// ============================================================================ - -struct hmx_fa_context { - const struct htp_ops_context * octx; - bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 - uint32_t n_threads; - - // Op parameters - float scale; - float max_bias; - float logit_softcap; - uint32_t n_head_log2; - float m0, m1; - - // Dimensions - uint32_t DK, DV; - uint32_t n_kv; // kv_len - uint32_t n_kv_heads; // number of KV heads - uint32_t n_heads; // number of Q heads - uint32_t G; // GQA factor = n_heads / n_kv_heads - struct fastdiv_values div_G; - uint32_t n_kv_blocks; - uint32_t neq1; // Q token count - - // Types - bool is_q_fp32; - bool is_dst_fp32; - - // Dynamic block sizes - uint32_t Br; // Q tokens per block (before GQA expansion) - uint32_t Bc; - uint32_t g_br; // hex_align_up(G * Br, 32) - actual tile row dim - - // VTCM buffers (allocated by vtcm_seq_alloc) - __fp16 * vtcm_q_tiles; // Q tile format [g_br, D] - __fp16 * vtcm_o_tiles[2]; // O ping-pong [g_br, D] - __fp16 * vtcm_k_fp16[2]; // K DMA double-buffer [Bc, D] - __fp16 * vtcm_v_fp16[2]; // V DMA double-buffer [Bc, D] - __fp16 * vtcm_k_tiles; // K tiles (transposed) - __fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered) - __fp16 * vtcm_s_tiles; // S = QK^T [g_br, Bc] - __fp16 * vtcm_p_tiles; // P = softmax(S) [g_br, Bc] - __fp16 * vtcm_d_tiles; // Diagonal rescale [g_br, g_br] - HVX_Vector * vtcm_m_vec; // Row max [g_br] - HVX_Vector * vtcm_l_vec; // Row sum [g_br] - HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] - HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] - HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] - uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) - uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) - __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br × m_line], DMA'd per KV block - __fp16 * vtcm_slopes; // ALiBi slopes [g_br] - size_t row_buf_stride; // HVX vectors per row buffer (Bc/64) - size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer - bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA) -}; - -// ============================================================================ -// Multi-thread K interleave phase -// ============================================================================ - -typedef struct { - struct hmx_fa_context * factx; - int kv_rows; - size_t src_stride; - size_t buf_idx; -} fa_k_int_args_t; - -static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { - fa_k_int_args_t * args = (fa_k_int_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const int total_rows = args->kv_rows; - const int rows_per_t = hex_align_up(hmx_ceil_div(total_rows, n), 2); // ensure even (row pairs) - const int start = i * rows_per_t; - const int end = hex_smin(start + rows_per_t, total_rows); - - if (start >= total_rows) { - return; - } - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start); - hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, factx->vtcm_k_fp16[args->buf_idx], total_rows, (int) factx->DK, - (int) args->src_stride, start, end); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start); -} - -static void fa_phase_k_interleave(struct hmx_fa_context * factx, int kv_rows, size_t src_stride, size_t buf_idx) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - fa_k_int_args_t args = { factx, kv_rows, src_stride, buf_idx }; - if (factx->n_threads > 1 && kv_rows >= (int) (factx->n_threads * 2)) { - worker_pool_run_func(wp, fa_k_interleave_thread, &args, factx->n_threads); - } else { - fa_k_interleave_thread(1, 0, &args); - } -} - -// ============================================================================ -// Multi-thread V interleave phase -// ============================================================================ - -typedef struct { - struct hmx_fa_context * factx; - int kv_rows; - size_t src_stride; - size_t buf_idx; - size_t n_col_tiles; -} fa_v_int_args_t; - -static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { - fa_v_int_args_t * args = (fa_v_int_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const int total_rows = args->kv_rows; - const int rows_per_t = hex_align_up(hmx_ceil_div(total_rows, n), 2); - const int start = i * rows_per_t; - const int end = hex_smin(start + rows_per_t, total_rows); - - if (start >= total_rows) { - return; - } - - __fp16 * v_tiles_dest = factx->pipeline ? factx->vtcm_v_tiles[args->buf_idx] : factx->vtcm_v_tiles[0]; - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start); - hmx_interleave_cols_to_tiles(v_tiles_dest, factx->vtcm_v_fp16[args->buf_idx], total_rows, (int) factx->DV, - (int) args->src_stride, (int) args->n_col_tiles, start, end); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start); -} - -static void fa_phase_v_interleave(struct hmx_fa_context * factx, - int kv_rows, - size_t src_stride, - size_t buf_idx, - size_t n_col_tiles) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - fa_v_int_args_t args = { factx, kv_rows, src_stride, buf_idx, n_col_tiles }; - if (factx->n_threads > 1 && kv_rows >= (int) (factx->n_threads * 2)) { - worker_pool_run_func(wp, fa_v_interleave_thread, &args, factx->n_threads); - } else { - fa_v_interleave_thread(1, 0, &args); - } -} - -// ============================================================================ -// Multi-thread Q load phase: read Q[G × neq1, DK] from DDR, convert F32→F16 -// (or deal F16 pairs), and write interleaved into vtcm_q_tiles. -// Each thread owns a disjoint range of row pairs; writes target distinct tile -// slots (r0 selects tile row, r1 selects intra-tile slot), so there is no -// write conflict. Padding fill (when n_rows_g < g_br) is done single-threaded -// by the caller before dispatching. -// ============================================================================ - -typedef struct { - struct hmx_fa_context * factx; - const struct htp_tensor * q; - uint32_t q_start; - uint32_t kv_head; - uint32_t ib3; - size_t n_rows_g; -} fa_q_load_args_t; - -static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { - fa_q_load_args_t * args = (fa_q_load_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const size_t n_rows_g = args->n_rows_g; - const size_t G = factx->G; - const size_t DK = factx->DK; - - // Partition row pairs across threads. Keep each thread's start even so r/r+1 - // are always in the same thread's range. - const size_t rows_per_t = hex_align_up(hmx_ceil_div(n_rows_g, n), 2); - const size_t start = (size_t) i * rows_per_t; - const size_t end = hex_smin(start + rows_per_t, n_rows_g); - - if (start >= n_rows_g) { - return; - } - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start); - - const struct htp_tensor * q = args->q; - const uint32_t q_start = args->q_start; - const uint32_t kv_head = args->kv_head; - const uint32_t ib3 = args->ib3; - - for (size_t r = start; r < end; r += 2) { - const bool next_row_valid = (r + 1) < n_rows_g; - - const size_t q_idx0 = fastdiv(r + 0, &factx->div_G); - const size_t h_idx0 = fastmodulo(r + 0, G, &factx->div_G); - const size_t q_idx1 = fastdiv(r + 1, &factx->div_G); - const size_t h_idx1 = fastmodulo(r + 1, G, &factx->div_G); - - const uint8_t * q_ptr0 = (const uint8_t *) q->data + (q_start + q_idx0) * q->nb[1] + - (kv_head * G + h_idx0) * q->nb[2] + ib3 * q->nb[3]; - const uint8_t * q_ptr1 = next_row_valid ? ((const uint8_t *) q->data + (q_start + q_idx1) * q->nb[1] + - (kv_head * G + h_idx1) * q->nb[2] + ib3 * q->nb[3]) : - NULL; - - size_t r0 = r / HMX_FP16_TILE_N_ROWS; - size_t r1 = r % HMX_FP16_TILE_N_ROWS; - __fp16 * out_base = factx->vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; - - if (factx->is_q_fp32) { - const HVX_Vector * pv_in0 = (const HVX_Vector *) q_ptr0; - const HVX_Vector * pv_in1 = q_ptr1 ? (const HVX_Vector *) q_ptr1 : NULL; - - for (uint32_t d = 0; d < DK / 32; ++d) { - HVX_Vector v0 = pv_in0[d]; - HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); - HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); - - HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); - out_tile[r1 / 2] = v_hf; - } - } else { - const HVX_Vector * pv_in0 = (const HVX_Vector *) q_ptr0; - const HVX_Vector * pv_in1 = q_ptr1 ? (const HVX_Vector *) q_ptr1 : NULL; - - for (uint32_t d = 0; d < DK / 64; ++d) { - HVX_Vector v0 = pv_in0[d]; - HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); - HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); - - __fp16 * out_dual_tile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; - HVX_Vector * pv_out0 = ((HVX_Vector *) out_dual_tile) + r1 / 2; - HVX_Vector * pv_out1 = pv_out0 + 16; - - *pv_out0 = Q6_V_lo_W(vp); - *pv_out1 = Q6_V_hi_W(vp); - } - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start); -} - -static void fa_phase_q_load(struct hmx_fa_context * factx, - const struct htp_tensor * q, - uint32_t q_start, - uint32_t kv_head, - uint32_t ib3, - size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - fa_q_load_args_t args = { factx, q, q_start, kv_head, ib3, n_rows_g }; - // Require >= 2 row pairs per thread so partitioning is worthwhile. - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - worker_pool_run_func(wp, fa_q_load_thread, &args, factx->n_threads); - } else { - fa_q_load_thread(1, 0, &args); - } -} - -// ============================================================================ -// Multi-thread O store phase: read O tiles from VTCM, convert F16->F32 (or -// deal F16 pairs), and write to strided DDR dst tensor. Each thread owns a -// disjoint row range; writes target distinct dst rows (different q_idx/h_idx -// pairs produced by r/G and r%G), so there is no write conflict. -// ============================================================================ - -typedef struct { - struct hmx_fa_context * factx; - const struct htp_tensor * dst; - const __fp16 * o_tile_src; - uint32_t q_start; - uint32_t kv_head; - uint32_t ib3; - size_t n_rows_g; -} fa_o_store_args_t; - -static void fa_o_store_thread(unsigned int n, unsigned int i, void * data) { - fa_o_store_args_t * args = (fa_o_store_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const size_t n_rows_g = args->n_rows_g; - const size_t G = factx->G; - const size_t DV = factx->DV; - - const size_t rows_per_t = hmx_ceil_div(n_rows_g, n); - const size_t start = (size_t) i * rows_per_t; - const size_t end = hex_smin(start + rows_per_t, n_rows_g); - - if (start >= n_rows_g) { - return; - } - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start); - - const struct htp_tensor * dst = args->dst; - const __fp16 * o_tile_src = args->o_tile_src; - const uint32_t q_start = args->q_start; - const uint32_t kv_head = args->kv_head; - const uint32_t ib3 = args->ib3; - - for (size_t r = start; r < end; ++r) { - const size_t q_idx = fastdiv(r, &factx->div_G); - const size_t h_idx = fastmodulo(r, G, &factx->div_G); - - // FIX(dst-indexing): ggml_flash_attn_ext() creates dst as permute(0,2,1,3) -> - // [DV, n_heads, n_tokens, n_seq], so head stride is nb[1] and token stride is nb[2]. - uint8_t * dst_row = (uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] + - (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]; - - size_t r0 = r / HMX_FP16_TILE_N_ROWS; - size_t r1 = r % HMX_FP16_TILE_N_ROWS; - const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV; - - if (factx->is_dst_fp32) { - float * out = (float *) dst_row; - for (uint32_t d = 0; d < DV / 32; ++d) { - const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS); - HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]); - if (r1 % 2 == 0) { - *(HVX_UVector *) (out + d * 32) = Q6_V_lo_W(vp); - } else { - *(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp); - } - } - } else { - __fp16 * out = (__fp16 *) dst_row; - for (uint32_t d = 0; d < DV / 64; ++d) { - const __fp16 * in_dual_tile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2; - const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dual_tile) + r1 / 2; - const HVX_Vector * pv_in1 = pv_in0 + 16; - HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2); - if (r1 % 2 == 0) { - *(HVX_UVector *) (out + d * 64) = Q6_V_lo_W(vp); - } else { - *(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp); - } - } - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, start); -} - -static void fa_phase_o_store(struct hmx_fa_context * factx, - const struct htp_tensor * dst, - const __fp16 * o_tile_src, - uint32_t q_start, - uint32_t kv_head, - uint32_t ib3, - size_t n_rows_g) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g }; - if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { - worker_pool_run_func(wp, fa_o_store_thread, &args, factx->n_threads); - } else { - fa_o_store_thread(1, 0, &args); - } -} - -// ============================================================================ -// Multi-thread softmax phase + serial m/l update + build_D -// ============================================================================ - -typedef struct { - struct hmx_fa_context * factx; - size_t kv_rows; - size_t n_rows_g; - size_t n_col_tiles; - size_t n_tiles_per_bc; - size_t n_row_tiles; - size_t n_row_tiles_g_br; - uint32_t Bc; - uint32_t G; - uint32_t kv_head; - uint32_t kv_start; - uint32_t q_start; - uint32_t ib3; - bool has_alibi; // true when max_bias != 0 (need slope * mask + add) - - // ALiBi per-head slopes (indexed by GQA-merged row: slope[r] for r in [0, n_rows_g)) - // slope[r] = 1.0 when max_bias == 0 (no ALiBi) - // Pointer into hmx_fa_context.vtcm_slopes (sized to g_br) - __fp16 * slopes; - - // Mask info (preloaded before softmax) - const struct htp_tensor * mask; - const __fp16 * mask_vtcm; // VTCM mask buffer base (NULL = DDR fallback) - size_t mask_vtcm_row_stride; // elements (__fp16) per row in VTCM mask buffer -} fa_softmax_args_t; - -static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { - fa_softmax_args_t * args = (fa_softmax_args_t *) data; - struct hmx_fa_context * factx = args->factx; - - const size_t n_rows_g = args->n_rows_g; - const size_t kv_rows = args->kv_rows; - const size_t Bc = args->Bc; - const size_t G = args->G; - const size_t n_tiles_per_bc = args->n_tiles_per_bc; - const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64); - - // Partition r_vec_idx across threads - const size_t vecs_per_t = hmx_ceil_div(n_row_vec_cnt, n); - const size_t vec_start = i * vecs_per_t; - const size_t vec_end = hex_smin(vec_start + vecs_per_t, n_row_vec_cnt); - - if (vec_start >= n_row_vec_cnt) { - return; - } - - struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, vec_start); - - // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride - const size_t row_buf_stride = factx->row_buf_stride; - HVX_Vector * my_row_buf0 = factx->vtcm_row_bufs + i * 2 * row_buf_stride; - HVX_Vector * my_row_buf1 = my_row_buf0 + row_buf_stride; - - const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); - - // Per-row accumulators: each fp16 lane in a 64-lane vector holds one row's scalar. - // CONTRACT: lane bits must be IEEE fp16 (hf), never qf16 — qf16 uses a different - // bit layout, so a later hf-domain read would silently produce wrong values. - // Convert first via Q6_Vhf_equals_Vqf16(). For reference: vtcm_m_vec/vtcm_s_rowmax - // are hf; vtcm_l_vec is qf16 — don't mix them up. - - for (size_t r_vec_idx = vec_start; r_vec_idx < vec_end; ++r_vec_idx) { - HVX_Vector rowmax_acc_v = v_neg_inf; - HVX_Vector rowsum_acc_v = Q6_V_vzero(); - HVX_Vector m_prev_v = factx->vtcm_m_vec[r_vec_idx]; - - for (int r_vec_off = 0; r_vec_off < 64; r_vec_off += 2) { - int r = r_vec_idx * 64 + r_vec_off; - if (r >= (int) hex_align_up(n_rows_g, 2)) { - break; - } - - int r0 = r / HMX_FP16_TILE_N_ROWS; - int r1 = r % HMX_FP16_TILE_N_ROWS; - - const __fp16 * s_ld_base = factx->vtcm_s_tiles + r0 * HMX_FP16_TILE_N_ROWS * Bc; - __fp16 * p_st_base = factx->vtcm_p_tiles + r0 * HMX_FP16_TILE_N_ROWS * Bc; - - // Decode 2 rows from S tiles into per-thread row buffers - HVX_Vector * pv_row_buf0 = my_row_buf0; - HVX_Vector * pv_row_buf1 = my_row_buf1; - for (size_t c = 0; c < kv_rows; c += 64) { - const __fp16 * in_dual_tile = s_ld_base + (c / 64) * HMX_FP16_TILE_N_ELMS * 2; - const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dual_tile) + r1 / 2; - const HVX_Vector * pv_s_in1 = pv_s_in0 + 16; - - HVX_VectorPair vp_s_dual_row = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2); - *pv_row_buf0++ = Q6_V_lo_W(vp_s_dual_row); - *pv_row_buf1++ = Q6_V_hi_W(vp_s_dual_row); - } - - // Apply softcap if enabled (in F32 precision) - if (factx->logit_softcap != 0.0f) { - // When EXP2_HF is on, fold log2(e) into v_cap so the output lands in - // log2(e)-scaled space for the downstream exp2. log2(e) is kept OUT - // of qk_scale in this configuration (see scale setup) so tanh sees - // the physical QK/(√d·c) argument. - float cap = factx->logit_softcap; -#ifdef HMX_FA_USE_EXP2_HF - cap *= 1.44269504f; // log2(e) -#endif - const HVX_Vector v_cap = hvx_vec_splat_f32(cap); - for (size_t c = 0; c < kv_rows; c += 64) { - size_t ci = c / 64; - - HVX_VectorPair r0_f32 = hvx_vec_f16_to_f32(my_row_buf0[ci]); - HVX_Vector t0_lo = hvx_vec_tanh_f32(Q6_V_lo_W(r0_f32)); - HVX_Vector t0_hi = hvx_vec_tanh_f32(Q6_V_hi_W(r0_f32)); - t0_lo = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(t0_lo, v_cap)); - t0_hi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(t0_hi, v_cap)); - my_row_buf0[ci] = hvx_vec_f32_to_f16(t0_lo, t0_hi); - - HVX_VectorPair r1_f32 = hvx_vec_f16_to_f32(my_row_buf1[ci]); - HVX_Vector t1_lo = hvx_vec_tanh_f32(Q6_V_lo_W(r1_f32)); - HVX_Vector t1_hi = hvx_vec_tanh_f32(Q6_V_hi_W(r1_f32)); - t1_lo = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(t1_lo, v_cap)); - t1_hi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(t1_hi, v_cap)); - my_row_buf1[ci] = hvx_vec_f32_to_f16(t1_lo, t1_hi); - } - } - - // Apply mask & compute rowmax(S) - // - // Optimizations over baseline: - // A. No-ALiBi fast path: when max_bias==0 (slope≡1.0), skip the - // slope multiplication — still add mask (additive bias) but - // avoid the mul_f16_f16. Saves 2 ops/dual-row vs ALiBi path. - // B. GQA mask row dedup: G consecutive Q rows share one mask row - // (qi = r / G). Reuse mask vector when qi is unchanged between - // row0 and row1 (saves ~75% of VTCM loads for G=4). - - // ALiBi slopes — only needed when has_alibi (scheme A) - HVX_Vector v_slope0, v_slope1; - if (args->has_alibi) { - HVX_Vector v_s = hvx_vmemu(args->slopes + r); - v_slope0 = hvx_vec_repl_f16(v_s); - v_slope1 = (r + 1 < (int) n_rows_g) ? hvx_vec_repl_f16(Q6_V_vror_VR(v_s, 2)) : Q6_V_vzero(); - } - - const HVX_Vector v_threshold = Q6_Vh_vsplat_R(0xcc00); // fp16 -16.0 (hoisted outside for-c) - - HVX_Vector v_s_rowmax0 = v_neg_inf; - HVX_Vector v_s_rowmax1 = v_neg_inf; - for (size_t c = 0; c < kv_rows; c += 64) { - size_t ci = c / 64; - const size_t ne = hex_smin(kv_rows - c, 64); - HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16)); - - if (args->mask) { - HVX_Vector v_mask0, v_mask1; - - if (args->mask_vtcm) { - // Read mask from VTCM buffer (DMA'd per KV block). - // GQA dedup (scheme B): skip load when qi unchanged. - const size_t qi0 = fastdiv(r + 0, &factx->div_G); - v_mask0 = *(const HVX_UVector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c); - v_mask1 = v_neg_inf; - if (r + 1 < (int) n_rows_g) { - const size_t qi1 = fastdiv(r + 1, &factx->div_G); - if (qi1 == qi0) { - v_mask1 = v_mask0; // scheme B: reuse — same mask row - } else { - v_mask1 = *(const HVX_UVector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c); - } - } - } else { - // Fallback: read mask directly from DDR (when mask->ne[2] > 1). - const struct htp_tensor * mask = args->mask; - const size_t q_idx0 = args->q_start + fastdiv(r + 0, &factx->div_G); - const size_t h_idx0 = args->kv_head * G + fastmodulo(r + 0, G, &factx->div_G); - const uint32_t im2_0 = h_idx0 % mask->ne[2]; - const uint32_t im3_0 = args->ib3 % mask->ne[3]; - - const __fp16 * m0_ptr = (const __fp16 *) ((const uint8_t *) mask->data + q_idx0 * mask->nb[1] + - im2_0 * mask->nb[2] + im3_0 * mask->nb[3]) + args->kv_start + c; - v_mask0 = *(const HVX_UVector *) m0_ptr; - v_mask1 = v_neg_inf; - - if (r + 1 < (int) n_rows_g) { - const size_t q_idx1 = args->q_start + fastdiv(r + 1, &factx->div_G); - if (q_idx1 == q_idx0) { - // scheme B: same mask row in DDR path - v_mask1 = v_mask0; - } else { - const size_t h_idx1 = args->kv_head * G + fastmodulo(r + 1, G, &factx->div_G); - const uint32_t im2_1 = h_idx1 % mask->ne[2]; - const uint32_t im3_1 = args->ib3 % mask->ne[3]; - const __fp16 * m1_ptr = (const __fp16 *) ((const uint8_t *) mask->data + q_idx1 * mask->nb[1] + - im2_1 * mask->nb[2] + im3_1 * mask->nb[3]) + args->kv_start + c; - v_mask1 = *(const HVX_UVector *) m1_ptr; - } - } - } - - // Threshold: mask values below -16.0 are treated as -inf (causal mask). - HVX_VectorPred q_keep0 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask0, v_threshold), q_tail_keep); - HVX_VectorPred q_keep1 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask1, v_threshold), q_tail_keep); - - if (args->has_alibi) { - // ALiBi path: S += slope * mask (full mul + add) - HVX_Vector v_sm0 = hvx_vec_mul_f16_f16(v_mask0, v_slope0); - HVX_Vector v_sm1 = hvx_vec_mul_f16_f16(v_mask1, v_slope1); - my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_sm0), v_neg_inf); - my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_sm1), v_neg_inf); - } else { - // No-ALiBi fast path (scheme A): slope≡1.0, skip the mul - // but still add mask (additive positional bias). vmux - // clamps mask < -16 to -inf as a numerical safeguard. - my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_mask0), v_neg_inf); - my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_mask1), v_neg_inf); - } - } else { - if (ne < 64) { - my_row_buf0[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf0[ci], v_neg_inf); - my_row_buf1[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf1[ci], v_neg_inf); - } - } - - v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]); - v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]); - } - - v_s_rowmax0 = hvx_vec_reduce_max_f16(v_s_rowmax0); - v_s_rowmax1 = hvx_vec_reduce_max_f16(v_s_rowmax1); - - // Splat m_prev[r], m_prev[r+1] from the per-row accumulator. - // vror brings the target lane to lane 0, then vdelta replicates it - // across all lanes — stays in the vector domain (no store/reload). - HVX_Vector v_m_prev0 = hvx_vec_repl_f16(Q6_V_vror_VR(m_prev_v, r_vec_off * 2)); - HVX_Vector v_m_prev1 = hvx_vec_repl_f16(Q6_V_vror_VR(m_prev_v, (r_vec_off + 1) * 2)); - - // HVX max — both operands are splats, so result is splat of m_new. - HVX_Vector v_dup_m0 = Q6_Vhf_vmax_VhfVhf(v_m_prev0, v_s_rowmax0); - HVX_Vector v_dup_m1 = Q6_Vhf_vmax_VhfVhf(v_m_prev1, v_s_rowmax1); - - // Insert row r, r+1 rowmax into rowmax_acc_v via 2-byte-wide vmux. - // Byte ranges: lane0 = [r_vec_off*2 .. r_vec_off*2+1], lane1 shifted by 2. - // vsetq2 handles the n=128 corner case when r_vec_off reaches 62. - { - HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); - HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); - HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); - HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); - HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); - rowmax_acc_v = Q6_V_vmux_QVV(p_lane0, v_dup_m0, rowmax_acc_v); - rowmax_acc_v = Q6_V_vmux_QVV(p_lane1, v_dup_m1, rowmax_acc_v); - } - - // Compute P = exp(S - m_new), using HVX exp - const HVX_Vector v_zero = Q6_V_vzero(); - HVX_Vector v_p_rowsum0 = v_zero; - HVX_Vector v_p_rowsum1 = v_zero; - -#ifdef HMX_FA_USE_EXP2_HF - // FP16 exp2 polynomial path (matches htp-ops-lib flash_attn.c): - // P = exp2(S - m_new) - for (size_t c = 0; c < kv_rows; c += 64) { - size_t ci = c / 64; - HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0); - HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1); - - HVX_Vector v_p_row0_hf = hvx_exp2_hf(Q6_Vhf_equals_Vqf16(v_s_minus_m0)); - HVX_Vector v_p_row1_hf = hvx_exp2_hf(Q6_Vhf_equals_Vqf16(v_s_minus_m1)); -#else - // F32 exp path: qf16 → f32 → exp → f32 → f16. Higher precision, - for (size_t c = 0; c < kv_rows; c += 64) { - size_t ci = c / 64; - HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0); - HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1); - - HVX_VectorPair vp0 = hvx_vec_f16_to_f32_shuff(Q6_Vhf_equals_Vqf16(v_s_minus_m0)); - HVX_Vector p0_lo = hvx_vec_exp_f32(Q6_V_lo_W(vp0)); - HVX_Vector p0_hi = hvx_vec_exp_f32(Q6_V_hi_W(vp0)); - HVX_Vector v_p_row0_hf = hvx_vec_f32_to_f16_shuff(p0_lo, p0_hi); - - HVX_VectorPair vp1 = hvx_vec_f16_to_f32_shuff(Q6_Vhf_equals_Vqf16(v_s_minus_m1)); - HVX_Vector p1_lo = hvx_vec_exp_f32(Q6_V_lo_W(vp1)); - HVX_Vector p1_hi = hvx_vec_exp_f32(Q6_V_hi_W(vp1)); - HVX_Vector v_p_row1_hf = hvx_vec_f32_to_f16_shuff(p1_lo, p1_hi); -#endif - // Write P to tile format. Dual-tile pattern assumes Bc is a - // multiple of 64 (enforced by bc_unit=64 in hmx_fa_find_chunk_size), - // so both tile halves are always in the current r0 block. - __fp16 * out_dual_tile = p_st_base + (c / 64) * HMX_FP16_TILE_N_ELMS * 2; - HVX_Vector * pv_p_out0 = ((HVX_Vector *) out_dual_tile) + r1 / 2; - HVX_Vector * pv_p_out1 = pv_p_out0 + 16; - - HVX_VectorPair vp_p_dual = Q6_W_vshuff_VVR(v_p_row1_hf, v_p_row0_hf, -2); - *pv_p_out0 = Q6_V_lo_W(vp_p_dual); - *pv_p_out1 = Q6_V_hi_W(vp_p_dual); - - HVX_VectorPair vp_p0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf); - HVX_VectorPair vp_p1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf); - - v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0), Q6_V_hi_W(vp_p0))); - v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1), Q6_V_hi_W(vp_p1))); - } - - HVX_Vector rowsum0_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum0)); - HVX_Vector rowsum1_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum1)); - { - // Both inputs are f32 splats, so the f32->f16 output is an fp16 splat. - HVX_Vector rv0_v = hvx_vec_f32_to_f16(rowsum0_sf, rowsum0_sf); - HVX_Vector rv1_v = hvx_vec_f32_to_f16(rowsum1_sf, rowsum1_sf); - - HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); - HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); - HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); - HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); - HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); - rowsum_acc_v = Q6_V_vmux_QVV(p_lane0, rv0_v, rowsum_acc_v); - rowsum_acc_v = Q6_V_vmux_QVV(p_lane1, rv1_v, rowsum_acc_v); - } - } - - factx->vtcm_s_rowmax[r_vec_idx] = rowmax_acc_v; - factx->vtcm_p_rowsum[r_vec_idx] = rowsum_acc_v; - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, vec_start); -} - -// Serial m/l update + build_D. Must run after softmax barrier (s_rowmax written by all threads). -// -// noinline: function boundary acts as a hard compiler barrier so the (size_t)addr scatter -// intrinsics inside cannot be hoisted past the call site. Mirrors the structural protection -// matmul gets for free via worker_pool function-pointer dispatch. Without this, the compiler -// can reorder the scatter past the subsequent hmx_queue_push and the HMX-queue worker thread -// reads stale VTCM (PPL → ~vocab-size). -static __attribute__((noinline)) void fa_ml_update_and_build_d(struct hmx_fa_context * factx, - size_t n_rows_g, - size_t n_row_tiles, - size_t n_row_tiles_g_br) { - // Reuse s_rowmax buffer for exp(m_diff) — safe because softmax is fully complete - HVX_Vector * const mvec_exp_m_diff = factx->vtcm_s_rowmax; - - const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64); - for (size_t i = 0; i < n_row_vec_cnt; ++i) { - HVX_Vector v_m_prev = factx->vtcm_m_vec[i]; - HVX_Vector v_m_curr = Q6_Vhf_vmax_VhfVhf(v_m_prev, factx->vtcm_s_rowmax[i]); - HVX_Vector v_m_diff = Q6_Vqf16_vsub_VhfVhf(v_m_prev, v_m_curr); - -#ifdef HMX_FA_USE_EXP2_HF - // Base-2 path: must match P = exp2(S - m_new) in fa_softmax_thread. - HVX_Vector v_exp_m_diff = hvx_exp2_hf(Q6_Vhf_equals_Vqf16(v_m_diff)); -#else - HVX_VectorPair vp_diff = hvx_vec_f16_to_f32_shuff(Q6_Vhf_equals_Vqf16(v_m_diff)); - HVX_Vector exp_lo = hvx_vec_exp_f32(Q6_V_lo_W(vp_diff)); - HVX_Vector exp_hi = hvx_vec_exp_f32(Q6_V_hi_W(vp_diff)); - HVX_Vector v_exp_m_diff = hvx_vec_f32_to_f16_shuff(exp_lo, exp_hi); -#endif - - HVX_Vector v_l_curr = Q6_Vqf16_vmpy_Vqf16Vhf(factx->vtcm_l_vec[i], v_exp_m_diff); - v_l_curr = Q6_Vqf16_vadd_Vqf16Vhf(v_l_curr, factx->vtcm_p_rowsum[i]); - - factx->vtcm_m_vec[i] = v_m_curr; - factx->vtcm_l_vec[i] = v_l_curr; - mvec_exp_m_diff[i] = v_exp_m_diff; - } - - // Build diagonal tile D = diag(exp(m_diff)) - const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; - const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); - for (size_t i = 0; i < n_row_tiles; ++i) { - const HVX_Vector v_content = Q6_V_vror_VR(mvec_exp_m_diff[i / 2], (i % 2) * 64); - __fp16 * out_base = factx->vtcm_d_tiles + i * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); - // Compiler barrier — Q6_vscatter takes (size_t)addr; without this the - // compiler may not recognize the volatile read below as aliasing and - // could reorder it before the scatter, defeating the HW drain. - __asm__ __volatile__("" ::: "memory"); - // Per-tile drain: scatter regions are disjoint (stride > tile size), - // so a single drain at tile 0 does NOT retire later tiles' entries. - (void) *(volatile HVX_Vector *) out_base; - } -} - -// Build D = diag(1/l) tile for the final O = D @ O normalization. -// -// noinline: same rationale as fa_ml_update_and_build_d — keeps Q6_vscatter from -// being hoisted past the subsequent hmx_queue_push at the o_norm call site. -static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx, - size_t n_row_tiles, - size_t n_row_tiles_g_br) { - const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; - const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); - const HVX_Vector one = hvx_vec_splat_f32(1.0f); - - HVX_Vector v_content = Q6_V_vzero(); - for (size_t i = 0; i < n_row_tiles; ++i) { - if ((i % 2) == 0) { - HVX_Vector v_l_hf = Q6_Vhf_equals_Vqf16(factx->vtcm_l_vec[i / 2]); - HVX_VectorPair vp_l = hvx_vec_f16_to_f32_shuff(v_l_hf); - HVX_Vector inv_lo = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(one, hvx_vec_inverse_f32(Q6_V_lo_W(vp_l)))); - HVX_Vector inv_hi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(one, hvx_vec_inverse_f32(Q6_V_hi_W(vp_l)))); - v_content = hvx_vec_f32_to_f16_shuff(inv_lo, inv_hi); - } else { - v_content = Q6_V_vror_VR(v_content, 64); - } - - __fp16 * out_base = factx->vtcm_d_tiles + i * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); - // Compiler barrier — see fa_ml_update_and_build_d for rationale. - __asm__ __volatile__("" ::: "memory"); - (void) *(volatile HVX_Vector *) out_base; - } -} - -// Combined: multi-thread softmax -> barrier -> serial m/l update + build_D -static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, - fa_softmax_args_t * sargs, - size_t n_row_tiles, - size_t n_row_tiles_g_br) { - worker_pool_context_t wp = factx->octx->ctx->worker_pool; - const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); - - if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { - uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); - worker_pool_run_func(wp, fa_softmax_thread, sargs, n_use); - } else { - fa_softmax_thread(1, 0, sargs); - } - // barrier implicit in worker_pool_run_func return - - fa_ml_update_and_build_d(factx, sargs->n_rows_g, n_row_tiles, n_row_tiles_g_br); -} - -// ============================================================================ -// HMX job structs and worker functions -// ============================================================================ - -typedef struct { - const __fp16 * q_tiles; - const __fp16 * k_tiles; - __fp16 * s_tiles; - size_t n_row_tiles; - size_t n_col_tiles; - size_t n_dot_tiles; // DK / 32 - size_t n_tiles_per_bc; - uint8_t * hmx_scales; -} hmx_fa_qk_job_t; - -static void hmx_fa_qk_dot_worker(void * data) { - hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_col_tiles = job->n_col_tiles; - const size_t n_dot_tiles = job->n_dot_tiles; - const size_t n_tiles_per_bc = job->n_tiles_per_bc; - const __fp16 * restrict q_tiles = job->q_tiles; - const __fp16 * restrict k_tiles = job->k_tiles; - __fp16 * restrict s_tiles = job->s_tiles; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < n_col_tiles; ++c) { - const __fp16 * row_tiles = q_tiles + r * HMX_FP16_TILE_N_ROWS * n_dot_tiles * HMX_FP16_TILE_N_COLS; - const __fp16 * col_tiles = k_tiles + c * HMX_FP16_TILE_N_COLS * n_dot_tiles * HMX_FP16_TILE_N_COLS; - __fp16 * out_tile = s_tiles + (r * n_tiles_per_bc + c) * HMX_FP16_TILE_N_ELMS; - - for (size_t k = 0; k < n_dot_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int) row_tiles, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) col_tiles, 2047); - row_tiles += HMX_FP16_TILE_N_ELMS; - col_tiles += HMX_FP16_TILE_N_ELMS; - } - Q6_mxmem_AR_after_hf(out_tile, 0); - } - } -} - -typedef struct { - __fp16 * o_curr; - const __fp16 * o_prev; - const __fp16 * p_tiles; - const __fp16 * v_tiles; - const __fp16 * d_tiles; - uint8_t * hmx_scales; - size_t n_row_tiles; - size_t n_col_tiles; - size_t n_row_tiles_g_br; - size_t n_tiles_per_bc; - size_t DV; -} hmx_fa_o_update_job_t; - -static void hmx_fa_o_update_worker(void * data) { - hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_col_tiles = job->n_col_tiles; - const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; - const size_t n_tiles_per_bc = job->n_tiles_per_bc; - const size_t DV_tiles = job->DV / 32; - const __fp16 * restrict d_tiles = job->d_tiles; - const __fp16 * restrict p_tiles = job->p_tiles; - const __fp16 * restrict v_tiles = job->v_tiles; - const __fp16 * restrict o_prev = job->o_prev; - __fp16 * restrict o_curr = job->o_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(DV_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - // D[r,r] @ O_prev[r,c] — only the diagonal tile - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = o_prev + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - Q6_activation_hf_mxmem_RR((unsigned int) d_diag, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) o_rc, 2047); - - // P @ V (accumulate on same accumulator) - const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - const __fp16 * v_tile_in = v_tiles + (c * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - for (size_t k = 0; k < n_col_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int) p_tile_in, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) v_tile_in, 2047); - p_tile_in += HMX_FP16_TILE_N_ELMS; - v_tile_in += HMX_FP16_TILE_N_ELMS; - } - - __fp16 * o_tile_out = o_curr + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - Q6_mxmem_AR_after_hf(o_tile_out, 0); - } - } -} - -typedef struct { - __fp16 * o_curr; // output (row-major tile layout) - const __fp16 * o_prev; // input (column-major tile layout) - const __fp16 * d_tiles; // diag(1/l) tiles - uint8_t * hmx_scales; - size_t n_row_tiles; - size_t n_row_tiles_g_br; - size_t DV; -} hmx_fa_o_norm_job_t; - -static void hmx_fa_o_norm_worker(void * data) { - hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; - const size_t n_row_tiles = job->n_row_tiles; - const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; - const size_t DV_tiles = job->DV / 32; - const __fp16 * restrict d_tiles = job->d_tiles; - const __fp16 * restrict o_prev = job->o_prev; - __fp16 * restrict o_curr = job->o_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(DV_tiles > 0); - - Q6_bias_mxmem2_A((void *) job->hmx_scales); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_tiles + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = o_prev + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_out = o_curr + (r * DV_tiles + c) * HMX_FP16_TILE_N_ELMS; - - Q6_activation_hf_mxmem_RR((unsigned int) d_diag, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) o_rc, 2047); - Q6_mxmem_AR_after_hf(o_out, 0); - } - } -} - -// Populate per-GQA-row ALiBi slopes for a given KV head. -// Row r in the GQA-merged block maps to Q head h = kv_head * G + r % G. -// slope(h) = m0^(h+1) when h < n_head_log2, else m1^(2*(h-n_head_log2)+1). -// When max_bias == 0, all slopes are 1.0 (no ALiBi). -static __attribute__((noinline)) void fa_compute_slopes( - const struct hmx_fa_context * factx, - uint32_t kv_head, - size_t n_rows_g) { - __fp16 * slopes = factx->vtcm_slopes; - if (factx->max_bias == 0.0f) { - hvx_splat_f16_a(slopes, 1.0f, n_rows_g); - return; - } - - const uint32_t G = factx->G; - const uint32_t n_head_log2 = factx->n_head_log2; - const float m0 = factx->m0; - const float m1 = factx->m1; - - __fp16 temp_slopes[512] __attribute__((aligned(128))); - if (G <= 32) { - // Fast path: Compute G unique slope values in vector registers - HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1); - - __fp16 temp_slopes_aligned[64] __attribute__((aligned(128))); - hvx_vmem(temp_slopes_aligned) = hvx_vec_f32_to_f16(v_val, Q6_V_vzero()); - - for (uint32_t i = 0; i < G; ++i) { - temp_slopes[i] = temp_slopes_aligned[i]; - } - } else { - // Fallback path: G > 32 (rare configurations) - for (uint32_t i = 0; i < G; ++i) { - temp_slopes[i] = (__fp16)alibi_slope(kv_head * G + i, n_head_log2, m0, m1); - } - } - - // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses) - __fp16 local_slopes[n_rows_g] __attribute__((aligned(128))); - for (size_t r = 0; r < n_rows_g; ++r) { - local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)]; - } - - // Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes) - hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g); -} - -// ============================================================================ -// Core HMX flash attention algorithm (GQA-merged) -// ============================================================================ - -int hmx_flash_attn_ext(struct htp_ops_context * octx) { - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL; - const struct htp_tensor * q = octx->src[0]; - const struct htp_tensor * k = octx->src[1]; - const struct htp_tensor * v = octx->src[2]; - const struct htp_tensor * mask = (octx->src[3] && octx->src[3]->data) ? octx->src[3] : NULL; - const struct htp_tensor * dst = octx->dst; - - struct htp_context * const ctx = octx->ctx; - - if (!ctx->hmx_enabled) { - return HTP_STATUS_NO_SUPPORT; - } - - // Dimensions - const uint32_t neq0 = q->ne[0]; // head_dim (DK) - const uint32_t neq1 = q->ne[1]; // n_tokens - const uint32_t neq2 = q->ne[2]; // n_heads - const uint32_t neq3 = q->ne[3]; // n_seqs - - const uint32_t nek0 = k->ne[0]; // head_dim - const uint32_t nek1 = k->ne[1]; // kv_len - - const uint32_t nev0 = v->ne[0]; // head_dim (DV) - - const uint32_t DK = neq0; - const uint32_t DV = nev0; - - // HMX requires head_dim to be multiple of 32 - if (DK % 32 != 0 || DV % 32 != 0) { - return HTP_STATUS_NO_SUPPORT; - } - - // GQA factor - const uint32_t n_kv_heads = k->ne[2]; - const uint32_t G = neq2 / n_kv_heads; - - // Thread count for multi-thread HVX phases - const uint32_t n_threads_init = octx->n_threads; - - // Compute dynamic block sizes (GQA-aware, accounting for per-thread row bufs) - size_t Br, Bc; - const size_t vtcm_budget = ctx->vtcm_size; - if (hmx_fa_find_chunk_size(&Br, &Bc, G, DK, DV, neq1, nek1, vtcm_budget, n_threads_init) != 0) { - return HTP_STATUS_VTCM_TOO_SMALL; - } - - const size_t g_br = hex_align_up(G * Br, HMX_FP16_TILE_N_ROWS); - - const uint32_t n_kv_blocks = (nek1 + Bc - 1) / Bc; - const bool pipeline = (n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads_init >= 2); - - // Bypass thread pool dispatch for small prompts/non-pipelined prefill by setting n_threads = 1 - const uint32_t n_threads = pipeline ? n_threads_init : 1; - - FARF(HIGH, "hmx-fa: neq1=%u nek1=%u DK=%u DV=%u G=%u Br=%zu Bc=%zu g_br=%zu n_kv_blocks=%u pipeline=%d vtcm=%zu", - neq1, nek1, DK, DV, G, Br, Bc, g_br, n_kv_blocks, pipeline, vtcm_budget); - - // ======== Build context ======== - struct hmx_fa_context factx; - memset(&factx, 0, sizeof(factx)); - factx.octx = octx; - factx.n_threads = n_threads; - factx.DK = DK; - factx.DV = DV; - factx.n_kv = nek1; - factx.n_kv_heads = n_kv_heads; - factx.n_heads = neq2; - factx.G = G; - factx.div_G = init_fastdiv_values(G); - factx.neq1 = neq1; - factx.Br = (uint32_t) Br; - factx.Bc = (uint32_t) Bc; - factx.g_br = (uint32_t) g_br; - factx.n_kv_blocks = n_kv_blocks; - factx.is_q_fp32 = (q->type == HTP_TYPE_F32); - factx.is_dst_fp32 = (dst->type == HTP_TYPE_F32); - factx.pipeline = pipeline; - factx.mask_broadcast = (mask != NULL && mask->ne[2] == 1); - - // Extract op parameters (mutable during softcap adjustment, then stored as const in factx) - float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; - memcpy(&scale, (float *) octx->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) octx->op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (float *) octx->op_params + 2, sizeof(float)); - - if (logit_softcap != 0.0f) { - scale /= logit_softcap; - } - -#ifdef HMX_FA_USE_EXP2_HF - // Pre-bake log2(e) into qk_scale so HMX-produced S tiles are in log2(e)-scaled - // space. Then exp2(S - m) in the softmax equals base-e exp((S - m) / log2(e)), - // preserving ggml's base-e softmax semantics. Matches htp-ops-lib flash_attn.c. - // - // When softcap is active we cannot pre-bake log2(e) here — it would land inside - // the tanh argument and shift the softcap knee from x≈c to x≈c/log2(e), giving - // numerically wrong softcapped values. Instead fold log2(e) into the post-tanh - // multiplier (see softcap block: v_cap absorbs log2(e)). - if (logit_softcap == 0.0f) { - scale *= 1.44269504f; // log2(e) - } -#endif - - factx.scale = scale; - factx.max_bias = max_bias; - factx.logit_softcap = logit_softcap; - - factx.n_head_log2 = 1u << (uint32_t) floor(log2(neq2)); - factx.m0 = powf(2.0f, -(max_bias) / factx.n_head_log2); - factx.m1 = powf(2.0f, -(max_bias / 2.0f) / factx.n_head_log2); - - // ======== VTCM allocation (GQA-aware) ======== - const size_t size_k_row = DK * sizeof(__fp16); - const size_t size_v_row = DV * sizeof(__fp16); - const size_t size_k_row_padded = hex_round_up(size_k_row, 128); - const size_t size_v_row_padded = hex_round_up(size_v_row, 128); - - const size_t q_tile_bytes = hex_align_up(g_br * DK * sizeof(__fp16), 4096); - const size_t o_tile_bytes = hex_align_up(g_br * DV * sizeof(__fp16), 4096); - const size_t k_dma_bytes = hex_align_up(Bc * size_k_row_padded, 4096); - const size_t v_dma_bytes = hex_align_up(Bc * size_v_row_padded, 4096); - const size_t k_tile_bytes = hex_align_up(Bc * DK * sizeof(__fp16), 4096); - const size_t v_tile_bytes = hex_align_up(Bc * DV * sizeof(__fp16), 4096); - const size_t s_tile_bytes = hex_align_up(g_br * Bc * sizeof(__fp16), 4096); - const size_t d_tile_bytes = hex_align_up(g_br * g_br * sizeof(__fp16), 4096); - const size_t col_vec_bytes = hex_align_up(g_br * sizeof(__fp16), 256); - const size_t row_vec_bytes = hex_align_up(Bc * sizeof(__fp16), 256); - const size_t m_line_bytes = hex_align_up(Bc * sizeof(__fp16), 128); - const size_t m_buf_bytes = hex_align_up(Br * m_line_bytes, 4096); - const size_t slopes_bytes = hex_align_up(g_br * sizeof(__fp16), 128); - - uint8_t * vtcm_cur = ctx->vtcm_base; - - factx.vtcm_q_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, q_tile_bytes); - factx.vtcm_o_tiles[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, o_tile_bytes); - factx.vtcm_o_tiles[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, o_tile_bytes); - factx.vtcm_k_fp16[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_dma_bytes); - factx.vtcm_k_fp16[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_dma_bytes); - factx.vtcm_v_fp16[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_dma_bytes); - factx.vtcm_v_fp16[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_dma_bytes); - factx.vtcm_k_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, k_tile_bytes); - factx.vtcm_v_tiles[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_tile_bytes); - if (pipeline) { - factx.vtcm_v_tiles[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, v_tile_bytes); - } else { - factx.vtcm_v_tiles[1] = NULL; - } - factx.vtcm_s_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, s_tile_bytes); - factx.vtcm_p_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, s_tile_bytes); - factx.vtcm_d_tiles = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, d_tile_bytes); - factx.vtcm_m_vec = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_l_vec = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_s_rowmax = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_p_rowsum = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, col_vec_bytes); - factx.vtcm_row_bufs = (HVX_Vector *) vtcm_seq_alloc(&vtcm_cur, row_vec_bytes * 2 * n_threads); - factx.row_buf_stride = row_vec_bytes / sizeof(HVX_Vector); - factx.vtcm_hmx_scales_id = vtcm_seq_alloc(&vtcm_cur, 256); - factx.vtcm_hmx_scales_qk = vtcm_seq_alloc(&vtcm_cur, 256); - factx.vtcm_mask_buf = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, m_buf_bytes); - factx.mask_buf_row_stride = m_line_bytes / sizeof(__fp16); - factx.vtcm_slopes = (__fp16 *) vtcm_seq_alloc(&vtcm_cur, slopes_bytes); - - if ((size_t) (vtcm_cur - ctx->vtcm_base) > ctx->vtcm_size) { - return HTP_STATUS_VTCM_TOO_SMALL; - } - - // ======== Initialize HMX output scales ======== - // Identity scale (1.0) for O updates and normalization - hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 - - // QK scale embedded in HMX output - hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale)); - - // ======== Skip compute if profiling ======== - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } - - - // ======== DMA setup ======== - dma_queue * const dma = ctx->dma[0]; - - // Padded row sizes for DMA (defined in outer scope) - - const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS; - const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS; - - // Q/O element size for Q load and O store - const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16); - - // ======== HMX lock strategy ======== - // Pipeline: queue thread auto-acquires HMX lock on first push; released by suspend. - // Fallback: main thread holds the lock (original behavior). - if (!factx.pipeline) { - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - } - - // ======== Reusable job descriptors for pipeline ======== - hmx_fa_qk_job_t qk_job; - hmx_fa_o_update_job_t ou_job; - hmx_fa_o_norm_job_t on_job; - - // ======== Main loop: per batch, per KV head, per Q block ======== - for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { - for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) { - const uint32_t ik2 = kv_head; - const uint32_t ik3 = ib3 / (neq3 / k->ne[3]); - const uint32_t iv2 = kv_head; - const uint32_t iv3 = ib3 / (neq3 / v->ne[3]); - - for (uint32_t q_start = 0; q_start < neq1; q_start += Br) { - const uint32_t n_q_rows = hex_smin(Br, neq1 - q_start); - const size_t n_rows_g = n_q_rows * G; - const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); - const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS; - - // ---- Load Q block [g_br, D] -> tiles, interleaving G heads ---- - if (n_rows_g < g_br) { - hvx_splat_u8_a(factx.vtcm_q_tiles, 0, q_tile_bytes); - } - fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g); - - // ---- Initialize per-block state ---- - hvx_splat_u8_a(factx.vtcm_l_vec, 0, col_vec_bytes); - hvx_splat_u8_a(factx.vtcm_d_tiles, 0, d_tile_bytes); - hvx_splat_u16_a(factx.vtcm_m_vec, 0xfbff, col_vec_bytes/2); - - __fp16 * o_tile_prev = factx.vtcm_o_tiles[0]; - __fp16 * o_tile_curr = factx.vtcm_o_tiles[1]; - hvx_splat_u8_a(o_tile_prev, 0, o_tile_bytes); - - // ---- KV block loop with DMA double-buffering ---- - size_t buf_idx = 0; - - fa_compute_slopes(&factx, kv_head, n_rows_g); - - // Prefetch first KV block - if (factx.n_kv_blocks > 0) { - const uint32_t kv_rows0 = hex_smin(Bc, nek1); - - const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], - size_k_row, kv_rows0); - - const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3]; - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], - size_v_row, kv_rows0); - } - - // Mask DMA: single 2D transfer of n_q_rows unique mask rows into VTCM buffer. - // Only when mask is head-broadcast (ne[2]==1); otherwise softmax reads DDR directly. - #define MASK_DMA_PUSH(kv_start_val, kv_rows_val, has_mask_dma_var) \ - do { \ - has_mask_dma_var = false; \ - if (mask && factx.mask_broadcast) { \ - const uint32_t _im3 = ib3 % mask->ne[3]; \ - const uint8_t * _ms = (const uint8_t *) mask->data + q_start * mask->nb[1] + _im3 * mask->nb[3] + \ - (kv_start_val) * sizeof(__fp16); \ - dma_queue_push(dma, dma_make_ptr(factx.vtcm_mask_buf, _ms), m_line_bytes, mask->nb[1], \ - (kv_rows_val) * sizeof(__fp16), n_q_rows); \ - has_mask_dma_var = true; \ - } \ - } while (0) - - #define MASK_DMA_POP(has_mask_dma_var) \ - do { \ - if (has_mask_dma_var) { \ - dma_queue_pop(dma); \ - } \ - } while (0) - - #define DMA_PREFETCH_KV(blk_val) \ - do { \ - if ((blk_val) < factx.n_kv_blocks) { \ - const uint32_t _ns = (blk_val) * Bc; \ - const uint32_t _nr = hex_smin(Bc, nek1 - _ns); \ - size_t _nb = 1 - buf_idx; \ - const uint8_t * _ks = (const uint8_t *) k->data + _ns * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; \ - dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[_nb], _ks), size_k_row_padded, k->nb[1], size_k_row, _nr); \ - const uint8_t * _vs = (const uint8_t *) v->data + _ns * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; \ - dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[_nb], _vs), size_v_row_padded, v->nb[1], size_v_row, _nr); \ - } \ - } while (0) - - const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); - const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); - - if (factx.pipeline) { - // ================================================================== - // Pipeline path: HVX phases ‖ HMX queue worker - // ================================================================== - struct hmx_queue * hmx_q = ctx->hmx_queue; - - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { - const uint32_t kv_start = kv_blk * Bc; - const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); - const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); - - // Wait for current KV DMA - dma_queue_pop(dma); // K - dma_queue_pop(dma); // V - - // Push mask DMA for this block (single 2D DMA when broadcast) - bool has_mask_dma = false; - MASK_DMA_PUSH(kv_start, kv_rows, has_mask_dma); - - // ---- Phase 1: K_int(blk) ‖ O_update(blk-1) ---- - if (kv_blk > 0) { - // Submit O_update for previous block (HMX worker) - ou_job.o_curr = o_tile_curr; - ou_job.o_prev = o_tile_prev; - ou_job.p_tiles = factx.vtcm_p_tiles; - ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; - ou_job.d_tiles = factx.vtcm_d_tiles; - ou_job.hmx_scales = factx.vtcm_hmx_scales_id; - ou_job.n_row_tiles = n_row_tiles; - ou_job.n_col_tiles = hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS); - ou_job.n_row_tiles_g_br = n_row_tiles_g_br; - ou_job.n_tiles_per_bc = n_tiles_per_bc; - ou_job.DV = DV; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); - } - fa_phase_k_interleave(&factx, kv_rows, k_src_stride, buf_idx); - - // ---- Phase 2: qk_dot(blk) on HMX ‖ V_int(blk) + DMA prefetch on HVX ---- - qk_job.q_tiles = factx.vtcm_q_tiles; - qk_job.k_tiles = factx.vtcm_k_tiles; - qk_job.s_tiles = factx.vtcm_s_tiles; - qk_job.n_row_tiles = n_row_tiles; - qk_job.n_col_tiles = n_col_tiles; - qk_job.n_dot_tiles = DK / 32; - qk_job.n_tiles_per_bc = n_tiles_per_bc; - qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job)); - - // DMA push next block (non-blocking, before worker_pool) - DMA_PREFETCH_KV(kv_blk + 1); - fa_phase_v_interleave(&factx, kv_rows, v_src_stride, buf_idx, n_tiles_per_bc); - - // Pop and swap previous block's output update (deferred HMX pop) - if (kv_blk > 0) { - hmx_queue_pop(hmx_q); - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - // Pop current block's dot product job - hmx_queue_pop(hmx_q); - - // ---- Phase 3: softmax(blk) + build_D(blk) | HMX idle ---- - // Pop mask DMA before softmax (ensures VTCM buffer is ready) - MASK_DMA_POP(has_mask_dma); - - fa_softmax_args_t sargs; - memset(&sargs, 0, sizeof(sargs)); - sargs.factx = &factx; - sargs.kv_rows = kv_rows; - sargs.n_rows_g = n_rows_g; - sargs.n_col_tiles = n_col_tiles; - sargs.n_tiles_per_bc = n_tiles_per_bc; - sargs.n_row_tiles = n_row_tiles; - sargs.n_row_tiles_g_br = n_row_tiles_g_br; - sargs.Bc = Bc; - sargs.G = G; - sargs.kv_head = kv_head; - sargs.kv_start = kv_start; - sargs.q_start = q_start; - sargs.ib3 = ib3; - sargs.has_alibi = (factx.max_bias != 0.0f); - sargs.mask = mask; - sargs.mask_vtcm = has_mask_dma ? (const __fp16 *) factx.vtcm_mask_buf : NULL; - sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; - sargs.slopes = factx.vtcm_slopes; - fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); - - buf_idx = 1 - buf_idx; - } // end KV block loop (pipeline) - - // Epilogue: O_update for last block - if (factx.n_kv_blocks > 0) { - const uint32_t last_blk = factx.n_kv_blocks - 1; - const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS); - ou_job.o_curr = o_tile_curr; - ou_job.o_prev = o_tile_prev; - ou_job.p_tiles = factx.vtcm_p_tiles; - ou_job.v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; - ou_job.d_tiles = factx.vtcm_d_tiles; - ou_job.hmx_scales = factx.vtcm_hmx_scales_id; - ou_job.n_row_tiles = n_row_tiles; - ou_job.n_col_tiles = last_cols; - ou_job.n_row_tiles_g_br = n_row_tiles_g_br; - ou_job.n_tiles_per_bc = n_tiles_per_bc; - ou_job.DV = DV; - hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); - hmx_queue_pop(hmx_q); - - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - } else { - // ================================================================== - // Fallback path: sequential with multi-thread HVX phases - // Main thread holds HMX lock, runs HMX inline. - // ================================================================== - - for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) { - const uint32_t kv_start = kv_blk * Bc; - const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); - const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); - dma_queue_pop(dma); // K - dma_queue_pop(dma); // V - - bool has_mask_dma = false; - MASK_DMA_PUSH(kv_start, kv_rows, has_mask_dma); - DMA_PREFETCH_KV(kv_blk + 1); - fa_phase_k_interleave(&factx, kv_rows, k_src_stride, buf_idx); - - { - const size_t n_dot_tiles = (size_t) (DK / 32); - const __fp16 * restrict q_base = factx.vtcm_q_tiles; - const __fp16 * restrict k_base = factx.vtcm_k_tiles; - __fp16 * restrict s_base = factx.vtcm_s_tiles; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_qk); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < n_col_tiles; ++c) { - const __fp16 * row_tiles = q_base + r * HMX_FP16_TILE_N_ROWS * DK; - const __fp16 * col_tiles = k_base + c * HMX_FP16_TILE_N_COLS * DK; - __fp16 * out_tile = s_base + (r * n_tiles_per_bc + c) * HMX_FP16_TILE_N_ELMS; - for (size_t k = 0; k < n_dot_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int) row_tiles, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) col_tiles, 2047); - row_tiles += HMX_FP16_TILE_N_ELMS; - col_tiles += HMX_FP16_TILE_N_ELMS; - } - Q6_mxmem_AR_after_hf(out_tile, 0); - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - } - - // Pop mask DMA - MASK_DMA_POP(has_mask_dma); - - // Softmax + build_D (multi-thread HVX + serial m/l update) - fa_softmax_args_t sargs; - memset(&sargs, 0, sizeof(sargs)); - sargs.factx = &factx; - sargs.kv_rows = kv_rows; - sargs.n_rows_g = n_rows_g; - sargs.n_col_tiles = n_col_tiles; - sargs.n_tiles_per_bc = n_tiles_per_bc; - sargs.n_row_tiles = n_row_tiles; - sargs.n_row_tiles_g_br = n_row_tiles_g_br; - sargs.Bc = Bc; - sargs.G = G; - sargs.kv_head = kv_head; - sargs.kv_start = kv_start; - sargs.q_start = q_start; - sargs.ib3 = ib3; - sargs.has_alibi = (factx.max_bias != 0.0f); - sargs.mask = mask; - sargs.mask_vtcm = has_mask_dma ? (const __fp16 *) factx.vtcm_mask_buf : NULL; - sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; - sargs.slopes = factx.vtcm_slopes; - fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); - fa_phase_v_interleave(&factx, kv_rows, v_src_stride, buf_idx, n_tiles_per_bc); - - { - const size_t DV_tiles = (size_t) (DV / 32); - const __fp16 * restrict d_base = factx.vtcm_d_tiles; - const __fp16 * restrict p_base = factx.vtcm_p_tiles; - const __fp16 * restrict v_base = factx.vtcm_v_tiles[0]; - const __fp16 * restrict op_base = o_tile_prev; - __fp16 * restrict oc_base = o_tile_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(DV_tiles > 0); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_id); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_base + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = op_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - Q6_activation_hf_mxmem_RR((unsigned int) d_diag, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) o_rc, 2047); - - const __fp16 * p_tile_in = p_base + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - const __fp16 * v_tile_in = v_base + (c * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; - for (size_t k = 0; k < n_col_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int) p_tile_in, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) v_tile_in, 2047); - p_tile_in += HMX_FP16_TILE_N_ELMS; - v_tile_in += HMX_FP16_TILE_N_ELMS; - } - - __fp16 * o_tile_out = oc_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - Q6_mxmem_AR_after_hf(o_tile_out, 0); - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); - } - - buf_idx = 1 - buf_idx; - } // end KV block loop (fallback) - } - - // ---- Final normalization: O = diag(1/l) @ O ---- - { - fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br); - - // HMX: O_final = diag(1/l) @ O_prev - if (factx.pipeline) { - on_job.o_curr = o_tile_curr; - on_job.o_prev = o_tile_prev; - on_job.d_tiles = factx.vtcm_d_tiles; - on_job.hmx_scales = factx.vtcm_hmx_scales_id; - on_job.n_row_tiles = n_row_tiles; - on_job.n_row_tiles_g_br = n_row_tiles_g_br; - on_job.DV = DV; - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job)); - hmx_queue_pop(ctx->hmx_queue); - } else { - const size_t DV_tiles = (size_t) (DV / 32); - const __fp16 * restrict d_base = factx.vtcm_d_tiles; - const __fp16 * restrict op_base = o_tile_prev; - __fp16 * restrict oc_base = o_tile_curr; - __builtin_assume(n_row_tiles > 0); - __builtin_assume(DV_tiles > 0); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - Q6_bias_mxmem2_A((void *) factx.vtcm_hmx_scales_id); - for (size_t r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < DV_tiles; ++c) { - const __fp16 * d_diag = d_base + r * (n_row_tiles_g_br + 1) * HMX_FP16_TILE_N_ELMS; - const __fp16 * o_rc = op_base + (c * n_row_tiles_g_br + r) * HMX_FP16_TILE_N_ELMS; - __fp16 * o_out = oc_base + (r * DV_tiles + c) * HMX_FP16_TILE_N_ELMS; - - Q6_activation_hf_mxmem_RR((unsigned int) d_diag, 2047); - Q6_weight_hf_mxmem_RR((unsigned int) o_rc, 2047); - Q6_mxmem_AR_after_hf(o_out, 0); - } - } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HMX_COMP, HTP_MAX_NTHREADS); - } - } - - // ---- Store O block ---- - fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g); - -#undef MASK_DMA_PUSH -#undef MASK_DMA_POP -#undef DMA_PREFETCH_KV - - } // end Q block loop - } // end KV head loop - } // end batch loop - - if (factx.pipeline) { - hmx_queue_suspend(ctx->hmx_queue); - } else { - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - } - - - - return HTP_STATUS_OK; -} diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index b7fba22a87f..740a8f87d61 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -506,7 +506,8 @@ static void dequantize_tiled_weight_to_fp16_task_q8_0( } } -static void convert_f16_weight_to_fp16_tiles_task( +static __attribute__((noinline)) +void convert_f16_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { @@ -543,17 +544,13 @@ static void convert_f16_weight_to_fp16_tiles_task( Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v1); v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); } - (void) *(volatile HVX_Vector *)(tile_base); } ++t; ++kt; } - - if (start_tile < end_tile) { - (void) *(volatile HVX_Vector *)(state->dst + (end_tile - 1) * HTP_MM_HMX_TILE_N_ELMS); - } } -static void quantize_f32_weight_to_fp16_tiles_task( +static __attribute__((noinline)) +void quantize_f32_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, uint32_t start_tile, uint32_t end_tile) { @@ -594,125 +591,193 @@ static void quantize_f32_weight_to_fp16_tiles_task( Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi); v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); } - (void) *(volatile HVX_Vector *)(tile_base); } ++t; ++kt; } - - if (start_tile < end_tile) { - (void) *(volatile HVX_Vector *)(state->dst + (end_tile - 1) * HTP_MM_HMX_TILE_N_ELMS); - } } // --- End tiled dequantizers --- -// requires external HMX lock -static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation, const __fp16 *restrict weight, const __fp16 *restrict scales, +// dot-chunk functions require external HMX lock + +static void core_dot_chunk_fp16_short(__fp16 *restrict output, const __fp16 *restrict activation, + const __fp16 *restrict weight, const __fp16 *restrict scales, uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) { __builtin_assume(n_row_tiles > 0); __builtin_assume(n_col_tiles > 0); __builtin_assume(n_dot_tiles > 0); + __builtin_assume(n_dot_tiles <= 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales)); + + const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + const uint32_t range = 2048u * n_dot_tiles - 1; + + for (uint32_t r = 0; r < n_row_tiles; ++r) { + const __fp16 *row_base = activation + r * dot_stride; + const __fp16 *col_base = weight; + __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + + for (size_t c = 0; c < n_col_tiles; ++c) { + asm volatile(HMX_CLRACC_F16()); + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base)); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory"); + col_base += dot_stride; + out_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } +} + +static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation, + const __fp16 *restrict weight, const __fp16 *restrict scales, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) { + if (n_dot_tiles <= 32) { + core_dot_chunk_fp16_short(output, activation, weight, scales, n_row_tiles, n_col_tiles, n_dot_tiles); + return; + } + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales)); + + const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; - Q6_bias_mxmem2_A((void *)scales); for (uint32_t r = 0; r < n_row_tiles; ++r) { + const __fp16 *row_base = activation + r * dot_stride; + const __fp16 *col_base = weight; + __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + for (size_t c = 0; c < n_col_tiles; ++c) { - Q6_mxclracc_hf(); - - const __fp16 *row_tiles = activation + r * n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; - const __fp16 *col_tiles = weight + c * n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; - - for (uint32_t k = 0, k_block; k < n_dot_tiles; k += k_block) { - k_block = hex_smin(n_dot_tiles - k, 32); - const uint32_t range = 2048u * (uint32_t)k_block - 1; - Q6_activation_hf_mxmem_RR_deep((unsigned int)row_tiles, range); - Q6_weight_hf_mxmem_RR((unsigned int)col_tiles, range); - row_tiles += k_block * HTP_MM_HMX_TILE_N_ELMS; - col_tiles += k_block * HTP_MM_HMX_TILE_N_ELMS; + const __fp16 *row_tiles = row_base; + const __fp16 *col_tiles = col_base; + + asm volatile(HMX_CLRACC_F16()); + + const uint32_t n_loops = n_dot_tiles / 32; + const uint32_t rem = n_dot_tiles % 32; + + for (uint32_t l = 0; l < n_loops; ++l) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles)); + row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; } - __fp16 *out_tile = output + (r * n_col_tiles + c) * HTP_MM_HMX_TILE_N_ELMS; - Q6_mxmem_AR_after_hf(out_tile, 0); + if (rem > 0) { + const uint32_t range = 2048u * rem - 1; + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles)); + } + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory"); + + col_base += dot_stride; + out_tile += HTP_MM_HMX_TILE_N_ELMS; } } } -// C += AB -static void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, +static void core_mma_chunk_fp16_short(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile, uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) { __builtin_assume(n_row_tiles > 0); __builtin_assume(n_col_tiles > 0); __builtin_assume(n_dot_tiles > 0); + __builtin_assume(n_dot_tiles <= 32); - Q6_bias_mxmem2_A((void *)col_scales); + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales)); const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + const uint32_t range = 2048u * n_dot_tiles - 1; + for (size_t i = 0; i < n_row_tiles; ++i) { const __fp16 *row_base = a + i * dot_tile_stride; __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + const __fp16 *col_base = b; + __fp16 *accum_tile = res_base; + for (size_t j = 0; j < n_col_tiles; ++j) { - Q6_mxclracc_hf(); + asm volatile(HMX_CLRACC_F16()); - const __fp16 *col_tiles = b + j * dot_tile_stride; - const __fp16 *row_tiles = row_base; - __fp16 *accum_tile = res_base + j * HTP_MM_HMX_TILE_N_ELMS; if (!zero_init) { - Q6_activation_hf_mxmem_RR((unsigned int)accum_tile, 2047); - Q6_weight_hf_mxmem_RR((unsigned int)eye_tile, 2047); + asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile)); } - for (uint32_t k = 0, k_block; k < n_dot_tiles; k += k_block) { - k_block = hex_smin(n_dot_tiles - k, 32); - const uint32_t range = 2048u * k_block - 1; - Q6_activation_hf_mxmem_RR_deep((unsigned int)row_tiles, range); - Q6_weight_hf_mxmem_RR((unsigned int)col_tiles, range); - row_tiles += k_block * HTP_MM_HMX_TILE_N_ELMS; - col_tiles += k_block * HTP_MM_HMX_TILE_N_ELMS; - } + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base)); - Q6_mxmem_AR_after_hf(accum_tile, 0); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory"); + + col_base += dot_tile_stride; + accum_tile += HTP_MM_HMX_TILE_N_ELMS; } } } -// --- Async HMX matmul job (for pipeline overlap) --- +static void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, + const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) { + if (n_dot_tiles <= 32) { + core_mma_chunk_fp16_short(c, a, b, col_scales, eye_tile, n_row_tiles, n_col_tiles, n_dot_tiles, zero_init); + return; + } + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales)); -typedef struct { - __fp16 * output; - const __fp16 * activation; - const __fp16 * weight; - const __fp16 * scales; - uint32_t n_row_tiles; - uint32_t n_col_tiles; - uint32_t n_dot_tiles; -} hmx_matmul_job_t; - -static void hmx_matmul_worker_fn(void * data) { - hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; - FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); - core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); -} + const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; -static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, - __fp16 * output, - const __fp16 * activation, - const __fp16 * weight, - const __fp16 * scales, - uint32_t n_row_tiles, - uint32_t n_col_tiles, - uint32_t n_dot_tiles) { - job->output = output; - job->activation = activation; - job->weight = weight; - job->scales = scales; - job->n_row_tiles = n_row_tiles; - job->n_col_tiles = n_col_tiles; - job->n_dot_tiles = n_dot_tiles; + for (size_t i = 0; i < n_row_tiles; ++i) { + const __fp16 *row_base = a + i * dot_tile_stride; + __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + const __fp16 *col_base = b; + __fp16 *accum_tile = res_base; + + for (size_t j = 0; j < n_col_tiles; ++j) { + const __fp16 *col_tiles = col_base; + const __fp16 *row_tiles = row_base; + + asm volatile(HMX_CLRACC_F16()); + + if (!zero_init) { + asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile)); + } + + const uint32_t n_loops = n_dot_tiles / 32; + const uint32_t rem = n_dot_tiles % 32; + + for (uint32_t l = 0; l < n_loops; ++l) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles)); + row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + } + + if (rem > 0) { + const uint32_t range = 2048u * rem - 1; + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles)); + } + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory"); + + col_base += dot_tile_stride; + accum_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } } // output : fp16 -> f32p -static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, uint32_t start_row, uint32_t n_rows, uint32_t n_cols, uint32_t dst_stride, uint32_t dst_cols) { +static void transfer_output_chunk_fp16_to_fp32( + float *restrict dst, + const float *restrict src2, + const __fp16 *restrict vtcm_src, + uint32_t start_row, + uint32_t n_rows, + uint32_t n_cols, + uint32_t dst_stride, + uint32_t src2_stride, + uint32_t dst_cols +) { assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); const size_t tile_row_stride = (n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; @@ -727,6 +792,7 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; float *output_row_base = dst + r * dst_stride; // global memory row base for row r (and r+1) + const float *src2_row_base = src2 ? (src2 + r * src2_stride) : NULL; #pragma unroll(4) for (size_t c = 0; c < limit_c_aligned; c += HTP_MM_HMX_TILE_N_COLS) { @@ -738,9 +804,20 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0); HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride); - *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + *pv_out0 = v_out0; + if (r + 1 < n_rows) { - *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + *pv_out1 = v_out1; } } @@ -752,9 +829,20 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 HVX_Vector v = ((const HVX_Vector *) tile)[r1]; HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp))); + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), v_out0); + if (r + 1 < n_rows) { - hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp))); + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), v_out1); } } } @@ -763,11 +851,13 @@ static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 typedef struct { const __fp16 *vtcm_src; float *dst; + const float *src2; uint32_t n_tasks; uint32_t n_tot_chunks; uint32_t n_chunks_per_task; uint32_t n_cols; uint32_t dst_stride; // DDR row stride + uint32_t src2_stride; // DDR row stride for residual uint32_t dst_cols; // Actual output columns struct htp_thread_trace * traces; } output_transfer_task_state_t; @@ -866,148 +956,55 @@ static void transfer_activation_chunk_fp32_to_fp16(__fp16 *restrict vtcm_dst, co } } -typedef struct { - __fp16 *dst; - const float *src; - uint32_t n_tasks; - uint32_t n_tot_chunks; - uint32_t n_chunks_per_task; - uint32_t k_block; - uint32_t k_stride; - uint32_t k_valid; - struct htp_thread_trace * traces; - struct htp_context * ctx; - float * vtcm_f32_act; -} activation_transfer_task_state_t; - -static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( - dma_queue *dma_q, +static void transfer_activation_row_pair_fp32_to_fp16( __fp16 *restrict vtcm_dst, - const float *restrict src, - uint32_t n_rows, + const float *restrict row0, + const float *restrict row1, + uint32_t r, uint32_t k_block, - uint32_t k_stride, uint32_t k_valid, - float *thread_f32_act) { + bool row0_valid, + bool row1_valid) { - const uint32_t R = HTP_MM_DMA_ACT_ROWS_PER_STEP; - const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx - const uint32_t n_steps = n_rows_padded / R; + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); - // pre-fetch step 0 - if (n_steps > 0 && n_rows > 0) { - uint32_t nrows_to_fetch = hex_smin(n_rows, R); - dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src), - k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); - } + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); - for (uint32_t s = 0; s < n_steps; ++s) { - uint32_t r = R * s; - float *curr_buf = thread_f32_act + (s % 2) * R * k_block; + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; - if (r < n_rows) { - dma_queue_pop(dma_q); - } - - uint32_t next_s = s + 1; - uint32_t next_r = R * next_s; - if (next_r < n_rows) { - uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); - const float *next_src = src + next_r * k_stride; - float *next_buf = thread_f32_act + (next_s % 2) * R * k_block; - dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), - k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); - } - - #pragma unroll - for (uint32_t i = 0; i < HTP_MM_DMA_ACT_ROWS_PER_STEP; i += 2) { - uint32_t curr_r = r + i; - const bool row0_valid = (curr_r < n_rows); - const bool row1_valid = (curr_r + 1) < n_rows; - - const float *ptr_in0 = curr_buf + i * k_block; - const float *ptr_in1 = curr_buf + (i + 1) * k_block; + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); - uint32_t c = 0; - for (; c + 32 <= k_valid; c += 32) { - HVX_Vector v0 = Q6_V_vzero(); - HVX_Vector v1 = Q6_V_vzero(); - if (row0_valid) v0 = *(const HVX_Vector *)(ptr_in0 + c); - if (row1_valid) v1 = *(const HVX_Vector *)(ptr_in1 + c); + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); - HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); - uint32_t r0 = curr_r / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = curr_r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx - uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index - uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; - HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); - tile[r1 / 2] = v_out; - } - if (c < k_block) { - HVX_Vector v0 = Q6_V_vzero(); - HVX_Vector v1 = Q6_V_vzero(); - if (row0_valid) v0 = *(const HVX_Vector *)(ptr_in0 + c); - if (row1_valid) v1 = *(const HVX_Vector *)(ptr_in1 + c); - - uint32_t rem = k_valid - c; - HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); - v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); - v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); - - HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); - - uint32_t r0 = curr_r / HTP_MM_HMX_TILE_N_ROWS; // tile row index - uint32_t r1 = curr_r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx - uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index - uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; - - HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); - tile[r1 / 2] = v_out; - } - } + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; } } -typedef struct { - const struct mmid_row_mapping *matrix_rows; - __fp16 *dst; - const float *src; - uint32_t n_tasks; - uint32_t n_tot_chunks; - uint32_t n_chunks_per_task; - uint32_t k_block; - uint32_t cur_a; - uint32_t mapping_stride; - uint32_t ne11; - struct fastdiv_values ne11_div; - size_t nb11; - size_t nb12; - uint32_t start_row; - uint32_t cne1; - uint32_t k_valid; - struct htp_thread_trace *traces; -} activation_transfer_gathered_task_state_t; - -typedef struct { - const struct mmid_row_mapping *matrix_rows; - const __fp16 *vtcm_src; - float *dst; - uint32_t n_tasks; - uint32_t n_tot_chunks; - uint32_t n_chunks_per_task; - uint32_t n_cols; - uint32_t cur_a; - uint32_t mapping_stride; - size_t dst_nb1; - size_t dst_nb2; - uint32_t start_row; - uint32_t cne1; - struct htp_thread_trace *traces; -} output_transfer_scattered_task_state_t; - static void transfer_activation_chunk_fp32_to_fp16_gathered( __fp16 *restrict vtcm_dst, const float *restrict src, diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index a0007539c56..5f6a5e206bb 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -6,6 +6,7 @@ #include #include +#include #include @@ -42,6 +43,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { case HMX_QUEUE_NOOP: /* noop */; break; case HMX_QUEUE_KILL: *killed = true; break; case HMX_QUEUE_SUSPEND: hmx_unlock(q); break; + case HMX_QUEUE_WAKEUP: hmx_lock(q); break; default: hmx_lock(q); htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); @@ -70,9 +72,14 @@ static void hmx_queue_thread(void * arg) { while (!killed) { unsigned int seqn = atomic_load(&q->seqn); if (seqn == prev_seqn) { + // drop HVX context while spinning + if (poll_cnt > 1 && poll_cnt == HMX_QUEUE_POLL_COUNT) { + qurt_hvx_unlock(); + } if (--poll_cnt) { hex_pause(); continue; } FARF(HIGH, "hmx-queue-thread: sleeping"); qurt_futex_wait(&q->seqn, prev_seqn); + poll_cnt = HMX_QUEUE_POLL_COUNT; continue; } prev_seqn = seqn; diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index 83135cd91d8..b176fa17961 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -18,13 +18,19 @@ extern "C" { #endif #define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024) -#define HMX_QUEUE_POLL_COUNT 2000 + +#if __HVX_ARCH__ > 79 +#define HMX_QUEUE_POLL_COUNT 2000 +#else +#define HMX_QUEUE_POLL_COUNT 1 +#endif typedef void (*hmx_queue_func)(void *); // Dummy funcs used as signals enum hmx_queue_signal { HMX_QUEUE_NOOP = 0, // aka NULL + HMX_QUEUE_WAKEUP, HMX_QUEUE_SUSPEND, HMX_QUEUE_KILL }; @@ -97,7 +103,7 @@ static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) { return q->capacity; } -static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) { unsigned int ip = q->idx_pop; unsigned int iw = q->idx_write; @@ -120,13 +126,28 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { return rd; } +static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { + while (1) { + struct hmx_queue_desc d = hmx_queue_pop_one(q); + + uint32_t sig = (uint32_t) d.func; + if (sig && sig <= HMX_QUEUE_KILL) + continue; + + return d; + } +} + static inline void hmx_queue_flush(struct hmx_queue * q) { - while (hmx_queue_pop(q).func != NULL) ; + while (hmx_queue_pop_one(q).func != NULL) ; +} + +static inline void hmx_queue_wakeup(struct hmx_queue * q) { + hmx_queue_signal(q, HMX_QUEUE_WAKEUP); } static inline void hmx_queue_suspend(struct hmx_queue *q) { hmx_queue_signal(q, HMX_QUEUE_SUSPEND); - hmx_queue_flush(q); } #ifdef __cplusplus diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index f448ee3372a..2a61ca7349d 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -42,14 +42,14 @@ static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VL // Full range: start_row=0, end_row=n_cols. static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, const __fp16 * restrict vtcm_src, - int n_cols, - int k, - int src_stride, - int start_row, - int end_row) { + uint32_t n_cols, + uint32_t k, + size_t src_stride, + uint32_t start_row, + uint32_t end_row) { assert(k % HMX_FP16_TILE_N_COLS == 0); - const int n_k_tiles = k / HMX_FP16_TILE_N_COLS; + const uint32_t n_k_tiles = k / HMX_FP16_TILE_N_COLS; const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); @@ -65,14 +65,14 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, if (pair_scatter) { // Step c by 64 fp16 (two K-tiles per scatter), advance dst by 2 tiles per iter. - const int c_step = 2 * HMX_FP16_TILE_N_COLS; - const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); - const size_t dst_step = 2 * (size_t) HMX_FP16_TILE_N_ELMS; - const int n_c_iters = k / c_step; - - for (int r = start_row; r < end_row; r += 2) { - const int ct = r / HMX_FP16_TILE_N_ROWS; - const int local_r = r % HMX_FP16_TILE_N_ROWS; + const uint32_t c_step = 2 * HMX_FP16_TILE_N_COLS; + const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); + const size_t dst_step = 2 * (size_t) HMX_FP16_TILE_N_ELMS; + const uint32_t n_c_iters = k / c_step; + + for (uint32_t r = start_row; r < end_row; r += 2) { + const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; + const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); @@ -86,7 +86,7 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, assert(c_byte_step % 128 == 0); if (p1) { - for (int i = 0; i < n_c_iters; ++i) { + for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step; HVX_Vector v1 = hvx_vmem(p1); p1 += c_byte_step; Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0); @@ -95,7 +95,7 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, } } else { const HVX_Vector vzero = Q6_V_vzero(); - for (int i = 0; i < n_c_iters; ++i) { + for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step; Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0); Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, vzero); @@ -105,14 +105,14 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, } } else { // Fallback: scatter one K-tile per call (region 2047, masked). - const int c_step = HMX_FP16_TILE_N_COLS; - const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); - const size_t dst_step = (size_t) HMX_FP16_TILE_N_ELMS; - const int n_c_iters = k / c_step; - - for (int r = start_row; r < end_row; r += 2) { - const int ct = r / HMX_FP16_TILE_N_ROWS; - const int local_r = r % HMX_FP16_TILE_N_ROWS; + const uint32_t c_step = HMX_FP16_TILE_N_COLS; + const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); + const size_t dst_step = (size_t) HMX_FP16_TILE_N_ELMS; + const uint32_t n_c_iters = k / c_step; + + for (uint32_t r = start_row; r < end_row; r += 2) { + const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; + const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); @@ -122,7 +122,7 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL; if (p1) { - for (int i = 0; i < n_c_iters; ++i) { + for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step; HVX_Vector v1 = hvx_vmemu(p1); p1 += c_byte_step; Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0); @@ -131,7 +131,7 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, } } else { const HVX_Vector vzero = Q6_V_vzero(); - for (int i = 0; i < n_c_iters; ++i) { + for (uint32_t i = 0; i < n_c_iters; ++i) { HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step; Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0); Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, vzero); @@ -148,24 +148,24 @@ static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, // Full range: start_row=0, end_row=n_rows. static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, const __fp16 * restrict src, - int n_rows, - int head_dim, - int src_stride, - int n_row_tiles, - int start_row, - int end_row) { + uint32_t n_rows, + uint32_t head_dim, + size_t src_stride, + uint32_t n_row_tiles, + uint32_t start_row, + uint32_t end_row) { __builtin_assume(head_dim > 0); const size_t tile_stride_elms = (size_t) n_row_tiles * HMX_FP16_TILE_N_ELMS; - for (int r = start_row; r < end_row; r += 2) { + for (uint32_t r = start_row; r < end_row; r += 2) { const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_rows; const HVX_Vector * pv_in0 = (const HVX_Vector *) (src + r * src_stride); const HVX_Vector * pv_in1 = next_row_valid ? (const HVX_Vector *) (src + (r + 1) * src_stride) : NULL; // Row-pair invariants hoisted out of the c loop. - const int r0 = r / HMX_FP16_TILE_N_ROWS; - const int r1_half = (r % HMX_FP16_TILE_N_ROWS) / 2; + const uint32_t r0 = r / HMX_FP16_TILE_N_ROWS; + const uint32_t r1_half = (r % HMX_FP16_TILE_N_ROWS) / 2; // tb0 starts at tile (c0=0, r0); tb1 at the adjacent dim-tile (c0=1, r0). // Each c step (+= 64) advances both by 2 dim-tiles worth of fp16. @@ -174,7 +174,7 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, const size_t tb_step = 2 * tile_stride_elms; if (pv_in1) { - for (int c = 0; c < head_dim; c += 64) { + for (uint32_t c = 0; c < head_dim; c += 64) { HVX_Vector v0 = *pv_in0++; HVX_Vector v1 = *pv_in1++; HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); @@ -185,7 +185,7 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, } } else { const HVX_Vector vzero = Q6_V_vzero(); - for (int c = 0; c < head_dim; c += 64) { + for (uint32_t c = 0; c < head_dim; c += 64) { HVX_Vector v0 = *pv_in0++; HVX_VectorPair vp = Q6_W_vshuff_VVR(vzero, v0, -2); ((HVX_Vector *) tb0)[r1_half] = Q6_V_lo_W(vp); @@ -197,4 +197,26 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, } } +// --- HMX inline asm macros for load-store packetization --- +#define HMX_LOAD_MPY_F16(act, wt, range) \ + "{\n" \ + " activation.hf = mxmem(" act ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ + "}\n" + +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ + " activation.hf = mxmem(" act ", " range "):deep\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ + "}\n" + +#define HMX_STORE_AFTER_F16(out, scale_reg) \ + "mxmem(" out ", " scale_reg "):after.hf = acc\n" + +#define HMX_SET_BIAS(scales) \ + "bias = mxmem2(" scales ")\n" + +#define HMX_CLRACC_F16() \ + "mxclracc.hf\n" + #endif // HMX_UTILS_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index d0409013578..c9d0b3539a9 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -60,6 +60,7 @@ enum htp_op_code { HTP_OP_MUL_MAT_ID, HTP_OP_MUL_MAT_QKV, HTP_OP_MUL_MAT_FFN, + HTP_OP_MUL_MAT_ADD, HTP_OP_RMS_NORM, HTP_OP_RMS_NORM_MUL, HTP_OP_UNARY_SILU, @@ -175,6 +176,11 @@ enum htp_trace_event_id { HTP_TRACE_EVT_HVX_W_DEQUANT = 23, HTP_TRACE_EVT_HVX_W_PREP = 24, HTP_TRACE_EVT_HVX_O_PROC = 25, + HTP_TRACE_EVT_HVX_FA_QK = 26, + HTP_TRACE_EVT_HVX_FA_SFM = 27, + HTP_TRACE_EVT_HVX_FA_Q_PREP = 28, + HTP_TRACE_EVT_HVX_FA_K_PREP = 29, + HTP_TRACE_EVT_HVX_FA_V_PREP = 30, HTP_TRACE_EVT_HMX_COMP = 40, }; diff --git a/ggml/src/ggml-hexagon/htp/htp-vtcm.h b/ggml/src/ggml-hexagon/htp/htp-vtcm.h new file mode 100644 index 00000000000..a1e90575fb1 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-vtcm.h @@ -0,0 +1,19 @@ +#ifndef HTP_VTCM_H +#define HTP_VTCM_H + +#include +#include + +static inline uint8_t *vtcm_seq_alloc(uint8_t **vtcm_ptr, size_t size) { + uint8_t *p = *vtcm_ptr; + *vtcm_ptr += size; + return p; +} + +#define VTCM_LAYOUT_ALLOC(off, field, sz) do { (L)->field = (off); (off) += (sz); } while (0) +#define VTCM_LAYOUT_ALLOC_OPTIONAL(off, field, sz, cond) do { if (cond) { VTCM_LAYOUT_ALLOC(off, field, sz); } else { (L)->field = 0; } } while (0) + +#define VTCM_LAYOUT_PTR(type, base, offset) ((type *)((uint8_t *)(base) + (offset))) +#define VTCM_LAYOUT_PTR_OPTIONAL(type, base, offset, cond) ((cond) ? VTCM_LAYOUT_PTR(type, base, offset) : NULL) + +#endif // HTP_VTCM_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h index 493b26c6e75..d7575b9ff10 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-base.h +++ b/ggml/src/ggml-hexagon/htp/hvx-base.h @@ -134,16 +134,7 @@ static inline HVX_Vector hvx_vec_f32_to_f16_shuff(HVX_Vector v0, HVX_Vector v1) } static inline HVX_Vector hvx_vec_f32_to_f16(HVX_Vector v0, HVX_Vector v1) { - HVX_Vector v = Q6_Vh_vdeal_Vh(hvx_vec_f32_to_f16_shuff(v0, v1)); - -#if __HVX_ARCH__ < 79 - // replace NaNs with -INF, older arches produce NaNs for (-INF + 0.0) - const HVX_Vector neg_inf = hvx_vec_splat_f16(-INFINITY); - HVX_VectorPred nan = hvx_vec_is_nan_f16(v); - v = Q6_V_vmux_QVV(nan, neg_inf, v); -#endif - - return v; + return Q6_Vh_vdeal_Vh(hvx_vec_f32_to_f16_shuff(v0, v1)); } #if __HVX_ARCH__ >= 79 @@ -170,8 +161,6 @@ static inline HVX_VectorPair hvx_vec_f16_to_f32(HVX_Vector v) { } #endif - - static inline HVX_Vector hvx_vec_i16_from_hf_rnd_sat(HVX_Vector vin) { // This looks complicated. // Ideally should just be Q6_Vh_equals_Vhf(vin) diff --git a/ggml/src/ggml-hexagon/htp/hvx-exp.h b/ggml/src/ggml-hexagon/htp/hvx-exp.h index e71ec4909a6..bcd3d2d32c6 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-exp.h +++ b/ggml/src/ggml-hexagon/htp/hvx-exp.h @@ -16,6 +16,7 @@ #define EXP_COEFF_0 (0x3F000000) // 0.5 = 1/(2!) #define EXP_LOGN2 (0x3F317218) // ln(2) = 0.6931471805 #define EXP_LOG2E (0x3FB8AA3B) // log2(e) = 1/ln(2) = 1.4426950408 +#define EXP_LOG2E_F 1.44269504f #define EXP_ONE (0x3f800000) // 1.0 #define EXP_RANGE_R (0x42B17218) // ln(FLT_MAX) approx = 88.7228 #define EXP_RANGE_L (0xC2B00000) // -88.0 (approx log(FLT_MIN)) @@ -213,4 +214,42 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict } } +static inline HVX_Vector hvx_vec_exp2_f16(HVX_Vector x_v) { + const HVX_Vector zero_v = Q6_V_vzero(); + const HVX_Vector half_hf_v = Q6_Vh_vsplat_R(0x3800); // fp16 0.5 + + // Clamp input to prevent integer underflow in FP16-to-INT16 conversion + const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f); + x_v = Q6_Vhf_vmax_VhfVhf(v_clamp_min, x_v); + + // k = round_toward_neg_inf(x); f = (float)k; frac = x - f + HVX_Vector x_minus_half = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(x_v, half_hf_v)); + HVX_Vector k_v = Q6_Vh_equals_Vhf(x_minus_half); // truncate to int16 + HVX_Vector f_v = Q6_Vhf_equals_Vh(k_v); // back to fp16 + + HVX_Vector x_qf16 = Q6_Vqf16_vsub_VhfVhf(x_v, f_v); // fractional part in qf16 + + // Horner: y = ((((E5*x + E4)*x + E3)*x + E2)*x + E1)*x + E0 + HVX_Vector y = Q6_Vqf16_vmpy_Vqf16Vqf16(Q6_Vh_vsplat_R(0x5082), x_qf16); // E5*x + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x157d)); // + E4 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x20ed)); // + E3 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x2b1b)); // + E2 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x33b0)); // + E1 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x398c)); // + E0 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); // y = y * x + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x3c00)); // + 1.0 + + // Combine polynomial (mantissa) with integer part (exponent): result = y * 2^k + y = Q6_Vhf_equals_Vqf16(y); + HVX_Vector y_exp = Q6_Vuh_vlsr_VuhR(Q6_Vh_vasl_VhR(y, 1), 11); + y_exp = Q6_Vh_vadd_VhVh(k_v, y_exp); + HVX_VectorPred q_underflow = Q6_Q_vcmp_gt_VhVh(zero_v, y_exp); + y = Q6_Vh_vaslacc_VhVhR(y, k_v, 10); + return Q6_V_vmux_QVV(q_underflow, zero_v, y); +} + #endif /* HVX_EXP_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h new file mode 100644 index 00000000000..c05bd0b8526 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h @@ -0,0 +1,232 @@ +#ifndef HVX_FA_KERNELS_H +#define HVX_FA_KERNELS_H + +#include +#include +#include "hvx-utils.h" + +// Little inner kernels for HVX + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +// This is a bit of a hack because the compiler is struggling to properly inline +// the default hvx_vec_f32_to_f16 with output into the local array. +static __attribute__((unused)) __attribute__((noinline)) void hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1) +{ + *(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1); +} + +// Dot product of two F16 vectors, accumulating to float +static inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) { + const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16 + const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]); + + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = HVX_OP_ADD_F32(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)); + rsum = HVX_OP_MUL_F32(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum)); + hvx_vec_store_u(r, 4, rsum); +} + +static inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y, + const uint8_t * restrict x, + const size_t stride_x, + const size_t nvec, + const size_t nloe) { + const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16 + const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16 + const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16 + const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16 + const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 + + HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + + uint32_t i = 0; + + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = vy[i]; + HVX_Vector x0_hf = vx0[i]; + HVX_Vector x1_hf = vx1[i]; + HVX_Vector x2_hf = vx2[i]; + HVX_Vector x3_hf = vx3[i]; + + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); + rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); + } + + if (nloe) { + // Load x (fp16) and zero-out unused elements + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]); + HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]); + HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]); + + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); + rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); + } + + HVX_Vector rsum0 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)); + HVX_Vector rsum1 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)); + HVX_Vector rsum2 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p)); + HVX_Vector rsum3 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p)); + + HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } }; + return hvx_vec_reduce_sum_f32x4(rsum0123); +} + +static inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y, + const uint8_t * restrict x, + const size_t stride_x, + const size_t n, + float s) { + + const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + const size_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector sums = Q6_V_vzero(); + const size_t stride_x_4 = stride_x * 4; + for (uint32_t j = 0; j < VLEN_FP32; j += 4) { + HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe); + HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32); + sums = Q6_V_vmux_QVV(pred, sums, sums_x4); + x += stride_x_4; + } + + return HVX_OP_MUL_F32(hvx_vec_splat_f32(s), sums); +} + +// MAD: y (F32) += x (F16) * s (F16) +static inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector S0 = hvx_vec_splat_f16(*s); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} + +// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16) +static inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1, + const __fp16 * restrict s0, const __fp16 * restrict s1, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0; + const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector S0 = hvx_vec_splat_f16(*s0); + HVX_Vector S1 = hvx_vec_splat_f16(*s1); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} + +static inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t n, HVX_Vector vs) { + assert((size_t) dst % 128 == 0); + assert((size_t) src % 128 == 0); + + const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src; + HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst; + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + uint32_t i = 0; + #pragma unroll(4) + for (; i < nvec; ++i) { + vdst[i] = HVX_OP_MUL_F32(vsrc[i], vs); + } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), HVX_OP_MUL_F32(vsrc[i], vs)); + } +} + +#endif /* HVX_FA_KERNELS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h index 52351b1039c..328a8311894 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h @@ -256,7 +256,7 @@ static inline void quantize_f16_f16_flat_kernel( // Dot kernels that consume flat (non-tiled) activations -static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -312,10 +312,14 @@ static void flat_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -397,11 +401,19 @@ static void flat_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -464,10 +476,14 @@ static void flat_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -561,11 +577,19 @@ static void flat_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -620,10 +644,14 @@ static void flat_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const v v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -704,11 +732,19 @@ static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -765,10 +801,14 @@ static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -851,11 +891,19 @@ static void flat_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -921,10 +969,14 @@ static void flat_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -1019,6 +1071,441 @@ static void flat_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +static inline void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_sf = y[i]; + HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_sf = x0[i]; + HVX_Vector r1_sf = x1[i]; + HVX_Vector c0_sf = y0[i]; + HVX_Vector c1_sf = y1[i]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + + HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); } + +static inline void vec_dot_f32_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { + const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; + const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector x_sf = vx[i]; + HVX_Vector y_sf = vy[i]; + + rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); + } + + if (nloe) { + HVX_Vector x_sf = vx[i]; + HVX_Vector y_sf = vy[i]; + + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + x_sf = Q6_V_vand_QV(bmask, x_sf); + y_sf = Q6_V_vand_QV(bmask, y_sf); + + rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); + } + + rsum = hvx_vec_reduce_sum_f32(rsum); + hvx_vec_store_u(&s[0], 4, rsum); +} + +#undef HVX_OP_ADD_F32 +#undef HVX_OP_MUL_F32 + +static inline void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); + hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + HVX_VectorPair rsum0_p = Q6_W_vzero(); + HVX_VectorPair rsum1_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = y[i]; + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + } + + HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); + HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + // Row sums (sf) - 4 accumulators for 2x2 tile + HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_hf = x0[i]; + HVX_Vector r1_hf = x1[i]; + HVX_Vector c0_hf = y0[i]; + HVX_Vector c1_hf = y1[i]; + + // Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1 + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + + HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); + HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); + HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); + HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 + hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 +} + +static inline void vec_dot_f16_f16_uu_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_UVector * restrict x = (const HVX_UVector *) vx; + const HVX_UVector * restrict y = (const HVX_UVector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]); + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); + hvx_vec_store_u(&s[0], 4, rsum); +} + +static inline void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { + const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; + const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + const HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + // Load y (fp32) and convert into fp16 + HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements + HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements + HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); + + // Load x (fp16) + HVX_Vector x_hf = vx[i]; + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + if (nloe) { + // Load y (fp32) and convert into fp16 + HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements + HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements + HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); + + // Load x (fp16) + HVX_Vector x_hf = vx[i]; + + // Zero-out unused elements + // Note that we need to clear both x and y because they may contain NANs + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + x_hf = Q6_V_vand_QV(bmask, x_hf); + y_hf = Q6_V_vand_QV(bmask, y_hf); + + HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + + rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + } + + // Convert into fp32 and reduce + rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); + hvx_vec_store_u(&s[0], 4, rsum); +} + +static inline void hvx_tensor_add_f32_grid( + const struct htp_tensor * restrict dst, + const struct htp_tensor * restrict src2, + uint32_t start_row, + uint32_t end_row, + uint32_t start_col, + uint32_t end_col, + const struct fastdiv_values * div_ne11_12, + const struct fastdiv_values * div_ne11 +) { + if (start_row >= end_row || start_col >= end_col) return; + const uint32_t nb1 = dst->nb[1]; // row stride in bytes + + const uint32_t ne11 = dst->ne[1]; + const uint32_t ne12 = dst->ne[2]; + const uint32_t ne11_12 = ne11 * ne12; + + const bool is_broadcast1 = (src2->ne[1] == 1); + const bool is_broadcast2 = (src2->ne[2] == 1); + const bool is_broadcast3 = (src2->ne[3] == 1); + + for (uint32_t r = start_row; r < end_row; r++) { + float * dst_row = (float *) ((uint8_t *) dst->data + r * nb1); + + uint32_t i13 = fastdiv(r, div_ne11_12); + uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11); + uint32_t i11 = r - i13 * ne11_12 - i12 * ne11; + + uint32_t i23 = is_broadcast3 ? 0 : i13; + uint32_t i22 = is_broadcast2 ? 0 : i12; + uint32_t i21 = is_broadcast1 ? 0 : i11; + + const float * src2_row = (const float *) ((const uint8_t *) src2->data + + i21 * src2->nb[1] + i22 * src2->nb[2] + i23 * src2->nb[3]); + + float * dst_ptr = &dst_row[start_col]; + const float * src2_ptr = &src2_row[start_col]; + int remaining = end_col - start_col; + while (remaining >= 32) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z); + dst_ptr += 32; + src2_ptr += 32; + remaining -= 32; + } + if (remaining > 0) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); + } + } +} + diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index bcb0b8f9e47..40b65aa3b55 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -378,7 +378,7 @@ static inline HVX_VectorPair accum_q8_0_32x2( return Q6_W_vcombine_VV(v_sum1, v_sum0); } -static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -401,10 +401,14 @@ static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -484,11 +488,19 @@ static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -519,10 +531,14 @@ static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -637,11 +653,19 @@ static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -663,10 +687,14 @@ static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -745,11 +773,19 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -773,10 +809,14 @@ static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); } - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -857,11 +897,19 @@ static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, floa v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); } - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } -static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows) { +static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -896,10 +944,14 @@ static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } } -static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows) { +static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y0_q = vy0; const uint8_t * restrict y1_q = vy1; @@ -1013,8 +1065,16 @@ static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, floa v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); - hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); - hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } } static inline void quantize_f32_q8_0_tiled_kernel( diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index 37f3e7b6fae..dd66dd84c95 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -3,6 +3,7 @@ #include "hvx-base.h" #include "hvx-inverse.h" +#include "hvx-exp.h" #define FAST_SIGMOID_LOG2F (0x3fb8aa3b) // 1.442695022 #define FAST_SIGMOID_C1 (0x3d009076) // 0.03138777 @@ -139,4 +140,42 @@ static inline void hvx_tanh_f32_aa(uint8_t * restrict dst, const uint8_t * restr hvx_tanh_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } +static inline HVX_Vector hvx_vec_fast_sigmoid_f16(HVX_Vector x_v) { + const HVX_Vector v_one = hvx_vec_splat_f16(1.0f); + const HVX_Vector v_neg_log2e = hvx_vec_splat_f16(-EXP_LOG2E_F); + const HVX_Vector em_mask = Q6_Vh_vsplat_R(0x7FFF); + + // Compute absolute value of x_v + HVX_Vector abs_x = Q6_V_vand_VV(x_v, em_mask); + + // Compute u = -abs_x * log2(e) <= 0. + HVX_Vector u = hvx_vec_mul_f16_f16(abs_x, v_neg_log2e); + + // Clamp input to prevent underflow in exp2 + const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f); + u = Q6_Vhf_vmax_VhfVhf(v_clamp_min, u); + + HVX_Vector exp_val = hvx_vec_exp2_f16(u); + HVX_Vector denom = hvx_vec_add_f16_f16(v_one, exp_val); + HVX_Vector sig_abs = hvx_vec_inverse_f16(denom); + + // check if x_v < 0 (using integer comparison on absolute value) + HVX_VectorPred is_neg = Q6_Q_vcmp_gt_VhVh(abs_x, x_v); + + // If x_v < 0, return 1.0f - sig_abs + HVX_Vector sig_neg = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(v_one, sig_abs)); + return Q6_V_vmux_QVV(is_neg, sig_neg, sig_abs); +} + +static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) { + // tanh(x) = 2 * sigmoid(2x) - 1 + const HVX_Vector v_two = hvx_vec_splat_f16(2.0f); + + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, v_two); + HVX_Vector sig2x = hvx_vec_fast_sigmoid_f16(x2); + + const HVX_Vector v_neg_one = hvx_vec_splat_f16(-1.0f); + return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one); +} + #endif /* HVX_SIGMOID_H */ diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index d76512ea4a3..bfb3e3138ea 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -575,6 +575,7 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) { static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { case HTP_OP_MUL_MAT: + case HTP_OP_MUL_MAT_ADD: return op_matmul(octx); case HTP_OP_MUL_MAT_ID: @@ -947,6 +948,8 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) { int op_status = HTP_STATUS_OK; uint32_t op_wakeup = n_ops / 2; // half-way throgh the batch + hmx_queue_wakeup(ctx->hmx_queue); + for (uint32_t i=0; i < n_ops; i++) { struct profile_data prof; @@ -975,6 +978,8 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) { } } + hmx_queue_suspend(ctx->hmx_queue); + struct htp_opbatch_rsp rsp; rsp.id = req.id; rsp.status = op_status; diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 81a0ffbebb8..1683131a813 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -9,6 +9,7 @@ #include #include +#include #include "hex-dma.h" #include "hvx-utils.h" @@ -19,10 +20,22 @@ #include "htp-ctx.h" #include "htp-ops.h" #include "matmul-ops.h" -#include "vtcm-utils.h" +#include "htp-vtcm.h" + +static void hvx_tensor_add_f32_grid( + const struct htp_tensor * restrict dst, + const struct htp_tensor * restrict src2, + uint32_t start_row, + uint32_t end_row, + uint32_t start_col, + uint32_t end_col, + const struct fastdiv_values * div_ne11_12, + const struct fastdiv_values * div_ne11 +); typedef struct { float *dst; + const float *src2; const float *activation; const __fp16 *weight; int m; @@ -31,6 +44,7 @@ typedef struct { int act_stride; int weight_stride; int dst_stride; + uint32_t src2_stride; int ne02; int ne03; int ne12; @@ -41,6 +55,8 @@ typedef struct { size_t src1_nb3; size_t dst_nb2; size_t dst_nb3; + size_t src2_nb2; + size_t src2_nb3; } hmx_mm_f16_f32_batched_params_t; struct htp_mm_context { @@ -61,11 +77,11 @@ struct htp_mm_context { void (*vec_dot_32x1)(const uint32_t n, float * restrict s, const void * restrict vx, - const void * restrict vy, uint32_t valid_rows); + const void * restrict vy, uint32_t valid_rows, + const float * restrict sz); // Precomputed values uint32_t src0_nrows_per_thread; - uint32_t src1_nrows_per_thread; struct fastdiv_values mm_div_ne12_ne1; struct fastdiv_values mm_div_ne1; @@ -73,11 +89,16 @@ struct htp_mm_context { struct fastdiv_values mm_div_r3; struct fastdiv_values mm_div_ne11; + // Per thread quant tasks // Precomputed block-parallel quantization values - uint32_t quant_ib_first[MAX_NUM_WORKERS]; - uint32_t quant_ib_last[MAX_NUM_WORKERS]; - uint32_t quant_r[MAX_NUM_WORKERS]; - uint32_t quant_c[MAX_NUM_WORKERS]; + worker_callback_t quant_task_func; + uint32_t quant_ib_first[MAX_NUM_WORKERS]; + uint32_t quant_ib_last[MAX_NUM_WORKERS]; + uint32_t quant_r[MAX_NUM_WORKERS]; + uint32_t quant_c[MAX_NUM_WORKERS]; + uint32_t n_quant_tasks; + uint32_t n_quant_rows_per_thread; + atomic_uint quant_barrier; // Fields for scattered mapping & HMX support in MUL_MAT_ID const uint32_t * matrix_row_counts; @@ -133,382 +154,7 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -#if __HVX_ARCH__ < 79 -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) -#else -#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) -#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) -#endif -static void vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_Vector * restrict x = (const HVX_Vector *) vx; - const HVX_Vector * restrict y = (const HVX_Vector *) vy; - - uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors - uint32_t nloe = n % VLEN_FP32; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); - rsum = HVX_OP_ADD_F32(rsum, prod); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); - rsum = HVX_OP_ADD_F32(rsum, prod); - } - - *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); -} - -static void vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y = (const HVX_Vector *) vy0; - - uint32_t nvec = n / VLEN_FP32; - uint32_t nloe = n % VLEN_FP32; - - HVX_Vector rsum0 = Q6_V_vzero(); - HVX_Vector rsum1 = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector y_sf = y[i]; - HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); - HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); - rsum0 = HVX_OP_ADD_F32(rsum0, prod0); - rsum1 = HVX_OP_ADD_F32(rsum1, prod1); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); - HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); - rsum0 = HVX_OP_ADD_F32(rsum0, prod0); - rsum1 = HVX_OP_ADD_F32(rsum1, prod1); - } - - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); - HVX_VectorAlias va; - va.v = rsum; - s0[0] = va.fp32[0]; - s0[1] = va.fp32[1]; -} - -static void vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; - const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; - - uint32_t nvec = n / VLEN_FP32; - uint32_t nloe = n % VLEN_FP32; - - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector r0_sf = x0[i]; - HVX_Vector r1_sf = x1[i]; - HVX_Vector c0_sf = y0[i]; - HVX_Vector c1_sf = y1[i]; - - r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); - r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); - r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); - r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - - HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); - HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); - - r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); - r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); - r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); - r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); - } - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - HVX_VectorAlias va0, va1; - va0.v = r0_r1_c0_sum; - va1.v = r0_r1_c1_sum; - s0[0] = va0.fp32[0]; - s0[1] = va0.fp32[1]; - s1[0] = va1.fp32[0]; - s1[1] = va1.fp32[1]; -} - -static void vec_dot_f32_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { - const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; - const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; - - uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors - uint32_t nloe = n % VLEN_FP32; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector x_sf = vx[i]; - HVX_Vector y_sf = vy[i]; - - rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); - } - - if (nloe) { - HVX_Vector x_sf = vx[i]; - HVX_Vector y_sf = vy[i]; - - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - x_sf = Q6_V_vand_QV(bmask, x_sf); - y_sf = Q6_V_vand_QV(bmask, y_sf); - - rsum = HVX_OP_ADD_F32(rsum, HVX_OP_MUL_F32(x_sf, y_sf)); - } - - rsum = hvx_vec_reduce_sum_f32(rsum); - hvx_vec_store_u(&s[0], 4, rsum); -} - -static void vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_Vector * restrict x = (const HVX_Vector *) vx; - const HVX_Vector * restrict y = (const HVX_Vector *) vy; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_VectorPair rsum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); - } - - HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); - hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); -} - -static void vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y = (const HVX_Vector *) vy0; - - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; - - HVX_VectorPair rsum0_p = Q6_W_vzero(); - HVX_VectorPair rsum1_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector y_hf = y[i]; - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - } - - HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); - HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); - hvx_vec_store_u(s0, 8, rsum); -} - -static void vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; - const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; - - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; - - // Row sums (sf) - 4 accumulators for 2×2 tile - HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector r0_hf = x0[i]; - HVX_Vector r1_hf = x1[i]; - HVX_Vector c0_hf = y0[i]; - HVX_Vector c1_hf = y1[i]; - - // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1 - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - - HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); - HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); - - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); - } - - HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); - HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); - HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); - HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 -} - -static void vec_dot_f16_f16_uu_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_UVector * restrict x = (const HVX_UVector *) vx; - const HVX_UVector * restrict y = (const HVX_UVector *) vy; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} - -static void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const void * restrict x, const void * restrict y) { - const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; - const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - const HVX_Vector zero = Q6_V_vzero(); - - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - if (nloe) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - // Zero-out unused elements - // Note that we need to clear both x and y because they may contain NANs - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - x_hf = Q6_V_vand_QV(bmask, x_hf); - y_hf = Q6_V_vand_QV(bmask, y_hf); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - // Convert into fp32 and reduce - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} #define htp_matmul_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ @@ -526,10 +172,10 @@ static void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const v const uint32_t ne12 = src1->ne[2]; \ const uint32_t ne13 = src1->ne[3]; \ \ - const uint32_t ne20 = src2->ne[0]; \ - const uint32_t ne21 = src2->ne[1]; \ - const uint32_t ne22 = src2->ne[2]; \ - const uint32_t ne23 = src2->ne[3]; \ + const uint32_t ne20 = src2 ? src2->ne[0] : 0; \ + const uint32_t ne21 = src2 ? src2->ne[1] : 0; \ + const uint32_t ne22 = src2 ? src2->ne[2] : 0; \ + const uint32_t ne23 = src2 ? src2->ne[3] : 0; \ \ const uint32_t ne0 = dst->ne[0]; \ const uint32_t ne1 = dst->ne[1]; \ @@ -558,6 +204,18 @@ static void vec_dot_f16_f32_uu_1x1(const uint32_t n, float * restrict s, const v uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ htp_matmul_tensors_preamble; +static inline void hvx_mm_run_quant_task(struct htp_mm_context * mmctx, unsigned int ith) { + if (mmctx->quant_task_func) { + if (ith < mmctx->n_quant_tasks) { + mmctx->quant_task_func(mmctx->n_quant_tasks, ith, mmctx); + atomic_fetch_sub(&mmctx->quant_barrier, 1); + } + while (atomic_load(&mmctx->quant_barrier) > 0) { + // spin + } + } +} + // *** matmul with support for 4d tensors and full broadcasting static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { @@ -631,6 +289,9 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) { } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start); + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, ir1_start, ir1_end, ir0_start, ir0_end, &mmctx->mm_div_ne12_ne1, &mmctx->mm_div_ne1); + } } #include "hmx-mm-kernels-tiled.h" @@ -648,10 +309,6 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ @@ -661,6 +318,7 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const size_t dst_row_size = nb1; \ const size_t src1_row_size = nb11; \ const size_t src1_stride = mmctx->vtcm_src1_stride; \ + const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; \ \ uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; \ uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; \ @@ -680,9 +338,17 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void uint32_t ct_end = (src0_end_row + 31) / 32; \ \ uint32_t push_ct = ct_start; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + if (src0_start_row < src0_end_row) { \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ } \ \ for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ @@ -702,7 +368,15 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void float * dst_ptr0 = &dst_row0[ct * 32]; \ float * dst_ptr1 = &dst_row1[ct * 32]; \ \ - DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows); \ + const float * src2_ptr0 = NULL; \ + const float * src2_ptr1 = NULL; \ + if (src2) { \ + const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((ir1+0) * src2_stride)); \ + const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((ir1+1) * src2_stride)); \ + src2_ptr0 = &src2_row0[ct * 32]; \ + src2_ptr1 = &src2_row1[ct * 32]; \ + } \ + DOT_2X2(ne10, dst_ptr0, dst_ptr1, w_tile, src1_col0, src1_col1, valid_rows, src2_ptr0, src2_ptr1); \ } \ \ for (; ir1 < src1_nrows; ++ir1) { \ @@ -710,7 +384,12 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); \ float * dst_ptr = &dst_row[ct * 32]; \ \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows); \ + const float * src2_ptr = NULL; \ + if (src2) { \ + const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + (ir1 * src2_stride)); \ + src2_ptr = &src2_row[ct * 32]; \ + } \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, src2_ptr); \ } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ @@ -731,10 +410,6 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ \ - if (src0_start_row >= src0_end_row) { \ - return; \ - } \ - \ struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \ \ const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \ @@ -752,6 +427,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void float * tmp = (float *) vtcm_dst_ptr; \ \ const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \ + \ const uint8_t * restrict src1_col = (const uint8_t *) src1_data; \ float * restrict dst_col = (float *) dst->data; \ \ @@ -767,9 +443,17 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void uint32_t ct_end = (src0_end_row + 31) / 32; \ \ uint32_t push_ct = ct_start; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + if (src0_start_row < src0_end_row) { \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ } \ \ for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ @@ -780,7 +464,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void valid_rows = MIN(32, MAX(0, valid_rows)); \ \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ - DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows); \ + DOT_2X1(ne10, dst_ptr, w_tile, src1_col, valid_rows, NULL); \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ if (push_ct < ct_end) { \ @@ -792,7 +476,24 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void \ int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; \ if (copy_cnt > 0) { \ - hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); \ + if (src2) { \ + float * dst_ptr = &dst_col[src0_start_row]; \ + const float * src2_ptr = (const float *) src2->data + src0_start_row; \ + float * tmp_ptr = tmp; \ + int remaining = copy_cnt; \ + while (remaining > 0) { \ + int n = MIN(remaining, 32); \ + HVX_Vector v_out = hvx_vmemu(tmp_ptr); \ + HVX_Vector v_z = hvx_vmemu(src2_ptr); \ + hvx_vec_store_u(dst_ptr, n * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); \ + dst_ptr += n; \ + src2_ptr += n; \ + tmp_ptr += n; \ + remaining -= n; \ + } \ + } else { \ + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); \ + } \ } \ } @@ -850,17 +551,22 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, const uint32_t start_row_kv = src0_nrows_per_thread_kv * ith; \ const uint32_t end_row_kv = MIN(start_row_kv + src0_nrows_per_thread_kv, src0_nrows_kv); \ \ - if (start_row_kv < end_row_kv) { \ - uint32_t ct_start_kv = start_row_kv / 32; \ - uint32_t ct_end_kv = (end_row_kv + 31) / 32; \ + uint32_t ct_start_kv = start_row_kv / 32; \ + uint32_t ct_end_kv = (end_row_kv + 31) / 32; \ \ - uint32_t push_ct = ct_start_kv; \ + uint32_t push_ct = ct_start_kv; \ + if (start_row_kv < end_row_kv) { \ for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end_kv; d++, push_ct++) { \ dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \ src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (start_row_kv < end_row_kv) { \ \ for (uint32_t ct = ct_start_kv; ct < ct_end_kv; ct++) { \ const uint8_t * w_tile_k = dma_queue_pop(dma_queue).dst; \ @@ -885,8 +591,8 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * dst_ptr0_v = &dst_row0_v[ct * 32]; \ float * dst_ptr1_v = &dst_row1_v[ct * 32]; \ \ - DOT_2X2(ne10, dst_ptr0_k, dst_ptr1_k, w_tile_k, src1_col0, src1_col1, valid_rows); \ - DOT_2X2(ne10, dst_ptr0_v, dst_ptr1_v, w_tile_v, src1_col0, src1_col1, valid_rows); \ + DOT_2X2(ne10, dst_ptr0_k, dst_ptr1_k, w_tile_k, src1_col0, src1_col1, valid_rows, NULL, NULL); \ + DOT_2X2(ne10, dst_ptr0_v, dst_ptr1_v, w_tile_v, src1_col0, src1_col1, valid_rows, NULL, NULL); \ } \ \ for (; ir1 < src1_nrows; ++ir1) { \ @@ -898,8 +604,8 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * restrict dst_row_v = (float *) (dst_v->data + (ir1 * dst_k_row_size)); \ float * dst_ptr_v = &dst_row_v[ct * 32]; \ \ - DOT_2X1(ne10, dst_ptr_k, w_tile_k, src1_col, valid_rows); \ - DOT_2X1(ne10, dst_ptr_v, w_tile_v, src1_col, valid_rows); \ + DOT_2X1(ne10, dst_ptr_k, w_tile_k, src1_col, valid_rows, NULL); \ + DOT_2X1(ne10, dst_ptr_v, w_tile_v, src1_col, valid_rows, NULL); \ } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ith); \ \ @@ -948,7 +654,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * dst_ptr0_q = &dst_row0_q[ct * 32]; \ float * dst_ptr1_q = &dst_row1_q[ct * 32]; \ \ - DOT_2X2(ne10, dst_ptr0_q, dst_ptr1_q, w_tile_q, src1_col0, src1_col1, valid_rows); \ + DOT_2X2(ne10, dst_ptr0_q, dst_ptr1_q, w_tile_q, src1_col0, src1_col1, valid_rows, NULL, NULL); \ } \ \ for (; ir1 < src1_nrows; ++ir1) { \ @@ -957,7 +663,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * restrict dst_row_q = (float *) (dst_q->data + (ir1 * dst_q_row_size)); \ float * dst_ptr_q = &dst_row_q[ct * 32]; \ \ - DOT_2X1(ne10, dst_ptr_q, w_tile_q, src1_col, valid_rows); \ + DOT_2X1(ne10, dst_ptr_q, w_tile_q, src1_col, valid_rows, NULL); \ } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ @@ -1019,11 +725,19 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, uint32_t ct_end = (src0_end_row + 31) / 32; \ \ uint32_t push_ct = ct_start; \ - for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \ - src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + if (src0_start_row < src0_end_row) { \ + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { \ + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, \ + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + d * tile_row_transfer_size_aligned, \ + src2_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); \ + } \ + } \ + \ + hvx_mm_run_quant_task(mmctx, ith); \ + \ + if (src0_start_row >= src0_end_row) { \ + return; \ } \ \ for (uint32_t ct = ct_start; ct < ct_end; ct++) { \ @@ -1049,8 +763,8 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * dst_ptr0_up = &dst_row0_up[ct * 32]; \ float * dst_ptr1_up = &dst_row1_up[ct * 32]; \ \ - DOT_2X2(ne10, dst_ptr0_gate, dst_ptr1_gate, w_tile_gate, src1_col0, src1_col1, valid_rows); \ - DOT_2X2(ne10, dst_ptr0_up, dst_ptr1_up, w_tile_up, src1_col0, src1_col1, valid_rows); \ + DOT_2X2(ne10, dst_ptr0_gate, dst_ptr1_gate, w_tile_gate, src1_col0, src1_col1, valid_rows, NULL, NULL); \ + DOT_2X2(ne10, dst_ptr0_up, dst_ptr1_up, w_tile_up, src1_col0, src1_col1, valid_rows, NULL, NULL); \ } \ \ for (; ir1 < src1_nrows; ++ir1) { \ @@ -1062,8 +776,8 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, float * restrict dst_row_up = (float *) (dst_up->data + (ir1 * dst_row_size)); \ float * dst_ptr_up = &dst_row_up[ct * 32]; \ \ - DOT_2X1(ne10, dst_ptr_gate, w_tile_gate, src1_col, valid_rows); \ - DOT_2X1(ne10, dst_ptr_up, w_tile_up, src1_col, valid_rows); \ + DOT_2X1(ne10, dst_ptr_gate, w_tile_gate, src1_col, valid_rows, NULL); \ + DOT_2X1(ne10, dst_ptr_up, w_tile_up, src1_col, valid_rows, NULL); \ } \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); \ \ @@ -1099,7 +813,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) { const uint32_t ne2 = src->ne[2]; \ const uint32_t ne3 = src->ne[3]; \ const uint32_t nrows = ne1 * ne2 * ne3; \ - const uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread; \ + const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ \ const uint32_t ir_first = nrows_per_thread * ith; \ if (ir_first >= nrows) { \ @@ -1115,7 +829,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) { const size_t dst_row_size = (dst_row_size_expr); \ const uint8_t * restrict src_data = (const uint8_t *) src->data + (src_row_size * ir_first); \ uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first); \ - uint8_t * restrict tmp_data = (uint8_t *) mmctx->vtcm_src0 + (mmctx->vtcm_src0_size_per_thread * ith); \ + uint8_t * restrict tmp_data = (uint8_t *) mmctx->vtcm_dst + (mmctx->vtcm_dst_size_per_thread * ith); \ kernel_fn(src_data, dst_data, tmp_data, ne0, ir_last - ir_first, src_row_size, dst_row_size); \ \ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ @@ -1140,7 +854,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo quantize_f32_q8_0_tiled_block_kernel( (const float *) src->data, mmctx->vtcm_src1, - (uint8_t *) mmctx->vtcm_src0 + (mmctx->vtcm_src0_size_per_thread * ith), + (uint8_t *) mmctx->vtcm_dst + (mmctx->vtcm_dst_size_per_thread * ith), src->ne[0], mmctx->quant_ib_first[ith], mmctx->quant_ib_last[ith], @@ -1164,7 +878,7 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo quantize_f32_q8_1_tiled_block_kernel( (const float *) src->data, mmctx->vtcm_src1, - (uint8_t *) mmctx->vtcm_src0 + (mmctx->vtcm_src0_size_per_thread * ith), + (uint8_t *) mmctx->vtcm_dst + (mmctx->vtcm_dst_size_per_thread * ith), src->ne[0], mmctx->quant_ib_first[ith], mmctx->quant_ib_last[ith], @@ -1230,11 +944,6 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; const size_t dst_row_size = nb1; @@ -1252,14 +961,21 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * restrict src0_row = (const uint8_t *) src0->data; // Prefill vtcm with src0 rows - #pragma unroll(4) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) { - break; + if (src0_start_row < src0_end_row) { + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); + } + + hvx_mm_run_quant_task(mmctx, ith); + + if (src0_start_row >= src0_end_row) { + return; } // Process src0 rows @@ -1311,6 +1027,9 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) { } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, 0, src1_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } } static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { @@ -1321,11 +1040,6 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_start_row = src0_nrows_per_thread * ith; const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; const size_t dst_row_size = nb1; @@ -1354,14 +1068,21 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t prefetch_mask = n_prefetch - 1; // Prefill vtcm with 2x src0 rows - #pragma unroll(2) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint32_t is0 = (ir0 - src0_start_row); - if (is0 >= n_prefetch) { - break; + if (src0_start_row < src0_end_row) { + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint32_t is0 = (ir0 - src0_start_row); + if (is0 >= n_prefetch) { + break; + } + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); + } + + hvx_mm_run_quant_task(mmctx, ith); + + if (src0_start_row >= src0_end_row) { + return; } // Process src0 rows @@ -1392,7 +1113,25 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } - hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, src0_end_row - src0_start_row); + int copy_cnt = src0_end_row - src0_start_row; + if (src2) { + float * dst_ptr = &dst_col[src0_start_row]; + const float * src2_ptr = (const float *) src2->data + src0_start_row; + float * tmp_ptr = tmp; + int remaining = copy_cnt; + while (remaining > 0) { + int n = MIN(remaining, 32); + HVX_Vector v_out = hvx_vmemu(tmp_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vec_store_u(dst_ptr, n * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); + dst_ptr += n; + src2_ptr += n; + tmp_ptr += n; + remaining -= n; + } + } else { + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); + } } #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * ids->ne[0] * ids->ne[1] + (i1)] @@ -1410,7 +1149,8 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_start_row = src0_nrows_per_thread * ith; const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - // no work for this thread + hvx_mm_run_quant_task(mmctx, ith); + if (src0_start_row >= src0_end_row) { return; } @@ -1476,7 +1216,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) { const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_stride); float * restrict dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0)); - mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows); + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); @@ -1498,7 +1238,8 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_start_row = src0_nrows_per_thread * ith; const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - // no work for this thread + hvx_mm_run_quant_task(mmctx, ith); + if (src0_start_row >= src0_end_row) { return; } @@ -1555,7 +1296,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) { valid_rows = MIN(32, MAX(0, valid_rows)); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); - mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows); + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, src1_col, valid_rows, NULL); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); if (push_ct < ct_end) { @@ -1625,9 +1366,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); size_t src1_row_size_padded; - worker_callback_t quant_job_func; + worker_callback_t quant_task_func; worker_callback_t matmul_job_func; - uint32_t n_quant_jobs = 1; + uint32_t n_quant_tasks = 1; if (src1_nrows > 1) { if (is_repacked) { switch (src0->type) { @@ -1660,7 +1401,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { switch (kparams->kernel_type) { case HTP_MM_KERNEL_HVX_F16_F16_VTCM: - quant_job_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16_flat : quantize_f16_f16_flat; + quant_task_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16_flat : quantize_f16_f16_flat; mmctx->type = "f16-f16"; mmctx->vec_dot_1x1 = vec_dot_f16_f16_aa_1x1; mmctx->vec_dot_2x1 = vec_dot_f16_f16_aa_2x1; @@ -1677,7 +1418,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->mm_div_r2 = kparams->div_r2; mmctx->mm_div_r3 = kparams->div_r3; need_quant = false; - quant_job_func = NULL; + quant_task_func = NULL; src1_row_size = nb11; break; @@ -1691,11 +1432,11 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { mmctx->mm_div_r3 = kparams->div_r3; src1_row_size = nb11; need_quant = false; - quant_job_func = NULL; + quant_task_func = NULL; break; case HTP_MM_KERNEL_HVX_F32_F32_VTCM: - quant_job_func = quantize_f32_f32_flat; + quant_task_func = quantize_f32_f32_flat; mmctx->type = "f32-f32"; mmctx->vec_dot_1x1 = vec_dot_f32_f32_aa_1x1; mmctx->vec_dot_2x1 = vec_dot_f32_f32_aa_2x1; @@ -1704,7 +1445,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { break; case HTP_MM_KERNEL_HVX_F32_F32_DDR: - quant_job_func = NULL; + quant_task_func = NULL; mmctx->type = "f32-f32"; mmctx->vec_dot_1x1 = vec_dot_f32_f32_uu_1x1; mmctx->mm_div_ne12_ne1 = kparams->div_ne12_ne1; @@ -1717,8 +1458,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { break; case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); if (src1_nrows > 1) { @@ -1755,55 +1496,44 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { const uint32_t total_nb = src1_nrows * nb; if (src1_nrows < octx->n_threads) { - n_quant_jobs = MIN(total_nb, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; - for (uint32_t ith = 0; ith < n_quant_jobs; ++ith) { - uint32_t ib_first = (total_nb * ith) / n_quant_jobs; - uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_jobs; + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; mmctx->quant_ib_first[ith] = ib_first; mmctx->quant_ib_last[ith] = ib_last; mmctx->quant_r[ith] = ib_first / nb; mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); break; } - size_t src0_sz = 0, src1_sz = 0, dst_sz = 0; - if (kparams->vtcm_src0_size > 0 || kparams->vtcm_src1_size > 0 || kparams->vtcm_dst_size > 0) { - src0_sz = kparams->vtcm_src0_size; - src1_sz = kparams->vtcm_src1_size; - dst_sz = kparams->vtcm_dst_size; - } else { - const uint32_t n_prefetch = kparams->n_prefetch; - assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - htp_mm_hvx_get_vtcm_sizes( - kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, - dst_row_size, src0_row_size, src1_row_size, n_prefetch, - &src0_sz, &src1_sz, &dst_sz - ); - } + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, + dst_row_size, src0_row_size, src1_row_size, kparams->n_prefetch, false, false, false); if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { - mmctx->vtcm_src1_size_per_thread = src1_sz; + mmctx->vtcm_src1_size_per_thread = L.src1_bytes; } else { - mmctx->vtcm_src1_size_per_thread = src1_sz / octx->n_threads; + mmctx->vtcm_src1_size_per_thread = L.src1_bytes / octx->n_threads; } - mmctx->vtcm_src0_size_per_thread = src0_sz / octx->n_threads; - mmctx->vtcm_dst_size_per_thread = dst_sz / octx->n_threads; + mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; + mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : (src1_sz + src0_sz + dst_sz); + size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, - src0_sz, src1_sz, dst_sz, vtcm_size); + L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); FARF(HIGH, "matmul-%s : %ux%ux%ux%u * %ux%ux%ux%u-> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], @@ -1815,10 +1545,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = vtcm_seq_alloc(&vtcm_ptr, src1_sz); - mmctx->vtcm_src0 = vtcm_seq_alloc(&vtcm_ptr, src0_sz); - mmctx->vtcm_dst = vtcm_seq_alloc(&vtcm_ptr, dst_sz); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); octx->src1_spad.src = NULL; octx->src0_spad.src = NULL; @@ -1831,8 +1561,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) { return HTP_STATUS_OK; if (need_quant) { - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); + mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->quant_task_func = quant_task_func; + mmctx->n_quant_tasks = n_quant_tasks; + atomic_init(&mmctx->quant_barrier, n_quant_tasks); + } else { + mmctx->quant_task_func = NULL; + mmctx->n_quant_tasks = 0; } const uint32_t n_matmul_jobs = octx->n_threads; @@ -1870,10 +1605,6 @@ static void hvx_mm_qkv_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - if (src0_start_row >= src0_end_row) { - return; - } - const size_t dst_k_row_size = dst_k->nb[1]; // K and V share output width const size_t dst_q_row_size = dst_q->nb[1]; // Q may be wider (GQA) const size_t src0_row_size = src0->nb[1]; @@ -1902,17 +1633,25 @@ static void hvx_mm_qkv_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * restrict src3_row = (const uint8_t *) src3->data; // Prefill spad with src0, src2, src3 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) { - break; + if (src0_start_row < src0_end_row) { + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), + src2_stride, src2_row_size, src2_row_size, 2); + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size), + src3_stride, src3_row_size, src3_row_size, 2); } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src3_ptr + is0 * src3_stride, src3_row + ir0 * src3_row_size), - src3_stride, src3_row_size, src3_row_size, 2); + } + + hvx_mm_run_quant_task(mmctx, ith); + + if (src0_start_row >= src0_end_row) { + return; } // Process rows @@ -2024,10 +1763,6 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) { const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - if (src0_start_row >= src0_end_row) { - return; - } - const size_t dst_row_size = dst_gate->nb[1]; const size_t src0_row_size = src0->nb[1]; const size_t src2_row_size = src2->nb[1]; @@ -2051,15 +1786,23 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) { const uint8_t * restrict src2_row = (const uint8_t *) src2->data; // Prefill spad with src0, src2 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= (int)n_prefetch) { - break; + if (src0_start_row < src0_end_row) { + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), + src2_stride, src2_row_size, src2_row_size, 2); } - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, src0_row_size, 2); - dma_queue_push(dma_queue, dma_make_ptr(vtcm_src2_ptr + is0 * src2_stride, src2_row + ir0 * src2_row_size), - src2_stride, src2_row_size, src2_row_size, 2); + } + + hvx_mm_run_quant_task(mmctx, ith); + + if (src0_start_row >= src0_end_row) { + return; } // Process rows @@ -2187,20 +1930,102 @@ static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); float *dst = st->dst + chunk_idx * st->dst_stride; - transfer_output_chunk_fp16_to_fp32(dst, st->vtcm_src, chunk_idx, chunk_size, st->n_cols, st->dst_stride, st->dst_cols); + const float *src2 = st->src2 ? (st->src2 + chunk_idx * st->src2_stride) : NULL; + transfer_output_chunk_fp16_to_fp32(dst, src2, st->vtcm_src, chunk_idx, chunk_size, st->n_cols, st->dst_stride, st->src2_stride, st->dst_cols); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); } +typedef struct { + const struct mmid_row_mapping *matrix_rows; + __fp16 *dst; + const float *src; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t k_block; + uint32_t k_stride; + uint32_t k_valid; + struct htp_thread_trace * traces; + struct htp_context * ctx; + float * vtcm_f32_act; + size_t vtcm_f32_act_bytes_per_thread; + uint32_t dma_step_rows; + uint32_t dma_step_rows_shift; +} activation_transfer_task_state_t; + +static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined( + dma_queue *dma_q, + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t k_valid, + float *thread_f32_act, + struct htp_thread_trace *tr, + uint32_t dma_step_rows, + uint32_t dma_step_rows_shift) { + + const uint32_t R = dma_step_rows; + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + + const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + + // Push step 0 + if (n_steps > 0 && n_rows > 0) { + uint32_t nrows_to_fetch = hex_smin(n_rows, R); + dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src), + k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + } + // Push step 1 (if valid) + if (n_steps > 1) { + uint32_t next_r = R * 1; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride; + float *next_buf = thread_f32_act + 1 * R * k_block; + dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src), + k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + } + } + for (uint32_t s = 0; s < n_steps; ++s) { + uint32_t r = s << dma_step_rows_shift; + float *curr_buf = thread_f32_act; + + if (r < n_rows) { + curr_buf = (float *) dma_queue_pop(dma_q).dst; + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + float *pair_buf = curr_buf + (p << 1) * k_block; + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); + + transfer_activation_row_pair_fp32_to_fp16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + + // Push step s + 2 + uint32_t next_s = s + 2; + uint32_t next_r = next_s << dma_step_rows_shift; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + const float *next_src = src + next_r * k_stride; + dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src), + k_block * sizeof(float), k_stride * sizeof(float), k_valid * sizeof(float), nrows_to_fetch); + } + } +} + static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; - int start_chunk_idx = i * st->n_chunks_per_task; - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, start_chunk_idx); - for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { int chunk_idx = task_id * st->n_chunks_per_task; size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); @@ -2209,18 +2034,55 @@ static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, const float *src = st->src + chunk_idx * st->k_stride; if (st->vtcm_f32_act) { - float *thread_f32_act = st->vtcm_f32_act + i * HTP_MM_DMA_ACT_MULTIPLIER * st->k_block; + float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread); transfer_activation_chunk_fp32_to_fp16_dma_pipelined( - st->ctx->dma[i], dst, src, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_f32_act + st->ctx->dma[i], dst, src, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift ); } else { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); transfer_activation_chunk_fp32_to_fp16(dst, src, chunk_size, st->k_block, st->k_stride, st->k_valid); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); } } - - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, start_chunk_idx); } +typedef struct { + const struct mmid_row_mapping *matrix_rows; + __fp16 *dst; + const float *src; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t k_block; + uint32_t cur_a; + uint32_t mapping_stride; + uint32_t ne11; + struct fastdiv_values ne11_div; + size_t nb11; + size_t nb12; + uint32_t start_row; + uint32_t cne1; + uint32_t k_valid; + struct htp_thread_trace *traces; +} activation_transfer_gathered_task_state_t; + +typedef struct { + const struct mmid_row_mapping *matrix_rows; + const __fp16 *vtcm_src; + float *dst; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t n_cols; + uint32_t cur_a; + uint32_t mapping_stride; + size_t dst_nb1; + size_t dst_nb2; + uint32_t start_row; + uint32_t cne1; + struct htp_thread_trace *traces; +} output_transfer_scattered_task_state_t; + static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { activation_transfer_gathered_task_state_t *st = data; struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL; @@ -2316,8 +2178,8 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( } } -static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const __fp16 *vtcm_src, - int n_rows, int n_cols, int dst_stride, int dst_cols, int n_threads) { +static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src, + int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) { assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); if (n_rows <= 0) return; @@ -2333,9 +2195,11 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, state.n_tot_chunks = n_tot_chunks; state.n_chunks_per_task = n_chunks_per_task; state.dst = dst; + state.src2 = src2; state.vtcm_src = vtcm_src; state.n_cols = n_cols; state.dst_stride = dst_stride; + state.src2_stride = src2_stride; state.dst_cols = dst_cols; state.traces = ctx->trace; @@ -2355,35 +2219,93 @@ static void transfer_activation_chunk_threaded( int k_stride, int n_threads, int k_valid, - float *vtcm_f32_act) { + float *vtcm_f32_act, + size_t vtcm_f32_act_bytes) { + if (n_rows <= 0) { + return; + } + assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) { + size_t thread_scratch_elements = vtcm_f32_act_bytes / (n_threads * sizeof(float)); + size_t dma_step_rows_max = (thread_scratch_elements / 2) / k_block; + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } else { + dma_step_rows = 2; + dma_step_rows_shift = 1; + } + } + activation_transfer_task_state_t state; - state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; - state.n_tot_chunks = n_tot_chunks; - state.n_chunks_per_task = n_chunks_per_task; - state.dst = dst; - state.src = src; - state.k_block = k_block; - state.k_stride = k_stride; - state.k_valid = k_valid; - state.traces = ctx->trace; - state.ctx = ctx; - state.vtcm_f32_act = vtcm_f32_act; + state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; + state.n_tot_chunks = n_tot_chunks; + state.n_chunks_per_task = n_chunks_per_task; + state.dst = dst; + state.src = src; + state.k_block = k_block; + state.k_stride = k_stride; + state.k_valid = k_valid; + state.traces = ctx->trace; + state.ctx = ctx; + state.vtcm_f32_act = vtcm_f32_act; + + int active_threads = hex_smin(n_threads, (int)state.n_tasks); + state.vtcm_f32_act_bytes_per_thread = (vtcm_f32_act_bytes / active_threads) & ~127u; + state.dma_step_rows = dma_step_rows; + state.dma_step_rows_shift = dma_step_rows_shift; if (state.n_tasks == 1 || n_threads == 1) { transfer_activation_chunk_worker_fn(1, 0, &state); } else { - int n_tasks = hex_smin((int) state.n_tasks, n_threads); - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, n_tasks); + worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads); } } +// --- Async HMX matmul job (for pipeline overlap) --- + +typedef struct { + __fp16 * output; + const __fp16 * activation; + const __fp16 * weight; + const __fp16 * scales; + uint32_t n_row_tiles; + uint32_t n_col_tiles; + uint32_t n_dot_tiles; +} hmx_matmul_job_t; + +static void hmx_matmul_worker_fn(void * data) { + hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; + FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); + core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); +} + +static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, + __fp16 * output, + const __fp16 * activation, + const __fp16 * weight, + const __fp16 * scales, + uint32_t n_row_tiles, + uint32_t n_col_tiles, + uint32_t n_dot_tiles) { + job->output = output; + job->activation = activation; + job->weight = weight; + job->scales = scales; + job->n_row_tiles = n_row_tiles; + job->n_col_tiles = n_col_tiles; + job->n_dot_tiles = n_dot_tiles; +} static int hmx_mm_2d_f32(struct htp_context *ctx, float *restrict dst, + const float *restrict src2, const float *activation, const uint8_t *weight, int m, int k, int n, @@ -2392,6 +2314,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, int weight_type, int k_valid, int dst_stride, + uint32_t src2_stride, int dst_cols, int m_chunk, int n_chunk, @@ -2425,57 +2348,47 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); + const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32); const size_t vec_dot_size = k * sizeof(__fp16); const size_t vtcm_budget = ctx->vtcm_size; + const uint32_t dma_dst_stride = is_quant ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; + size_t m_chunk_n_rows = m_chunk; size_t n_chunk_n_cols = n_chunk; size_t vtcm_used = vtcm_size; const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; - const size_t act_f32_size = hex_align_up((size_t)act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), HTP_MM_HMX_TILE_SIZE); + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, false, pipeline, act_threads, aligned_tile_size); - const size_t weight_area_size = is_quant - ? hex_align_up((n_chunk_n_cols / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) - : hex_align_up(n_chunk_n_cols * row_stride, HTP_MM_HMX_TILE_SIZE); - const size_t act_area_size = hex_align_up(m_chunk_n_rows * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t output_area_size = hex_align_up(m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); - - size_t scratch0_size, scratch1_size, scratch2_size; - scratch0_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HTP_MM_HMX_TILE_SIZE); // dequant buf 0 - scratch1_size = pipeline ? scratch0_size : 0; // dequant buf 1 - scratch2_size = pipeline ? output_area_size : 0; // output buf 1 - - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight_raw[2] = { NULL, NULL }; - if (weight_area_size) { - if (pipeline) { - vtcm_weight_raw[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - vtcm_weight_raw[1] = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - } else { - vtcm_weight_raw[0] = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - } - } - __fp16 *vtcm_f16_act = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, act_area_size); - float *vtcm_f32_act = (float *) vtcm_seq_alloc(&vtcm_ptr, act_f32_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_size); - void *vtcm_scratch1 = scratch1_size ? vtcm_seq_alloc(&vtcm_ptr, scratch1_size) : NULL; - void *vtcm_scratch2 = scratch2_size ? vtcm_seq_alloc(&vtcm_ptr, scratch2_size) : NULL; - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - - vtcm_used = vtcm_ptr - (uint8_t *) ctx->vtcm_base; + vtcm_used = L.total_bytes; if (vtcm_used > vtcm_budget) { FARF(ERROR, "hmx-mm-2d-precomputed: VTCM overflow: used %zu budget %zu, m %d k %d n %d mc %zu nc %zu", vtcm_used, vtcm_budget, m, k, n, m_chunk_n_rows, n_chunk_n_cols); return -1; } + uint8_t * const base = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight_raw[2] = { + VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]), + VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_weight[1], pipeline) + }; + + __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); + float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); + __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); + void *vtcm_scratch0 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[0]); + void *vtcm_scratch1 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_scratch[1], pipeline); + void *vtcm_scratch2 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_dst[1], pipeline); + __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - FARF(HIGH, "hmx-mm-2d-precomputed: standard : m %d k %d n %d wtype %d mc %zu nc %zu vtcm %zu/%zu", + FARF(HIGH, "hmx-mm-2d: m %d k %d n %d wtype %d mc %zu nc %zu vtcm %zu/%zu", m, k, n, weight_type, m_chunk_n_rows, n_chunk_n_cols, vtcm_used, vtcm_budget); int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); @@ -2490,124 +2403,128 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act); + transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) - const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - if (is_quant) { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), aligned_tile_size, tile_size, tile_size, (n_cols_A0 / 32) * n_k_tiles); - } else { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), row_stride, weight_stride, row_stride, n_cols_A0); - } + const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A0 = is_quant ? (n_cols_A0 / 32) * n_k_tiles : n_cols_A0; + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { - const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - if (is_quant) { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), aligned_tile_size, tile_size, tile_size, (n_cols_A1 / 32) * n_k_tiles); - } else { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), row_stride, weight_stride, row_stride, n_cols_A1); - } + const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A1 = is_quant ? (n_cols_A1 / 32) * n_k_tiles : n_cols_A1; + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[1], weight + n_chunk_n_cols * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } - // pop A0 -> dequantize A0 -> submit C0 - dma_queue_pop(ctx->dma[0]); - dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_weight_bufs[0], vtcm_weight_raw[0], - n_cols_A0, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); - - hmx_matmul_job_init(&job_slots[0], (__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_f16_act, - (__fp16 *) vtcm_weight_bufs[0], vtcm_scales, - hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), - hmx_ceil_div(n_cols_A0, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[0])); - - // Main loop: pop/dequantize A_{i+1} -> push A_{i+2} -> submit C_{i+1} -> wait C_i and store D_i + // Main loop: pop A_i -> dequantize A_i -> push A_{i+2} -> submit C_i -> wait C_{i-1} and store D_{i-1} for (int i = 0; i < n_chunk_cnt; ++i) { const size_t nc = i * n_chunk_n_cols; - const size_t nc_p1 = nc + 1 * n_chunk_n_cols; const size_t nc_p2 = nc + 2 * n_chunk_n_cols; const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); - const size_t n_cols_p1 = hex_smin(n - nc_p1, n_chunk_n_cols); const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); - // 1. pop A_{i+1} and dequantize it (if i+1 < n_chunk_cnt) - if (i + 1 < n_chunk_cnt) { - dma_queue_pop(ctx->dma[0]); - dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_weight_bufs[(i + 1) % 2], vtcm_weight_raw[(i + 1) % 2], - n_cols_p1, k, row_stride, weight_type, - n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); - } + // 1. pop A_i + void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + + // 2. dequantize A_i + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 2], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); - // 2. push A_{i+2} (if i+2 < n_chunk_cnt) + // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { - if (is_quant) { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[(i + 2) % 2], weight + nc_p2 * weight_stride), aligned_tile_size, tile_size, tile_size, (n_cols_p2 / 32) * n_k_tiles); - } else { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[(i + 2) % 2], weight + nc_p2 * weight_stride), row_stride, weight_stride, row_stride, n_cols_p2); - } + const uint32_t height_p2 = is_quant ? (n_cols_p2 / 32) * n_k_tiles : n_cols_p2; + dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_p2 * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } - // 3. submit C_{i+1} (if i+1 < n_chunk_cnt) - if (i + 1 < n_chunk_cnt) { - hmx_matmul_job_init(&job_slots[(i + 1) % 2], (__fp16 *) vtcm_output_bufs[(i + 1) % 2], - (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], - vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), - hmx_ceil_div(n_cols_p1, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[(i + 1) % 2])); + // 4. submit C_i + hmx_matmul_job_init(&job_slots[i % 2], (__fp16 *) vtcm_output_bufs[i % 2], + (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], + vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), + hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[i % 2])); + + // 5. wait C_{i-1} and store D_{i-1} (multi-thread HVX, parallel with C_i) + if (i > 0) { + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_prev = (i - 1) * n_chunk_n_cols; + const size_t n_cols_prev = hex_smin(n - nc_prev, n_chunk_n_cols); + float *output_chunk = dst + (mr * dst_stride + nc_prev); + const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc_prev) : NULL; + int chunk_dst_cols = dst_cols - (int)nc_prev; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); + } } + } - // 4. wait C_i and store D_i (multi-thread HVX, parallel with C_{i+1}) - hmx_queue_pop(ctx->hmx_queue); - float *output_chunk = dst + (mr * dst_stride + nc); - int chunk_dst_cols = dst_cols - (int)nc; - if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, vtcm_output_bufs[i % 2], n_rows, n_cols, dst_stride, chunk_dst_cols, n_threads); - } + // Epilogue: wait C_{last} and store D_{last} + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_last = (n_chunk_cnt - 1) * n_chunk_n_cols; + const size_t n_cols_last = hex_smin(n - nc_last, n_chunk_n_cols); + float *output_chunk = dst + (mr * dst_stride + nc_last); + const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc_last) : NULL; + int chunk_dst_cols = dst_cols - (int)nc_last; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); } } - hmx_queue_suspend(ctx->hmx_queue); } else { - // --- Synchronous Un-pipelined loop (m <= 32 or fallback) --- - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + // --- Synchronous loop (m <= 32 or fallback) --- + hmx_matmul_job_t job; for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act); + transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes); + + // A0: Pre-fetch the first weight chunk (nc = 0) + if (n > 0) { + const size_t n_cols = hex_smin(n, n_chunk_n_cols); + const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + } for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); - // A: Weight DMA (Synchronous) - if (is_quant) { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight + nc * weight_stride), aligned_tile_size, tile_size, tile_size, (n_cols / 32) * n_k_tiles); - } else { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight_raw[0], weight + nc * weight_stride), row_stride, weight_stride, row_stride, n_cols); - } - dma_queue_pop(ctx->dma[0]); + // A: Wait for weight DMA + void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_scratch0, vtcm_weight_raw[0], + ctx, vtcm_scratch0, curr_raw, n_cols, k, row_stride, weight_type, n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); - // C: HMX Compute (Synchronous) - core_dot_chunk_fp16(vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + // Start weight DMA for the next chunk early + const size_t nc_next = nc + n_chunk_n_cols; + if (nc_next < n) { + const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); + const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; + dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + } + + // C: HMX Compute (Queue-based) + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); // D: Output Store float *output_chunk = dst + (mr * dst_stride + nc); + const float *src2_chunk = src2 ? (src2 + mr * src2_stride + nc) : NULL; int chunk_dst_cols = dst_cols - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output_chunk, vtcm_output, n_rows, n_cols, dst_stride, chunk_dst_cols, n_threads); + transfer_output_chunk_threaded(ctx, output_chunk, src2_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); } } } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); } return 0; @@ -2644,6 +2561,13 @@ static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_f16_f32_batched_params_t (size_t) dst_b3 * params->dst_nb3); } +static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_params_t *params, + int src2_b2, int src2_b3) { + return params->src2 ? (const float *) ((const uint8_t *) params->src2 + + (size_t) src2_b2 * params->src2_nb2 + + (size_t) src2_b3 * params->src2_nb3) : NULL; +} + static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params, int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) { @@ -2651,11 +2575,12 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx, for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { ret = hmx_mm_2d_f32(ctx, hmx_mm_dst_batch_ptr(params, b2, b3), + hmx_mm_src2_batch_ptr(params, b2, b3), hmx_mm_activation_batch_ptr(params, b2, b3), (const uint8_t *)hmx_mm_weight_batch_ptr(params, b2, b3), params->m, params->k, params->n, params->act_stride, params->weight_stride * (int)sizeof(__fp16), - HTP_TYPE_F16, params->k, params->n, params->n, + HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n, m_chunk, n_chunk, pipeline, n_threads, act_threads, 0, 0, vtcm_size); } @@ -2692,37 +2617,34 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ size_t n_chunk_n_cols = n_chunk; size_t vtcm_used = vtcm_size; - const size_t act_head_stride = m_chunk_n_rows * (size_t) params->k; // fp16 elements between heads - const size_t weight_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t activation_area_size = hex_align_up(group_size * m_chunk_n_rows * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t output_area_size = hex_align_up(m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); - const size_t scratch_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - __fp16 *vtcm_f16_act = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, activation_area_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - void *vtcm_scratch1 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - float *vtcm_f32_act = use_dma_activation ? (float *) vtcm_seq_alloc(&vtcm_ptr, f32_scratch_size) : NULL; - - if ((size_t) (vtcm_ptr - (uint8_t *) ctx->vtcm_base) > vtcm_budget) { + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, use_dma_activation, false, act_threads, 0); + + if (L.total_bytes > vtcm_budget) { FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size); } + uint8_t * const base = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight = VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]); + __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); + __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); + void *vtcm_scratch0 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[0]); + void *vtcm_scratch1 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[1]); + __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); + float *vtcm_f32_act = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_act_f32, use_dma_activation); + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 FARF(HIGH, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, params->m, params->k, params->n, group_size, params->ne13, m_chunk_n_rows, n_chunk_n_cols, - (size_t) (vtcm_ptr - (uint8_t *) ctx->vtcm_base), vtcm_budget); + L.total_bytes, vtcm_budget); const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_matmul_job_t job; for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { @@ -2739,63 +2661,59 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ // thrashing from HVX loads at large strides. for (int g = 0; g < group_size; ++g) { const float *activation_chunk = hmx_mm_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride; - __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * act_head_stride; - if (use_dma_activation) { - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - activation_chunk, (int) n_rows, - params->k, params->act_stride, act_threads, params->k, vtcm_f32_act); - } else { - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - activation_chunk, (int) n_rows, - params->k, params->act_stride, act_threads, params->k, NULL); - } + __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; + transfer_activation_chunk_threaded(ctx, vtcm_act_g, + activation_chunk, (int) n_rows, + params->k, params->act_stride, act_threads, params->k, vtcm_f32_act, L.act_f32_bytes); } - void *buf_curr = vtcm_scratch0; - void *buf_next = vtcm_scratch1; - + // Prologue: Push A0 and A1 (if exists) { const size_t n_cols_first = hex_smin((size_t) params->n, n_chunk_n_cols); - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_curr, weight_group), + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_scratch0, weight_group), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); } + if (n_chunk_n_cols < (size_t) params->n) { + const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols); + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_scratch1, weight_group + params->weight_stride), + fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second); + } for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { - const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); + const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HTP_MM_HMX_TILE_N_COLS); { - dma_queue_pop(ctx->dma[0]); + void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + + hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) curr_raw, n_cols, params->k, params->k, 0, n_cols); - const size_t nc_next = nc + n_chunk_n_cols; + const size_t nc_next = nc + n_chunk_n_cols * 2; if (nc_next < (size_t) params->n) { const size_t n_cols_next = hex_smin((size_t) params->n - nc_next, n_chunk_n_cols); const __fp16 *next_weight_chunk = weight_group + nc_next * params->weight_stride; - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_next, next_weight_chunk), + dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, next_weight_chunk), fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); } - - hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) buf_curr, n_cols, params->k, params->k, 0, n_cols); - hex_swap_ptr(&buf_curr, &buf_next); } // Reuse the interleaved weight for every q_head in this GQA group for (int g = 0; g < group_size; ++g) { - struct htp_thread_trace * tr = &ctx->trace[HTP_MAX_NTHREADS]; - htp_trace_event_start(tr, HTP_TRACE_EVT_HMX_COMP, g); { - const __fp16 * vtcm_act_g = vtcm_f16_act + (size_t) g * act_head_stride; - core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, - params->k / 32); + const __fp16 * vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; + hmx_matmul_job_init(&job, vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, params->k / 32); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HMX_COMP, g); { float *output = hmx_mm_dst_batch_ptr(params, b2_base + g, b3) + mr * params->dst_stride + nc; + const float *src2_chunk = params->src2 ? (hmx_mm_src2_batch_ptr(params, b2_base + g, b3) + mr * params->src2_stride + nc) : NULL; int chunk_dst_cols = params->n - (int)nc; if (chunk_dst_cols > 0) { - transfer_output_chunk_threaded(ctx, output, vtcm_output, (int) n_rows, (int) n_cols, params->dst_stride, chunk_dst_cols, ctx->n_threads); + transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols, + params->dst_stride, params->src2_stride, chunk_dst_cols, ctx->n_threads); } } } @@ -2804,8 +2722,6 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ } } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - return 0; } @@ -2956,6 +2872,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, int tile_size = htp_mm_get_weight_tile_size(weight_type); int aligned_tile_size = htp_mm_get_weight_aligned_tile_size(weight_type); + const uint32_t dma_dst_stride = is_quant ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_quant ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_quant ? tile_size : row_stride; + const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; const size_t weight_row_stride = is_quant ? qweight_row_stride : row_stride; @@ -2978,12 +2898,12 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, size_t scratch0_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = weight_area_size ? (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size) : NULL; - __fp16 *vtcm_f16_act = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, act_area_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_size); - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); + uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight = weight_area_size ? (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size) : NULL; + __fp16 *vtcm_f16_act = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, act_area_size); + __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); + void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_size); + __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); vtcm_used = vtcm_ptr - (uint8_t *) ctx->vtcm_base; if (vtcm_used > vtcm_budget) { @@ -2993,7 +2913,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_matmul_job_t job; for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin(m_padded - mr, m_chunk_n_rows); @@ -3003,39 +2923,52 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k, matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid); + // A0: Pre-fetch the first weight chunk (nc = 0) + if (n > 0) { + const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols); + const uint32_t height = is_quant ? (n_cols / 32) * n_k_tiles : n_cols; + dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight, weight), + dma_dst_stride, dma_src_stride, dma_width_bytes, height); + } + for (size_t nc = 0; nc < (size_t) n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); - if (is_quant) { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight, weight + nc * weight_stride), aligned_tile_size, tile_size, tile_size, (n_cols / 32) * n_k_tiles); - } else { - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_weight, weight + nc * weight_stride), row_stride, weight_stride, row_stride, n_cols); - } - dma_queue_pop(ctx->dma[0]); + // A: Wait for weight DMA + void * curr_raw = dma_queue_pop(ctx->dma[0]).dst; + // B: Weight Dequantize (Threaded) dequantize_tiled_weight_chunk_to_fp16_tiles( - ctx, vtcm_scratch0, vtcm_weight, + ctx, vtcm_scratch0, curr_raw, n_cols, k, row_stride, weight_type, n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads ); - struct htp_thread_trace * tr = &ctx->trace[HTP_MAX_NTHREADS]; - htp_trace_event_start(tr, HTP_TRACE_EVT_HMX_COMP, nc); - core_dot_chunk_fp16(vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HMX_COMP, nc); + // Start weight DMA for the next chunk early + const size_t nc_next = nc + n_chunk_n_cols; + if (nc_next < (size_t) n) { + const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols); + const uint32_t height_next = is_quant ? (n_cols_next / 32) * n_k_tiles : n_cols_next; + dma_queue_push(ctx->dma[0], dma_make_ptr(curr_raw, weight + nc_next * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + } + // C: HMX Compute (Queue-based) + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, vtcm_scratch0, vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); + + // D: Output Store transfer_output_chunk_scattered_threaded( ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols, matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads); } } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); return 0; } - // --- Dispatchers and Public Entry Points --- static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams) { @@ -3051,11 +2984,23 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k return HTP_STATUS_OK; } + const float * src2_ptr = NULL; + uint32_t src2_stride = 0; + size_t src2_nb2 = 0; + size_t src2_nb3 = 0; + if (src2) { + src2_ptr = (const float *) src2->data; + src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); + src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; + src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; + } + int ret = -1; const int n_threads = MIN(kparams->n_threads, (int) octx->n_threads); if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { hmx_mm_f16_f32_batched_params_t batch_params = { .dst = (float *) dst->data, + .src2 = src2_ptr, .activation = (float *) src1->data, .weight = (const __fp16 *) src0->data, .m = m_total, @@ -3064,6 +3009,7 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k .act_stride = act_stride, .weight_stride = wgt_stride, .dst_stride = (int) (dst->nb[1] / sizeof(float)), + .src2_stride = src2_stride, .ne02 = ne02, .ne03 = ne03, .ne12 = ne12, @@ -3074,6 +3020,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k .src1_nb3 = src1->nb[3], .dst_nb2 = dst->nb[2], .dst_nb3 = dst->nb[3], + .src2_nb2 = src2_nb2, + .src2_nb3 = src2_nb3, }; ret = hmx_mm_f16_f32_batched(octx->ctx, &batch_params, kparams->m_chunk, kparams->n_chunk, @@ -3082,9 +3030,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k kparams->vtcm_size); } else { ret = hmx_mm_2d_f32( - octx->ctx, (float*) dst->data, (float*) src1->data, (const uint8_t *) src0->data, + octx->ctx, (float*) dst->data, src2_ptr, (float*) src1->data, (const uint8_t *) src0->data, m_total, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0], - (int)(dst->nb[1] / sizeof(float)), (int)dst->ne[0], + (int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0], kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, kparams->n_act_threads, kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size @@ -3145,7 +3093,7 @@ static int hmx_mm_op_matmul_id( return HTP_STATUS_OK; } -static int hvx_mm_op_matmul_id( +static int hvx_mm_matmul_id( struct htp_ops_context * octx, struct htp_mm_context * mmctx, size_t src0_row_size_padded, @@ -3163,41 +3111,33 @@ static int hvx_mm_op_matmul_id( const uint32_t nb = (ne10 + qk - 1) / qk; const uint32_t total_nb = src1_nrows * nb; - worker_callback_t quant_job_func; - uint32_t n_quant_jobs = 1; + worker_callback_t quant_task_func; + uint32_t n_quant_tasks = 1; if (src1_nrows < octx->n_threads) { - n_quant_jobs = MIN(total_nb, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; - for (uint32_t ith = 0; ith < n_quant_jobs; ++ith) { - uint32_t ib_first = (total_nb * ith) / n_quant_jobs; - uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_jobs; + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; mmctx->quant_ib_first[ith] = ib_first; mmctx->quant_ib_last[ith] = ib_last; mmctx->quant_r[ith] = ib_first / nb; mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - // Scratchpad sizes are computed on the host (htp_mm_hvx_id_get_vtcm_sizes) and passed in. - // The ID layout is routing-independent, so the host has exact visibility -- consume it here - // rather than recomputing, to keep host budgeting and device allocation in lockstep. - size_t src0_sz = kparams->vtcm_src0_size; - size_t src1_sz = kparams->vtcm_src1_size; - size_t src2_sz = 0; // mapping lives in DDR - size_t dst_sz = 0; // ID kernels scatter straight to DDR - size_t vtcm_size = kparams->vtcm_size; + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads, + 0, src0_row_size, src1_row_size, kparams->n_prefetch, true, false, false); - size_t src0_sz_per_thread = src0_sz / octx->n_threads; - size_t src1_sz_per_thread = src1_sz; - size_t src2_sz_per_thread = 0; - size_t dst_sz_per_thread = 0; + size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; - FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size %zu dst-spad-size %zu (%zu)\n", mmctx->type, - src0_sz, src1_sz, src2_sz, dst_sz, vtcm_size); + FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, + L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size); FARF(HIGH, "matmul-id-%s : %ux%ux%ux%u * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], @@ -3211,11 +3151,11 @@ static int hvx_mm_op_matmul_id( return HTP_STATUS_VTCM_TOO_SMALL; } - uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = vtcm_seq_alloc(&vtcm_ptr, src1_sz); - mmctx->vtcm_src0 = vtcm_seq_alloc(&vtcm_ptr, src0_sz); - mmctx->vtcm_src2 = vtcm_seq_alloc(&vtcm_ptr, src2_sz); - mmctx->vtcm_dst = vtcm_seq_alloc(&vtcm_ptr, dst_sz); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_src2 = NULL; + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); octx->src1_spad.src = NULL; octx->src0_spad.src = NULL; @@ -3225,13 +3165,15 @@ static int hvx_mm_op_matmul_id( mmctx->vtcm_src0_stride = src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = src0_sz_per_thread; - mmctx->vtcm_src1_size_per_thread = src1_sz_per_thread; - mmctx->vtcm_src2_size_per_thread = src2_sz_per_thread; - mmctx->vtcm_dst_size_per_thread = dst_sz_per_thread; + mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; + mmctx->vtcm_src1_size_per_thread = L.src1_bytes; + mmctx->vtcm_src2_size_per_thread = 0; + mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); + mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->quant_task_func = quant_task_func; + mmctx->n_quant_tasks = n_quant_tasks; + atomic_init(&mmctx->quant_barrier, n_quant_tasks); const uint32_t n_matmul_jobs = octx->n_threads; worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs); @@ -3259,7 +3201,7 @@ int op_matmul_id(struct htp_ops_context * octx) { const uint32_t src0_nrows = ne01; // per expert const uint32_t src1_nrows = ne11 * ne12 * ne13; - worker_callback_t quant_job_func; + worker_callback_t quant_task_func; worker_callback_t matmul_id_job_func = src1_nrows > 1 ? hvx_mm_id : hvx_mv_id; // Compute src0_nrows_per_thread @@ -3329,7 +3271,7 @@ int op_matmul_id(struct htp_ops_context * octx) { return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping); } - return hvx_mm_op_matmul_id(octx, mmctx, src0_row_size_padded, src1_nrows, matmul_id_job_func, mapping_buf, must_free_mapping); + return hvx_mm_matmul_id(octx, mmctx, src0_row_size_padded, src1_nrows, matmul_id_job_func, mapping_buf, must_free_mapping); } int op_matmul_qkv(struct htp_ops_context * octx) { @@ -3373,25 +3315,25 @@ int op_matmul_qkv(struct htp_ops_context * octx) { const uint32_t nb = (src1->ne[0] + qk - 1) / qk; const uint32_t total_nb = src1_nrows * nb; - worker_callback_t quant_job_func; - uint32_t n_quant_jobs = 1; + worker_callback_t quant_task_func; + uint32_t n_quant_tasks = 1; if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; } else if (src1_nrows < octx->n_threads) { - n_quant_jobs = MIN(total_nb, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; - for (uint32_t ith = 0; ith < n_quant_jobs; ++ith) { - uint32_t ib_first = (total_nb * ith) / n_quant_jobs; - uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_jobs; + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; mmctx->quant_ib_first[ith] = ib_first; mmctx->quant_ib_last[ith] = ib_last; mmctx->quant_r[ith] = ib_first / nb; mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } size_t src1_row_size; @@ -3401,17 +3343,11 @@ int op_matmul_qkv(struct htp_ops_context * octx) { src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]); } - // Set up scratchpads using precomputed sizes from the host - size_t src0_sz = kparams->vtcm_src0_size; - size_t src1_sz = kparams->vtcm_src1_size; - size_t src2_sz = kparams->vtcm_src2_size; - size_t src3_sz = kparams->vtcm_src3_size; - size_t vtcm_size = kparams->vtcm_size; + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads, + 0, src0_row_size, src1_row_size, kparams->n_prefetch, false, true, false); - size_t src0_sz_per_thread = src0_sz / octx->n_threads; - size_t src1_sz_per_thread = src1_sz; - size_t src2_sz_per_thread = src2_sz / octx->n_threads; - size_t src3_sz_per_thread = src3_sz / octx->n_threads; + size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-qkv: current VTCM reservation %zu is too small, needed %zu\n", @@ -3419,33 +3355,37 @@ int op_matmul_qkv(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = vtcm_seq_alloc(&vtcm_ptr, src1_sz); - mmctx->vtcm_src0 = vtcm_seq_alloc(&vtcm_ptr, src0_sz); - mmctx->vtcm_src2 = vtcm_seq_alloc(&vtcm_ptr, src2_sz); - mmctx->vtcm_src3 = vtcm_seq_alloc(&vtcm_ptr, src3_sz); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2); + mmctx->vtcm_src3 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src3); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); octx->src1_spad.src = NULL; octx->src0_spad.src = NULL; octx->src2_spad.src = NULL; octx->src3_spad.src = NULL; + octx->dst_spad.src = NULL; mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src3_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = src0_sz_per_thread; - mmctx->vtcm_src1_size_per_thread = src1_sz_per_thread; - mmctx->vtcm_src2_size_per_thread = src2_sz_per_thread; - mmctx->vtcm_src3_size_per_thread = src3_sz_per_thread; + mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; + mmctx->vtcm_src1_size_per_thread = L.src1_bytes; + mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; + mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads; + mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return HTP_STATUS_OK; - // Run quantization once - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); + mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->quant_task_func = quant_task_func; + mmctx->n_quant_tasks = n_quant_tasks; + atomic_init(&mmctx->quant_barrier, n_quant_tasks); // Run fused matmul const uint32_t n_matmul_jobs = octx->n_threads; @@ -3517,25 +3457,25 @@ int op_matmul_ffn(struct htp_ops_context * octx) { const uint32_t nb = (src1->ne[0] + qk - 1) / qk; const uint32_t total_nb = src1_nrows * nb; - worker_callback_t quant_job_func; - uint32_t n_quant_jobs = 1; + worker_callback_t quant_task_func; + uint32_t n_quant_tasks = 1; if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat; } else if (src1_nrows < octx->n_threads) { - n_quant_jobs = MIN(total_nb, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; - for (uint32_t ith = 0; ith < n_quant_jobs; ++ith) { - uint32_t ib_first = (total_nb * (ith + 0)) / n_quant_jobs; - uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_jobs; + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * (ith + 0)) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; mmctx->quant_ib_first[ith] = ib_first; mmctx->quant_ib_last[ith] = ib_last; mmctx->quant_r[ith] = ib_first / nb; mmctx->quant_c[ith] = ib_first % nb; } } else { - n_quant_jobs = MIN(src1_nrows, octx->n_threads); - quant_job_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; + n_quant_tasks = MIN(src1_nrows, octx->n_threads); + quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; } size_t src1_row_size; @@ -3545,44 +3485,44 @@ int op_matmul_ffn(struct htp_ops_context * octx) { src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(src1->ne[0]) : htp_mm_q8_0_tiled_row_size(src1->ne[0]); } - // Set up scratchpads using precomputed sizes from the host - size_t src0_sz = kparams->vtcm_src0_size; - size_t src1_sz = kparams->vtcm_src1_size; - size_t src2_sz = kparams->vtcm_src2_size; - size_t vtcm_size = kparams->vtcm_size; + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, src1->ne[0], src1_nrows, octx->n_threads, + 0, src0_row_size, src1_row_size, kparams->n_prefetch, false, false, true); - size_t src0_sz_per_thread = src0_sz / octx->n_threads; - size_t src1_sz_per_thread = src1_sz; - size_t src2_sz_per_thread = src2_sz / octx->n_threads; + size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes; if (octx->ctx->vtcm_size < vtcm_size) { FARF(ERROR, "matmul-ffn: current VTCM reservation %zu is too small, needed %zu\n", octx->ctx->vtcm_size, vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } - uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; - mmctx->vtcm_src1 = vtcm_seq_alloc(&vtcm_ptr, src1_sz); - mmctx->vtcm_src0 = vtcm_seq_alloc(&vtcm_ptr, src0_sz); - mmctx->vtcm_src2 = vtcm_seq_alloc(&vtcm_ptr, src2_sz); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src2); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); octx->src1_spad.src = NULL; octx->src0_spad.src = NULL; octx->src2_spad.src = NULL; + octx->dst_spad.src = NULL; mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src2_stride = is_repacked ? 0 : src0_row_size_padded; mmctx->vtcm_src1_stride = src1_row_size; - mmctx->vtcm_src0_size_per_thread = src0_sz_per_thread; - mmctx->vtcm_src1_size_per_thread = src1_sz_per_thread; - mmctx->vtcm_src2_size_per_thread = src2_sz_per_thread; + mmctx->vtcm_src0_size_per_thread = L.src0_bytes / octx->n_threads; + mmctx->vtcm_src1_size_per_thread = L.src1_bytes; + mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads; + mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads; if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return HTP_STATUS_OK; - // Run quantization once - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); + mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->quant_task_func = quant_task_func; + mmctx->n_quant_tasks = n_quant_tasks; + atomic_init(&mmctx->quant_barrier, n_quant_tasks); // Run fused matmul const uint32_t n_matmul_jobs = octx->n_threads; diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index a94d5430dab..2e131bc3d02 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -6,6 +6,7 @@ #include "htp-ops.h" #include "hex-fastdiv.h" #include "hex-common.h" +#include "htp-vtcm.h" #ifdef __cplusplus extern "C" { @@ -44,7 +45,7 @@ extern "C" { // --- DMA Activation Transfer Configuration --- #define HTP_MM_DMA_ACT_ROWS_PER_STEP 2 -#define HTP_MM_DMA_ACT_MULTIPLIER 4 +#define HTP_MM_DMA_ACT_MULTIPLIER (2 * HTP_MM_DMA_ACT_ROWS_PER_STEP) enum htp_mm_kernel_type { HTP_MM_KERNEL_UNSUPPORTED = 0, @@ -295,210 +296,351 @@ static inline void htp_mm_hmx_get_batched_chunk_costs( *size_per_mn_out = sizeof(uint16_t); } -static inline size_t htp_mm_hmx_get_2d_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size -) { - const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; - const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32); - const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k); - const size_t vec_dot_size = k * sizeof(uint16_t); - - const size_t act_f32_size = htp_mm_round_up(act_threads * 4 * k * sizeof(float), HTP_MM_HMX_TILE_SIZE); - size_t weight_area_size = is_quant - ? htp_mm_round_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) - : htp_mm_round_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); - if (pipeline) { - weight_area_size *= 2; - } - const size_t act_area_size = htp_mm_round_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t output_area_size = htp_mm_round_up(mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); +struct htp_mm_hmx_vtcm_layout { + // Byte offsets from vtcm_base for each region + size_t off_weight[2]; // [1] is only used when pipelined + size_t off_act; + size_t off_act_f32; // fp32 activation conversion scratch + size_t off_dst[2]; // [1] is only used when pipelined + size_t off_scratch[2]; // dequantization scratch pads + size_t off_scales; // HMX scales (256 bytes) + + // Cached sizes of regions for HMX kernel use + size_t weight_area_bytes; + size_t act_area_bytes; + size_t act_f32_bytes; + size_t output_area_bytes; + size_t scratch_bytes[2]; + size_t act_head_stride; + + size_t total_bytes; +}; - size_t scratch0_size = htp_mm_round_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - size_t scratch1_size = pipeline ? scratch0_size : 0; - size_t scratch2_size = pipeline ? output_area_size : 0; +struct htp_mm_hvx_vtcm_layout { + // Byte offsets from vtcm_base for each region + size_t off_src1; // vtcm_src1 (activation) + size_t off_src0; // vtcm_src0 (weight/Wk) + size_t off_src2; // vtcm_src2 (Wq / fused only) + size_t off_src3; // vtcm_src3 (Wv / fused only) + size_t off_dst; // vtcm_dst (output scratch) + + // Cached sizes + size_t src0_bytes; + size_t src1_bytes; + size_t src2_bytes; + size_t src3_bytes; + size_t dst_bytes; + + size_t total_bytes; +}; - return weight_area_size + act_area_size + act_f32_size + output_area_size + - scratch0_size + scratch1_size + scratch2_size + 256; -} +static inline void htp_mm_hmx_vtcm_layout_build( + struct htp_mm_hmx_vtcm_layout * L, + int kernel_type, + int wtype, + uint32_t k, + size_t mc, + size_t nc, + uint32_t group_size, + bool use_dma_activation, + bool pipeline, + uint32_t act_threads, + uint32_t aligned_tile_size +) { + size_t off = 0; + + if (kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) { + const size_t vec_dot_size = k * sizeof(uint16_t); + const size_t act_head_stride = mc * k; + const size_t weight_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); + const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); + const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t min_f32_size = use_dma_activation + ? hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128) : 0; + + // Group A: Permanent activation tiles and scales + size_t off_group_a = 0; + VTCM_LAYOUT_ALLOC(off_group_a, off_act, activation_area_size); + VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data + + // Group B: Compute-only buffers (starts at off_group_a) + size_t off_group_b = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, false); + VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, false); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch_area_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[1], scratch_area_size); + + const size_t group_b_size = off_group_b - off_group_a; + + // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) + const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t act_f32_size = use_dma_activation + ? hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128) : 0; + size_t off_group_c = off_group_a; + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_c, off_act_f32, act_f32_size, use_dma_activation); + + const size_t group_c_size = off_group_c - off_group_a; + + L->weight_area_bytes = weight_area_size; + L->act_area_bytes = activation_area_size; + L->act_f32_bytes = act_f32_size; + L->output_area_bytes = output_area_size; + L->scratch_bytes[0] = scratch_area_size; + L->scratch_bytes[1] = scratch_area_size; + L->act_head_stride = act_head_stride; + + off = off_group_a + hex_smax(group_b_size, group_c_size); + } else { + // HTP_MM_KERNEL_HMX_2D + const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32); + const size_t row_stride = htp_mm_get_tiled_row_stride(wtype, k); + const size_t vec_dot_size = k * sizeof(uint16_t); + const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + + const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t weight_area_size = is_quant + ? hex_align_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) + : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); + const size_t act_area_size = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); + + const size_t scratch0_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t scratch1_size = pipeline ? scratch0_size : 0; + + // Group A: Scales and activation tiles (must not overlap with Group B or C) + size_t off_group_a = 0; + VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data + VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_area_size); + + // Group B: Compute-only buffers (starts at off_group_a) + size_t off_group_b = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch0_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_scratch[1], scratch0_size, pipeline); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, pipeline); + + const size_t group_b_size = off_group_b - off_group_a; + + // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) + const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); + size_t off_group_c = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); + + const size_t group_c_size = off_group_c - off_group_a; + + L->weight_area_bytes = weight_area_size; + L->act_area_bytes = act_area_size; + L->act_f32_bytes = act_f32_size; + L->output_area_bytes = output_area_size; + L->scratch_bytes[0] = scratch0_size; + L->scratch_bytes[1] = scratch1_size; + L->act_head_stride = 0; + + off = off_group_a + hex_smax(group_b_size, group_c_size); + } -static inline size_t htp_mm_hmx_get_batched_vtcm_size( - int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool use_dma_activation, bool pipeline, uint32_t act_threads) { - (void)wtype; - (void)pipeline; - const size_t vec_dot_size = k * sizeof(uint16_t); - const size_t f32_scratch_size = use_dma_activation - ? htp_mm_round_up(act_threads * 4 * k * sizeof(float), HTP_MM_HMX_TILE_SIZE) : 0; - - const size_t act_head_stride = mc * k; - const size_t weight_area_size = htp_mm_round_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - const size_t act_area_size = htp_mm_round_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); - const size_t output_area_size = htp_mm_round_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); - const size_t scratch_area_size = htp_mm_round_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - - return weight_area_size + act_area_size + output_area_size + - 2 * scratch_area_size + 256 + f32_scratch_size; + L->total_bytes = off; } -static inline size_t htp_mm_hvx_get_vtcm_sizes( +static inline void htp_mm_hvx_vtcm_layout_build( + struct htp_mm_hvx_vtcm_layout * L, int kernel_type, int wtype, uint32_t ne10, // k - uint32_t src1_nrows, // m_total (or act_nrows) + uint32_t src1_nrows, // m_total uint32_t n_threads, size_t dst_row_size, size_t src0_row_size, size_t src1_row_size, uint32_t n_prefetch, - size_t * vtcm_src0_size_out, - size_t * vtcm_src1_size_out, - size_t * vtcm_dst_size_out + bool is_matmul_id, + bool is_fused_qkv, + bool is_fused_ffn ) { - size_t vtcm_src0_size = 0; - size_t vtcm_src1_size = 0; - size_t vtcm_dst_size = 0; + size_t src0_sz = 0; + size_t src1_sz = 0; + size_t src2_sz = 0; + size_t src3_sz = 0; + size_t dst_sz = 0; const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || wtype == HTP_TYPE_MXFP4); - const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); - const size_t dst_nrows = (src1_nrows > 1) ? 0 : 1; + if (is_fused_qkv || is_fused_ffn) { + const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + const size_t quant_scratch_size = hex_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads; - switch (kernel_type) { - case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { - size_t f16_src1_row_size = htp_mm_round_up(ne10 * 2, 128); - vtcm_src1_size = htp_mm_round_up(f16_src1_row_size * src1_nrows, 256); - vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; - break; - } - case HTP_MM_KERNEL_HVX_F16_F32_DDR: - case HTP_MM_KERNEL_HVX_F16_F16_DDR: - case HTP_MM_KERNEL_HVX_F32_F32_DDR: - case HTP_MM_KERNEL_HVX_F32_F16_DDR: { - vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size, 256) * n_threads; - vtcm_src1_size = htp_mm_round_up(n_prefetch * src1_row_size, 256) * n_threads; - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; - break; - } - case HTP_MM_KERNEL_HVX_F32_F32_VTCM: { - size_t f32_src1_row_size = htp_mm_round_up(ne10 * 4, 128); - vtcm_src1_size = htp_mm_round_up(f32_src1_row_size * src1_nrows, 256); - vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; - break; - } - case HTP_MM_KERNEL_HVX_QUANT_BLOCK: - case HTP_MM_KERNEL_HVX_QUANT_ROW: { - size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; - vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - vtcm_src1_size = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - - // src0 spad is also used in dynamic quantizer to store padded src1 rows - size_t src1_row_size_padded = htp_mm_round_up(q_src1_row_size, QK_Q8_0_TILED * sizeof(float)); - if (vtcm_src0_size < src1_row_size_padded) { - vtcm_src0_size = src1_row_size_padded; - } + size_t src0_sz_per_thread = 0; + size_t src2_sz_per_thread = 0; + size_t src3_sz_per_thread = 0; - vtcm_src0_size = vtcm_src0_size * n_threads; - vtcm_dst_size = vtcm_dst_size * n_threads; + if (is_repack) { + uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = ne10 / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - vtcm_src0_size = repacked_vtcm_size * n_threads; + src0_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + src2_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + if (is_fused_qkv) { + src3_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + } + } else { + src0_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); + src2_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); + if (is_fused_qkv) { + src3_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128); } - break; } - case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { - size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - vtcm_dst_size = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; - vtcm_src0_size = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - vtcm_src1_size = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); + size_t flat_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); + size_t tiled_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - size_t src1_row_size_padded = htp_mm_round_up(q_src1_row_size, 256); - if (vtcm_src0_size < src1_row_size_padded) { - vtcm_src0_size = src1_row_size_padded; - } + if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) { + src1_sz = hex_round_up(flat_src1_row_size * src1_nrows, 128); + } else { + src1_sz = hex_round_up(tiled_src1_row_size * src1_nrows, 128); + } - vtcm_src0_size = vtcm_src0_size * n_threads; - vtcm_dst_size = vtcm_dst_size * n_threads; + src0_sz = src0_sz_per_thread * n_threads; + src2_sz = src2_sz_per_thread * n_threads; + src3_sz = src3_sz_per_thread * n_threads; + dst_sz = quant_scratch_size; + } else if (is_matmul_id) { + const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); + const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10); + + size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); + src1_sz = htp_mm_round_up(src1_row_size_tiled * src1_nrows, 256); + + if (is_repack) { + const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + const uint32_t n_k_tiles = ne10 / 32; + const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); + src0_sz_per_thread = repacked_vtcm_size; + } - if (is_repack) { - uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - uint32_t n_k_tiles = ne10 / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; - } - vtcm_src0_size = repacked_vtcm_size * n_threads; + src0_sz = src0_sz_per_thread * n_threads; + dst_sz = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)) * n_threads; + } else { + const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); + const size_t dst_nrows = (src1_nrows > 1) ? 0 : 1; + + switch (kernel_type) { + case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { + size_t f16_src1_row_size = htp_mm_round_up(ne10 * 2, 128); + src1_sz = htp_mm_round_up(f16_src1_row_size * src1_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + break; } - break; - } - default: - break; - } + case HTP_MM_KERNEL_HVX_F16_F32_DDR: + case HTP_MM_KERNEL_HVX_F16_F16_DDR: + case HTP_MM_KERNEL_HVX_F32_F32_DDR: + case HTP_MM_KERNEL_HVX_F32_F16_DDR: { + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size, 256) * n_threads; + src1_sz = htp_mm_round_up(n_prefetch * src1_row_size, 256) * n_threads; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + break; + } + case HTP_MM_KERNEL_HVX_F32_F32_VTCM: { + size_t f32_src1_row_size = htp_mm_round_up(ne10 * 4, 128); + src1_sz = htp_mm_round_up(f32_src1_row_size * src1_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + break; + } + case HTP_MM_KERNEL_HVX_QUANT_BLOCK: + case HTP_MM_KERNEL_HVX_QUANT_ROW: { + size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); - *vtcm_src0_size_out = vtcm_src0_size; - *vtcm_src1_size_out = vtcm_src1_size; - *vtcm_dst_size_out = vtcm_dst_size; + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); + src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - return vtcm_src0_size + vtcm_src1_size + vtcm_dst_size; -} + src0_sz = src0_sz * n_threads; -static inline size_t htp_mm_hvx_id_get_vtcm_sizes( - int wtype, - uint32_t ne10, // k - uint32_t src1_nrows, - uint32_t n_threads, - size_t src0_row_size, // nb01 - uint32_t n_prefetch, - size_t * vtcm_src0_size_out, - size_t * vtcm_src1_size_out -) { - const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || - wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || - wtype == HTP_TYPE_MXFP4); + if (is_repack) { + uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + uint32_t n_k_tiles = ne10 / 32; + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); + src0_sz = repacked_vtcm_size * n_threads; + } - const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); - const size_t src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) - : htp_mm_q8_0_tiled_row_size(ne10); + size_t quant_scratch_size_per_thread = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + size_t dst_size_per_thread = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; + if (dst_size_per_thread < quant_scratch_size_per_thread) { + dst_size_per_thread = quant_scratch_size_per_thread; + } + dst_sz = dst_size_per_thread * n_threads; + break; + } + case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: { + size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); - size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); - size_t src1_sz = htp_mm_round_up(src1_row_size * src1_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256); + src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256); - // src0 spad also holds temporary transposed src1 columns during dynamic quantization. - const size_t src1_row_size_padded = htp_mm_round_up(src1_row_size, QK_Q8_0_TILED * sizeof(float)); - if (src0_sz_per_thread < src1_row_size_padded) { - src0_sz_per_thread = src1_row_size_padded; - } + src0_sz = src0_sz * n_threads; + + if (is_repack) { + uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + uint32_t n_k_tiles = ne10 / 32; + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); + src0_sz = repacked_vtcm_size * n_threads; + } - if (is_repack) { - const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); - const uint32_t n_k_tiles = ne10 / 32; - const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; - size_t repacked_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); - if (repacked_vtcm_size < src1_row_size_padded) { - repacked_vtcm_size = src1_row_size_padded; + size_t quant_scratch_size_per_thread = htp_mm_round_up(ne10 * sizeof(float), QK_Q8_0_TILED * sizeof(float)); + size_t dst_size_per_thread = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) : 0; + if (dst_size_per_thread < quant_scratch_size_per_thread) { + dst_size_per_thread = quant_scratch_size_per_thread; + } + dst_sz = dst_size_per_thread * n_threads; + break; + } + default: + break; } - src0_sz_per_thread = repacked_vtcm_size; } - const size_t vtcm_src0_size = src0_sz_per_thread * n_threads; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src1, src1_sz); + VTCM_LAYOUT_ALLOC(off, off_src0, src0_sz); + VTCM_LAYOUT_ALLOC(off, off_src2, src2_sz); + VTCM_LAYOUT_ALLOC(off, off_src3, src3_sz); + VTCM_LAYOUT_ALLOC(off, off_dst, dst_sz); + + L->src0_bytes = src0_sz; + L->src1_bytes = src1_sz; + L->src2_bytes = src2_sz; + L->src3_bytes = src3_sz; + L->dst_bytes = dst_sz; + L->total_bytes = off; +} - *vtcm_src0_size_out = vtcm_src0_size; - *vtcm_src1_size_out = src1_sz; +static inline size_t htp_mm_hmx_get_2d_vtcm_size( + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size +) { + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, false, pipeline, act_threads, aligned_tile_size); + return L.total_bytes; +} - return vtcm_src0_size + src1_sz; +static inline size_t htp_mm_hmx_get_batched_vtcm_size( + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool use_dma_activation, bool pipeline, uint32_t act_threads) { + (void)pipeline; + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_F16_BATCHED, wtype, k, mc, nc, group_size, use_dma_activation, false, act_threads, 0); + return L.total_bytes; } #ifdef __cplusplus diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index c839044b84f..d16dc7d38ef 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -23,6 +23,7 @@ #define HTP_ROPE_TYPE_NORMAL 0 #define HTP_ROPE_TYPE_NEOX 2 #define HTP_ROPE_TYPE_MROPE 8 +#define HTP_ROPE_TYPE_VISION 24 #define HTP_ROPE_TYPE_IMROPE 40 #define HTP_ROPE_SPAD_NROWS 16 @@ -70,7 +71,9 @@ struct htp_rope_context { struct htp_ops_context * octx; size_t src0_row_size; + size_t src0_row_stride; size_t dst_row_size; + size_t dst_row_stride; size_t src0_row_size_aligned; size_t dst_row_size_aligned; size_t theta_cache_offset; @@ -210,6 +213,7 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float pos_e, const int32_t sections[4], const bool is_imrope, + const bool indep_sects, const float freq_scale, const float * freq_factors, float * corr_dims, @@ -231,6 +235,14 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; const int sector = (i0 / 2) % sect_dims; + if (indep_sects) { + // Reset theta when crossing into a new section. + if (sector == 0) { theta_t = pos_t; } + else if (sector == sections[0]) { theta_h = pos_h; } + else if (sector == sec_w) { theta_w = pos_w; } + else if (sector == sec_e) { theta_e = pos_e; } + } + float theta; if (is_imrope) { // Interleaved: sector mod 3 selects component @@ -422,6 +434,17 @@ static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restr } } +static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, + uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { + #pragma unroll(4) + for (uint32_t i = 0; i < nr; i++) { + float * d = (float *) (dst + i * rctx->dst_row_size_aligned); + float * s = (float *) (src + i * rctx->src0_row_size_aligned); + + hvx_rope_neox_f32_aa(d, s, ne0, theta_cache); + } +} + static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { struct htp_rope_context * rctx = (struct htp_rope_context *) data; struct htp_ops_context * octx = rctx->octx; @@ -447,8 +470,9 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint64_t tt = HAP_perf_get_qtimer_count(); const int32_t mode = rctx->mode; - // MROPE and IMROPE use NEOX-style pairing for the rotation + // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); + const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -496,8 +520,10 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned; - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_size, pnr); + + // Copy only the row payload while striding the DDR source + dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } @@ -516,7 +542,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { (float) pos[i2 + ne2], (float) pos[i2 + ne2 * 2], (float) pos[i2 + ne2 * 3], - rctx->sections, is_imrope, + rctx->sections, is_imrope, is_vision, rctx->freq_scale, freq_factors, rctx->corr_dims, ne0, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale); @@ -542,14 +568,19 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); - if (is_neox) { + if (is_vision) { + rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + } else if (is_neox) { rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); } else { rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); } uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_addr, dst_spad), rctx->dst_row_size, rctx->dst_row_size_aligned, cnr); + + // Write only the row payload while striding the DDR dst + dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad), + rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr); // Prefetch more rows (if any) if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { @@ -558,8 +589,8 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_size, pnr); + dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } @@ -598,12 +629,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - const size_t src0_row_size = src0->nb[1]; - const size_t dst_row_size = dst->nb[1]; + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t src0_row_stride = src0->nb[1]; + const size_t dst_row_size = dst->ne[0] * sizeof(float); + const size_t dst_row_stride = dst->nb[1]; // Aligned row sizes for VTCM const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); + const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN); const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256); // Calculate spad sizes per thread @@ -652,8 +685,10 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); - rctx.src0_row_size = src0_row_size; - rctx.dst_row_size = dst_row_size; + rctx.src0_row_size = src0_row_size; + rctx.src0_row_stride = src0_row_stride; + rctx.dst_row_size = dst_row_size; + rctx.dst_row_stride = dst_row_stride; rctx.src0_row_size_aligned = src0_row_size_aligned; rctx.dst_row_size_aligned = dst_row_size_aligned; rctx.theta_cache_offset = theta_cache_size_aligned; diff --git a/ggml/src/ggml-hexagon/htp/vtcm-utils.h b/ggml/src/ggml-hexagon/htp/vtcm-utils.h deleted file mode 100644 index b129fb74e31..00000000000 --- a/ggml/src/ggml-hexagon/htp/vtcm-utils.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef VTCM_UTILS_H -#define VTCM_UTILS_H - -#include "hex-utils.h" - -#include -#include -#include - -static inline uint8_t *vtcm_seq_alloc(uint8_t **vtcm_ptr, size_t size) { - uint8_t *p = *vtcm_ptr; - *vtcm_ptr += size; - return p; -} - -#endif // VTCM_UTILS_H diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.c b/ggml/src/ggml-hexagon/htp/worker-pool.c index 172e28908eb..50960d2c75d 100644 --- a/ggml/src/ggml-hexagon/htp/worker-pool.c +++ b/ggml/src/ggml-hexagon/htp/worker-pool.c @@ -1,6 +1,9 @@ #include "worker-pool.h" +#include "hex-utils.h" #include +#include + #include #include #include @@ -9,7 +12,6 @@ #include "HAP_farf.h" -#define WORKER_THREAD_STACK_SZ (2 * 16384) #define LOWEST_USABLE_QURT_PRIO (254) struct worker_pool_s; @@ -42,17 +44,27 @@ static void worker_pool_main(void * context) { FARF(HIGH, "worker-pool: thread %u started", me->id); unsigned int prev_seqn = 0; + unsigned int poll_cnt = WORKER_POOL_POLL_COUNT; while (!atomic_load(&pool->killed)) { unsigned int seqn = atomic_load(&pool->seqn); if (seqn == prev_seqn) { - // Nothing to do + // drop HVX context while spinning + if (poll_cnt > 1 && poll_cnt == WORKER_POOL_POLL_COUNT) { + qurt_hvx_unlock(); + } + if (--poll_cnt) { + hex_pause(); + continue; + } qurt_futex_wait(&pool->seqn, prev_seqn); + poll_cnt = WORKER_POOL_POLL_COUNT; continue; } - // New job prev_seqn = seqn; + poll_cnt = WORKER_POOL_POLL_COUNT; + // New job unsigned int n = atomic_load(&pool->n_jobs); unsigned int i = atomic_fetch_add(&pool->next_job, 1); if (i >= n) { diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.h b/ggml/src/ggml-hexagon/htp/worker-pool.h index 6f8c9056c4b..cba692126ad 100644 --- a/ggml/src/ggml-hexagon/htp/worker-pool.h +++ b/ggml/src/ggml-hexagon/htp/worker-pool.h @@ -24,9 +24,17 @@ typedef struct { void * data; } worker_pool_job_t; +#define WORKER_THREAD_STACK_SZ (2 * 16384) + /// Maximum supported number of worker threads. #define MAX_NUM_WORKERS 10 +#if __HVX_ARCH__ > 79 +#define WORKER_POOL_POLL_COUNT 2000 +#else +#define WORKER_POOL_POLL_COUNT 1 +#endif + // Initialize worker pool. WORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads); diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index a7d4e0ea2b5..7121193f1c8 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -130,6 +130,9 @@ if (GGML_HIP_EXPORT_METRICS) set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -Rpass-analysis=kernel-resource-usage --save-temps") endif() +# Fast math for HIP, like CUDA's -use_fast_math. Not -ffast-math: that implies -ffinite-math-only, which breaks ggml's INFINITY masking and produces NaNs. +set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -funsafe-math-optimizations") + if (NOT GGML_CUDA_FA) add_compile_definitions(GGML_CUDA_NO_FA) endif() @@ -155,3 +158,5 @@ if (GGML_HIP_RCCL) endif() target_link_libraries(ggml-hip PRIVATE ggml-base hip::host roc::rocblas roc::hipblas) + +target_compile_options(ggml-hip PRIVATE "$<$:-ffast-math;-fno-finite-math-only>") diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 0e1f1de4577..15290c3d109 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -160,11 +160,15 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows(ggml_me return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, ggml_type tidx, ggml_type tdst) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; - snprintf(base, 256, "kernel_set_rows_%s_%s", ggml_type_name(tdst), ggml_type_name(tidx)); + const auto tsrc = op->src[0]->type; + const auto tidx = op->src[1]->type; + const auto tdst = op->type; + + snprintf(base, 256, "kernel_set_rows_%s_%s_%s", ggml_type_name(tsrc), ggml_type_name(tidx), ggml_type_name(tdst)); snprintf(name, 256, "%s", base); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -1800,6 +1804,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1 return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_COL2IM_1D); + + GGML_ASSERT(ggml_is_contiguous(op->src[0])); + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_col2im_1d_%s", ggml_type_name(op->src[0]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_CONV_TRANSPOSE_2D); @@ -1845,6 +1869,29 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d(ggml_met return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) { + assert(op->op == GGML_OP_CONV_2D_DW); + + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_conv_2d_dw%s_%s_%s", + tiled ? "_tiled" : "", + ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_CONV_3D); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index d465f31c083..9d4aca12159 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -112,7 +112,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cpy struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_1d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_2d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows (ggml_metal_library_t lib, enum ggml_type tsrc); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, enum ggml_type tidx, enum ggml_type tdst); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc); @@ -150,7 +150,9 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index a7cbc60ebe4..5d29250f654 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1157,6 +1157,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_COL2IM_1D: + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && + op->type == op->src[0]->type && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op); case GGML_OP_CONV_3D: return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && @@ -1193,6 +1198,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + case GGML_OP_CONV_2D_DW: + return op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); case GGML_OP_UPSCALE: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_POOL_1D: @@ -1329,7 +1338,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return op->src[0]->type != GGML_TYPE_NVFP4; case GGML_OP_SET_ROWS: { - if (op->src[0]->type != GGML_TYPE_F32) { + if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) { return false; } diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index ff74cafb5b7..d6761023b76 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -603,6 +603,16 @@ typedef struct { uint64_t nb1; } ggml_metal_kargs_conv_transpose_1d; +typedef struct { + int32_t T_in; + int32_t T_out; + int32_t OC; + int32_t K; + int32_t K_OC; + int32_t s0; + int32_t p0; +} ggml_metal_kargs_col2im_1d; + typedef struct { int32_t IC; int32_t IH; @@ -646,6 +656,34 @@ typedef struct { int32_t d1; } ggml_metal_kargs_conv_2d; +typedef struct { + uint64_t nb00; // kernel strides + uint64_t nb01; + uint64_t nb02; + uint64_t nb10; // input strides + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + uint64_t nb0; // output strides + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t IW; // input width + int32_t IH; // input height + int32_t KW; // kernel width + int32_t KH; // kernel height + int32_t C; // channels (IC == OC for depthwise) + int32_t OW; // output width + int32_t OH; // output height + int32_t N; // batch size + int32_t s0; // stride x + int32_t s1; // stride y + int32_t p0; // padding x + int32_t p1; // padding y + int32_t d0; // dilation x + int32_t d1; // dilation y +} ggml_metal_kargs_conv_2d_dw; + typedef struct { uint64_t ofs0; uint64_t ofs1; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 18656b346f2..45909c4777b 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -387,6 +387,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_conv_2d(ctx, idx); } break; + case GGML_OP_CONV_2D_DW: + { + n_fuse = ggml_metal_op_conv_2d_dw(ctx, idx); + } break; case GGML_OP_CONV_TRANSPOSE_1D: { n_fuse = ggml_metal_op_conv_transpose_1d(ctx, idx); @@ -395,6 +399,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_conv_transpose_2d(ctx, idx); } break; + case GGML_OP_COL2IM_1D: + { + n_fuse = ggml_metal_op_col2im_1d(ctx, idx); + } break; case GGML_OP_CONV_3D: { n_fuse = ggml_metal_op_conv_3d(ctx, idx); @@ -1198,7 +1206,7 @@ int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op->src[1]->type, op->type); + auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op); const int32_t nk0 = ne0/ggml_blck_size(op->type); @@ -3738,6 +3746,86 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_conv_2d_dw(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + + GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + + const int32_t s0 = ((const int32_t *) op->op_params)[0]; + const int32_t s1 = ((const int32_t *) op->op_params)[1]; + const int32_t p0 = ((const int32_t *) op->op_params)[2]; + const int32_t p1 = ((const int32_t *) op->op_params)[3]; + const int32_t d0 = ((const int32_t *) op->op_params)[4]; + const int32_t d1 = ((const int32_t *) op->op_params)[5]; + + ggml_metal_kargs_conv_2d_dw args = { + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb03, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + /*.IW =*/ ne10, + /*.IH =*/ ne11, + /*.KW =*/ ne00, + /*.KH =*/ ne01, + /*.C =*/ ne12, + /*.OW =*/ ne0, + /*.OH =*/ ne1, + /*.N =*/ ne13, + /*.s0 =*/ s0, + /*.s1 =*/ s1, + /*.p0 =*/ p0, + /*.p1 =*/ p1, + /*.d0 =*/ d0, + /*.d1 =*/ d1, + }; + + const bool use_tiled = (nb12 < nb10); + + auto pipeline = ggml_metal_library_get_pipeline_conv_2d_dw(lib, op, use_tiled); + + int nth = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); + nth = std::min(nth, 256); + nth = std::max(nth, 1); + + const int32_t OW = ne0; + const int32_t OH = ne1; + const int32_t C = ne12; + const int32_t N = ne13; + + const int tg_x = use_tiled ? (C + nth - 1) / nth : (OW + nth - 1) / nth; + const int tg_y = OH; + const int tg_z = use_tiled ? OW * N : C * N; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, tg_x, tg_y, tg_z, nth, 1, 1); + + return 1; +} + int ggml_metal_op_conv_3d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -3854,6 +3942,47 @@ int ggml_metal_op_conv_transpose_1d(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_col2im_1d(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const int32_t s0 = ((const int32_t *)(op->op_params))[0]; + const int32_t OC = ((const int32_t *)(op->op_params))[1]; + const int32_t p0 = ((const int32_t *)(op->op_params))[2]; + + const int32_t K_OC = (int32_t) op->src[0]->ne[0]; + const int32_t T_in = (int32_t) op->src[0]->ne[1]; + const int32_t K = K_OC / OC; + const int32_t T_out = (int32_t) op->ne[0]; + + ggml_metal_kargs_col2im_1d args = { + /*.T_in =*/ T_in, + /*.T_out =*/ T_out, + /*.OC =*/ OC, + /*.K =*/ K, + /*.K_OC =*/ K_OC, + /*.s0 =*/ s0, + /*.p0 =*/ p0, + }; + + auto pipeline = ggml_metal_library_get_pipeline_col2im_1d(lib, op); + + const int total = T_out * OC; + const int nth = 256; + const int ntg = (total + nth - 1) / nth; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + + return 1; +} + int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 36c61071b4f..0bebd836a18 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -75,9 +75,11 @@ int ggml_metal_op_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_rope (ggml_metal_op_t ctx, int idx); int ggml_metal_op_im2col (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_conv_2d_dw (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_3d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_transpose_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_transpose_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_col2im_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_upscale (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad_reflect_1d (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 25e78e10089..6b6f9fd870c 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -42,6 +42,8 @@ typedef matrix bfloat4x4; typedef matrix bfloat2x4; #endif +#define QK_NL 16 + constexpr constant static float kvalues_iq4nl_f[16] = { -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f }; @@ -4906,6 +4908,202 @@ kernel void kernel_conv_2d( uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]); +// grid: x = C tile, y = OH, z = OW * N (for channel-contiguous layouts) +template +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int32_t c = (int32_t)(tgpig.x * ntg.x + tpitg.x); + if (c >= args.C) { + return; + } + + const int32_t oh = tgpig.y; + const int32_t own = tgpig.z; + const int32_t ow = own % args.OW; + const int32_t n = own / args.OW; + + const int32_t base_y = oh*args.s1 - args.p1; + + int32_t ky_start = 0; + if (base_y < 0) { + ky_start = (-base_y + args.d1 - 1)/args.d1; + } + int32_t ky_end = args.KH; + const int32_t y_max = args.IH - 1 - base_y; + if (y_max < 0) { + ky_end = ky_start; + } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { + ky_end = min(ky_end, y_max/args.d1 + 1); + } + + const int32_t base_x = ow*args.s0 - args.p0; + + int32_t kx_start = 0; + if (base_x < 0) { + kx_start = (-base_x + args.d0 - 1)/args.d0; + } + int32_t kx_end = args.KW; + const int32_t x_max = args.IW - 1 - base_x; + if (x_max < 0) { + kx_end = kx_start; + } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { + kx_end = min(kx_end, x_max/args.d0 + 1); + } + + float acc = 0.0f; + + if (ky_start < ky_end && kx_start < kx_end) { + const uint64_t w_base = (uint64_t) c * args.nb02; + const uint64_t src_base = (uint64_t) n * args.nb13 + (uint64_t) c * args.nb12; + + for (int32_t ky = ky_start; ky < ky_end; ++ky) { + const int32_t iy = base_y + ky*args.d1; + const uint64_t src_row = src_base + (uint64_t) iy * args.nb11; + const uint64_t w_row = w_base + (uint64_t) ky * args.nb01; + + for (int32_t kx = kx_start; kx < kx_end; ++kx) { + const int32_t ix = base_x + kx*args.d0; + const float x = *(device const float *)(src + src_row + (uint64_t) ix * args.nb10); + const float w = (float)(*(device const TK *)(weights + w_row + (uint64_t) kx * args.nb00)); + acc += x * w; + } + } + } + + const uint64_t dst_offs = + (uint64_t) n * args.nb3 + + (uint64_t) c * args.nb2 + + (uint64_t) oh * args.nb1 + + (uint64_t) ow * args.nb0; + + *(device float *)(dst + dst_offs) = acc; +} + +// grid: x = OW tile, y = OH, z = C * N (for spatially-contiguous layouts) +template +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int32_t oh = tgpig.y; + const int32_t cn = tgpig.z; + const int32_t c = cn % args.C; + const int32_t n = cn / args.C; + + const int32_t base_y = oh*args.s1 - args.p1; + + int32_t ky_start = 0; + if (base_y < 0) { + ky_start = (-base_y + args.d1 - 1)/args.d1; + } + int32_t ky_end = args.KH; + const int32_t y_max = args.IH - 1 - base_y; + if (y_max < 0) { + ky_end = ky_start; + } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { + ky_end = min(ky_end, y_max/args.d1 + 1); + } + + const uint64_t w_base = (uint64_t) c * args.nb02; + const uint64_t src_base = (uint64_t) n * args.nb13 + (uint64_t) c * args.nb12; + + const int32_t ow = (int32_t)(tgpig.x * ntg.x + tpitg.x); + if (ow >= args.OW) { + return; + } + + float acc = 0.0f; + + const int32_t base_x = ow*args.s0 - args.p0; + + int32_t kx_start = 0; + if (base_x < 0) { + kx_start = (-base_x + args.d0 - 1)/args.d0; + } + int32_t kx_end = args.KW; + const int32_t x_max = args.IW - 1 - base_x; + if (x_max < 0) { + kx_end = kx_start; + } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { + kx_end = min(kx_end, x_max/args.d0 + 1); + } + + if (ky_start < ky_end && kx_start < kx_end) { + for (int32_t ky = ky_start; ky < ky_end; ++ky) { + const int32_t iy = base_y + ky*args.d1; + const uint64_t src_row = src_base + (uint64_t) iy * args.nb11; + const uint64_t w_row = w_base + (uint64_t) ky * args.nb01; + + for (int32_t kx = kx_start; kx < kx_end; ++kx) { + const int32_t ix = base_x + kx*args.d0; + const float x = *(device const float *)(src + src_row + (uint64_t) ix * args.nb10); + const float w = (float)(*(device const TK *)(weights + w_row + (uint64_t) kx * args.nb00)); + acc += x * w; + } + } + } + + const uint64_t dst_offs = + (uint64_t) n * args.nb3 + + (uint64_t) c * args.nb2 + + (uint64_t) oh * args.nb1 + + (uint64_t) ow * args.nb0; + + *(device float *)(dst + dst_offs) = acc; +} + +template [[host_name("kernel_conv_2d_dw_f32_f32")]] +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_f16_f32")]] +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_tiled_f32_f32")]] +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_tiled_f16_f32")]] +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + typedef void (conv_transpose_1d_t)( constant ggml_metal_kargs_conv_transpose_1d & args, device const float * src0, @@ -4977,6 +5175,49 @@ kernel void kernel_conv_transpose_1d( uint3 tgpg[[threadgroups_per_grid]]); +template +kernel void kernel_col2im_1d( + constant ggml_metal_kargs_col2im_1d & args, + device const T * col, + device T * dst, + uint tgpig [[threadgroup_position_in_grid]], + uint tpitg [[thread_position_in_threadgroup]], + uint ntg [[threads_per_threadgroup]]) { + + const int idx = tgpig * ntg + tpitg; + if (idx >= args.T_out * args.OC) { + return; + } + + const int t_out = idx % args.T_out; + const int oc = idx / args.T_out; + const int t_abs = t_out + args.p0; // absolute position in uncropped signal + + int t_in_min = (t_abs - args.K + args.s0) / args.s0; // ceil((t_abs - K + 1) / s0) + if (t_in_min < 0) { + t_in_min = 0; + } + int t_in_max = t_abs / args.s0; + if (t_in_max >= args.T_in) { + t_in_max = args.T_in - 1; + } + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; t_in++) { + const int k = t_abs - t_in * args.s0; + sum += float(col[(oc * args.K + k) + t_in * args.K_OC]); + } + + dst[t_out + oc * args.T_out] = T(sum); +} + +template [[host_name("kernel_col2im_1d_f32")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const float *, device float *, uint, uint, uint); +template [[host_name("kernel_col2im_1d_f16")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const half *, device half *, uint, uint, uint); +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_col2im_1d_bf16")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const bfloat *, device bfloat *, uint, uint, uint); +#endif + + typedef void (conv_transpose_2d_t)( constant ggml_metal_kargs_conv_transpose_2d & args, device const float * src0, @@ -9343,7 +9584,40 @@ kernel void kernel_get_rows_f( } } -template +typedef decltype(kernel_get_rows_f) get_rows_f_t; + +template [[host_name("kernel_get_rows_f32")]] kernel get_rows_f_t kernel_get_rows_f; +template [[host_name("kernel_get_rows_f16")]] kernel get_rows_f_t kernel_get_rows_f; +template [[host_name("kernel_get_rows_i32")]] kernel get_rows_f_t kernel_get_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f; +#endif + +typedef decltype(kernel_get_rows_q) get_rows_q_t; + +template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_1")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q8_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_mxfp4")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q2_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q3_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q; + +template kernel void kernel_set_rows_q32( constant ggml_metal_kargs_set_rows & args, device const void * src0, @@ -9367,14 +9641,14 @@ kernel void kernel_set_rows_q32( const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); - const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { quantize_func(src_row + 32*ind, dst_row[ind]); } } -template +template kernel void kernel_set_rows_f( constant ggml_metal_kargs_set_rows & args, device const void * src0, @@ -9397,14 +9671,47 @@ kernel void kernel_set_rows_f( const int32_t i10 = i01; const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; - device T * dst_row = ( device T *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); - const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + device TD * dst_row = ( device TD *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); + const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { - dst_row[ind] = (T) src_row[ind]; + dst_row[ind] = (TD) src_row[ind]; } } +typedef decltype(kernel_set_rows_f) set_rows_f_t; + +template [[host_name("kernel_set_rows_f32_i64_f32")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_f32")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i64_f16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_f16")]] kernel set_rows_f_t kernel_set_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_set_rows_f32_i64_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +#endif + +template [[host_name("kernel_set_rows_f16_i64_f16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f16_i32_f16")]] kernel set_rows_f_t kernel_set_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_set_rows_bf16_i64_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_bf16_i32_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +#endif + +typedef decltype(kernel_set_rows_q32) set_rows_q32_t; + +template [[host_name("kernel_set_rows_f32_i64_q8_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q8_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q4_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q4_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q4_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q4_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q5_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q5_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q5_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q5_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32; + kernel void kernel_diag_f32( constant ggml_metal_kargs_diag & args, device const char * src0, @@ -10147,75 +10454,6 @@ kernel void kernel_mul_mm_id( } } -#define QK_NL 16 - -// -// get rows -// - -typedef decltype(kernel_get_rows_f) get_rows_f_t; - -template [[host_name("kernel_get_rows_f32")]] kernel get_rows_f_t kernel_get_rows_f; -template [[host_name("kernel_get_rows_f16")]] kernel get_rows_f_t kernel_get_rows_f; -template [[host_name("kernel_get_rows_i32")]] kernel get_rows_f_t kernel_get_rows_f; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f; -#endif - -typedef decltype(kernel_get_rows_q) get_rows_q_t; - -template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_1")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q8_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_mxfp4")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q2_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q3_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q; - -// -// set rows -// - -typedef decltype(kernel_set_rows_f) set_rows_f_t; - -template [[host_name("kernel_set_rows_f32_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f32_i32")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f16_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f16_i32")]] kernel set_rows_f_t kernel_set_rows_f; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_set_rows_bf16_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_bf16_i32")]] kernel set_rows_f_t kernel_set_rows_f; -#endif - -typedef decltype(kernel_set_rows_q32) set_rows_q32_t; - -template [[host_name("kernel_set_rows_q8_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q8_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_1_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_1_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_1_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_1_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_iq4_nl_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_iq4_nl_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; - // // matrix-matrix multiplication // diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 82ce61d72c6..9ec3268b7f4 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -31,6 +31,11 @@ if (GGML_OPENCL_EMBED_KERNELS) target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/autogenerated") endif () +if (GGML_OPENCL_USE_ADRENO_BIN_KERNELS) + message(STATUS "OpenCL will use precompiled binary kernels for Adreno (improved performance on some platforms)") + add_compile_definitions(GGML_OPENCL_USE_ADRENO_BIN_KERNELS) +endif () + function(ggml_opencl_add_kernel KNAME) set(KERN_HDR ${CMAKE_CURRENT_BINARY_DIR}/autogenerated/${KNAME}.cl.h) set(KERN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${KNAME}.cl) @@ -78,6 +83,8 @@ set(GGML_OPENCL_KERNELS mul_mv_f16_f32_l4 mul_mv_f16_f32 mul_mv_f32_f32 + mul_mv_q1_0_f32 + mul_mv_q1_0_f32_flat mul_mv_q4_0_f32 mul_mv_q4_0_f32_v mul_mv_q4_0_f32_8x_flat @@ -128,6 +135,7 @@ set(GGML_OPENCL_KERNELS moe_sort_by_expert mul_mm_f32_f32_l4_lm mul_mm_f16_f32_l4_lm + mul_mm_q1_0_f32_l4_lm mul_mm_q4_0_f32_l4_lm mul_mm_q4_1_f32_l4_lm mul_mm_q5_0_f32_l4_lm @@ -137,6 +145,8 @@ set(GGML_OPENCL_KERNELS mul_mm_q4_k_f32_l4_lm mul_mm_q5_k_f32_l4_lm mul_mm_q6_k_f32_l4_lm + gemv_noshuffle_q1_0_f32 + gemm_noshuffle_q1_0_f32 gemv_noshuffle_q4_0_f32 gemv_noshuffle_q4_0_f32_spec gemm_noshuffle_q4_0_f32 @@ -192,7 +202,10 @@ set(GGML_OPENCL_KERNELS mul_mm_f16_f32_kq_kqv conv2d conv2d_f16_f32 + flash_attn_pre_f16 flash_attn_f32_f16 + flash_attn_f32_q8_0 + flash_attn_f32_q4_0 flash_attn_f16 flash_attn_f32 ) diff --git a/ggml/src/ggml-opencl/fa_tune.h b/ggml/src/ggml-opencl/fa_tune.h new file mode 100644 index 00000000000..447f0cf7e04 --- /dev/null +++ b/ggml/src/ggml-opencl/fa_tune.h @@ -0,0 +1,92 @@ +#pragma once + +// Flash-attention per-(dk,dv) tile tuning for the Adreno OpenCL backend. +// Isolated from ggml-opencl.cpp so the tuning numbers are easy to find and +// edit; the FA dispatch and kernel-compile logic stay in the main file. +// This header is a file section — it is #included exactly once, at the point +// in ggml-opencl.cpp where the ggml logging macros are already in scope. + +// Per-(dk, dv) FA config; shared by dispatch and supports_op. +struct ggml_opencl_fa_dim { + int dk; int dv; int bm; int bn; int n_split; int nkv_split_threshold; +}; + +// Split variant fires when n_kv >= threshold (threshold=0 -> always split). +// Default tuning covers Adreno 7xx/8xx mobile and X1-series laptop GPUs. +static const ggml_opencl_fa_dim g_fa_dims_adreno_default[] = { + { 40, 40, 64, 32, 1, 0}, { 64, 64, 64, 32, 2, 64}, + { 80, 80, 64, 32, 2, 64}, { 96, 96, 64, 32, 2, 64}, + {112, 112, 64, 32, 2, 64}, {128, 128, 64, 32, 2, 64}, + {192, 128, 16, 16, 1, 0}, + {192, 192, 16, 16, 1, 0}, + {256, 256, 16, 16, 16, 0}, + {512, 512, 8, 16, 64, 0}, +}; + +struct ggml_opencl_fa_dim_table { + const ggml_opencl_fa_dim * data; + size_t count; + + const ggml_opencl_fa_dim * begin() const { return data; } + const ggml_opencl_fa_dim * end() const { return data + count; } +}; + +// Mutable copy of the active table; GGML_OPENCL_FA_TUNE patches entries here +// at backend init without touching the const source table. +static ggml_opencl_fa_dim g_fa_dims_runtime[ + sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0])]; + +static ggml_opencl_fa_dim_table g_opencl_fa_dims = { + g_fa_dims_adreno_default, + sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0]), +}; + +// GGML_OPENCL_FA_TUNE=dk:dv:bm:bn:nsplit:thr[,…] — patches matching entries +// in the active table at backend init, before the first FA kernel compiles. +// Unmatched (dk,dv) pairs are warned and ignored. +static void ggml_opencl_fa_apply_env_overrides() { + const char * e = std::getenv("GGML_OPENCL_FA_TUNE"); + if (!e || !e[0]) { + return; + } + + std::string s = e; + size_t pos = 0; + while (pos < s.size()) { + size_t comma = s.find(',', pos); + std::string entry = s.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); + int dk, dv, bm, bn, nsplit, thr; + if (std::sscanf(entry.c_str(), "%d:%d:%d:%d:%d:%d", &dk, &dv, &bm, &bn, &nsplit, &thr) == 6) { + bool patched = false; + for (size_t i = 0; i < g_opencl_fa_dims.count; ++i) { + ggml_opencl_fa_dim & d = g_fa_dims_runtime[i]; + if (d.dk == dk && d.dv == dv) { + d.bm = bm; d.bn = bn; d.n_split = nsplit; d.nkv_split_threshold = thr; + GGML_LOG_INFO("ggml_opencl: FA tune override DK=%d DV=%d -> bm=%d bn=%d n_split=%d thr=%d\n", + dk, dv, bm, bn, nsplit, thr); + patched = true; + break; + } + } + if (!patched) { + GGML_LOG_WARN("ggml_opencl: FA tune override DK=%d DV=%d ignored (no matching dim)\n", dk, dv); + } + } else { + GGML_LOG_WARN("ggml_opencl: FA tune override entry malformed: '%s'\n", entry.c_str()); + } + if (comma == std::string::npos) break; + pos = comma + 1; + } +} + +// Copy the default table into the mutable runtime buffer and apply any +// GGML_OPENCL_FA_TUNE overrides. A per-generation table can be added here +// once it has been tuned on hardware. +static void ggml_cl_init_fa_dims_table() { + const size_t count = sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0]); + for (size_t i = 0; i < count; ++i) { + g_fa_dims_runtime[i] = g_fa_dims_adreno_default[i]; + } + g_opencl_fa_dims = { g_fa_dims_runtime, count }; + ggml_opencl_fa_apply_env_overrides(); +} diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 00f20b09b8f..38600f31a7e 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -13,6 +13,22 @@ #include "ggml-backend-impl.h" #include "ggml.h" +#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS +#include "libdl.h" +#ifdef _WIN32 +#define KERNEL_LIB_NAME "adreno-opencl-kernels.dll" +#else +#define KERNEL_LIB_NAME "libadreno-opencl-kernels.so" +#endif // _WIN32 +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS + +typedef const void * (*get_adreno_bin_kernel_func_t)( + const char * name, + const char * gpu_name, + const char * compiler_ver, + size_t * out_size +); + #include #include @@ -29,6 +45,8 @@ #include #include #include +#include +#include #undef MIN #undef MAX @@ -53,6 +71,9 @@ //------------------------------------------------------------------------------ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor); +static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor); +static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor); +static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); // See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. // Precompute mp (m' in the paper) and L such that division @@ -96,6 +117,7 @@ enum ADRENO_GPU_GEN { A7X, A8X, X1E, + X2E, }; enum ADRENO_CL_COMPILER_TYPE { @@ -236,6 +258,10 @@ static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { return ADRENO_GPU_GEN::X1E; } + if (strstr(device_name, "X2")) { + return ADRENO_GPU_GEN::X2E; + } + return ADRENO_GPU_GEN::ADRENO_UNKNOWN; } @@ -368,7 +394,7 @@ struct ggml_backend_opencl_device_context { cl_device_type device_type; std::string device_version; - // Initialized by ggml_cl2_init(). + // Initialized by ggml_cl_init(). ggml_backend_opencl_context * backend_ctx = nullptr; // Initialized by ggml_backend_opencl_device_get_buffer_type() @@ -384,6 +410,82 @@ struct ggml_backend_opencl_device_context { size_t global_mem_size = 0; }; +// Lazily-compiled flash-attention kernels and their per-(dk,dv) tile metadata. +// One map per (Q/KV dtype, decode/prefill, split) combination; the int maps +// hold tile dims (bm/bn), workgroup sizes and the n_kv split thresholds. +struct ggml_opencl_fa_kernels { + // f16 Q / f16 KV + std::map, cl_kernel> f16; + std::map, cl_kernel> f16_q1; + // f32 Q / f32 KV + std::map, cl_kernel> f32; + std::map, cl_kernel> f32_q1; + // f32 Q / f16 KV (mixed) + std::map, cl_kernel> f32_f16; + std::map, cl_kernel> f32_f16_split; // N_SPLIT>1 variant + std::map, cl_kernel> f32_f16_split_k_img; // DK=512 prefill split, K via image1d_buffer_t + std::map, cl_kernel> f32_f16_q1; + std::map, cl_kernel> f32_f16_q1_split; // flash-decoding K-split + // vec decode + std::map, cl_kernel> f32_f16_q1_vec; + // kv-head-coalesced vec decode + std::map, cl_kernel> f32_f16_q1_vec_mq; + // kv-head-coalesced + flash-decoding split + std::map, cl_kernel> f32_f16_q1_vec_mq_split; + // MQ_GQA=8 specializations + std::map, cl_kernel> f32_f16_q1_vec_mq_g8; + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8; + // k-image variant of MQ_G8 vec_mq_split + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_k_img; + // k-image variant of MQ_GQA=4 vec_mq_split + std::map, cl_kernel> f32_f16_q1_vec_mq_split_k_img; + // alternative decode + std::map, cl_kernel> f32_f16_q1_local_tile; + // hybrid local-tile + MQ + FD-split kernel for DK=DV=128 only + std::map, cl_kernel> f32_f16_q1_local_mq_split; + std::map, cl_kernel> f32_f16_q1_local_mq_split_g8; + std::map, int> f32_f16_bm; + std::map, int> f32_f16_bn; + std::map, int> f32_f16_wg_size; + std::map, int> f32_f16_split_wg_size; + std::map, int> f32_f16_split_nkv_threshold; + // f32 Q / native q8_0 KV + std::map, cl_kernel> f32_q8_0_q1; // decode + std::map, cl_kernel> f32_q8_0_q1_vec; // DV-split + multi-subgroup decode + std::map, cl_kernel> f32_q8_0_q1_split; // flash-decoding pass 1 + // KV-head-coalesced + flash-decoding split for q8_0 KV + std::map, cl_kernel> f32_q8_0_q1_vec_mq_split; + std::map, cl_kernel> f32_q8_0_q1_vec_mq_split_g8; + std::map, cl_kernel> f32_q8_0; // prefill (baseline) + std::map, cl_kernel> f32_q8_0_split; // N_SPLIT>1 variant + std::map, int> f32_q8_0_split_wg_size; // wg_size = bm*n_split + std::map, int> f32_q8_0_split_nkv_threshold; // use split when n_kv >= this + std::map, int> f32_q8_0_split_bm; // per-split BLOCK_M + // f32 Q / native q4_0 KV + std::map, cl_kernel> f32_q4_0_q1; + std::map, cl_kernel> f32_q4_0_q1_vec; // DV-split + multi-subgroup decode + std::map, cl_kernel> f32_q4_0_q1_split; + // kv-head-coalesced + flash-decoding split for q4_0 kv (dp4a K dot) + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split; + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_g8; + std::map, cl_kernel> f32_q4_0; + std::map, cl_kernel> f32_q4_0_split; + std::map, int> f32_q4_0_split_wg_size; + std::map, int> f32_q4_0_split_nkv_threshold; + std::map, int> f32_q4_0_split_bm; + // shared: flash-decoding merge + prefill prepass (kv-pad, mask-pad, blk class) + std::map, cl_kernel> f32_merge; + std::map, cl_kernel> kv_pad_f16; + std::map, cl_kernel> mask_pad_f16; + std::map, cl_kernel> blk_f16; + // generic prefill tile dims (f16 / f32 paths) + std::map, int> bm; + std::map, int> bn; + // attempted (variant, (dk, dv)) + // all attempted FA kernels appear here, but those not registered failed compilation + std::set>> variant_attempted; +}; + // backend context struct ggml_backend_opencl_context { int ref_count; @@ -397,9 +499,6 @@ struct ggml_backend_opencl_context { // argsort is loaded in supports_op because its availability depends on how // many workgroups are allowed, which requires kernel compilation. bool kernels_loaded_argsort = false; - // flash attn is loaded in supports_op because it contains multiple variants - // and takes time to compile, so we want to only compile it when needed. - bool kernels_loaded_flash_attn = false; // rest of the kernels are currently always loaded in alloc_buffer. bool kernels_loaded = false; @@ -414,13 +513,22 @@ struct ggml_backend_opencl_context { size_t max_workgroup_size; bool fp16_support; bool has_vector_subgroup_broadcast; - bool has_qcom_subgroup_shuffle = false; // cl_qcom_subgroup_shuffle + bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle + bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle bool disable_fusion; + // ragged moe, use int to directly pass to kernel + cl_uint adreno_use_moe_ragged; + cl_uint adreno_moe_ragged_skip_gran; + bool adreno_has_large_buffer; bool adreno_use_large_buffer; + bool adreno_use_bin_kernels; + get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr; ggml_cl_compiler_version adreno_cl_compiler_version; + std::string kernel_compile_opts; // cached for lazy-compiled kernels. + int adreno_wave_size; cl_bool non_uniform_workgroups; @@ -438,6 +546,30 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_scales_trans; ggml_cl_buffer prealloc_act_trans; + // pool of persistent image1d_buffer views over kv-cache layers, keyed by + // (parent buffer, offset within parent) + // used by the img-variant KQ/KQV dispatch paths to avoid per-call + // clCreateSubBuffer + clCreateImage + pending-release-queue on long-context decode + struct ImagePoolKey { + uintptr_t buf; + uint64_t offset; + bool operator<(const ImagePoolKey & o) const { + if (buf != o.buf) return buf < o.buf; + return offset < o.offset; + } + }; + struct ImagePoolEntry { + cl_mem sub_buffer = nullptr; + cl_mem image = nullptr; + size_t k_bytes = 0; + cl_channel_type channel_data_type = CL_FLOAT; + }; + std::map kq_img_pool; + std::map kqv_img_pool; + + // pool for the on-device f16 buffer for kv-cache with non-FA quantized-K (q8_0/q4_0) + std::map dequant_f16_pool; + // prealloc buffers for src0 and src1 ggml_cl_buffer prealloc_src0; ggml_cl_buffer prealloc_src1; @@ -546,16 +678,13 @@ struct ggml_backend_opencl_context { cl_kernel kernel_diag_f32; cl_kernel kernel_soft_max, kernel_soft_max_4; cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16; - std::map, cl_kernel> kernels_flash_attn_f16; - std::map, cl_kernel> kernels_flash_attn_f16_q1; - std::map, cl_kernel> kernels_flash_attn_f32; - std::map, cl_kernel> kernels_flash_attn_f32_q1; - std::map, cl_kernel> kernels_flash_attn_f32_f16; - std::map, cl_kernel> kernels_flash_attn_f32_f16_q1; - std::map, int> kernels_flash_attn_bm; - std::map, int> kernels_flash_attn_bn; + ggml_opencl_fa_kernels fa; cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0; cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32; + cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32; + cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32; + cl_kernel kernel_set_rows_q4_0_i64, kernel_set_rows_q4_0_i32; + cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32; cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32; @@ -567,6 +696,15 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mat_f16_f32_l4_dr; cl_kernel kernel_mul_mat_f16_f32_l4_dr_ls; cl_kernel kernel_mul_mat_f16_f32_l4_dr_lq; + cl_kernel kernel_mul_mat_f16_f32_l4_x8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_tiled; cl_kernel kernel_adreno_xmem_pack_src_f32; cl_kernel kernel_adreno_xmem_prepack_weight_f16; @@ -575,6 +713,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mm_f16_f32_kqv; cl_kernel kernel_mul_mm_f16_f32_kq; cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v; + cl_kernel kernel_convert_block_q1_0, kernel_restore_block_q1_0; cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0; cl_kernel kernel_convert_block_q4_0_trans4_ns, kernel_restore_block_q4_0_trans4_ns; cl_kernel kernel_convert_block_q4_1, kernel_restore_block_q4_1; @@ -589,6 +728,10 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_mxfp4, kernel_convert_block_mxfp4_trans, kernel_restore_block_mxfp4, kernel_restore_block_mxfp4_trans; cl_kernel kernel_convert_block_mxfp4_trans4_ns, kernel_restore_block_mxfp4_trans4_ns; cl_kernel kernel_convert_block_q8_0, kernel_restore_block_q8_0, kernel_restore_block_q8_0_trans; + cl_kernel kernel_dequant_q8_0_f16_view_aos; + cl_kernel kernel_dequant_q8_0_f32_view_aos; + cl_kernel kernel_dequant_q4_0_f16_view_aos; + cl_kernel kernel_dequant_q4_0_f32_view_aos; cl_kernel kernel_convert_block_q6_K_noshuffle, kernel_restore_block_q6_K_noshuffle; cl_kernel kernel_convert_bf16_to_f16, kernel_convert_f16_to_bf16; cl_kernel kernel_mul_mat_q4_0_f32_8x_flat; @@ -610,6 +753,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_iq4_nl, kernel_restore_block_iq4_nl; cl_kernel kernel_convert_block_iq4_nl_noshuffle; cl_kernel kernel_restore_block_iq4_nl_noshuffle; + cl_kernel kernel_mul_mv_q1_0_f32, kernel_mul_mv_q1_0_f32_flat; cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat; cl_kernel kernel_mul_mv_q4_1_f32; cl_kernel kernel_mul_mv_q4_1_f32_flat; @@ -656,15 +800,15 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; cl_kernel kernel_timestep_embedding; - cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns; - cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns; + cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin; + cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin; cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns; cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns; - cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns; + cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin; cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns; cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns; cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32; - cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns; + cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin; cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; @@ -673,6 +817,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_id_mxfp4_f32_flat; cl_kernel kernel_mul_mm_f32_f32_l4_lm; cl_kernel kernel_mul_mm_f16_f32_l4_lm; + cl_kernel kernel_mul_mm_q1_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_1_f32_l4_lm; cl_kernel kernel_mul_mm_q5_0_f32_l4_lm; @@ -807,6 +952,20 @@ struct ggml_backend_opencl_context { #endif } + const void * get_adreno_bin_kernel(const std::string &kernel_name, size_t *bin_size) const { + if (!get_adreno_bin_kernel_func) { + return nullptr; + } + + size_t sz; + const void * kernel_bin = get_adreno_bin_kernel_func( + kernel_name.c_str(), device_name.c_str(), driver_version.c_str(), &sz); + if (bin_size) { + *bin_size = sz; + } + return kernel_bin; + } + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Transpose kernels cl_program program_transpose; @@ -828,8 +987,10 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; cl_kernel kernel_gemv_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q4_1_f32; - cl_kernel kernel_gemm_noshuffle_q8_0_f32; + cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; cl_kernel kernel_gemv_noshuffle_q8_0_f32; + cl_kernel kernel_gemm_noshuffle_q1_0_f32; + cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32; cl_kernel kernel_gemm_noshuffle_q4_k_f32; cl_kernel kernel_gemv_noshuffle_q6_K_f32; @@ -854,6 +1015,21 @@ struct ggml_backend_opencl_context { write_profiling_info(); profiling_results.clear(); #endif + // release pooled image1d_buffer views over KV cache layers. + for (auto & kv : kq_img_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } + } + kq_img_pool.clear(); + for (auto & kv : kqv_img_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } + } + kqv_img_pool.clear(); + for (auto & kv : dequant_f16_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + } + dequant_f16_pool.clear(); } } }; @@ -877,7 +1053,13 @@ inline std::string read_file(const std::string &path) { return text; } -static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts) { +// fatal=false returns NULL on compile failure instead of aborting; used for +// optional FA variants that may exhaust the Adreno compiler at large DK. +// when the compiler returns CL_OUT_OF_HOST_MEMORY/CL_OUT_OF_RESOURCES (seen with DK>=256/512) +// for FA programs, do clFinish the queue to free up resources, then rebuild (up to 3x) +// if retry_queue is provided +static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, bool fatal, const char *tag = nullptr, cl_command_queue retry_queue = nullptr) { + if (tag) { GGML_LOG_INFO("ggml_opencl: compiling %s\n", tag); } cl_program p; char *program_log; size_t program_size; @@ -886,9 +1068,57 @@ static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, co program_size = strlen(program_buffer); - p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err); + const int max_attempts = retry_queue ? 3 : 1; + for (int attempt = 0; attempt < max_attempts; ++attempt) { + p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err); + if(err < 0) { + GGML_LOG_ERROR("OpenCL error creating program"); + if (fatal) exit(1); + return NULL; + } + + err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL); + if (err == CL_SUCCESS) { + return p; + } + + const bool transient = (err == CL_OUT_OF_HOST_MEMORY || err == CL_OUT_OF_RESOURCES); + if (retry_queue && transient && attempt + 1 < max_attempts) { + clReleaseProgram(p); + GGML_LOG_WARN("ggml_opencl: transient compile failure (err=%d)%s%s — clFinish + retry (%d/%d)\n", + err, tag ? " building " : "", tag ? tag : "", attempt + 2, max_attempts); + clFinish(retry_queue); // drain in-flight ops holding driver host-heap + continue; + } + + clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*) malloc(log_size + 1); + program_log[log_size] = '\0'; + clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + GGML_LOG_ERROR("ggml_opencl: kernel compile error (err=%d)%s%s:\n\n%s\n", err, tag ? " building " : "", tag ? tag : "", program_log); + free(program_log); + clReleaseProgram(p); + if (fatal) { + exit(1); + } + return nullptr; + } + return NULL; +} + +static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts) { + return build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true); +} + +static cl_program build_program_from_binary(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, size_t bin_size = 0) { + cl_program p; + char *program_log; + size_t log_size; + int err; + + p = clCreateProgramWithBinary(ctx, 1, &dev, &bin_size, (const unsigned char**)&program_buffer, NULL, &err); if(err < 0) { - GGML_LOG_ERROR("OpenCL error creating program"); + GGML_LOG_ERROR("OpenCL error creating program from binary"); exit(1); } @@ -932,82 +1162,15 @@ static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { } } -static void load_cl_kernels_flash_attn(ggml_backend_opencl_context *backend_ctx) { - // compiler options for general kernels - auto opencl_c_std = - std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); - std::string compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable -cl-unsafe-math-optimizations" - " -cl-finite-math-only -cl-fast-relaxed-math"; - - // flash_attn - if (!backend_ctx->kernels_loaded_flash_attn) { - cl_int err; - - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_f16 { - #include "flash_attn_f16.cl.h" - }; - const std::string kernel_src_f32 { - #include "flash_attn_f32.cl.h" - }; - const std::string kernel_src_f32_f16 { - #include "flash_attn_f32_f16.cl.h" - }; - #else - const std::string kernel_src_f16 = read_file("flash_attn_f16.cl"); - const std::string kernel_src_f32 = read_file("flash_attn_f32.cl"); - const std::string kernel_src_f32_f16 = read_file("flash_attn_f32_f16.cl"); - #endif - - if (!kernel_src_f16.empty() && !kernel_src_f32.empty() && !kernel_src_f32_f16.empty()) { - const struct { int dk; int dv; int bm; int bn; } fa_dims[] = { - { 40, 40, 32, 32}, { 64, 64, 64, 64}, { 80, 80, 64, 32}, { 96, 96, 64, 32}, - {112, 112, 32, 32}, {128, 128, 32, 32}, {192, 128, 16, 16}, - {192, 192, 16, 16}, {256, 256, 16, 16}, - }; - - for (size_t i = 0; i < sizeof(fa_dims)/sizeof(fa_dims[0]); ++i) { - const int dk = fa_dims[i].dk; - const int dv = fa_dims[i].dv; - const int bm = fa_dims[i].bm; - const int bn = fa_dims[i].bn; - std::string OPTS = compile_opts + - " -D DK=" + std::to_string(dk) + - " -D DV=" + std::to_string(dv) + - " -D BLOCK_M=" + std::to_string(bm) + - " -D BLOCK_N=" + std::to_string(bn); - - cl_program prog_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f16.c_str(), OPTS); - cl_kernel k_f16, k_f16_q1; - CL_CHECK((k_f16 = clCreateKernel(prog_f16, "flash_attn_f16", &err), err)); - CL_CHECK((k_f16_q1 = clCreateKernel(prog_f16, "flash_attn_f16_q1", &err), err)); - backend_ctx->kernels_flash_attn_f16[{dk, dv}] = k_f16; - backend_ctx->kernels_flash_attn_f16_q1[{dk, dv}] = k_f16_q1; - CL_CHECK(clReleaseProgram(prog_f16)); - - cl_program prog_f32 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32.c_str(), OPTS); - cl_kernel k_f32, k_f32_q1; - CL_CHECK((k_f32 = clCreateKernel(prog_f32, "flash_attn_f32", &err), err)); - CL_CHECK((k_f32_q1 = clCreateKernel(prog_f32, "flash_attn_f32_q1", &err), err)); - backend_ctx->kernels_flash_attn_f32[{dk, dv}] = k_f32; - backend_ctx->kernels_flash_attn_f32_q1[{dk, dv}] = k_f32_q1; - CL_CHECK(clReleaseProgram(prog_f32)); - - cl_program prog_f32_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32_f16.c_str(), OPTS); - cl_kernel k_f32_f16, k_f32_f16_q1; - CL_CHECK((k_f32_f16 = clCreateKernel(prog_f32_f16, "flash_attn_f32_f16", &err), err)); - CL_CHECK((k_f32_f16_q1 = clCreateKernel(prog_f32_f16, "flash_attn_f32_f16_q1", &err), err)); - backend_ctx->kernels_flash_attn_f32_f16[{dk, dv}] = k_f32_f16; - backend_ctx->kernels_flash_attn_f32_f16_q1[{dk, dv}] = k_f32_f16_q1; - CL_CHECK(clReleaseProgram(prog_f32_f16)); - - backend_ctx->kernels_flash_attn_bm[{dk, dv}] = bm; - backend_ctx->kernels_flash_attn_bn[{dk, dv}] = bn; - } - backend_ctx->kernels_loaded_flash_attn = true; - } +static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) { +#ifndef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + return false; +#else + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; } + return backend_ctx->adreno_use_bin_kernels; +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS } static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { @@ -1028,6 +1191,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { compile_opts += " -qcom-enable-large-buffer "; } + backend_ctx->kernel_compile_opts = compile_opts; + GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels"); // add @@ -1150,6 +1315,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { backend_ctx->program_cvt = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_convert_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q1_0", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q1_0", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0", &err), err)); @@ -1189,6 +1356,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_convert_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0_trans", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q8_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f16_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q8_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f32_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q4_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f16_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q4_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f32_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K_noshuffle", &err), err)); @@ -1680,6 +1851,40 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mv_q1_0_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q1_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q1_0_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q1_0_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_iq4_nl_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1796,6 +2001,51 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); } + + cl_int err_x8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8", &err_x8); + if (err_x8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = nullptr; } + + cl_int err_x8p = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_pair", &err_x8p); + if (err_x8p != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; } + + cl_int err_x8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4", &err_x8g); + if (err_x8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; } + + cl_int err_x8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4_img", &err_x8gi); + if (err_x8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; } + + cl_int err_x8gi_r4 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img", &err_x8gi_r4); + if (err_x8gi_r4 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; } + + cl_int err_r2dk256 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img", &err_r2dk256); + if (err_r2dk256 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; } + + cl_int err_y8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8", &err_y8); + if (err_y8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = nullptr; } + + cl_int err_y8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa", &err_y8g); + if (err_y8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; } + + cl_int err_y8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa_img", &err_y8gi); + if (err_y8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; } GGML_LOG_CONT("."); } @@ -1985,6 +2235,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mm_q1_0_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q1_0_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mm_iq4_nl_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2680,6 +2947,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i32", &err), err)); GGML_LOG_CONT("."); } @@ -2926,6 +3201,44 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q1_0_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q1_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q1_0_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q1_0_f32.cl.h" + }; +#else + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl"); +#endif + + cl_program prog = build_program_from_source( + backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_noshuffle_general { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -3214,6 +3527,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q8_0_f32_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_f32_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + // gemv_noshuffle_general_q8_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -3315,6 +3646,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_moe_q4_1_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_1_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + // gemv_moe_mxfp4_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3381,6 +3730,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_moe_q4_0_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + // gemv_moe_q5_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3483,6 +3850,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_moe_q4_k_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + // gemv_moe_q5_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3580,9 +3965,27 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); - CL_CHECK(clReleaseProgram(prog)); - GGML_LOG_CONT("."); + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_moe_mxfp4_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } // moe_reorder_b @@ -3704,78 +4107,817 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { backend_ctx->kernels_loaded = true; } -// XXX static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { -// XXX static bool initialized = false; -// XXX static ggml_backend_opencl_context *backend_ctx = nullptr; - static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev); static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev); -namespace /* anonymous */ { -extern struct ggml_backend_device_i ggml_backend_opencl_device_i; -} - -// Look for available and suitable devices. -static std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) { - std::vector found_devices; +// FA per-(dk,dv) tile tuning table + GGML_OPENCL_FA_TUNE override parsing. +#include "fa_tune.h" + +// FA variant key for the per-(dk,dv,variant) lazy compile cache. +// Kernel built on first dispatch to reduce kernel loading time. +// NB - a warmup run is recommended to get all necessary FA variants compiled +// before actual runs. +enum ggml_opencl_fa_variant { + FA_VARIANT_PRE = 0, // prepass kernels (kv_pad, mask_pad, blk) + FA_VARIANT_F16 = 1, + FA_VARIANT_F32 = 2, + FA_VARIANT_F32_F16 = 3, + FA_VARIANT_Q8_0 = 4, + FA_VARIANT_Q4_0 = 5, + FA_VARIANT_F32_F16_SPLIT = 6, + FA_VARIANT_Q8_0_SPLIT = 7, + FA_VARIANT_Q4_0_SPLIT = 8, +}; -#ifdef GGML_OPENCL_PROFILING - GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); +static std::string ggml_opencl_fa_kernel_src(ggml_opencl_fa_variant v) { +#ifdef GGML_OPENCL_EMBED_KERNELS + switch (v) { + case FA_VARIANT_F16: + return std::string{ + #include "flash_attn_f16.cl.h" + }; + case FA_VARIANT_F32: + return std::string{ + #include "flash_attn_f32.cl.h" + }; + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: + return std::string{ + #include "flash_attn_f32_f16.cl.h" + }; + case FA_VARIANT_PRE: + return std::string{ + #include "flash_attn_pre_f16.cl.h" + }; + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q8_0.cl.h" + }; + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q4_0.cl.h" + }; + } + return {}; +#else + switch (v) { + case FA_VARIANT_F16: return read_file("flash_attn_f16.cl"); + case FA_VARIANT_F32: return read_file("flash_attn_f32.cl"); + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: return read_file("flash_attn_f32_f16.cl"); + case FA_VARIANT_PRE: return read_file("flash_attn_pre_f16.cl"); + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: return read_file("flash_attn_f32_q8_0.cl"); + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: return read_file("flash_attn_f32_q4_0.cl"); + } + return {}; #endif +} - struct cl_device; - struct cl_platform { - cl_platform_id id; - unsigned number; - char name[128]; - char vendor[128]; - struct cl_device * devices; - unsigned n_devices; - struct cl_device * default_device; - }; +static std::string ggml_opencl_fa_compile_opts(ggml_backend_opencl_context * backend_ctx, + const ggml_opencl_fa_dim * cfg, + ggml_opencl_fa_variant variant) { + std::string opts = backend_ctx->kernel_compile_opts + + " -D DK=" + std::to_string(cfg->dk) + + " -D DV=" + std::to_string(cfg->dv) + + " -D BLOCK_M=" + std::to_string(cfg->bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn); - struct cl_device { - struct cl_platform * platform; - cl_device_id id; - unsigned number; - cl_device_type type; - char name[128]; - char version[128]; - }; + if (backend_ctx->gpu_family == INTEL) { + opts += " -D FA_SG=32"; + } - enum { NPLAT = 16, NDEV = 16 }; + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + if (is_split) { + opts += " -D N_SPLIT=" + std::to_string(cfg->n_split); + if (backend_ctx->has_subgroup_shuffle) { + opts += backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + } + return opts; +} - struct cl_platform platforms[NPLAT]; - unsigned n_platforms = 0; - struct cl_device devices[NDEV]; - unsigned n_devices = 0; - struct cl_device * default_device = NULL; - unsigned default_platform_number = 0; +// only register when the kernel's required dispatch workgroup size is within +// the limit of the device's maximum workgroup size +static bool ggml_opencl_fa_kernel_fits_wg(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, size_t required_wg, + const char * name, int dk, int dv) { + if (kernel == NULL) { return false; } + const size_t dev_max = backend_ctx->max_workgroup_size; + if (dev_max < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d requires WG %zu > device max %zu; skipping registration (will fall back)\n", + name, dk, dv, required_wg, dev_max); + return false; + } + size_t kwg = 0; + cl_int err = clGetKernelWorkGroupInfo(kernel, backend_ctx->device, + CL_KERNEL_WORK_GROUP_SIZE, + sizeof(kwg), &kwg, NULL); + if (err != CL_SUCCESS) { + GGML_LOG_INFO("ggml_opencl: clGetKernelWorkGroupInfo failed for %s DK=%d DV=%d (err=%d); skipping registration\n", + name, dk, dv, err); + return false; + } + if (kwg < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d per-kernel max %zu < required %zu; skipping registration (will fall back)\n", + name, dk, dv, kwg, required_wg); + return false; + } + return true; +} - cl_platform_id platform_ids[NPLAT]; - if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { - GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); - return found_devices; +// Log private memory for an FA kernel. Enable via `GGML_OPENCL_FA_LOG_SPILL=1`. +// On Adreno non-zero private_mem means spilling to global memory due to resource +// constraint and usually causes performance degradation. +// (per-work-item, no cache locality) — a strong signal to pick a config +// with smaller per-thread state (e.g. larger N_SPLIT). +static void ggml_opencl_log_fa_kernel_spill(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, const char * name, int dk, int dv) { + static const bool enabled = []{ + const char * e = std::getenv("GGML_OPENCL_FA_LOG_SPILL"); + return e && e[0] && e[0] != '0'; + }(); + + if (!enabled || kernel == nullptr) { + return; } - for (unsigned i = 0; i < n_platforms; i++) { - struct cl_platform * p = &platforms[i]; - p->number = i; - p->id = platform_ids[i]; - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + cl_ulong priv_mem = 0; + if (clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, + sizeof(priv_mem), &priv_mem, NULL) == CL_SUCCESS) { + const char * tag = priv_mem > 0 ? "SPILL" : "ok"; + GGML_LOG_INFO("ggml_opencl: [%s] %s DK=%d DV=%d private_mem=%llu bytes\n", + tag, name, dk, dv, (unsigned long long) priv_mem); + } +} - cl_device_id device_ids[NDEV]; - cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); - if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { - p->n_devices = 0; - } else { - CL_CHECK(clGetDeviceIDsError); +static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backend_ctx, int dk, int dv) { + const std::pair dk_dv = {dk, dv}; + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; } - p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; - p->default_device = NULL; + } - for (unsigned j = 0; j < p->n_devices; j++) { + if (cfg == nullptr) { + GGML_ABORT("ggml_opencl: no flash_attn config for DK=%d DV=%d", dk, dv); + } + + // BM-tile metadata is consumed by the prefill dispatch (n_q_blocks / wg + // sizing) regardless of whether the prepass kernels are needed for this + // n_kv — set it unconditionally + backend_ctx->fa.f32_f16_bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.f32_f16_bn[{dk, dv}] = cfg->bn; + backend_ctx->fa.f32_f16_wg_size[{dk, dv}] = cfg->bm; + backend_ctx->fa.bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.bn[{dk, dv}] = cfg->bn; + + if (backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0) { return; } + + GGML_LOG_INFO("ggml_opencl: lazy-compiling flash_attn prepass for DK=%d DV=%d\n", dk, dv); + cl_int err; + const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_PRE); + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_PRE); + // retry when kernel compile fails + cl_program prog_pre_f16 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, "fa prepass f16", backend_ctx->queue); + if (!prog_pre_f16) { return; } + cl_kernel k_kv_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_kv_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_mask_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_mask_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_blk_f16 = clCreateKernel(prog_pre_f16, "flash_attn_blk_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseKernel(k_mask_pad_f16); clReleaseProgram(prog_pre_f16); return; } + backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; + backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; + backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; + clReleaseProgram(prog_pre_f16); +} + +// DK=512 prefill BM-tile +static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) { + const int dk = 512, dv = 512; + const std::pair dk_dv = {dk, dv}; + auto & target = split ? backend_ctx->fa.f32_f16_split : backend_ctx->fa.f32_f16; + if (target.count(dk_dv) > 0) { return true; } + + static bool failed[2] = { false, false }; + if (failed[split ? 1 : 0]) { return false; } + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { cfg = &d; break; } + } + if (cfg == nullptr) { failed[split ? 1 : 0] = true; return false; } + if (split && cfg->n_split <= 1) { failed[1] = true; return false; } + + const ggml_opencl_fa_variant variant = split ? FA_VARIANT_F32_F16_SPLIT : FA_VARIANT_F32_F16; + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY"; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, + /*fatal=*/false, split ? "fa f32_f16 prefill512 split" : "fa f32_f16 prefill512", + backend_ctx->queue); + if (!prog) { failed[split ? 1 : 0] = true; return false; } + + cl_int err; + cl_kernel k = clCreateKernel(prog, "flash_attn_f32_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog); failed[split ? 1 : 0] = true; return false; } + target[dk_dv] = k; + if (split) { + backend_ctx->fa.f32_f16_split_wg_size[dk_dv] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[dk_dv] = cfg->nkv_split_threshold; + } + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, + split ? "flash_attn_f32_f16 (prefill512 split)" : "flash_attn_f32_f16 (prefill512)", dk, dv); + clReleaseProgram(prog); + + // determine whether to use the K-image variant of the split tile + static const char * pkimg_build_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_build = (pkimg_build_env != NULL) && (pkimg_build_env[0] != '0'); + if (split && pkimg_build && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) == 0) { + std::string opts_img = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + + " -D FA_PREFILL_ONLY -D FA_K_IMG -D FA_TILE_NAME=flash_attn_f32_f16_k_img"; + cl_program prog_img = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts_img, + /*fatal=*/false, "fa f32_f16 prefill512 split k_img", backend_ctx->queue); + if (prog_img) { + cl_int err_img; + cl_kernel k_img = clCreateKernel(prog_img, "flash_attn_f32_f16_k_img", &err_img); + if (err_img == CL_SUCCESS) { + backend_ctx->fa.f32_f16_split_k_img[dk_dv] = k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_img, + "flash_attn_f32_f16 (prefill512 split k_img)", dk, dv); + } + clReleaseProgram(prog_img); + } + } + return true; +} + +// Compile one (variant, dk, dv); memoised. false = compiler rejected. +static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_ctx, int dk, int dv, ggml_opencl_fa_variant variant) { + const std::pair dk_dv = {dk, dv}; + + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + if (cfg == nullptr) { + return false; + } + + // if a variant has already been compiled + switch (variant) { + case FA_VARIANT_F16: { + if (backend_ctx->fa.f16.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32: { + if (backend_ctx->fa.f32.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16: { + // The DK=512 decode-only program does not create the f32_f16 + // prefill kernel; check the q1 kernel instead so that repeated + // calls return a consistent result. + const bool decode_only = (dk == 512); + if (decode_only ? (backend_ctx->fa.f32_f16_q1.count(dk_dv) > 0) + : (backend_ctx->fa.f32_f16.count(dk_dv) > 0)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0: { + if (backend_ctx->fa.f32_q8_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0: { + if (backend_ctx->fa.f32_q4_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16_SPLIT: { + if (backend_ctx->fa.f32_f16_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0_SPLIT: { + if (backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0_SPLIT: { + if (backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_PRE: { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, dk, dv); + return true; + } + } + + // not registered but attempted - meaning these kernels failed to compile + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } + backend_ctx->fa.variant_attempted.insert(attempt_key); + + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + const bool is_quant = variant == FA_VARIANT_Q8_0 || variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0 || variant == FA_VARIANT_Q4_0_SPLIT; + if (is_quant && (dk % 32 != 0 || dv % 32 != 0)) { + return false; + } + if (is_split && cfg->n_split <= 1) { + return false; + } + if ((variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT) && + ((dk / 32) % cfg->n_split != 0 || (dv / 4) % cfg->n_split != 0)) { + return false; + } + + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { return false; } + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant); + + // bypass kernels for DK=512 + const bool fa_decode_only = (variant == FA_VARIANT_F32_F16 && dk == 512); + if (fa_decode_only) { + opts += " -D FA_DECODE_ONLY -D FA_DECODE_MINIMAL"; + } + + const char * tag = nullptr; + switch (variant) { + case FA_VARIANT_F16: tag = "fa f16"; break; + case FA_VARIANT_F32: tag = "fa f32"; break; + case FA_VARIANT_F32_F16: tag = "fa f32_f16"; break; + case FA_VARIANT_Q8_0: tag = "fa q8_0"; break; + case FA_VARIANT_Q4_0: tag = "fa q4_0"; break; + case FA_VARIANT_F32_F16_SPLIT: tag = "fa f32_f16 split"; break; + case FA_VARIANT_Q8_0_SPLIT: tag = "fa q8_0 split"; break; + case FA_VARIANT_Q4_0_SPLIT: tag = "fa q4_0 split"; break; + default: break; + } + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, tag, backend_ctx->queue); + if (!prog) { return false; } + + cl_int err; + switch (variant) { + case FA_VARIANT_F16: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f16", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f16_q1", &err), err)); + backend_ctx->fa.f16[{dk, dv}] = k; + backend_ctx->fa.f16_q1[{dk, dv}] = kq1; + break; + } + case FA_VARIANT_F32: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_q1", &err), err)); + backend_ctx->fa.f32[{dk, dv}] = k; + backend_ctx->fa.f32_q1[{dk, dv}] = kq1; + break; + } + case FA_VARIANT_F32_F16: { + cl_kernel kq1; + // BM-tile prefill kernel is excluded from the decode-only (DK=512) + if (!fa_decode_only) { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, "flash_attn_f32_f16", dk, dv); + } + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_f16_q1", &err), err)); + backend_ctx->fa.f32_f16_q1[{dk, dv}] = kq1; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, "flash_attn_f32_f16_q1", dk, dv); + cl_kernel k_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_split", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_f16_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, "flash_attn_f32_f16_q1_split", dk, dv); + } + // q1_vec decode kernel (DV-split + subgroup reduce) + cl_kernel k_q1_vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + "flash_attn_f32_f16_q1_vec", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, "flash_attn_f32_f16_q1_vec", dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } + // KV-head-coalesced vec for high-GQA small models + cl_kernel k_q1_vec_mq = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq, 256, + "flash_attn_f32_f16_q1_vec_mq", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq[{dk, dv}] = k_q1_vec_mq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq, "flash_attn_f32_f16_q1_vec_mq", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq); + } + } + // KV-head-coalesced + flash-decoding split, reuses merge kernel + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + "flash_attn_f32_f16_q1_vec_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + // K-image variant of MQ_GQA=4 split + cl_kernel k_q1_vec_mq_split_k_img = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_k_img, 256, + "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img[{dk, dv}] = k_q1_vec_mq_split_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_k_img, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_k_img); + } + } + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + // local-tile decode variant + if (dk == 128 && dv == 128) { + cl_kernel k_lt = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_tile", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lt, 128, + "flash_attn_f32_f16_q1_local_tile", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_tile[{dk, dv}] = k_lt; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lt, "flash_attn_f32_f16_q1_local_tile", dk, dv); + } else { + clReleaseKernel(k_lt); + } + } + // hybrid local-tile + MQ + FD-split + cl_kernel k_lmq = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq, 64, + "flash_attn_f32_f16_q1_local_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split[{dk, dv}] = k_lmq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq, "flash_attn_f32_f16_q1_local_mq_split", dk, dv); + } else { + clReleaseKernel(k_lmq); + } + } + } + + // second compile of the same source with -DMQ_GQA=8. + // FA_MQ_ONLY keeps only the vec_mq kernels so that the program + // compiles within the Adreno compiler's memory budget at DK>=256. + const std::string opts_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3 -D FA_MQ_ONLY"; + cl_program prog_g8 = fa_decode_only ? nullptr : build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8, + /*fatal=*/false, "fa f32_f16 MQ_GQA=8", backend_ctx->queue); + if (prog_g8) { + const size_t mq_g8_required_wg = 192; // Q1_WG_SIZE(64) * MQ_NSG_SPLIT(3) + cl_kernel k_q1_vec_mq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_g8[{dk, dv}] = k_q1_vec_mq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_g8, "flash_attn_f32_f16_q1_vec_mq_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_g8); + } + } + cl_kernel k_q1_vec_mq_split_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8[{dk, dv}] = k_q1_vec_mq_split_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8, "flash_attn_f32_f16_q1_vec_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8); + } + } + // K-image variant + cl_kernel k_q1_vec_mq_split_g8_k_img = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_k_img, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split_k_img (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img[{dk, dv}] = k_q1_vec_mq_split_g8_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_k_img, "flash_attn_f32_f16_q1_vec_mq_split_g8_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8_k_img); + } + } + // hybrid local-tile + MQ_GQA=8 + if (dk == 128 && dv == 128) { + cl_kernel k_lmq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq_g8, 64, + "flash_attn_f32_f16_q1_local_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split_g8[{dk, dv}] = k_lmq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq_g8, "flash_attn_f32_f16_q1_local_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_lmq_g8); + } + } + } + clReleaseProgram(prog_g8); + } + break; + } + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q4_0: { + const bool is_q8 = variant == FA_VARIANT_Q8_0; + const std::string base = is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0"; + const std::string name_q1 = base + "_q1"; + const std::string name_q1_split = base + "_q1_split"; + auto & m_q1 = is_q8 ? backend_ctx->fa.f32_q8_0_q1 : backend_ctx->fa.f32_q4_0_q1; + auto & m_prefill = is_q8 ? backend_ctx->fa.f32_q8_0 : backend_ctx->fa.f32_q4_0; + auto & m_q1_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_split : backend_ctx->fa.f32_q4_0_q1_split; + + cl_kernel k, kq1; + CL_CHECK((kq1 = clCreateKernel(prog, name_q1.c_str(), &err), err)); + CL_CHECK((k = clCreateKernel(prog, base.c_str(), &err), err)); + m_q1[{dk, dv}] = kq1; + m_prefill[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, name_q1.c_str(), dk, dv); + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, base.c_str(), dk, dv); + cl_kernel k_split = clCreateKernel(prog, name_q1_split.c_str(), &err); + if (err == CL_SUCCESS) { + m_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, name_q1_split.c_str(), dk, dv); + } + + // DV-split decode variant (q1_vec) + auto & m_q1_vec = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec : backend_ctx->fa.f32_q4_0_q1_vec; + const std::string name_q1_vec = name_q1 + "_vec"; + cl_kernel k_q1_vec = clCreateKernel(prog, name_q1_vec.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + name_q1_vec.c_str(), dk, dv)) { + m_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, name_q1_vec.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } + + // KV-head-coalesced + flash-decoding split + auto & m_mq_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split; + const std::string name_mq_split = name_q1 + "_vec_mq_split"; + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + name_mq_split.c_str(), dk, dv)) { + m_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + if (!backend_ctx->fa.f32_merge.count({dk, dv})) { + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + } + // Second compile with MQ_GQA=8, MQ_NSG=3, MQ_NSG_SPLIT=3 + auto & m_mq_split_g8 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8 + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8; + const std::string opts_mq_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3"; + cl_program prog_mq_g8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_mq_g8, + /*fatal=*/false, is_q8 ? "fa q8_0 MQ_GQA=8" : "fa q4_0 MQ_GQA=8", + backend_ctx->queue); + if (prog_mq_g8) { + const size_t mq_g8_required_wg = 192; + cl_kernel k_g8 = clCreateKernel(prog_mq_g8, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8, mq_g8_required_wg, + name_mq_split.c_str(), dk, dv)) { + m_mq_split_g8[{dk, dv}] = k_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_g8); + } + } + clReleaseProgram(prog_mq_g8); + } + break; + } + case FA_VARIANT_F32_F16_SPLIT: { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16_split[{dk, dv}] = k; + backend_ctx->fa.f32_f16_split_wg_size[{dk, dv}] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[{dk, dv}] = cfg->nkv_split_threshold; + break; + } + case FA_VARIANT_Q8_0_SPLIT: + case FA_VARIANT_Q4_0_SPLIT: { + const bool is_q8 = variant == FA_VARIANT_Q8_0_SPLIT; + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0", &err), err)); + auto & split = is_q8 ? backend_ctx->fa.f32_q8_0_split : backend_ctx->fa.f32_q4_0_split; + auto & split_wg = is_q8 ? backend_ctx->fa.f32_q8_0_split_wg_size : backend_ctx->fa.f32_q4_0_split_wg_size; + auto & split_bm = is_q8 ? backend_ctx->fa.f32_q8_0_split_bm : backend_ctx->fa.f32_q4_0_split_bm; + auto & split_thresh = is_q8 ? backend_ctx->fa.f32_q8_0_split_nkv_threshold : backend_ctx->fa.f32_q4_0_split_nkv_threshold; + split[{dk, dv}] = k; + split_wg[{dk, dv}] = cfg->bm * cfg->n_split; + split_bm[{dk, dv}] = cfg->bm; + split_thresh[{dk, dv}] = 0; // quant prefill: always split + break; + } + default: + break; + } + CL_CHECK(clReleaseProgram(prog)); + return true; +} + +// Compile a quant FA split kernel with a hand-picked (BLOCK_M, N_SPLIT) that +// overrides the default fa_dims tuning, for the DK values where the default +// N_SPLIT is degenerate for quant prefill: +// DK=256: default N_SPLIT=16 leaves DK/32=8 blocks -> 0 blocks/split. +// Override N_SPLIT=8 (1 block/split), BLOCK_M=16. +// DK=96 : DK/32 = 3 blocks, not divisible by the default N_SPLIT=2 -> +// override N_SPLIT=3. BLOCK_M must be 16, not 32: the N_SPLIT=3 +// QK-partial reduction uses sub_group_shuffle, so all 3 split +// threads of a query must land in one subgroup — WG_SIZE = +// BLOCK_M*N_SPLIT must be <= the 64-lane Adreno subgroup (16*3=48). +static bool ggml_opencl_ensure_fa_quant_split_override( + ggml_backend_opencl_context * backend_ctx, + int dk, int dv, int quant_bm, int quant_n_split, bool is_q8_0 +) { + const std::pair dk_dv = {dk, dv}; + if (is_q8_0 && backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + if (!is_q8_0 && backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } + + const ggml_opencl_fa_variant variant = is_q8_0 ? FA_VARIANT_Q8_0_SPLIT : FA_VARIANT_Q4_0_SPLIT; + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } + + backend_ctx->fa.variant_attempted.insert(attempt_key); + + std::string shuffle_opts; + if (backend_ctx->has_subgroup_shuffle) { + shuffle_opts = backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + if (cfg == nullptr) { + return false; + } + + // BLK_PREPASS_BM is the prepass-kernel BLOCK_M, needed so the quant kernel + // indexes the blk[] classification buffer correctly. + std::string opts = backend_ctx->kernel_compile_opts + shuffle_opts + + " -D DK=" + std::to_string(dk) + + " -D DV=" + std::to_string(dv) + + " -D BLOCK_M=" + std::to_string(quant_bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn) + + " -D N_SPLIT=" + std::to_string(quant_n_split) + + " -D BLK_PREPASS_BM=" + std::to_string(cfg->bm); + + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { + return false; + } + + const std::string tag = std::string("fa ") + (is_q8_0 ? "q8_0" : "q4_0") + + " split DK=" + std::to_string(dk); + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, tag.c_str(), backend_ctx->queue); + if (!prog) { return false; } + cl_int err; + cl_kernel k; + if (is_q8_0) { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q8_0", &err), err)); + backend_ctx->fa.f32_q8_0_split[dk_dv] = k; + backend_ctx->fa.f32_q8_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q8_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q8_0_split_nkv_threshold[dk_dv] = 0; + } else { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q4_0", &err), err)); + backend_ctx->fa.f32_q4_0_split[dk_dv] = k; + backend_ctx->fa.f32_q4_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q4_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q4_0_split_nkv_threshold[dk_dv] = 0; + } + CL_CHECK(clReleaseProgram(prog)); + return true; +} + +namespace /* anonymous */ { +extern struct ggml_backend_device_i ggml_backend_opencl_device_i; +} + +// Look for available and suitable devices. +static std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) { + std::vector found_devices; + +#ifdef GGML_OPENCL_PROFILING + GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); +#endif + + struct cl_device; + struct cl_platform { + cl_platform_id id; + unsigned number; + char name[128]; + char vendor[128]; + struct cl_device * devices; + unsigned n_devices; + struct cl_device * default_device; + }; + + struct cl_device { + struct cl_platform * platform; + cl_device_id id; + unsigned number; + cl_device_type type; + char name[128]; + char version[128]; + }; + + enum { NPLAT = 16, NDEV = 16 }; + + struct cl_platform platforms[NPLAT]; + unsigned n_platforms = 0; + struct cl_device devices[NDEV]; + unsigned n_devices = 0; + struct cl_device * default_device = NULL; + unsigned default_platform_number = 0; + + cl_platform_id platform_ids[NPLAT]; + if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { + GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); + return found_devices; + } + + for (unsigned i = 0; i < n_platforms; i++) { + struct cl_platform * p = &platforms[i]; + p->number = i; + p->id = platform_ids[i]; + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + + cl_device_id device_ids[NDEV]; + cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); + if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { + p->n_devices = 0; + } else { + CL_CHECK(clGetDeviceIDsError); + } + p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; + p->default_device = NULL; + + for (unsigned j = 0; j < p->n_devices; j++) { struct cl_device * d = &devices[n_devices]; d->number = n_devices++; d->id = device_ids[j]; @@ -3955,6 +5097,8 @@ static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * backend_ctx->driver_version.c_str()); GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: subgroup shuffle support: %s\n", + backend_ctx->has_subgroup_shuffle ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", backend_ctx->fp16_support ? "true" : "false"); GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", @@ -4111,6 +5255,8 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { backend_ctx->gpu_family = dev_ctx->gpu_family; backend_ctx->adreno_gen = dev_ctx->adreno_gen; if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + ggml_cl_init_fa_dims_table(); + // Use wave size of 64 for all Adreno GPUs. backend_ctx->adreno_wave_size = 64; } @@ -4156,6 +5302,11 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { // check Adreno large buffer support backend_ctx->adreno_has_large_buffer = strstr(ext_buffer, "cl_qcom_large_buffer") != NULL; + // subgroup shuffle support (N_SPLIT>1 FA kernel) + backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer, "cl_qcom_subgroup_shuffle") != NULL; + backend_ctx->has_subgroup_shuffle = + strstr(ext_buffer, "cl_khr_subgroup_shuffle") != NULL || + backend_ctx->has_qcom_subgroup_shuffle; cl_uint base_align_in_bits; CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); @@ -4195,6 +5346,36 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && backend_ctx->gpu_family == GPU_FAMILY::ADRENO; + // ragged moe, unspecified or non-zero means enabled, set to 0 to disable + static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16"); + backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0); + + // ragged moe, tile-skip granularity (columns per skip-group): 8 = quarter (default), + // 16 = half (legacy), 32 = disabled. Override with GGML_OPENCL_MOE_RAGGED_GRAN={8,16,32} + static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); + backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; + +#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + // try loading adreno binary kernels if enabled + // if fails to load, builtin kernels will be used + { + dl_handle * kernel_lib_handle = dl_load_library(KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = false; + + if (kernel_lib_handle) { + backend_ctx->get_adreno_bin_kernel_func = (get_adreno_bin_kernel_func_t)dl_get_sym(kernel_lib_handle, "get_adreno_kernels"); + if (backend_ctx->get_adreno_bin_kernel_func) { + GGML_LOG_INFO("ggml_opencl: loaded bin kernel library %s\n", KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = true; + } else { + GGML_LOG_INFO("ggml_opencl: bin kernel library %s is invalid, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } else { + GGML_LOG_INFO("ggml_opencl: failed to load %s, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS + cl_int err; // A local ref of cl_context for convenience @@ -4350,20 +5531,53 @@ struct ggml_tensor_extra_cl { } }; -// Additional tensor extra structs for quantized tensors. -// These tensors are loaded from files and should not be allocated in scratch -- -// they should always be allocated from the pool. Hence, they do not have an -// `offset`, which indicate their locations in the scratch buffer. -struct ggml_tensor_extra_cl_q4_0 { - // Quantized values. +struct ggml_tensor_extra_cl_q1_0 { cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. cl_mem q_img = nullptr; - // Scales. + cl_mem d = nullptr; - // Scales in image1d_buffer_t. cl_mem d_img = nullptr; - // Size of quantized values. + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q1_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +// Additional tensor extra structs for quantized tensors. +// These tensors are loaded from files and should not be allocated in scratch -- +// they should always be allocated from the pool. Hence, they do not have an +// `offset`, which indicate their locations in the scratch buffer. +struct ggml_tensor_extra_cl_q4_0 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Size of quantized values. size_t size_q = 0; // Size of scales. size_t size_d = 0; @@ -5028,8 +6242,14 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c threshold_ne0 = 128; threshold_ne1 = 128; } - return tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && + bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && tensor->ne[2] == 1 && tensor->ne[3] == 1; + + // q6_K adreno kernels requires ne1 is multiple of 128 + if (tensor->type == GGML_TYPE_Q6_K) { + return threashold_ok && tensor->ne[1] % 128 == 0; + } + return threashold_ok; } inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { @@ -5059,6 +6279,19 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) { // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. // note that this forces large M weights to use LM GEMM. + // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the + // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of + // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. + // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe + // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every + // odd column of the transposed layout (gross garbage) and dropping the last row; + // other non-multiples over-dispatch and write past the end of dst. Route such + // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back + // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of + // 128 and keep the noshuffle path. + if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1; } @@ -5100,6 +6333,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te switch (op->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_0: return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); default: return false; @@ -5175,9 +6410,10 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_UNARY_OP_TANH: case GGML_UNARY_OP_NEG: case GGML_UNARY_OP_EXP: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + // Adreno F16 exp/expm1 overflow even post-half->float convert. + return op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_EXPM1: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + return op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_SOFTPLUS: return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; default: @@ -5250,7 +6486,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return true; } else if (op->src[0]->type == GGML_TYPE_F32) { return op->src[1]->type == GGML_TYPE_F32; - } else if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_1 || + } else if (op->src[0]->type == GGML_TYPE_Q1_0) { + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q4_0) { + // Non-contig src0 routes through on-device dequant-to-f16. + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || op->src[0]->type == GGML_TYPE_MXFP4 || op->src[0]->type == GGML_TYPE_IQ4_NL || @@ -5339,43 +6580,81 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_FLASH_ATTN_EXT: - { - load_cl_kernels_flash_attn(backend_ctx); + case GGML_OP_FLASH_ATTN_EXT: { + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * v = op->src[2]; + + const int dk = q->ne[0]; + const int dv = v->ne[0]; + + const struct { int dk; int dv; } supported_dims[] = { + { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, + {112, 112}, {128, 128}, {192, 128}, + {192, 192}, {256, 256}, + {512, 512}, + }; - const ggml_tensor * q = op->src[0]; - const ggml_tensor * k = op->src[1]; - const ggml_tensor * v = op->src[2]; + bool dims_supported = false; + for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { + if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { + dims_supported = true; + break; + } + } + if (!dims_supported) { + return false; + } - const int dk = q->ne[0]; - const int dv = v->ne[0]; + const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && + v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; + const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && + v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; + const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && + v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; + + // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. + auto is_kv_type_ok = [](ggml_type t) { + return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || + t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q8_0; + }; + const bool is_f32_asym = q->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + k->type != v->type && + is_kv_type_ok(k->type) && is_kv_type_ok(v->type); - const struct { int dk; int dv; } supported_dims[] = { - { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, - {112, 112}, {128, 128}, {192, 128}, - {192, 192}, {256, 256}, - }; + const bool kv_combo_ok = is_f32_f32 || is_f16_f16 || is_f32_f16 || + is_f32_q8_0 || is_f32_q4_0 || is_f32_asym; + if (!kv_combo_ok) { + return false; + } - bool dims_supported = false; - for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { - if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { - dims_supported = true; - break; - } + if (dk == 512) { + if (backend_ctx->gpu_family == INTEL) { + return false; } - if (!dims_supported) { + if (!is_f32_f16) { return false; } - - const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && - v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; - const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; - - return is_f32_f32 || is_f16_f16 || is_f32_f16; + if (q->ne[1] == 1) { + // DK=512 decode is bandwidth-bound and slower on the GPU + // than on the CPU; decline it here so it runs on the CPU. + // Prefill (n_q > 1) stays on the GPU. + return false; + } else { + // prefill, BM-tile in its own FA_PREFILL_ONLY program + if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) { + return false; + } + } } + return true; + } default: return false; } @@ -5491,6 +6770,12 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { delete e; } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) { + delete e; + } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + delete e; + } for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) { delete e; } @@ -5532,6 +6817,21 @@ struct ggml_backend_opencl_buffer_context { return extra; } + ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() { + ggml_tensor_extra_cl_q1_0 * extra; + if (temp_tensor_extras_q1_0.empty()) { + extra = new ggml_tensor_extra_cl_q1_0(); + } else { + extra = temp_tensor_extras_q1_0.back(); + temp_tensor_extras_q1_0.pop_back(); + } + + temp_tensor_extras_q1_0_in_use.push_back(extra); + + extra->reset(); + return extra; + } + ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { ggml_tensor_extra_cl_q4_0 * extra; if (temp_tensor_extras_q4_0.empty()) { @@ -5688,6 +6988,11 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_in_use.clear(); + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + temp_tensor_extras_q1_0.push_back(e); + } + temp_tensor_extras_q1_0_in_use.clear(); + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { temp_tensor_extras_q4_0.push_back(e); } @@ -5737,6 +7042,9 @@ struct ggml_backend_opencl_buffer_context { temp_tensor_extras_q6_K.push_back(e); } temp_tensor_extras_q6_K_in_use.clear(); + + q8_0_soa_tensors.clear(); + q4_0_soa_tensors.clear(); } // Pools for extras. Available extras are in `temp_tensor_extras`. Extras @@ -5746,6 +7054,8 @@ struct ggml_backend_opencl_buffer_context { // for reuse. std::vector temp_tensor_extras; std::vector temp_tensor_extras_in_use; + std::vector temp_tensor_extras_q1_0; + std::vector temp_tensor_extras_q1_0_in_use; std::vector temp_tensor_extras_q4_0; std::vector temp_tensor_extras_q4_0_in_use; std::vector temp_tensor_extras_q4_1; @@ -5767,6 +7077,17 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; + // q8_0 tensors with AoS->SoA layout conversion installed by set_tensor. + // Two types of tensors get SOA'ed - normal weights and MoE weights. + // In Q8_0's case, we only have normal weights. If we ever have Q8_0 as MoE + // weights, they need to be added to this set in `set_tensors`. + std::unordered_set q8_0_soa_tensors; + + // Same for q4_0. KV-cache q4_0 tensors are allocated but never pass + // through set_tensor, so they stay AoS and aren't in this set. + // In Q4_0's case, in addition to normal weights, we have MoE weights. + std::unordered_set q4_0_soa_tensors; + // The buffer_context is initially created by ggml_backend_buft_alloc_buffer // before any tensor is initialized (at the beginning of alloc_tensor_range). // Hence, there is always a buffer object in this vector. When each tensor is @@ -5842,12 +7163,92 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, cl_command_queue queue = backend_ctx->queue; #ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0(); + + // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + + // q is uint32 (32 sign bits each); d is one half per 128-block. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + int M = tensor->ne[1]; // ne01 + int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % 128 == 0); + GGML_ASSERT(M % 4 == 0); + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M); + } // end transpose +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + return; + } // We separate the quantized bits and scale from block_q4_0 by using an // additional kernel, where each thread handles a block. We first read the // original weights into a temporary buffer, then create two separate // buffers for quantized bits and scales, which are then populated by the // conversion kernel. if (tensor->type == GGML_TYPE_Q4_0) { + // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). + if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + return; + } // Tensors should have been preallocated, therefore they should // already have ggml_tensor_extra_cl as extra. ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; @@ -5937,6 +7338,8 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, }; extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); tensor->extra = extra; + // MoE tensors are also SOA'ed + ctx->q4_0_soa_tensors.insert(tensor); return; } @@ -5965,6 +7368,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; + ctx->q4_0_soa_tensors.insert(tensor); // transpose the weights and scales #ifdef GGML_OPENCL_USE_ADRENO_KERNELS @@ -6516,6 +7920,11 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, return; } if (tensor->type == GGML_TYPE_Q8_0) { + // Views share the parent's buffer; parent owns SoA conversion. + if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + return; + } + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); @@ -6571,6 +7980,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); tensor->extra = extra; + ctx->q8_0_soa_tensors.insert(tensor); // Transpose the weights and scales #ifdef GGML_OPENCL_USE_ADRENO_KERNELS @@ -7219,6 +8629,63 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, sync_with_other_backends(backend_ctx); #ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_q; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } // In end-to-end runs, get_tensor is usually used to get back the logits, // where we can simply do clEnqueueReadBuffer since they are f32. // However, in test-backend-ops, the GPU graph is copied to the CPU backend, @@ -7226,7 +8693,18 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, // To properly support this, we need to restore block_q4_0 struct arrays // from the flattened buffers. if (tensor->type == GGML_TYPE_Q4_0) { - ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)tensor->extra; + // KV-cache q4_0 stays AoS — direct readback, no SoA restore. + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueReadBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + return; + } + // SoA extra lives on the parent tensor — follow view_src. + const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; + ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)extra_src->extra; #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, tensor)) { @@ -7697,7 +9175,18 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } if (tensor->type == GGML_TYPE_Q8_0) { - ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)tensor->extra; + // KV-cache q8_0 stays AoS (see Q4_0 branch). + if (!ggml_cl_is_q8_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueReadBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + return; + } + // SoA extra lives on the parent — follow view_src. + const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; + ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; cl_int err; cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, @@ -8306,12 +9795,30 @@ static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer UNUSED(buft); } +static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); +#ifdef GGML_OPENCL_SOA_Q + // set_tensor carves quantized weights into per-component subbuffers (d/q, + // ql/qh/s/d, ...) whose origins are each rounded up to the device base + // alignment. When a component's size is not a multiple of the alignment + // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte + // alignment), the aligned carve extends past ggml_nbytes and the last + // subbuffer would overlap the next tensor in the pool. Reserve the worst-case + // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. + if (ggml_is_quantized(tensor->type)) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; + size += 4 * dev_ctx->backend_ctx->alignment; + } +#endif // GGML_OPENCL_SOA_Q + return size; +} + static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, - /* .get_alloc_size = */ NULL, + /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, /* .is_host = */ NULL, }; @@ -8821,6 +10328,34 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +// check if a Q8_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q8_0 || tensor->buffer == nullptr) { + return false; + } + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q8_0_soa_tensors.count(key) > 0; +} + +// check if a Q4_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q4_0 || tensor->buffer == nullptr) { + return false; + } + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q4_0_soa_tensors.count(key) > 0; +} + static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -8834,26 +10369,14 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c // ne2 = ne02 // ne3 = ne03 - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; - - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - const int ne0 = dst->ne[0]; + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); const int nblk0 = ne0/ggml_blck_size(dst->type); @@ -8861,31 +10384,49 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + + const bool q8_0_soa = dst->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(dst); + const bool q4_0_soa = dst->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(dst); + const bool is_soa = q8_0_soa || q4_0_soa; cl_kernel kernel; - switch (dst->type) { - case GGML_TYPE_F32: - if (src1->type == GGML_TYPE_I64) { - kernel = backend_ctx->kernel_set_rows_f32_i64; - } else { - kernel = backend_ctx->kernel_set_rows_f32_i32; - } - break; - case GGML_TYPE_F16: - if (src1->type == GGML_TYPE_I64) { - kernel = backend_ctx->kernel_set_rows_f16_i64; - } else { - kernel = backend_ctx->kernel_set_rows_f16_i32; - } - break; - default: - GGML_ABORT("not implemented"); + if (q8_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_soa_i64 + : backend_ctx->kernel_set_rows_q8_0_soa_i32; + } else if (q4_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_soa_i64 + : backend_ctx->kernel_set_rows_q4_0_soa_i32; + } else { + switch (dst->type) { + case GGML_TYPE_F32: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f32_i64 + : backend_ctx->kernel_set_rows_f32_i32; + break; + case GGML_TYPE_F16: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f16_i64 + : backend_ctx->kernel_set_rows_f16_i32; + break; + case GGML_TYPE_Q8_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_i64 + : backend_ctx->kernel_set_rows_q8_0_i32; + break; + case GGML_TYPE_Q4_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_i64 + : backend_ctx->kernel_set_rows_q4_0_i32; + break; + default: + GGML_ABORT("not implemented"); + } } fastdiv_vals ne11_ = init_fastdiv_values(ne11); @@ -8895,21 +10436,65 @@ static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, c CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); + + if (is_soa) { + // The q/d subbuffers in q8_0/q4_0 extras are interchangeable here. + // For views (e.g. ggml_set_rows' `out`), follow view_src for the SoA extra. + const ggml_tensor * soa_src = dst->view_src != nullptr ? dst->view_src : dst; + cl_mem q_mem = nullptr; + cl_mem d_mem = nullptr; + if (q8_0_soa) { + ggml_tensor_extra_cl_q8_0 * e = (ggml_tensor_extra_cl_q8_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } else { + ggml_tensor_extra_cl_q4_0 * e = (ggml_tensor_extra_cl_q4_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } + cl_ulong offset_q = 0; + cl_ulong offset_d = 0; + const int ne1_dst = dst->ne[1]; + const int ne2_dst = dst->ne[2]; + const int ne3_dst = dst->ne[3]; + + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &q_mem)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_mem)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_d)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1_dst)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne2_dst)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne3_dst)); + } else { + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); + } int nth0 = 64; if (backend_ctx->gpu_family == INTEL) { @@ -11483,14 +13068,386 @@ static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); } +struct ggml_cl_flash_attn_temp_buffer { + cl_mem data = nullptr; + + ~ggml_cl_flash_attn_temp_buffer() { + if (data != nullptr) { + CL_CHECK(clReleaseMemObject(data)); + data = nullptr; + } + } +}; + +// Resolve the source buffer + strides for an FA KV tensor: keep the +// caller-supplied AoS buffer if non-NULL, else fall back to tensor->extra. +static void ggml_cl_flash_attn_resolve_src( + const ggml_tensor * tensor, + cl_mem & buf, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3) { + if (buf != NULL) { + return; + } + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + buf = extra->data_device; + offset = extra->offset + tensor->view_offs; + nb1 = tensor->nb[1]; + nb2 = tensor->nb[2]; + nb3 = tensor->nb[3]; +} + +// Read a (possibly strided-view) tensor from device into a tight host buffer. +// dim 0 is always tight; a strided view is gathered row-by-row. +static void ggml_cl_flash_attn_read_tensor_host( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem src_buffer, cl_ulong src_offset, + cl_ulong src_nb1, cl_ulong src_nb2, cl_ulong src_nb3, + size_t row_bytes, void * dst, size_t total_bytes +) { + const bool contiguous_layout = + src_nb1 == row_bytes && + src_nb2 == row_bytes * (cl_ulong) tensor->ne[1] && + src_nb3 == src_nb2 * (cl_ulong) tensor->ne[2]; + + if (contiguous_layout) { + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + src_offset, total_bytes, dst, 0, NULL, NULL)); + return; + } + + size_t dst_off = 0; + for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { + const cl_ulong row_src_off = src_offset + + (cl_ulong) i3 * src_nb3 + + (cl_ulong) i2 * src_nb2 + + (cl_ulong) i1 * src_nb1; + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + row_src_off, row_bytes, + (uint8_t *) dst + dst_off, 0, NULL, NULL)); + dst_off += row_bytes; + } + } + } + GGML_ASSERT(dst_off == total_bytes); +} + +// forward decl: used by the FA decode dispatch (K-image variant) below. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type); + +// Rebuild AoS q8_0/q4_0 bytes from a SoA tensor into a temp buffer. +// Returns false if the tensor is not SoA-quantised (already AoS). +static bool ggml_cl_flash_attn_reconstruct_aos( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + if (tensor == nullptr) { + return false; + } + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(tensor); + const bool is_q4_0 = tensor->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(tensor); + if (!is_q8_0 && !is_q4_0) { + return false; + } + + // For views, SoA extra is on view_src (view->extra is pre-SoA). + // Noshuffle layout only applies to 2D weights, as determined by `use_adreno_kernels`, + // where ne2 == 1 and ne3 == 1 -- these are never FA inputs. + // Therefore, we use `restore_block_qk_0` kernels, not `restore_block_qk_0_noshuffle`. + const ggml_tensor * soa_src = tensor->view_src ? tensor->view_src : tensor; + cl_mem extra_q = NULL; + cl_mem extra_d = NULL; + if (is_q8_0) { + auto * e = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; + } else { + auto * e = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; + } + + // Reconstruct the whole parent; view offsets then work naturally. + const size_t parent_nbytes = ggml_nbytes(soa_src); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = is_q8_0 ? backend_ctx->kernel_restore_block_q8_0 + : backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &temp.data)); + + const size_t n_blocks = (size_t) ggml_nelements(soa_src) / ggml_blck_size(soa_src->type); + size_t global_work_size[] = { n_blocks, 1, 1 }; + size_t local_work_size[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); + + out_buf = temp.data; + out_offset = tensor->view_offs; + out_nb1 = tensor->nb[1]; + out_nb2 = tensor->nb[2]; + out_nb3 = tensor->nb[3]; + return true; +} + +// GPU dequant of a contiguous q4_0/q8_0 KV tensor to f16/f32. Caller supplies +// src_buf when reconstructing from SoA. Returns false for non-contig layouts +// (the kernel indexes blocks tightly within ne[0]) so the caller can fall back +// to the host path. +static bool ggml_cl_flash_attn_dequant_kv_gpu( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + cl_mem in_src_buf, + cl_ulong in_src_offset, + cl_ulong in_src_nb1, + cl_ulong in_src_nb2, + cl_ulong in_src_nb3, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q4_0); + GGML_ASSERT(target_type == GGML_TYPE_F16 || target_type == GGML_TYPE_F32); + + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0; + + cl_mem src_buf = in_src_buf; + cl_ulong src_offset = in_src_offset; + cl_ulong src_nb1 = in_src_nb1; + cl_ulong src_nb2 = in_src_nb2; + cl_ulong src_nb3 = in_src_nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buf, src_offset, src_nb1, src_nb2, src_nb3); + + if (tensor->nb[0] != (cl_ulong) ggml_type_size(tensor->type)) { + return false; + } + + const size_t n_blocks = (size_t) ggml_nelements(tensor) / 32; // block size is 32 + const size_t elem_size = ggml_type_size(target_type); + const size_t out_bytes = n_blocks * 32 * elem_size; + const cl_int nblk0_arg = (cl_int) (tensor->ne[0] / 32); + const cl_int ne1_arg = (cl_int) tensor->ne[1]; + const cl_int ne2_arg = (cl_int) tensor->ne[2]; + const cl_int ne3_arg = (cl_int) tensor->ne[3]; + + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); + + cl_kernel kernel; + if (target_type == GGML_TYPE_F16) { + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + } else { + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f32_view_aos + : backend_ctx->kernel_dequant_q4_0_f32_view_aos; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &nblk0_arg)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne1_arg)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne2_arg)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne3_arg)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &temp.data)); + + size_t global_ws[3] = { (size_t) nblk0_arg, (size_t) ne1_arg, (size_t) ne2_arg * (size_t) ne3_arg }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_ws, NULL, 0, NULL, NULL)); + + out_buf = temp.data; + out_offset = 0; + out_nb1 = (cl_ulong) tensor->ne[0] * elem_size; + out_nb2 = out_nb1 * (cl_ulong) tensor->ne[1]; + out_nb3 = out_nb2 * (cl_ulong) tensor->ne[2]; + return true; +} + +static bool ggml_cl_flash_attn_prepare_quantized_tensor( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (!ggml_is_quantized(tensor->type)) { + return false; + } + + // Caller-supplied AoS buffer wins over tensor->extra when present. + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); + + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) (tensor->ne[0] / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + // tight-packed byte count (ggml_nbytes includes stride gaps). + const size_t total_bytes = (size_t) (n / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + std::vector host_quant(total_bytes); + + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_quant.data(), total_bytes); + + std::vector host_f32(n); + ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); + + const size_t bytes_per_elem = ggml_type_size(target_type); + const size_t buffer_size = (size_t) n * bytes_per_elem; + + std::vector host_linear(buffer_size); + if (target_type == GGML_TYPE_F32) { + memcpy(host_linear.data(), host_f32.data(), buffer_size); + } else { + GGML_ASSERT(target_type == GGML_TYPE_F16); + ggml_fp32_to_fp16_row(host_f32.data(), (ggml_fp16_t *) host_linear.data(), n); + } + + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, buffer_size, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, buffer_size, host_linear.data(), 0, NULL, NULL)); + + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * bytes_per_elem); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); + + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention dequantizes GPU-resident quantized KV cache into temporary linear buffers; performance may be poor\n"); + warned = true; + } + + return true; +} + +// Host-side F16 -> F32 for the asymmetric-KV F32 fallback path. +static bool ggml_cl_flash_attn_convert_f16_to_f32( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (tensor->type != GGML_TYPE_F16) { + return false; + } + + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); + + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) tensor->ne[0] * sizeof(ggml_fp16_t); + const size_t total_bytes = (size_t) n * sizeof(ggml_fp16_t); + std::vector host_f16(total_bytes); + + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_f16.data(), total_bytes); + + std::vector host_f32(n); + ggml_fp16_to_fp32_row((const ggml_fp16_t *) host_f16.data(), host_f32.data(), n); + + const size_t f32_bytes = (size_t) n * sizeof(float); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, f32_bytes, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, + f32_bytes, host_f32.data(), 0, NULL, NULL)); + + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * sizeof(float)); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); + + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention asymmetric KV converts an F16 cache to F32 host-side; performance may be poor\n"); + warned = true; + } + + return true; +} + +// Flash-Decoding (K-split) dispatch thresholds. FD fires for non-causal +// attention with n_kv >= FD_MIN_N_KV and d_head <= FD_MAX_DK; the KV range is +// split into ~n_kv/FD_KV_PER_SPLIT partials, clamped to [FD_MIN_SPLITS, +// FD_MAX_SPLITS]. Multi-query FD is restricted to small heads +// (d_head <= FD_MAX_DK_MULTI) and capped at FD_MAX_N_Q_MULTI queries. +static constexpr int FD_MIN_N_KV = 2048; +static constexpr int FD_KV_PER_SPLIT = 2048; +// f16 KV decode wants more splits than the 2048 default; quantized KV keeps 2048. +static constexpr int FD_KV_PER_SPLIT_F16 = 512; +static constexpr int FD_MIN_SPLITS = 2; +static constexpr int FD_MAX_SPLITS = 16; +static constexpr int FD_MAX_DK = 128; +static constexpr int FD_MAX_DK_MULTI = 64; +static constexpr int FD_MAX_N_Q_MULTI = 8; +// MQ FD split-groups have few subgroups (MQ_NSG_SPLIT), so use a smaller +// kv_per_split to keep the softmax recurrence short; non-MQ keeps FD_KV_PER_SPLIT. +static constexpr int FD_MQ_KV_PER_SPLIT = 256; +static constexpr int FD_MQ_MAX_SPLITS = 128; + static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { const ggml_tensor * v = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; + GGML_ASSERT(q->extra); GGML_ASSERT(k->extra); GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); + if (mask) { GGML_ASSERT(mask->extra); } @@ -11508,87 +13465,715 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co const int n_head_kv = k->ne[2]; const int n_batch = q->ne[3]; + // DK=512 (Gemma-4 global layers) runs decode-only (q1 / q1_split) on + // Adreno - it never uses the BM-tile path, and the prepass + split-tile + // programs OOM the compiler at DK=512; supports_op only admits + // n_q==1 here and prefill goes to CPU + const bool fa_decode_only_512 = (d_head_q == 512); + + // per-variant lazy compile for this (dk, dv) + // DK=512 decode (n_q==1) needs no prepass + // DK=512 prefill (n_q>1) does, so compile it only when needed + if (!fa_decode_only_512 || n_q > 1) { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, d_head_q, d_head_v); + } + cl_kernel kernel = NULL; + bool use_prefill_k_img = false; // K is image1d_buffer_t for DK=512 prefill const bool is_f16 = q->type == GGML_TYPE_F16; - const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16; - const std::pair dk_dv = {d_head_q, d_head_v}; - - if (n_q == 1) { - if (is_mixed) { - kernel = backend_ctx->kernels_flash_attn_f32_f16_q1.at(dk_dv); - } else if (is_f16) { - kernel = backend_ctx->kernels_flash_attn_f16_q1.at(dk_dv); + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; + + if (is_f16) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); + } else if (is_mixed) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16); + if (fa_decode_only_512) { + // DK=512: the BM-tile prefill kernels are specifically compiled from + // FA_PREFILL_ONLY + if (n_q > 1) { + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false); + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/true); + } } else { - kernel = backend_ctx->kernels_flash_attn_f32_q1.at(dk_dv); - } - } else { - if (is_mixed) { - kernel = backend_ctx->kernels_flash_attn_f32_f16.at(dk_dv); - } else if (is_f16) { - kernel = backend_ctx->kernels_flash_attn_f16.at(dk_dv); + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16_SPLIT); + } + } else if (is_q8_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/true); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/true); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0_SPLIT); + } + } else if (is_q4_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/false); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/false); } else { - kernel = backend_ctx->kernels_flash_attn_f32.at(dk_dv); + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0_SPLIT); } + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32); } - GGML_ASSERT(kernel != NULL); + + const std::pair dk_dv = {d_head_q, d_head_v}; + const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && + backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; + // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. + const bool use_native_q8_0 = is_q8_0 && n_q > 1 && + backend_ctx->fa.f32_q8_0.count(dk_dv) > 0; + const bool use_native_q4_0_q1 = is_q4_0 && n_q == 1 && + backend_ctx->fa.f32_q4_0_q1.count(dk_dv) > 0; + const bool use_native_q4_0 = is_q4_0 && n_q > 1 && + backend_ctx->fa.f32_q4_0.count(dk_dv) > 0; + const int block_m = n_q > 1 + ? (is_mixed ? backend_ctx->fa.f32_f16_bm.at(dk_dv) : backend_ctx->fa.bm.at(dk_dv)) + : 0; + // block_n is only used by the n_q > 1 prefill path; its map is not + // populated for DK=512 decode, so do not read it for decode. + const int block_n = (n_q > 1) + ? (is_mixed ? backend_ctx->fa.f32_f16_bn.at(dk_dv) + : backend_ctx->fa.bn.at(dk_dv)) + : 0; + // Pick split variant only when n_kv crosses the per-(dk,dv) threshold. + // the N_SPLIT>1 prefill tile reduces DK partials via subgroup shuffle, + // on Intel it uses the non-split BM tile and does not depend on subgroup size + const bool use_split_kernel = (n_q > 1 && is_mixed && + backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_f16_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_f16_split_nkv_threshold.at(dk_dv)); + const bool use_split_q8_0 = (use_native_q8_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q8_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q8_0_split_nkv_threshold.at(dk_dv)); + const bool use_split_q4_0 = (use_native_q4_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q4_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q4_0_split_nkv_threshold.at(dk_dv)); + const int wg_size_fa = (n_q > 1 && is_mixed) + ? (use_split_kernel + ? backend_ctx->fa.f32_f16_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_f16_wg_size.at(dk_dv)) + : block_m; ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; - ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; - ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; + // SoA q8_0/q4_0 K/V: data_device aliases the `q` subbuffer; reconstruct + // AoS into a temp buffer below. AoS tensors use extra_k/v->data_device. + const bool k_soa = ggml_cl_is_q8_0_soa(k) || ggml_cl_is_q4_0_soa(k); + const bool v_soa = ggml_cl_is_q8_0_soa(v) || ggml_cl_is_q4_0_soa(v); + ggml_tensor_extra_cl * extra_k = k_soa ? nullptr : (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = v_soa ? nullptr : (ggml_tensor_extra_cl *)v->extra; + cl_ulong offset_q = extra_q->offset + q->view_offs; - cl_ulong offset_k = extra_k->offset + k->view_offs; - cl_ulong offset_v = extra_v->offset + v->view_offs; + cl_ulong offset_k = k_soa ? 0 : extra_k->offset + k->view_offs; + cl_ulong offset_v = v_soa ? 0 : extra_v->offset + v->view_offs; cl_ulong offset_o = extra_o->offset + dst->view_offs; cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; - const cl_ulong q_nb1 = q->nb[1], q_nb2 = q->nb[2], q_nb3 = q->nb[3]; - const cl_ulong k_nb1 = k->nb[1], k_nb2 = k->nb[2], k_nb3 = k->nb[3]; - const cl_ulong v_nb1 = v->nb[1], v_nb2 = v->nb[2], v_nb3 = v->nb[3]; - const cl_ulong o_nb1 = dst->nb[1], o_nb2 = dst->nb[2], o_nb3 = dst->nb[3]; + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; + + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; + + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; + + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; const int mask_ne2 = mask ? mask->ne[2] : 0; const int mask_ne3 = mask ? mask->ne[3] : 0; - float scale, max_bias, logit_softcap; + float scale; + float max_bias; + float logit_softcap; + const float * params = (const float *)dst->op_params; scale = params[0]; max_bias = params[1]; logit_softcap = params[2]; + bool use_q1_vec = false; + bool use_q1_vec_mq = false; + bool use_local_tile = false; + // KV-head-coalesced gate: gqa_ratio == compile-time MQ_GQA + // restricts to DK=DV=256 for now due to local memory size + const int gqa_ratio_dispatch = n_head_kv > 0 ? (n_head / n_head_kv) : 0; + if (n_q == 1) { + if (use_native_q8_0_q1) { + if (d_head_v >= 256 && + backend_ctx->fa.f32_q8_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q8_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); + } + } else if (use_native_q4_0_q1) { + // q4_0 vec kernel uses per-lane dp4a (cl_khr_integer_dot_product) + const char * q4vec_env = getenv("GGML_OPENCL_FA_Q4_VEC"); + const bool q4vec_off = (q4vec_env != NULL) && (q4vec_env[0] == '0'); + if (!q4vec_off && d_head_v >= 256 && + backend_ctx->fa.f32_q4_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q4_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); + } + } else if (is_mixed) { + static const char * lt_env = getenv("GGML_OPENCL_FA_LOCAL_TILE"); + static const bool lt_on = (lt_env != NULL) && (lt_env[0] != '0'); + if (lt_on && d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_local_tile.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_local_tile.at(dk_dv); + use_local_tile = true; + } else { + static const char * f16_vec_dk128_env = getenv("GGML_OPENCL_FA_F16_VEC_DK128"); + static const bool f16_vec_dk128_off = (f16_vec_dk128_env != NULL) && (f16_vec_dk128_env[0] == '0'); + const int dv_gate = f16_vec_dk128_off ? 256 : 128; + if (d_head_v >= dv_gate && + backend_ctx->fa.f32_f16_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); + } + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16_q1.at(dk_dv); + } else { + kernel = backend_ctx->fa.f32_q1.at(dk_dv); + } + } else { + if (use_native_q8_0) { + kernel = use_split_q8_0 + ? backend_ctx->fa.f32_q8_0_split.at(dk_dv) + : backend_ctx->fa.f32_q8_0.at(dk_dv); + } else if (use_native_q4_0) { + kernel = use_split_q4_0 + ? backend_ctx->fa.f32_q4_0_split.at(dk_dv) + : backend_ctx->fa.f32_q4_0.at(dk_dv); + } else if (is_mixed) { + if (use_split_kernel) { + // DK=512 prefill: opt-in texture-cache K reads (image1d_buffer_t). + static const char * pkimg_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_on = (pkimg_env != NULL) && (pkimg_env[0] != '0'); + if (d_head_q == 512 && pkimg_on && + backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_split_k_img.at(dk_dv); + use_prefill_k_img = true; + } else { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + } + } else { + kernel = backend_ctx->fa.f32_f16.at(dk_dv); + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16.at(dk_dv); + } else { + kernel = backend_ctx->fa.f32.at(dk_dv); + } + } + + // Intel goes to the basic q1 kernel + if (backend_ctx->gpu_family == INTEL && n_q == 1) { + use_q1_vec = use_q1_vec_mq = use_local_tile = false; + if (is_mixed && backend_ctx->fa.f32_f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } + else if (is_f16 && backend_ctx->fa.f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } + else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } + else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } + else if (backend_ctx->fa.f32_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } + } + GGML_ASSERT(kernel != NULL); + + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_pad; + ggml_cl_flash_attn_temp_buffer temp_v_pad; + ggml_cl_flash_attn_temp_buffer temp_mask_pad; + ggml_cl_flash_attn_temp_buffer temp_blk; + const ggml_type kv_target_type = is_f16 ? GGML_TYPE_F16 : GGML_TYPE_F32; + + cl_mem k_data_device = k_soa ? NULL : extra_k->data_device; + cl_mem v_data_device = v_soa ? NULL : extra_v->data_device; + + // SoA q8_0/q4_0 -> reconstruct AoS for downstream kernels that expect + // tight records (no-op when k/v is already AoS). + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, k, temp_k_aos, + k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, v, temp_v_aos, + v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + + // currently FA kernels support KV cache with f16, f32, q4_0 and q8_0. + // there two cases that these kernels cannot cover, + // 1. KV cache types are q4_0 or q8_0, but the FA kernels fail to compile + // 2. KV cache types not currently supported by an FA kernel, e.g., q4_1 + // these two cases are supported here by dequantizing to f32/f16 and this + // causes performance degradation. + // For q4_0 or q8_0 cases that fail kernel compilation, dequant happens in GPU; + // for types that do not have FA kernels, dequant happens on host. + if (!use_native_q8_0_q1 && !use_native_q8_0 && + !use_native_q4_0_q1 && !use_native_q4_0) { + // for q4_0, q8_0 FA kernels that fail to compile + bool k_done = false; + bool v_done = false; + if (k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q4_0) { + k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, kv_target_type, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (v->type == GGML_TYPE_Q8_0 || v->type == GGML_TYPE_Q4_0) { + v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, kv_target_type, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + if (!k_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, k, kv_target_type, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (!v_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, v, kv_target_type, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + // Asymmetric KV on the F32 fallback path: convert the F16 side to F32 + // too. (Symmetric F16 / mixed paths handle F16 directly.) + if (kv_target_type == GGML_TYPE_F32 && !is_mixed && !is_f16) { + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, k, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, v, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + } + + cl_mem k_pad_buffer = NULL; + cl_mem v_pad_buffer = NULL; + cl_mem mask_pad_buffer = NULL; + cl_mem blk_buffer = NULL; + cl_ulong mask_pad_nb1 = 0; + cl_ulong mask_pad_nb2 = 0; + cl_ulong mask_pad_nb3 = 0; + + // Flash-Decoding K-split decision. Resolved here, before the prefill + // prepass, because KV-pad and blk prepass are pure overhead when FD fires. const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); + const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1; + cl_kernel fd_k_split = NULL; + bool use_fd_mq = false; + size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4) + bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset) + + { + const char * mq_env = getenv("GGML_OPENCL_FA_MQ"); + const bool mq_enabled = (mq_env == NULL) ? true : (mq_env[0] != '0'); + const bool mq_kv_ok = is_mixed || is_q8_0 || is_q4_0; + + const char * lmq_env = getenv("GGML_OPENCL_FA_LOCAL_MQ_SPLIT"); + const bool lmq_on = (lmq_env != NULL) && (lmq_env[0] != '0'); + + static const char * vec_nq_env = getenv("GGML_OPENCL_FA_VEC_NQ"); + static const int N_MAX_VEC_NQ = (vec_nq_env != NULL && vec_nq_env[0] != '\0') + ? atoi(vec_nq_env) : 1; + + const bool nq_in_vec_range = (n_q >= 1) && (n_q <= N_MAX_VEC_NQ); + const bool nq1_only = (n_q == 1); + if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal && + backend_ctx->gpu_family != INTEL && + !use_local_tile && + n_kv >= FD_MIN_N_KV && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 8 && + backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_local_mq_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 4 && + ((d_head_q == 256 && d_head_v == 256) || + (d_head_q == 128 && d_head_v == 128)) && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + const bool k_img_on = d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.count(dk_dv) > 0; + if (k_img_on) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.at(dk_dv); + use_fd_mq = true; + use_fa_k_img = true; + } else { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + use_fd_mq = true; + } + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + use_fa_k_img = true; + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split.at(dk_dv); + use_fd_mq = true; + } else if (nq1_only && is_q4_0) { + const char * q4_mq_env = getenv("GGML_OPENCL_FA_Q4_MQ"); + const bool q4_mq_on = (q4_mq_env != NULL) && (q4_mq_env[0] != '0'); + if (q4_mq_on && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split.at(dk_dv); + use_fd_mq = true; + } + } + } + } + if (fd_k_split == NULL && + n_q >= 1 && n_q <= fd_max_n_q && n_kv >= FD_MIN_N_KV && !is_causal && + d_head_q <= FD_MAX_DK && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (is_mixed && backend_ctx->fa.f32_f16_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_split.at(dk_dv); + } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_split.at(dk_dv); + } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_split.at(dk_dv); + } + } + const bool use_fd = (fd_k_split != NULL); + + const int n_q_blocks = n_q > 1 ? (n_q + block_m - 1) / block_m : 0; + const int n_kv_blocks = (n_kv > 0 && block_n > 0) ? (n_kv + block_n - 1) / block_n : 0; + // KV pad + blk prepass are pure overhead when FD will fire — skip them. + const bool use_mixed_prepass = is_mixed && n_q > 1 && !use_fd; + // make sure prepass kernels are compiled + const bool have_kv_pad = backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0; + const bool have_blk = backend_ctx->fa.blk_f16.count(dk_dv) > 0; + const bool use_kv_pad = use_mixed_prepass && (n_kv % block_n != 0) && have_kv_pad; + // blk prepass: per-KV-tile mask class (0=masked, 1=mixed, 2=unmasked). + // Consumed identically by f32_f16, q8_0 and q4_0 prefill kernels. + const bool use_quant_prepass = (use_native_q8_0 || use_native_q4_0) && !use_fd; + const bool use_blk_mask = (use_mixed_prepass || use_quant_prepass) && mask_buffer != NULL && have_blk; + + if (use_kv_pad) { + cl_int err; + + const size_t k_pad_size = (size_t) k_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_k_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_pad_size, NULL, &err); + CL_CHECK(err); + k_pad_buffer = temp_k_pad.data; + + const size_t v_pad_size = (size_t) v_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_v_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_pad_size, NULL, &err); + CL_CHECK(err); + v_pad_buffer = temp_v_pad.data; + + cl_kernel kernel_kv_pad = backend_ctx->fa.kv_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 0, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 1, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 2, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 3, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 4, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 5, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 6, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 7, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 8, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 9, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 10, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 11, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 12, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 13, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 14, sizeof(cl_ulong), &v_nb3)); + + size_t global_work_size[] = { (size_t) block_n, (size_t) n_head_kv, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel_kv_pad, 3, global_work_size, NULL, dst); + + if (mask_buffer != NULL) { + mask_pad_nb1 = (cl_ulong) block_n * (cl_ulong) sizeof(ggml_fp16_t); + mask_pad_nb2 = (cl_ulong) n_q * mask_pad_nb1; + mask_pad_nb3 = (cl_ulong) mask_ne2 * mask_pad_nb2; + + const size_t mask_pad_size = (size_t) mask_ne3 * (size_t) mask_pad_nb3; + temp_mask_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, mask_pad_size, NULL, &err); + CL_CHECK(err); + mask_pad_buffer = temp_mask_pad.data; + + cl_kernel kernel_mask_pad = backend_ctx->fa.mask_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 2, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 9, sizeof(int), &mask_ne3)); + + size_t global_work_size_mask[] = { (size_t) block_n, (size_t) n_q, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_mask_pad, 3, global_work_size_mask, NULL, dst); + } + } + + if (use_blk_mask) { + cl_int err; + const size_t blk_size = (size_t) n_kv_blocks * (size_t) n_q_blocks * (size_t) mask_ne2 * (size_t) mask_ne3; + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + if (err != CL_SUCCESS) { + // Flush before retry — reclaim deferred driver deallocations. + CL_CHECK(clFinish(backend_ctx->queue)); + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + } + CL_CHECK(err); + blk_buffer = temp_blk.data; + + cl_kernel kernel_blk = backend_ctx->fa.blk_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_blk, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_blk, 2, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_blk, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_blk, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_blk, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_blk, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_blk, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_blk, 9, sizeof(int), &mask_ne3)); + + size_t global_work_size_blk[] = { (size_t) n_kv_blocks, (size_t) n_q_blocks, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_blk, 3, global_work_size_blk, NULL, dst); + } const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_k->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra_v->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); + if (use_fd) { + static const int fd_env_kv_per_split = []{ + const char * e = getenv("GGML_OPENCL_FD_KV_PER_SPLIT"); + return (e && e[0]) ? atoi(e) : 0; + }(); + static const int fd_env_max_splits = []{ + const char * e = getenv("GGML_OPENCL_FD_MAX_SPLITS"); + return (e && e[0]) ? atoi(e) : 0; + }(); + + int fd_kv_per_split = use_fd_mq ? FD_MQ_KV_PER_SPLIT + : (is_mixed ? FD_KV_PER_SPLIT_F16 : FD_KV_PER_SPLIT); + int fd_max_splits = use_fd_mq ? FD_MQ_MAX_SPLITS : FD_MAX_SPLITS; + if (fd_env_kv_per_split > 0) { fd_kv_per_split = fd_env_kv_per_split; } + if (fd_env_max_splits > 0) { fd_max_splits = fd_env_max_splits; } + int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split; + if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; } + if (n_splits > fd_max_splits) { n_splits = fd_max_splits; } + const int kv_per_split = (n_kv + n_splits - 1) / n_splits; + + const int fa_partial_floats = 2 + d_head_v; + const size_t partial_size_bytes = + (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float); + + ggml_cl_flash_attn_temp_buffer temp_partial; + cl_int err; + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + if (err != CL_SUCCESS) { + CL_CHECK(clFinish(backend_ctx->queue)); + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + } + CL_CHECK(err); + + cl_kernel k_split = fd_k_split; + int argi = 0; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_q)); + if (use_fa_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + cl_mem k_img = nullptr; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } + + // if image creation fails, fallback to buffer based kernels + if (k_img == nullptr) { + if (gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + } else { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + } + use_fa_k_img = false; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_img)); + } + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &temp_partial.data)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split)); + + // MQ split kernel uses MQ_NSG_SPLIT subgroups and one WG per (kv_head, batch, split) + // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192) + const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64; + const size_t fd_head_dim = use_fd_mq + ? (size_t)(n_head_kv * n_batch) + : (size_t)(n_head * n_batch); + size_t fd_lws[3] = { fd_wg, 1, 1 }; + // gid(2) packs q_idx * n_splits + split_idx. + size_t fd_gws[3] = { fd_wg, fd_head_dim, (size_t)(n_splits * n_q) }; + backend_ctx->enqueue_ndrange_kernel(k_split, 3, fd_gws, fd_lws, dst); + + cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv); + argi = 0; + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &temp_partial.data)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &sinks_buffer)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_sinks)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_q)); + + const size_t merge_wg = (size_t) (d_head_v / 4); // one lane per float4 + size_t merge_lws[3] = { merge_wg, 1, 1 }; + size_t merge_gws[3] = { merge_wg, (size_t)(n_head * n_batch), (size_t) n_q }; + backend_ctx->enqueue_ndrange_kernel(k_merge, 3, merge_gws, merge_lws, dst); + return; + } + + cl_mem prefill_k_img = nullptr; + if (use_prefill_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + prefill_k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } + if (prefill_k_img == nullptr) { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + use_prefill_k_img = false; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), use_prefill_k_img ? &prefill_k_img : &k_data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); @@ -11604,15 +14189,57 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); + if (n_q > 1 && is_mixed) { + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel, 44, sizeof(int), &n_kv_blocks)); + CL_CHECK(clSetKernelArg(kernel, 45, sizeof(cl_ulong), &mask_pad_nb1)); + CL_CHECK(clSetKernelArg(kernel, 46, sizeof(cl_ulong), &mask_pad_nb2)); + CL_CHECK(clSetKernelArg(kernel, 47, sizeof(cl_ulong), &mask_pad_nb3)); + } else if (use_native_q8_0 || use_native_q4_0) { + // arg 40 = blk classification buffer (NULL disables prepass opt). + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &blk_buffer)); + } if (n_q == 1) { - const size_t wg_size = 64; - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { wg_size, (size_t)(n_head * n_batch) }; + if (use_local_tile) { + const size_t lt_wg = 128; + size_t local_work_size[] = { lt_wg, 1, 1 }; + size_t global_work_size[] = { lt_wg, (size_t) n_head, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // q1_vec dispatches with NSG subgroups + const size_t q1_wg = backend_ctx->gpu_family == INTEL ? 32 : 64; + const size_t wg_size = use_q1_vec ? 256 : q1_wg; + const size_t head_dim_global = use_q1_vec_mq + ? (size_t)(n_head_kv * n_batch) + : (size_t)(n_head * n_batch); + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { wg_size, head_dim_global }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } + } else if (use_native_q8_0 || use_native_q4_0) { + // Native quant prefill. The split variant may override BLOCK_M + // (e.g. DK=96 quant uses BM=16). + const bool use_split = use_native_q8_0 ? use_split_q8_0 : use_split_q4_0; + int bm; + size_t wg_size; + if (use_split) { + bm = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_bm.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_bm.at(dk_dv); + wg_size = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_wg_size.at(dk_dv); + } else { + bm = backend_ctx->fa.bm.at(dk_dv); + wg_size = (size_t) bm; + } + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { (size_t)((n_q + bm - 1) / bm) * wg_size, (size_t)(n_head * n_batch) }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } else { - const int block_m = backend_ctx->kernels_flash_attn_bm.at(dk_dv); - const size_t wg_size = block_m; + const size_t wg_size = (size_t) wg_size_fa; size_t local_work_size[] = { wg_size, 1 }; size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); @@ -12021,34 +14648,231 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten int offset_src0 = 0; int offset_src1 = 0; - // set kernel args - // <--------------------------------------------> // - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + // set kernel args + // <--------------------------------------------> // + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + + size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)}; + size_t local_work_size[3] = {64, 1, 2}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + // deallocate sub buffers and images + // <--------------------------------------------> // + CL_CHECK(clReleaseMemObject(A_image1d)); + CL_CHECK(clReleaseMemObject(D_image1d)); + CL_CHECK(clReleaseMemObject(A_sub_buffer)); + CL_CHECK(clReleaseMemObject(B_sub_buffer)); + CL_CHECK(clReleaseMemObject(D_sub_buffer)); +} + +static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_ASSERT(src0->type == GGML_TYPE_Q1_0); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + GGML_ASSERT(src1->view_offs == 0); + GGML_ASSERT(dst->view_offs == 0); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 == ne10); + GGML_ASSERT((ne00 % 128) == 0); + GGML_ASSERT(ne0 == ne01); + + cl_context context = backend_ctx->context; + cl_kernel kernel; + + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + int M = ne01; + int N = ne1; + int K = ne00; + + if (ne1 == 1) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; + + // image for q (uint32: each texel packs 32 sign bits) + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; + img_desc.buffer = extra0_q1_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // create a sub_buffer for B + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemv_noshuffle_q1_0_f32; + + int r2 = 1; + int r3 = 1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[] = { wavesize, 4, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } + + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32; + int padded_N = N + padding; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)}; - size_t local_work_size[3] = {64, 1, 2}; + size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; + size_t local_work_size[] = { 2, 128, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // deallocate sub buffers and images - // <--------------------------------------------> // - CL_CHECK(clReleaseMemObject(A_image1d)); - CL_CHECK(clReleaseMemObject(D_image1d)); - CL_CHECK(clReleaseMemObject(A_sub_buffer)); - CL_CHECK(clReleaseMemObject(B_sub_buffer)); - CL_CHECK(clReleaseMemObject(D_sub_buffer)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -13004,7 +15828,9 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; + // SoA extra lives on view_src (view->extra is pre-SoA). + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; @@ -13095,6 +15921,99 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { + // use bin kernel if available + if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { + int K_pad = K; + + cl_mem b_sub_buf = nullptr; + cl_mem d_sub_buf = nullptr; + + cl_mem a_img = nullptr; + cl_mem s_img = nullptr; + cl_mem b_img = nullptr; + cl_mem d_img = nullptr; + + // subbuffer for activations + region.origin = offset1; + region.size = K_pad * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // Create subbuffer and image1d_buffer for dst + region.origin = (extrad->offset); // + dst->view_offs; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // create an image for A + img_fmt = { CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 4; // Divide by 4 for char -> float + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((a_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // create an image for Scale + img_fmt = { CL_R, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; // Block size is 32 + img_desc.buffer = extra0_q8_0->d; + CL_CHECK((s_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // create an image for B from sub_buffer + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K_pad * N; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // img for d + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * N; + img_desc.buffer = d_sub_buf; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin; + + bool layoutA_Mfirst = true; + bool layoutS_Mfirst = true; + bool layoutB_Nfirst = false; + bool layoutC_Mfirst = true; + + cl_uint lineStrideMatrixAinBytes = layoutA_Mfirst ? M * 4 : K; // int8 + cl_uint lineStrideMatrixSinBytes = layoutS_Mfirst ? M * 2 : (K / 32) * 2; // fp16 + cl_uint lineStrideMatrixBinBytes = layoutB_Nfirst ? N * 4 : K_pad * 4; // fp32 + cl_uint lineStrideMatrixCinBytes = layoutC_Mfirst ? M * 4 : N * 4; // fp32 + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &lineStrideMatrixAinBytes)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &lineStrideMatrixSinBytes)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &lineStrideMatrixBinBytes)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &lineStrideMatrixCinBytes)); + + size_t global_work_size[] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N, 64)}; + size_t local_work_size[] = { 64, 2, 2 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(a_img)); + CL_CHECK(clReleaseMemObject(s_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(d_img)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -13756,6 +16675,264 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t #endif } +// Dequant a possibly-strided q4_0/q8_0 tensor to tight-packed f16. Returns a +// temp cl_mem the caller must release. SoA inputs are reconstructed into a +// temp AoS buffer reported via *extra_reconstruct (also caller-released). +// this is for quantized K cache without FA. +static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem * extra_reconstruct /* out, may be NULL */ +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); + + if (extra_reconstruct) { + *extra_reconstruct = NULL; + } + + cl_mem src_buf; + cl_ulong src_offset; + cl_ulong src_nb1; + cl_ulong src_nb2; + cl_ulong src_nb3; + + uintptr_t pool_key_buf = 0; + cl_ulong pool_key_off = (cl_ulong) tensor->view_offs; + + const bool is_soa = tensor->type == GGML_TYPE_Q4_0 + ? ggml_cl_is_q4_0_soa(tensor) + : ggml_cl_is_q8_0_soa(tensor); + + cl_mem aos = nullptr; + if (is_soa) { + // Reconstruct full parent AoS; view's own nb[] then index it correctly. + const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor; + const ggml_tensor * soa_src = parent; + const size_t block_bytes = (size_t) ggml_type_size(tensor->type); + const size_t blck_size = (size_t) ggml_blck_size(tensor->type); + const size_t parent_row_blocks = (size_t) parent->ne[0] / blck_size; + const size_t parent_row_bytes = parent_row_blocks * block_bytes; + const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes; + + cl_int err; + aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); + + // large q4_0/q8_0 WEIGHTS are stored transposed and small weights + // (and the AoS KV-cache, handled in the else branch above) are not. + // choose a proper restore kernel based on this. + bool restored = false; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + const int p_ne00 = (int) parent->ne[0]; + const int p_ne01 = (int) parent->ne[1]; + if (tensor->type == GGML_TYPE_Q8_0 && enable_adreno_trans_weight(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &p_ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &p_ne01)); + size_t gws[] = { (size_t)(((p_ne01 + 63) / 64) * 64), 1, 1 }; + size_t lws[] = { 64, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + restored = true; + } else if (tensor->type == GGML_TYPE_Q4_0 && + use_adreno_kernels(backend_ctx, parent) && + !use_adreno_moe_kernels(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + const size_t size_q = (size_t) ggml_nelements(parent) / blck_size * (blck_size / 2); + const size_t size_d = (size_t) ggml_nelements(parent) / blck_size * sizeof(ggml_fp16_t); + cl_int err2 = CL_SUCCESS; + cl_mem buf_tq = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_q, NULL, &err2); CL_CHECK(err2); + cl_mem buf_td = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_d, NULL, &err2); CL_CHECK(err2); + transpose_2d_as_16b(backend_ctx, extra->q, buf_tq, size_q, p_ne01, p_ne00 / 4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_td, size_d, p_ne01, p_ne00 / 32); + cl_uchar mask_0F = 0x0F, mask_F0 = 0xF0; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_tq)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_td)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + const size_t n_blk = parent_nbytes / block_bytes; + size_t gws[] = { n_blk, 1, 1 }; + size_t lws[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + + CL_CHECK(clReleaseMemObject(buf_tq)); + CL_CHECK(clReleaseMemObject(buf_td)); + restored = true; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + if (!restored) { + cl_kernel kernel; + if (tensor->type == GGML_TYPE_Q8_0) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q8_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } else { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } + + const size_t n_blocks = parent_nbytes / block_bytes; + size_t gws_rec[] = { n_blocks, 1, 1 }; + size_t lws_rec[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws_rec, lws_rec, 0, NULL, NULL)); + } + + (void) parent_row_blocks; + (void) parent_row_bytes; + src_buf = aos; + src_offset = tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; + + if (extra_reconstruct) { + *extra_reconstruct = aos; + } + } else { + auto * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + src_buf = extra->data_device; + src_offset = extra->offset + tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; + pool_key_buf = (uintptr_t) extra->data_device; + pool_key_off = (cl_ulong) src_offset; + } + + const cl_int nblk0 = (cl_int) (tensor->ne[0] / ggml_blck_size(tensor->type)); + const cl_int ne1_ = (cl_int) tensor->ne[1]; + const cl_int ne2_ = (cl_int) tensor->ne[2]; + const cl_int ne3_ = (cl_int) tensor->ne[3]; + + const size_t out_bytes = (size_t) ggml_nelements(tensor) * sizeof(ggml_fp16_t); + + // reuse a pooled f16 buffer for this KV-cache view across decode steps instead of + // allocating new one per attention op + cl_mem out = nullptr; + { + auto & pool = backend_ctx->dequant_f16_pool; + ggml_backend_opencl_context::ImagePoolKey key{pool_key_buf, (uint64_t) pool_key_off}; + auto it = pool.find(key); + if (it != pool.end() && it->second.k_bytes >= out_bytes && it->second.image) { + out = it->second.image; + } else { + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + pool.erase(it); + } + cl_int err = CL_SUCCESS; + out = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.image = out; + entry.k_bytes = out_bytes; + pool[key] = entry; + } + } + + cl_kernel dq_kernel = tensor->type == GGML_TYPE_Q8_0 + ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + + CL_CHECK(clSetKernelArg(dq_kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(dq_kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(dq_kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(dq_kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(dq_kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(dq_kernel, 5, sizeof(cl_int), &nblk0)); + CL_CHECK(clSetKernelArg(dq_kernel, 6, sizeof(cl_int), &ne1_)); + CL_CHECK(clSetKernelArg(dq_kernel, 7, sizeof(cl_int), &ne2_)); + CL_CHECK(clSetKernelArg(dq_kernel, 8, sizeof(cl_int), &ne3_)); + CL_CHECK(clSetKernelArg(dq_kernel, 9, sizeof(cl_mem), &out)); + + size_t gws[3] = { (size_t) nblk0, (size_t) ne1_, (size_t) (ne2_ * ne3_) }; + size_t lws[3] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + + // release the reconstructed aos if + // 1. it was actually reconstructed + // 2. the caller didn't request it to be returned + // src_buf may refer to aos, so we should release after this enqueue + if (aos && !extra_reconstruct) { + CL_CHECK(clReleaseMemObject(aos)); + } + return out; +} + +// look up or create a pooled image1d_buffer over a KV-cache view. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type +) { + ggml_backend_opencl_context::ImagePoolKey key{(uintptr_t)data_device, (uint64_t)offset0}; + auto it = pool.find(key); + if (it != pool.end() + && it->second.k_bytes >= required_bytes + && it->second.channel_data_type == channel_data_type + && it->second.image != nullptr) { + return it->second.image; + } + + // need to create or recreate and release any stale entry first. + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + if (it->second.sub_buffer) {CL_CHECK(clReleaseMemObject(it->second.sub_buffer)); } + pool.erase(it); + } + + cl_int status = CL_SUCCESS; + cl_buffer_region region = {}; + region.origin = (size_t)offset0; + region.size = required_bytes; + cl_mem sub = clCreateSubBuffer(data_device, 0, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + if (status != CL_SUCCESS) { + return nullptr; + } + + const size_t pixel_size = (channel_data_type == CL_HALF_FLOAT) ? 8 : 16; + cl_image_format fmt = {CL_RGBA, channel_data_type}; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = required_bytes / pixel_size; + desc.buffer = sub; + cl_mem img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, + &fmt, &desc, NULL, &status); + if (status != CL_SUCCESS) { + CL_CHECK(clReleaseMemObject(sub)); + return nullptr; + } + + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.sub_buffer = sub; + entry.image = img; + entry.k_bytes = required_bytes; + entry.channel_data_type = channel_data_type; + pool[key] = entry; + return img; +} + static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -13770,6 +16947,30 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // quant kv without FA + // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1) + // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B) + if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && + (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) { + cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr); + + ggml_tensor fake_src0 = *src0; + ggml_tensor_extra_cl fake_extra = {}; + fake_extra.data_device = f16_buf; + fake_extra.offset = 0; + fake_src0.type = GGML_TYPE_F16; + fake_src0.extra = &fake_extra; + fake_src0.view_src = nullptr; + fake_src0.view_offs = 0; + fake_src0.nb[0] = sizeof(ggml_fp16_t); + fake_src0.nb[1] = fake_src0.nb[0] * src0->ne[0]; + fake_src0.nb[2] = fake_src0.nb[1] * src0->ne[1]; + fake_src0.nb[3] = fake_src0.nb[2] * src0->ne[2]; + + ggml_cl_mul_mat(backend, &fake_src0, src1, dst); + return; + } + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; @@ -13779,16 +16980,20 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co cl_ulong offsetd = extrad->offset + dst->view_offs; #ifdef GGML_OPENCL_SOA_Q - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; - ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; - ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; - ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; - ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)src0->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)src0->extra; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; + // view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV. + // Follow view_src to reach the real SoA extra. + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; + ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; #endif GGML_TENSOR_LOCALS(int, ne0, src0, ne); @@ -13835,6 +17040,228 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } } + + static const char * mm_kq_gqa_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_IMG"); + static const bool mm_kq_gqa_img_on = (mm_kq_gqa_img_env == nullptr || mm_kq_gqa_img_env[0] != '0'); + static const char * mm_kq_gqa_r4_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_R4_IMG"); + static const bool mm_kq_gqa_r4_img_on = (mm_kq_gqa_r4_img_env == nullptr || mm_kq_gqa_r4_img_env[0] != '0'); + const bool img_r4_gate = + mm_kq_gqa_r4_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 4 && (ne13 / ne03) == 1; + if (mm_kq_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // r2=4 specialization + if (img_r4_gate) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // DK=256, r2=2 specialization + static const char * mm_kq_r2_dk256_env = getenv("GGML_OPENCL_MM_KQ_GQA_R2_DK256_IMG"); + static const bool mm_kq_r2_dk256_on = (mm_kq_r2_dk256_env != nullptr && mm_kq_r2_dk256_env[0] != '0'); + if (mm_kq_r2_dk256_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 256 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 2 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // GQA-coalesced KQV for decode using image1d_buffer_t + static const char * mm_kqv_gqa_img_env = getenv("GGML_OPENCL_MM_KQV_GQA_IMG"); + static const bool mm_kqv_gqa_img_on = (mm_kqv_gqa_img_env != nullptr && mm_kqv_gqa_img_env[0] != '0'); + if (mm_kqv_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img != nullptr && + ne11 == 1 && ne01 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t v_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t v_bytes = (v_bytes_span + 7) & ~(size_t)7; + const size_t v_pixels = v_bytes >> 3; + if (v_pixels > 0 && v_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img; + cl_mem V_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kqv_img_pool, + extra0->data_device, offset0, v_bytes, CL_HALF_FLOAT); + if (V_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &V_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } } if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { @@ -13842,6 +17269,13 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that // limit, so the check is omitted. + // q1_0 x fp32 + if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight(backend_ctx, src0)) { + ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst); + return; + } + // q4_0 x fp32 if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst); @@ -14045,6 +17479,48 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } + case GGML_TYPE_Q1_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } case GGML_TYPE_Q4_0: { if (ne11 < 32) { break; @@ -14592,12 +18068,50 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; nrows = 1; } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - if (ne11 == 1) { + // multi-output decode variants when Q is a single row + static const char * mm_force_l4_env = getenv("GGML_OPENCL_MM_F16_FORCE_L4"); + static const bool mm_force_l4_on = (mm_force_l4_env != nullptr && mm_force_l4_env[0] != '0'); + const bool can_multi_out = !mm_force_l4_on && ne11 == 1 && ne01 >= 64 && ne01 % 8 == 0; + // paired-K-row variant that doubles per-wave-cycle + static const char * mm_kq_pair_env = getenv("GGML_OPENCL_MM_KQ_PAIR"); + static const bool mm_kq_pair_on = (mm_kq_pair_env != nullptr && mm_kq_pair_env[0] != '0'); + // GQA-coalesced variant that reads each K-row once and + // emits gqa_ratio outputs + static const char * mm_kq_gqa_env = getenv("GGML_OPENCL_MM_KQ_GQA"); + static const bool mm_kq_gqa_on = (mm_kq_gqa_env != nullptr && mm_kq_gqa_env[0] != '0'); + // GQA-coalesced KQV variant (DK=128/r2=8/r3=1) that reads + // each V slab once per K-head and emits all r2 Q-heads + static const char * mm_kqv_gqa_env = getenv("GGML_OPENCL_MM_KQV_GQA"); + static const bool mm_kqv_gqa_on = (mm_kqv_gqa_env != nullptr && mm_kqv_gqa_env[0] != '0'); + if (can_multi_out && (ne01 % 16) == 0 && ne00 == 128 && r2 == 8 && r3 == 1 && mm_kq_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && mm_kq_pair_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8; + nrows = 1; + } else if (can_multi_out && ne01 == 128 && r2 == 8 && r3 == 1 && mm_kqv_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa; + nrows = 1; + } else if (can_multi_out && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8; + nrows = 1; + } else if (ne11 == 1) { + // Decode shapes that don't satisfy the x8/y8 row + // constraints (ne01 < 64 or ne01 % 8 != 0) fall back to + // upstream's 4-output _dr kernel. kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; nrows = 1; // not used by this kernel } else { kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; - nrows = ne11; + nrows = ne11; } } else { kernel = backend_ctx->kernel_mul_mat_f16_f32; @@ -14633,6 +18147,81 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); break; + case GGML_TYPE_Q1_0: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat; + + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q1_0_f32; + + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } case GGML_TYPE_Q4_0: // This should have been satisfied. GGML_ASSERT(ne11 == ne1); @@ -15347,6 +18936,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || + src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_Q2_K) { // Each SIMD group produces N_DST values in the result. Assuming each @@ -15375,6 +18965,26 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8 || + kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair || + kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8) { + // multi-output decode variants: each WG processes 8 outputs along ne01, ne11 == 1 + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4) { + // GQA-coalesced KQ: one WG per K-head emits N_K_ROWS_GQA=16 K-rows * r2 Q-heads + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa) { + // GQA-coalesced KQV: one WG per K-head emits 8 DV-rows * r2 Q-heads + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr) { @@ -15543,15 +19153,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, GGML_UNUSED(offset0); #ifdef GGML_OPENCL_SOA_Q - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; - ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; - ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)src0->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)src0->extra; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; - ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; + // SoA extra lives on view_src (view->extra is pre-SoA). + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; + ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; + #endif // TODO: general MoE for the following types @@ -15675,6 +19288,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns; + if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin; + } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -15720,6 +19336,11 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); @@ -15767,6 +19388,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -15892,6 +19515,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns; + if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin; + } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -15937,6 +19563,11 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); @@ -15985,6 +19616,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -16161,6 +19794,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -16338,6 +19973,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -16498,6 +20135,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns; + if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin; + } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -16539,6 +20179,11 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(status); cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); @@ -16587,6 +20232,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -16765,6 +20412,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -16940,6 +20589,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); @@ -17022,6 +20673,9 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, } else { // for gemm kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns; + if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin; + } // Reorder router if called from test-backend-ops or when new router is generated. // Otherwise reuse the reordered result from previous mul_mat_id call. @@ -17068,6 +20722,11 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_image_desc image_desc_buf_src1; image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); CL_CHECK(status); @@ -17115,6 +20774,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); // set thread grid global_size[1] = static_cast((ne01 + 63) / 64); diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 226b127ab3b..bf0335a6989 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -27,6 +27,8 @@ #define QR5_1 2 #define QK8_0 32 #define QR8_0 1 +#define QK1_0 128 +#define QR1_0 1 #define QK_K 256 #define K_SCALE_SIZE (3 * QK_K / 64) #define K_QUANTS_PER_ITERATION 2 @@ -38,6 +40,14 @@ typedef ushort uint16_t; typedef int int32_t; typedef uint uint32_t; +//------------------------------------------------------------------------------ +// block_q1_0 +//------------------------------------------------------------------------------ +typedef struct { + half d; // delta + uchar qs[QK1_0/8]; // 1-bit signs (16 bytes) +} block_q1_0; + //------------------------------------------------------------------------------ // block_q4_0 //------------------------------------------------------------------------------ @@ -159,6 +169,42 @@ kernel void kernel_convert_f16_to_bf16( } } +//------------------------------------------------------------------------------ +// kernel_convert_block_q1_0 +// Convert block_q1_0 (AOS) to 2 separate arrays (SOA): quant bytes + scales. +// q1_0 bits are stored in natural order (bit j of byte i -> weight 8*i + j) +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q1_0( + global block_q1_0 * src0, + global uchar * dst_q, + global half * dst_d +) { + global block_q1_0 * b = (global block_q1_0 *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + (QK1_0/8)*get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + + for (int i = 0; i < QK1_0/8; ++i) { + q[i] = b->qs[i]; + } +} + +kernel void kernel_restore_block_q1_0( + global uchar * src_q, + global half * src_d, + global block_q1_0 * dst +) { + global block_q1_0 * b = (global block_q1_0 *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + (QK1_0/8)*get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + for (int i = 0; i < QK1_0/8; ++i) { + b->qs[i] = q[i]; + } +} + //------------------------------------------------------------------------------ // kernel_convert_block_q4_0 // Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA). @@ -1582,6 +1628,158 @@ kernel void kernel_restore_block_q8_0( } } +// View-aware AoS q8_0 -> f32 dequant (f32/f32 FA path). +kernel void kernel_dequant_q8_0_f32_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global float * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0); + float d = vload_half(0, (global half *)block); + global char * qs = block + 2; + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global float * out = dst + (dst_row_base + blk_i0) * QK8_0; + + for (int i = 0; i < QK8_0; ++i) { + out[i] = d * (float)qs[i]; + } +} + +// View-aware AoS q8_0 -> f16 dequant. Rows tight, batch strides may be gapped. +kernel void kernel_dequant_q8_0_f16_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global half * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0); + float d = vload_half(0, (global half *)block); + global char * qs = block + 2; + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global half * out = dst + (dst_row_base + blk_i0) * QK8_0; + + for (int i = 0; i < QK8_0; ++i) { + out[i] = (half)(d * (float)qs[i]); + } +} + +// View-aware AoS q4_0 -> f32 dequant (mirrors the q8_0 view variant). +kernel void kernel_dequant_q4_0_f32_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global float * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2); + float d = vload_half(0, (global half *)block); + global uchar * qs = (global uchar *)(block + 2); + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global float * out = dst + (dst_row_base + blk_i0) * QK4_0; + + for (int i = 0; i < QK4_0/2; ++i) { + uchar byte = qs[i]; + int q0 = (int)(byte & 0x0F) - 8; + int q1 = (int)(byte >> 4) - 8; + out[i] = d * (float)q0; + out[i + QK4_0/2] = d * (float)q1; + } +} + +// View-aware AoS q4_0 -> f16 dequant (mirrors the q8_0 view variant). +kernel void kernel_dequant_q4_0_f16_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global half * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2); + float d = vload_half(0, (global half *)block); + global uchar * qs = (global uchar *)(block + 2); + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global half * out = dst + (dst_row_base + blk_i0) * QK4_0; + + for (int i = 0; i < QK4_0/2; ++i) { + uchar byte = qs[i]; + int q0 = (int)(byte & 0x0F) - 8; + int q1 = (int)(byte >> 4) - 8; + out[i] = (half)(d * (float)q0); + out[i + QK4_0/2] = (half)(d * (float)q1); + } +} + kernel void kernel_restore_block_q8_0_trans( global uchar * src_q, global half * src_d, diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl index 8f43c4f27d5..fc58a22eccd 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl @@ -4,13 +4,30 @@ #define ACC_TYPE4 float4 #define DATA_TYPE half #define DATA_TYPE4 half4 -#define CONVERT_ACC4(x) convert_float4(x) -#define CONVERT_DATA4(x) convert_half4(x) +#define CONVERT_ACC4(x) ((float4)((float)(x).s0, (float)(x).s1, (float)(x).s2, (float)(x).s3)) +#define CONVERT_DATA4(x) ((half4)((half)(x).s0, (half)(x).s1, (half)(x).s2, (half)(x).s3)) #define DK_VEC (DK/4) #define DV_VEC (DV/4) #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +// q1 reduces over a Q1_WG_SIZE-wide WG via work-group barriers; the launch WG +// must match. Defaults to the Adreno sg (64); host passes -D FA_SG=32 on Intel. +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -81,18 +98,18 @@ __kernel void flash_attn_f16( if (my_query_row < n_q) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } } ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -125,49 +142,72 @@ __kernel void flash_attn_f16( continue; } - for (int j = 0; j < BLOCK_N; j += 2) { + for (int j = 0; j < BLOCK_N; j += 4) { const int k_row0 = k_start + j; const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + const ACC_TYPE4 qk = q_priv[k]; + dot_acc0 = mad(qk, CONVERT_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_ACC4(l_k[j+3][k]), dot_acc3); } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; } - - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); } - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p0 = native_exp(s0 - m_new); + const ACC_TYPE p1 = native_exp(s1 - m_new); + const ACC_TYPE p2 = native_exp(s2 - m_new); + const ACC_TYPE p3 = native_exp(s3 - m_new); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]); + o_acc[i] = mad(p3, CONVERT_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); } - l_i = l_i * scale_prev + p0 + p1; + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; m_i = m_new; } } @@ -179,7 +219,7 @@ __kernel void flash_attn_f16( const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -191,12 +231,12 @@ __kernel void flash_attn_f16( global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (DATA_TYPE4)(0.0f); } @@ -258,7 +298,7 @@ __kernel void flash_attn_f16_q1( ACC_TYPE4 q_priv[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } @@ -270,12 +310,12 @@ __kernel void flash_attn_f16_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } @@ -293,7 +333,7 @@ __kernel void flash_attn_f16_q1( __local ACC_TYPE local_m[Q1_WG_SIZE]; local_m[tid] = m_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); barrier(CLK_LOCAL_MEM_FENCE); @@ -301,7 +341,7 @@ __kernel void flash_attn_f16_q1( const ACC_TYPE m_final = local_m[0]; ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -311,7 +351,7 @@ __kernel void flash_attn_f16_q1( const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } @@ -325,7 +365,7 @@ __kernel void flash_attn_f16_q1( } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]); } @@ -335,7 +375,7 @@ __kernel void flash_attn_f16_q1( __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; local_l[tid] = l_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_l[tid] += local_l[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -354,7 +394,7 @@ __kernel void flash_attn_f16_q1( for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -364,7 +404,7 @@ __kernel void flash_attn_f16_q1( } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f); } } diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl index a6d74790375..599877bdbae 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl @@ -11,7 +11,24 @@ #define DK_VEC (DK/4) #define DV_VEC (DV/4) #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +// q1 reduces over a Q1_WG_SIZE-wide WG via work-group barriers; the launch WG +// must match. Defaults to the Adreno sg (64); host passes -D FA_SG=32 on Intel. +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -82,18 +99,18 @@ __kernel void flash_attn_f32( if (my_query_row < n_q) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } } ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -102,6 +119,15 @@ __kernel void flash_attn_f32( __local DATA_TYPE4 l_v[BLOCK_N][DV_VEC]; for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { +#if FA_SG < 64 + // WAR on l_k/l_v: threads with my_query_row >= n_q skip the compute below + // (continue) and would race ahead to reload the tiles while active threads + // still read them. A single 64-wide Adreno subgroup (WG == sg) runs lockstep + // and hides this; a WG that spans multiple narrower subgroups (Intel sg=32) + // corrupts the result. All threads reach this each iteration (no-op on the + // first), so it does not diverge with the continue. Compiled out at sg=64. + barrier(CLK_LOCAL_MEM_FENCE); +#endif for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) { const int row = i / DK_VEC; const int col = i % DK_VEC; @@ -126,49 +152,72 @@ __kernel void flash_attn_f32( continue; } - for (int j = 0; j < BLOCK_N; j += 2) { + for (int j = 0; j < BLOCK_N; j += 4) { const int k_row0 = k_start + j; const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + const ACC_TYPE4 qk = q_priv[k]; + dot_acc0 = mad(qk, CONVERT_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_ACC4(l_k[j+3][k]), dot_acc3); } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; } - - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); } - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p0 = native_exp(s0 - m_new); + const ACC_TYPE p1 = native_exp(s1 - m_new); + const ACC_TYPE p2 = native_exp(s2 - m_new); + const ACC_TYPE p3 = native_exp(s3 - m_new); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]); + o_acc[i] = mad(p3, CONVERT_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); } - l_i = l_i * scale_prev + p0 + p1; + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; m_i = m_new; } } @@ -180,7 +229,7 @@ __kernel void flash_attn_f32( const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -192,12 +241,12 @@ __kernel void flash_attn_f32( global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (DATA_TYPE4)(0.0f); } @@ -259,7 +308,7 @@ __kernel void flash_attn_f32_q1( ACC_TYPE4 q_priv[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } @@ -271,12 +320,12 @@ __kernel void flash_attn_f32_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } @@ -294,7 +343,7 @@ __kernel void flash_attn_f32_q1( __local ACC_TYPE local_m[Q1_WG_SIZE]; local_m[tid] = m_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); barrier(CLK_LOCAL_MEM_FENCE); @@ -302,7 +351,7 @@ __kernel void flash_attn_f32_q1( const ACC_TYPE m_final = local_m[0]; ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -312,7 +361,7 @@ __kernel void flash_attn_f32_q1( const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } @@ -326,7 +375,7 @@ __kernel void flash_attn_f32_q1( } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]); } @@ -336,7 +385,7 @@ __kernel void flash_attn_f32_q1( __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; local_l[tid] = l_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_l[tid] += local_l[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -355,7 +404,7 @@ __kernel void flash_attn_f32_q1( for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -365,7 +414,7 @@ __kernel void flash_attn_f32_q1( } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f); } } diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl index ec7361b9e37..a27b1b56a2e 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl @@ -1,5 +1,37 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +// subgroup size for q1 kernels +#ifndef FA_SG +#define FA_SG 64 +#endif +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define REQD_FA_SG __attribute__((intel_reqd_sub_group_size(FA_SG))) +#else +#define REQD_FA_SG +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#endif + #define ACC_TYPE float #define ACC_TYPE4 float4 #define Q_DATA_TYPE4 float4 @@ -12,8 +44,37 @@ #define DK_VEC (DK/4) #define DV_VEC (DV/4) + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif + +// N_SPLIT>1 splits DK/DV across threads to cut per-thread register use. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) + +#if N_SPLIT > 1 +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -26,9 +87,20 @@ inline float get_alibi_slope( return pow(base, exph); } -__kernel void flash_attn_f32_f16( + +// Adreno compiler crashes when attempting to compile the entire program for DK=512, +// FA_DECODE_ONLY allows bypass the encoding kernel. +#if !defined(FA_DECODE_ONLY) && !defined(FA_MQ_ONLY) +#ifndef FA_TILE_NAME +#define FA_TILE_NAME flash_attn_f32_f16 +#endif +__kernel void FA_TILE_NAME( const global void * q_void, ulong q_offset, +#ifdef FA_K_IMG + __read_only image1d_buffer_t k_img, ulong k_offset_unused, +#else const global void * k_void, ulong k_offset, +#endif const global void * v_void, ulong v_offset, global void * o_void, ulong o_offset, const float scale, @@ -54,48 +126,83 @@ __kernel void flash_attn_f32_f16( const int mask_ne2, const int mask_ne3, const global void* sinks_void, - const ulong sinks_offset + const ulong sinks_offset, + const global void * k_pad_void, + const global void * v_pad_void, + const global void * mask_pad_void, + const global char * blk, + const int n_kv_blocks, + const ulong mask_pad_nb1, + const ulong mask_pad_nb2, + const ulong mask_pad_nb3 ) { const int tid = get_local_id(0); const int block_q_idx = get_group_id(0); const int head_batch_idx = get_global_id(1); - const int my_query_row = block_q_idx * BLOCK_M + tid; +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; const int batch_idx = head_batch_idx / n_head; const int head_idx = head_batch_idx % n_head; const int gqa_ratio = n_head / n_head_kv; const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; const global char* q_base = (const global char*)q_void + q_offset; +#ifndef FA_K_IMG const global char* k_base = (const global char*)k_void + k_offset; +#endif const global char* v_base = (const global char*)v_void + v_offset; global char* o_base = (global char*)o_void + o_offset; const global char* mask_base = NULL; if (mask_void != NULL) { - const int mask_head_idx = head_idx % mask_ne2; - const int mask_batch_idx = batch_idx % mask_ne3; mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; } + const global char* mask_pad_base = NULL; + if (mask_pad_void != NULL) { + mask_pad_base = (const global char*)mask_pad_void + mask_batch_idx * mask_pad_nb3 + mask_head_idx * mask_pad_nb2; + } + const global char* blk_base = NULL; + if (blk != NULL) { + const int n_q_blocks = (n_q + BLOCK_M - 1) / BLOCK_M; + blk_base = blk + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks + block_q_idx) * n_kv_blocks; + } - ACC_TYPE4 q_priv[DK_VEC]; - if (my_query_row < n_q) { + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + const int dk_off = split_idx * SPLIT_DK_VEC; + if (query_valid) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll - for (int i = 0; i < DK_VEC; ++i) { - q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + FA_UNROLL + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[dk_off + i]); + } + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = (ACC_TYPE4)(0.0f); } } - ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll - for (int i = 0; i < DV_VEC; ++i) { + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -103,86 +210,388 @@ __kernel void flash_attn_f32_f16( __local KV_DATA_TYPE4 l_k[BLOCK_N][DK_VEC]; __local KV_DATA_TYPE4 l_v[BLOCK_N][DV_VEC]; +#if N_SPLIT > 1 && !defined(HAS_SUBGROUP_SHUFFLE) + __local ACC_TYPE local_partial[BLOCK_N][WG_SIZE]; + __local ACC_TYPE local_p[BLOCK_M][BLOCK_N]; + __local ACC_TYPE local_softmax_scale[BLOCK_M]; + __local ACC_TYPE local_l_inv[BLOCK_M]; +#endif + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + const int use_kv_pad = k_pad_void != NULL && k_start + BLOCK_N > n_kv; + const int k_tile_start = use_kv_pad ? 0 : k_start; + const ulong k_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * k_nb1 : k_nb2; + const ulong k_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * k_tile_nb2 : k_nb3; + const ulong v_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * v_nb1 : v_nb2; + const ulong v_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * v_tile_nb2 : v_nb3; +#ifdef FA_K_IMG + // K via texture cache for the bulk (aligned) tiles; the ragged last + // tile (use_kv_pad) still reads the f32-strided pad buffer from global. + const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : (const global char*) 0; + const int k_pitch_px_row = (int)(k_nb1 >> 3); + const int k_pitch_px_head = (int)(k_nb2 >> 3); + const int k_pitch_px_batch = (int)(k_nb3 >> 3); +#else + const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : k_base; +#endif + const global char* v_tile_base = use_kv_pad ? (const global char*) v_pad_void : v_base; + for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) { const int row = i / DK_VEC; const int col = i % DK_VEC; - const int k_row_idx = k_start + row; - if (k_row_idx < n_kv) { - const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; - l_k[row][col] = ((__global KV_DATA_TYPE4*)(k_base + k_row_offset))[col]; + const int k_row_idx = k_tile_start + row; + if (use_kv_pad || k_row_idx < n_kv) { +#ifdef FA_K_IMG + if (use_kv_pad) { + const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1; + l_k[row][col] = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col]; + } else { + const int k_row_px = batch_idx * k_pitch_px_batch + head_kv_idx * k_pitch_px_head + k_row_idx * k_pitch_px_row; + l_k[row][col] = read_imageh(k_img, k_row_px + col); + } +#else + const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1; + l_k[row][col] = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col]; +#endif + } else { + l_k[row][col] = (KV_DATA_TYPE4)(0.0h); } } for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) { const int row = i / DV_VEC; const int col = i % DV_VEC; - const int v_row_idx = k_start + row; - if (v_row_idx < n_kv) { - const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; - l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_base + v_row_offset))[col]; + const int v_row_idx = k_tile_start + row; + if (use_kv_pad || v_row_idx < n_kv) { + const ulong v_row_offset = batch_idx * v_tile_nb3 + head_kv_idx * v_tile_nb2 + v_row_idx * v_nb1; + l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_tile_base + v_row_offset))[col]; + } else { + l_v[row][col] = (KV_DATA_TYPE4)(0.0h); } } barrier(CLK_LOCAL_MEM_FENCE); - if (my_query_row >= n_q) { - continue; - } +#if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE) + { + const int dv_off = split_idx * SPLIT_DV_VEC; + for (int j = 0; j < BLOCK_N; j += 2) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; - for (int j = 0; j < BLOCK_N; j += 2) { - const int k_row0 = k_start + j; - const int k_row1 = k_start + j + 1; + ACC_TYPE partial0 = 0.0f; + ACC_TYPE partial1 = 0.0f; + FA_UNROLL + for (int k = 0; k < SPLIT_DK_VEC; k++) { + const ACC_TYPE4 qk = q_priv[k]; + ACC_TYPE4 dot0 = qk * CONVERT_KV_ACC4(l_k[j ][dk_off + k]); + ACC_TYPE4 dot1 = qk * CONVERT_KV_ACC4(l_k[j+1][dk_off + k]); + partial0 += dot0.s0 + dot0.s1 + dot0.s2 + dot0.s3; + partial1 += dot1.s0 + dot1.s1 + dot1.s2 + dot1.s3; + } - ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); - ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll - for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1); - } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + FA_UNROLL + for (int step = 1; step < N_SPLIT; step <<= 1) { + partial0 += sub_group_shuffle_xor(partial0, step); + partial1 += sub_group_shuffle_xor(partial1, step); + } + + ACC_TYPE score0 = partial0 * scale; + ACC_TYPE score1 = partial1 * scale; + + if (!query_valid) { score0 = FA_M_INIT; score1 = FA_M_INIT; } + if (is_causal) { + if (k_row0 > (n_kv - n_q + my_query_row)) score0 = FA_M_INIT; + if (k_row1 > (n_kv - n_q + my_query_row)) score1 = FA_M_INIT; + } + if (k_row0 >= n_kv) score0 = FA_M_INIT; + if (k_row1 >= n_kv) score1 = FA_M_INIT; - if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + if (query_valid && mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + score0 += slope * (ACC_TYPE)mask_ptr[j]; + score1 += slope * (ACC_TYPE)mask_ptr[j + 1]; + } else { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + } + } + + if (logit_softcap > 0.0f) { + score0 = logit_softcap * tanh(score0 / logit_softcap); + score1 = logit_softcap * tanh(score1 / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, max(score0, score1)); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE sp = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(score0 - m_exp); + const ACC_TYPE p1 = native_exp(score1 - m_exp); + + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] = o_acc[i] * sp + + p0 * CONVERT_KV_ACC4(l_v[j ][dv_off + i]) + + p1 * CONVERT_KV_ACC4(l_v[j+1][dv_off + i]); + } + l_i = l_i * sp + p0 + p1; + m_i = m_new; + } + } +#elif N_SPLIT > 1 + // N_SPLIT>1 fallback (no shuffle): 3-phase local-memory reduction. + // Phase 1 — partial dots for all BLOCK_N tokens. + for (int j = 0; j < BLOCK_N; ++j) { + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + FA_UNROLL + for (int k = 0; k < SPLIT_DK_VEC; k++) { + dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j][dk_off + k]), dot_acc); } + local_partial[j][tid] = + dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3; + } + barrier(CLK_LOCAL_MEM_FENCE); // 1 barrier: partial dots visible + + // Phase 2 — split_idx==0 reduces partial sums and computes block softmax. + if (split_idx == 0) { + if (query_valid) { + ACC_TYPE m_new = m_i; + for (int j = 0; j < BLOCK_N; ++j) { + const int k_row = k_start + j; + ACC_TYPE score = 0.0f; + FA_UNROLL + for (int s = 0; s < N_SPLIT; s++) { + score += local_partial[j][q_lane * N_SPLIT + s]; + } + score *= scale; + + if (is_causal && k_row > (n_kv - n_q + my_query_row)) score = FA_M_INIT; + if (k_row >= n_kv) score = FA_M_INIT; - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + if (mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + score += slope * (ACC_TYPE)mask_ptr[j]; + } else { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row < n_kv) score += slope * (ACC_TYPE)mask_ptr[k_row]; + } + } - if (mask_base != NULL) { - const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + m_new = max(m_new, score); + local_p[q_lane][j] = score; + } + + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE sp = native_exp(m_i - m_exp); + ACC_TYPE l_new = l_i * sp; + for (int j = 0; j < BLOCK_N; ++j) { + const ACC_TYPE p = native_exp(local_p[q_lane][j] - m_exp); + local_p[q_lane][j] = p; + l_new += p; + } + local_softmax_scale[q_lane] = sp; + l_i = l_new; + m_i = m_new; + } else { + local_softmax_scale[q_lane] = 1.0f; + for (int j = 0; j < BLOCK_N; ++j) local_p[q_lane][j] = 0.0f; } + } + barrier(CLK_LOCAL_MEM_FENCE); - if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + // Phase 3 — V accumulate using broadcast probabilities. + { + const ACC_TYPE sp_block = local_softmax_scale[q_lane]; + const int dv_off = split_idx * SPLIT_DV_VEC; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] *= sp_block; + } + for (int j = 0; j < BLOCK_N; ++j) { + const ACC_TYPE p = local_p[q_lane][j]; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] = mad(p, CONVERT_KV_ACC4(l_v[j][dv_off + i]), o_acc[i]); + } } + } +#else + // N_SPLIT==1: j+=4 unroll. Requires BLOCK_N % 4 == 0. + if (query_valid) { + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL + for (int k = 0; k < DK_VEC; k++) { + const ACC_TYPE4 qk = q_priv[k]; + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k]), dot_acc3); + } + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; - #pragma unroll - for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_KV_ACC4(l_v[j][i]) + p1 * CONVERT_KV_ACC4(l_v[j+1][i]); + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + s0 += slope * (ACC_TYPE)mask_ptr[j]; + s1 += slope * (ACC_TYPE)mask_ptr[j + 1]; + s2 += slope * (ACC_TYPE)mask_ptr[j + 2]; + s3 += slope * (ACC_TYPE)mask_ptr[j + 3]; + } else { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; + } + } + + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + + FA_UNROLL + for (int i = 0; i < DV_VEC; ++i) { + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_KV_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); + } + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } +#endif + // End of tile: every thread must finish reading l_k/l_v before the + // next iteration's load overwrites them (WAR hazard on local memory). + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. +#if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE) + if (query_valid) { + ACC_TYPE sinks_sp = 1.0f; + if (sinks_void != NULL) { + const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + sinks_sp = exp(m_i - m_final); + l_i = l_i * sinks_sp + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const int dv_off = split_idx * SPLIT_DV_VEC; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + if (l_inv > 0.0f) { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv); } - l_i = l_i * scale_prev + p0 + p1; - m_i = m_new; + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f); + } + } + } +#elif N_SPLIT > 1 + if (split_idx == 0) { + ACC_TYPE sinks_sp = 1.0f; + if (query_valid && sinks_void != NULL) { + const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + sinks_sp = exp(m_i - m_final); + l_i = l_i * sinks_sp + exp(m_sink - m_final); + m_i = m_final; } + local_softmax_scale[q_lane] = sinks_sp; + local_l_inv[q_lane] = (query_valid && l_i > 0.0f) ? (1.0f / l_i) : 0.0f; } + barrier(CLK_LOCAL_MEM_FENCE); - if (my_query_row < n_q) { + if (query_valid) { + const ACC_TYPE sinks_sp = local_softmax_scale[q_lane]; + const ACC_TYPE l_inv = local_l_inv[q_lane]; + const int dv_off = split_idx * SPLIT_DV_VEC; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + if (l_inv > 0.0f) { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv); + } + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f); + } + } + } +#else + if (query_valid) { if (sinks_void != NULL) { const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); const ACC_TYPE m_sink = sinks_ptr[head_idx]; const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -194,19 +603,25 @@ __kernel void flash_attn_f32_f16( global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_O_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (O_DATA_TYPE4)(0.0f); } } } +#endif } +#endif // !FA_DECODE_ONLY +// allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs +#ifndef FA_PREFILL_ONLY +#ifndef FA_MQ_ONLY // q1 excluded from the MQ-only (g8) program +REQD_FA_SG __kernel void flash_attn_f32_f16_q1( const global void * q_void, ulong q_offset, const global void * k_void, ulong k_offset, @@ -247,7 +662,9 @@ __kernel void flash_attn_f32_f16_q1( const int head_kv_idx = head_idx / gqa_ratio; const global char* q_base = (const global char*)q_void + q_offset; +#ifndef FA_K_IMG const global char* k_base = (const global char*)k_void + k_offset; +#endif const global char* v_base = (const global char*)v_void + v_offset; global char* o_base = (global char*)o_void + o_offset; @@ -258,13 +675,16 @@ __kernel void flash_attn_f32_f16_q1( mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; } - ACC_TYPE4 q_priv[DK_VEC]; + // Q is uniform across WG threads (n_q=1). Share via local memory to + // avoid per-thread q_priv[DK_VEC] dynamic-indexed private array that + // spills to DDR on Adreno. + __local ACC_TYPE4 q_shared[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll - for (int i = 0; i < DK_VEC; ++i) { - q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); } + sub_group_barrier(CLK_LOCAL_MEM_FENCE); float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -273,14 +693,14 @@ __kernel void flash_attn_f32_f16_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; if (mask_base != NULL) { @@ -293,18 +713,10 @@ __kernel void flash_attn_f32_f16_q1( m_i = max(m_i, score); } - __local ACC_TYPE local_m[Q1_WG_SIZE]; - local_m[tid] = m_i; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll - for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); - barrier(CLK_LOCAL_MEM_FENCE); - } - const ACC_TYPE m_final = local_m[0]; + const ACC_TYPE m_final = sub_group_reduce_max(m_i); ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -314,9 +726,9 @@ __kernel void flash_attn_f32_f16_q1( const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset); const global KV_DATA_TYPE4* v_ptr = (const global KV_DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; if (mask_base != NULL) { @@ -328,25 +740,18 @@ __kernel void flash_attn_f32_f16_q1( } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]); } } - __local ACC_TYPE local_l[Q1_WG_SIZE]; __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; - local_l[tid] = l_i; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll - for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) local_l[tid] += local_l[tid + s]; - barrier(CLK_LOCAL_MEM_FENCE); - } + const ACC_TYPE l_red = sub_group_reduce_add(l_i); const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); - ACC_TYPE l_final = local_l[0]; + ACC_TYPE l_final = l_red; if (sinks_ptr != NULL) { l_final += exp(sinks_ptr[head_idx] - m_final); @@ -356,18 +761,1512 @@ __kernel void flash_attn_f32_f16_q1( const ACC_TYPE l_inv = 1.0f / l_final; for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; - barrier(CLK_LOCAL_MEM_FENCE); + sub_group_barrier(CLK_LOCAL_MEM_FENCE); } if (tid == 0) { o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); } } + +#endif // !FA_MQ_ONLY (q1) +// decode variant for large DV (e.g. Gemma-4 DK=DV=512 global layers). +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +// allow bypassing the kernel to avoid compiler crash for DK=512 on Adreno GPUs +#if !defined(FA_DECODE_MINIMAL) && !defined(FA_MQ_ONLY) +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; // subgroup index (0..VEC_NSG-1) + const int tid_sg = tid % Q1_WG_SIZE; // lane within subgroup + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // Q is uniform across the WG — stage in __local once. All WG threads load. + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + // per-thread DV slice within its subgroup + // DV=512 -> 2x float4 = 32 bytes; DV=256 -> 1x float4 - no spill + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + // each subgroup independently runs the FA-2 online softmax over its slice of n_kv. + // sinks are not folded into per-subgroup m_i — they're added once in + // the cross-subgroup merge to avoid double-counting. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + // Q*K^T: each thread accumulates its DK slice; subgroup-reduce the partial. + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + dot4 = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + // FA-2 online update. All threads in the subgroup see the same score, + // so m_i and l_i evolve identically across lanes within the subgroup. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + o_acc[idx] = mad(p, CONVERT_KV_ACC4(v_ptr[dv_idx]), o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + // Cross-subgroup merge via __local. Each subgroup publishes (m_i, l_i) + // and its o_acc slice; subgroup 0 then folds them into the final norm + // and writes the row. + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + // m_final = max over all subgroups' m_i, plus the sink (if any). + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + // Each thread in subgroup 0 writes its DV slice, folding all subgroups' + // contributions with the rescale factor. + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +#endif // !FA_DECODE_MINIMAL + +#ifndef FA_DECODE_ONLY + +// flash_attn_f32_f16_q1_local_tile +// one WG per (q_idx, q_head) + +#define LT_KC 32 +#define LT_WG 128 + +#ifndef FA_MQ_ONLY // q1_local_tile excluded from the MQ-only (g8) program +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_local_tile( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void * sinks_void, + const ulong sinks_offset +) { + const int q_idx = get_global_id(0) / LT_WG; + const int head_idx = get_global_id(1); + const int batch_idx = get_global_id(2); + const int tid = get_local_id(0); + + const int gqa_ratio = n_head_kv > 0 ? (n_head / n_head_kv) : 1; + const int head_kv_idx = head_idx / gqa_ratio; + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + __local half k_tile[LT_KC * DK]; // 32*128*2 = 8 KB at DK=128 + __local half v_tile[LT_KC * DV]; // 8 KB + __local float red[LT_WG]; // 512 B reduction scratch + __local float score_shared; // broadcast score (each K-step) + + // Each thread owns one float of Q at index `tid` (assumes LT_WG == DK). + const global char * q_row_base = (const global char *) q_void + q_offset + + batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + float q_val = ((const global float *) q_row_base)[tid]; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + float o_val = 0.0f; + float m_i = FA_M_INIT; + float l_i = 0.0f; + + for (int kb = 0; kb < n_kv; kb += LT_KC) { + const int tile_len = min(LT_KC, n_kv - kb); + + // Stage K and V tiles into __local. + for (int i = tid; i < tile_len * DK; i += LT_WG) { + const int j = i / DK; + const int d = i % DK; + const int kv_idx = kb + j; + const global char * k_row = (const global char *) k_void + k_offset + + batch_idx * k_nb3 + head_kv_idx * k_nb2 + + (ulong) kv_idx * k_nb1; + const global char * v_row = (const global char *) v_void + v_offset + + batch_idx * v_nb3 + head_kv_idx * v_nb2 + + (ulong) kv_idx * v_nb1; + k_tile[j * DK + d] = ((const global half *) k_row)[d]; + v_tile[j * DV + d] = ((const global half *) v_row)[d]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int j = 0; j < tile_len; ++j) { + const int kv_idx = kb + j; + + // Q·K dot via __local tree-reduce. + red[tid] = q_val * convert_float(k_tile[j * DK + tid]); + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = LT_WG >> 1; stride > 0; stride >>= 1) { + if (tid < stride) { + red[tid] += red[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (tid == 0) { + float s = red[0] * scale; + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + s += slope * (float) mask_ptr[kv_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score_shared = s; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float s = score_shared; + const float m_new = fmax(m_i, s); + const float alpha = native_exp(m_i - m_new); + const float beta = native_exp(s - m_new); + + o_val = o_val * alpha + beta * convert_float(v_tile[j * DV + tid]); + l_i = l_i * alpha + beta; + m_i = m_new; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Fold attention sinks into the running (m, l, o), if present. + if (sinks_void != NULL) { + const global float * sinks_ptr = + (const global float *) ((const global char *) sinks_void + sinks_offset); + const float m_sink = sinks_ptr[head_idx]; + const float m_new = fmax(m_i, m_sink); + const float alpha = native_exp(m_i - m_new); + const float beta = native_exp(m_sink - m_new); + o_val = o_val * alpha; + l_i = l_i * alpha + beta; + m_i = m_new; + } + + const float l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + global float * o_row = (global float *) ((global char *) o_void + o_offset + + batch_idx * o_nb3 + head_idx * o_nb1 + + (ulong) q_idx * o_nb2); + o_row[tid] = o_val * l_inv; +} + +// flash_attn_f32_f16_q1_local_mq_split + +#define LMQ_WG 64 +#define LMQ_KC 32 +#define LMQ_DPL 2 // DK / LMQ_WG at DK=128 + +#endif // !FA_MQ_ONLY (q1_local_tile) +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif + +#ifndef FA_MQ_ONLY // q1_local_mq_split excluded from the MQ-only (g8) program +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_local_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); // 0..LMQ_WG-1 + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // Empty split — write sentinel for each Q-head so merge treats it as 0. + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // Stage MQ_GQA Q rows in __local (MQ_GQA × DK floats). + __local float q_shared[MQ_GQA * DK]; + for (int i = tid; i < MQ_GQA * DK; i += LMQ_WG) { + const int h = i / DK; + const int d = i % DK; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_off = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global float * q_ptr = (const global float *) (q_base + q_row_off); + q_shared[h * DK + d] = q_ptr[d]; + } + + // K/V tile staging buffers (16 KB combined at DK=DV=128 KC=32). + __local half k_tile[LMQ_KC * DK]; + __local half v_tile[LMQ_KC * DV]; + + // Per-h state held in private registers. + float o_acc[MQ_GQA][LMQ_DPL]; + float m_i[MQ_GQA]; + float l_i[MQ_GQA]; + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) o_acc[h][p] = 0.0f; + } + + // Per-h mask pointers. + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + barrier(CLK_LOCAL_MEM_FENCE); // Ensure Q staged before first dot. + + for (int kb = kv_start; kb < kv_end; kb += LMQ_KC) { + const int tile_len = min((int) LMQ_KC, kv_end - kb); + + // Cooperative load K + V tile. + for (int i = tid; i < tile_len * DK; i += LMQ_WG) { + const int j = i / DK; + const int d = i % DK; + const int kv_idx = kb + j; + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + (ulong) kv_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + (ulong) kv_idx * v_nb1; + k_tile[j * DK + d] = ((const global half *) k_row)[d]; + v_tile[j * DV + d] = ((const global half *) v_row)[d]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Process each cache row in the tile. + for (int j = 0; j < tile_len; ++j) { + const int kv_idx = kb + j; + + // Dot product per h: lane owns LMQ_DPL D-elements at (tid*LMQ_DPL..). + float score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + float contrib = 0.0f; + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + contrib += q_shared[h * DK + d] * (float) k_tile[j * DK + d]; + } + float s = sub_group_reduce_add(contrib) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (float) mask_ptr[kv_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } + + // Online softmax update + V accumulation per h. + float p_h[MQ_GQA]; + float sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const float m_new = fmax(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + const float v_val = (float) v_tile[j * DV + d]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][p] = o_acc[h][p] * sp_h[h] + p_h[h] * v_val; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); // Before next tile load overwrites k/v_tile. + } + + // write partial records: one per (h, split) + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float * rec_o = rec + 2; + + if (tid == 0) { + rec[0] = m_i[h]; + rec[1] = l_i[h]; + } + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + rec_o[d] = o_acc[h][p]; + } + } +} + +#endif // !FA_MQ_ONLY (q1_local_mq_split) +#ifndef MQ_NSG +#define MQ_NSG 4 +#endif +#define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; // subgroup 0..MQ_NSG-1 + const int tid_sg = tid % Q1_WG_SIZE; // lane 0..63 + const int kvhead_batch_idx = get_global_id(1); + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + // per-h ALiBi slope + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + // per-h mask row pointer + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + // per-thread per-h DV slice. + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // each subgroup independently sweeps its slice of n_kv. + const int kv_per_sg = (n_kv + MQ_NSG - 1) / MQ_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + // Q*K^T: load each K stride once, dot against all MQ_GQA Q rows. + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } + + // FA-2 online softmax update — V load amortized across MQ_GQA heads. + // p, scale_prev are computed per h; the V vector is loaded once + // per dv stride and reused MQ_GQA times. + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + // cross subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG]; + __local ACC_TYPE4 sg_o[MQ_NSG][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + // each subgroup publishes its o_acc slice for head h. + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_final = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG; ++s) { + m_final = max(m_final, sg_m[h][s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG; ++s) { + l_final += sg_l[h][s] * native_exp(sg_m[h][s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv); + } + } + // Barrier guards next h's overwrite of sg_o. + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE (Q1_WG_SIZE * MQ_NSG_SPLIT) + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // write sentinel for each of the MQ_GQA Q-heads so the + // merge pass treats this slot as dropped + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // stage MQ_GQA Q rows in __local once (uniform across WG) + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // each subgroup independently sweeps its slice of the split's kv range. + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + // per-h cross-subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + // fold per-subgroup (m, l) into split-level (m_c, l_c) + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + // each thread writes its DV slice of the merged O. + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img( + const global void * q_void, ulong q_offset, + __read_only image1d_buffer_t k_img, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // K pitches in pixel units, pixel = 1 half4 = 8 B -> byte_stride >> 3. + const int pitch_px_row = (int)(k_nb1 >> 3); + const int pitch_px_head = (int)(k_nb2 >> 3); + const int pitch_px_batch = (int)(k_nb3 >> 3); + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const int k_row_px = batch_idx * pitch_px_batch + + head_kv_idx * pitch_px_head + + k_idx * pitch_px_row; + + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const half4 k_h4 = read_imageh(k_img, k_row_px + k); + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_h4); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} +#endif // !FA_DECODE_ONLY + +#ifndef FA_MQ_ONLY // q1_split + merge excluded from the MQ-only (g8) program +__kernel void flash_attn_f32_f16_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + // Empty split: leave sentinel partial for merge. + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + // share Q via local memory (n_q=1 per split -> uniform across WG). + __local ACC_TYPE4 q_shared[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // pass 1a — split-local max. + ACC_TYPE m_i = FA_M_INIT; + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset); + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < DK_VEC; ++k) { + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + } + ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + m_i = max(m_i, score); + } + + const ACC_TYPE m_c = sub_group_reduce_max(m_i); + + // pass 1b — softmax-weighted V accumulate. + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + ACC_TYPE l_i = 0.0f; + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_offset); + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < DK_VEC; ++k) { + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + } + ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + const ACC_TYPE p = exp(score - m_c); + l_i += p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) { + o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]); + } + } + + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + const ACC_TYPE l_c = sub_group_reduce_add(l_i); + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +// FD Pass 2: merge per-split partials into final O +// empty splits drop via exp(-INF)=0. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); // 0..DV_VEC-1 + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} +#endif // !FA_MQ_ONLY (q1_split + merge) +#endif // !FA_PREFILL_ONLY (decode kernels) diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl new file mode 100644 index 00000000000..444c5545e2d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl @@ -0,0 +1,1615 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#define FA_HAVE_INT_DOT 1 +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#endif + +// Flash attention: Q=f32, K=q4_0, V=q4_0. +// Block = half d + uchar qs[16]; qs[j] low/high nibble -> elem j / j+16. +// Dequant: val[i] = d * (nibble_i - 8). dp4a path runs on raw 0..15 nibbles +// and applies the -8*sum(q) correction once per block (needs Q q_sum). + +#define ACC_TYPE float +#define ACC_TYPE4 float4 +#define Q_DATA_TYPE4 float4 +#define O_DATA_TYPE4 float4 +#define MASK_DATA_TYPE half +#define CONVERT_Q_ACC4(x) (x) +#define CONVERT_O_DATA4(x) (x) + +#define DK_VEC (DK/4) +#define DV_VEC (DV/4) + +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +#define QK4_0 32 +#define Q4_0_BLOCK_SIZE 18 + +#define DK_Q4_BLOCKS (DK / QK4_0) +#define DV_Q4_BLOCKS (DV / QK4_0) + +inline float dot_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) { + float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + + float sum = 0.0f; + // Low nibbles -> elems 0..15. + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 nv = (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + sum += dot(q_slice[g], nv); + } + // High nibbles -> elems 16..31. + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 nv = (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + sum += dot(q_slice[4 + g], nv); + } + return sum * d; +} + +#ifdef FA_HAVE_INT_DOT +inline uint pack_i8x4(char a, char b, char c, char d) { + return ((uint)(uchar)a) | + ((uint)(uchar)b) << 8 | + ((uint)(uchar)c) << 16 | + ((uint)(uchar)d) << 24; +} + +// Returns (qd, q_sum); q_sum feeds the -8*sum(q) bias correction. +typedef struct { + float qd; + int q_sum; +} q4_q_block_info; + +inline q4_q_block_info quant_q_block_int8_packed_q4(const ACC_TYPE4 * q_block, + uint * out_packed) { + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 av = fabs(q_block[i]); + amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3))); + } + float qd = amax / 127.0f; + float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f; + + int q_sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 v = q_block[i] * qid; + char a = (char)((int)round(v.s0)); + char b = (char)((int)round(v.s1)); + char c = (char)((int)round(v.s2)); + char d = (char)((int)round(v.s3)); + out_packed[i] = pack_i8x4(a, b, c, d); + q_sum += (int)a + (int)b + (int)c + (int)d; + } + q4_q_block_info info = { qd, q_sum }; + return info; +} + +// k_packed[0..3] = low nibbles (Q elems 0..15), k_packed[4..7] = high (16..31). +inline void pack_q4_0_nibbles(const global uchar * qs, uint * k_packed) { + #pragma unroll + for (int g = 0; g < 4; ++g) { + uchar b0 = qs[g*4 + 0]; + uchar b1 = qs[g*4 + 1]; + uchar b2 = qs[g*4 + 2]; + uchar b3 = qs[g*4 + 3]; + k_packed[g] = + ((uint)(b0 & 0x0F)) | + ((uint)(b1 & 0x0F)) << 8 | + ((uint)(b2 & 0x0F)) << 16 | + ((uint)(b3 & 0x0F)) << 24; + k_packed[4 + g] = + ((uint)(b0 >> 4)) | + ((uint)(b1 >> 4)) << 8 | + ((uint)(b2 >> 4)) << 16 | + ((uint)(b3 >> 4)) << 24; + } +} + +inline float dot_q4_0_int(const global char * k_block_ptr, + const uint * q_packed, + float q_d, + int q_sum) { + float kd = vload_half(0, (const global half *)k_block_ptr); + const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2); + + uint k_packed[8]; + pack_q4_0_nibbles(k_qs, k_packed); + + int sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed[i], sum); + } + // Correct raw-nibble sum: (nibble - 8) bias -> subtract 8 * q_sum. + return (float)(sum - 8 * q_sum) * q_d * kd; +} +#endif // FA_HAVE_INT_DOT + +inline void dequant_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * out) { + float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + + #pragma unroll + for (int g = 0; g < 4; ++g) { + out[g] = d * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + out[4 + g] = d * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + } +} + +// max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier. +inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) { + if (max_bias <= 0.0f) return 1.0f; + float base = (head_idx < n_head_log2) ? m0 : m1; + int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1); + return pow(base, (float)exph); +} + +// q1 decode: one query row per WG, threads sweep KV positions. +__kernel void flash_attn_f32_q4_0_q1( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char* q_base = (const global char*)q_void + q_offset; + const global char* k_base = (const global char*)k_void + k_offset; + const global char* v_base = (const global char*)v_void + v_offset; + global char* o_base = (global char*)o_void + o_offset; + + const global char* mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + // Quantise Q once per thread: 8 uints + qd + q_sum per block. + uint q_packed[DK_Q4_BLOCKS * 8]; + float q_d_scale[DK_Q4_BLOCKS]; + int q_sum_arr[DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]); + q_d_scale[b] = info.qd; + q_sum_arr[b] = info.q_sum; + } +#endif + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE* sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + } + + // One-pass online softmax (FA-2): single sweep over kv positions, + // updating per-thread (m_i, l_i, o_acc) per K. Eliminates the second + // K read of the original two-pass implementation. + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { + const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; b++) { +#ifdef FA_HAVE_INT_DOT + score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]); +#else + score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + + if (mask_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); + score += slope * (ACC_TYPE)mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q4_BLOCKS; b++) { + ACC_TYPE4 v_dequant[8]; + dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; i++) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_final, rescale per-thread l_i and + // o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_final = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_final); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + ACC_TYPE l_final = local_l[0]; + + if (sinks_ptr != NULL) { + l_final += exp(sinks_ptr[head_idx] - m_final); + } + + if (l_final > 0.0f) { + const ACC_TYPE l_inv = 1.0f / l_final; + for (int i = 0; i < DV_VEC; i++) { + local_o_comp[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); + } + } + } else if (tid == 0) { + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); + } +} + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +// Dequant one float4 lane (0..7) from a q4_0 block. +// Lanes 0..3 → low nibbles of qs[0..15], lanes 4..7 → high nibbles. +inline float4 dequant_q4_0_lane(const global char * block_ptr, int lane) { + const float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + const int g = lane & 3; + const int shift = (lane < 4) ? 0 : 4; + return d * (float4)((float)((qs[g*4+0] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+1] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+2] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+3] >> shift) & 0x0F) - 8.0f); +} + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q4_0_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + +#ifdef FA_HAVE_INT_DOT + // quantize Q to int8-packed uints + per-block (qd, q_sum) once per WG for dp4a + // one thread per Q block, remaining threads idle this step + __local uint q_packed_shared[DK_Q4_BLOCKS * 8]; + __local float q_d_shared[DK_Q4_BLOCKS]; + __local int q_sum_shared[DK_Q4_BLOCKS]; + if (tid < DK_Q4_BLOCKS) { + ACC_TYPE4 q_block[8]; + #pragma unroll + for (int i = 0; i < 8; ++i) q_block[i] = q_shared[tid * 8 + i]; + uint packed[8]; + q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed); + #pragma unroll + for (int i = 0; i < 8; ++i) q_packed_shared[tid * 8 + i] = packed[i]; + q_d_shared[tid] = info.qd; + q_sum_shared[tid] = info.q_sum; + } + barrier(CLK_LOCAL_MEM_FENCE); +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + +#ifdef FA_HAVE_INT_DOT + // per-lane dp4a: each lane packs 4 raw q4_0 nibbles into a uint, + // then dot_acc_sat_4x8packed_ss_int against the matching uint. + ACC_TYPE lane_contrib = 0.0f; + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane_in_block = qk % 8; + const int g = lane_in_block & 3; + const int shift = (lane_in_block < 4) ? 0 : 4; + const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE; + const float kd = vload_half(0, (const global half *)k_block); + const global uchar * k_qs = (const global uchar *)(k_block + 2); + const uchar b0 = k_qs[g*4 + 0]; + const uchar b1 = k_qs[g*4 + 1]; + const uchar b2 = k_qs[g*4 + 2]; + const uchar b3 = k_qs[g*4 + 3]; + const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) | + ((uint)((b1 >> shift) & 0x0F)) << 8 | + ((uint)((b2 >> shift) & 0x0F)) << 16 | + ((uint)((b3 >> shift) & 0x0F)) << 24; + const uint q_packed_lane = q_packed_shared[block_idx * 8 + lane_in_block]; + const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0); + const float qd = q_d_shared[block_idx]; + const float block_scale = qd * kd; + float contrib = (float)raw_dot * block_scale; + if (lane_in_block == 0) { + // block bias correction is per-block + const int q_sum_b = q_sum_shared[block_idx]; + contrib -= 8.0f * block_scale * (float)q_sum_b; + } + lane_contrib += contrib; + } + ACC_TYPE score = sub_group_reduce_add(lane_contrib) * scale; +#else + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane); + dot4 = mad(q_shared[qk], k_v, dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; +#endif + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane); + o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged); + } + o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +// Flash-decoding split pass for q4_0 KV. Merge kernel is type-agnostic and +// shared with the f16/q8_0 FA kernels. +#define FA_PARTIAL_FLOATS (2 + DV) + +__kernel void flash_attn_f32_q4_0_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed[DK_Q4_BLOCKS * 8]; + float q_d_scale[DK_Q4_BLOCKS]; + int q_sum_arr[DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]); + q_d_scale[b] = info.qd; + q_sum_arr[b] = info.q_sum; + } +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // One-pass online softmax (FA-2): single sweep over the split's K range. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { +#ifdef FA_HAVE_INT_DOT + score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]); +#else + score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q4_BLOCKS; ++b) { + ACC_TYPE4 v_dequant[8]; + dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; ++i) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_c, rescale per-thread l_i and o_acc + // by alpha = exp(m_i_thread - m_c) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_c = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_c); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE l_c = local_l[0]; + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +// Prefill: q4_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling. +// K in local as packed nibbles + per-block scale; V dequant -> half in local. +// Requires DK % QK4_0 == 0 and DV % QK4_0 == 0. +#define KV_DATA_TYPE4 half4 +#define CONVERT_KV_ACC4(x) convert_float4(x) + +#define DK_Q4_BLOCKS_PREFILL (DK / QK4_0) +#define DV_Q4_BLOCKS_PREFILL (DV / QK4_0) + +// N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs +// sub_group_shuffle_xor and DK_Q4_BLOCKS_PREFILL % N_SPLIT == 0. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#if N_SPLIT > 1 +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) +#define SPLIT_DK_Q4_BLOCKS (DK_Q4_BLOCKS_PREFILL / N_SPLIT) +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else +#define SPLIT_DK_VEC DK_VEC +#define SPLIT_DV_VEC DV_VEC +#define SPLIT_DK_Q4_BLOCKS DK_Q4_BLOCKS_PREFILL +#define WG_SIZE BLOCK_M +#endif + +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE_Q4 (Q1_WG_SIZE * MQ_NSG_SPLIT) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q4_0_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q4) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + +#ifdef FA_HAVE_INT_DOT + __local uint q_packed_shared[MQ_GQA * DK_Q4_BLOCKS * 8]; + __local float q_d_shared[MQ_GQA * DK_Q4_BLOCKS]; + __local int q_sum_shared[MQ_GQA * DK_Q4_BLOCKS]; + { + const int active = MQ_GQA * DK_Q4_BLOCKS; + if (tid < active) { + const int h = tid / DK_Q4_BLOCKS; + const int block_id = tid % DK_Q4_BLOCKS; + ACC_TYPE4 q_block[8]; + #pragma unroll + for (int i = 0; i < 8; ++i) q_block[i] = q_shared[h * DK_VEC + block_id * 8 + i]; + uint packed[8]; + q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed); + #pragma unroll + for (int i = 0; i < 8; ++i) q_packed_shared[(h * DK_Q4_BLOCKS + block_id) * 8 + i] = packed[i]; + q_d_shared[h * DK_Q4_BLOCKS + block_id] = info.qd; + q_sum_shared[h * DK_Q4_BLOCKS + block_id] = info.q_sum; + } + } + barrier(CLK_LOCAL_MEM_FENCE); +#endif + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + +#ifdef FA_HAVE_INT_DOT + ACC_TYPE lane_contrib[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) lane_contrib[h] = 0.0f; + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane_in_block = qk % 8; + const int g = lane_in_block & 3; + const int shift = (lane_in_block < 4) ? 0 : 4; + const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE; + const float kd = vload_half(0, (const global half *)k_block); + const global uchar * k_qs = (const global uchar *)(k_block + 2); + const uchar b0 = k_qs[g*4 + 0]; + const uchar b1 = k_qs[g*4 + 1]; + const uchar b2 = k_qs[g*4 + 2]; + const uchar b3 = k_qs[g*4 + 3]; + const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) | + ((uint)((b1 >> shift) & 0x0F)) << 8 | + ((uint)((b2 >> shift) & 0x0F)) << 16 | + ((uint)((b3 >> shift) & 0x0F)) << 24; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const uint q_packed_lane = q_packed_shared[(h * DK_Q4_BLOCKS + block_idx) * 8 + lane_in_block]; + const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0); + const float qd = q_d_shared[h * DK_Q4_BLOCKS + block_idx]; + const float block_scale = qd * kd; + float contrib = (float) raw_dot * block_scale; + if (lane_in_block == 0) { + const int q_sum_b = q_sum_shared[h * DK_Q4_BLOCKS + block_idx]; + contrib -= 8.0f * block_scale * (float) q_sum_b; + } + lane_contrib[h] += contrib; + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = sub_group_reduce_add(lane_contrib[h]) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } +#else + // fallback float-dequant K dot + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } +#endif + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]); + } + } + } + + // per-h cross-subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +__kernel void flash_attn_f32_q4_0( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset, + // blk: per-(qblock,kvblock) class from flash_attn_blk_f16 + // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt. + const global void * blk_void +) { + const int tid = get_local_id(0); + const int block_q_idx = get_group_id(0); + const int head_batch_idx = get_global_id(1); + +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx. + #ifndef BLK_PREPASS_BM + #define BLK_PREPASS_BM BLOCK_M + #endif + const global char * blk_base = NULL; + int n_kv_blocks = 0; + if (blk_void != NULL) { + n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM; + const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM; + blk_base = (const global char *) blk_void + + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks; + } + + const int dk_off_vec = split_idx * SPLIT_DK_VEC; + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + if (query_valid) { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; + const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = q_ptr[dk_off_vec + i]; + } + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed_pf[SPLIT_DK_Q4_BLOCKS * 8]; + float q_d_pf[SPLIT_DK_Q4_BLOCKS]; + int q_sum_pf[SPLIT_DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < SPLIT_DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed_pf[b * 8]); + q_d_pf[b] = info.qd; + q_sum_pf[b] = info.q_sum; + } +#endif + + const int dv_off_vec = split_idx * SPLIT_DV_VEC; + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + +#ifdef FA_HAVE_INT_DOT + __local uint l_k_packed[BLOCK_N][DK_Q4_BLOCKS_PREFILL * 8]; + __local float l_k_scale [BLOCK_N][DK_Q4_BLOCKS_PREFILL]; +#else + __local half4 l_k[BLOCK_N][DK_VEC]; +#endif + + __local half4 l_v[BLOCK_N][DV_VEC]; + + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + // Skip fully-masked KV tiles (uniform branch across WG). + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + { +#ifdef FA_HAVE_INT_DOT + const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + l_k_scale[row][blk] = df; + uint k_packed[8]; + pack_q4_0_nibbles(qs, k_packed); + #pragma unroll + for (int j = 0; j < 8; ++j) { + l_k_packed[row][blk * 8 + j] = k_packed[j]; + } + } else { + l_k_scale[row][blk] = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) l_k_packed[row][blk * 8 + j] = 0u; + } + } +#else + // Fallback: dequant q4_0 -> half in local memory. + const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + l_k[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3); + l_k[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h); + } + } +#endif + } + // V tile load — dequant V -> half in local memory. + { + const int v_blocks_per_row = DV_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + l_v[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3); + l_v[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor. +#if N_SPLIT > 1 + { +#else + if (query_valid) { +#endif + const int k_blk_base = split_idx * SPLIT_DK_Q4_BLOCKS; + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; + + ACC_TYPE s0, s1, s2, s3; +#ifdef FA_HAVE_INT_DOT + s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f; + #pragma unroll + for (int b_local = 0; b_local < SPLIT_DK_Q4_BLOCKS; ++b_local) { + const int b = k_blk_base + b_local; + int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3); + } + const float qd = q_d_pf[b_local]; + const int q_sum = q_sum_pf[b_local]; + s0 += (float)(sum0 - 8 * q_sum) * qd * l_k_scale[j ][b]; + s1 += (float)(sum1 - 8 * q_sum) * qd * l_k_scale[j+1][b]; + s2 += (float)(sum2 - 8 * q_sum) * qd * l_k_scale[j+2][b]; + s3 += (float)(sum3 - 8 * q_sum) * qd * l_k_scale[j+3][b]; + } +#else + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < SPLIT_DK_VEC; ++k) { + const ACC_TYPE4 qk = q_priv[k]; + const int k_abs = dk_off_vec + k; + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3); + } + s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3; + s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3; + s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3; + s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3; +#endif + +#if N_SPLIT > 1 + // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96): + // explicit 3-lane shuffle. + #if (N_SPLIT & (N_SPLIT - 1)) == 0 + #pragma unroll + for (int step = 1; step < N_SPLIT; step <<= 1) { + s0 += sub_group_shuffle_xor(s0, step); + s1 += sub_group_shuffle_xor(s1, step); + s2 += sub_group_shuffle_xor(s2, step); + s3 += sub_group_shuffle_xor(s3, step); + } + #else + const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT; + s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2); + s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2); + s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); + s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); + #endif + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } +#endif + s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (query_valid && mask_base != NULL && blk_cur != 2) { + const global MASK_DATA_TYPE * mask_ptr = + (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; + } + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + const int i_abs = dv_off_vec + i; + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]), + mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]), + o_acc[i] * scale_prev)))); + } + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. + if (query_valid) { + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + const ACC_TYPE scale_o = exp(m_i - m_final); + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o; + l_i = l_i * scale_o + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global float4 * o_row = (global float4 *) (o_base + o_row_offset); + if (l_inv > 0.0f) { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv; + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f); + } + } +} + +// FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA +// source owns a copy since kernels compile per-source-program. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl new file mode 100644 index 00000000000..547e6bc6a8f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl @@ -0,0 +1,1486 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#define FA_HAVE_INT_DOT 1 +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#endif + +// Flash attention: Q=f32, K=q8_0, V=q8_0. + +#define ACC_TYPE float +#define ACC_TYPE4 float4 +#define Q_DATA_TYPE4 float4 +#define O_DATA_TYPE4 float4 +#define MASK_DATA_TYPE half +#define CONVERT_Q_ACC4(x) (x) +#define CONVERT_O_DATA4(x) (x) + +#define DK_VEC (DK/4) +#define DV_VEC (DV/4) + +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// q8_0 block: 2B scale (half) + 32B int8 quants. +#define QK8_0 32 +#define Q8_0_BLOCK_SIZE 34 + +#define DK_Q8_BLOCKS (DK / QK8_0) +#define DV_Q8_BLOCKS (DV / QK8_0) + +inline float dot_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) { + float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2; + + float sum = 0.0f; + #pragma unroll + for (int i = 0; i < 8; i++) { + float4 qv = (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]); + sum += dot(q_slice[i], qv); + } + return sum * d; +} + +#ifdef FA_HAVE_INT_DOT +inline uint pack_i8x4(char a, char b, char c, char d) { + return ((uint)(uchar)a) | + ((uint)(uchar)b) << 8 | + ((uint)(uchar)c) << 16 | + ((uint)(uchar)d) << 24; +} + +inline float quant_q_block_int8_packed(const ACC_TYPE4 * q_block, + uint * out_packed) { + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 av = fabs(q_block[i]); + amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3))); + } + float qd = amax / 127.0f; + float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f; + + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 v = q_block[i] * qid; + char a = (char)((int)round(v.s0)); + char b = (char)((int)round(v.s1)); + char c = (char)((int)round(v.s2)); + char d = (char)((int)round(v.s3)); + out_packed[i] = pack_i8x4(a, b, c, d); + } + return qd; +} + +inline float dot_q8_0_int(const global char * k_block_ptr, + const uint * q_packed, + float q_d) { + float kd = vload_half(0, (const global half *)k_block_ptr); + const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2); + + // k_qs is 2-byte aligned; pack chars per iteration rather than cast to uint*. + int sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + uint k_packed = + (uint)k_qs[i*4 + 0] | + ((uint)k_qs[i*4 + 1]) << 8 | + ((uint)k_qs[i*4 + 2]) << 16 | + ((uint)k_qs[i*4 + 3]) << 24; + sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed, sum); + } + return (float)sum * q_d * kd; +} +#endif // FA_HAVE_INT_DOT + +inline void dequant_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * out) { + float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2; + + #pragma unroll + for (int i = 0; i < 8; i++) { + out[i] = d * (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]); + } +} + +// max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier. +inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) { + if (max_bias <= 0.0f) return 1.0f; + float base = (head_idx < n_head_log2) ? m0 : m1; + int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1); + return pow(base, (float)exph); +} + +// q1 decode: one query row per WG, threads sweep KV positions. +__kernel void flash_attn_f32_q8_0_q1( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char* q_base = (const global char*)q_void + q_offset; + const global char* k_base = (const global char*)k_void + k_offset; + const global char* v_base = (const global char*)v_void + v_offset; + global char* o_base = (global char*)o_void + o_offset; + + const global char* mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + // Quantise Q once per thread; q_priv stays as fp for the V accumulate. + uint q_packed[DK_Q8_BLOCKS * 8]; + float q_d_scale[DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { + q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]); + } +#endif + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE* sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + } + + // One-pass online softmax: per-thread maintains running (m_i, l_i, o_acc), + // updating each as new K positions are processed. Eliminates the second + // K read of the original two-pass implementation. After the loop, threads + // are merged via the standard FA-2 cross-thread reduction (rescale each + // thread's l_i and o_acc by alpha=exp(m_i_thread - m_final), then sum). + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { + const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; b++) { +#ifdef FA_HAVE_INT_DOT + score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b]); +#else + score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + + if (mask_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); + score += slope * (ACC_TYPE)mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q8_BLOCKS; b++) { + ACC_TYPE4 v_dequant[8]; + dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; i++) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_final, then rescale per-thread l_i + // and o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_final = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_final); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + ACC_TYPE l_final = local_l[0]; + + if (sinks_ptr != NULL) { + l_final += exp(sinks_ptr[head_idx] - m_final); + } + + if (l_final > 0.0f) { + const ACC_TYPE l_inv = 1.0f / l_final; + for (int i = 0; i < DV_VEC; i++) { + local_o_comp[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); + } + } + } else if (tid == 0) { + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); + } +} + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +inline float4 dequant_q8_0_lane(const global char * block_ptr, int lane) { + const float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2 + lane * 4; + return d * (float4)((float)qs[0], (float)qs[1], (float)qs[2], (float)qs[3]); +} + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q8_0_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane); + dot4 = mad(q_shared[qk], k_v, dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane); + o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged); + } + o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +// Flash-decoding split pass for q8_0 KV. Partial record: [m, l, O[DV]]. +// Merge kernel from flash_attn_f32_f16.cl is type-agnostic and reused. +#define FA_PARTIAL_FLOATS (2 + DV) + +__kernel void flash_attn_f32_q8_0_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + // Empty split: leave sentinel partial for merge. + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed[DK_Q8_BLOCKS * 8]; + float q_d_scale[DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { + q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]); + } +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // One-pass online softmax (FA-2): single sweep over the split's K range, + // updating per-thread (m_i, l_i, o_acc) per position. Eliminates the + // second K read of the original two-pass implementation. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { +#ifdef FA_HAVE_INT_DOT + score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE, &q_packed[b * 8], q_d_scale[b]); +#else + score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q8_BLOCKS; ++b) { + ACC_TYPE4 v_dequant[8]; + dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; ++i) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_c, then rescale per-thread l_i and + // o_acc by alpha = exp(m_i_thread - m_c) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_c = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_c); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE l_c = local_l[0]; + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +// Prefill: q8_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling. +// K path keeps packed int8 in local for dp4a QK dot; V path dequant -> half in local. +// Requires DK % QK8_0 == 0 and DV % QK8_0 == 0 (gated in supports_op). +#define KV_DATA_TYPE4 half4 +#define CONVERT_KV_ACC4(x) convert_float4(x) + +#define DK_Q8_BLOCKS_PREFILL (DK / QK8_0) +#define DV_Q8_BLOCKS_PREFILL (DV / QK8_0) + +// N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs +// sub_group_shuffle_xor and DK_Q8_BLOCKS_PREFILL % N_SPLIT == 0. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#if N_SPLIT > 1 +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) +#define SPLIT_DK_Q8_BLOCKS (DK_Q8_BLOCKS_PREFILL / N_SPLIT) +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else +#define SPLIT_DK_VEC DK_VEC +#define SPLIT_DV_VEC DV_VEC +#define SPLIT_DK_Q8_BLOCKS DK_Q8_BLOCKS_PREFILL +#define WG_SIZE BLOCK_M +#endif + +// FA_V_STRATEGY: 0 = dequant V to half in local (default); 2 = keep packed +// int8 in local, dequant in the accumulate loop (smaller local, slightly slower). +#ifndef FA_V_STRATEGY +#define FA_V_STRATEGY 0 +#endif + +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE_Q8 (Q1_WG_SIZE * MQ_NSG_SPLIT) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q8_0_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // Empty split — write sentinel for each of the MQ_GQA Q-heads. + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q8) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]); + } + } + } + + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +__kernel void flash_attn_f32_q8_0( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset, + // blk: per-(qblock,kvblock) class from flash_attn_blk_f16 + // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt. + const global void * blk_void +) { + const int tid = get_local_id(0); + const int block_q_idx = get_group_id(0); + const int head_batch_idx = get_global_id(1); + +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx. + #ifndef BLK_PREPASS_BM + #define BLK_PREPASS_BM BLOCK_M + #endif + const global char * blk_base = NULL; + int n_kv_blocks = 0; + if (blk_void != NULL) { + n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM; + const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM; + blk_base = (const global char *) blk_void + + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks; + } + + const int dk_off_vec = split_idx * SPLIT_DK_VEC; + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + if (query_valid) { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; + const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = q_ptr[dk_off_vec + i]; + } + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed_pf[SPLIT_DK_Q8_BLOCKS * 8]; + float q_d_pf[SPLIT_DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < SPLIT_DK_Q8_BLOCKS; ++b) { + q_d_pf[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed_pf[b * 8]); + } +#endif + + const int dv_off_vec = split_idx * SPLIT_DV_VEC; + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + +#ifdef FA_HAVE_INT_DOT + __local uint l_k_packed[BLOCK_N][DK_Q8_BLOCKS_PREFILL * 8]; + __local float l_k_scale [BLOCK_N][DK_Q8_BLOCKS_PREFILL]; +#else + __local half4 l_k[BLOCK_N][DK_VEC]; +#endif + +#if FA_V_STRATEGY == 2 + __local uint l_v_packed[BLOCK_N][DV_Q8_BLOCKS_PREFILL * 8]; + __local float l_v_scale [BLOCK_N][DV_Q8_BLOCKS_PREFILL]; +#else + __local half4 l_v[BLOCK_N][DV_VEC]; +#endif + + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + // Skip fully-masked KV tiles (uniform branch across WG). + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + { +#ifdef FA_HAVE_INT_DOT + const int k_blocks_per_row = DK_Q8_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + l_k_scale[row][blk] = df; + #pragma unroll + for (int j = 0; j < 8; ++j) { + uint k_packed = + (uint) qs[j*4 + 0] | + ((uint) qs[j*4 + 1]) << 8 | + ((uint) qs[j*4 + 2]) << 16 | + ((uint) qs[j*4 + 3]) << 24; + l_k_packed[row][blk * 8 + j] = k_packed; + } + } else { + l_k_scale[row][blk] = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) l_k_packed[row][blk * 8 + j] = 0u; + } + } +#else + // Fallback: dequant q8_0 -> half in local memory. + const int k_blocks_per_row = DK / QK8_0; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global char * qs = blk_ptr + 2; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = df * (float4)((float) qs[j*4 + 0], + (float) qs[j*4 + 1], + (float) qs[j*4 + 2], + (float) qs[j*4 + 3]); + l_k[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h); + } + } +#endif + } + // V tile load — strategy-dependent. +#if FA_V_STRATEGY == 2 + { + // Int8 packed V in local memory + per-block scale. Accumulate + // step unpacks inline. + const int v_blocks_per_row = DV_Q8_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + l_v_scale[row][blk] = df; + #pragma unroll + for (int j = 0; j < 8; ++j) { + uint v_packed = + (uint) qs[j*4 + 0] | + ((uint) qs[j*4 + 1]) << 8 | + ((uint) qs[j*4 + 2]) << 16 | + ((uint) qs[j*4 + 3]) << 24; + l_v_packed[row][blk * 8 + j] = v_packed; + } + } else { + l_v_scale[row][blk] = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) l_v_packed[row][blk * 8 + j] = 0u; + } + } + } +#else + { + // Default: dequant V -> half in local memory. + const int v_blocks_per_row = DV / QK8_0; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global char * qs = blk_ptr + 2; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = df * (float4)((float) qs[j*4 + 0], + (float) qs[j*4 + 1], + (float) qs[j*4 + 2], + (float) qs[j*4 + 3]); + l_v[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h); + } + } + } +#endif + barrier(CLK_LOCAL_MEM_FENCE); + + // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor. +#if N_SPLIT > 1 + { +#else + if (query_valid) { +#endif + const int k_blk_base = split_idx * SPLIT_DK_Q8_BLOCKS; + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; + + ACC_TYPE s0, s1, s2, s3; +#ifdef FA_HAVE_INT_DOT + // dp4a-accelerated QK dot over owned blocks. + s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f; + #pragma unroll + for (int b_local = 0; b_local < SPLIT_DK_Q8_BLOCKS; ++b_local) { + const int b = k_blk_base + b_local; + int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3); + } + const float qd = q_d_pf[b_local]; + s0 += (float)sum0 * qd * l_k_scale[j ][b]; + s1 += (float)sum1 * qd * l_k_scale[j+1][b]; + s2 += (float)sum2 * qd * l_k_scale[j+2][b]; + s3 += (float)sum3 * qd * l_k_scale[j+3][b]; + } +#else + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < SPLIT_DK_VEC; ++k) { + const ACC_TYPE4 qk = q_priv[k]; + const int k_abs = dk_off_vec + k; + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3); + } + s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3; + s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3; + s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3; + s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3; +#endif + +#if N_SPLIT > 1 + // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96): 3-way shuffle. + #if (N_SPLIT & (N_SPLIT - 1)) == 0 + #pragma unroll + for (int step = 1; step < N_SPLIT; step <<= 1) { + s0 += sub_group_shuffle_xor(s0, step); + s1 += sub_group_shuffle_xor(s1, step); + s2 += sub_group_shuffle_xor(s2, step); + s3 += sub_group_shuffle_xor(s3, step); + } + #else + const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT; + s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2); + s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2); + s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); + s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); + #endif + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } +#endif + s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (query_valid && mask_base != NULL && blk_cur != 2) { + const global MASK_DATA_TYPE * mask_ptr = + (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; + } + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + +#if FA_V_STRATEGY == 2 + #pragma unroll + for (int b_local = 0; b_local < DV_Q8_BLOCKS_PREFILL / N_SPLIT; ++b_local) { + const int b_abs = split_idx * (DV_Q8_BLOCKS_PREFILL / N_SPLIT) + b_local; + const float d0 = l_v_scale[j ][b_abs]; + const float d1 = l_v_scale[j+1][b_abs]; + const float d2 = l_v_scale[j+2][b_abs]; + const float d3 = l_v_scale[j+3][b_abs]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + const int lane_abs = b_abs * 8 + g; + const int lane_local = b_local * 8 + g; + uint pk0 = l_v_packed[j ][lane_abs]; + uint pk1 = l_v_packed[j+1][lane_abs]; + uint pk2 = l_v_packed[j+2][lane_abs]; + uint pk3 = l_v_packed[j+3][lane_abs]; + float4 v0 = d0 * (float4)((float)(char)(pk0 & 0xff), (float)(char)((pk0>>8)&0xff), (float)(char)((pk0>>16)&0xff), (float)(char)((pk0>>24)&0xff)); + float4 v1 = d1 * (float4)((float)(char)(pk1 & 0xff), (float)(char)((pk1>>8)&0xff), (float)(char)((pk1>>16)&0xff), (float)(char)((pk1>>24)&0xff)); + float4 v2 = d2 * (float4)((float)(char)(pk2 & 0xff), (float)(char)((pk2>>8)&0xff), (float)(char)((pk2>>16)&0xff), (float)(char)((pk2>>24)&0xff)); + float4 v3 = d3 * (float4)((float)(char)(pk3 & 0xff), (float)(char)((pk3>>8)&0xff), (float)(char)((pk3>>16)&0xff), (float)(char)((pk3>>24)&0xff)); + o_acc[lane_local] = mad(p3, v3, + mad(p2, v2, + mad(p1, v1, + mad(p0, v0, + o_acc[lane_local] * scale_prev)))); + } + } +#else // FA_V_STRATEGY == 0 + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + const int i_abs = dv_off_vec + i; + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]), + mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]), + o_acc[i] * scale_prev)))); + } +#endif + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. With N_SPLIT>1 each thread writes its SPLIT_DV_VEC slice. + if (query_valid) { + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + const ACC_TYPE scale_o = exp(m_i - m_final); + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o; + l_i = l_i * scale_o + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global float4 * o_row = (global float4 *) (o_base + o_row_offset); + if (l_inv > 0.0f) { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv; + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f); + } + } +} + +// FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA +// source owns a copy since kernels compile per-source-program. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl new file mode 100644 index 00000000000..88ead4bcb51 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl @@ -0,0 +1,156 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +__kernel void flash_attn_kv_pad_f16( + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * k_pad_void, + global void * v_pad_void, + const int n_kv, + const int n_head_kv, + const int n_batch, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3 +) { + const int row_idx = get_global_id(0); + const int head_kv_idx = get_global_id(1); + const int batch_idx = get_global_id(2); + + if (row_idx >= BLOCK_N || head_kv_idx >= n_head_kv || batch_idx >= n_batch) { + return; + } + + const int tail_start = n_kv - (n_kv % BLOCK_N); + const int src_row_idx = tail_start + row_idx; + + const global char * k_src = (const global char *) k_void + k_offset; + const global char * v_src = (const global char *) v_void + v_offset; + global char * k_pad = (global char *) k_pad_void; + global char * v_pad = (global char *) v_pad_void; + + const ulong k_dst_offset = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx) * ((ulong) BLOCK_N * k_nb1) + (ulong) row_idx * k_nb1; + const ulong v_dst_offset = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx) * ((ulong) BLOCK_N * v_nb1) + (ulong) row_idx * v_nb1; + + if (src_row_idx < n_kv) { + const ulong k_src_offset = (ulong) batch_idx * k_nb3 + (ulong) head_kv_idx * k_nb2 + (ulong) src_row_idx * k_nb1; + const ulong v_src_offset = (ulong) batch_idx * v_nb3 + (ulong) head_kv_idx * v_nb2 + (ulong) src_row_idx * v_nb1; + + for (ulong i = 0; i < k_nb1; ++i) { + k_pad[k_dst_offset + i] = k_src[k_src_offset + i]; + } + for (ulong i = 0; i < v_nb1; ++i) { + v_pad[v_dst_offset + i] = v_src[v_src_offset + i]; + } + } else { + for (ulong i = 0; i < k_nb1; ++i) { + k_pad[k_dst_offset + i] = 0; + } + for (ulong i = 0; i < v_nb1; ++i) { + v_pad[v_dst_offset + i] = 0; + } + } +} + +__kernel void flash_attn_mask_pad_f16( + const global void * mask_void, ulong mask_offset, + global void * mask_pad_void, + const int n_q, + const int n_kv, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3 +) { + const int col_idx = get_global_id(0); + const int q_row = get_global_id(1); + const int mask_slice = get_global_id(2); + + if (col_idx >= BLOCK_N || q_row >= n_q || mask_slice >= mask_ne2 * mask_ne3) { + return; + } + + const int tail_start = n_kv - (n_kv % BLOCK_N); + const int src_col_idx = tail_start + col_idx; + const int mask_head_idx = mask_slice % mask_ne2; + const int mask_batch_idx = mask_slice / mask_ne2; + + const global char * mask_src_base = (const global char *) mask_void + mask_offset + + (ulong) mask_batch_idx * mask_nb3 + + (ulong) mask_head_idx * mask_nb2 + + (ulong) q_row * mask_nb1; + const global half * mask_src = (const global half *) mask_src_base; + + global half * mask_pad = (global half *) mask_pad_void; + const ulong dst_idx = + (((ulong) mask_batch_idx * (ulong) mask_ne2 + (ulong) mask_head_idx) * (ulong) n_q + (ulong) q_row) * (ulong) BLOCK_N + + (ulong) col_idx; + + mask_pad[dst_idx] = src_col_idx < n_kv ? mask_src[src_col_idx] : (half) (-INFINITY); +} + +// Per-KV-tile mask class. 0=all -inf (skip tile), 1=mixed (apply mask), +// 2=all zero, no -inf (skip mask lookup). Causal diagonal tiles are class 1. +__kernel void flash_attn_blk_f16( + const global void * mask_void, ulong mask_offset, + global char * blk, + const int n_q, + const int n_kv, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3 +) { + const int kv_block_idx = get_global_id(0); + const int q_block_idx = get_global_id(1); + const int mask_slice = get_global_id(2); + + const int n_q_blocks = (n_q + BLOCK_M - 1) / BLOCK_M; + const int n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + if (kv_block_idx >= n_kv_blocks || q_block_idx >= n_q_blocks || mask_slice >= mask_ne2 * mask_ne3) { + return; + } + + const int mask_head_idx = mask_slice % mask_ne2; + const int mask_batch_idx = mask_slice / mask_ne2; + const int q_start = q_block_idx * BLOCK_M; + const int k_start = kv_block_idx * BLOCK_N; + const int q_count = min(BLOCK_M, n_q - q_start); + const int k_count = min(BLOCK_N, n_kv - k_start); + + const half neg_max_half = (half) (-65504.0f); + char has_unmasked = 0; + char has_masked = 0; + char has_nonzero = 0; + + const global char * mask_base = (const global char *) mask_void + mask_offset + + (ulong) mask_batch_idx * mask_nb3 + + (ulong) mask_head_idx * mask_nb2; + + for (int qi = 0; qi < q_count; ++qi) { + const global half * mask_row = (const global half *) (mask_base + (ulong) (q_start + qi) * mask_nb1) + k_start; + for (int ki = 0; ki < k_count; ++ki) { + const half v = mask_row[ki]; + if (v <= neg_max_half) { + has_masked = 1; + } else { + has_unmasked = 1; + if (v != (half) 0.0f) { + has_nonzero = 1; + } + } + } + if (has_masked && has_unmasked) break; // mixed tile — short-circuit. + } + + char res; + if (has_unmasked == 0) { + res = 0; + } else if (has_masked || has_nonzero) { + res = 1; + } else { + res = 2; + } + + blk[((ulong) mask_slice * (ulong) n_q_blocks + (ulong) q_block_idx) * (ulong) n_kv_blocks + (ulong) kv_block_idx] = res; +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl index 02cdbdd9fb1..834050a4f9a 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl @@ -132,6 +132,46 @@ static inline half8 mxfp4_to_fp16_packed8(ushort2 fp4x8) { c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + static inline half e8m0_to_fp16(uchar x) { ushort bits; @@ -157,7 +197,9 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -167,6 +209,28 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -216,9 +280,11 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 8 elements reduction for better precision - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Repeat for second sub-block uint half_step = step + TILESIZE_K; @@ -244,8 +310,10 @@ kernel void kernel_gemm_moe_mxfp4_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 3-levels reduction for better precision - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl index d403ed0cab1..58f79323617 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl @@ -98,6 +98,46 @@ c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair kernel void kernel_gemm_moe_q4_0_f32_ns( @@ -109,7 +149,9 @@ kernel void kernel_gemm_moe_q4_0_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -119,6 +161,28 @@ kernel void kernel_gemm_moe_q4_0_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -167,9 +231,11 @@ kernel void kernel_gemm_moe_q4_0_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 8 elements reduction for better precision - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Repeat for second sub-block uint half_step = step + TILESIZE_K; @@ -194,8 +260,10 @@ kernel void kernel_gemm_moe_q4_0_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 3-levels reduction for better precision - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl index b2bddf3f73a..65931244634 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl @@ -98,6 +98,46 @@ c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair kernel void kernel_gemm_moe_q4_1_f32_ns( @@ -110,7 +150,9 @@ kernel void kernel_gemm_moe_q4_1_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -120,6 +162,28 @@ kernel void kernel_gemm_moe_q4_1_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -169,9 +233,11 @@ kernel void kernel_gemm_moe_q4_1_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 8 elements reduction for better precision - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Repeat for second sub-block uint half_step = step + TILESIZE_K; @@ -196,8 +262,10 @@ kernel void kernel_gemm_moe_q4_1_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 3-levels reduction for better precision - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl index ab8228d18ca..f96189692e1 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl @@ -114,6 +114,46 @@ inline void get_scale_min_k4( c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) kernel void kernel_gemm_moe_q4_k_f32_ns( @@ -127,7 +167,9 @@ kernel void kernel_gemm_moe_q4_k_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -137,6 +179,25 @@ kernel void kernel_gemm_moe_q4_k_f32_ns( return; } + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -199,9 +260,11 @@ kernel void kernel_gemm_moe_q4_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Second half (next 16 elements, same sub-block scale) uint half_step = step + TILESIZE_K; @@ -221,8 +284,10 @@ kernel void kernel_gemm_moe_q4_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl index d1a35d58bb2..4b8dc52c295 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl @@ -98,6 +98,46 @@ c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair kernel void kernel_gemm_moe_q5_0_f32_ns( @@ -110,7 +150,9 @@ kernel void kernel_gemm_moe_q5_0_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -120,6 +162,28 @@ kernel void kernel_gemm_moe_q5_0_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -171,9 +235,11 @@ kernel void kernel_gemm_moe_q5_0_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 8 elements reduction for better precision - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Repeat for second sub-block uint half_step = step + TILESIZE_K; @@ -198,8 +264,10 @@ kernel void kernel_gemm_moe_q5_0_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 3-levels reduction for better precision - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl index 90d345ecf51..ceb977096ef 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl @@ -98,6 +98,46 @@ c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair kernel void kernel_gemm_moe_q5_1_f32_ns( @@ -111,7 +151,9 @@ kernel void kernel_gemm_moe_q5_1_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -121,6 +163,28 @@ kernel void kernel_gemm_moe_q5_1_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -173,9 +237,11 @@ kernel void kernel_gemm_moe_q5_1_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 8 elements reduction for better precision - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Repeat for second sub-block uint half_step = step + TILESIZE_K; @@ -200,8 +266,10 @@ kernel void kernel_gemm_moe_q5_1_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); // 32 16x16 fp16 dot product with 3-levels reduction for better precision - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl index 13c26f6f3b6..0cbea13f085 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl @@ -114,6 +114,46 @@ inline void get_scale_min_k4( c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) kernel void kernel_gemm_moe_q5_k_f32_ns( @@ -128,7 +168,9 @@ kernel void kernel_gemm_moe_q5_k_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -138,6 +180,28 @@ kernel void kernel_gemm_moe_q5_k_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -204,9 +268,11 @@ kernel void kernel_gemm_moe_q5_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Second half uint half_step = step + TILESIZE_K; @@ -226,8 +292,10 @@ kernel void kernel_gemm_moe_q5_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl index 85ccebec78c..f32114fcfb8 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl @@ -98,6 +98,46 @@ c_reg.lo += convert_float8(acc.lo); \ c_reg.hi += convert_float8(acc.hi); \ +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + __attribute__((qcom_wave_pair_mode(1))) kernel void kernel_gemm_moe_q6_k_f32_ns( @@ -111,7 +151,9 @@ kernel void kernel_gemm_moe_q6_k_f32_ns( __write_only image1d_buffer_t dst, __global int * total_tiles, uint ne00, - uint ne01 + uint ne01, + uint is_ragged, + uint skip_gran ) { uint block_id_m = get_global_id(1); // m_tile uint block_id_n = get_global_id(2); // n_tile @@ -121,6 +163,28 @@ kernel void kernel_gemm_moe_q6_k_f32_ns( return; } + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + __private half16 reg_a; __private float32 reg_c = (float32)(0); __local half4 shared_b[128]; @@ -183,9 +247,11 @@ kernel void kernel_gemm_moe_q6_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - half16 acc; - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } // Second half uint half_step = step + TILESIZE_K; @@ -205,8 +271,10 @@ kernel void kernel_gemm_moe_q6_k_f32_ns( sub_group_barrier(CLK_LOCAL_MEM_FENCE); - dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); - dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } } if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl new file mode 100644 index 00000000000..ba46cc75c92 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl @@ -0,0 +1,94 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +// each work-item computes a 4 (rows of A / m) x 8 (cols of B / n) output tile. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q1_0_f32( + global const uint * src0_q, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + int k, + int m, + int n, + int n_no_padding, + ulong offsetd +) { + int n_4 = n >> 2; + + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 2; + dst = (global float *)((global char*)dst + offsetd); + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + half8 B; + + global const uint* wptr = src0_q + gx_2; + global const half* sptr = src0_d + gx_2; + + // 32 weights per uint32, 128 weights (one block / one scale) per 4 uint32. + for (int i = 0; i < k; i += 32) { + uint4 pack4 = vload4(0, wptr + (i / 32) * m); // 4 rows, 32 K-values each + half4 scale = vload4(0, sptr + (i / 128) * m); // 4 rows, one scale per 128 + + for (int j = 0; j < 32; ++j) { + B.s0123 = read_imageh(src1, gy * 2 + (i + j) * n_4); + B.s4567 = read_imageh(src1, gy * 2 + (i + j) * n_4 + 1); + + // sign bit -> +-1 (half arithmetic avoids unsigned underflow) + half4 wj = (half4)( + 2.0h * (half)((pack4.s0 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s1 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s2 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s3 >> j) & 1u) - 1.0h) * scale; + + c0 += B * wj.s0; + c1 += B * wj.s1; + c2 += B * wj.s2; + c3 += B * wj.s3; + } + } + + int idx = (gy << 3) * m + (gx << 2); + + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl new file mode 100644 index 00000000000..e83c5d06893 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl @@ -0,0 +1,121 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK1_0 128 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_q1(total, bits, scale, regB, lb) \ + total += (2.0f*(float)((bits >> 0) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+0); \ + total += (2.0f*(float)((bits >> 1) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+0); \ + total += (2.0f*(float)((bits >> 2) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+0); \ + total += (2.0f*(float)((bits >> 3) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+0); \ + total += (2.0f*(float)((bits >> 4) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+0); \ + total += (2.0f*(float)((bits >> 5) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+0); \ + total += (2.0f*(float)((bits >> 6) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+0); \ + total += (2.0f*(float)((bits >> 7) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+0); \ + total += (2.0f*(float)((bits >> 8) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+1); \ + total += (2.0f*(float)((bits >> 9) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+1); \ + total += (2.0f*(float)((bits >> 10) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+1); \ + total += (2.0f*(float)((bits >> 11) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+1); \ + total += (2.0f*(float)((bits >> 12) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+1); \ + total += (2.0f*(float)((bits >> 13) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+1); \ + total += (2.0f*(float)((bits >> 14) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+1); \ + total += (2.0f*(float)((bits >> 15) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+1); \ + total += (2.0f*(float)((bits >> 16) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+2); \ + total += (2.0f*(float)((bits >> 17) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+2); \ + total += (2.0f*(float)((bits >> 18) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+2); \ + total += (2.0f*(float)((bits >> 19) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+2); \ + total += (2.0f*(float)((bits >> 20) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+2); \ + total += (2.0f*(float)((bits >> 21) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+2); \ + total += (2.0f*(float)((bits >> 22) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+2); \ + total += (2.0f*(float)((bits >> 23) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+2); \ + total += (2.0f*(float)((bits >> 24) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+3); \ + total += (2.0f*(float)((bits >> 25) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+3); \ + total += (2.0f*(float)((bits >> 26) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+3); \ + total += (2.0f*(float)((bits >> 27) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+3); \ + total += (2.0f*(float)((bits >> 28) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+3); \ + total += (2.0f*(float)((bits >> 29) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+3); \ + total += (2.0f*(float)((bits >> 30) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+3); \ + total += (2.0f*(float)((bits >> 31) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+3); + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q1_0_f32( + read_only image1d_buffer_t src0_q, + global half * src0_d, + read_only image1d_buffer_t src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + uint BLOCK_STRIDE_A = 4 * M; + + uint4 regA; + half regS; + float8 regB; + + float totalSum = 0.0f; + + #pragma unroll 1 + for (uint kb = groupId; kb < (K / QK1_0); kb += N_SIMDGROUP) { + regS = src0_d[gid + kb * LINE_STRIDE_A]; // each fiber loads its row's scale + + // first 16 fibers load 8 B values each -> 128 activations for this block + if (slid < 16) { + regB.s0123 = read_imagef(src1, (slid * 2 + kb * 32)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + kb * 32)); + } + + // load this row's 4 uint32 (128 sign bits) + regA.s0 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + + float scale = (float)regS; + dequantizeBlockAccum_q1(totalSum, regA.s0, scale, regB, 0); + dequantizeBlockAccum_q1(totalSum, regA.s1, scale, regB, 4); + dequantizeBlockAccum_q1(totalSum, regA.s2, scale, regB, 8); + dequantizeBlockAccum_q1(totalSum, regA.s3, scale, regB, 12); + } + + // reduction in local memory, assumes #wave = N_SIMDGROUP = 4 + local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + dst[gid] = totalSum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl new file mode 100644 index 00000000000..67710fac4b4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl @@ -0,0 +1,156 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// LOAD_VEC_A is 8 because one q1_0 quant byte expands to 8 weights along K. +#define LOAD_VEC_A 8 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#define TM 4 +#define TN 8 + +kernel void kernel_mul_mm_q1_0_f32_l4_lm( + global uchar * src0_q, + global half * src0_d, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a; + int ib = idx / 16; // 16 quant bytes per q1_0 block + + float d = (float)src0_d[ib]; + uint bits = src0_q[idx]; + + // use float to avoid unsigned underflow of (2*0 - 1). + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 0) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 1) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 2) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 3) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 4) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 4) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 5) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 5) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 6) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 6) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 7) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 7) & 1) - 1.0f); + } else { + for (int b = 0; b < LOAD_VEC_A; ++b) { + buf_a[(loadr_a * LOAD_VEC_A + b) * BM + loadc_a + l] = 0.0f; + } + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const int offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl index a639ec664b1..da2e14ae993 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl @@ -18,6 +18,14 @@ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) #endif +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#endif + // Assumes row size (ne00) is a multiple of 4 #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 @@ -378,3 +386,848 @@ kernel void kernel_mul_mat_f16_f32_l4_dr_lq( } } #endif // ADRENO_GPU + +#define N_ROWS_PER_WG 8 +#define N_OUTS_PER_WG 8 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_ROWS_PER_WG; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + __local float4 q_loc[64]; // ne00/4 max for sub_group_size 64 + if (sgs_lid < ne00 / 4) { + q_loc[sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int dr = 0; dr < N_ROWS_PER_WG; ++dr) { + const int r0 = r0_base + dr; + if (r0 >= ne01) return; + + const ulong offset_src0 = r0 * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + const half4 k4 = x4[i]; + const float4 q = q_loc[i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + const float all_sum = sub_group_reduce_add(sumf); + if (sgs_lid == 0) { + dst[im * ne1 * ne0 + r0] = all_sum; // ne11 == 1, so r1==0 + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_OUTS_PER_WG; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + global half4 * x4_o[N_OUTS_PER_WG]; + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + const ulong off = r0c * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + x4_o[o] = (global half4 *)(src0 + off); + } + + float sum[N_OUTS_PER_WG] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + const float4 q4 = y4[i]; + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const half4 v4 = x4_o[o][i]; + sum[o] += convert_float(v4.s0) * q4.s0 + + convert_float(v4.s1) * q4.s1 + + convert_float(v4.s2) * q4.s2 + + convert_float(v4.s3) * q4.s3; + } + } + + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const int r0 = r0_base + o; + const float s = sub_group_reduce_add(sum[o]); + if (sgs_lid == 0 && r0 < ne01) { + dst[im * ne1 * ne0 + r0] = s; + } + } +} + +#define N_OUTS_PAIR 8 +#define N_PAIRS_PAIR (N_OUTS_PAIR / 2) + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_pair( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int half_id = sgs_lid >> 5; // 0 = lower half, 1 = upper half + const int lane_h = sgs_lid & 31; // lane 0..31 within half + + const int r0_base = get_group_id(0) * N_OUTS_PAIR; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + __local float4 q_loc[64]; // ne00/4 max for sub_group_size 64 + if (sgs_lid < ne00 / 4) { + q_loc[sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int dk_vec = ne00 / 4; + + #pragma unroll + for (int p = 0; p < N_PAIRS_PAIR; ++p) { + const int r0 = r0_base + 2 * p + half_id; + + const ulong offset_src0 = r0 * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + for (int i = lane_h; i < dk_vec; i += 32) { + const half4 k4 = x4[i]; + const float4 q = q_loc[i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 16); + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_h == 0) { + dst[im * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_K_ROWS_GQA 16 +#define GQA_RATIO_GQA 8 +#define LANES_PER_QH 8 // 64 / GQA_RATIO_GQA +#define DK_VEC_GQA 32 // DK / 4 for DK=128 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa4( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 3; // 0..7: which Q-head (8 per WG) + const int lane_q = sgs_lid & 7; // 0..7: lane within Q-head partition + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; // K-head index (also K2 batch) + const int i03 = im_kv / ne02; // n13 batch index + + const int q_head_lo = i02 * GQA_RATIO_GQA; + + __local float4 q_loc[GQA_RATIO_GQA * DK_VEC_GQA]; // 4 × 32 = 128 float4 + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + + if (sgs_lid < DK_VEC_GQA) { + q_loc[qh * DK_VEC_GQA + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + // K base offset for this WG. All 8 K-rows × 4 Q-heads share this K-head. + const ulong offset_src0_base = (i02) * nb02 + (i03 / r3) * nb03; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA; ++dr) { + const int r0 = r0_base + dr; + + const ulong offset_src0 = r0 * nb01 + offset_src0_base; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + #pragma unroll + for (int t = 0; t < 4; ++t) { + const int i = lane_q + t * LANES_PER_QH; // 8, 16, 24-step + const half4 k4 = x4[i]; + const float4 q = q_loc[q_id * DK_VEC_GQA + i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_DV_ROWS_Y8GQA 8 +#define GQA_RATIO_Y8GQA 8 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8_gqa( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_DV_ROWS_Y8GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; // K-head index + const int i03 = im_kv / ne02; // n13 batch index + + // GQA Q-heads sharing this K-head. + const int q_head_lo = i02 * GQA_RATIO_Y8GQA; + + global float4 * y4_q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + y4_q[qh] = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + } + + global half4 * x4_o[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + const ulong off = r0c * nb01 + (i02) * nb02 + (i03 / r3) * nb03; + x4_o[o] = (global half4 *)(src0 + off); + } + + float sum[N_DV_ROWS_Y8GQA][GQA_RATIO_Y8GQA] = { {0.0f} }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + // load 8 V values (one per DV row), same K-head, K-pos = i. + half4 v[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + v[o] = x4_o[o][i]; + } + + // load 8 softmax values (one per Q-head). + float4 q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + q[qh] = y4_q[qh][i]; + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const float4 vf = (float4)(convert_float(v[o].s0), + convert_float(v[o].s1), + convert_float(v[o].s2), + convert_float(v[o].s3)); + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + sum[o][qh] += vf.s0 * q[qh].s0 + + vf.s1 * q[qh].s1 + + vf.s2 * q[qh].s2 + + vf.s3 * q[qh].s3; + } + } + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const float s = sub_group_reduce_add(sum[o][qh]); + if (sgs_lid == 0 && r0 < ne01) { + const int im_out = i03 * ne12 + (q_head_lo + qh); + dst[im_out * ne1 * ne0 + r0] = s; + } + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa4_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 3; // 0..7: which Q-head (8 per WG) + const int lane_q = sgs_lid & 7; // 0..7: lane within Q-head partition + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_GQA; + + __local float4 q_loc[GQA_RATIO_GQA * DK_VEC_GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + if (sgs_lid < DK_VEC_GQA) { + q_loc[qh * DK_VEC_GQA + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + float sumf = 0.0f; + #pragma unroll + for (int t = 0; t < 2; ++t) { + const int p = lane_q + t * LANES_PER_QH; // pixel idx in row, 0..15 + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; // first half4 idx + const float4 qa = q_loc[q_id * DK_VEC_GQA + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_GQA + i0 + 1]; + sumf += convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8_gqa_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_DV_ROWS_Y8GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_Y8GQA; + + // Q (= softmax(KQ)) base pointers per Q-head + global float4 * y4_q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + y4_q[qh] = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + } + + const int pitch_px_row = (int)(nb01 >> 3); + const int pitch_px_head = (int)(nb02 >> 3); + const int pitch_px_n13 = (int)(nb03 >> 3); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + // per-DV-row pixel base + int row_px_base[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + row_px_base[o] = r0c * pitch_px_row + head_px_base; + } + + float sum[N_DV_ROWS_Y8GQA][GQA_RATIO_Y8GQA] = { {0.0f} }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + half4 v[N_DV_ROWS_Y8GQA]; + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + v[o] = read_imageh(src0_img, row_px_base[o] + i); + } + + float4 q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + q[qh] = y4_q[qh][i]; + } + // 64 mads. + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const float4 vf = (float4)(convert_float(v[o].s0), + convert_float(v[o].s1), + convert_float(v[o].s2), + convert_float(v[o].s3)); + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + sum[o][qh] += vf.s0 * q[qh].s0 + + vf.s1 * q[qh].s1 + + vf.s2 * q[qh].s2 + + vf.s3 * q[qh].s3; + } + } + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const float s = sub_group_reduce_add(sum[o][qh]); + if (sgs_lid == 0 && r0 < ne01) { + const int im_out = i03 * ne12 + (q_head_lo + qh); + dst[im_out * ne1 * ne0 + r0] = s; + } + } + } +} + +#define N_K_ROWS_GQA_R4 16 +#define GQA_RATIO_R4 4 +#define LANES_PER_QH_R4 16 // = 64 / GQA_RATIO_R4 +#define DK_VEC_R4 32 // DK / 4 for DK=128 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 4; // 0..3 + const int lane_q = sgs_lid & 15; // 0..15 + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA_R4; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_R4; + + __local float4 q_loc[GQA_RATIO_R4 * DK_VEC_R4]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_R4; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + if (sgs_lid < DK_VEC_R4) { + q_loc[qh * DK_VEC_R4 + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA_R4; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + const int p = lane_q; + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; + const float4 qa = q_loc[q_id * DK_VEC_R4 + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_R4 + i0 + 1]; + + float sumf = + convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_K_ROWS_GQA_R2_DK256 16 +#define GQA_RATIO_R2 2 +#define LANES_PER_QH_R2 32 // = 64 / GQA_RATIO_R2 +#define DK_VEC_DK256 64 // DK / 4 for DK=256 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 5; // 0..1 + const int lane_q = sgs_lid & 31; // 0..31 + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA_R2_DK256; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_R2; + + __local float4 q_loc[GQA_RATIO_R2 * DK_VEC_DK256]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_R2; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + q_loc[qh * DK_VEC_DK256 + sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA_R2_DK256; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + const int p = lane_q; + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; + const float4 qa = q_loc[q_id * DK_VEC_DK256 + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_DK256 + i0 + 1]; + + float sumf = + convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + + sumf += sub_group_shuffle_xor(sumf, 16); + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl new file mode 100644 index 00000000000..8746e2a3aaa --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl @@ -0,0 +1,141 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK1_0 128 +typedef struct { + half d; + uchar qs[QK1_0/8]; +} block_q1_0; + +#define NB_Q1_0 16 + +#ifdef INTEL_GPU +#define N_R0_Q1_0 4 // number of rows each subgroup works on +#define N_SG_Q1_0 2 // number of subgroups in a work group +#define N_SIMDWIDTH 16 // subgroup size +#elif defined (ADRENO_GPU) +#define N_R0_Q1_0 4 +#define N_SG_Q1_0 2 +#define N_SIMDWIDTH 64 +#endif + +inline float block_q_1_0_dot_y(global block_q1_0 * qb, float sumy, float yl[NB_Q1_0], short il) { + global uchar * qs = qb->qs + il*2; + uint b0 = qs[0]; + uint b1 = qs[1]; + + float acc = 0.f; + acc += yl[ 0]*(float)((b0 >> 0) & 1) + yl[ 1]*(float)((b0 >> 1) & 1); + acc += yl[ 2]*(float)((b0 >> 2) & 1) + yl[ 3]*(float)((b0 >> 3) & 1); + acc += yl[ 4]*(float)((b0 >> 4) & 1) + yl[ 5]*(float)((b0 >> 5) & 1); + acc += yl[ 6]*(float)((b0 >> 6) & 1) + yl[ 7]*(float)((b0 >> 7) & 1); + + acc += yl[ 8]*(float)((b1 >> 0) & 1) + yl[ 9]*(float)((b1 >> 1) & 1); + acc += yl[10]*(float)((b1 >> 2) & 1) + yl[11]*(float)((b1 >> 3) & 1); + acc += yl[12]*(float)((b1 >> 4) & 1) + yl[13]*(float)((b1 >> 5) & 1); + acc += yl[14]*(float)((b1 >> 6) & 1) + yl[15]*(float)((b1 >> 7) & 1); + + return qb->d * (2.0f*acc - sumy); +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q1_0_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + int nb = ne00/QK1_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0; + + uint i12 = im%ne12; + uint i13 = im/ne12; + + ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // pointers to src0 rows + global block_q1_0 * ax[N_R0_Q1_0]; + for (int row = 0; row < N_R0_Q1_0; ++row) { + ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ax[row] = (global block_q1_0 *) ((global char *) src0 + offset_src0); + } + + float yl[NB_Q1_0]; + float sumf[N_R0_Q1_0] = { 0.f }; + + const short ix = get_sub_group_local_id()/8; + const short il = get_sub_group_local_id()%8; + + global float * yb = y + ix*QK1_0 + il*NB_Q1_0; + + // each thread handles NB_Q1_0 quants at a time + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + float sumy = 0.f; + for (short i = 0; i < NB_Q1_0; ++i) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + for (short row = 0; row < N_R0_Q1_0; row++) { + sumf[row] += block_q_1_0_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += N_SIMDWIDTH*NB_Q1_0; + } + + global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0; + + for (int row = 0; row < N_R0_Q1_0; ++row) { + float tot = sub_group_reduce_add(sumf[row]); + + if (get_sub_group_local_id() == 0 && first_row + row < ne01) { + dst_f32[first_row + row] = tot; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl new file mode 100644 index 00000000000..870e36d8661 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl @@ -0,0 +1,190 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK1_0 128 +#define QK1_0_BYTES (QK1_0/8) // 16 quant bytes per block +#define QK1_0_BLK_BYTES (QK1_0_BYTES + 2) // d + qs in original tensor = 18 + +#define NB_Q1_0 16 // quants handled per thread (two qs bytes) + +#ifdef INTEL_GPU +#define N_R0_Q1_0 4 // number of rows each subgroup works on +#define N_SG_Q1_0 2 // number of subgroups in a work group +#define N_SIMDWIDTH 16 // subgroup size +#elif defined (ADRENO_GPU) +#define N_R0_Q1_0 4 +#define N_SG_Q1_0 2 +#define N_SIMDWIDTH 64 +#endif + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q1_0_f32_flat( + global char * src0_q, + global half * src0_d, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + int nb = ne00/QK1_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0; + + uint i12 = im%ne12; + uint i13 = im/ne12; + + ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // pointers to src0 rows (flat: q bytes + scales) + uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + + global uchar * ax0, * ax1, * ax2, * ax3; + global half * ad0, * ad1, * ad2, * ad3; + uint offset_src0; + + offset_src0 = (offset_src0_base + 0*nb01) / QK1_0_BLK_BYTES; + ax0 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 1*nb01) / QK1_0_BLK_BYTES; + ax1 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 2*nb01) / QK1_0_BLK_BYTES; + ax2 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 3*nb01) / QK1_0_BLK_BYTES; + ax3 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + const short ix = get_sub_group_local_id()/8; + const short il = get_sub_group_local_id()%8; + + global float * yb = y + ix*QK1_0 + il*NB_Q1_0; + + float8 yl_lo; + float8 yl_hi; + float4 sumf = 0.f; + + // each thread handles NB_Q1_0 = 16 quants (two qs bytes) at a time + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + yl_lo = vload8(0, yb); + yl_hi = vload8(0, yb + 8); + float sumy = yl_lo.s0 + yl_lo.s1 + yl_lo.s2 + yl_lo.s3 + + yl_lo.s4 + yl_lo.s5 + yl_lo.s6 + yl_lo.s7 + + yl_hi.s0 + yl_hi.s1 + yl_hi.s2 + yl_hi.s3 + + yl_hi.s4 + yl_hi.s5 + yl_hi.s6 + yl_hi.s7; + + uint b0, b1; + float acc; + + b0 = ax0[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax0[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s0 += (float)ad0[ib] * (2.0f*acc - sumy); + + b0 = ax1[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax1[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s1 += (float)ad1[ib] * (2.0f*acc - sumy); + + b0 = ax2[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax2[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s2 += (float)ad2[ib] * (2.0f*acc - sumy); + + b0 = ax3[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax3[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s3 += (float)ad3[ib] * (2.0f*acc - sumy); + + yb += N_SIMDWIDTH*NB_Q1_0; + } + + global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0; + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), + sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), + sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) dst_f32[first_row + 0] = tot.s0; + if (first_row + 1 < ne01) dst_f32[first_row + 1] = tot.s1; + if (first_row + 2 < ne01) dst_f32[first_row + 2] = tot.s2; + if (first_row + 3 < ne01) dst_f32[first_row + 3] = tot.s3; + } +} diff --git a/ggml/src/ggml-opencl/kernels/set_rows.cl b/ggml/src/ggml-opencl/kernels/set_rows.cl index fc3ff7aa1e7..4ad5af13f13 100644 --- a/ggml/src/ggml-opencl/kernels/set_rows.cl +++ b/ggml/src/ggml-opencl/kernels/set_rows.cl @@ -158,6 +158,239 @@ kernel void kernel_set_rows_f32_i32( } } +// f32 -> q8_0 quantize set_rows. Block = half d + char qs[32]. +#define QK8_0 32 + +inline void quantize_q8_0_block(global float * x, global char * qs, global half * d_out) { + float amax = 0.0f; + for (int j = 0; j < QK8_0; j++) { + amax = fmax(amax, fabs(x[j])); + } + + float d = amax / 127.0f; + float id = (d != 0.0f) ? 127.0f / amax : 0.0f; + + vstore_half(d, 0, d_out); + + for (int j = 0; j < QK8_0; j++) { + qs[j] = (char)((int)round(x[j] * id)); + } +} + +kernel void kernel_set_rows_q8_0_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * y = dst_row + blk * (2 + QK8_0); + + quantize_q8_0_block(x, y + 2, (global half *)y); + } +} + +kernel void kernel_set_rows_q8_0_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * y = dst_row + blk * (2 + QK8_0); + + quantize_q8_0_block(x, y + 2, (global half *)y); + } +} + +// SoA q8_0 variants. dst_q: int8[QK8_0] per block; dst_d: fp16 scale per block. +// Layout matches kernel_convert_block_q8_0; block index follows dst element order. +kernel void kernel_set_rows_q8_0_soa_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0; + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * q = q_row + blk * QK8_0; + + quantize_q8_0_block(x, q, d_row + blk); + } +} + +kernel void kernel_set_rows_q8_0_soa_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0; + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * q = q_row + blk * QK8_0; + + quantize_q8_0_block(x, q, d_row + blk); + } +} + kernel void kernel_set_rows_f16_i32( global char * src0, ulong offset0, @@ -206,3 +439,270 @@ kernel void kernel_set_rows_f16_i32( dst_row[ind] = src_row[ind]; } } + +// f32 -> q4_0 quantize set_rows. Block = half d + uchar qs[16] (shuffled +// nibbles: qs[j] low/high = elem j / j+16). +// Dequant: val[i] = d * (nibble_i - 8) +// nblk0 = number of q4_0 blocks per row = ne00 / 32. +#define QK4_0 32 +#define Q4_0_BLOCK_SIZE 18 + +inline void quantize_q4_0_block(global float * x, global uchar * qs, global half * d_out) { + // Find the signed value with the largest absolute magnitude (matches ggml ref). + float max = 0.0f; + float amax = 0.0f; + for (int j = 0; j < QK4_0; j++) { + float v = x[j]; + float a = fabs(v); + if (a > amax) { + amax = a; + max = v; + } + } + + float d = max / -8.0f; + float id = (d != 0.0f) ? 1.0f / d : 0.0f; + + vstore_half(d, 0, d_out); + + for (int j = 0; j < QK4_0/2; j++) { + float x0 = x[j] * id; + float x1 = x[j + QK4_0/2] * id; + + int i0 = (int)(x0 + 8.5f); + int i1 = (int)(x1 + 8.5f); + if (i0 < 0) i0 = 0; + if (i0 > 15) i0 = 15; + if (i1 < 0) i1 = 0; + if (i1 > 15) i1 = 15; + + qs[j] = (uchar)i0 | ((uchar)i1 << 4); + } +} + +kernel void kernel_set_rows_q4_0_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global char * y = dst_row + blk * Q4_0_BLOCK_SIZE; + global half * yd = (global half *)(y); + global uchar * yqs = (global uchar *)(y + 2); + + quantize_q4_0_block(x, yqs, yd); + } +} + +kernel void kernel_set_rows_q4_0_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global char * y = dst_row + blk * Q4_0_BLOCK_SIZE; + global half * yd = (global half *)(y); + global uchar * yqs = (global uchar *)(y + 2); + + quantize_q4_0_block(x, yqs, yd); + } +} + +// SoA variants for q4_0 dst. Used when the backend has split block_q4_0 records +// into separate quant (dst_q) and scale (dst_d) sub-buffers — same pattern as +// the q8_0 SoA variants above. +// +// Layout (matches kernel_convert_block_q4_0, the "shuffled" variant): +// dst_q: contiguous 16 packed nibbles per block, block i at offset i * 16 bytes. +// dst_d: contiguous fp16 scales, block i at offset i * 2 bytes. +// Nibble layout inside each byte is unchanged from AoS: qs[j] low nibble = element j, +// qs[j] high nibble = element j+16. kernel_restore_block_q4_0 copies bytes as-is. +kernel void kernel_set_rows_q4_0_soa_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2); + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global uchar * qs = q_row + blk * (QK4_0/2); + global half * d_bk = d_row + blk; + + quantize_q4_0_block(x, qs, d_bk); + } +} + +kernel void kernel_set_rows_q4_0_soa_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2); + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global uchar * qs = q_row + blk * (QK4_0/2); + global half * d_bk = d_row + blk; + + quantize_q4_0_block(x, qs, d_bk); + } +} diff --git a/ggml/src/ggml-opencl/libdl.h b/ggml/src/ggml-opencl/libdl.h new file mode 100644 index 00000000000..8ca5016f039 --- /dev/null +++ b/ggml/src/ggml-opencl/libdl.h @@ -0,0 +1,79 @@ +#pragma once + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +#else +# include +# include +#endif +#include + +namespace fs = std::filesystem; + +#ifdef _WIN32 + +using dl_handle = std::remove_pointer_t; + +struct dl_handle_deleter { + void operator()(HMODULE handle) { + FreeLibrary(handle); + } +}; + +static inline dl_handle * dl_load_library(const fs::path & path) { + // suppress error dialogs for missing DLLs + DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); + SetErrorMode(old_mode | SEM_FAILCRITICALERRORS); + + HMODULE handle = LoadLibraryW(path.wstring().c_str()); + + SetErrorMode(old_mode); + + return handle; +} + +static inline void * dl_get_sym(dl_handle * handle, const char * name) { + DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); + SetErrorMode(old_mode | SEM_FAILCRITICALERRORS); + + void * p = (void *) GetProcAddress(handle, name); + + SetErrorMode(old_mode); + + return p; +} + +static inline const char * dl_error() { + return ""; +} + +#else + +using dl_handle = void; + +struct dl_handle_deleter { + void operator()(void * handle) { + dlclose(handle); + } +}; + +static inline dl_handle * dl_load_library(const fs::path & path) { + dl_handle * handle = dlopen(path.string().c_str(), RTLD_NOW | RTLD_LOCAL); + return handle; +} + +static inline void * dl_get_sym(dl_handle * handle, const char * name) { + return dlsym(handle, name); +} + +static inline const char * dl_error() { + const char *rslt = dlerror(); + return rslt != nullptr ? rslt : ""; +} + +#endif diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index b6df4f0fbb7..48c63e4d70f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1270,77 +1270,14 @@ void GgmlOvDecoder::visit_subgraph(std::function ops = { - {GGML_OP_NONE, "GGML_OP_NONE" }, - {GGML_OP_ACC, "GGML_OP_ACC" }, - {GGML_OP_ADD, "GGML_OP_ADD" }, - {GGML_OP_ADD1, "GGML_OP_ADD1" }, - {GGML_OP_ADD_ID, "GGML_OP_ADD_ID" }, - {GGML_OP_CONCAT, "GGML_OP_CONCAT" }, - {GGML_OP_CONT, "GGML_OP_CONT" }, - {GGML_OP_DIV, "GGML_OP_DIV" }, - {GGML_OP_DUP, "GGML_OP_DUP" }, - {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, - {GGML_OP_MUL, "GGML_OP_MUL" }, - {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, - {GGML_OP_MUL_MAT_ID, "GGML_OP_MUL_MAT_ID" }, - {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, - {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, - {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, - {GGML_OP_NORM, "GGML_OP_NORM" }, - {GGML_OP_ROPE, "GGML_OP_ROPE" }, - {GGML_OP_SCALE, "GGML_OP_SCALE" }, - {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, - {GGML_OP_SUM_ROWS, "GGML_OP_SUM_ROWS" }, - {GGML_OP_SUB, "GGML_OP_SUB" }, - {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE" }, - {GGML_OP_VIEW, "GGML_OP_VIEW" }, - {GGML_OP_SET_ROWS, "GGML_OP_SET_ROWS" }, - {GGML_OP_CPY, "GGML_OP_CPY" }, - {GGML_OP_FLASH_ATTN_EXT, "GGML_OP_FLASH_ATTN_EXT" }, - {GGML_OP_L2_NORM, "GGML_OP_L2_NORM" }, - {GGML_OP_CLAMP, "GGML_OP_CLAMP" }, - {GGML_OP_PAD, "GGML_OP_PAD" }, - {GGML_OP_SSM_CONV, "GGML_OP_SSM_CONV" }, - {GGML_OP_GATED_DELTA_NET, "GGML_OP_GATED_DELTA_NET"}, - {GGML_OP_ARGSORT, "GGML_OP_ARGSORT" }, - {GGML_OP_REPEAT, "GGML_OP_REPEAT" }, - {GGML_OP_IM2COL, "GGML_OP_IM2COL" } - }; - static const std::map unary_ops = { - {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS" }, - {GGML_UNARY_OP_SGN, "GGML_UNARY_OP_SGN" }, - {GGML_UNARY_OP_NEG, "GGML_UNARY_OP_NEG" }, - {GGML_UNARY_OP_STEP, "GGML_UNARY_OP_STEP" }, - {GGML_UNARY_OP_TANH, "GGML_UNARY_OP_TANH" }, - {GGML_UNARY_OP_ELU, "GGML_UNARY_OP_ELU" }, - {GGML_UNARY_OP_RELU, "GGML_UNARY_OP_RELU" }, - {GGML_UNARY_OP_SIGMOID, "GGML_UNARY_OP_SIGMOID" }, - {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU" }, - {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK" }, - {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU" }, - {GGML_UNARY_OP_SOFTPLUS, "GGML_UNARY_OP_SOFTPLUS" }, - {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH" }, - {GGML_UNARY_OP_HARDSIGMOID, "GGML_UNARY_OP_HARDSIGMOID"}, - {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP" }, - {GGML_UNARY_OP_COUNT, "GGML_UNARY_OP_COUNT" } - }; - static const std::map glu_ops = { - {GGML_GLU_OP_SWIGLU, "GGML_GLU_OP_SWIGLU"}, - {GGML_GLU_OP_GEGLU, "GGML_GLU_OP_GEGLU" }, - {GGML_GLU_OP_REGLU, "GGML_GLU_OP_REGLU" } - }; - switch (node->op) { case GGML_OP_UNARY: - return unary_ops.at(ggml_get_unary_op(node)); + return std::string("GGML_UNARY_OP_") + ggml_unary_op_name(ggml_get_unary_op(node)); case GGML_OP_GLU: - return glu_ops.at(ggml_get_glu_op(node)); + return std::string("GGML_GLU_OP_") + ggml_glu_op_name(ggml_get_glu_op(node)); default: - return ops.at(node->op); + return std::string("GGML_OP_") + ggml_op_name(node->op); } - static const std::string unknown_op = "UNKNOWN_GGML_OP"; - return unknown_op; } const std::string & GgmlOvDecoder::get_op_type(int node_idx) const { diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 943aef86453..659dbd4b5ac 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1053,6 +1053,10 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { (op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) { return true; } + // CPY into a strided view of a larger buffer (recurrent-state snapshots) not supported + if (op->view_src && ggml_nbytes(op) != ggml_nbytes(op->view_src)) { + return true; + } break; } case GGML_OP_MUL_MAT: { diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp index c8bf0815224..e54d700d421 100644 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -17,6 +17,22 @@ namespace frontend { namespace ggml { namespace op { +static ov::Output reshape_add_id_input_to_2d(const ov::Output & input, + const ov::PartialShape & input_shape, + const std::vector & dims) { + const auto actual_shape = input.get_partial_shape(); + if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { + return input; + } + + if (input_shape.rank().is_static() && input_shape.rank().get_length() == 2) { + return input; + } + + auto shape = std::make_shared(input, ov::element::i64); + return std::make_shared(input, get_dimensions(shape, dims), false); +} + OutputVector translate_add_id(const NodeContext & context) { num_inputs_check(context, 3, 3); @@ -28,11 +44,9 @@ OutputVector translate_add_id(const NodeContext & context) { // input: [1, n_token, n_used, n_embd] // bias: [1, 1, n_expert, n_embd] // ids: [1, 1, n_token, n_used] - auto bias_shape_4d = std::make_shared(bias, ov::element::i64); - auto ids_shape_4d = std::make_shared(ids, ov::element::i64); - - bias = std::make_shared(bias, get_dimensions(bias_shape_4d, {2, 3}), false); - ids = std::make_shared(ids, get_dimensions(ids_shape_4d, {2, 3}), false); + // Model bias constants may already be stored as [n_expert, n_embd]. + bias = reshape_add_id_input_to_2d(bias, context.get_input_shape(1), {2, 3}); + ids = reshape_add_id_input_to_2d(ids, context.get_input_shape(2), {2, 3}); if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { ids = std::make_shared(ids, ov::element::i32); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 5c46e071375..d220f2f584a 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -3,8 +3,11 @@ #include "../utils.h" #include +#include #include #include +#include +#include #include #include #include @@ -15,7 +18,7 @@ namespace frontend { namespace ggml { namespace op { -OutputVector translate_glu_swiglu(const NodeContext & context) { +static std::pair, ov::Output> get_glu_inputs(const NodeContext & context) { num_inputs_check(context, 1, 2); ov::Output src0; @@ -52,6 +55,12 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { std::swap(src0, src1); } + return {src0, src1}; +} + +OutputVector translate_glu_swiglu(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + auto sigmoid = std::make_shared(src0); auto silu = std::make_shared(src0, sigmoid); auto res = std::make_shared(silu, src1); @@ -59,6 +68,27 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { return rename_outputs_with_suffix({res}, context.get_name()); } +OutputVector translate_glu_swiglu_oai(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + + const int32_t * params = context.get_output_op_params(); + const float alpha = reinterpret_cast(params)[2]; + const float limit = reinterpret_cast(params)[3]; + + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); + auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); + auto scaled_gate = std::make_shared(gate, alpha_const); + auto sigmoid = std::make_shared(scaled_gate); + auto out_glu = std::make_shared(gate, sigmoid); + + auto up = std::make_shared(src1, -limit, limit); + auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); + auto up_plus_one = std::make_shared(up, one); + auto res = std::make_shared(out_glu, up_plus_one); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + } // namespace op } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index 09e29d4cce2..6df2784c2e4 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -2,23 +2,135 @@ #include "../op_table.h" #include "../utils.h" +#include +#include +#include #include +#include +#include #include #include #include #include #include #include +#include #include #include -#include +#include #include +#include namespace ov { namespace frontend { namespace ggml { namespace op { +namespace { + +std::shared_ptr const_i64(const std::vector & values) { + return ov::op::v0::Constant::create(ov::element::i64, ov::Shape{values.size()}, values); +} + +ov::Output slice_axis(const ov::Output & input, int64_t axis, int64_t begin, int64_t end) { + return std::make_shared(input, const_i64({begin}), const_i64({end}), const_i64({1}), + const_i64({axis})); +} + +ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context, + ov::Output expert_weights, + ov::Output activations, + ov::Output ids) { + auto packed_shape = expert_weights.get_partial_shape().to_shape(); + FRONT_END_OP_CONVERSION_CHECK(packed_shape.size() == 5 && packed_shape[4] == 17, + "Expected packed MXFP4 expert weights with shape [1, n_expert, m, k_blocks, 17]"); + + const int64_t n_expert = static_cast(packed_shape[1]); + const int64_t rows = static_cast(packed_shape[2]); + const int64_t k_blocks = static_cast(packed_shape[3]); + const int64_t qk = 32; + const int64_t cols = k_blocks * qk; + + auto packed_shape_4d = const_i64({n_expert, rows, k_blocks, 17}); + expert_weights = std::make_shared(expert_weights, packed_shape_4d, false); + + auto activations_shape_4d = std::make_shared(activations, ov::element::i64); + auto ids_shape_4d = std::make_shared(ids, ov::element::i64); + auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); + auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); + + activations = std::make_shared(activations, activations_shape_3d, false); + ids = std::make_shared(ids, ids_shape_2d, false); + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + + static const std::vector f4e2m1_lut = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; + std::vector e8m0_lut(256); + for (size_t i = 0; i < e8m0_lut.size(); ++i) { + uint32_t bits = static_cast(i) << 23; + memcpy(&e8m0_lut[i], &bits, sizeof(float)); + } + e8m0_lut[0] = std::numeric_limits::min() / 2.0f; + e8m0_lut[255] = std::numeric_limits::quiet_NaN(); + + auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut); + auto scale_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{e8m0_lut.size()}, e8m0_lut); + + auto selected_packed_weights = std::make_shared(expert_weights, ids, gather_axis); + auto scale_byte = slice_axis(selected_packed_weights, 4, 0, 1); + auto qs = slice_axis(selected_packed_weights, 4, 1, 17); + auto low = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY); + auto high_shift = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY); + auto nibbles = std::make_shared(ov::OutputVector{low, high_shift}, 4); + auto nibble_indices = std::make_shared(nibbles, ov::element::i32); + auto weights_f32 = std::make_shared(f4_lut, nibble_indices, gather_axis); + + auto scale_indices = std::make_shared(scale_byte, ov::element::i32); + auto scales_f32 = std::make_shared(scale_lut, scale_indices, gather_axis); + ov::Output selected_weights = std::make_shared(weights_f32, scales_f32, + ov::op::AutoBroadcastType::NUMPY); + + auto ids_shape = std::make_shared(ids, ov::element::i64); + auto selected_weights_target_dims = std::make_shared( + ov::OutputVector{get_dimensions(ids_shape, {0, 1}), const_i64({rows, cols})}, 0); + selected_weights = std::make_shared(selected_weights, selected_weights_target_dims, false); + + auto activations_shape = std::make_shared(activations, ov::element::i64); + ov::Output acts_target_dims = std::make_shared( + ov::OutputVector{ + get_dimensions(activations_shape, {0}), + get_dimensions(ids_shape, {1}), + get_dimensions(activations_shape, {2}), + }, + 0); + ov::Output acts_broadcasted = + std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); + + auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2})); + ov::Output result = + std::make_shared(activations_expanded, selected_weights, false, true); + + auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {rows}); + auto result_target_dims = std::make_shared( + ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); + result = std::make_shared(result, result_target_dims, false); + + const auto output_type = context.get_output_type(); + if (result.get_element_type() != output_type) { + result = std::make_shared(result, output_type); + } + return result; +} + +} // namespace + OutputVector translate_mul_mat_id(const NodeContext & context) { num_inputs_check(context, 3, 3); @@ -26,6 +138,12 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { auto activations = process_view_input_new(context, 1); auto ids = process_view_input_new(context, 2); + if (expert_weights.get_element_type() == ov::element::u8 && expert_weights.get_partial_shape().rank().is_static() && + expert_weights.get_partial_shape().rank().get_length() == 5) { + return rename_outputs_with_suffix({translate_mul_mat_id_mxfp4_packed(context, expert_weights, activations, ids)}, + context.get_name()); + } + // OpenVINO sees GGML tensors in reversed dimension order: // weights: [1, n_expert, m, k] // activations: [1, n_tokens, n_used_or_1, k] diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 287faedbb53..b391d3f9107 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -6,12 +6,16 @@ #include #include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include @@ -20,12 +24,31 @@ namespace frontend { namespace ggml { namespace op { +static bool is_static_one(const ov::Dimension & dim) { + return dim.is_static() && dim.get_length() == 1; +} + +static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) { + return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length(); +} + +static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) { + if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 || + logits_shape.rank().get_length() != 4) { + return false; + } + + return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) && + same_static_dim(candidate[3], logits_shape[1]); +} + // Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: // 1) logits = src0 * scale // 2) logits += mask (if provided) -// 3) softmax over the last dimension +// 3) append attention sinks as hidden logits (if provided) +// 4) softmax over the last dimension and remove the hidden sink column OutputVector translate_soft_max(const NodeContext & context) { - num_inputs_check(context, 1, 2); + num_inputs_check(context, 1, 3); float scale = 1.0f; float max_bias = 0.0f; @@ -33,6 +56,11 @@ OutputVector translate_soft_max(const NodeContext & context) { memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); ov::Output logits = context.get_input(0); + const bool second_input_is_sinks = + context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape()); + const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks; + const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; + const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; // Apply scale first: logits = src0 * scale if (scale != 1.0f) { @@ -41,12 +69,12 @@ OutputVector translate_soft_max(const NodeContext & context) { logits = std::make_shared(logits, scale_const); } - FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && context.get_input_size() < 2), + FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask), "OpenVINO softmax ALiBi path requires mask input"); // Optional mask add: logits += mask // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. - if (context.get_input_size() > 1) { + if (has_mask) { ov::Output mask = context.get_input(1); // For stateful @@ -94,8 +122,40 @@ OutputVector translate_soft_max(const NodeContext & context) { logits = std::make_shared(logits, mask); } + ov::Output softmax_input = logits; + if (has_sinks) { + ov::Output sinks = context.get_input(sinks_input_idx); + if (sinks.get_element_type() != logits.get_element_type()) { + sinks = std::make_shared(sinks, logits.get_element_type()); + } + + auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1}); + auto sinks_4d = std::make_shared(sinks, sink_shape, false); + + auto logits_shape = std::make_shared(logits, ov::element::i64); + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4}); + auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + + auto sink_prefix_shape = std::make_shared(logits_shape, zero, three, one, shape_axis); + auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto sink_broadcast_shape = std::make_shared( + ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0); + auto sink_column = std::make_shared(sinks_4d, sink_broadcast_shape, + ov::op::BroadcastType::BIDIRECTIONAL); + softmax_input = std::make_shared(ov::OutputVector{logits, sink_column}, 3); + + auto softmax_with_sink = std::make_shared(softmax_input, -1); + auto original_last_dim = std::make_shared(logits_shape, three, four, one, shape_axis); + auto res = std::make_shared(softmax_with_sink, zero, original_last_dim, one, three); + + return rename_outputs_with_suffix({res}, context.get_name()); + } + // Softmax along last dimension (equivalent to ggml softmax over ne[0]). - auto res = std::make_shared(logits, -1); + auto res = std::make_shared(softmax_input, -1); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index f84a1bf931a..59fd26df8cd 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -47,6 +47,7 @@ std::unordered_map get_supported_ops() { {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, {"GGML_OP_VIEW", op::translate_view }, {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, + {"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai }, {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, {"GGML_OP_SET_ROWS", op::translate_set_rows }, {"GGML_OP_CPY", op::translate_cpy }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index c90ff837790..1d695fa1258 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -32,6 +32,7 @@ GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); GGML_OP_CONVERTER(translate_view); GGML_OP_CONVERTER(translate_glu_swiglu); +GGML_OP_CONVERTER(translate_glu_swiglu_oai); GGML_OP_CONVERTER(translate_glu_geglu); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 15d231f70c0..1ebc50a763f 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -71,6 +71,44 @@ void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_REST } } +void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + // Compute scale as max absolute value in the block + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float a = fabsf(x[i*qk + j]); + if (a > amax) amax = a; + } + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + // Clear quant bytes + for (int j = 0; j < qk / 4; ++j) { + y[i].qs[j] = 0; + } + + // Encode 2-bit values: round(w/d) clamped to [-1, 2], then add 1 + // 00 (-1) = -scale, 01 (0) = 0, 10 (+1) = +scale, 11 (+2) = 2*scale + for (int j = 0; j < qk; ++j) { + const float w = x[i*qk + j]; + int q = (int)roundf(w * id) + 1; + if (q < 0) q = 0; + if (q > 3) q = 3; + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + y[i].qs[byte_index] |= ((uint8_t)q << bit_offset); + } + } +} + // reference implementation for deterministic creation of model files void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -398,6 +436,26 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI } } +void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(x[i].d); + + for (int j = 0; j < qk; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const uint8_t q = (x[i].qs[byte_index] >> bit_offset) & 0x03; + // 00=-1, 01=0, 10=+1, 11=+2 + y[i*qk + j] = ((int)q - 1) * d; + } + } +} + void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -2052,6 +2110,20 @@ size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } +size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + if (!quant_weights) { + quantize_row_q2_0_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + } + size_t row_size = ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_q2_0_ref(src, (block_q2_0*)qrow, n_per_row); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { if (!quant_weights) { @@ -5461,6 +5533,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb); } break; + case GGML_TYPE_Q2_0: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb); + } break; case GGML_TYPE_Q4_0: { VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index d56c86da890..75188f1af18 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -15,6 +15,7 @@ extern "C" { // Quantization GGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k); @@ -43,6 +44,7 @@ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_ // Dequantization GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -93,6 +95,7 @@ GGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index 1f5a9127266..2d92a95661e 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -14,6 +14,7 @@ #define GGML_SYCL_BACKEND_HPP #include "binbcast.hpp" +#include "col2im-1d.hpp" #include "common.hpp" #include "concat.hpp" #include "conv.hpp" diff --git a/ggml/src/ggml-sycl/col2im-1d.cpp b/ggml/src/ggml-sycl/col2im-1d.cpp new file mode 100644 index 00000000000..c4f090040e3 --- /dev/null +++ b/ggml/src/ggml-sycl/col2im-1d.cpp @@ -0,0 +1,102 @@ +#include "col2im-1d.hpp" + +template +static void col2im_1d_sycl( + const T * col, + T * dst, + const int T_in, + const sycl::uint3 T_out_fd, + const int K, + const int K_OC, + const int32_t s0, + const int32_t p0, + const int total, + dpct::queue_ptr stream) { + + const uint32_t block_size = SYCL_COL2IM_1D_BLOCK_SIZE; + const uint32_t num_blocks = (uint32_t) ((total + block_size - 1) / block_size); + + stream->parallel_for( + sycl::nd_range<3>( + sycl::range<3>(1, 1, num_blocks * block_size), + sycl::range<3>(1, 1, block_size)), + [=](sycl::nd_item<3> item_ct1) { + const int idx = (int) item_ct1.get_global_id(2); + if (idx >= total) { + return; + } + + const sycl::uint2 qr = fast_div_modulo((uint32_t) idx, T_out_fd); + const int oc = (int) qr.x(); + const int t_out = (int) qr.y(); + const int t_abs = t_out + p0; + + int t_in_min = (t_abs - K + s0) / s0; + if (t_in_min < 0) { + t_in_min = 0; + } + int t_in_max = t_abs / s0; + if (t_in_max >= T_in) { + t_in_max = T_in - 1; + } + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; ++t_in) { + const int k = t_abs - t_in * s0; + sum += static_cast(col[(oc * K + k) + t_in * K_OC]); + } + + dst[idx] = static_cast(sum); + }); +} + +void ggml_sycl_op_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(src0 != nullptr); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(src0->type == dst->type); + + const int32_t s0 = ((const int32_t *) dst->op_params)[0]; + const int32_t OC = ((const int32_t *) dst->op_params)[1]; + const int32_t p0 = ((const int32_t *) dst->op_params)[2]; + + const int K_OC = (int) src0->ne[0]; + const int T_in = (int) src0->ne[1]; + const int K = K_OC / OC; + const int T_out = (int) dst->ne[0]; + + GGML_ASSERT(OC > 0); + GGML_ASSERT(K_OC % OC == 0); + + const sycl::uint3 T_out_fd = init_fastdiv_values((uint32_t) T_out); + + const int total = T_out * OC; + + dpct::queue_ptr stream = ctx.stream(); + + switch (src0->type) { + case GGML_TYPE_F32: + col2im_1d_sycl( + (const float *) src0->data, + (float *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; + case GGML_TYPE_F16: + col2im_1d_sycl( + (const sycl::half *) src0->data, + (sycl::half *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; +#ifdef GGML_SYCL_HAS_BF16 + case GGML_TYPE_BF16: + col2im_1d_sycl( + (const sycl::ext::oneapi::bfloat16 *) src0->data, + (sycl::ext::oneapi::bfloat16 *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; +#endif + default: + GGML_ABORT("col2im_1d: unsupported type %d", src0->type); + } +} diff --git a/ggml/src/ggml-sycl/col2im-1d.hpp b/ggml/src/ggml-sycl/col2im-1d.hpp new file mode 100644 index 00000000000..cfb53da5da3 --- /dev/null +++ b/ggml/src/ggml-sycl/col2im-1d.hpp @@ -0,0 +1,8 @@ +#ifndef GGML_SYCL_COL2IM_1D_HPP +#define GGML_SYCL_COL2IM_1D_HPP + +#include "common.hpp" + +void ggml_sycl_op_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_COL2IM_1D_HPP diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 8534bd3581e..fcc97611e15 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -59,7 +59,7 @@ void ggml_sycl_host_free(void* ptr); extern int g_ggml_sycl_debug; -extern int g_ggml_sycl_disable_optimize; +extern int g_ggml_sycl_enable_optimize; extern int g_ggml_sycl_prioritize_dmmv; extern int g_ggml_sycl_enable_flash_attention; extern int g_ggml_sycl_dev2dev_memcpy; diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index 96709554cf6..5d0f9a89fd3 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -1,6 +1,7 @@ #include "cpy.hpp" #include +#include #include "dequantize.hpp" #include "ggml-sycl/common.hpp" @@ -50,6 +51,57 @@ static void cpy_1_i32_i32(const char * cxi, char * cdsti) { *dsti = *xi; } +static void cpy_1_f32_i32(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + int32_t * dsti = (int32_t *) cdsti; + + *dsti = (int32_t) *xi; +} + +static void cpy_1_i32_f32(const char * cxi, char * cdsti) { + const int32_t * xi = (const int32_t *) cxi; + float * dsti = (float *) cdsti; + + *dsti = (float) *xi; +} + +#ifdef GGML_SYCL_HAS_BF16 +static void cpy_1_f32_bf16(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = sycl::ext::oneapi::bfloat16(*xi); +} + +static void cpy_1_bf16_f32(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + float * dsti = (float *) cdsti; + + *dsti = static_cast(*xi); +} + +static void cpy_1_bf16_bf16(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = *xi; +} + +static void cpy_1_f16_bf16(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = sycl::ext::oneapi::bfloat16(static_cast(*xi)); +} + +static void cpy_1_bf16_f16(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + sycl::half * dsti = (sycl::half *) cdsti; + + *dsti = sycl::half(static_cast(*xi)); +} +#endif + template static void cpy_f32_f16(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, @@ -247,6 +299,38 @@ static void ggml_cpy_f32_f16_sycl(const char * cx, char * cdst, const int ne, co } } +static void ggml_cpy_f32_i32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + { + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); + } +} + +static void ggml_cpy_i32_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + { + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); + } +} + static void ggml_cpy_f32_q8_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -376,6 +460,19 @@ static void ggml_cpy_q5_1_f32_sycl(const char * cx, char * cdst, const int ne, c }); } +static void ggml_cpy_mxfp4_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ne; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_f32, QK_MXFP4>(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -389,6 +486,269 @@ static void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, }); } +static void cpy_blck_f16_q4_0(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK4_0]; + + for (int j = 0; j < QK4_0; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q4_0((const char *) xf, cdsti); +} + +static void cpy_blck_f16_q4_1(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK4_1]; + + for (int j = 0; j < QK4_1; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q4_1((const char *) xf, cdsti); +} + +static void cpy_blck_f16_q5_0(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK5_0]; + + for (int j = 0; j < QK5_0; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q5_0((const char *) xf, cdsti); +} + +static void ggml_cpy_f16_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK4_0 == 0); + const int num_blocks = ne / QK4_0; + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_q4_1_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK4_1 == 0); + const int num_blocks = ne / QK4_1; + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK5_0 == 0); + const int num_blocks = ne / QK5_0; + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static bool ggml_sycl_is_quantized_type(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +static bool ggml_sycl_can_quantize_rows_sycl(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +template +static inline float ggml_sycl_src_to_f32(const SrcScalar & x) { + return (float) x; +} + +#ifdef GGML_SYCL_HAS_BF16 +template <> +inline float ggml_sycl_src_to_f32(const sycl::ext::oneapi::bfloat16 & x) { + return static_cast(x); +} + +template <> +inline float ggml_sycl_src_to_f32(const ggml_bf16_t & x) { + union { + uint32_t u32; + float f32; + } value; + + value.u32 = (uint32_t) x.bits << 16; + return value.f32; +} +#endif + +template +static void ggml_sycl_quantize_rows_q(const char * cx, char * cdst, const int64_t ne, + const int64_t ne00, const int64_t ne01, const int64_t ne02, + const size_t nb00, const size_t nb01, const size_t nb02, const size_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, + const size_t nb10, const size_t nb11, const size_t nb12, const size_t nb13, + queue_ptr stream) { + GGML_ASSERT(ne % qk == 0); + GGML_ASSERT(ne00 % qk == 0); + + const int64_t total_blocks = ne / qk; + constexpr int block_size = 256; + const int64_t grid_size = ceil_div(total_blocks, (int64_t) block_size); + + stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), [=](sycl::nd_item<1> item_ct1) { + const int64_t block_idx = item_ct1.get_global_linear_id(); + if (block_idx >= total_blocks) { + return; + } + + const int64_t i = block_idx * qk; + + const int64_t i03 = i / (ne00 * ne01 * ne02); + const int64_t i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01); + const int64_t i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00; + const int64_t i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00; + const size_t x_offset = i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03; + + const int64_t i13 = i / (ne10 * ne11 * ne12); + const int64_t i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11); + const int64_t i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10; + const int64_t i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10; + const size_t dst_offset = (i10 / qk) * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13; + + float xf[qk]; + if (nb00 == sizeof(SrcScalar)) { + const SrcScalar * src_row = (const SrcScalar *) (cx + x_offset); + for (int j = 0; j < qk; ++j) { + xf[j] = ggml_sycl_src_to_f32(src_row[j]); + } + } else { + for (int j = 0; j < qk; ++j) { + const SrcScalar * src_val = (const SrcScalar *) (cx + x_offset + j * nb00); + xf[j] = ggml_sycl_src_to_f32(*src_val); + } + } + + quantize_block((const char *) xf, cdst + dst_offset); + }); +} + +template +static void ggml_sycl_quantize_rows_sycl(const char * cx, char * cdst, const ggml_tensor * src0, const ggml_tensor * src1, + const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, + const size_t nb00, const size_t nb01, const size_t nb02, const size_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, + const size_t nb11, const size_t nb12, const size_t nb13, queue_ptr stream) { + GGML_UNUSED(src0); + GGML_UNUSED(src1); + + switch (src1->type) { + case GGML_TYPE_Q8_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q1_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q5_1: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q5_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q4_1: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q4_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_IQ4_NL: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + case GGML_TYPE_MXFP4: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + case GGML_TYPE_NVFP4: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + default: + GGML_ABORT("unsupported quantized target type in sycl quantizer src1->type=%s\n", + ggml_type_name(src1->type)); + } +} + static void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -509,8 +869,269 @@ static void ggml_cpy_q4_1_q4_1(const char * cx, char * cdst, const int ne, const }); } +static void ggml_cpy_q1_0_q1_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_mxfp4_mxfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_nvfp4_nvfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q2_K_q2_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q3_K_q3_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q4_K_q4_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q5_K_q5_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q6_K_q6_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_xxs_iq2_xxs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_xs_iq2_xs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_s_iq2_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq3_xxs_iq3_xxs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq1_s_iq1_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq1_m_iq1_m(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq4_nl_iq4_nl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq3_s_iq3_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq4_xs_iq4_xs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +#ifdef GGML_SYCL_HAS_BF16 +static void ggml_cpy_f32_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) { + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} +#endif + void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1) try { // Unlike other operators ggml_sycl_cpy takes 2 distinct tensors instead of a dst ggml_tensor and rely on its src field + GGML_SYCL_DEBUG("ggml_sycl_cpy: src0->type=%s, src1->type=%s\n", + ggml_type_name(src0->type), ggml_type_name(src1->type)); scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0)); const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); @@ -525,12 +1146,31 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co if ((src0->type == src1->type) && (ggml_is_contiguous(src0) && ggml_is_contiguous(src1))) { GGML_SYCL_DEBUG("%s: memcpy path\n", __func__); main_stream->memcpy(src1_ddc, src0_ddc, ggml_nbytes(src0)); + } else if (src0->type == GGML_TYPE_F32 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, + main_stream); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_BF16 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, main_stream); +#endif } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { ggml_cpy_f32_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) { ggml_cpy_f32_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) { + ggml_cpy_f32_i32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) { ggml_cpy_f32_q8_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -546,12 +1186,24 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { ggml_cpy_f16_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q4_0) { + ggml_cpy_f16_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q4_1) { + ggml_cpy_f16_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q5_0) { + ggml_cpy_f16_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { ggml_cpy_i16_i16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) { ggml_cpy_i32_i32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_F32) { + ggml_cpy_i32_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q4_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -573,6 +1225,9 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) { ggml_cpy_q5_1_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_F32) { + ggml_cpy_mxfp4_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { ggml_cpy_f32_iq4_nl_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -586,6 +1241,57 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co ggml_cpy_q4_0_q4_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_q4_1_q4_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q1_0 && src1->type == GGML_TYPE_Q1_0) { + ggml_cpy_q1_0_q1_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_MXFP4) { + ggml_cpy_mxfp4_mxfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_NVFP4) { + ggml_cpy_nvfp4_nvfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_K && src1->type == GGML_TYPE_Q2_K) { + ggml_cpy_q2_K_q2_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q3_K && src1->type == GGML_TYPE_Q3_K) { + ggml_cpy_q3_K_q3_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q4_K && src1->type == GGML_TYPE_Q4_K) { + ggml_cpy_q4_K_q4_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q5_K && src1->type == GGML_TYPE_Q5_K) { + ggml_cpy_q5_K_q5_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q6_K && src1->type == GGML_TYPE_Q6_K) { + ggml_cpy_q6_K_q6_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_XXS && src1->type == GGML_TYPE_IQ2_XXS) { + ggml_cpy_iq2_xxs_iq2_xxs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_XS && src1->type == GGML_TYPE_IQ2_XS) { + ggml_cpy_iq2_xs_iq2_xs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_S && src1->type == GGML_TYPE_IQ2_S) { + ggml_cpy_iq2_s_iq2_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ3_XXS && src1->type == GGML_TYPE_IQ3_XXS) { + ggml_cpy_iq3_xxs_iq3_xxs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ1_S && src1->type == GGML_TYPE_IQ1_S) { + ggml_cpy_iq1_s_iq1_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ1_M && src1->type == GGML_TYPE_IQ1_M) { + ggml_cpy_iq1_m_iq1_m(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ4_NL && src1->type == GGML_TYPE_IQ4_NL) { + ggml_cpy_iq4_nl_iq4_nl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ3_S && src1->type == GGML_TYPE_IQ3_S) { + ggml_cpy_iq3_s_iq3_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ4_XS && src1->type == GGML_TYPE_IQ4_XS) { + ggml_cpy_iq4_xs_iq4_xs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_f32_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32) { + ggml_cpy_bf16_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_bf16_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_f16_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F16) { + ggml_cpy_bf16_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); +#endif } else { GGML_LOG_ERROR("%s: unsupported type combination (%s to %s)\n", __func__, ggml_type_name(src0->type), ggml_type_name(src1->type)); diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index 62ff34c8793..c4cfd961da2 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -317,7 +317,7 @@ inline void cpy_blck_f32_nvfp4(const char * cxi, char * cdsti) { const uint8_t ue = ggml_fp32_to_ue4m3(amax / 6.0f); dsti->d[s] = ue; - const float d = ggml_ue4m3_to_fp32(ue); + const float d = ggml_sycl_ue4m3_to_fp32(ue); for (int j = 0; j < QK_NVFP4_SUB / 2; ++j) { const uint8_t x0 = best_index_mxfp4(xb[0 + j], d); diff --git a/ggml/src/ggml-sycl/cross_entropy_loss.cpp b/ggml/src/ggml-sycl/cross_entropy_loss.cpp new file mode 100644 index 00000000000..c9d12a5590f --- /dev/null +++ b/ggml/src/ggml-sycl/cross_entropy_loss.cpp @@ -0,0 +1,255 @@ +#include "cross_entropy_loss.hpp" + +#include +#include + +template +static __dpct_inline__ void cross_entropy_loss_f32_kernel( + const float * __restrict__ logits, + const float * __restrict__ labels, + float * __restrict__ row_loss, + const int nclasses, + const int nrows, + float * __restrict__ smem, + const sycl::nd_item<3> & item) { + + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + + logits += (int64_t) row * nclasses; + labels += (int64_t) row * nclasses; + + float max_logit = -INFINITY; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = logits[i]; + max_logit = sycl::fmax(max_logit, v); + if (has_shared) { + smem[i] = v; + } + } + max_logit = warp_reduce_max(max_logit); + + float sum_exp = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = has_shared ? smem[i] : logits[i]; + sum_exp += sycl::exp(v - max_logit); + } + sum_exp = warp_reduce_sum(sum_exp); + const float log_sum = sycl::log(sum_exp); + + float loss = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = has_shared ? smem[i] : logits[i]; + loss += (v - max_logit - log_sum) * labels[i]; + } + loss = -warp_reduce_sum(loss) / (float) nrows; + + if (tid == 0) { + row_loss[row] = loss; + } +} + +template +static __dpct_inline__ void cross_entropy_loss_back_f32_kernel( + const float * __restrict__ grad, + const float * __restrict__ logits, + const float * __restrict__ labels, + float * __restrict__ dst, + const int nclasses, + const int nrows, + float * __restrict__ smem, + const sycl::nd_item<3> & item) { + + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + + logits += (int64_t) row * nclasses; + labels += (int64_t) row * nclasses; + dst += (int64_t) row * nclasses; + + float max_logit = -INFINITY; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = logits[i]; + max_logit = sycl::fmax(max_logit, v); + if (has_shared) { + smem[i] = v; + } + } + max_logit = warp_reduce_max(max_logit); + + float sum_exp = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = sycl::exp((has_shared ? smem[i] : logits[i]) - max_logit); + sum_exp += v; + if (has_shared) { + smem[i] = v; + } else { + dst[i] = v; + } + } + sum_exp = warp_reduce_sum(sum_exp); + const float inv_sum = 1.0f / sum_exp; + + const float d_by_nrows = grad[0] / (float) nrows; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float sm_num = has_shared ? smem[i] : dst[i]; + dst[i] = (sm_num * inv_sum - labels[i]) * d_by_nrows; + } +} + +static void cross_entropy_reduce_rows( + ggml_backend_sycl_context & ctx, + const float * row_loss, + float * dst, + const int64_t nrows) { + if (nrows == 1) { + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(dst, row_loss, sizeof(float)))); + return; + } + + ggml_sycl_pool_alloc tmp_alloc(ctx.pool(), nrows); + float * tmp = tmp_alloc.get(); + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(tmp, row_loss, nrows * sizeof(float)))); + + int64_t cur = nrows; + while (cur > 1) { + const int64_t out = (cur + WARP_SIZE - 1) / WARP_SIZE; + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, out); + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + const int64_t i = (int64_t) row * WARP_SIZE + tid; + float v = i < cur ? tmp[i] : 0.0f; + v = warp_reduce_sum(v); + if (tid == 0) { + tmp[row] = v; + } + }); + cur = out; + } + + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(dst, tmp, sizeof(float)))); +} + +void ggml_sycl_cross_entropy_loss(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, src1)); + GGML_ASSERT(ggml_is_scalar(dst)); + + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const int64_t nclasses = src0->ne[0]; + const int64_t nrows = ggml_nrows(src0); + + const float * logits_d = (const float *) src0->data; + const float * labels_d = (const float *) src1->data; + float * dst_d = (float *) dst->data; + + ggml_sycl_pool_alloc row_loss_alloc(ctx.pool(), nrows); + float * row_loss = row_loss_alloc.get(); + + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, nrows); + const size_t nbytes_shared = (size_t) nclasses * sizeof(float); + const size_t smpbo = ggml_sycl_info().devices[ctx.device].smpbo; + + if (nbytes_shared <= smpbo) { + ctx.stream()->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem(sycl::range<1>(nclasses), cgh); + cgh.parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_f32_kernel( + logits_d, labels_d, row_loss, + (int) nclasses, (int) nrows, + get_pointer(smem), item); + }); + }); + } else { + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_f32_kernel( + logits_d, labels_d, row_loss, + (int) nclasses, (int) nrows, + nullptr, item); + }); + } + + cross_entropy_reduce_rows(ctx, row_loss, dst_d, nrows); +} + +void ggml_sycl_cross_entropy_loss_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3); + + const ggml_tensor * grad = dst->src[0]; + const ggml_tensor * src0f = dst->src[1]; + const ggml_tensor * src1f = dst->src[2]; + + GGML_ASSERT(grad->type == GGML_TYPE_F32); + GGML_ASSERT(src0f->type == GGML_TYPE_F32); + GGML_ASSERT(src1f->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_scalar(grad)); + GGML_ASSERT(ggml_is_contiguous(grad)); + GGML_ASSERT(ggml_is_contiguous(src0f)); + GGML_ASSERT(ggml_is_contiguous(src1f)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0f, src1f)); + GGML_ASSERT(ggml_are_same_shape(src0f, dst)); + + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const int64_t nclasses = src0f->ne[0]; + const int64_t nrows = ggml_nrows(src0f); + + const float * grad_d = (const float *) grad->data; + const float * logits_d = (const float *) src0f->data; + const float * labels_d = (const float *) src1f->data; + float * dst_d = (float *) dst->data; + + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, nrows); + const size_t nbytes_shared = (size_t) nclasses * sizeof(float); + const size_t smpbo = ggml_sycl_info().devices[ctx.device].smpbo; + + if (nbytes_shared <= smpbo) { + ctx.stream()->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem(sycl::range<1>(nclasses), cgh); + cgh.parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_back_f32_kernel( + grad_d, logits_d, labels_d, dst_d, + (int) nclasses, (int) nrows, + get_pointer(smem), item); + }); + }); + } else { + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_back_f32_kernel( + grad_d, logits_d, labels_d, dst_d, + (int) nclasses, (int) nrows, + nullptr, item); + }); + } +} diff --git a/ggml/src/ggml-sycl/cross_entropy_loss.hpp b/ggml/src/ggml-sycl/cross_entropy_loss.hpp new file mode 100644 index 00000000000..3f1cb817ba1 --- /dev/null +++ b/ggml/src/ggml-sycl/cross_entropy_loss.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "common.hpp" + +void ggml_sycl_cross_entropy_loss(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +void ggml_sycl_cross_entropy_loss_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index fb8a1757f19..5c6835e1d23 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -680,14 +680,14 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, q16[2] = q2[0] & 0x0f0f; q16[3] = q2[0] & 0xf0f0; - float4 s = {0.f, 0.f, 0.f, 0.f}; + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; float smin = 0; for (int l = 0; l < 2; ++l) { - s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2]; - s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6]; + s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2]; + s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6]; smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; } - tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin; + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin; #endif } @@ -835,14 +835,14 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, q16[2] = q2[0] & 0x0f0f; q16[3] = q2[0] & 0xf0f0; - float4 s = {0.f, 0.f, 0.f, 0.f}; + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; float smin = 0; for (int l = 0; l < 2; ++l) { - s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2]; - s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6]; + s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2]; + s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6]; smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; } - tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin; + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin; #endif } @@ -1126,7 +1126,7 @@ static void dequantize_mul_mat_vec_q5_k_reorder(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -1762,10 +1762,13 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const sycl::range<3> block_dims(1, 1, QK_WARP_SIZE); + const int ny = 2 / K_QUANTS_PER_ITERATION; + const int block_num_y = (nrows + ny - 1) / ny; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1); }); } diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 0c82ceb969f..bae157a487a 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -9,9 +9,12 @@ #define SYCL_LOCAL_ID_CALC(ITEM, IDX) \ (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX)) -static void acc_f32(const float * x, const float * y, float * dst, const int64_t ne, - const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, - const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) { +static void acc_f32(const char * x, const char * y, float * dst, const int64_t ne, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) { auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); const int64_t i = SYCL_LOCAL_ID_CALC(item_ct1, 2); @@ -30,9 +33,18 @@ static void acc_f32(const float * x, const float * y, float * dst, const int64_t tmp -= i11 * s11; const int64_t i10 = tmp; - float val = x[i]; + int64_t tmp_dst = i; + const int64_t i3 = tmp_dst / (ne2*ne1*ne0); + tmp_dst -= i3 * (ne2*ne1*ne0); + const int64_t i2 = tmp_dst / (ne1*ne0); + tmp_dst -= i2 * (ne1*ne0); + const int64_t i1 = tmp_dst / ne0; + tmp_dst -= i1 * ne0; + const int64_t i0 = tmp_dst; + + float val = *(const float *) (x + i0*nb00 + i1*nb01 + i2*nb02 + i3*nb03); if (src1_idx >= 0 && i10 < ne10 && i11 < ne11 && i12 < ne12 && i13 < ne13) { - val += y[((i13*ne12 + i12) * ne11 + i11) * ne10 + i10]; + val += *(const float *) (y + i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13); } dst[i] = val; } @@ -422,15 +434,24 @@ static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const } namespace ggml_sycl_detail { -static void acc_f32_sycl(const float *x, const float *y, float *dst, - const int64_t n_elements, const int64_t ne10, const int64_t ne11, - const int64_t ne12, const int64_t ne13, const int64_t s1, const int64_t s2, const int64_t s3, +static void acc_f32_sycl(const char *x, const char *y, float *dst, + const int64_t n_elements, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + const int64_t s1, const int64_t s2, const int64_t s3, const int64_t offset, queue_ptr stream) { const int num_blocks = (n_elements + SYCL_ACC_BLOCK_SIZE - 1) / SYCL_ACC_BLOCK_SIZE; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, ne13, s1, s2, s3, offset); + acc_f32(x, y, dst, n_elements, + ne0, ne1, ne2, ne3, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, ne13, + nb10, nb11, nb12, nb13, + s1, s2, s3, offset); }); } @@ -843,8 +864,8 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - const float * src0_d = (const float *) src0->data; - const float * src1_d = (const float *) src1->data; + const char * src0_d = (const char *) src0->data; + const char * src1_d = (const char *) src1->data; float * dst_d = (float *) dst->data; dpct::queue_ptr stream = ctx.stream(); @@ -853,17 +874,20 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src1)); GGML_ASSERT(dst->nb[0] == ggml_element_size(dst)); GGML_ASSERT(ggml_is_contiguously_allocated(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, dst)); - const int64_t s1 = dst->op_params[0] / sizeof(float); - const int64_t s2 = dst->op_params[1] / sizeof(float); - const int64_t s3 = dst->op_params[2] / sizeof(float); - const int64_t offset = dst->op_params[3] / sizeof(float); + const int64_t s1 = (int64_t) ((const int32_t *) dst->op_params)[0] / (int64_t) sizeof(float); + const int64_t s2 = (int64_t) ((const int32_t *) dst->op_params)[1] / (int64_t) sizeof(float); + const int64_t s3 = (int64_t) ((const int32_t *) dst->op_params)[2] / (int64_t) sizeof(float); + const int64_t offset = (int64_t) ((const int32_t *) dst->op_params)[3] / (int64_t) sizeof(float); ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst), + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], s1, s2, s3, offset, stream); } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 41449db665e..5226fb184a9 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -41,7 +41,7 @@ #if SYCL_EXT_ONEAPI_VIRTUAL_MEM # include # include -# define GGML_SYCL_USE_VMM +# define GGML_SYCL_SUPPORT_VMM #endif #include @@ -74,15 +74,16 @@ #include "ggml-sycl/solve_tri.hpp" #include "ggml-sycl/gated_delta_net.hpp" #include "ggml-sycl/pool.hpp" +#include "ggml-sycl/cross_entropy_loss.hpp" #define MEM_SIZE_2M 0x00200000 #define MEM_SIZE_1G 0x40000000 static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; -int g_ggml_sycl_disable_optimize = 0; -int g_ggml_sycl_disable_graph = 0; -int g_ggml_sycl_disable_dnn = 0; +int g_ggml_sycl_enable_optimize = 1; +int g_ggml_sycl_enable_graph = 0; +int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_enable_vmm = 1; int g_ggml_sycl_prioritize_dmmv = 0; int g_ggml_sycl_use_async_mem_op = 0; @@ -117,7 +118,7 @@ static ggml_sycl_device_info ggml_sycl_init() { SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( prop, device))); -#if !defined(GGML_SYCL_USE_VMM) +#if !defined(GGML_SYCL_SUPPORT_VMM) info.devices[i].vmm = 0; #else info.devices[i].vmm = device.has(sycl::aspect::ext_oneapi_virtual_mem); @@ -265,14 +266,24 @@ void ggml_backend_sycl_print_sycl_devices() { print_device_opt_feature(device_count); } +static const char* dev2dev_int2str(int dev2dev) { + if (dev2dev == DEV2DEV_MEMCPY_SYCL) { + return "SYCL API"; + } else if (dev2dev == DEV2DEV_MEMCPY_L0) { + return "Level Zero API"; + } else { + return "Unknown"; + } +} + static void ggml_check_sycl() try { static bool initialized = false; if (!initialized) { g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); - g_ggml_sycl_disable_optimize = ggml_sycl_get_env("GGML_SYCL_DISABLE_OPT", 0); - g_ggml_sycl_disable_graph = ggml_sycl_get_env("GGML_SYCL_DISABLE_GRAPH", 1); - g_ggml_sycl_disable_dnn = ggml_sycl_get_env("GGML_SYCL_DISABLE_DNN", 0); + g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); + g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); + g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); @@ -292,72 +303,87 @@ static void ggml_check_sycl() try { GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("Build with Macros:\n"); -#if defined(GGML_SYCL_FORCE_MMQ) - GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); +#if defined(GGML_SYCL_DNNL) + GGML_LOG_INFO(" GGML_SYCL_DNNL: yes\n"); #else - GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); + GGML_LOG_INFO(" GGML_SYCL_DNNL: no\n"); #endif + #if defined(GGML_SYCL_F16) GGML_LOG_INFO(" GGML_SYCL_F16: yes\n"); #else GGML_LOG_INFO(" GGML_SYCL_F16: no\n"); #endif + +#if defined(GGML_SYCL_FORCE_MMQ) + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); +#else + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); +#endif + #if defined(GGML_SYCL_GRAPH) GGML_LOG_INFO(" GGML_SYCL_GRAPH: yes\n"); #else GGML_LOG_INFO(" GGML_SYCL_GRAPH: no\n"); #endif -#if defined(GGML_SYCL_DNNL) - GGML_LOG_INFO(" GGML_SYCL_DNNL: yes\n"); -#else - GGML_LOG_INFO(" GGML_SYCL_DNNL: no\n"); -#endif + #if defined(GGML_SYCL_SUPPORT_LEVEL_ZERO_API) GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: yes\n"); #else GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: no\n"); #endif -#if defined(GGML_SYCL_USE_VMM) - GGML_LOG_INFO(" GGML_SYCL_USE_VMM: yes\n"); +#if defined(GGML_SYCL_SUPPORT_VMM) + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_VMM: yes\n"); #else - GGML_LOG_INFO(" GGML_SYCL_USE_VMM: no\n"); + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_VMM: no\n"); #endif GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); - GGML_LOG_INFO(" GGML_SYCL_DISABLE_OPT: %d\n", g_ggml_sycl_disable_optimize); -#ifdef GGML_SYCL_GRAPH - GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: %d\n", g_ggml_sycl_disable_graph); + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s)\n", g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); #else - GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: graph disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s), enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO_API\n", + g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); #endif -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: %d\n", g_ggml_sycl_use_level_zero_api); - GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d\n", g_ggml_sycl_dev2dev_memcpy); + +#if defined(GGML_SYCL_DNNL) + GGML_LOG_INFO(" GGML_SYCL_ENABLE_DNN: %d\n", g_ggml_sycl_enable_dnn); #else - GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: Disable Level Zero API usage by compile flag\n"); - GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d, enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO_API\n", - g_ggml_sycl_dev2dev_memcpy); + GGML_LOG_INFO(" GGML_SYCL_ENABLE_DNN: DNN disabled by compile flag\n"); #endif -#if GGML_SYCL_DNNL - GGML_LOG_INFO(" GGML_SYCL_DISABLE_DNN: %d\n", g_ggml_sycl_disable_dnn); + +#ifdef SYCL_FLASH_ATTN + GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d\n", g_ggml_sycl_enable_flash_attention); +#else + GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d disabled by compile flag\n", + g_ggml_sycl_enable_flash_attention); +#endif + +#ifdef GGML_SYCL_GRAPH + GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: %d\n", g_ggml_sycl_enable_graph); #else - GGML_LOG_INFO(" GGML_SYCL_DISABLE_DNN: DNN disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: graph disabled by compile flag\n"); #endif -#if defined(GGML_SYCL_USE_VMM) + + GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); + +#if defined(GGML_SYCL_SUPPORT_VMM) GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm); #else GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: virtual memory extension is not available\n"); #endif + GGML_LOG_INFO(" GGML_SYCL_PRIORITIZE_DMMV: %d\n", g_ggml_sycl_prioritize_dmmv); + g_ggml_sycl_use_async_mem_op_requested = ggml_sycl_get_env("GGML_SYCL_USE_ASYNC_MEM_OP", 1); GGML_LOG_INFO(" GGML_SYCL_USE_ASYNC_MEM_OP: %d\n", g_ggml_sycl_use_async_mem_op_requested); -#ifdef SYCL_FLASH_ATTN - GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d\n", g_ggml_sycl_enable_flash_attention); +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: %d\n", g_ggml_sycl_use_level_zero_api); #else - GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d disabled by compile flag\n", - g_ggml_sycl_enable_flash_attention); + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: Disable Level Zero API usage by compile flag\n"); #endif GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system); @@ -373,7 +399,7 @@ static void ggml_check_sycl() try { // staging path while preserving queue ordering semantics. Graph support still depends on the extension being // available, but it no longer needs to control the non-graph fast path. #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC - g_ggml_sycl_use_async_mem_op = g_ggml_sycl_use_async_mem_op_requested || !g_ggml_sycl_disable_graph; + g_ggml_sycl_use_async_mem_op = g_ggml_sycl_use_async_mem_op_requested || g_ggml_sycl_enable_graph; if (g_ggml_sycl_use_async_mem_op) { for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) { if (!dpct::dev_mgr::instance().get_device(i).has(sycl::aspect::ext_oneapi_async_memory_alloc)) { @@ -516,12 +542,14 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, return GGML_STATUS_SUCCESS; } - if (!g_ggml_sycl_disable_optimize) { + if (g_ggml_sycl_enable_optimize) { // set reorder extra buffer based on supported type switch (tensor->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K:{ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; tensor->extra = extra; @@ -1562,7 +1590,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { }; // pool with virtual memory management -#if defined(GGML_SYCL_USE_VMM) +#if defined(GGML_SYCL_SUPPORT_VMM) struct ggml_sycl_pool_vmm : public ggml_sycl_pool { static const size_t SYCL_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB @@ -1674,7 +1702,7 @@ struct ggml_sycl_pool_vmm : public ggml_sycl_pool { GGML_ASSERT(ptr == reinterpret_cast(pool_addr + pool_used)); } }; -#endif // defined(GGML_SYCL_USE_VMM) +#endif // defined(GGML_SYCL_SUPPORT_VMM) struct ggml_sycl_pool_host : public ggml_sycl_pool { queue_ptr qptr; @@ -1756,11 +1784,11 @@ std::unique_ptr ggml_backend_sycl_context::new_pool_for_host(que } std::unique_ptr ggml_backend_sycl_context::new_pool_for_device(queue_ptr qptr, int device) { -#if defined(GGML_SYCL_USE_VMM) +#if defined(GGML_SYCL_SUPPORT_VMM) if (g_ggml_sycl_enable_vmm && ggml_sycl_info().devices[device].vmm) { return std::unique_ptr(new ggml_sycl_pool_vmm(qptr, device)); } -#endif // defined(GGML_SYCL_USE_VMM) +#endif // defined(GGML_SYCL_SUPPORT_VMM) return std::unique_ptr(new ggml_sycl_pool_leg(qptr, device)); } @@ -2088,11 +2116,148 @@ static int next_power_of_2(int x) { return n; } +static void init_argsort_indices_padded( + int * idx, + const int nrows, + const int ncols_pad, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols_pad; + + if (gid >= total) { + return; + } + + idx[gid] = (int) (gid % (size_t) ncols_pad); +} + +template +static void argsort_f32_i32_global_pass(const float * x, + int * idx, + const int ncols, + const int nrows, + const int ncols_pad, + const int j, + const int k, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols_pad; + + if (gid >= total) { + return; + } + + const int row = (int) (gid / (size_t) ncols_pad); + const int col = (int) (gid % (size_t) ncols_pad); + const int ixj = col ^ j; + + if (ixj <= col || ixj >= ncols_pad) { + return; + } + + const size_t base = (size_t) row * (size_t) ncols_pad; + const size_t pos_a = base + (size_t) col; + const size_t pos_b = base + (size_t) ixj; + + const int a = idx[pos_a]; + const int b = idx[pos_b]; + + bool do_swap = false; + + if ((col & k) == 0) { + if (a >= ncols || + (b < ncols && + (order == GGML_SORT_ORDER_ASC ? + x[(size_t) row * (size_t) ncols + (size_t) a] > x[(size_t) row * (size_t) ncols + (size_t) b] : + x[(size_t) row * (size_t) ncols + (size_t) a] < x[(size_t) row * (size_t) ncols + (size_t) b]))) { + do_swap = true; + } + } else { + if (b >= ncols || + (a < ncols && + (order == GGML_SORT_ORDER_ASC ? + x[(size_t) row * (size_t) ncols + (size_t) a] < x[(size_t) row * (size_t) ncols + (size_t) b] : + x[(size_t) row * (size_t) ncols + (size_t) a] > x[(size_t) row * (size_t) ncols + (size_t) b]))) { + do_swap = true; + } + } + + if (do_swap) { + idx[pos_a] = b; + idx[pos_b] = a; + } +} + +static void copy_argsort_indices_unpadded(const int * idx_padded, + int * dst, + const int nrows, + const int ncols, + const int ncols_pad, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols; + + if (gid >= total) { + return; + } + + const int row = (int) (gid / (size_t) ncols); + const int col = (int) (gid % (size_t) ncols); + + dst[(size_t) row * (size_t) ncols + (size_t) col] = idx_padded[(size_t) row * (size_t) ncols_pad + (size_t) col]; +} + static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, const int nrows, ggml_sort_order order, - queue_ptr stream, int device) { + queue_ptr stream, int device, ggml_sycl_pool & pool) { // bitonic sort requires ncols to be power of 2 const int ncols_pad = next_power_of_2(ncols); + const size_t shared_mem = (size_t) ncols_pad * sizeof(int); + const size_t smpbo = ggml_sycl_info().devices[device].smpbo; + + if (shared_mem > smpbo) { + ggml_sycl_pool_alloc idx_padded_alloc(pool, (size_t) nrows * (size_t) ncols_pad); + int * idx_padded = idx_padded_alloc.get(); + + constexpr size_t block_size = 256; + const size_t total_padded = (size_t) nrows * (size_t) ncols_pad; + const size_t nblocks_padded = (total_padded + block_size - 1) / block_size; + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { init_argsort_indices_padded(idx_padded, nrows, ncols_pad, item_ct1); }); + + for (int k = 2; k <= ncols_pad; k *= 2) { + for (int j = k / 2; j > 0; j /= 2) { + if (order == GGML_SORT_ORDER_ASC) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + argsort_f32_i32_global_pass(x, idx_padded, ncols, nrows, ncols_pad, j, + k, item_ct1); + }); + } else if (order == GGML_SORT_ORDER_DESC) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + argsort_f32_i32_global_pass(x, idx_padded, ncols, nrows, ncols_pad, j, + k, item_ct1); + }); + } else { + GGML_ABORT("invalid sort order"); + } + } + } + + const size_t total = (size_t) nrows * (size_t) ncols; + const size_t nblocks = (total + block_size - 1) / block_size; + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>(nblocks * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + copy_argsort_indices_unpadded(idx_padded, dst, nrows, ncols, ncols_pad, item_ct1); + }); + + return; + } int nth = 1; int max_block_size = ggml_sycl_info().max_work_group_sizes[device]; @@ -2105,8 +2270,6 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, const sycl::range<3> block_dims(1, 1, nth); const sycl::range<3> block_nums(1, nrows, 1); - const size_t shared_mem = ncols_pad * sizeof(int); - GGML_ASSERT(shared_mem<=ggml_sycl_info().devices[device].smpbo); if (order == GGML_SORT_ORDER_ASC) { stream->submit([&](sycl::handler &cgh) { @@ -2429,7 +2592,7 @@ inline void ggml_sycl_op_mul_mat_sycl( #if GGML_SYCL_DNNL && defined(GGML_SYCL_HAS_BF16) // Fast path for bf16 src0 - if (src0->type == GGML_TYPE_BF16 && !g_ggml_sycl_disable_dnn && ggml_is_contiguous(src0) && + if (src0->type == GGML_TYPE_BF16 && g_ggml_sycl_enable_dnn && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) { using bf16_t = sycl::ext::oneapi::bfloat16; ggml_sycl_pool_alloc src1_as_bf16(ctx.pool(), src1_ncols*ne10); @@ -2482,7 +2645,7 @@ inline void ggml_sycl_op_mul_mat_sycl( : src1_as_f16.get(); #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn) { + if (g_ggml_sycl_enable_dnn) { DnnlGemmWrapper::row_gemm(ctx,row_diff, src1_ncols , ne10, src0_ptr, DnnlGemmWrapper::to_dt(), src1_ptr, DnnlGemmWrapper::to_dt(), dst_dd_i, DnnlGemmWrapper::to_dt(), stream); @@ -2532,7 +2695,7 @@ inline void ggml_sycl_op_mul_mat_sycl( const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10; const bool use_mkl_direct = gemm_flops < 256 * 256 * 256; #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn && !use_mkl_direct) { + if (g_ggml_sycl_enable_dnn && !use_mkl_direct) { DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i, DnnlGemmWrapper::to_dt(), src1_ddf1_i, DnnlGemmWrapper::to_dt(), dst_dd_i, DnnlGemmWrapper::to_dt(), stream); @@ -2625,7 +2788,7 @@ inline void ggml_sycl_op_argsort(ggml_backend_sycl_context & ctx, ggml_tensor * enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0]; argsort_f32_i32_sycl(src0_dd, (int *)dst_dd, ncols, nrows, order, - main_stream, ctx.device); + main_stream, ctx.device, ctx.pool()); } static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -3352,7 +3515,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons const int64_t r3 = ne13 / ne03; #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn) { + if (g_ggml_sycl_enable_dnn) { int64_t str_a0 = nb00 / type_size_src0; int64_t str_a1 = nb01 / type_size_src0; int64_t str_a2 = nb02 / type_size_src0; @@ -3527,6 +3690,10 @@ inline bool ggml_sycl_supports_reorder_dmmv(enum ggml_type type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: return true; default: return false; @@ -4092,12 +4259,12 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { } static bool should_reorder_tensor(ggml_backend_sycl_context& ctx, const ggml_tensor * dst) { - return !g_ggml_sycl_disable_optimize && //allow optimize, controlled by $GGML_SYCL_DISABLE_OPT - ctx.opt_feature.reorder && //allow this device due to good perf, skip the devices with bad perf. - dst->op == GGML_OP_MUL_MAT && //limit to some supported cases of Q4_0, to do for more cases. - // ne[1] <= 8 so multi-column decode (spec / MTP verify) also bootstraps the reorder; - // all reorderable types have a _switch_ncols kernel. - dst->src[1]->ne[1] <= 8 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1; + return g_ggml_sycl_enable_optimize && //allow optimize, controlled by $GGML_SYCL_ENABLE_OPT + ctx.opt_feature.reorder && //allow this device due to good perf, skip the devices with bad perf. + dst->op == GGML_OP_MUL_MAT && //limit to some supported cases of Q4_0, to do for more cases. + // ne[1] <= 8 so multi-column decode (spec / MTP verify) also bootstraps the reorder; + // all reorderable types have a _switch_ncols kernel. + dst->src[1]->ne[1] <= 8 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1; } static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * src0, const ggml_tensor * /* src1 */, @@ -4136,7 +4303,7 @@ static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * // Lazily reorder supported MoE expert weights once their fused path is used. static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tensor * src0) { - if (g_ggml_sycl_disable_optimize || !ctx->opt_feature.reorder) { + if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) { return; } if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) { @@ -4604,6 +4771,11 @@ static void ggml_sycl_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * d ggml_sycl_op_im2col_3d(ctx, dst); } +static void ggml_sycl_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_col2im_1d(ctx, dst); +} + static void ggml_sycl_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); ggml_sycl_op_conv_3d(ctx, dst); @@ -4912,6 +5084,12 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_SOFT_MAX_BACK: ggml_sycl_op_soft_max_back(ctx, dst); break; + case GGML_OP_CROSS_ENTROPY_LOSS: + ggml_sycl_cross_entropy_loss(ctx, dst); + break; + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + ggml_sycl_cross_entropy_loss_back(ctx, dst); + break; case GGML_OP_ROPE: ggml_sycl_rope(ctx, dst); break; @@ -4924,6 +5102,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_IM2COL_3D: ggml_sycl_im2col_3d(ctx, dst); break; + case GGML_OP_COL2IM_1D: + ggml_sycl_col2im_1d(ctx, dst); + break; case GGML_OP_POOL_2D: ggml_sycl_pool2d(ctx, dst); break; @@ -5204,7 +5385,10 @@ static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_ auto * sycl_ctx = static_cast(backend->context); #ifdef GGML_SYCL_GRAPH - bool use_sycl_graph = !g_ggml_sycl_disable_graph && check_graph_compatibility(cgraph); + bool use_sycl_graph = false; + if (g_ggml_sycl_enable_graph) { + use_sycl_graph = check_graph_compatibility(cgraph); + } if (use_sycl_graph) { const bool graph_support = dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_limited_graph); if (!graph_support) { @@ -5470,7 +5654,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons // TODO: This specific configuration can fail with oneDNN and needs more debugging if (!ggml_is_permuted(a) && ggml_is_permuted(b) && b->ne[2] > 1 && b->ne[3] > 1 && a->ne[0] > 128 && a->ne[2] == 1 && src0_type == GGML_TYPE_F16) { - printf("zjy 2\n"); return false; } return true; @@ -5538,70 +5721,99 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons { ggml_type src0_type = op->src[0]->type; ggml_type src1_type = op->src[1]->type; - if (src0_type == src1_type && (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) && src0_type != GGML_TYPE_BF16) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) { - return true; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return true; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q8_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) { - return true; - } - if (src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) { - return true; - } - if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) { - return true; - } - if(src0_type == GGML_TYPE_Q8_0 && src1_type == GGML_TYPE_Q8_0) { - return true; + + if (src0_type == GGML_TYPE_F16) { + if (src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_Q5_0) { - return true; + + if (src0_type == GGML_TYPE_BF16) { + if (src1_type == GGML_TYPE_Q4_0 || //big error in ut + src1_type == GGML_TYPE_Q4_1 || //big error in ut + src1_type == GGML_TYPE_Q8_0 || //big error in ut + src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_Q5_1) { - return true; + + if (src0_type == GGML_TYPE_F32) { + if (src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_Q4_0) { - return true; + + if (src1_type == GGML_TYPE_F32) { + if (src0_type == GGML_TYPE_Q1_0 || + src0_type == GGML_TYPE_NVFP4 || + src0_type == GGML_TYPE_Q2_K || + src0_type == GGML_TYPE_Q3_K || + src0_type == GGML_TYPE_Q4_K || + src0_type == GGML_TYPE_Q5_K || + src0_type == GGML_TYPE_Q6_K || + src0_type == GGML_TYPE_IQ2_XXS || + src0_type == GGML_TYPE_IQ2_XS || + src0_type == GGML_TYPE_IQ2_S || + src0_type == GGML_TYPE_IQ3_XXS || + src0_type == GGML_TYPE_IQ1_S || + src0_type == GGML_TYPE_IQ1_M || + src0_type == GGML_TYPE_IQ3_S || + src0_type == GGML_TYPE_IQ4_NL || + src0_type == GGML_TYPE_IQ4_XS + ) { + return false; + } } - if(src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_Q4_1) { - return true; + + if (src0_type == src1_type) { + if (src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M) { + return false; + } } - return false; + + return true; } case GGML_OP_REPEAT_BACK: { @@ -5643,7 +5855,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SCALE: return true; case GGML_OP_CONT: - return op->src[0]->type != GGML_TYPE_BF16; + return true; case GGML_OP_TRI: { const ggml_tensor * src0 = op->src[0]; @@ -5666,6 +5878,14 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_IM2COL_3D: case GGML_OP_UPSCALE: return true; + case GGML_OP_COL2IM_1D: + return ggml_is_contiguous(op->src[0]) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 +#ifdef GGML_SYCL_HAS_BF16 + || op->type == GGML_TYPE_BF16 +#endif + ) && + op->src[0]->type == op->type; case GGML_OP_CONV_3D: return op->type == GGML_TYPE_F32 && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && @@ -5677,8 +5897,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_MEAN: return ggml_is_contiguous(op->src[0]); case GGML_OP_ARGSORT: - return op->src[0]->ne[0] * sizeof(int) <= - ggml_sycl_info().devices[device].smpbo; + return true; case GGML_OP_TOP_K: { const ggml_tensor * src0 = op->src[0]; const int k = op->ne[0]; @@ -5690,9 +5909,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons } case GGML_OP_POOL_2D: case GGML_OP_POOL_1D: - return true; case GGML_OP_ACC: - return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + return true; case GGML_OP_PAD: if (ggml_get_op_params_i32(op, 8) != 0) { return false; @@ -5725,6 +5943,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_FILL: case GGML_OP_CUMSUM: case GGML_OP_DIAG: + case GGML_OP_CROSS_ENTROPY_LOSS: + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: return true; case GGML_OP_SOLVE_TRI: return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K; diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 09fce1280ad..c4472e4bd66 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -2,8 +2,10 @@ #include "ggml-sycl/common.hpp" #include "ggml-sycl/presets.hpp" -static void norm_f32(const float* x, float* dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps, const sycl::nd_item<3>& item_ct1, sycl::float2* s_sum, int block_size) { +static void norm_f32(const float* x, float* dst, const int ncols, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, + const float eps, const sycl::nd_item<3>& item_ct1, sycl::float2* s_sum, int block_size) { const int nrows = item_ct1.get_group_range(2); const int nchannels = item_ct1.get_group_range(1); @@ -16,16 +18,16 @@ static void norm_f32(const float* x, float* dst, const int ncols, const int64_t const int tid = item_ct1.get_local_id(2); const int nwarps = nthreads / WARP_SIZE; - const auto strided_offset = calculate_offset<3>({stride_sample, stride_channel, stride_row}, {sample, channel, row}); - const auto packed_offset = calculate_offset<3>({nchannels * nrows * ncols, nrows * ncols, ncols}, {sample, channel, row}); + const auto src_offset = calculate_offset<3>({src_stride_sample, src_stride_channel, src_stride_row}, {sample, channel, row}); + const auto dst_offset = calculate_offset<3>({dst_stride_sample, dst_stride_channel, dst_stride_row}, {sample, channel, row}); - x += strided_offset; - dst += packed_offset; + x += src_offset; + dst += dst_offset; sycl::float2 mean_var = sycl::float2(0.f, 0.f); for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; + const float xi = x[col * src_stride_col]; mean_var.x() += xi; mean_var.y() += xi * xi; } @@ -54,7 +56,7 @@ static void norm_f32(const float* x, float* dst, const int ncols, const int64_t const float inv_std = sycl::rsqrt(var + eps); for (int col = tid; col < ncols; col += block_size) { - dst[col] = (x[col] - mean) * inv_std; + dst[col * dst_stride_col] = (x[col * src_stride_col] - mean) * inv_std; } } @@ -145,8 +147,10 @@ static void group_norm_f32(const float* x, float* dst, const int group_size, con } } -static void rms_norm_f32(const float* x, float* dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size) { +static void rms_norm_f32(const float* x, float* dst, const int ncols, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, + const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size) { const int nrows = item_ct1.get_group_range(2); const int nchannels = item_ct1.get_group_range(1); @@ -160,17 +164,17 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const int6 const int tid = item_ct1.get_local_id(2); const int nwarps = nthreads / WARP_SIZE; - const auto strided_offset = calculate_offset<3>({stride_sample, stride_channel, stride_row}, {sample, channel, row}); - const auto packed_offset = calculate_offset<3>({nchannels * nrows * ncols, nrows * ncols, ncols}, {sample, channel, row}); + const auto src_offset = calculate_offset<3>({src_stride_sample, src_stride_channel, src_stride_row}, {sample, channel, row}); + const auto dst_offset = calculate_offset<3>({dst_stride_sample, dst_stride_channel, dst_stride_row}, {sample, channel, row}); - x += strided_offset; - dst += packed_offset; + x += src_offset; + dst += dst_offset; float tmp = 0.0f; // partial sum for thread in warp for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; + const float xi = x[col * src_stride_col]; tmp += xi * xi; } @@ -198,14 +202,15 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const int6 const float scale = sycl::rsqrt(mean + eps); for (int col = tid; col < ncols; col += block_size) { - dst[col] = scale * x[col]; + dst[col * dst_stride_col] = scale * x[col * src_stride_col]; } } template static void l2_norm_f32(const float * x, float * dst, const int ncols, - const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, + const int64_t src_stride_sample, const int64_t dst_stride_col, const int64_t dst_stride_row, + const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, const int block_size) { const int nrows = item_ct1.get_group_range(2); const int nchannels = item_ct1.get_group_range(1); @@ -215,13 +220,13 @@ static void l2_norm_f32(const float * x, float * dst, const int ncols, const int sample = item_ct1.get_group(0); const int tid = item_ct1.get_local_id(2); - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; + x += sample*src_stride_sample + channel*src_stride_channel + row*src_stride_row; + dst += sample*dst_stride_sample + channel*dst_stride_channel + row*dst_stride_row; float tmp = 0.0f; // partial sum for thread in warp for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; + const float xi = x[col * src_stride_col]; tmp += xi * xi; } @@ -229,12 +234,13 @@ static void l2_norm_f32(const float * x, float * dst, const int ncols, const float scale = sycl::rsqrt(sycl::fmax(tmp, eps * eps)); for (int col = tid; col < ncols; col += block_size) { - dst[col] = scale * x[col]; + dst[col * dst_stride_col] = scale * x[col * src_stride_col]; } } static void norm_f32_sycl(const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, - const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, const float eps, queue_ptr stream, int device) { const sycl::range<3> global_dims(nsamples, nchannels, nrows); @@ -245,7 +251,10 @@ static void norm_f32_sycl(const float * x, float * dst, const int ncols, const i sycl::nd_range<3>(global_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, eps, item_ct1, nullptr, WARP_SIZE); + norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, nullptr, WARP_SIZE); }); }); } @@ -265,7 +274,10 @@ static void norm_f32_sycl(const float * x, float * dst, const int ncols, const i sycl::nd_range<3>(global_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); + norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); }); } @@ -319,7 +331,9 @@ static void group_norm_f32_sycl(const float* x, float* dst, } static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, const int nrows, const int nchannels, const int nsamples, - const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, queue_ptr stream, int device) { + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, + const float eps, queue_ptr stream, int device) { // printf("%s ncols=%d, nrows=%d, WARP_SIZE=%d\n", __func__, ncols, nrows, WARP_SIZE); const sycl::range<3> global_dims(nsamples, nchannels, nrows); @@ -330,7 +344,10 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, const sycl::nd_range<3>(global_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - rms_norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, eps, item_ct1, nullptr, WARP_SIZE); + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, nullptr, WARP_SIZE); }); }); } @@ -350,7 +367,10 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, const sycl::nd_range<3>(global_dims * block_dims, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - rms_norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); }); } @@ -363,9 +383,14 @@ static void l2_norm_f32_sycl(const float * x, const int nrows, const int nchannels, const int nsamples, - const int64_t stride_row, - const int64_t stride_channel, - const int64_t stride_sample, + const int64_t src_stride_col, + const int64_t src_stride_row, + const int64_t src_stride_channel, + const int64_t src_stride_sample, + const int64_t dst_stride_col, + const int64_t dst_stride_row, + const int64_t dst_stride_channel, + const int64_t dst_stride_sample, const float eps, queue_ptr stream, int device) { @@ -379,7 +404,10 @@ static void l2_norm_f32_sycl(const float * x, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { - l2_norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, eps, item_ct1, + l2_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, nullptr, warp_size); }); }); @@ -398,7 +426,9 @@ static void l2_norm_f32_sycl(const float * x, block_dims), [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { - l2_norm_f32(x, dst, ncols, stride_row, stride_channel, stride_sample, + l2_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size); }); }); @@ -421,12 +451,20 @@ void ggml_sycl_op_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { memcpy(&eps, dst->op_params, sizeof(float)); GGML_ASSERT(eps >= 0.0f); const size_t ts0 = ggml_type_size(src0->type); - GGML_ASSERT(nb00 == ts0); - const int64_t s01 = nb01 / ts0; - const int64_t s02 = nb02 / ts0; - const int64_t s03 = nb03 / ts0; - - norm_f32_sycl(src0_dd, dst_dd, ne00, ne01, ne02, ne03, s01, s02, s03, eps, main_stream, ctx.device); + const size_t tdst = ggml_type_size(dst->type); + GGML_ASSERT(nb00 % ts0 == 0 && nb01 % ts0 == 0 && nb02 % ts0 == 0 && nb03 % ts0 == 0); + GGML_ASSERT(nb0 % tdst == 0 && nb1 % tdst == 0 && nb2 % tdst == 0 && nb3 % tdst == 0); + const int64_t ss0 = nb00 / ts0; + const int64_t ss1 = nb01 / ts0; + const int64_t ss2 = nb02 / ts0; + const int64_t ss3 = nb03 / ts0; + const int64_t ds0 = nb0 / tdst; + const int64_t ds1 = nb1 / tdst; + const int64_t ds2 = nb2 / tdst; + const int64_t ds3 = nb3 / tdst; + + norm_f32_sycl(src0_dd, dst_dd, ne00, ne01, ne02, ne03, + ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, main_stream, ctx.device); } void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { @@ -465,11 +503,19 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_TENSOR_UNARY_OP_LOCALS const size_t ts0 = ggml_type_size(src0->type); - GGML_ASSERT(nb00 == ts0); - const int64_t s01 = nb01 / ts0; - const int64_t s02 = nb02 / ts0; - const int64_t s03 = nb03 / ts0; - rms_norm_f32_sycl(src0_dd, dst_dd, ne00, ne01, ne02, ne03, s01, s02, s03, eps, main_stream, ctx.device); + const size_t tdst = ggml_type_size(dst->type); + GGML_ASSERT(nb00 % ts0 == 0 && nb01 % ts0 == 0 && nb02 % ts0 == 0 && nb03 % ts0 == 0); + GGML_ASSERT(nb0 % tdst == 0 && nb1 % tdst == 0 && nb2 % tdst == 0 && nb3 % tdst == 0); + const int64_t ss0 = nb00 / ts0; + const int64_t ss1 = nb01 / ts0; + const int64_t ss2 = nb02 / ts0; + const int64_t ss3 = nb03 / ts0; + const int64_t ds0 = nb0 / tdst; + const int64_t ds1 = nb1 / tdst; + const int64_t ds2 = nb2 / tdst; + const int64_t ds3 = nb3 / tdst; + rms_norm_f32_sycl(src0_dd, dst_dd, ne00, ne01, ne02, ne03, + ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, main_stream, ctx.device); } void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -644,13 +690,21 @@ void ggml_sycl_op_l2_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { GGML_ASSERT(eps >= 0.0f); const size_t ts0 = ggml_type_size(src0->type); - GGML_ASSERT(nb00 == ts0); - const int64_t s01 = nb01 / ts0; - const int64_t s02 = nb02 / ts0; - const int64_t s03 = nb03 / ts0; + const size_t tdst = ggml_type_size(dst->type); + GGML_ASSERT(nb00 % ts0 == 0 && nb01 % ts0 == 0 && nb02 % ts0 == 0 && nb03 % ts0 == 0); + GGML_ASSERT(nb0 % tdst == 0 && nb1 % tdst == 0 && nb2 % tdst == 0 && nb3 % tdst == 0); + const int64_t ss0 = nb00 / ts0; + const int64_t ss1 = nb01 / ts0; + const int64_t ss2 = nb02 / ts0; + const int64_t ss3 = nb03 / ts0; + const int64_t ds0 = nb0 / tdst; + const int64_t ds1 = nb1 / tdst; + const int64_t ds2 = nb2 / tdst; + const int64_t ds3 = nb3 / tdst; /*support both WARP_SIZE or WARP_32_SIZE in code choose by hardware for better performance */ - l2_norm_f32_sycl(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, ctx.device); + l2_norm_f32_sycl(src0_d, dst_d, ne00, ne01, ne02, ne03, + ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, stream, ctx.device); } diff --git a/ggml/src/ggml-sycl/presets.hpp b/ggml/src/ggml-sycl/presets.hpp index 54566316ad1..502e3b61050 100644 --- a/ggml/src/ggml-sycl/presets.hpp +++ b/ggml/src/ggml-sycl/presets.hpp @@ -19,6 +19,7 @@ #define WARP_SIZE GGML_SYCL_WARP_SIZE #define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses +#define SYCL_COL2IM_1D_BLOCK_SIZE 256 #define SYCL_GELU_BLOCK_SIZE 256 #define SYCL_SILU_BLOCK_SIZE 256 #define SYCL_TANH_BLOCK_SIZE 256 @@ -62,7 +63,7 @@ #endif #ifndef K_QUANTS_PER_ITERATION -#define K_QUANTS_PER_ITERATION 2 +#define K_QUANTS_PER_ITERATION 1 #else static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUANTS_PER_ITERATION must be 1 or 2"); #endif diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5fbebc6d751..a483d22c1a2 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -129,7 +129,7 @@ typedef struct VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE { #endif #define ROUNDUP_POW2(M, N) (((M) + (N) - 1) & ~((N) - 1)) -#define CEIL_DIV(M, N) (((M) + (N)-1) / (N)) +#define CEIL_DIV(M, N) (((M) / (N)) + (((M) % (N)) != 0)) static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } #define VK_VENDOR_ID_AMD 0x1002 @@ -308,6 +308,7 @@ enum vk_device_architecture { AMD_RDNA1, AMD_RDNA2, AMD_RDNA3, + INTEL_XE1, INTEL_XE2, NVIDIA_PRE_TURING, NVIDIA_TURING, @@ -365,21 +366,26 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& const std::vector ext_props = device.enumerateDeviceExtensionProperties(); bool subgroup_size_control = false; + bool integer_dot_product = false; for (const auto& properties : ext_props) { if (strcmp("VK_EXT_subgroup_size_control", properties.extensionName) == 0) { subgroup_size_control = true; + } else if (strcmp("VK_KHR_shader_integer_dot_product", properties.extensionName) == 0) { + integer_dot_product = true; } } - if (!subgroup_size_control) { + if (!subgroup_size_control || !integer_dot_product) { return vk_device_architecture::OTHER; } vk::PhysicalDeviceProperties2 props2; vk::PhysicalDeviceSubgroupSizeControlPropertiesEXT subgroup_size_control_props; + vk::PhysicalDeviceShaderIntegerDotProductPropertiesKHR integer_dot_props; props2.pNext = &subgroup_size_control_props; + subgroup_size_control_props.pNext = &integer_dot_props; device.getProperties2(&props2); if (subgroup_size_control_props.minSubgroupSize == 16) { @@ -388,6 +394,9 @@ static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& // https://www.intel.com/content/www/us/en/content-details/824434/2024-intel-tech-tour-xe2-and-lunar-lake-s-gpu.html // https://www.intel.com/content/www/us/en/docs/oneapi/optimization-guide-gpu/2025-0/intel-xe-gpu-architecture.html return vk_device_architecture::INTEL_XE2; + } else if (subgroup_size_control_props.minSubgroupSize == 8 && + integer_dot_product && integer_dot_props.integerDotProduct4x8BitPackedSignedAccelerated) { + return vk_device_architecture::INTEL_XE1; } } else if (props.vendorID == VK_VENDOR_ID_NVIDIA) { const std::vector ext_props = device.enumerateDeviceExtensionProperties(); @@ -1898,6 +1907,38 @@ static bool vk_enable_sync_logger = false; static uint32_t vk_perf_logger_frequency = 1; static std::string vk_pipeline_stats_filter; +static uint64_t ggml_vk_get_node_flops(const ggml_tensor * node) { + if (node->op == GGML_OP_MUL_MAT || node->op == GGML_OP_MUL_MAT_ID) { + const uint64_t m = node->ne[0]; + const uint64_t n = node->ne[1]; + const uint64_t k = node->src[1]->ne[0]; + const uint64_t batch = node->ne[2] * node->ne[3]; + return m * n * (k + (k - 1)) * batch; + } + if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { + const ggml_tensor * knl = node->src[0]; + const uint64_t Cout = node->ne[2]; + const uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; + const uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; + return Cout * size_N * (size_K + (size_K - 1)); + } + if (node->op == GGML_OP_CONV_3D) { + const ggml_tensor * knl = node->src[0]; + const uint64_t OC = ggml_get_op_params_i32(node, 11); + const uint64_t IC = ggml_get_op_params_i32(node, 9); + const uint64_t size_K = IC * knl->ne[0] * knl->ne[1] * knl->ne[2]; + const uint64_t size_N = node->ne[3] / OC * node->ne[0] * node->ne[1] * node->ne[2]; + return OC * size_N * (size_K + (size_K - 1)); + } + if (node->op == GGML_OP_FLASH_ATTN_EXT) { + const ggml_tensor * q = node->src[0]; + const ggml_tensor * k = node->src[1]; + const ggml_tensor * v = node->src[2]; + return 2ull * q->ne[1] * q->ne[2] * (k->ne[0] + v->ne[0]) * k->ne[1] * q->ne[3]; + } + return 0; +} + class vk_perf_logger { public: void print_timings(bool force = false) { @@ -1946,7 +1987,7 @@ class vk_perf_logger { } std::string get_node_fusion_name(const ggml_tensor * node, const char *fusion_name, uint64_t *n_flops) { - *n_flops = 0; + *n_flops = ggml_vk_get_node_flops(node); std::string fusion_str; if (fusion_name) { fusion_str = fusion_name + std::string(" "); @@ -1973,35 +2014,22 @@ class vk_perf_logger { if (batch > 1) { name += " batch=" + std::to_string(batch); } - name = fusion_str + name; - *n_flops = m * n * (k + (k - 1)) * batch; - return name; + return fusion_str + name; } if (node->op == GGML_OP_CONV_2D || node->op == GGML_OP_CONV_TRANSPOSE_2D) { std::string name = ggml_op_name(node->op); - ggml_tensor * knl = node->src[0]; - uint64_t OW = node->ne[0]; - uint64_t OH = node->ne[1]; - uint64_t N = node->ne[3]; + const ggml_tensor * knl = node->src[0]; uint64_t Cout = node->ne[2]; - uint64_t KW = knl->ne[0]; - uint64_t KH = knl->ne[1]; - uint64_t Cin = node->src[1]->ne[2]; - // KxCRS @ CRSxNPQ = KxNPQ -> M=K, K=CRS, N=NPQ - uint64_t size_M = Cout; - uint64_t size_K = Cin * KW * KH; - uint64_t size_N = N * OW * OH; - *n_flops = size_M * size_N * (size_K + (size_K - 1)); - name += " M=Cout=" + std::to_string(size_M) + ", K=Cin*KW*KH=" + std::to_string(size_K) + + uint64_t size_K = node->src[1]->ne[2] * knl->ne[0] * knl->ne[1]; + uint64_t size_N = node->ne[3] * node->ne[0] * node->ne[1]; + name += " M=Cout=" + std::to_string(Cout) + ", K=Cin*KW*KH=" + std::to_string(size_K) + ", N=N*OW*OH=" + std::to_string(size_N); - name = fusion_str + name; - return name; + return fusion_str + name; } if (node->op == GGML_OP_RMS_NORM) { std::string name = ggml_op_name(node->op); name += "(" + std::to_string(node->ne[0]) + "," + std::to_string(node->ne[1]) + "," + std::to_string(node->ne[2]) + "," + std::to_string(node->ne[3]) + ")"; - name = fusion_str + name; - return name; + return fusion_str + name; } if (node->op == GGML_OP_FLASH_ATTN_EXT) { const ggml_tensor * dst = node; @@ -2017,7 +2045,6 @@ class vk_perf_logger { " k(" << k->ne[0] << "," << k->ne[1] << "," << k->ne[2] << "," << k->ne[3] << "), " << " v(" << v->ne[0] << "," << v->ne[1] << "," << v->ne[2] << "," << v->ne[3] << "), " << " m(" << (m?m->ne[0]:0) << "," << (m?m->ne[1]:0) << "," << (m?m->ne[2]:0) << "," << (m?m->ne[3]:0) << ")"; - *n_flops = 2ull * q->ne[1] * q->ne[2] * (k->ne[0] + v->ne[0]) * k->ne[1] * q->ne[3]; return name.str(); } if (node->op == GGML_OP_TOP_K) { @@ -2081,7 +2108,7 @@ struct ggml_backend_vk_context { bool do_add_rms_partials_offset_calculation; bool do_add_rms_partials; - uint64_t last_total_mul_mat_bytes {}; + uint64_t last_total_flops {UINT64_MAX}; // Cache most recent tensor that was converted into prealloc_y, and what pipeline it used to convert. vk_pipeline_struct * prealloc_y_last_pipeline_used {}; @@ -2448,6 +2475,85 @@ static bool ggml_vk_strip_decode_vector(const uint32_t * code, size_t word_count return true; } +// Remove the loop unrolling hint of the matmul shader's BK loop +// and replace it with the dont_unroll hint for better performance on +// hardware like Apple M1/M2. +// Assumes 1. code comes from mul_mm.comp 2. the K-tile loop has no loop +// control hint and 3. the BK loop is the last loop nested directly inside +// the K-tile loop. +// Returns true when the input was modified; returns false otherwise +// without touching `out`. +static bool ggml_vk_roll_bk_loop(const uint32_t * code, size_t word_count, std::vector & out) { + if (word_count < 5) { + return false; + } + + struct vk_spv_loop { + size_t header; + size_t end; + uint32_t control; + }; + + std::vector loops; + + // Collect a list of all loops in the module. + for (size_t pos = 5; pos < word_count; ) { + const uint32_t wc = code[pos] >> spv::WordCountShift; + const uint32_t op = code[pos] & spv::OpCodeMask; + if (wc == 0 || pos + wc > word_count) { + return false; + } + + if (op == spv::OpLoopMerge && wc >= 4) { loops.push_back({ pos, 0, code[pos + 3] }); } + + if (op == spv::OpLabel && wc >= 2) { + for (auto & l : loops) { + if (l.end == 0 && code[l.header + 1] == code[pos + 1]) { l.end = pos; } + } + } + + pos += wc; + } + + auto encloses = [](const vk_spv_loop & a, const vk_spv_loop & b) { + return a.header < b.header && b.header < a.end; + }; + + // Find the BK loop. + const vk_spv_loop * bk = nullptr; + for (const auto & h : loops) { + if (h.control != spv::LoopControlUnrollMask) { + continue; + } + const vk_spv_loop * parent = nullptr; + bool has_child = false; + for (const auto & g : loops) { + if (encloses(g, h) && (!parent || g.header > parent->header)) { + parent = &g; + } + if (encloses(h, g)) { + has_child = true; + } + } + // BK loop should be the last loop nested inside the loop with no hint + // and have at least one child loop. + if (parent && + parent->control == spv::LoopControlMaskNone && + has_child && + (!bk || h.header > bk->header)) { + bk = &h; + } + } + if (!bk) { + return false; + } + + // set DontUnroll instead of Unroll + out.assign(code, code + word_count); + out[bk->header + 3] = spv::LoopControlDontUnrollMask; + return true; +} + static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipeline, size_t spv_size, const void* spv_data, const std::string entrypoint, uint32_t parameter_count, std::array wg_denoms, std::vector specialization_constants, bool disable_robustness, bool require_full_subgroups, uint32_t required_subgroup_size) { @@ -2531,6 +2637,22 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin } #endif +#if VK_HEADER_VERSION >= 287 + // Roll the mul_mm BK loop on Asahi Linux. Skip bf16 and the mul_mmq pipelines. + if (device->driver_id == vk::DriverId::eMesaHoneykrisp && + pipeline->name.rfind("matmul", 0) == 0 && + pipeline->name.find("bf16") == std::string::npos && + pipeline->name.find("q8_1") == std::string::npos) { + const uint32_t * src = spirv.empty() ? reinterpret_cast(spv_data) : spirv.data(); + size_t src_n = spirv.empty() ? spv_size / sizeof(uint32_t) : spirv.size(); + std::vector rolled; + if (ggml_vk_roll_bk_loop(src, src_n, rolled)) { + spirv = std::move(rolled); + shader_module_create_info = vk::ShaderModuleCreateInfo({}, spirv.size() * sizeof(uint32_t), spirv.data()); + } + } +#endif + pipeline->shader_module = device->device.createShaderModule(shader_module_create_info); vk::PushConstantRange pcr( @@ -3837,7 +3959,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { l_warptile = { 256, 128, 128, 16, subgroup_size_8, 64, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; l_warptile_mmq = l_warptile_mmq_int = { 256, 128, 128, 32, subgroup_size_8, 64, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; l_warptile_mmq_int_k = { 256, 128, 128, 32, subgroup_size_16, 64, 1, 4, 2, 1, subgroup_size_16 }; - } else if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support && device->architecture == INTEL_XE2) { + } else if (device->vendor_id == VK_VENDOR_ID_INTEL && device->coopmat_support) { // Xe2/Xe3 with coopmat enabled - warptile performance tuning l_warptile = { 512, 128, 128, 16, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; @@ -4710,7 +4832,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } uint32_t rm_iq = 2 * rm_kq; - const bool use_subgroups = device->subgroup_arithmetic && device->architecture != vk_device_architecture::AMD_GCN; + const bool use_subgroups = device->subgroup_arithmetic; // Ensure a subgroup size >= 16 is available const bool use_subgroups16 = use_subgroups && subgroup_min_size_16; @@ -6361,9 +6483,8 @@ static vk_device ggml_vk_get_device(size_t idx) { break; case VK_VENDOR_ID_INTEL: { // Current Windows driver does not expose BF16 support. - // We only want to use l_warptile if coopmat is available and is Xe2+ - const bool xe2_with_coopmat = device->coopmat_support && device->architecture == INTEL_XE2; - const bool use_l_warptile = (i == GGML_TYPE_BF16) ? (device->coopmat_bf16_support && xe2_with_coopmat) : xe2_with_coopmat; + // We only want to use l_warptile if coopmat is available + const bool use_l_warptile = (i == GGML_TYPE_BF16) ? (device->coopmat_bf16_support && device->coopmat_support) : device->coopmat_support; device->mul_mat_l[i] = use_l_warptile; device->mul_mat_id_l[i] = use_l_warptile; device->mul_mat_m[i] = true; @@ -10189,7 +10310,8 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx } // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. - bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16; + bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 + && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, mask != nullptr, use_mask_opt, logit_softcap != 0, k->type, v->type); @@ -16180,22 +16302,34 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } // Submit after enough work has accumulated, to overlap CPU cmdbuffer generation with GPU execution. - // Estimate the amount of matmul work by looking at the weight matrix size, and submit every 100MB - // (and scaled down based on model size, so smaller models submit earlier). - int submitted_nodes = 0; - int submit_count = 0; - uint64_t mul_mat_bytes = 0; - uint64_t total_mul_mat_bytes = 0; - uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u); + // Estimate the amount of compute work using flops, and submit every 200 GFLOP + // (and scaled down based on total graph flops, so smaller models submit earlier). + // Also submit at least every 100 nodes, in case there are workloads without heavy compute. + uint32_t submitted_nodes = 0; + uint32_t submit_count = 0; + uint64_t batch_flops = 0; + uint64_t total_flops = 0; + uint64_t flops_cap = 200'000'000'000ULL; + + // On weaker AMD GPUs larger submissions can hit a driver timeout, submit more often to avoid this + if (ctx->device->vendor_id == VK_VENDOR_ID_AMD && ctx->device->shader_core_count > 0) { + if (ctx->device->architecture == AMD_GCN && ctx->device->shader_core_count < 32) { + flops_cap = 500'000'000ULL * ctx->device->shader_core_count; + } else if (ctx->device->architecture != AMD_GCN && ctx->device->shader_core_count < 24) { + flops_cap = 2'000'000'000ULL * ctx->device->shader_core_count; + } + } + uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u); + for (int i = 0; i < cgraph->n_nodes; i++) { if (first_node_in_batch) { submit_node_idx = i; } - if (cgraph->nodes[i]->op == GGML_OP_MUL_MAT || cgraph->nodes[i]->op == GGML_OP_MUL_MAT_ID) { - auto bytes = ggml_nbytes(cgraph->nodes[i]->src[0]); - mul_mat_bytes += bytes; - total_mul_mat_bytes += bytes; + { + auto node_flops = ggml_vk_get_node_flops(cgraph->nodes[i]); + batch_flops += node_flops; + total_flops += node_flops; } // op_srcs_fused_elementwise indicates whether an op's srcs all contribute to @@ -16407,8 +16541,8 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // Signal the almost_ready fence when the graph is mostly complete (< 20% remaining) bool almost_ready = (cgraph->n_nodes - i) < cgraph->n_nodes / 5; - bool submit = ((uint32_t)submitted_nodes >= ctx->device->max_nodes_per_submit) || - (mul_mat_bytes_per_submit != 0 && mul_mat_bytes >= mul_mat_bytes_per_submit) || + bool submit = (submitted_nodes >= ctx->device->max_nodes_per_submit) || + (flops_per_submit != 0 && batch_flops >= flops_per_submit) || (i + ctx->num_additional_fused_ops >= last_node) || (almost_ready && !ctx->almost_ready_fence_pending); @@ -16442,9 +16576,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg if (submit && enqueued) { first_node_in_batch = true; submitted_nodes = 0; - mul_mat_bytes = 0; + batch_flops = 0; if (submit_count < 3) { - mul_mat_bytes_per_submit *= 2; + flops_per_submit *= 2; } submit_count++; } @@ -16453,7 +16587,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg ctx->fused_ops_write_mask = 0; } - ctx->last_total_mul_mat_bytes = total_mul_mat_bytes; + ctx->last_total_flops = total_flops; if (vk_perf_logger_enabled) { // End the command buffer and submit/wait @@ -17248,21 +17382,24 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_SET_ROWS: { - switch (op->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_Q1_0: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - return true; - default: - return false; + if (op->src[0]->type == GGML_TYPE_F32) { + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + return true; + default: + return false; + } } + return false; } case GGML_OP_CONT: case GGML_OP_CPY: @@ -17890,9 +18027,9 @@ static bool ggml_vk_device_is_supported(const vk::PhysicalDevice & vkdev) { static bool ggml_vk_khr_cooperative_matrix_support(const vk::PhysicalDeviceProperties& props, const vk::PhysicalDeviceDriverProperties& driver_props, vk_device_architecture arch) { switch (props.vendorID) { case VK_VENDOR_ID_INTEL: - // Only allowing Xe2 GPU at the moment since Xe2 GPU can gain significant performance boost, - // while some older hardware (ex. Arc A770) has performance regressions - return arch == vk_device_architecture::INTEL_XE2; + // Only allowing Xe2/Xe3 GPU and integrated Xe GPUs at the moment since older hardware (ex. Arc A770) has performance regressions. + return (arch == vk_device_architecture::INTEL_XE2) || + (arch == vk_device_architecture::INTEL_XE1 && props.deviceType == vk::PhysicalDeviceType::eIntegratedGpu && driver_props.driverID == vk::DriverId::eIntelProprietaryWindows); case VK_VENDOR_ID_AMD: if (driver_props.driverID == vk::DriverId::eAmdProprietary || driver_props.driverID == vk::DriverId::eAmdOpenSource) { // Workaround for AMD proprietary driver reporting support on all GPUs @@ -17940,6 +18077,8 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) case 0xE20B: // B580 case 0xE211: // Pro B60 return 20; + case 0xB080: // PTL Xe3 LPG 2x6 (12 subslices) + return 12; default: return 0; } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp index fd84c3c91d8..7bbee577fb7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp @@ -28,13 +28,10 @@ vec2 cache_b_ds; #include "mul_mat_vecq_funcs.glsl" -void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i) { +void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint col, const uint b_qs_idx) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - const uint col = i*BLOCK_SIZE + tid*K_PER_ITER; - // Preload data_b block const uint b_block_idx = (j*p.batch_stride_b + col) / QUANT_K_Q8_1 + b_offset; - const uint b_qs_idx = tid % (32 / K_PER_ITER); const uint b_block_idx_outer = b_block_idx / 4; const uint b_block_idx_inner = b_block_idx % 4; cache_b_ds = vec2(data_b[b_block_idx_outer].ds[b_block_idx_inner]); @@ -91,35 +88,35 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } } - uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE); - if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) { + const uint col_stride = K_PER_ITER * BLOCK_SIZE; + uint num_iters = p.ncols / col_stride; + if (num_iters * col_stride + K_PER_ITER * tid < p.ncols) { num_iters++; } - int unroll_count = 4; - uint unrolled_iters = num_iters & ~(unroll_count - 1); - uint i = 0; - while (i < unrolled_iters) { + const uint b_qs_idx = tid % (32 / K_PER_ITER); + uint col = tid * K_PER_ITER; + while (num_iters >= 4) { // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; + [[unroll]] for (uint k = 0; k < 4; ++k) { + iter(temp, first_row, num_rows, col, b_qs_idx); + col += col_stride; } - } - unroll_count = 2; - unrolled_iters = num_iters & ~(unroll_count - 1); + num_iters -= 4; + } - while (i < unrolled_iters) { + if (num_iters >= 2) { // Manually partially unroll the loop - [[unroll]] for (uint k = 0; k < unroll_count; ++k) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; - } + iter(temp, first_row, num_rows, col, b_qs_idx); + col += col_stride; + iter(temp, first_row, num_rows, col, b_qs_idx); + col += col_stride; + num_iters -= 2; } - while (i < num_iters) { - iter(temp, first_row, num_rows, tid, i*K_PER_ITER); - i++; + + if (num_iters > 0) { + iter(temp, first_row, num_rows, col, b_qs_idx); } reduce_result(temp, d_offset, first_row, num_rows, tid); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp index c62bce82555..5ee5275d278 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp @@ -42,7 +42,7 @@ float op_leaky_relu(float x) { } float op_step(float x) { - return x >= 0.0f ? 1.0f : 0.0f; + return x > 0.0f ? 1.0f : 0.0f; } float op_tanh(float x) { diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index c00a2e9ee9b..d7692363a1d 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -1563,6 +1563,7 @@ class ggml_webgpu_shader_lib { case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: { // Quantized types using u32 buffers for portability. defines.push_back("SRC_TYPE=u32"); @@ -1593,6 +1594,8 @@ class ggml_webgpu_shader_lib { } else if ((key.src_type >= GGML_TYPE_Q4_0 && key.src_type <= GGML_TYPE_Q8_1) || key.src_type == GGML_TYPE_IQ4_NL || key.src_type == GGML_TYPE_MXFP4) { defines.push_back("BLOCK_SIZE=32u"); + } else if (key.src_type == GGML_TYPE_NVFP4) { + defines.push_back("BLOCK_SIZE=64u"); } else if (key.src_type >= GGML_TYPE_Q2_K) { defines.push_back("BLOCK_SIZE=256u"); } else { @@ -1960,6 +1963,7 @@ class ggml_webgpu_shader_lib { defines.push_back(type_upper + "_TABLES"); break; case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: defines.push_back(type_upper + "_LUT"); break; default: @@ -2103,6 +2107,7 @@ class ggml_webgpu_shader_lib { defines.push_back(type_upper + "_TABLES"); break; case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: defines.push_back(type_upper + "_LUT"); break; default: @@ -2274,6 +2279,7 @@ class ggml_webgpu_shader_lib { defines.push_back(type_upper + "_TABLES"); break; case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: defines.push_back(type_upper + "_LUT"); break; default: @@ -2394,6 +2400,7 @@ class ggml_webgpu_shader_lib { defines.push_back(type_upper + "_TABLES"); break; case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: defines.push_back(type_upper + "_LUT"); break; default: @@ -2814,23 +2821,16 @@ class ggml_webgpu_shader_lib { variant.resize(variant.size() - (sizeof("_mask") - 1)); variant += "_mask_blk"; } - uint32_t vec_ne = 1u; - if (key.common.k_type == GGML_TYPE_F16 && key.common.v_type == GGML_TYPE_F16 && - key.common.head_dim_qk == key.common.head_dim_v) { - switch (key.common.head_dim_qk) { - case 64: - case 192: - case 576: - vec_ne = 2u; - break; - case 96: - vec_ne = 4u; - break; - default: - break; - } + + uint32_t d_split = context.min_subgroup_size; + if (key.common.k_type == GGML_TYPE_F16 && key.common.v_type == GGML_TYPE_F16) { + const uint32_t D = key.common.head_dim_qk | key.common.head_dim_v; + const uint32_t D_lsb = D & (~(D - 1u)); + d_split = std::min(std::min(context.min_subgroup_size, 4u), std::max(D_lsb / 4u, 1u)); } - defines.push_back(std::string("VEC_NE=") + std::to_string(vec_ne) + "u"); + + defines.push_back(std::string("D_SPLIT=") + std::to_string(d_split)); + variant += "_dsplit" + std::to_string(d_split); auto pipeline_decisions = std::make_shared(decisions); webgpu_pipeline pipeline = diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index f0ec18abd9a..29025e9ba4e 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4056,6 +4056,7 @@ static bool ggml_webgpu_supported_qtype(ggml_type type) { case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: return true; default: return false; @@ -4156,6 +4157,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: supports_op = true; break; default: @@ -4196,6 +4198,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: supports_op = true; break; default: diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl index 758efa17d77..6634fbd6578 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl @@ -896,9 +896,23 @@ const kvalues_iq4nl = array( #endif -#ifdef MXFP4_LUT +#if defined(MXFP4_LUT) || defined(NVFP4_LUT) const kvalues_mxfp4 = array( 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12 ); -#endif +#endif // MXFP4_LUT || NVFP4_LUT +#ifdef NVFP4_LUT +fn ue4m3_to_fp32(u: u32) -> f32 { + if (u == 0u || u == 127u) { + return 0.0; + } + let exp = (u >> 3u) & 15u; + let man = u & 7u; + if (exp == 0u) { + return f32(man) * (1.0 / 512.0); + } + let bits = ((exp + 120u) << 23u) | (man << 20u); + return bitcast(bits); +} +#endif // NVFP4_LUT diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl index 30ed97cca0c..d5127624196 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl @@ -39,9 +39,6 @@ enable subgroups; #define KV_GRANULARITY 8 #define KV_TILE 16 #define WG_SIZE 64 -#ifndef VEC_NE -#define VEC_NE 4u -#endif #define KV_BLOCKS (KV_TILE / KV_GRANULARITY) @@ -367,11 +364,11 @@ fn main(@builtin(workgroup_id) wg_id: vec3, // accumulate q block * k block into registers across the entire KV tile if (!skip_tile) { - let num_of_threads = subgroup_size / VEC_NE; + let num_of_threads:u32 = D_SPLIT; let tx = sg_inv_id % num_of_threads; let ty = sg_inv_id / num_of_threads; if (subgroup_id == 0u && q_row_start < params.seq_len_q) { - for (var kv_base : u32 = 0u; kv_base < KV_TILE; kv_base += VEC_NE) { + for (var kv_base : u32 = 0u; kv_base < KV_TILE; kv_base += subgroup_size / D_SPLIT) { let kv_idx = kv_base + ty; var partial_sum: f32 = 0.0; let kv_valid = kv_idx < KV_TILE && (kv_tile + kv_idx) < params.seq_len_kv; @@ -486,15 +483,18 @@ fn main(@builtin(workgroup_id) wg_id: vec3, if (!skip_tile) { // we have P (KV_TILE) in inter_shmem and V (KV_TILE x head_dim_v) in kv_shmem // we want to compute O += P * V across the full KV tile - let ne_threads : u32 = VEC_NE; + let ne_threads : u32 = subgroup_size / D_SPLIT; let nl_threads = max(1u, subgroup_size / ne_threads); let tx_pv = sg_inv_id % nl_threads; let ty_pv = sg_inv_id / nl_threads; if (subgroup_id == 0u && q_row_start < params.seq_len_q) { for (var vec_col = tx_pv; vec_col < (HEAD_DIM_V / 4u); vec_col += nl_threads) { var lo = vec4(0.0, 0.0, 0.0, 0.0); - for (var cc = 0u; cc < KV_TILE / ne_threads; cc += 1u) { + for (var cc = 0u; cc * ne_threads < KV_TILE; cc += 1u) { let kv_idx = cc * ne_threads + ty_pv; + if (kv_idx >= KV_TILE) { + continue; + } let v_row = kv_tile + kv_idx; if (v_row >= params.seq_len_kv) { continue; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl index 78d61a93d28..487edb3275f 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl @@ -672,6 +672,27 @@ fn copy_elements(src_base: u32, dst_base: u32, offset: u32) { } #endif +#ifdef NVFP4 +fn copy_elements(src_base: u32, dst_base: u32, offset: u32) { + let block_byte_base = (src_base + offset) * 36; + let d_word = load_u32_at_src(block_byte_base); + for (var sub: u32 = 0u; sub < 4; sub++) { + let d = ue4m3_to_fp32(get_byte(d_word, sub)) * 0.5; + for (var j: u32 = 0u; j < 2; j++) { + let q_packed = load_u32_at_src(block_byte_base + 4 + sub * 8 + j * 4); + for (var k: u32 = 0; k < 4; k++) { + let q_byte = get_byte(q_packed, k); + let q_lo = f32(kvalues_mxfp4[q_byte & 0xFu]) * d; + let q_hi = f32(kvalues_mxfp4[(q_byte >> 4) & 0xF]) * d; + let dst_offset = dst_base + offset * 64 + sub * 16 + j * 4 + k; + dst[dst_offset] = q_lo; + dst[dst_offset + 8u] = q_hi; + } + } + } +} +#endif + @group(0) @binding(0) var src: array; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl index 6a2eb8c824e..13996ab5157 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl @@ -241,7 +241,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 #endif // INIT_SRC0_SHMEM_Q8_1 #if defined(INIT_SRC0_SHMEM_MXFP4) - let block_byte_base = src0_idx * 17u; + let block_byte_base = src0_idx * 17u; // BLOCK_SIZE_BYTES = 17u; let eu8 = get_byte(load_u32_at_src0_aligned(block_byte_base), block_byte_base & 3u); let e = ldexp(1.0, i32(eu8) - 128); @@ -263,6 +263,47 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 } #endif // legacy-quants +#if defined(INIT_SRC0_SHMEM_NVFP4) +const BLOCK_SIZE = 64u; +const BLOCK_SIZE_BYTES = 36u; +const SUB_BLOCK_SIZE = 16u; // elements sharing one UE4M3 scale +const NQ = 16u; +const BYTES_PER_THREAD = 8u; +const BYTES_PER_INNER_LOOP = 4u; + +fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) { + for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) { + let tile_m = i / TILE_K; + let tile_k_start = i % TILE_K; + let global_m = offset_m + tile_m; + let global_k_start = k_outer + tile_k_start; + + if (global_m >= params.m) { + break; + } + + let block_k = global_k_start / BLOCK_SIZE; + let sub_block = (global_k_start % BLOCK_SIZE) / SUB_BLOCK_SIZE; + let src0_idx = batch_offset + global_m * params.stride_01 + block_k; + + let block_byte_base = src0_idx * BLOCK_SIZE_BYTES; + let d_byte_base = block_byte_base; + let qs_byte_base = block_byte_base + 4u; + + let d = ue4m3_to_fp32(get_byte(load_u32_at_src0_aligned(d_byte_base), sub_block)) * 0.5; + + for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j++) { + let q_packed = load_u32_at_src0_aligned(qs_byte_base + sub_block * 8u + j * 4u); + for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) { + let q_byte = get_byte(q_packed, k); + shmem[i + j * BYTES_PER_INNER_LOOP + k] = f16(f32(kvalues_mxfp4[q_byte & 0xF]) * d); + shmem[i + j * BYTES_PER_INNER_LOOP + k + 8u] = f16(f32(kvalues_mxfp4[(q_byte >> 4) & 0xF]) * d); + } + } + } +} +#endif // INIT_SRC0_SHMEM_NVFP4 + // k-quants #if defined(INIT_SRC0_SHMEM_Q2_K) || defined(INIT_SRC0_SHMEM_Q3_K) || defined(INIT_SRC0_SHMEM_Q4_K) || defined(INIT_SRC0_SHMEM_Q5_K) || defined(INIT_SRC0_SHMEM_Q6_K) const BLOCK_SIZE = 256u; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl index b0703fe9062..8fd0d1907cf 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl @@ -1505,3 +1505,49 @@ fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src return acc; } #endif + +#ifdef MUL_ACC_NVFP4 +#define BLOCK_SIZE 64 +#define BLOCK_SIZE_BYTES 36 +#define THREADS_PER_BLOCK 4 +#define ELEMS_PER_THREAD (BLOCK_SIZE/THREADS_PER_BLOCK) +fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src1_idx_base: u32) -> array, NUM_COLS> { + var acc: array, NUM_COLS>; + + let num_blocks = params.k / BLOCK_SIZE; + let sub = thread_id % THREADS_PER_BLOCK; + for (var block = thread_id/THREADS_PER_BLOCK; block < num_blocks; block += WG_SIZE/THREADS_PER_BLOCK) { + let x_base = src1_idx_base + block * BLOCK_SIZE + sub * ELEMS_PER_THREAD; + var x_block: array, NUM_COLS>; + for (var col = 0u; col < NUM_COLS;col += 1) { + for (var i = 0u; i < ELEMS_PER_THREAD / 2; i++) { + x_block[col][i] = f32(src1[x_base + col * params.stride_11 + i]); + x_block[col][i + 8] = f32(src1[x_base + col * params.stride_11 + i + 8]); + } + } + for (var row = 0u; row < OUTPUTS_PER_WG; row++) { + let output_row = row_base + row; + if (output_row < params.m) { + let block_byte_base = (src0_batch_offset + output_row * params.stride_01 + block) * BLOCK_SIZE_BYTES; + let d = ue4m3_to_fp32(get_byte(load_u32_at_src0_aligned(block_byte_base), sub)) * 0.5; + let q_w0 = load_u32_at_src0_aligned(block_byte_base + 4u + 8u * sub); + let q_w1 = load_u32_at_src0_aligned(block_byte_base + 8u + 8u * sub); + for (var col = 0u;col < NUM_COLS;col += 1) { + var row_sum = 0.0; + for (var l = 0u; l < 8u; l++) { + let q_word = select(q_w0, q_w1, l >= 4u); + let q_byte = get_byte(q_word, l % 4u); + let q_lo = f32(kvalues_mxfp4[q_byte & 0xFu]) * d; + let q_hi = f32(kvalues_mxfp4[(q_byte >> 4u) & 0xFu]) * d; + row_sum += q_lo * x_block[col][l]; + row_sum += q_hi * x_block[col][l + 8u]; + } + acc[col][row] += row_sum; + } + } + } + } + + return acc; +} +#endif diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 0f682fd1856..de0321d9ffd 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -525,7 +525,11 @@ const char * ggml_commit(void) { #if defined(_MSC_VER) || defined(__MINGW32__) static int64_t timer_freq, timer_start; -void ggml_time_init(void) { +static BOOL CALLBACK ggml_time_init_once(PINIT_ONCE once, PVOID param, PVOID *ctx) { + UNUSED(once); + UNUSED(param); + UNUSED(ctx); + LARGE_INTEGER t; QueryPerformanceFrequency(&t); timer_freq = t.QuadPart; @@ -535,6 +539,12 @@ void ggml_time_init(void) { // We subtract the program start time to reduce the likelihood of that happening. QueryPerformanceCounter(&t); timer_start = t.QuadPart; + + return TRUE; +} +void ggml_time_init(void) { + static INIT_ONCE once = INIT_ONCE_STATIC_INIT; + InitOnceExecuteOnce(&once, ggml_time_init_once, NULL, NULL); } int64_t ggml_time_ms(void) { LARGE_INTEGER t; @@ -671,6 +681,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_q1_0, .from_float_ref = (ggml_from_float_t) quantize_row_q1_0_ref, }, + [GGML_TYPE_Q2_0] = { + .type_name = "q2_0", + .blck_size = QK2_0, + .type_size = sizeof(block_q2_0), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q2_0, + .from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref, + }, [GGML_TYPE_Q4_0] = { .type_name = "q4_0", .blck_size = QK4_0, @@ -1407,6 +1425,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break; case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break; case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break; + case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break; case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break; case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break; case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break; @@ -3907,7 +3926,7 @@ struct ggml_tensor * ggml_set_rows( GGML_ASSERT(b->ne[2] % c->ne[1] == 0); GGML_ASSERT(b->ne[3] % c->ne[2] == 0); GGML_ASSERT(c->ne[3] == 1); - GGML_ASSERT(b->type == GGML_TYPE_F32); + GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16); GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32); GGML_ASSERT(ggml_is_contiguous_rows(a)); @@ -7409,6 +7428,10 @@ static int ggml_node_list_find_tensor(const struct ggml_cgraph * cgraph, return -1; } +static bool ggml_is_constant(const struct ggml_tensor * tensor) { + return tensor->buffer != NULL && ggml_backend_buffer_get_usage(tensor->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && (tensor->flags & GGML_TENSOR_FLAG_PARAM) == 0; +} + bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph, const int * node_idxs, int count, @@ -7454,10 +7477,11 @@ bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph, return false; } - // if node is a view, check if the view_src and all it's parent view_srcs are within the subgraph + // if node is a view, check if the view_src and all its parent view_srcs are within the subgraph. + // external view sources are allowed only for weight tensors, which are constant for this graph execution. struct ggml_tensor * view_src = node->view_src; while (view_src) { - if (ggml_node_list_find_tensor(cgraph, node_idxs, count, view_src) == -1) { + if (ggml_node_list_find_tensor(cgraph, node_idxs, count, view_src) == -1 && !ggml_is_constant(view_src)) { return false; } view_src = view_src->view_src; @@ -7729,6 +7753,7 @@ size_t ggml_quantize_chunk( switch (type) { case GGML_TYPE_Q1_0: result = quantize_q1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q2_0: result = quantize_q2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_0: result = quantize_q4_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q4_1: result = quantize_q4_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q5_0: result = quantize_q5_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 27bab1a8ea6..b841bbc7bfe 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -eced84c86f8b012c752c016f7fe789adea168e1e +524f974bb21a1013408f76d71c15732482c0c3fe