diff --git a/.planning/architecture/kernel_aarch64.md b/.planning/architecture/kernel_aarch64.md index 88f5b913..5e5822fb 100644 --- a/.planning/architecture/kernel_aarch64.md +++ b/.planning/architecture/kernel_aarch64.md @@ -86,13 +86,18 @@ stateDiagram-v2 ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8_] / exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4_] / exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_] / exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8_] / exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4_] / exec_simd_q4_vector_packed_q8_rhs_bl4_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_] / exec_simd_q4_vector_packed_q8_rhs_bl4_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4_] / exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_q8_rhs_] / exec_simd_q4_vector_q8_rhs_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_f32_rhs_] / exec_simd_q4_vector_f32_rhs_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm_] / exec_simd_q6_vector_q8_argmax_prepared_i8mm_op_mul_mat_argmax_ + ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_] / exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_argmax_ + ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_] / exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs_] / exec_simd_q6_vector_packed_q8_rhs_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_] / exec_simd_q6_vector_prepared_q8_rhs_op_mul_mat_ @@ -150,8 +155,9 @@ stateDiagram-v2 ready --> ready : dispatch_op_soft_max [dispatch_op_soft_max__] / dispatch_op_soft_max__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ - ready --> ready : dispatch_op_rope [rope_mode_is_false___] / mark_done_op__false__ - ready --> ready : dispatch_op_rope [rope_mode_is_true___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_mode_is_0___] / mark_done_op__false__ + ready --> ready : dispatch_op_rope [rope_mode_is_2___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_timestep_is__] / mark_done_op__ ready --> ready : dispatch_op_rope [dispatch_op_rope__] / dispatch_op_rope__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ @@ -232,14 +238,15 @@ stateDiagram-v2 ready --> ready : dispatch_op_unary [abs___] / abs__ ready --> ready : dispatch_op_unary [neg___] / neg__ ready --> ready : dispatch_op_unary [relu___] / relu__ - ready --> ready : dispatch_op_unary [abs___] / abs__ - ready --> ready : dispatch_op_unary [neg___] / neg__ - ready --> ready : dispatch_op_unary [relu___] / relu__ + ready --> ready : dispatch_op_unary [silu___] / silu__ + ready --> ready : dispatch_op_unary [abs__] / abs__ + ready --> ready : dispatch_op_unary [neg__] / neg__ + ready --> ready : dispatch_op_unary [relu__] / relu__ ready --> ready : dispatch_op_unary [exp___] / exp__ ready --> ready : dispatch_op_unary [tanh___] / tanh__ ready --> ready : dispatch_op_unary [elu___] / elu__ ready --> ready : dispatch_op_unary [gelu___] / gelu__ - ready --> ready : dispatch_op_unary [silu___] / silu__ + ready --> ready : dispatch_op_unary [silu__] / silu__ ready --> ready : dispatch_op_unary [dispatch_op_unary__] / dispatch_op_unary__ ready --> ready : dispatch_op_map_custom1 [dispatch_op_map_custom1__] / dispatch_op_map_custom1__ ready --> ready : dispatch_op_map_custom1 [dispatch_op_map_custom1__] / dispatch_op_map_custom1__ @@ -344,13 +351,18 @@ stateDiagram-v2 | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x8_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x4_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_q8_rhs_bl4_matrix_x4_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_q8_rhs_bl4_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_q8_rhs>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_q8_rhs_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q4_vector_f32_rhs>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_f32_rhs_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat_argmax`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_q8_argmax_prepared_i8mm_op_mul_mat_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat_argmax`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat_argmax`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat_argmax`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_packed_q8_rhs_op_mul_mat_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat_argmax`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_mul_mat`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`simd_op_mul_mat_q6_vector_prepared_q8_rhs>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exec_simd_q6_vector_prepared_q8_rhs_op_mul_mat>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | @@ -408,8 +420,9 @@ stateDiagram-v2 | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`rope_mode_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`mark_done_op, false>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`rope_mode_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`mark_done_op, true>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`rope_mode_is<0>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`mark_done_op, false>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`rope_mode_is<2>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`mark_done_op, true>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`rope_timestep_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`mark_done_op>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | @@ -490,14 +503,15 @@ stateDiagram-v2 | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`abs>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`neg>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`relu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exp>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`exp>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`tanh>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`tanh>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`elu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`elu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`gelu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`gelu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`silu>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_unary>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`dispatch_op_map_custom1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/aarch64/sm.hpp) | diff --git a/.planning/architecture/kernel_x86_64.md b/.planning/architecture/kernel_x86_64.md index 9b4b4706..879b8ff6 100644 --- a/.planning/architecture/kernel_x86_64.md +++ b/.planning/architecture/kernel_x86_64.md @@ -131,8 +131,9 @@ stateDiagram-v2 ready --> ready : dispatch_op_soft_max [dispatch_op_soft_max__] / dispatch_op_soft_max__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ - ready --> ready : dispatch_op_rope [rope_mode_is_false___] / mark_done_op__false__ - ready --> ready : dispatch_op_rope [rope_mode_is_true___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_mode_is_0___] / mark_done_op__false__ + ready --> ready : dispatch_op_rope [rope_mode_is_2___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_timestep_is__] / mark_done_op__ ready --> ready : dispatch_op_rope [dispatch_op_rope__] / dispatch_op_rope__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ @@ -369,8 +370,9 @@ stateDiagram-v2 | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_soft_max_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`rope_mode_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`mark_done_op, false>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | -| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`rope_mode_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`mark_done_op, true>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`rope_mode_is<0>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`mark_done_op, false>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`rope_mode_is<2>>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`mark_done_op, true>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`rope_timestep_is>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`mark_done_op>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`dispatch_op_rope_back>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/x86_64/sm.hpp) | diff --git a/.planning/architecture/logits_sampler.md b/.planning/architecture/logits_sampler.md index 70248226..c887550c 100644 --- a/.planning/architecture/logits_sampler.md +++ b/.planning/architecture/logits_sampler.md @@ -27,12 +27,21 @@ stateDiagram-v2 sample_call_decision --> errored : completion_sample_logits_runtime_ [sampler_call_failed_] / mark_sampler_error_ sample_complete_decision --> done : completion_sample_logits_runtime_ [selected_token_valid_] / none sample_complete_decision --> errored : completion_sample_logits_runtime_ [selected_token_missing_or_invalid_] / mark_invalid_request_ + ready --> state_temperature_top_k_request_decision : sample_temperature_top_k_runtime [always] / none + state_temperature_top_k_request_decision --> state_temperature_top_k_scale : completion_sample_temperature_top_k_runtime_ [temperature_top_k_request_valid_] / scale_temperature_logits_ + state_temperature_top_k_request_decision --> errored : completion_sample_temperature_top_k_runtime_ [temperature_top_k_request_invalid_] / mark_invalid_request_ + state_temperature_top_k_scale --> state_temperature_top_k_probabilities : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_probabilities_ + state_temperature_top_k_probabilities --> state_temperature_top_k_rank : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_top_k_ + state_temperature_top_k_rank --> state_temperature_top_k_select : completion_sample_temperature_top_k_runtime_ [always] / select_temperature_top_k_ + state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [always] / none done --> ready : completion_configure_runtime_ [always] / publish_done_ errored --> ready : completion_configure_runtime_ [always] / publish_error_ done --> ready : completion_sample_logits_runtime_ [always] / publish_done_ errored --> ready : completion_sample_logits_runtime_ [always] / publish_error_ done --> ready : completion_sample_preselected_runtime_ [always] / publish_done_ errored --> ready : completion_sample_preselected_runtime_ [always] / publish_error_ + done --> ready : completion_sample_temperature_top_k_runtime_ [always] / publish_done_ + errored --> ready : completion_sample_temperature_top_k_runtime_ [always] / publish_error_ ready --> ready : _ [always] / on_unexpected_ request_logits_decision --> ready : _ [always] / on_unexpected_ request_preselected_decision --> ready : _ [always] / on_unexpected_ @@ -42,6 +51,11 @@ stateDiagram-v2 sample_call --> ready : _ [always] / on_unexpected_ sample_call_decision --> ready : _ [always] / on_unexpected_ sample_complete_decision --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_request_decision --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_scale --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_probabilities --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_rank --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_select --> ready : _ [always] / on_unexpected_ done --> ready : _ [always] / on_unexpected_ errored --> ready : _ [always] / on_unexpected_ ``` @@ -69,12 +83,21 @@ stateDiagram-v2 | [`sample_call_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`sampler_call_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`mark_sampler_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`sample_complete_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`selected_token_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`sample_complete_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`selected_token_missing_or_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`mark_invalid_request>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`sample_temperature_top_k_runtime`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_request_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_request_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`temperature_top_k_request_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`scale_temperature_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_request_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`temperature_top_k_request_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`mark_invalid_request>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`compute_temperature_probabilities>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_probabilities`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_probabilities`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`compute_temperature_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_rank`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_rank`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`select_temperature_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`request_logits_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`request_preselected_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | @@ -84,5 +107,10 @@ stateDiagram-v2 | [`sample_call`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`sample_call_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`sample_complete_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_request_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_probabilities`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_rank`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | diff --git a/.planning/architecture/memory_streaming.md b/.planning/architecture/memory_streaming.md new file mode 100644 index 00000000..9da98108 --- /dev/null +++ b/.planning/architecture/memory_streaming.md @@ -0,0 +1,80 @@ +# memory_streaming + +Source: [`emel/memory/streaming/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_empty : initialize [guard_configuration_valid_] / effect_initialize_ + state_uninitialized --> state_uninitialized : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_empty --> state_empty : initialize [always] / already_initialized__ + state_filling --> state_filling : initialize [always] / already_initialized__ + state_full --> state_full : initialize [always] / already_initialized__ + state_errored --> state_empty : initialize [guard_configuration_valid_] / effect_initialize_ + state_errored --> state_errored : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_empty --> state_full : advance [guard_capacity_one_] / wrap__ + state_empty --> state_filling : advance [guard_capacity_many_] / advance__ + state_filling --> state_filling : advance [guard_filling_remains_partial_] / advance__ + state_filling --> state_full : advance [guard_filling_becomes_full_] / wrap__ + state_full --> state_full : advance [guard_full_position_available_before_wrap_] / advance__ + state_full --> state_full : advance [guard_full_position_available_at_wrap_] / wrap__ + state_full --> state_full : advance [guard_full_position_overflow_] / position_overflow__ + state_full --> state_full : advance [guard_full_cursor_invalid_] / internal_error__ + state_empty --> state_empty : reset [always] / effect_reset_ + state_filling --> state_empty : reset [always] / effect_reset_ + state_full --> state_empty : reset [always] / effect_reset_ + state_empty --> state_empty : capture_view [always] / empty__ + state_filling --> state_filling : capture_view [always] / filling__ + state_full --> state_full : capture_view [always] / full__ + state_uninitialized --> state_uninitialized : advance [always] / uninitialized__ + state_uninitialized --> state_uninitialized : reset [always] / uninitialized__ + state_uninitialized --> state_uninitialized : capture_view [always] / uninitialized__ + state_errored --> state_errored : advance [always] / internal_error__ + state_errored --> state_errored : reset [always] / internal_error__ + state_errored --> state_errored : capture_view [always] / internal_error__ + state_uninitialized --> state_errored : _ [always] / effect_unexpected_ + state_empty --> state_errored : _ [always] / effect_unexpected_ + state_filling --> state_errored : _ [always] / effect_unexpected_ + state_full --> state_errored : _ [always] / effect_unexpected_ + state_errored --> state_errored : _ [always] / effect_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_configuration_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_initialize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_configuration_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`invalid_configuration>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_configuration_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_initialize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_configuration_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`invalid_configuration>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_capacity_one>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`wrap>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_capacity_many>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_filling_remains_partial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_filling_becomes_full>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`wrap>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_full_position_available_before_wrap>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_full_position_available_at_wrap>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`wrap>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_full_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`guard_full_cursor_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`capture_view`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`empty>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`capture_view`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`filling>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`capture_view`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`full>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`capture_view`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`capture_view`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_empty`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_filling`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/memory/streaming/sm.hpp) | diff --git a/.planning/architecture/mermaid/kernel_aarch64.mmd b/.planning/architecture/mermaid/kernel_aarch64.mmd index e82058f0..82c43c50 100644 --- a/.planning/architecture/mermaid/kernel_aarch64.mmd +++ b/.planning/architecture/mermaid/kernel_aarch64.mmd @@ -79,13 +79,18 @@ stateDiagram-v2 ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8_] / exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4_] / exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_] / exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8_] / exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4_] / exec_simd_q4_vector_packed_q8_rhs_bl4_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_] / exec_simd_q4_vector_packed_q8_rhs_bl4_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4_] / exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_q8_rhs_] / exec_simd_q4_vector_q8_rhs_op_mul_mat_ + ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q4_vector_f32_rhs_] / exec_simd_q4_vector_f32_rhs_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_op_mul_mat_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm_] / exec_simd_q6_vector_q8_argmax_prepared_i8mm_op_mul_mat_argmax_ + ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_] / exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_argmax_ + ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_] / exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs_] / exec_simd_q6_vector_packed_q8_rhs_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat_argmax [simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm_] / exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_argmax_ ready --> ready : dispatch_op_mul_mat [simd_op_mul_mat_q6_vector_prepared_q8_rhs_] / exec_simd_q6_vector_prepared_q8_rhs_op_mul_mat_ @@ -143,8 +148,9 @@ stateDiagram-v2 ready --> ready : dispatch_op_soft_max [dispatch_op_soft_max__] / dispatch_op_soft_max__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ - ready --> ready : dispatch_op_rope [rope_mode_is_false___] / mark_done_op__false__ - ready --> ready : dispatch_op_rope [rope_mode_is_true___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_mode_is_0___] / mark_done_op__false__ + ready --> ready : dispatch_op_rope [rope_mode_is_2___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_timestep_is__] / mark_done_op__ ready --> ready : dispatch_op_rope [dispatch_op_rope__] / dispatch_op_rope__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ @@ -225,14 +231,15 @@ stateDiagram-v2 ready --> ready : dispatch_op_unary [abs___] / abs__ ready --> ready : dispatch_op_unary [neg___] / neg__ ready --> ready : dispatch_op_unary [relu___] / relu__ - ready --> ready : dispatch_op_unary [abs___] / abs__ - ready --> ready : dispatch_op_unary [neg___] / neg__ - ready --> ready : dispatch_op_unary [relu___] / relu__ + ready --> ready : dispatch_op_unary [silu___] / silu__ + ready --> ready : dispatch_op_unary [abs__] / abs__ + ready --> ready : dispatch_op_unary [neg__] / neg__ + ready --> ready : dispatch_op_unary [relu__] / relu__ ready --> ready : dispatch_op_unary [exp___] / exp__ ready --> ready : dispatch_op_unary [tanh___] / tanh__ ready --> ready : dispatch_op_unary [elu___] / elu__ ready --> ready : dispatch_op_unary [gelu___] / gelu__ - ready --> ready : dispatch_op_unary [silu___] / silu__ + ready --> ready : dispatch_op_unary [silu__] / silu__ ready --> ready : dispatch_op_unary [dispatch_op_unary__] / dispatch_op_unary__ ready --> ready : dispatch_op_map_custom1 [dispatch_op_map_custom1__] / dispatch_op_map_custom1__ ready --> ready : dispatch_op_map_custom1 [dispatch_op_map_custom1__] / dispatch_op_map_custom1__ diff --git a/.planning/architecture/mermaid/kernel_x86_64.mmd b/.planning/architecture/mermaid/kernel_x86_64.mmd index 9f7dcc08..35568018 100644 --- a/.planning/architecture/mermaid/kernel_x86_64.mmd +++ b/.planning/architecture/mermaid/kernel_x86_64.mmd @@ -124,8 +124,9 @@ stateDiagram-v2 ready --> ready : dispatch_op_soft_max [dispatch_op_soft_max__] / dispatch_op_soft_max__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ ready --> ready : dispatch_op_soft_max_back [dispatch_op_soft_max_back__] / dispatch_op_soft_max_back__ - ready --> ready : dispatch_op_rope [rope_mode_is_false___] / mark_done_op__false__ - ready --> ready : dispatch_op_rope [rope_mode_is_true___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_mode_is_0___] / mark_done_op__false__ + ready --> ready : dispatch_op_rope [rope_mode_is_2___] / mark_done_op__true__ + ready --> ready : dispatch_op_rope [rope_timestep_is__] / mark_done_op__ ready --> ready : dispatch_op_rope [dispatch_op_rope__] / dispatch_op_rope__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ ready --> ready : dispatch_op_rope_back [dispatch_op_rope_back__] / dispatch_op_rope_back__ diff --git a/.planning/architecture/mermaid/logits_sampler.mmd b/.planning/architecture/mermaid/logits_sampler.mmd index faf1f10b..5c230682 100644 --- a/.planning/architecture/mermaid/logits_sampler.mmd +++ b/.planning/architecture/mermaid/logits_sampler.mmd @@ -20,12 +20,21 @@ stateDiagram-v2 sample_call_decision --> errored : completion_sample_logits_runtime_ [sampler_call_failed_] / mark_sampler_error_ sample_complete_decision --> done : completion_sample_logits_runtime_ [selected_token_valid_] / none sample_complete_decision --> errored : completion_sample_logits_runtime_ [selected_token_missing_or_invalid_] / mark_invalid_request_ + ready --> state_temperature_top_k_request_decision : sample_temperature_top_k_runtime [always] / none + state_temperature_top_k_request_decision --> state_temperature_top_k_scale : completion_sample_temperature_top_k_runtime_ [temperature_top_k_request_valid_] / scale_temperature_logits_ + state_temperature_top_k_request_decision --> errored : completion_sample_temperature_top_k_runtime_ [temperature_top_k_request_invalid_] / mark_invalid_request_ + state_temperature_top_k_scale --> state_temperature_top_k_probabilities : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_probabilities_ + state_temperature_top_k_probabilities --> state_temperature_top_k_rank : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_top_k_ + state_temperature_top_k_rank --> state_temperature_top_k_select : completion_sample_temperature_top_k_runtime_ [always] / select_temperature_top_k_ + state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [always] / none done --> ready : completion_configure_runtime_ [always] / publish_done_ errored --> ready : completion_configure_runtime_ [always] / publish_error_ done --> ready : completion_sample_logits_runtime_ [always] / publish_done_ errored --> ready : completion_sample_logits_runtime_ [always] / publish_error_ done --> ready : completion_sample_preselected_runtime_ [always] / publish_done_ errored --> ready : completion_sample_preselected_runtime_ [always] / publish_error_ + done --> ready : completion_sample_temperature_top_k_runtime_ [always] / publish_done_ + errored --> ready : completion_sample_temperature_top_k_runtime_ [always] / publish_error_ ready --> ready : _ [always] / on_unexpected_ request_logits_decision --> ready : _ [always] / on_unexpected_ request_preselected_decision --> ready : _ [always] / on_unexpected_ @@ -35,5 +44,10 @@ stateDiagram-v2 sample_call --> ready : _ [always] / on_unexpected_ sample_call_decision --> ready : _ [always] / on_unexpected_ sample_complete_decision --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_request_decision --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_scale --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_probabilities --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_rank --> ready : _ [always] / on_unexpected_ + state_temperature_top_k_select --> ready : _ [always] / on_unexpected_ done --> ready : _ [always] / on_unexpected_ errored --> ready : _ [always] / on_unexpected_ diff --git a/.planning/architecture/mermaid/memory_streaming.mmd b/.planning/architecture/mermaid/memory_streaming.mmd new file mode 100644 index 00000000..8e5b4428 --- /dev/null +++ b/.planning/architecture/mermaid/memory_streaming.mmd @@ -0,0 +1,35 @@ +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_empty : initialize [guard_configuration_valid_] / effect_initialize_ + state_uninitialized --> state_uninitialized : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_empty --> state_empty : initialize [always] / already_initialized__ + state_filling --> state_filling : initialize [always] / already_initialized__ + state_full --> state_full : initialize [always] / already_initialized__ + state_errored --> state_empty : initialize [guard_configuration_valid_] / effect_initialize_ + state_errored --> state_errored : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_empty --> state_full : advance [guard_capacity_one_] / wrap__ + state_empty --> state_filling : advance [guard_capacity_many_] / advance__ + state_filling --> state_filling : advance [guard_filling_remains_partial_] / advance__ + state_filling --> state_full : advance [guard_filling_becomes_full_] / wrap__ + state_full --> state_full : advance [guard_full_position_available_before_wrap_] / advance__ + state_full --> state_full : advance [guard_full_position_available_at_wrap_] / wrap__ + state_full --> state_full : advance [guard_full_position_overflow_] / position_overflow__ + state_full --> state_full : advance [guard_full_cursor_invalid_] / internal_error__ + state_empty --> state_empty : reset [always] / effect_reset_ + state_filling --> state_empty : reset [always] / effect_reset_ + state_full --> state_empty : reset [always] / effect_reset_ + state_empty --> state_empty : capture_view [always] / empty__ + state_filling --> state_filling : capture_view [always] / filling__ + state_full --> state_full : capture_view [always] / full__ + state_uninitialized --> state_uninitialized : advance [always] / uninitialized__ + state_uninitialized --> state_uninitialized : reset [always] / uninitialized__ + state_uninitialized --> state_uninitialized : capture_view [always] / uninitialized__ + state_errored --> state_errored : advance [always] / internal_error__ + state_errored --> state_errored : reset [always] / internal_error__ + state_errored --> state_errored : capture_view [always] / internal_error__ + state_uninitialized --> state_errored : _ [always] / effect_unexpected_ + state_empty --> state_errored : _ [always] / effect_unexpected_ + state_filling --> state_errored : _ [always] / effect_unexpected_ + state_full --> state_errored : _ [always] / effect_unexpected_ + state_errored --> state_errored : _ [always] / effect_unexpected_ diff --git a/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd b/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd new file mode 100644 index 00000000..4da0f26d --- /dev/null +++ b/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd @@ -0,0 +1,308 @@ +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_bind_contract_decision : initialize_run [always] / none + state_bind_contract_decision --> state_input_embedding_contract_decision : completion_initialize_run_ [guard_bind_contract_valid_] / effect_bind_contract_ + state_bind_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bind_contract_invalid_] / effect_mark_bind_failed_ + state_input_embedding_contract_decision --> state_temporal_projection_layout_decision : completion_initialize_run_ [guard_bound_root_operands_supported_] / none + state_input_embedding_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bound_root_operands_unsupported_] / effect_mark_bind_failed_ + state_temporal_projection_layout_decision --> state_depformer_projection_layout_decision : completion_initialize_run_ [guard_temporal_split_projection_layout_supported_] / effect_bind_temporal_split_projection_layout_ + state_temporal_projection_layout_decision --> state_depformer_projection_layout_decision : completion_initialize_run_ [guard_temporal_fused_projection_layout_supported_] / effect_bind_temporal_fused_projection_layout_ + state_temporal_projection_layout_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_temporal_projection_layout_unsupported_] / effect_mark_bind_failed_ + state_depformer_projection_layout_decision --> state_sampling_seed_decision : completion_initialize_run_ [guard_depformer_split_projection_layout_supported_] / effect_bind_depformer_split_projection_layout_ + state_depformer_projection_layout_decision --> state_sampling_seed_decision : completion_initialize_run_ [guard_depformer_fused_projection_layout_supported_] / effect_bind_depformer_fused_projection_layout_ + state_depformer_projection_layout_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_depformer_projection_layout_unsupported_] / effect_mark_bind_failed_ + state_sampling_seed_decision --> state_text_sampling_top_k_decision : completion_initialize_run_ [guard_sampling_seed_nonzero_] / effect_bind_nonzero_sampling_seed_ + state_sampling_seed_decision --> state_text_sampling_top_k_decision : completion_initialize_run_ [guard_sampling_seed_zero_] / effect_bind_zero_sampling_seed_ + state_text_sampling_top_k_decision --> state_audio_sampling_top_k_decision : completion_initialize_run_ [guard_text_sampling_top_k_within_card_] / effect_bind_requested_text_sampling_top_k_ + state_text_sampling_top_k_decision --> state_audio_sampling_top_k_decision : completion_initialize_run_ [guard_text_sampling_top_k_exceeds_card_] / effect_bind_full_card_text_sampling_top_k_ + state_audio_sampling_top_k_decision --> state_init_error_out_decision : completion_initialize_run_ [guard_audio_sampling_top_k_within_card_] / effect_bind_requested_audio_sampling_top_k_ + state_audio_sampling_top_k_decision --> state_init_error_out_decision : completion_initialize_run_ [guard_audio_sampling_top_k_exceeds_card_] / effect_bind_full_card_audio_sampling_top_k_ + state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ + state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / none + state_init_failed_error_out_decision --> state_init_failed_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ + state_init_failed_error_out_decision --> state_init_failed_callback_decision : completion_initialize_run_ [initialize_run__] / none + state_init_callback_decision --> state_ready : completion_initialize_run_ [guard_has_initialize_done_callback_] / effect_emit_initialize_done_ + state_init_callback_decision --> state_ready : completion_initialize_run_ [guard_no_initialize_done_callback_] / none + state_init_failed_callback_decision --> state_uninitialized : completion_initialize_run_ [guard_has_initialize_error_callback_] / effect_emit_initialize_error_ + state_init_failed_callback_decision --> state_uninitialized : completion_initialize_run_ [guard_no_initialize_error_callback_] / none + state_ready --> state_step_model_decision : step_run [always] / none + state_step_model_decision --> state_step_shape_decision : completion_step_run_ [guard_step_model_matches_] / none + state_step_model_decision --> state_step_error_out_decision : completion_step_run_ [guard_step_model_mismatch_] / effect_mark_model_mismatch_ + state_step_shape_decision --> state_step_execution_decision : completion_step_run_ [guard_step_shape_valid_] / none + state_step_shape_decision --> state_step_error_out_decision : completion_step_run_ [guard_step_shape_invalid_] / effect_mark_request_shape_ + state_step_execution_decision --> state_input_embedding : completion_step_run_ [guard_input_embedding_supported_] / effect_begin_input_embedding_ + state_step_execution_decision --> state_step_error_out_decision : completion_step_run_ [guard_input_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_embedding --> state_input_embedding_result_decision : completion_step_run_ [guard_external_input_embedding_supported_] / effect_apply_external_input_embedding_ + state_input_embedding --> state_input_text_embedding_bind : completion_step_run_ [guard_token_input_embedding_supported_] / effect_bind_input_text_embedding_ + state_input_embedding --> state_step_error_out_decision : completion_step_run_ [guard_token_input_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_bind --> state_input_text_embedding_bind_result_decision : completion_step_run_ [always] / none + state_input_text_embedding_bind_result_decision --> state_input_text_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_input_text_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_row --> state_input_text_embedding_row_result_decision : completion_step_run_ [always] / none + state_input_text_embedding_row_result_decision --> state_input_text_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_input_text_embedding_row_ + state_input_text_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_apply --> state_input_text_embedding_result_decision : completion_step_run_ [guard_input_text_embedding_succeeded_] / none + state_input_text_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_input_text_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_result_decision --> state_input_audio_token_decision : completion_step_run_ [always] / none + state_input_audio_token_decision --> state_input_audio_embedding : completion_step_run_ [guard_current_input_audio_token_present_] / effect_bind_input_audio_embedding_ + state_input_audio_token_decision --> state_input_audio_embedding_result_decision : completion_step_run_ [guard_current_input_audio_token_zero_] / effect_skip_zero_input_audio_embedding_ + state_input_audio_token_decision --> state_step_error_out_decision : completion_step_run_ [guard_current_input_audio_token_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding --> state_input_audio_embedding_bind_result_decision : completion_step_run_ [always] / none + state_input_audio_embedding_bind_result_decision --> state_input_audio_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_input_audio_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_row --> state_input_audio_embedding_row_result_decision : completion_step_run_ [always] / none + state_input_audio_embedding_row_result_decision --> state_input_audio_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_input_audio_embedding_row_ + state_input_audio_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_apply --> state_input_audio_embedding_result_decision : completion_step_run_ [guard_input_audio_embedding_succeeded_] / none + state_input_audio_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_input_audio_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_result_decision --> state_input_audio_codebook_advance : completion_step_run_ [guard_more_input_audio_codebooks_] / effect_advance_input_audio_codebook_ + state_input_audio_codebook_advance --> state_input_audio_token_decision : completion_step_run_ [always] / none + state_input_audio_embedding_result_decision --> state_input_embedding_result_decision : completion_step_run_ [guard_input_audio_codebooks_complete_] / effect_finish_input_embedding_ + state_input_embedding_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_input_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_embedding_result_decision --> state_bind_temporal_kv : completion_step_run_ [guard_temporal_kv_binding_present_] / effect_bind_temporal_kv_ + state_input_embedding_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_kv_binding_missing_] / effect_mark_graph_execution_unsupported_ + state_bind_temporal_kv --> state_temporal_kv_result_decision : completion_step_run_ [guard_temporal_kv_bound_] / none + state_bind_temporal_kv --> state_step_error_out_decision : completion_step_run_ [guard_temporal_kv_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_kv_result_decision --> state_temporal_position_result_decision : completion_step_run_ [always] / effect_advance_temporal_position_ + state_temporal_position_result_decision --> state_temporal_position_valid_decision : completion_step_run_ [guard_temporal_position_advance_succeeded_] / none + state_temporal_position_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_position_advance_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_position_valid_decision --> state_temporal_layer_norm : completion_step_run_ [guard_temporal_layer_norm_supported_] / effect_run_temporal_layer_norm_rms_ + state_temporal_position_valid_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm --> state_temporal_layer_norm_rms_result_decision : completion_step_run_ [always] / none + state_temporal_layer_norm_rms_result_decision --> state_temporal_layer_norm_scale : completion_step_run_ [guard_temporal_layer_norm_rms_succeeded_] / effect_run_temporal_layer_norm_scale_ + state_temporal_layer_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm_scale --> state_temporal_layer_norm_result_decision : completion_step_run_ [guard_temporal_layer_norm_succeeded_] / none + state_temporal_layer_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm_result_decision --> state_temporal_layer_projection : completion_step_run_ [guard_temporal_layer_projection_supported_] / effect_bind_temporal_layer_projection_ + state_temporal_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection --> state_temporal_layer_projection_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_projection_ + state_temporal_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection_run --> state_temporal_layer_projection_result_decision : completion_step_run_ [guard_temporal_layer_projection_succeeded_] / none + state_temporal_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection_result_decision --> state_temporal_layer_rope : completion_step_run_ [guard_temporal_layer_rope_supported_] / effect_run_temporal_layer_rope_0__ + state_temporal_layer_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_rope --> state_temporal_layer_rope_result_decision : completion_step_run_ [guard_temporal_layer_rope_succeeded_] / effect_copy_temporal_layer_rope_0__ + state_temporal_layer_rope --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_rope_result_decision --> state_temporal_layer_key_rope : completion_step_run_ [always] / effect_run_temporal_layer_rope_1__ + state_temporal_layer_key_rope --> state_temporal_layer_key_rope_result_decision : completion_step_run_ [guard_temporal_layer_rope_succeeded_] / effect_copy_temporal_layer_rope_1__ + state_temporal_layer_key_rope --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_key_rope_result_decision --> state_temporal_layer_cache_write : completion_step_run_ [guard_temporal_layer_cache_write_supported_] / effect_write_temporal_layer_kv_cache_ + state_temporal_layer_key_rope_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write --> state_temporal_layer_cache_write_result_decision : completion_step_run_ [guard_temporal_layer_cache_write_succeeded_] / none + state_temporal_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_supported_] / effect_run_temporal_layer_attention_ + state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_attention --> state_temporal_layer_attention_result_decision : completion_step_run_ [guard_temporal_layer_attention_succeeded_] / none + state_temporal_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_attention_result_decision --> state_temporal_layer_out_projection : completion_step_run_ [guard_temporal_layer_out_projection_supported_] / effect_bind_temporal_layer_out_projection_ + state_temporal_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection --> state_temporal_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_out_projection_ + state_temporal_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection_run --> state_temporal_layer_out_projection_result_decision : completion_step_run_ [guard_temporal_layer_out_projection_succeeded_] / none + state_temporal_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection_result_decision --> state_temporal_layer_residual : completion_step_run_ [always] / effect_apply_temporal_layer_attention_residual_ + state_temporal_layer_residual --> state_temporal_layer_residual_result_decision : completion_step_run_ [guard_temporal_layer_residual_succeeded_] / none + state_temporal_layer_residual --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_residual_result_decision --> state_temporal_layer_norm2 : completion_step_run_ [guard_temporal_layer_norm2_supported_] / effect_run_temporal_layer_norm2_rms_ + state_temporal_layer_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2 --> state_temporal_layer_norm2_rms_result_decision : completion_step_run_ [always] / none + state_temporal_layer_norm2_rms_result_decision --> state_temporal_layer_norm2_scale : completion_step_run_ [guard_temporal_layer_norm2_rms_succeeded_] / effect_run_temporal_layer_norm2_scale_ + state_temporal_layer_norm2_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2_scale --> state_temporal_layer_norm2_result_decision : completion_step_run_ [guard_temporal_layer_norm2_succeeded_] / none + state_temporal_layer_norm2_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2_result_decision --> state_temporal_layer_gating_in : completion_step_run_ [guard_temporal_layer_gating_in_supported_] / effect_bind_temporal_layer_gating_in_ + state_temporal_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in --> state_temporal_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_in_ + state_temporal_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in_run --> state_temporal_layer_gating_in_result_decision : completion_step_run_ [guard_temporal_layer_gating_in_succeeded_] / none + state_temporal_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in_result_decision --> state_temporal_layer_silu_gate : completion_step_run_ [guard_temporal_layer_silu_gate_supported_] / effect_run_temporal_layer_silu_gate_silu_ + state_temporal_layer_gating_in_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate --> state_temporal_layer_silu_gate_silu_result_decision : completion_step_run_ [always] / none + state_temporal_layer_silu_gate_silu_result_decision --> state_temporal_layer_silu_gate_mul : completion_step_run_ [guard_temporal_layer_silu_gate_silu_succeeded_] / effect_run_temporal_layer_silu_gate_mul_ + state_temporal_layer_silu_gate_silu_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_silu_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate_mul --> state_temporal_layer_silu_gate_result_decision : completion_step_run_ [guard_temporal_layer_silu_gate_succeeded_] / none + state_temporal_layer_silu_gate_mul --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate_result_decision --> state_temporal_layer_gating_out : completion_step_run_ [guard_temporal_layer_gating_out_supported_] / effect_bind_temporal_layer_gating_out_ + state_temporal_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out --> state_temporal_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_out_ + state_temporal_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out_run --> state_temporal_layer_gating_out_result_decision : completion_step_run_ [guard_temporal_layer_gating_out_succeeded_] / none + state_temporal_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out_result_decision --> state_temporal_layer_ff_residual : completion_step_run_ [always] / effect_apply_temporal_layer_ff_residual_ + state_temporal_layer_ff_residual --> state_temporal_layer_ff_residual_result_decision : completion_step_run_ [guard_temporal_layer_ff_residual_succeeded_] / none + state_temporal_layer_ff_residual --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_ff_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_ff_residual_result_decision --> state_temporal_layer_advance : completion_step_run_ [guard_more_temporal_layers_] / effect_advance_temporal_layer_ + state_temporal_layer_ff_residual_result_decision --> state_temporal_out_norm : completion_step_run_ [guard_temporal_layers_complete_] / effect_run_temporal_out_norm_rms_ + state_temporal_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_advance --> state_temporal_layer_norm : completion_step_run_ [guard_temporal_layer_norm_supported_] / effect_run_temporal_layer_norm_rms_ + state_temporal_layer_advance --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm --> state_temporal_out_norm_rms_result_decision : completion_step_run_ [always] / none + state_temporal_out_norm_rms_result_decision --> state_temporal_out_norm_scale : completion_step_run_ [guard_temporal_out_norm_rms_succeeded_] / effect_run_temporal_out_norm_scale_ + state_temporal_out_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm_scale --> state_temporal_out_norm_result_decision : completion_step_run_ [guard_temporal_out_norm_succeeded_] / effect_publish_temporal_out_norm_ + state_temporal_out_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm_result_decision --> state_text_logits_phase_decision : completion_step_run_ [guard_full_graph_phase_] / none + state_temporal_out_norm_result_decision --> state_sampling_ready : completion_step_run_ [guard_prediction_graph_phase_] / effect_publish_prediction_state_ + state_text_logits_phase_decision --> state_text_logits : completion_step_run_ [guard_text_logits_supported_] / effect_bind_text_token_logits_ + state_text_logits_phase_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_unsupported_] / effect_mark_graph_execution_unsupported_ + state_sampling_ready --> state_text_logits : step_run [guard_sampling_step_valid_] / effect_restore_prediction_state_and_bind_text_logits_ + state_sampling_ready --> state_step_error_out_decision : step_run [guard_sampling_step_invalid_] / effect_mark_request_shape_ + state_text_logits --> state_text_logits_bind_result_decision : completion_step_run_ [always] / none + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_valid_and_sampling_consume_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_token_valid_without_sampling_consume_] / effect_publish_forced_text_token_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_sampling_config_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_text_logits_run : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_argmax_] / effect_select_text_token_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_sample_projection --> state_text_logits_sample_projection_result_decision : completion_step_run_ [always] / none + state_text_logits_sample_projection_result_decision --> state_text_logits_sample_select : completion_step_run_ [guard_text_logits_matmul_succeeded_] / effect_select_text_sampling_token_ + state_text_logits_sample_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_matmul_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_sample_select --> state_text_logits_result_decision : completion_step_run_ [guard_sampled_text_token_ready_] / effect_publish_sampled_text_token_ + state_text_logits_sample_select --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_sampling_consumed_] / effect_publish_forced_text_token_ + state_text_logits_sample_select --> state_step_error_out_decision : completion_step_run_ [guard_text_sampling_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_run --> state_text_logits_result_decision : completion_step_run_ [guard_text_logits_succeeded_] / none + state_text_logits_run --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_result_decision --> state_bind_depformer_kv : completion_step_run_ [guard_depformer_kv_binding_present_] / effect_bind_depformer_kv_ + state_text_logits_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_kv_binding_missing_] / effect_mark_graph_execution_unsupported_ + state_bind_depformer_kv --> state_depformer_kv_result_decision : completion_step_run_ [guard_depformer_kv_bound_] / effect_reset_depformer_positions_ + state_bind_depformer_kv --> state_step_error_out_decision : completion_step_run_ [guard_depformer_kv_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_kv_result_decision --> state_depformer_position_reset_result_decision : completion_step_run_ [always] / none + state_depformer_position_reset_result_decision --> state_depformer_position_advance_result_decision : completion_step_run_ [guard_depformer_position_reset_succeeded_] / effect_advance_depformer_position_ + state_depformer_position_reset_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_position_reset_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_position_advance_result_decision --> state_depformer_weight_route_decision : completion_step_run_ [guard_depformer_position_advance_succeeded_] / none + state_depformer_position_advance_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_position_advance_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_weight_route_decision --> state_depformer_weight_decision : completion_step_run_ [guard_depformer_scheduled_weight_present_] / effect_use_depformer_scheduled_weight_ + state_depformer_weight_route_decision --> state_depformer_weight_decision : completion_step_run_ [guard_depformer_scheduled_weight_absent_] / effect_use_depformer_codebook_weight_ + state_depformer_weight_route_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_scheduled_weight_invalid_] / effect_mark_graph_execution_unsupported_ + state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_text_input_supported_] / effect_bind_depformer_text_input_projection_ + state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_audio_input_supported_] / effect_bind_depformer_audio_input_projection_ + state_depformer_weight_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_input --> state_depformer_input_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_bound_] / effect_run_depformer_input_projection_ + state_depformer_input_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_projection --> state_depformer_input_projection_result_decision : completion_step_run_ [always] / none + state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_text_input_projection_succeeded_] / effect_bind_depformer_text_input_embedding_ + state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_audio_input_projection_succeeded_] / effect_bind_depformer_audio_input_embedding_ + state_depformer_input_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_bind --> state_depformer_input_embedding_bind_result_decision : completion_step_run_ [always] / none + state_depformer_input_embedding_bind_result_decision --> state_depformer_input_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_depformer_input_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_row --> state_depformer_input_embedding_row_result_decision : completion_step_run_ [always] / none + state_depformer_input_embedding_row_result_decision --> state_depformer_input_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_depformer_input_embedding_row_ + state_depformer_input_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_apply --> state_depformer_input_result_decision : completion_step_run_ [guard_depformer_input_succeeded_] / none + state_depformer_input_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_result_decision --> state_depformer_layer_norm : completion_step_run_ [guard_depformer_layer_norm_supported_] / effect_run_depformer_layer_norm_rms_ + state_depformer_input_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm --> state_depformer_layer_norm_rms_result_decision : completion_step_run_ [always] / none + state_depformer_layer_norm_rms_result_decision --> state_depformer_layer_norm_scale : completion_step_run_ [guard_depformer_layer_norm_rms_succeeded_] / effect_run_depformer_layer_norm_scale_ + state_depformer_layer_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm_scale --> state_depformer_layer_norm_result_decision : completion_step_run_ [guard_depformer_layer_norm_succeeded_] / none + state_depformer_layer_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm_result_decision --> state_depformer_layer_projection : completion_step_run_ [guard_depformer_layer_projection_supported_] / effect_bind_depformer_layer_projection_ + state_depformer_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection --> state_depformer_layer_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_projection_ + state_depformer_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection_run --> state_depformer_layer_projection_result_decision : completion_step_run_ [guard_depformer_layer_projection_succeeded_] / none + state_depformer_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection_result_decision --> state_depformer_layer_cache_write : completion_step_run_ [guard_depformer_layer_cache_write_supported_] / effect_write_depformer_layer_kv_cache_ + state_depformer_layer_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_cache_write --> state_depformer_layer_cache_write_result_decision : completion_step_run_ [guard_depformer_layer_cache_write_succeeded_] / none + state_depformer_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_cache_write_result_decision --> state_depformer_layer_attention : completion_step_run_ [guard_depformer_layer_attention_supported_] / effect_run_depformer_layer_attention_ + state_depformer_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_attention --> state_depformer_layer_attention_result_decision : completion_step_run_ [guard_depformer_layer_attention_succeeded_] / none + state_depformer_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_attention_result_decision --> state_depformer_layer_out_projection : completion_step_run_ [guard_depformer_layer_out_projection_supported_] / effect_bind_depformer_layer_out_projection_ + state_depformer_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection --> state_depformer_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_out_projection_ + state_depformer_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection_run --> state_depformer_layer_out_projection_result_decision : completion_step_run_ [guard_depformer_layer_out_projection_succeeded_] / none + state_depformer_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection_result_decision --> state_depformer_layer_residual : completion_step_run_ [always] / effect_apply_depformer_layer_attention_residual_ + state_depformer_layer_residual --> state_depformer_layer_residual_result_decision : completion_step_run_ [guard_depformer_layer_residual_succeeded_] / none + state_depformer_layer_residual --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_residual_result_decision --> state_depformer_layer_norm2 : completion_step_run_ [guard_depformer_layer_norm2_supported_] / effect_run_depformer_layer_norm2_rms_ + state_depformer_layer_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2 --> state_depformer_layer_norm2_rms_result_decision : completion_step_run_ [always] / none + state_depformer_layer_norm2_rms_result_decision --> state_depformer_layer_norm2_scale : completion_step_run_ [guard_depformer_layer_norm2_rms_succeeded_] / effect_run_depformer_layer_norm2_scale_ + state_depformer_layer_norm2_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2_scale --> state_depformer_layer_norm2_result_decision : completion_step_run_ [guard_depformer_layer_norm2_succeeded_] / none + state_depformer_layer_norm2_scale --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2_result_decision --> state_depformer_layer_gating_in : completion_step_run_ [guard_depformer_layer_gating_in_supported_] / effect_bind_depformer_layer_gating_in_ + state_depformer_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in --> state_depformer_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_in_ + state_depformer_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in_run --> state_depformer_layer_gating_in_result_decision : completion_step_run_ [guard_depformer_layer_gating_in_succeeded_] / none + state_depformer_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in_result_decision --> state_depformer_layer_silu_gate : completion_step_run_ [guard_depformer_layer_silu_gate_supported_] / effect_run_depformer_layer_silu_gate_silu_ + state_depformer_layer_gating_in_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate --> state_depformer_layer_silu_gate_silu_result_decision : completion_step_run_ [always] / none + state_depformer_layer_silu_gate_silu_result_decision --> state_depformer_layer_silu_gate_mul : completion_step_run_ [guard_depformer_layer_silu_gate_silu_succeeded_] / effect_run_depformer_layer_silu_gate_mul_ + state_depformer_layer_silu_gate_silu_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_silu_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate_mul --> state_depformer_layer_silu_gate_result_decision : completion_step_run_ [guard_depformer_layer_silu_gate_succeeded_] / none + state_depformer_layer_silu_gate_mul --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate_result_decision --> state_depformer_layer_gating_out : completion_step_run_ [guard_depformer_layer_gating_out_supported_] / effect_bind_depformer_layer_gating_out_ + state_depformer_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out --> state_depformer_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_out_ + state_depformer_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out_run --> state_depformer_layer_gating_out_result_decision : completion_step_run_ [guard_depformer_layer_gating_out_succeeded_] / none + state_depformer_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out_result_decision --> state_depformer_layer_ff_residual : completion_step_run_ [always] / effect_apply_depformer_layer_ff_residual_ + state_depformer_layer_ff_residual --> state_depformer_layer_ff_residual_result_decision : completion_step_run_ [guard_depformer_layer_ff_residual_succeeded_] / none + state_depformer_layer_ff_residual --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_ff_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_ff_residual_result_decision --> state_depformer_layer_advance : completion_step_run_ [guard_more_depformer_layers_] / effect_advance_depformer_layer_ + state_depformer_layer_ff_residual_result_decision --> state_depformer_logits : completion_step_run_ [guard_depformer_logits_supported_] / effect_bind_depformer_token_logits_ + state_depformer_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits --> state_depformer_logits_bind_result_decision : completion_step_run_ [always] / none + state_depformer_logits_bind_result_decision --> state_depformer_logits_run : completion_step_run_ [guard_depformer_logits_projection_bound_argmax_] / effect_select_depformer_token_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_] / effect_compute_depformer_token_logits_ + state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_invalid_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_sample_projection --> state_depformer_logits_sample_projection_result_decision : completion_step_run_ [always] / none + state_depformer_logits_sample_projection_result_decision --> state_depformer_logits_sample_select : completion_step_run_ [guard_depformer_logits_matmul_succeeded_] / effect_select_depformer_sampling_token_ + state_depformer_logits_sample_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_matmul_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_advance --> state_depformer_layer_norm : completion_step_run_ [guard_depformer_layer_norm_supported_] / effect_run_depformer_layer_norm_rms_ + state_depformer_layer_advance --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_run --> state_depformer_token_publish : completion_step_run_ [guard_depformer_logits_succeeded_] / effect_publish_depformer_token_ + state_depformer_logits_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_sample_select --> state_depformer_token_publish : completion_step_run_ [guard_depformer_sampling_succeeded_] / effect_publish_depformer_token_ + state_depformer_logits_sample_select --> state_step_error_out_decision : completion_step_run_ [guard_depformer_sampling_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_token_publish --> state_depformer_logits_result_decision : completion_step_run_ [always] / none + state_depformer_logits_result_decision --> state_depformer_codebook_advance : completion_step_run_ [guard_more_depformer_codebooks_] / effect_advance_depformer_codebook_ + state_depformer_logits_result_decision --> state_ready : completion_step_run_ [guard_depformer_codebooks_complete_] / none + state_depformer_codebook_advance --> state_depformer_position_advance_result_decision : completion_step_run_ [always] / effect_advance_depformer_position_ + state_step_error_out_decision --> state_ready : completion_step_run_ [step_run__] / step_run__ + state_step_error_out_decision --> state_ready : completion_step_run_ [step_run__] / none + state_uninitialized --> state_uninit_step_error_out_decision : step_run [always] / effect_mark_not_initialized_ + state_uninit_step_error_out_decision --> state_uninitialized : completion_step_run_ [step_run__] / step_run__ + state_uninit_step_error_out_decision --> state_uninitialized : completion_step_run_ [step_run__] / none + state_ready --> state_reset_temporal_positions_decision : reset_run [always] / none + state_sampling_ready --> state_reset_temporal_positions_decision : reset_run [always] / none + state_reset_temporal_positions_decision --> state_reset_temporal_positions_result_decision : completion_reset_run_ [guard_reset_temporal_positions_present_] / effect_reset_temporal_positions_ + state_reset_temporal_positions_decision --> state_reset_depformer_positions_decision : completion_reset_run_ [guard_reset_temporal_positions_missing_] / none + state_reset_temporal_positions_result_decision --> state_reset_depformer_positions_decision : completion_reset_run_ [guard_reset_temporal_positions_succeeded_] / none + state_reset_temporal_positions_result_decision --> state_reset_failed : completion_reset_run_ [guard_reset_temporal_positions_failed_] / effect_mark_reset_failed_ + state_reset_depformer_positions_decision --> state_reset_depformer_positions_result_decision : completion_reset_run_ [guard_reset_depformer_positions_present_] / effect_reset_bound_depformer_positions_ + state_reset_depformer_positions_decision --> state_uninitialized : completion_reset_run_ [guard_reset_depformer_positions_missing_] / effect_reset_session_ + state_reset_depformer_positions_result_decision --> state_uninitialized : completion_reset_run_ [guard_reset_depformer_positions_succeeded_] / effect_reset_session_ + state_reset_depformer_positions_result_decision --> state_reset_failed : completion_reset_run_ [guard_reset_depformer_positions_failed_] / effect_mark_reset_failed_ + state_reset_failed --> state_uninitialized : completion_reset_run_ [always] / effect_reset_session_ + state_uninitialized --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_uninitialized --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_ready --> state_ready : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_ready --> state_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_sampling_ready --> state_sampling_ready : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_sampling_ready --> state_sampling_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ diff --git a/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd b/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd new file mode 100644 index 00000000..3281e4e8 --- /dev/null +++ b/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd @@ -0,0 +1,70 @@ +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_ready : initialize [guard_configuration_valid_] / effect_initialize_ + state_uninitialized --> state_uninitialized : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_ready --> state_ready : initialize [always] / already_initialized__ + state_prepared_full --> state_prepared_full : initialize [always] / already_initialized__ + state_prepared_generated --> state_prepared_generated : initialize [always] / already_initialized__ + state_prepared_tail --> state_prepared_tail : initialize [always] / already_initialized__ + state_errored --> state_ready : initialize [guard_configuration_valid_] / effect_initialize_ + state_errored --> state_errored : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_ready --> state_prepared_full : tokenize [guard_tokenize_full_] / effect_tokenize_full_ + state_ready --> state_prepared_tail : tokenize [guard_tokenize_tail_] / effect_tokenize_tail_ + state_ready --> state_prepared_generated : tokenize [guard_tokenize_empty_] / effect_tokenize_empty_ + state_ready --> state_ready : tokenize [guard_tokenize_invalid_] / request_shape__ + state_ready --> state_ready : tokenize [guard_tokenize_position_overflow_] / position_overflow__ + state_prepared_full --> state_prepared_full : tokenize [always] / phase_order__ + state_prepared_generated --> state_prepared_generated : tokenize [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : tokenize [always] / phase_order__ + state_prepared_full --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_full --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_generated --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_generated --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : detokenize_run [guard_position_overflow_] / position_overflow__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_position_overflow_] / position_overflow__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_position_overflow_] / position_overflow__ + state_commit_full_zero --> state_output_decision : completion_detokenize_run_ [always] / zero__ + state_commit_full_generated --> state_output_decision : completion_detokenize_run_ [always] / generated__ + state_commit_generated_zero --> state_output_decision : completion_detokenize_run_ [always] / zero__ + state_commit_generated --> state_output_decision : completion_detokenize_run_ [always] / generated__ + state_output_decision --> state_ready : completion_detokenize_run_ [guard_before_output_delay_] / effect_publish_no_output_ + state_output_decision --> state_output_validation : completion_detokenize_run_ [guard_past_output_delay_] / effect_collect_output_ + state_output_validation --> state_ready : completion_detokenize_run_ [guard_output_complete_] / effect_publish_output_ + state_output_validation --> state_ready : completion_detokenize_run_ [guard_output_incomplete_] / effect_publish_no_output_ + state_ready --> state_ready : restore_cache [guard_restore_valid_] / effect_restore_column_major_cache_ + state_ready --> state_ready : restore_cache [guard_restore_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : restore_cache [always] / phase_order__ + state_prepared_generated --> state_prepared_generated : restore_cache [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : restore_cache [always] / phase_order__ + state_prepared_full --> state_ready : advance [guard_advance_position_available_] / effect_advance_ + state_prepared_full --> state_prepared_full : advance [guard_advance_position_overflow_] / position_overflow__ + state_prepared_generated --> state_prepared_generated : advance [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : advance [always] / phase_order__ + state_ready --> state_ready : advance [always] / phase_order__ + state_ready --> state_ready : reset [always] / effect_reset_ + state_prepared_full --> state_ready : reset [always] / effect_reset_ + state_prepared_generated --> state_ready : reset [always] / effect_reset_ + state_prepared_tail --> state_ready : reset [always] / effect_reset_ + state_errored --> state_ready : reset [always] / effect_reset_ + state_uninitialized --> state_uninitialized : tokenize [always] / uninitialized__ + state_uninitialized --> state_uninitialized : detokenize_run [always] / uninitialized__ + state_uninitialized --> state_uninitialized : restore_cache [always] / uninitialized__ + state_uninitialized --> state_uninitialized : advance [always] / uninitialized__ + state_uninitialized --> state_uninitialized : reset [always] / uninitialized__ + state_ready --> state_ready : detokenize_run [always] / phase_order__ + state_errored --> state_errored : tokenize [always] / internal_error__ + state_errored --> state_errored : detokenize_run [always] / internal_error__ + state_errored --> state_errored : restore_cache [always] / internal_error__ + state_errored --> state_errored : advance [always] / internal_error__ + state_uninitialized --> state_errored : _ [always] / effect_unexpected_ + state_ready --> state_errored : _ [always] / effect_unexpected_ + state_prepared_full --> state_errored : _ [always] / effect_unexpected_ + state_prepared_generated --> state_errored : _ [always] / effect_unexpected_ + state_prepared_tail --> state_errored : _ [always] / effect_unexpected_ + state_errored --> state_errored : _ [always] / effect_unexpected_ diff --git a/.planning/architecture/speech_predictor_moshi_executor.md b/.planning/architecture/speech_predictor_moshi_executor.md new file mode 100644 index 00000000..6008b89d --- /dev/null +++ b/.planning/architecture/speech_predictor_moshi_executor.md @@ -0,0 +1,626 @@ +# speech_predictor_moshi_executor + +Source: [`emel/speech/predictor/moshi/executor/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_bind_contract_decision : initialize_run [always] / none + state_bind_contract_decision --> state_input_embedding_contract_decision : completion_initialize_run_ [guard_bind_contract_valid_] / effect_bind_contract_ + state_bind_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bind_contract_invalid_] / effect_mark_bind_failed_ + state_input_embedding_contract_decision --> state_temporal_projection_layout_decision : completion_initialize_run_ [guard_bound_root_operands_supported_] / none + state_input_embedding_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bound_root_operands_unsupported_] / effect_mark_bind_failed_ + state_temporal_projection_layout_decision --> state_depformer_projection_layout_decision : completion_initialize_run_ [guard_temporal_split_projection_layout_supported_] / effect_bind_temporal_split_projection_layout_ + state_temporal_projection_layout_decision --> state_depformer_projection_layout_decision : completion_initialize_run_ [guard_temporal_fused_projection_layout_supported_] / effect_bind_temporal_fused_projection_layout_ + state_temporal_projection_layout_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_temporal_projection_layout_unsupported_] / effect_mark_bind_failed_ + state_depformer_projection_layout_decision --> state_sampling_seed_decision : completion_initialize_run_ [guard_depformer_split_projection_layout_supported_] / effect_bind_depformer_split_projection_layout_ + state_depformer_projection_layout_decision --> state_sampling_seed_decision : completion_initialize_run_ [guard_depformer_fused_projection_layout_supported_] / effect_bind_depformer_fused_projection_layout_ + state_depformer_projection_layout_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_depformer_projection_layout_unsupported_] / effect_mark_bind_failed_ + state_sampling_seed_decision --> state_text_sampling_top_k_decision : completion_initialize_run_ [guard_sampling_seed_nonzero_] / effect_bind_nonzero_sampling_seed_ + state_sampling_seed_decision --> state_text_sampling_top_k_decision : completion_initialize_run_ [guard_sampling_seed_zero_] / effect_bind_zero_sampling_seed_ + state_text_sampling_top_k_decision --> state_audio_sampling_top_k_decision : completion_initialize_run_ [guard_text_sampling_top_k_within_card_] / effect_bind_requested_text_sampling_top_k_ + state_text_sampling_top_k_decision --> state_audio_sampling_top_k_decision : completion_initialize_run_ [guard_text_sampling_top_k_exceeds_card_] / effect_bind_full_card_text_sampling_top_k_ + state_audio_sampling_top_k_decision --> state_init_error_out_decision : completion_initialize_run_ [guard_audio_sampling_top_k_within_card_] / effect_bind_requested_audio_sampling_top_k_ + state_audio_sampling_top_k_decision --> state_init_error_out_decision : completion_initialize_run_ [guard_audio_sampling_top_k_exceeds_card_] / effect_bind_full_card_audio_sampling_top_k_ + state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ + state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / none + state_init_failed_error_out_decision --> state_init_failed_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ + state_init_failed_error_out_decision --> state_init_failed_callback_decision : completion_initialize_run_ [initialize_run__] / none + state_init_callback_decision --> state_ready : completion_initialize_run_ [guard_has_initialize_done_callback_] / effect_emit_initialize_done_ + state_init_callback_decision --> state_ready : completion_initialize_run_ [guard_no_initialize_done_callback_] / none + state_init_failed_callback_decision --> state_uninitialized : completion_initialize_run_ [guard_has_initialize_error_callback_] / effect_emit_initialize_error_ + state_init_failed_callback_decision --> state_uninitialized : completion_initialize_run_ [guard_no_initialize_error_callback_] / none + state_ready --> state_step_model_decision : step_run [always] / none + state_step_model_decision --> state_step_shape_decision : completion_step_run_ [guard_step_model_matches_] / none + state_step_model_decision --> state_step_error_out_decision : completion_step_run_ [guard_step_model_mismatch_] / effect_mark_model_mismatch_ + state_step_shape_decision --> state_step_execution_decision : completion_step_run_ [guard_step_shape_valid_] / none + state_step_shape_decision --> state_step_error_out_decision : completion_step_run_ [guard_step_shape_invalid_] / effect_mark_request_shape_ + state_step_execution_decision --> state_input_embedding : completion_step_run_ [guard_input_embedding_supported_] / effect_begin_input_embedding_ + state_step_execution_decision --> state_step_error_out_decision : completion_step_run_ [guard_input_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_embedding --> state_input_embedding_result_decision : completion_step_run_ [guard_external_input_embedding_supported_] / effect_apply_external_input_embedding_ + state_input_embedding --> state_input_text_embedding_bind : completion_step_run_ [guard_token_input_embedding_supported_] / effect_bind_input_text_embedding_ + state_input_embedding --> state_step_error_out_decision : completion_step_run_ [guard_token_input_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_bind --> state_input_text_embedding_bind_result_decision : completion_step_run_ [always] / none + state_input_text_embedding_bind_result_decision --> state_input_text_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_input_text_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_row --> state_input_text_embedding_row_result_decision : completion_step_run_ [always] / none + state_input_text_embedding_row_result_decision --> state_input_text_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_input_text_embedding_row_ + state_input_text_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_apply --> state_input_text_embedding_result_decision : completion_step_run_ [guard_input_text_embedding_succeeded_] / none + state_input_text_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_input_text_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_text_embedding_result_decision --> state_input_audio_token_decision : completion_step_run_ [always] / none + state_input_audio_token_decision --> state_input_audio_embedding : completion_step_run_ [guard_current_input_audio_token_present_] / effect_bind_input_audio_embedding_ + state_input_audio_token_decision --> state_input_audio_embedding_result_decision : completion_step_run_ [guard_current_input_audio_token_zero_] / effect_skip_zero_input_audio_embedding_ + state_input_audio_token_decision --> state_step_error_out_decision : completion_step_run_ [guard_current_input_audio_token_unsupported_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding --> state_input_audio_embedding_bind_result_decision : completion_step_run_ [always] / none + state_input_audio_embedding_bind_result_decision --> state_input_audio_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_input_audio_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_row --> state_input_audio_embedding_row_result_decision : completion_step_run_ [always] / none + state_input_audio_embedding_row_result_decision --> state_input_audio_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_input_audio_embedding_row_ + state_input_audio_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_apply --> state_input_audio_embedding_result_decision : completion_step_run_ [guard_input_audio_embedding_succeeded_] / none + state_input_audio_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_input_audio_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_audio_embedding_result_decision --> state_input_audio_codebook_advance : completion_step_run_ [guard_more_input_audio_codebooks_] / effect_advance_input_audio_codebook_ + state_input_audio_codebook_advance --> state_input_audio_token_decision : completion_step_run_ [always] / none + state_input_audio_embedding_result_decision --> state_input_embedding_result_decision : completion_step_run_ [guard_input_audio_codebooks_complete_] / effect_finish_input_embedding_ + state_input_embedding_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_input_embedding_failed_] / effect_mark_graph_execution_unsupported_ + state_input_embedding_result_decision --> state_bind_temporal_kv : completion_step_run_ [guard_temporal_kv_binding_present_] / effect_bind_temporal_kv_ + state_input_embedding_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_kv_binding_missing_] / effect_mark_graph_execution_unsupported_ + state_bind_temporal_kv --> state_temporal_kv_result_decision : completion_step_run_ [guard_temporal_kv_bound_] / none + state_bind_temporal_kv --> state_step_error_out_decision : completion_step_run_ [guard_temporal_kv_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_kv_result_decision --> state_temporal_position_result_decision : completion_step_run_ [always] / effect_advance_temporal_position_ + state_temporal_position_result_decision --> state_temporal_position_valid_decision : completion_step_run_ [guard_temporal_position_advance_succeeded_] / none + state_temporal_position_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_position_advance_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_position_valid_decision --> state_temporal_layer_norm : completion_step_run_ [guard_temporal_layer_norm_supported_] / effect_run_temporal_layer_norm_rms_ + state_temporal_position_valid_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm --> state_temporal_layer_norm_rms_result_decision : completion_step_run_ [always] / none + state_temporal_layer_norm_rms_result_decision --> state_temporal_layer_norm_scale : completion_step_run_ [guard_temporal_layer_norm_rms_succeeded_] / effect_run_temporal_layer_norm_scale_ + state_temporal_layer_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm_scale --> state_temporal_layer_norm_result_decision : completion_step_run_ [guard_temporal_layer_norm_succeeded_] / none + state_temporal_layer_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm_result_decision --> state_temporal_layer_projection : completion_step_run_ [guard_temporal_layer_projection_supported_] / effect_bind_temporal_layer_projection_ + state_temporal_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection --> state_temporal_layer_projection_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_projection_ + state_temporal_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection_run --> state_temporal_layer_projection_result_decision : completion_step_run_ [guard_temporal_layer_projection_succeeded_] / none + state_temporal_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_projection_result_decision --> state_temporal_layer_rope : completion_step_run_ [guard_temporal_layer_rope_supported_] / effect_run_temporal_layer_rope_0__ + state_temporal_layer_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_rope --> state_temporal_layer_rope_result_decision : completion_step_run_ [guard_temporal_layer_rope_succeeded_] / effect_copy_temporal_layer_rope_0__ + state_temporal_layer_rope --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_rope_result_decision --> state_temporal_layer_key_rope : completion_step_run_ [always] / effect_run_temporal_layer_rope_1__ + state_temporal_layer_key_rope --> state_temporal_layer_key_rope_result_decision : completion_step_run_ [guard_temporal_layer_rope_succeeded_] / effect_copy_temporal_layer_rope_1__ + state_temporal_layer_key_rope --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_rope_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_key_rope_result_decision --> state_temporal_layer_cache_write : completion_step_run_ [guard_temporal_layer_cache_write_supported_] / effect_write_temporal_layer_kv_cache_ + state_temporal_layer_key_rope_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write --> state_temporal_layer_cache_write_result_decision : completion_step_run_ [guard_temporal_layer_cache_write_succeeded_] / none + state_temporal_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_supported_] / effect_run_temporal_layer_attention_ + state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_attention --> state_temporal_layer_attention_result_decision : completion_step_run_ [guard_temporal_layer_attention_succeeded_] / none + state_temporal_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_attention_result_decision --> state_temporal_layer_out_projection : completion_step_run_ [guard_temporal_layer_out_projection_supported_] / effect_bind_temporal_layer_out_projection_ + state_temporal_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection --> state_temporal_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_out_projection_ + state_temporal_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection_run --> state_temporal_layer_out_projection_result_decision : completion_step_run_ [guard_temporal_layer_out_projection_succeeded_] / none + state_temporal_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_out_projection_result_decision --> state_temporal_layer_residual : completion_step_run_ [always] / effect_apply_temporal_layer_attention_residual_ + state_temporal_layer_residual --> state_temporal_layer_residual_result_decision : completion_step_run_ [guard_temporal_layer_residual_succeeded_] / none + state_temporal_layer_residual --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_residual_result_decision --> state_temporal_layer_norm2 : completion_step_run_ [guard_temporal_layer_norm2_supported_] / effect_run_temporal_layer_norm2_rms_ + state_temporal_layer_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2 --> state_temporal_layer_norm2_rms_result_decision : completion_step_run_ [always] / none + state_temporal_layer_norm2_rms_result_decision --> state_temporal_layer_norm2_scale : completion_step_run_ [guard_temporal_layer_norm2_rms_succeeded_] / effect_run_temporal_layer_norm2_scale_ + state_temporal_layer_norm2_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2_scale --> state_temporal_layer_norm2_result_decision : completion_step_run_ [guard_temporal_layer_norm2_succeeded_] / none + state_temporal_layer_norm2_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm2_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_norm2_result_decision --> state_temporal_layer_gating_in : completion_step_run_ [guard_temporal_layer_gating_in_supported_] / effect_bind_temporal_layer_gating_in_ + state_temporal_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in --> state_temporal_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_in_ + state_temporal_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in_run --> state_temporal_layer_gating_in_result_decision : completion_step_run_ [guard_temporal_layer_gating_in_succeeded_] / none + state_temporal_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_in_result_decision --> state_temporal_layer_silu_gate : completion_step_run_ [guard_temporal_layer_silu_gate_supported_] / effect_run_temporal_layer_silu_gate_silu_ + state_temporal_layer_gating_in_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate --> state_temporal_layer_silu_gate_silu_result_decision : completion_step_run_ [always] / none + state_temporal_layer_silu_gate_silu_result_decision --> state_temporal_layer_silu_gate_mul : completion_step_run_ [guard_temporal_layer_silu_gate_silu_succeeded_] / effect_run_temporal_layer_silu_gate_mul_ + state_temporal_layer_silu_gate_silu_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_silu_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate_mul --> state_temporal_layer_silu_gate_result_decision : completion_step_run_ [guard_temporal_layer_silu_gate_succeeded_] / none + state_temporal_layer_silu_gate_mul --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_silu_gate_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_silu_gate_result_decision --> state_temporal_layer_gating_out : completion_step_run_ [guard_temporal_layer_gating_out_supported_] / effect_bind_temporal_layer_gating_out_ + state_temporal_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out --> state_temporal_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_out_ + state_temporal_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out_run --> state_temporal_layer_gating_out_result_decision : completion_step_run_ [guard_temporal_layer_gating_out_succeeded_] / none + state_temporal_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_gating_out_result_decision --> state_temporal_layer_ff_residual : completion_step_run_ [always] / effect_apply_temporal_layer_ff_residual_ + state_temporal_layer_ff_residual --> state_temporal_layer_ff_residual_result_decision : completion_step_run_ [guard_temporal_layer_ff_residual_succeeded_] / none + state_temporal_layer_ff_residual --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_ff_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_ff_residual_result_decision --> state_temporal_layer_advance : completion_step_run_ [guard_more_temporal_layers_] / effect_advance_temporal_layer_ + state_temporal_layer_ff_residual_result_decision --> state_temporal_out_norm : completion_step_run_ [guard_temporal_layers_complete_] / effect_run_temporal_out_norm_rms_ + state_temporal_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_advance --> state_temporal_layer_norm : completion_step_run_ [guard_temporal_layer_norm_supported_] / effect_run_temporal_layer_norm_rms_ + state_temporal_layer_advance --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm --> state_temporal_out_norm_rms_result_decision : completion_step_run_ [always] / none + state_temporal_out_norm_rms_result_decision --> state_temporal_out_norm_scale : completion_step_run_ [guard_temporal_out_norm_rms_succeeded_] / effect_run_temporal_out_norm_scale_ + state_temporal_out_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm_scale --> state_temporal_out_norm_result_decision : completion_step_run_ [guard_temporal_out_norm_succeeded_] / effect_publish_temporal_out_norm_ + state_temporal_out_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_temporal_out_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_temporal_out_norm_result_decision --> state_text_logits_phase_decision : completion_step_run_ [guard_full_graph_phase_] / none + state_temporal_out_norm_result_decision --> state_sampling_ready : completion_step_run_ [guard_prediction_graph_phase_] / effect_publish_prediction_state_ + state_text_logits_phase_decision --> state_text_logits : completion_step_run_ [guard_text_logits_supported_] / effect_bind_text_token_logits_ + state_text_logits_phase_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_unsupported_] / effect_mark_graph_execution_unsupported_ + state_sampling_ready --> state_text_logits : step_run [guard_sampling_step_valid_] / effect_restore_prediction_state_and_bind_text_logits_ + state_sampling_ready --> state_step_error_out_decision : step_run [guard_sampling_step_invalid_] / effect_mark_request_shape_ + state_text_logits --> state_text_logits_bind_result_decision : completion_step_run_ [always] / none + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_valid_and_sampling_consume_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_token_valid_without_sampling_consume_] / effect_publish_forced_text_token_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_sampling_config_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_text_logits_run : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_argmax_] / effect_select_text_token_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_sample_projection --> state_text_logits_sample_projection_result_decision : completion_step_run_ [always] / none + state_text_logits_sample_projection_result_decision --> state_text_logits_sample_select : completion_step_run_ [guard_text_logits_matmul_succeeded_] / effect_select_text_sampling_token_ + state_text_logits_sample_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_matmul_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_sample_select --> state_text_logits_result_decision : completion_step_run_ [guard_sampled_text_token_ready_] / effect_publish_sampled_text_token_ + state_text_logits_sample_select --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_sampling_consumed_] / effect_publish_forced_text_token_ + state_text_logits_sample_select --> state_step_error_out_decision : completion_step_run_ [guard_text_sampling_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_run --> state_text_logits_result_decision : completion_step_run_ [guard_text_logits_succeeded_] / none + state_text_logits_run --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_failed_] / effect_mark_graph_execution_unsupported_ + state_text_logits_result_decision --> state_bind_depformer_kv : completion_step_run_ [guard_depformer_kv_binding_present_] / effect_bind_depformer_kv_ + state_text_logits_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_kv_binding_missing_] / effect_mark_graph_execution_unsupported_ + state_bind_depformer_kv --> state_depformer_kv_result_decision : completion_step_run_ [guard_depformer_kv_bound_] / effect_reset_depformer_positions_ + state_bind_depformer_kv --> state_step_error_out_decision : completion_step_run_ [guard_depformer_kv_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_kv_result_decision --> state_depformer_position_reset_result_decision : completion_step_run_ [always] / none + state_depformer_position_reset_result_decision --> state_depformer_position_advance_result_decision : completion_step_run_ [guard_depformer_position_reset_succeeded_] / effect_advance_depformer_position_ + state_depformer_position_reset_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_position_reset_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_position_advance_result_decision --> state_depformer_weight_route_decision : completion_step_run_ [guard_depformer_position_advance_succeeded_] / none + state_depformer_position_advance_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_position_advance_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_weight_route_decision --> state_depformer_weight_decision : completion_step_run_ [guard_depformer_scheduled_weight_present_] / effect_use_depformer_scheduled_weight_ + state_depformer_weight_route_decision --> state_depformer_weight_decision : completion_step_run_ [guard_depformer_scheduled_weight_absent_] / effect_use_depformer_codebook_weight_ + state_depformer_weight_route_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_scheduled_weight_invalid_] / effect_mark_graph_execution_unsupported_ + state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_text_input_supported_] / effect_bind_depformer_text_input_projection_ + state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_audio_input_supported_] / effect_bind_depformer_audio_input_projection_ + state_depformer_weight_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_input --> state_depformer_input_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_bound_] / effect_run_depformer_input_projection_ + state_depformer_input_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_projection --> state_depformer_input_projection_result_decision : completion_step_run_ [always] / none + state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_text_input_projection_succeeded_] / effect_bind_depformer_text_input_embedding_ + state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_audio_input_projection_succeeded_] / effect_bind_depformer_audio_input_embedding_ + state_depformer_input_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_embedding_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_bind --> state_depformer_input_embedding_bind_result_decision : completion_step_run_ [always] / none + state_depformer_input_embedding_bind_result_decision --> state_depformer_input_embedding_row : completion_step_run_ [guard_embedding_view_bound_] / effect_run_embedding_row_fetch_ + state_depformer_input_embedding_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_row --> state_depformer_input_embedding_row_result_decision : completion_step_run_ [always] / none + state_depformer_input_embedding_row_result_decision --> state_depformer_input_embedding_apply : completion_step_run_ [guard_embedding_row_succeeded_] / effect_apply_depformer_input_embedding_row_ + state_depformer_input_embedding_row_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_embedding_row_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_embedding_apply --> state_depformer_input_result_decision : completion_step_run_ [guard_depformer_input_succeeded_] / none + state_depformer_input_embedding_apply --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_input_result_decision --> state_depformer_layer_norm : completion_step_run_ [guard_depformer_layer_norm_supported_] / effect_run_depformer_layer_norm_rms_ + state_depformer_input_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm --> state_depformer_layer_norm_rms_result_decision : completion_step_run_ [always] / none + state_depformer_layer_norm_rms_result_decision --> state_depformer_layer_norm_scale : completion_step_run_ [guard_depformer_layer_norm_rms_succeeded_] / effect_run_depformer_layer_norm_scale_ + state_depformer_layer_norm_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm_scale --> state_depformer_layer_norm_result_decision : completion_step_run_ [guard_depformer_layer_norm_succeeded_] / none + state_depformer_layer_norm_scale --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm_result_decision --> state_depformer_layer_projection : completion_step_run_ [guard_depformer_layer_projection_supported_] / effect_bind_depformer_layer_projection_ + state_depformer_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection --> state_depformer_layer_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_projection_ + state_depformer_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection_run --> state_depformer_layer_projection_result_decision : completion_step_run_ [guard_depformer_layer_projection_succeeded_] / none + state_depformer_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_projection_result_decision --> state_depformer_layer_cache_write : completion_step_run_ [guard_depformer_layer_cache_write_supported_] / effect_write_depformer_layer_kv_cache_ + state_depformer_layer_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_cache_write --> state_depformer_layer_cache_write_result_decision : completion_step_run_ [guard_depformer_layer_cache_write_succeeded_] / none + state_depformer_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_cache_write_result_decision --> state_depformer_layer_attention : completion_step_run_ [guard_depformer_layer_attention_supported_] / effect_run_depformer_layer_attention_ + state_depformer_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_attention --> state_depformer_layer_attention_result_decision : completion_step_run_ [guard_depformer_layer_attention_succeeded_] / none + state_depformer_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_attention_result_decision --> state_depformer_layer_out_projection : completion_step_run_ [guard_depformer_layer_out_projection_supported_] / effect_bind_depformer_layer_out_projection_ + state_depformer_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection --> state_depformer_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_out_projection_ + state_depformer_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection_run --> state_depformer_layer_out_projection_result_decision : completion_step_run_ [guard_depformer_layer_out_projection_succeeded_] / none + state_depformer_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_out_projection_result_decision --> state_depformer_layer_residual : completion_step_run_ [always] / effect_apply_depformer_layer_attention_residual_ + state_depformer_layer_residual --> state_depformer_layer_residual_result_decision : completion_step_run_ [guard_depformer_layer_residual_succeeded_] / none + state_depformer_layer_residual --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_residual_result_decision --> state_depformer_layer_norm2 : completion_step_run_ [guard_depformer_layer_norm2_supported_] / effect_run_depformer_layer_norm2_rms_ + state_depformer_layer_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2 --> state_depformer_layer_norm2_rms_result_decision : completion_step_run_ [always] / none + state_depformer_layer_norm2_rms_result_decision --> state_depformer_layer_norm2_scale : completion_step_run_ [guard_depformer_layer_norm2_rms_succeeded_] / effect_run_depformer_layer_norm2_scale_ + state_depformer_layer_norm2_rms_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_rms_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2_scale --> state_depformer_layer_norm2_result_decision : completion_step_run_ [guard_depformer_layer_norm2_succeeded_] / none + state_depformer_layer_norm2_scale --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm2_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_norm2_result_decision --> state_depformer_layer_gating_in : completion_step_run_ [guard_depformer_layer_gating_in_supported_] / effect_bind_depformer_layer_gating_in_ + state_depformer_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in --> state_depformer_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_in_ + state_depformer_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in_run --> state_depformer_layer_gating_in_result_decision : completion_step_run_ [guard_depformer_layer_gating_in_succeeded_] / none + state_depformer_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_in_result_decision --> state_depformer_layer_silu_gate : completion_step_run_ [guard_depformer_layer_silu_gate_supported_] / effect_run_depformer_layer_silu_gate_silu_ + state_depformer_layer_gating_in_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate --> state_depformer_layer_silu_gate_silu_result_decision : completion_step_run_ [always] / none + state_depformer_layer_silu_gate_silu_result_decision --> state_depformer_layer_silu_gate_mul : completion_step_run_ [guard_depformer_layer_silu_gate_silu_succeeded_] / effect_run_depformer_layer_silu_gate_mul_ + state_depformer_layer_silu_gate_silu_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_silu_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate_mul --> state_depformer_layer_silu_gate_result_decision : completion_step_run_ [guard_depformer_layer_silu_gate_succeeded_] / none + state_depformer_layer_silu_gate_mul --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_silu_gate_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_silu_gate_result_decision --> state_depformer_layer_gating_out : completion_step_run_ [guard_depformer_layer_gating_out_supported_] / effect_bind_depformer_layer_gating_out_ + state_depformer_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out --> state_depformer_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_out_ + state_depformer_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out_run --> state_depformer_layer_gating_out_result_decision : completion_step_run_ [guard_depformer_layer_gating_out_succeeded_] / none + state_depformer_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_gating_out_result_decision --> state_depformer_layer_ff_residual : completion_step_run_ [always] / effect_apply_depformer_layer_ff_residual_ + state_depformer_layer_ff_residual --> state_depformer_layer_ff_residual_result_decision : completion_step_run_ [guard_depformer_layer_ff_residual_succeeded_] / none + state_depformer_layer_ff_residual --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_ff_residual_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_ff_residual_result_decision --> state_depformer_layer_advance : completion_step_run_ [guard_more_depformer_layers_] / effect_advance_depformer_layer_ + state_depformer_layer_ff_residual_result_decision --> state_depformer_logits : completion_step_run_ [guard_depformer_logits_supported_] / effect_bind_depformer_token_logits_ + state_depformer_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits --> state_depformer_logits_bind_result_decision : completion_step_run_ [always] / none + state_depformer_logits_bind_result_decision --> state_depformer_logits_run : completion_step_run_ [guard_depformer_logits_projection_bound_argmax_] / effect_select_depformer_token_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_] / effect_compute_depformer_token_logits_ + state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_invalid_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_sample_projection --> state_depformer_logits_sample_projection_result_decision : completion_step_run_ [always] / none + state_depformer_logits_sample_projection_result_decision --> state_depformer_logits_sample_select : completion_step_run_ [guard_depformer_logits_matmul_succeeded_] / effect_select_depformer_sampling_token_ + state_depformer_logits_sample_projection_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_matmul_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_layer_advance --> state_depformer_layer_norm : completion_step_run_ [guard_depformer_layer_norm_supported_] / effect_run_depformer_layer_norm_rms_ + state_depformer_layer_advance --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_norm_unsupported_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_run --> state_depformer_token_publish : completion_step_run_ [guard_depformer_logits_succeeded_] / effect_publish_depformer_token_ + state_depformer_logits_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_logits_sample_select --> state_depformer_token_publish : completion_step_run_ [guard_depformer_sampling_succeeded_] / effect_publish_depformer_token_ + state_depformer_logits_sample_select --> state_step_error_out_decision : completion_step_run_ [guard_depformer_sampling_failed_] / effect_mark_graph_execution_unsupported_ + state_depformer_token_publish --> state_depformer_logits_result_decision : completion_step_run_ [always] / none + state_depformer_logits_result_decision --> state_depformer_codebook_advance : completion_step_run_ [guard_more_depformer_codebooks_] / effect_advance_depformer_codebook_ + state_depformer_logits_result_decision --> state_ready : completion_step_run_ [guard_depformer_codebooks_complete_] / none + state_depformer_codebook_advance --> state_depformer_position_advance_result_decision : completion_step_run_ [always] / effect_advance_depformer_position_ + state_step_error_out_decision --> state_ready : completion_step_run_ [step_run__] / step_run__ + state_step_error_out_decision --> state_ready : completion_step_run_ [step_run__] / none + state_uninitialized --> state_uninit_step_error_out_decision : step_run [always] / effect_mark_not_initialized_ + state_uninit_step_error_out_decision --> state_uninitialized : completion_step_run_ [step_run__] / step_run__ + state_uninit_step_error_out_decision --> state_uninitialized : completion_step_run_ [step_run__] / none + state_ready --> state_reset_temporal_positions_decision : reset_run [always] / none + state_sampling_ready --> state_reset_temporal_positions_decision : reset_run [always] / none + state_reset_temporal_positions_decision --> state_reset_temporal_positions_result_decision : completion_reset_run_ [guard_reset_temporal_positions_present_] / effect_reset_temporal_positions_ + state_reset_temporal_positions_decision --> state_reset_depformer_positions_decision : completion_reset_run_ [guard_reset_temporal_positions_missing_] / none + state_reset_temporal_positions_result_decision --> state_reset_depformer_positions_decision : completion_reset_run_ [guard_reset_temporal_positions_succeeded_] / none + state_reset_temporal_positions_result_decision --> state_reset_failed : completion_reset_run_ [guard_reset_temporal_positions_failed_] / effect_mark_reset_failed_ + state_reset_depformer_positions_decision --> state_reset_depformer_positions_result_decision : completion_reset_run_ [guard_reset_depformer_positions_present_] / effect_reset_bound_depformer_positions_ + state_reset_depformer_positions_decision --> state_uninitialized : completion_reset_run_ [guard_reset_depformer_positions_missing_] / effect_reset_session_ + state_reset_depformer_positions_result_decision --> state_uninitialized : completion_reset_run_ [guard_reset_depformer_positions_succeeded_] / effect_reset_session_ + state_reset_depformer_positions_result_decision --> state_reset_failed : completion_reset_run_ [guard_reset_depformer_positions_failed_] / effect_mark_reset_failed_ + state_reset_failed --> state_uninitialized : completion_reset_run_ [always] / effect_reset_session_ + state_uninitialized --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_uninitialized --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_ready --> state_ready : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_ready --> state_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_sampling_ready --> state_sampling_ready : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_sampling_ready --> state_sampling_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_bind_contract_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_contract>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_embedding_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_bind_contract_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_bound_root_operands_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_bound_root_operands_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_split_projection_layout_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_split_projection_layout>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_fused_projection_layout_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_fused_projection_layout>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_projection_layout_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_split_projection_layout_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_split_projection_layout>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_sampling_seed_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_fused_projection_layout_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_fused_projection_layout>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_sampling_seed_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_projection_layout_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_projection_layout_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_seed_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_seed_nonzero>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_nonzero_sampling_seed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_seed_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_seed_zero>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_zero_sampling_seed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_sampling_top_k_within_card>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_requested_text_sampling_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_audio_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_sampling_top_k_exceeds_card>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_full_card_text_sampling_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_audio_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_audio_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_audio_sampling_top_k_within_card>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_requested_audio_sampling_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_audio_sampling_top_k_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_audio_sampling_top_k_exceeds_card>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_full_card_audio_sampling_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_init_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_has_initialize_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_emit_initialize_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_no_initialize_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_has_initialize_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_emit_initialize_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_init_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_no_initialize_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_model_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_model_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_step_model_matches>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_shape_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_model_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_step_model_mismatch>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_model_mismatch>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_shape_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_step_shape_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_execution_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_shape_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_step_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_request_shape>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_execution_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_embedding_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_begin_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_execution_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_embedding_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_external_input_embedding_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_external_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_token_input_embedding_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_input_text_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_token_input_embedding_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_embedding_row_fetch>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_input_text_embedding_row>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_text_embedding_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_text_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_text_embedding_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_text_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_token_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_token_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_current_input_audio_token_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_input_audio_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_token_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_current_input_audio_token_zero>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_skip_zero_input_audio_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_token_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_current_input_audio_token_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_embedding_row_fetch>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_input_audio_embedding_row>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_audio_embedding_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_audio_embedding_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_more_input_audio_codebooks>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_input_audio_codebook>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_codebook_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_codebook_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_audio_token_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_audio_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_audio_codebooks_complete>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_finish_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_input_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_input_embedding_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_kv_binding_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_kv>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_bind_temporal_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_input_embedding_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_kv_binding_missing>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_temporal_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_kv_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_kv_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_temporal_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_kv_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_kv_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_temporal_position>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_position_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_position_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_position_advance_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_position_valid_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_position_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_position_advance_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_position_valid_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_norm_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_position_valid_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_rms_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_norm_scale>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_rms_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_rope<0>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_copy_temporal_layer_rope<0>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_rope<1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_key_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_key_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_copy_temporal_layer_rope<1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_key_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_key_rope`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_rope_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_key_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_write_temporal_layer_kv_cache>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_key_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_temporal_layer_attention_residual>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_residual_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_residual_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_norm2_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm2`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_rms_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_norm2_scale>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_rms_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm2_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_silu_gate_silu>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_silu_gate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_silu_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_silu_gate_mul>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_silu_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_silu_gate_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_temporal_layer_ff_residual>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_ff_residual_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_ff_residual_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_more_temporal_layers>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_temporal_layer>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layers_complete>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_out_norm_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_out_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_out_norm_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_norm_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_norm_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_out_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_out_norm_rms_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_out_norm_scale>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_out_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_out_norm_rms_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_out_norm_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_temporal_out_norm>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_out_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_out_norm_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_full_graph_phase>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_phase_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_out_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_prediction_graph_phase>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_prediction_state>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_phase_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_text_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_phase_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_step_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_restore_prediction_state_and_bind_text_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_step_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_request_shape>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_valid_and_sampling_consume>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_valid_without_sampling_consume>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_forced_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_sampling_config_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_sampling>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_matmul_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_text_sampling_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_matmul_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampled_text_token_ready>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_sampled_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_sampling_consumed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_forced_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_sampling_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_kv_binding_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_kv>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_bind_depformer_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_kv_binding_missing>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_depformer_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_kv_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_depformer_positions>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_kv_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_bind_depformer_kv`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_kv_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_kv_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_position_reset_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_position_reset_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_position_reset_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_depformer_position>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_position_advance_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_position_reset_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_position_reset_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_position_advance_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_position_advance_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_weight_route_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_position_advance_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_position_advance_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_route_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_scheduled_weight_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_use_depformer_scheduled_weight>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_route_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_scheduled_weight_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_use_depformer_codebook_weight>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_route_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_scheduled_weight_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_text_input_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_text_input_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_audio_input_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_audio_input_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_input_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_text_input_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_text_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_audio_input_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_audio_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_embedding_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_embedding_row_fetch>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_row`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_depformer_input_embedding_row>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_row_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_embedding_row_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_embedding_apply`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_norm_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_rms_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_norm_scale>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_rms_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_cache_write_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_write_depformer_layer_kv_cache>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_cache_write_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_cache_write_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_cache_write_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_attention_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_attention>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_attention_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_attention_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_attention_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_depformer_layer_attention_residual>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_residual_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_residual_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_norm2_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm2`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_rms_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_norm2_scale>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_rms_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_rms_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm2_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_silu_gate_silu>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_silu_gate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_silu_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_silu_gate_mul>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_silu_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_silu_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_mul`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_silu_gate_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_apply_depformer_layer_ff_residual>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_ff_residual_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_ff_residual`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_ff_residual_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_more_depformer_layers>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_depformer_layer>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_depformer_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_sampling>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_depformer_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_sampling_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_matmul_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_depformer_sampling_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_matmul_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_norm_rms>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_norm`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_norm_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_depformer_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_token_publish`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_sampling_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_depformer_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_token_publish`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_sample_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_sampling_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_token_publish`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_more_depformer_codebooks>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_depformer_codebook>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_codebook_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_codebooks_complete>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_codebook_advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_advance_depformer_position>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_position_advance_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_not_initialized>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninit_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_uninit_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_uninit_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`reset_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_temporal_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`reset_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_temporal_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_temporal_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_temporal_positions_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_temporal_positions>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_temporal_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_temporal_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_temporal_positions_missing>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_depformer_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_temporal_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_temporal_positions_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_depformer_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_temporal_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_temporal_positions_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_reset_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_depformer_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_depformer_positions_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_bound_depformer_positions>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_depformer_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_depformer_positions_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_depformer_positions_missing>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_session>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_depformer_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_depformer_positions_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_session>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_depformer_positions_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_reset_depformer_positions_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_reset_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_reset_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_reset_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_reset_session>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | diff --git a/.planning/architecture/speech_tokenizer_moshi.md b/.planning/architecture/speech_tokenizer_moshi.md new file mode 100644 index 00000000..4d936f23 --- /dev/null +++ b/.planning/architecture/speech_tokenizer_moshi.md @@ -0,0 +1,150 @@ +# speech_tokenizer_moshi + +Source: [`emel/speech/tokenizer/moshi/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_uninitialized + state_uninitialized --> state_ready : initialize [guard_configuration_valid_] / effect_initialize_ + state_uninitialized --> state_uninitialized : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_ready --> state_ready : initialize [always] / already_initialized__ + state_prepared_full --> state_prepared_full : initialize [always] / already_initialized__ + state_prepared_generated --> state_prepared_generated : initialize [always] / already_initialized__ + state_prepared_tail --> state_prepared_tail : initialize [always] / already_initialized__ + state_errored --> state_ready : initialize [guard_configuration_valid_] / effect_initialize_ + state_errored --> state_errored : initialize [guard_configuration_invalid_] / invalid_configuration__ + state_ready --> state_prepared_full : tokenize [guard_tokenize_full_] / effect_tokenize_full_ + state_ready --> state_prepared_tail : tokenize [guard_tokenize_tail_] / effect_tokenize_tail_ + state_ready --> state_prepared_generated : tokenize [guard_tokenize_empty_] / effect_tokenize_empty_ + state_ready --> state_ready : tokenize [guard_tokenize_invalid_] / request_shape__ + state_ready --> state_ready : tokenize [guard_tokenize_position_overflow_] / position_overflow__ + state_prepared_full --> state_prepared_full : tokenize [always] / phase_order__ + state_prepared_generated --> state_prepared_generated : tokenize [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : tokenize [always] / phase_order__ + state_prepared_full --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_full --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_generated --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_generated --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : detokenize_run [guard_position_overflow_] / position_overflow__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_position_overflow_] / position_overflow__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_position_overflow_] / position_overflow__ + state_commit_full_zero --> state_output_decision : completion_detokenize_run_ [always] / zero__ + state_commit_full_generated --> state_output_decision : completion_detokenize_run_ [always] / generated__ + state_commit_generated_zero --> state_output_decision : completion_detokenize_run_ [always] / zero__ + state_commit_generated --> state_output_decision : completion_detokenize_run_ [always] / generated__ + state_output_decision --> state_ready : completion_detokenize_run_ [guard_before_output_delay_] / effect_publish_no_output_ + state_output_decision --> state_output_validation : completion_detokenize_run_ [guard_past_output_delay_] / effect_collect_output_ + state_output_validation --> state_ready : completion_detokenize_run_ [guard_output_complete_] / effect_publish_output_ + state_output_validation --> state_ready : completion_detokenize_run_ [guard_output_incomplete_] / effect_publish_no_output_ + state_ready --> state_ready : restore_cache [guard_restore_valid_] / effect_restore_column_major_cache_ + state_ready --> state_ready : restore_cache [guard_restore_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : restore_cache [always] / phase_order__ + state_prepared_generated --> state_prepared_generated : restore_cache [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : restore_cache [always] / phase_order__ + state_prepared_full --> state_ready : advance [guard_advance_position_available_] / effect_advance_ + state_prepared_full --> state_prepared_full : advance [guard_advance_position_overflow_] / position_overflow__ + state_prepared_generated --> state_prepared_generated : advance [always] / phase_order__ + state_prepared_tail --> state_prepared_tail : advance [always] / phase_order__ + state_ready --> state_ready : advance [always] / phase_order__ + state_ready --> state_ready : reset [always] / effect_reset_ + state_prepared_full --> state_ready : reset [always] / effect_reset_ + state_prepared_generated --> state_ready : reset [always] / effect_reset_ + state_prepared_tail --> state_ready : reset [always] / effect_reset_ + state_errored --> state_ready : reset [always] / effect_reset_ + state_uninitialized --> state_uninitialized : tokenize [always] / uninitialized__ + state_uninitialized --> state_uninitialized : detokenize_run [always] / uninitialized__ + state_uninitialized --> state_uninitialized : restore_cache [always] / uninitialized__ + state_uninitialized --> state_uninitialized : advance [always] / uninitialized__ + state_uninitialized --> state_uninitialized : reset [always] / uninitialized__ + state_ready --> state_ready : detokenize_run [always] / phase_order__ + state_errored --> state_errored : tokenize [always] / internal_error__ + state_errored --> state_errored : detokenize_run [always] / internal_error__ + state_errored --> state_errored : restore_cache [always] / internal_error__ + state_errored --> state_errored : advance [always] / internal_error__ + state_uninitialized --> state_errored : _ [always] / effect_unexpected_ + state_ready --> state_errored : _ [always] / effect_unexpected_ + state_prepared_full --> state_errored : _ [always] / effect_unexpected_ + state_prepared_generated --> state_errored : _ [always] / effect_unexpected_ + state_prepared_tail --> state_errored : _ [always] / effect_unexpected_ + state_errored --> state_errored : _ [always] / effect_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_configuration_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_initialize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_configuration_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`invalid_configuration>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`already_initialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_configuration_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_initialize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`initialize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_configuration_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`invalid_configuration>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_tokenize_full>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_tokenize_full>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_tokenize_tail>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_tokenize_tail>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_tokenize_empty>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_tokenize_empty>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_tokenize_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_tokenize_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_commit_full_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`zero>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_commit_full_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`generated>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_commit_generated_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`zero>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_commit_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`generated>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_before_output_delay>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_publish_no_output>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_output_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_past_output_delay>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_collect_output>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_output_validation`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_output_validation`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_output_complete>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_publish_output>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_output_validation`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_output_incomplete>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_publish_no_output>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_restore_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_restore_column_major_cache>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_restore_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_advance_position_available>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_advance>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_advance_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_reset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`reset`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`uninitialized>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`restore_cache`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`advance`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`internal_error>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | diff --git a/.planning/debug/moshi-personaplex-audio-parity.md b/.planning/debug/moshi-personaplex-audio-parity.md new file mode 100644 index 00000000..e8e5f665 --- /dev/null +++ b/.planning/debug/moshi-personaplex-audio-parity.md @@ -0,0 +1,453 @@ +--- +status: resolved +trigger: "Make EMEL PersonaPlex Moshi generate real speech end to end from the same macOS say WAV input as moshi.cpp, with fixed-seed reference comparison showing closely matching audio/token behavior and an output WAV the user can hear. CPU only." +created: 2026-07-09 +updated: 2026-07-10 +--- + +# Moshi PersonaPlex Audio Parity + +## Symptoms + +- expected: EMEL and moshi.cpp consume the same macOS `say` WAV with seed 1234 and produce closely matching PersonaPlex speech audio and token behavior on CPU. +- actual: both CPU lanes complete and emit non-silent speech, but the original moshi.cpp + reference run dropped 8,277 samples from the input WAV. Its prior output and the EMEL + schedule derived from it are not valid evidence of correct end-to-end input handling. +- errors: none in the completed EMEL or CPU-only reference run. +- timeline: implementation and E2E comparison are complete; scoped repository gates remain. +- reproduction: run `build/emel_moshi_say_e2e_current` with the converted Mimi, Moshi LM, NATF0 voice, and `build/moshi_e2e/say_input_24k.wav` inputs. + +## Current Focus + +- hypothesis: confirmed. Temporal timestep-embedding RoPE was the first divergent + arithmetic operation because EMEL contracted reference-separated f32 products into FMA. +- test: the captured frame-9/layer-26/head-9 pair now dispatches through the public CPU + kernel actor and requires reference bits `0x3f908001` / `0x3ea6cd3b`. +- result: the fresh unchanged-threshold 125-frame comparator passes with exact public + input, output, first-four, and text tokens plus effectively identical audio behavior. +- next_action: commit and push the guarded kernel route, public regression, maintained + comparison evidence, and audible 10-second WAV locations. +- reasoning_checkpoint: the causal chain is closed from one BF16 query rounding boundary + through raw scores, softmax, head output, sampled tokens, and decoded audio. +- tdd_checkpoint: the public kernel actor regression failed before the supported + split-layout route was implemented and passes with the reference arithmetic. + +## Evidence + +- timestamp: 2026-07-09T13:00:00-05:00 + observation: the latest EMEL log completed 51 voice-prefill frames, 12 prompt-prefill frames, and generated frames 0 through 24, but no WAV exists. + implication: PersonaPlex setup and generation dispatch run on CPU; audible output and parity remain unproven. + +- timestamp: 2026-07-09T19:10:00-05:00 + observation: the initial sampler regression failed on the raw-logit shortcut; after correcting the fixture to the exact `ggml_argsort_top_k` no-swap path, a pinned 2048-card case showed old token 1639 versus reference token 1582. + implication: full-card f32 softmax probability materialization is behaviorally significant under fixed-seed exponential sampling. + +- timestamp: 2026-07-09T19:30:00-05:00 + observation: Moshi sampling now runs through explicit projection, scale, softmax, full descending argsort/top-k, and exponential-selection states; the Park-Miller random state is executor-owned and seeded by initialization dependency injection. + implication: sampling no longer depends on process-global `rand`/`srand`, and forced text consumes the same reference sampling chain before publishing its forced token. + +- timestamp: 2026-07-09T19:40:00-05:00 + observation: `speech_moshi_executor_sampling*` passed 2 tests and 304 assertions, including a full-card token/state regression and two independent `process_event` executor actors backed by distinct model instances. + implication: the arithmetic fix and actor isolation are verified locally; frame-0 PersonaPlex parity remains the next evidence boundary. + +- timestamp: 2026-07-09T20:30:00-05:00 + observation: the AArch64 Q4_K dot product accumulated each block as one combined + expression, while GGML subtracts the minimum term and then adds the quantized dot term. + A 64-block regression failed before the fix and now matches exactly. + implication: temporal and depformer projections now preserve the reference f32 + accumulation order. + +- timestamp: 2026-07-09T20:50:00-05:00 + observation: EMEL widened each RMS input to double before squaring; GGML squares in f32 + and only widens the product for accumulation. A 4096-element bit regression failed + before the fix and now passes 4098 assertions. + implication: RMS normalization follows the pinned GGML operand order instead of a + mathematically equivalent but seed-sensitive rewrite. + +- timestamp: 2026-07-09T21:45:00-05:00 + observation: EMEL completed 46 input frames and 120 output frames, wrote a 24 kHz mono + s16 WAV lasting 9.600 s (peak 0.7645, RMS 0.05242), and `afplay` exited successfully. + implication: the maintained EMEL loader, generator, executor, sampler, Mimi encoder, + and Mimi decoder run end to end and produce audible non-silent output. + +- timestamp: 2026-07-09T22:05:00-05:00 + observation: a separately built GGML 0.10.2 reference lane had Metal, CUDA, Vulkan, + and OpenCL disabled; its device list contained only BLAS and CPU. It completed 125 + frames and wrote a 10.000 s WAV from the same input, seed, model, and voice. + implication: the reference evidence is CPU-only rather than a CPU-selected run that + still initializes Metal. + +- timestamp: 2026-07-09T22:15:00-05:00 + observation: EMEL/reference duration is 9.6/10.0 s, RMS differs by 0.87 dB, + active-frame ratio is 43.3%/41.6%, and the best 80 ms log-energy-envelope correlation + is 0.521 at a one-frame lag. The early live text pattern includes the same + `0,2769,261` burst at steps 10-12; stochastic codec tokens and waveform samples diverge. + implication: turn timing, activity, level, and early text behavior closely match, while + results must not be represented as bit-exact token or waveform parity. + +- timestamp: 2026-07-09T22:55:00-05:00 + observation: the changed-file quality gate passed the Zig build, Moshi LM benchmark + (701333 ns/op), speech and kernel/graph tests, 90.2% changed-line coverage, 53.1% + changed-branch coverage, live GGML CPU kernel parity, and cross-process determinism. + `lint_snapshot` reported only unrelated pre-existing formatter drift; all task-owned + files were removed from that delta after changed-hunk formatting. + implication: implementation-specific gates are green. The global lint snapshot was + intentionally not updated because project policy requires explicit user consent. + +- timestamp: 2026-07-09T23:40:00-05:00 + observation: inspection and raw-sample comparison proved moshi.cpp's PersonaPlex file + loop fed only 88,320 of 96,597 samples to Mimi. A local reference-tool correction removed + the unsubmitted pre-read, preserved every pending resampler frame until consumption, and + flushed the padded tail exactly once. + implication: the previous reference WAV and its audio-envelope comparison are invalid; + matching the reference's broken PCM schedule was not correct same-WAV parity. + +- timestamp: 2026-07-09T23:45:00-05:00 + observation: two corrected CPU-only seed-1234 reference runs each consumed 51 PCM frames. + Frame checkpoints 0/15/30/45/50 exactly matched WAV samples + 0/28800/57600/86400/96000, and the two 10-second output WAVs were byte-identical with + SHA-256 9ee9b7f686b3911ddb42589a3220eb029b3c5f4d2bed0361aef6b8f75a1c9564. + implication: the corrected moshi.cpp reference now receives the complete WAV contiguously + and is deterministic; EMEL must be realigned before parity can be claimed. + +- timestamp: 2026-07-10T00:15:00-05:00 + observation: a new macOS Samantha utterance, `Hey, I'm Gabe. How are you doing?`, was + generated as 2.267208 seconds of mono 24 kHz s16 PCM. Corrected moshi.cpp consumed 29 + contiguous frames, including one padded tail frame, and emitted a 10-second CPU WAV. + implication: the comparison now uses a natural conversational input instead of the old + opaque utterance while preserving the same fixed-seed and model/voice contract. + +- timestamp: 2026-07-10T00:25:00-05:00 + observation: EMEL's runner was corrected to consume contiguous PCM from sample zero, + encode fresh zero-PCM frames through the streaming Mimi actor after EOF, and use an + injected 125-output-frame target. Runner dimensions/diagnostic phase now live in an + injected context rather than globals. It consumed 29 input plus 96 silence frames and + emitted exactly 125 output frames. + implication: both lanes now receive the same real PCM and streaming silence schedule; + EMEL no longer mirrors moshi.cpp's former dropped-frame bug or repeats the final speech + token frame during flush. + +- timestamp: 2026-07-10T00:35:00-05:00 + observation: both lanes are byte-deterministic across two seed-1234 runs. EMEL/reference + duration is 10.0/10.0 seconds, active-frame ratio is 47.2%/48.0%, RMS is + 0.06268/0.07828 (EMEL 1.93 dB quieter), and best 80 ms energy-envelope correlation is + 0.457 at an 11-frame lag. Raw text behavior shares the response onset `0,293` before + subsequent sampled tokens diverge. + implication: end-to-end input scheduling, output duration, activity, level, response + onset, and determinism now align on the organic utterance; exact stochastic token or + waveform parity remains unproven. + +- timestamp: 2026-07-10T01:05:00-05:00 + observation: current-source O0 and O3 frame-0 runs produce the same EMEL sequence + `1049,1770,19,...`; the CPU reference produces `1049,488,1632,...`, while the older Metal + reference starts `1049,1770,646,...`. + implication: compiler optimization is eliminated. Voice-prefill frame 0 is + input-independent and already diverges from the required CPU lane at codebook 1. + +- timestamp: 2026-07-10T01:10:00-05:00 + observation: the post-Q4-order artifact predates the RMS change and produced + `1049,488,...`; its temporal output begins + `-0.180639,0.082387,0.385717`, close to CPU reference + `-0.186217,0.083980,0.380608` and far from Metal reference + `-0.060614,0.170016,0.397496`. The Q4 artifact already used the final separate + `sum -= minimum; sum += dot` accumulation order. + implication: the Q4 accumulation fix is not the chronological cause of the later + codebook-1 switch. The subsequent RMS float-square-order change is the only remaining + numeric pivot in that commit, although its local arithmetic matches GGML CPU source and + must not be reverted solely from a sampled-token coincidence. + +- timestamp: 2026-07-10T01:15:00-05:00 + observation: the existing RMS and Q4 regressions call detail helpers directly. They prove + isolated arithmetic but do not exercise the maintained actor dispatch path or compounded + temporal/depformer graph. + implication: the next failing regression must dispatch a captured earliest unequal + operand through `emel::kernel::sm::process_event`, or run the pinned frame-0 contract + through `moshi_executor::sm::process_event`, without adding test-only production hooks. + +- timestamp: 2026-07-09T19:45:00-05:00 + observation: full layer-0 array dumps are bit-equal through norm2 and the 22,528-value + Q4_K gating projection. The first unequal byte is gated element 13, where CPU GGML's + AArch64 vector SiLU returns `0xbe3c4e19` and EMEL's scalar `std::exp` route returns + `0xbe3c4e18`. + implication: Q4_K is eliminated as the earliest frame-0 mismatch; AArch64 SiLU routing is + the first authoritative divergent operand. + +- timestamp: 2026-07-09T19:55:00-05:00 + observation: an actor-level AArch64 unary regression failed on the captured Moshi + activation before the fix and now passes all four GGML reference bits. SiLU is selected + by an explicit guard/transition, executes the reference NEON polynomial, and the old + runtime-indexed unary function table was removed. + implication: the parity fix adds no hidden behavior selection and also replaces a scalar + hot-path activation with four-lane SIMD. + +- timestamp: 2026-07-09T20:00:00-05:00 + observation: the release CPU runner now exactly matches CPU moshi.cpp for all 16 sampled + tokens in voice frame 0 and for every voice frame through frame 8 (144 consecutive audio + tokens). The first mismatch is frame 9, codebook 4: EMEL selects 655 while the reference + selects 855. + implication: the SiLU correction moves deterministic token parity from one codebook to + nine complete voice frames. Remaining sampled divergence is later and seed-sensitive, + rather than a gross model, RNG, or input-schedule mismatch. + +- timestamp: 2026-07-09T20:05:00-05:00 + observation: immediately before EMEL's frame-9/codebook-4 exponential draw, token 655 is + rank 5 with probability 0.0470813811 and token 855 is rank 6 with probability + 0.0437657684; the injected Park-Miller state is 1942131806 and deterministically selects + rank 5. + implication: the next comparison should test whether the CPU reference swaps these + adjacent candidates, as happened at the earlier frame-0 boundary. + +- timestamp: 2026-07-09T20:10:00-05:00 + observation: CPU moshi.cpp ranks 855 fifth and 655 sixth at voice frame 9/codebook 4; + EMEL ranks 655 fifth and 855 sixth. Both lanes begin the draw from Park-Miller state + 1942131806 and therefore select their respective fifth-ranked token. + implication: the remaining divergence is confirmed as accumulated adjacent-logit + ordering, not different RNG state, draw count, top-k size, or sampling algorithm. + +- timestamp: 2026-07-09T20:45:00-05:00 + observation: debugger-only captures from the public E2E actor dispatch show that the + frame-9/codebook-4 temporal `transformer_out` is already unequal in every f32 element + (maximum absolute difference 0.150205, mean absolute difference 0.016466, RMSE + 0.021238, correlation 0.999879). The following depformer input projection remains close + (maximum absolute difference 0.031275, correlation 0.999500). + implication: the first sampled-token mismatch does not originate in the sampler or the + depformer input projection. Its earliest currently proven operand is the temporal output + handed to depformer. + +- timestamp: 2026-07-09T20:47:00-05:00 + observation: frame-9/codebook-4 depformer layer-0 arrays remain closely correlated after + the unequal temporal operand: norm2/gating input correlation 0.999207, Q4_K gating + projection 0.999828, gated activation 0.999166, and FF update 0.998632. The respective + maximum absolute differences are 0.09743, 0.06839, 0.03532, and 0.01666. + implication: no new actor-level depformer regression is warranted. Layer 0 propagates an + upstream temporal difference rather than exposing a discrete new arithmetic failure. + +- timestamp: 2026-07-09T20:50:00-05:00 + observation: the maintained isolated 30-frame comparison feeds the same 240 public Mimi + tokens to both lanes exactly. EMEL/reference output RMS is 0.00213/0.10288, active-frame + ratio is 0.033/0.400, all-codebook token match is 0.2042, first-four match is 0.3833, + text match is 0.5667, and energy correlation is 0.4998. + implication: the remaining audible-output failure is not caused by WAV framing or Mimi + input tokens. The voice/prompt Moshi numeric path remains the active fault boundary. + +- timestamp: 2026-07-09T20:55:00-05:00 + observation: `tools/bench/speech/personaplex_emel_runner.cpp` is a maintained isolated + EMEL lane, but its prefill/live/flush sequencing, completion/error choices, lazy cache + allocation, and produced/decode routing are still implemented in procedural helpers, + lambdas, loops, and `if` branches. Its graph callback also performs synchronous logging + while the parent generator dispatch is still inside its RTC boundary. + implication: the file is suitable as a diagnostic benchmark adapter, not yet as the + requested architecture-compliant runtime entrypoint. Promotion requires a variant-named + SML session actor, preallocated injected cache/storage, and guard/transition-owned phase, + validation, completion, and failure choices; the CLI should remain a thin preload/write + adapter around that actor. + +- timestamp: 2026-07-09T22:20:00-05:00 + observation: commit `0e8737de` promotes PersonaPlex orchestration into a variant-named + SML session actor with injected temporal/depformer caches, storage, sampling policy, and + frame target. The maintained 125-frame CPU comparison produces both 10-second WAVs. The + 29 actual WAV frames match all 232 public Mimi input tokens exactly; output match is + 0.161 overall, 0.192 for the first four codebooks, and 0.608 for text. Audio RMS is + 0.03488/0.07828, active ratio is 0.168/0.480, and best log-energy correlation is 0.3603. + implication: the maintained architecture and same-WAV input contract are satisfied. The + only maintained report threshold still failing is first-four output-token match + 0.192 < 0.300, so numeric temporal-transformer isolation remains required. + +- timestamp: 2026-07-09T22:35:00-05:00 + observation: scheduled CPU-reference copies and debugger-only EMEL copies from the public + actor lane show frame-9 temporal layer 0 is bit-for-bit identical at norm1 (4,096 f32), + QKV projection (12,288), attention pre-output (4,096), attention residual (4,096), + norm2/gating input (4,096), gating projection (22,528), gated activation (11,264), FF + update (4,096), and final layer output (4,096). + implication: temporal layer 0, including its history-dependent attention result and both + Q4_K projections, is eliminated. A regression against any layer-0 operation would be + already passing and would not reproduce the remaining fault; isolation must continue at + layer 1 or later before production changes. + +- timestamp: 2026-07-09T22:42:00-05:00 + observation: frame-9 temporal outputs remain bit-exact through layer 25. Layer 26 is the + first unequal layer: its QKV projection is exact, but attention pre-output differs in + exactly 128 of 4,096 values, indices 1,152-1,279 (head 9). The head-local maximum is + 0.000115395. Subsequent layer-26 feed-forward work amplifies the difference to a + 0.00133740 maximum at the layer output and layer 31 reaches 0.0398528. + implication: the first maintained temporal divergence is isolated to layer-26 attention + head 9 rather than a layer-specific Q4_K projection, gating operation, or residual path. + +- timestamp: 2026-07-09T22:44:00-05:00 + observation: layer 26 uses the same causal RoPE/RMSNorm/SiLU block, tensor shapes, and + Q4_K projection types as neighboring layers; there is no cross-attention, layer scale, + weights-per-step schedule, or model-specific route. EMEL and reference BF16 score-dot + disassembly also use the same 16-element f32 multiply and sequential f64 accumulation. + implication: a hidden layer route, dtype dispatch, and attention-score reduction order + are eliminated; query rotation, cached K/V, or softmax remained the candidate boundary. + +- timestamp: 2026-07-09T22:46:00-05:00 + observation: all ten frame-9 layer-26/head-9 K and V cache rows are bit-exact BF16. The + post-RoPE query differs by one ULP in 23 of 128 f32 lanes, but only lane 44 crosses a + BF16 tie: EMEL `0x3f908000` rounds to `0x3f90`, while reference `0x3f908001` rounds to + `0x3f91`. Replacing that one query BF16 value makes every one of the ten raw score dots + bit-exact. Before substitution, raw scores differ by at most 0.0102539 and normalized + weights by at most 0.0000842214. + implication: RoPE is the earliest causal operand divergence. K/V state, dot accumulation, + masking, and softmax only propagate the changed BF16 query. + +- timestamp: 2026-07-09T22:48:00-05:00 + observation: AArch64 disassembly shows EMEL lowers + `real * cos - imag * sin` to `fnmul` + `fmadd` and the imaginary result to `fmul` + + `fmadd`. The reference builds four independent GGML multiply tensors before add/subtract, + forcing each product to round to f32 first. + implication: the arithmetic defect is contraction across a reference-mandated f32 round + point, not a different RoPE frequency, position, pairing, or trigonometric function. + +- timestamp: 2026-07-09T22:56:00-05:00 + observation: after routing split-layout timestep RoPE through explicit CPU kernel actor + transitions with separate f32 product round points, the maintained 30-frame comparison + matches 232/232 actual-WAV input tokens, 240/240 public output tokens, 120/120 first-four + tokens, and 30/30 text tokens. EMEL/reference RMS is 0.10287923/0.10287953, both active + ratios are 0.400, and log-energy correlation is 0.999999957 at zero lag. + implication: the isolated RoPE defect fully explains the previously failing public-token + report at this focused horizon. The unchanged 125-frame comparison is now the required + maintained end-to-end verification rather than another arithmetic search. + +- timestamp: 2026-07-09T23:02:00-05:00 + observation: the fresh maintained 125-frame comparison at + `build/personaplex_compare_organic_rope_fix_10s/personaplex_compare.json` passes every + unchanged threshold. Actual-WAV input is 232/232, public output is 1,000/1,000, + first-four output is 500/500, and text is 125/125. EMEL/reference RMS is + 0.078284165/0.078284263, active ratio is 0.480/0.480, and log-energy correlation is + 0.999999967 at zero lag. EMEL took 57.90 seconds versus 259.12 seconds for the pinned + one-thread CPU reference, a 4.475x reference/EMEL ratio. + implication: the same-WAV, fixed-seed, CPU-only PersonaPlex comparison is fully passing; + the earlier 0.192 first-four failure is eliminated without weakening any threshold. + +- timestamp: 2026-07-09T23:18:00-05:00 + observation: after rebuilding the maintained runner from the final visible guarded and + unchecked actor source, the unchanged 125-frame comparison at + `build/personaplex_compare_organic_rope_guarded_10s/personaplex_compare.json` reproduced + the exact prior WAV hashes and passed again: 232/232 actual-WAV input tokens, 1,000/1,000 + public output tokens, 500/500 first-four output tokens, and 125/125 text tokens. Audio + energy correlation is 0.999999967 at zero lag with equal 0.480 activity. EMEL took + 59.20 seconds and the one-thread CPU moshi.cpp reference took 298.05 seconds, so the + measured reference/EMEL ratio is 5.035x. + implication: the result is reproducible from the committed architecture rather than a + stale diagnostic binary, and both audible WAVs are retained with the final report. + +- timestamp: 2026-07-09T23:26:00-05:00 + observation: the changed-file gate passed both affected test shards, 96.5% changed-line + coverage, 58.2% changed-branch coverage, live GGML kernel parity, and cross-process + determinism. The aggregate command remained nonzero because a stale repository benchmark + dependency manifest forced a full-suite comparison that lacks the pre-existing + `sm_scheduler/idle_async` and `sm_scheduler/busy_worker_async` entries, and the global + lint snapshot has unrelated drift. After local formatting, neither task-owned file is in + the lint delta; no snapshot was updated. + implication: all implementation-specific lanes are green, while the remaining gate + failures are structural repository baselines outside this change. + +- timestamp: 2026-07-09T20:15:00-05:00 + observation: the corrected release EMEL lane generated a 10.000-second, mono 24 kHz s16 + WAV from the organic input in 58.90 seconds on one CPU thread. The matching CPU + moshi.cpp run took 314.356 seconds; EMEL is 5.34x faster for this end-to-end run. + implication: the native unpacked EMEL path now has both audible output and measured + single-thread throughput evidence against the same effective Q4_K model class. + +- timestamp: 2026-07-09T20:25:00-05:00 + observation: the final scoped gates passed 75 AArch64 tests (8,139 assertions), 90.2% + changed-line coverage, 58.4% changed-branch coverage, the + AArch64 benchmark lane, live GGML kernel parity, and cross-process determinism. The only + final command failure is the repository-wide pre-existing lint snapshot delta; task-owned + AArch64 guard/SM entries were removed from that delta and the snapshot was not updated. + implication: implementation-specific verification is green without weakening gates or + changing the user-controlled lint baseline. + +- timestamp: 2026-07-10T00:10:00-05:00 + observation: the direct AArch64 Q4_K x8 BL8 operand path now preserves the native + per-block f32 scale/min accumulation order without reconstructing native rows or + materializing f32 weights. The 125-frame packed report at + `build/personaplex_compare_packed_parity_10s_manual/personaplex_compare.json` matches + 232/232 input tokens, 1,000/1,000 output tokens, 500/500 first-four tokens, and 125/125 + text tokens. Audio-energy correlation is 0.999999967 at zero lag. + implication: the packed artifact is a true native packed operand path with fixed-seed + behavior equivalent to the unpacked path, not a packed-to-native fallback. + +- timestamp: 2026-07-10T00:15:00-05:00 + observation: packed EMEL completed 125 frames in 50.671 seconds versus 57.99 seconds for + a fresh unpacked run and 261.648 seconds for one-thread CPU moshi.cpp. Packed and unpacked + EMEL WAVs are byte-identical; packed throughput is 2.467 frames/s, 1.144x the unpacked + throughput and 5.164x the reference throughput. + implication: direct packing improves end-to-end single-thread throughput by 14.4% while + preserving the audible/reference behavior. + +- timestamp: 2026-07-10T00:25:00-05:00 + observation: the PersonaPlex LM contains 4,313.8 MiB of Q4_K matrices, 176.8 MiB of + already-Q4_0 embedding tables, and only 1.06 MiB of F32 norms. The separate 331.2 MiB + Mimi artifact still contains 193.0 MiB of transformer/RVQ projection matrices. Its + maintained Q8_0 converter reduces those projections to 51.5 MiB and the complete Mimi + artifact to 189.7 MiB. + implication: Mimi projections are the meaningful non-Q4_K quantization target. LM norms + and the 0.4 MiB voice prompt cannot provide material size savings. + +- timestamp: 2026-07-10T00:30:00-05:00 + observation: the injected Q8_0 Mimi artifact completed the same packed-LM 125-frame run + in 42.74 seconds, a 1.186x speedup over the float-Mimi packed run. It retained 99.14% + actual-WAV input-code agreement, but autoregressive output-token agreement versus the + exact-parity EMEL baseline fell to 43.3%, text agreement to 70.4%, and waveform + correlation to 0.670. + implication: Q8_0 Mimi is a valid 141.5 MiB smaller/faster optional artifact, but it must + not replace the exact-reference artifact in parity claims. + +- timestamp: 2026-07-10T00:35:00-05:00 + observation: a late live-audio macOS sample profile placed 40.0% of top-of-stack samples + in the packed Q4_K kernel, 21.3% in Mimi transformer compute, 12.7% in temporal + attention, and roughly another 19% in Mimi convolution, im2col, f16 matmul, and tensor + access work. The 8-codebook PersonaPlex Mimi artifact also retains 24 unused 2 MiB + acoustic codebooks from the 32-codebook source. + implication: further speed work should optimize the packed LM kernel and Mimi compute; + further zero-drift size work should first prune the unused 48 MiB of Mimi codebooks. + +- timestamp: 2026-07-10T00:40:00-05:00 + observation: the final scoped gate passed the AArch64 benchmark, 98.0% changed-line and + 66.2% changed-branch coverage, live GGML kernel parity, cross-process determinism, and + the affected tests. The only aggregate failure is unrelated pre-existing global + clang-format snapshot drift, which was not updated. + implication: packed/Q8 changes and evidence are implementation-green without weakening + gates or changing user-owned snapshots. + +## Eliminated + +- hypothesis: the Moshi reference swaps the first two top-k indices before drawing exponentials. + evidence: `moshi_sample_top_k` calls `ggml_argsort_top_k`, which is a full descending argsort plus a view; the swap exists only in the separate `GGML_OP_TOP_K` CPU kernel. + conclusion: preserve descending argsort order without the `GGML_OP_TOP_K` swap. + +- hypothesis: PersonaPlex depformer requires RoPE. + evidence: the converted model and source config carry `moshi.lm.depformer.pos_emb = + "none"`; the reference tensor diagnostic labeled `REF_DEP_Q_ROPE` is the unrotated Q + tensor for this model. + conclusion: RoPE remains a kernel primitive selected by model metadata; no PersonaPlex + depformer RoPE actor or hardcoded route was added. + +## Resolution + +- root_cause: Moshi's split-layout timestep RoPE used the right frequency, position, + pairing, trigonometric values, and K/V state, but its action-level expression contracted + `real*cos - imag*sin` and `real*sin + imag*cos` into AArch64 FMA instructions. The pinned + GGML reference materializes four independent f32 multiply nodes before add/subtract. + At frame 9, temporal layer 26, head 9, query lane 44, that missing f32 round point changed + `0x3f908001` to `0x3f908000`, crossing BF16 `0x3f91` to `0x3f90`. One changed query lane + altered all ten scores and became the first sampled-token divergence. +- fix: add an explicit guarded split-timestep RoPE route to both CPU kernel actors, keep + each product's f32 round point, and route temporal query/key rotation through the + executor's injected kernel actor. Query rotate/copy and key rotate/copy are explicit, + statically bounded completion phases; success/error choices remain in guards and + transitions. Model period, tensor dimensions, position, caches, seed, temperatures, + top-k values, session storage, and frame target remain dependency-injected or event/model + supplied rather than process-global. +- verification: the captured public kernel actor regression matches both reference output + bits. The focused 30-frame comparison reaches exact public token/text parity and + 0.999999957 audio-energy correlation. The final rebuilt 125-frame CPU-only report passes with + exact 232/232 actual-WAV input tokens, 1,000/1,000 public output tokens, 500/500 + first-four tokens, 125/125 text tokens, 0.999999967 energy correlation, equal 0.480 + activity, and nearly identical RMS. It reproduces the prior output hashes from the final + actor source. Both lanes emit audible 10-second 24 kHz mono WAVs. +- files_changed: CPU kernel timestep-RoPE detail, AArch64/x86 guards/actions/transitions, + Moshi executor query/key RoPE phases, public kernel actor regression/helpers, and this + persistent evidence record. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a94cea4..915bf176 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,7 +168,6 @@ if(EMEL_ENABLE_TESTS) tests/gbnf/lexer_tests.cpp tests/gbnf/parser_tests.cpp tests/gbnf/sampler_tests.cpp - tests/text/generator/legacy_compatibility_tests.cpp tests/text/generator/lifecycle_tests.cpp tests/text/generator/action_guard_tests.cpp tests/text/generator/detail_tests.cpp @@ -188,9 +187,14 @@ if(EMEL_ENABLE_TESTS) tests/diarization/sortformer/transformer/lifecycle_tests.cpp tests/speech/codec/mimi_detail_tests.cpp tests/speech/codec/mimi_lifecycle_tests.cpp + tests/speech/generator/lifecycle_tests.cpp + tests/speech/predictor/moshi_lifecycle_tests.cpp + tests/speech/predictor/moshi_executor_detail_tests.cpp + tests/speech/predictor/moshi_executor_streaming_tests.cpp tests/speech/encoder/whisper/lifecycle_tests.cpp tests/speech/decoder/whisper/lifecycle_tests.cpp tests/speech/encoder/whisper/detail_tests.cpp + tests/speech/tokenizer/moshi/lifecycle_tests.cpp tests/speech/transcriber/lifecycle_tests.cpp tests/speech/tokenizer/whisper_tests.cpp tests/embeddings/text_embedding_lane_tests.cpp @@ -232,6 +236,7 @@ if(EMEL_ENABLE_TESTS) tests/memory/kv/lifecycle_tests.cpp tests/memory/recurrent/lifecycle_tests.cpp tests/memory/hybrid/lifecycle_tests.cpp + tests/memory/streaming/lifecycle_tests.cpp tests/graph/tensor/lifecycle_tests.cpp tests/model/tensor/lifecycle_tests.cpp tests/model/tensor/window/lifecycle_tests.cpp diff --git a/README.md b/README.md index 4a8c1fe8..acfdc401 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ environments, while Zig remains the default for day-to-day builds. - [`.planning/architecture/memory_hybrid.md`](.planning/architecture/memory_hybrid.md) - [`.planning/architecture/memory_kv.md`](.planning/architecture/memory_kv.md) - [`.planning/architecture/memory_recurrent.md`](.planning/architecture/memory_recurrent.md) +- [`.planning/architecture/memory_streaming.md`](.planning/architecture/memory_streaming.md) - [`.planning/architecture/model_loader.md`](.planning/architecture/model_loader.md) - [`.planning/architecture/model_tensor.md`](.planning/architecture/model_tensor.md) - [`.planning/architecture/model_tensor_window.md`](.planning/architecture/model_tensor_window.md) @@ -182,6 +183,8 @@ environments, while Zig remains the default for day-to-day builds. - [`.planning/architecture/speech_codec_mimi.md`](.planning/architecture/speech_codec_mimi.md) - [`.planning/architecture/speech_decoder_whisper.md`](.planning/architecture/speech_decoder_whisper.md) - [`.planning/architecture/speech_encoder_whisper.md`](.planning/architecture/speech_encoder_whisper.md) +- [`.planning/architecture/speech_predictor_moshi_executor.md`](.planning/architecture/speech_predictor_moshi_executor.md) +- [`.planning/architecture/speech_tokenizer_moshi.md`](.planning/architecture/speech_tokenizer_moshi.md) - [`.planning/architecture/speech_tokenizer_whisper.md`](.planning/architecture/speech_tokenizer_whisper.md) - [`.planning/architecture/speech_transcriber.md`](.planning/architecture/speech_transcriber.md) - [`.planning/architecture/tensor_view.md`](.planning/architecture/tensor_view.md) diff --git a/docs/benchmarking.md b/docs/benchmarking.md index df890ed8..d6f58261 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -344,6 +344,12 @@ fetch pinned artifacts and produce the EMEL-lane GGUF, but it does not build, li moshi.cpp reference lane. the guard is meant to fail if the maintained EMEL Moshi model contract rejects a valid converted LM GGUF. +the converted GGUF owns the PersonaPlex inference contract. the converter writes +`moshi.lm.inference.dep_q`, `moshi.lm.inference.pre_text_silence_frames`, +`moshi.lm.inference.post_text_silence_frames`, and `moshi.lm.inference.prompt_tokens`; EMEL loads +those fields from model metadata instead of hardcoding prompt or delayed-codebook constants in the +generator. + the default model discovery order is: - `--model` @@ -366,6 +372,61 @@ scripts/bench_moshi_lm_compare.sh \ pass `--system` on macOS hosts where zig links against a newer SDK than the runtime OS can load. + +## CPU-only PersonaPlex speech comparison + +to run real speech end to end through both EMEL and pinned moshi.cpp with the same fixed seed and +input samples: + +```bash +scripts/bench_personaplex_compare.sh +``` + +on macOS, the default input is generated with Samantha at 175 words per minute saying +`Hey, I'm Gabe. How are you doing?` directly as a mono 24 kHz signed 16-bit WAV. pass +`--audio /path/to/input.wav` on another Unix host or to use an existing recording. +Changed-file quality gates pass the deterministic 24 kHz tone generated by +`scripts/setup_moshi_cpp_reference.sh` explicitly through `--audio`, so the maintained lane is +portable to non-macOS hosts. Set `EMEL_QUALITY_GATES_PERSONAPLEX_AUDIO` to inject another gate +fixture. + +the comparison builds two isolated Release executables. `personaplex_emel_runner` links only EMEL +and drives the maintained generic speech generator with injected Mimi, batch-planner, tokenizer, +Moshi predictor, sampler, streaming-memory, hybrid KV/recurrent-memory, and decoder actors. Each +streamed or flushed frame is planned through the shared batch planner before prediction; the typed +one-frame plan is validated +by the predictor before memory allocation. The generic generator then dispatches graph execution +and sampling as distinct injected actor phases; the Moshi predictor enforces the typed +`prediction_ready -> execution_ready -> session_ready` protocol. Its executor stops graph RTC +after temporal normalization in `sampling_ready`, then the sample RTC restores the injected +temporal-state buffer and performs text logits, text sampling, depformer execution, and audio +sampling. `moshi_reference_driver personaplex` links only the pinned moshi.cpp/ggml reference lane. +the reference configure hard-disables Metal and the script injects `--threads 1`, seed `1234`, 125 +frames, and temperatures. The shared inference contract injects the fixed reference-compatible +audio/text top-k limits into EMEL; asymmetric command-line sampling overrides are intentionally not +exposed. There are no process-global EMEL sampling defaults. +The EMEL composition also injects the Park-Miller modulus and zero-seed replacement state; nonzero +external seeds are normalized into the valid actor-owned RNG state range before sampling. + +the pinned upstream JSON remains untouched for the reference lane. EMEL model conversion injects +the separate `tools/bench/personaplex-inference.json` contract, keeping prompt tokens, public +codebook count, silence framing, and sampling limits explicit and reproducible. the setup keeps the +full 16-codebook Mimi parity artifact separate from PersonaPlex's 8-public-codebook Mimi artifact. + +the report and both audible WAVs are written under `build/personaplex_compare/`. the report records +the input SHA256, EMEL/reference LM and Mimi paths and SHA256 values, exact public Mimi input-code +match, sampled output-token match, exact text frame/token match, lagged 80 ms energy correlation, +activity ratio, non-silence, output hashes, and end-to-end wall time. A text frame-count or token +drift fails the comparison. sampled audio output is compared as behavior rather than raw-waveform +equality because a one-rank floating-point difference can avalanche later fixed-seed draws even +when the RNG state and sampling policy agree. + +use `--emel-lm` and `--emel-mimi` (or `EMEL_PERSONAPLEX_EMEL_LM` and +`EMEL_PERSONAPLEX_EMEL_MIMI`) to inject packed or quantized EMEL artifacts without changing the +reference lane. use `--frames`, `--seed`, `--threads`, `--output-dir`, `--build-only`, or +`--run-only` to override the remaining injected run configuration. `--run-only` never fetches, +converts, or rebuilds artifacts. + ## generation-specific local overrides the canonical generation case has its own bounded local validation knobs: diff --git a/scripts/bench_mimi_compare.sh b/scripts/bench_mimi_compare.sh index f9b45ce3..83e4a482 100755 --- a/scripts/bench_mimi_compare.sh +++ b/scripts/bench_mimi_compare.sh @@ -18,6 +18,7 @@ BUILD_DIR="${EMEL_MIMI_COMPARE_BUILD_DIR:-$ROOT_DIR/build/mimi_compare_tools}" OUTPUT_DIR="${EMEL_MIMI_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/mimi_compare}" ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" MLX_ARTIFACT_DIR="${EMEL_PERSONAPLEX_MLX_ARTIFACT_DIR:-$ROOT_DIR/build/personaplex_mlx_reference}" +MODEL_CONFIG="$ARTIFACT_DIR/personaplex-config.json" BUILD_ONLY=false RUN_ONLY=false REFERENCE_BACKEND="moshi_cpp" @@ -47,13 +48,6 @@ while [[ $# -gt 0 ]]; do esac done -for tool in cmake ninja git; do - if ! command -v "$tool" >/dev/null 2>&1; then - echo "error: required tool missing: $tool" >&2 - exit 1 - fi -done - # Discover the interpreter the same way the MLX setup script does, so hosts # that ship python3.12/python3.13 without a python3 shim (the environment the # setup path supports) can run the WAV generation and comparison steps too. @@ -70,8 +64,13 @@ if [[ -z "$PYTHON_BIN" ]]; then echo "error: required tool missing: python3 (or python3.12/python3.13)" >&2 exit 1 fi - if ! $RUN_ONLY; then + for tool in cmake ninja git; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "error: required build tool missing: $tool" >&2 + exit 1 + fi + done # full mode: fetch + sha256 verify + run the emel converter (no build) # build_targets is always non-empty: the MLX lane drives its reference # through the Python shim only (no reference build target), while the @@ -97,6 +96,21 @@ if ! $RUN_ONLY; then --target "${build_targets[@]}" fi +if [[ ! -f "$MODEL_CONFIG" ]]; then + echo "error: missing pinned PersonaPlex model config: $MODEL_CONFIG" >&2 + exit 1 +fi +MIMI_N_Q="$($PYTHON_BIN - "$MODEL_CONFIG" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")).get("n_q") +if not isinstance(value, int) or value <= 0: + raise SystemExit("Mimi n_q must be a positive integer") +print(value) +PY +)" + if $BUILD_ONLY; then echo "mimi compare lanes built in $BUILD_DIR" exit 0 @@ -158,6 +172,7 @@ PY --emel-model "$EMEL_MODEL" \ --reference-model "$REFERENCE_MODEL" \ --audio "$AUDIO" \ + --n-q "$MIMI_N_Q" \ --compare-decode \ --prefix-streams "${EMEL_MIMI_MLX_PREFIX_STREAMS:-4}" \ --min-prefix-match "${EMEL_MIMI_MLX_MIN_PREFIX_MATCH:-0.90}" \ @@ -174,6 +189,7 @@ else --emel-model "$EMEL_MODEL" \ --reference-model "$REFERENCE_MODEL" \ --audio "$AUDIO" \ + --n-q "$MIMI_N_Q" \ --compare-decode \ --min-decode-psnr-db "${EMEL_MIMI_MIN_DECODE_PSNR_DB:-30}" \ --require-token-exact \ diff --git a/scripts/bench_personaplex_compare.sh b/scripts/bench_personaplex_compare.sh new file mode 100755 index 00000000..7e31a570 --- /dev/null +++ b/scripts/bench_personaplex_compare.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# CPU-only, fixed-seed PersonaPlex speech comparison using isolated subprocess lanes. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/build_jobs.sh +source "$ROOT_DIR/scripts/build_jobs.sh" + +BUILD_DIR="${EMEL_PERSONAPLEX_COMPARE_BUILD_DIR:-$ROOT_DIR/build/personaplex_compare_tools_zig}" +OUTPUT_DIR="${EMEL_PERSONAPLEX_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/personaplex_compare}" +ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" +AUDIO="" +EMEL_LM="${EMEL_PERSONAPLEX_EMEL_LM:-}" +EMEL_MIMI="${EMEL_PERSONAPLEX_EMEL_MIMI:-}" +FRAMES=125 +SEED=1234 +THREADS=1 +BUILD_ONLY=false +RUN_ONLY=false + +usage() { + cat <<'USAGE' +usage: scripts/bench_personaplex_compare.sh [options] + + --audio PATH 24 kHz mono s16 WAV; default uses macOS say for + "Hey, I'm Gabe. How are you doing?" + --emel-lm PATH enriched EMEL LM artifact; defaults to the unpacked + model in the configured artifact directory + --emel-mimi PATH enriched EMEL Mimi artifact; defaults to the float + model in the configured artifact directory + --frames N generated frames per lane (default: 125 / 10 seconds) + --seed N fixed sampling seed (default: 1234) + --threads N CPU threads in moshi.cpp reference lane (default: 1) + --output-dir PATH output WAV/log/report directory + --build-only fetch/configure/build without running + --run-only use an existing build and artifacts +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --audio) AUDIO="${2:-}"; shift 2 ;; + --emel-lm) EMEL_LM="${2:-}"; shift 2 ;; + --emel-mimi) EMEL_MIMI="${2:-}"; shift 2 ;; + --frames) FRAMES="${2:-}"; shift 2 ;; + --seed) SEED="${2:-}"; shift 2 ;; + --threads) THREADS="${2:-}"; shift 2 ;; + --output-dir) OUTPUT_DIR="${2:-}"; shift 2 ;; + --build-only) BUILD_ONLY=true; shift ;; + --run-only) RUN_ONLY=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage; exit 1 ;; + esac +done + +if $BUILD_ONLY && $RUN_ONLY; then + echo "error: --build-only and --run-only are mutually exclusive" >&2 + exit 1 +fi +for value in "$FRAMES" "$SEED" "$THREADS"; do + if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "error: frames, seed, and threads must be positive integers" >&2 + exit 1 + fi +done +if ! command -v python3 >/dev/null 2>&1; then + echo "error: required tool missing: python3" >&2 + exit 1 +fi + +if ! $RUN_ONLY; then + for tool in cmake ninja git zig; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "error: required build tool missing: $tool" >&2 + exit 1 + fi + done + "$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh" >/dev/null + zig_bin="$(command -v zig)" + cmake -S "$ROOT_DIR/tools/bench" -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DEMEL_BENCH_SUITE_FILTER=speech_dialogue_moshi \ + -DEMEL_BENCH_SKIP_MOSHI_REFERENCE=OFF \ + -DCMAKE_C_COMPILER="$zig_bin" \ + -DCMAKE_C_COMPILER_ARG1=cc \ + -DCMAKE_CXX_COMPILER="$zig_bin" \ + -DCMAKE_CXX_COMPILER_ARG1=c++ \ + -DCMAKE_ASM_COMPILER="$zig_bin" \ + -DCMAKE_ASM_COMPILER_ARG1=cc \ + -DCMAKE_C_FLAGS=-fno-sanitize=undefined \ + -DCMAKE_CXX_FLAGS=-fno-sanitize=undefined + cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" \ + --target personaplex_emel_runner moshi_reference_driver +fi + +if $BUILD_ONLY; then + echo "PersonaPlex CPU comparison lanes built in $BUILD_DIR" + exit 0 +fi + +mkdir -p "$OUTPUT_DIR" +if [[ -z "$AUDIO" ]]; then + if ! command -v say >/dev/null 2>&1; then + echo "error: default organic input needs macOS say; pass --audio" >&2 + exit 1 + fi + AUDIO="$OUTPUT_DIR/say_gabe_how_are_you_24k.wav" + say -v Samantha -r 175 --file-format=WAVE --data-format=LEI16@24000 \ + --channels=1 -o "$AUDIO" "Hey, I'm Gabe. How are you doing?" +fi + +EMEL_RUNNER="$BUILD_DIR/personaplex_emel_runner" +REFERENCE_DRIVER="$BUILD_DIR/moshi_reference_driver" +if [[ -z "$EMEL_MIMI" ]]; then + EMEL_MIMI="$ARTIFACT_DIR/mimi-e351c8d8-125-personaplex-emel.gguf" +fi +REFERENCE_MIMI="$ARTIFACT_DIR/mimi-e351c8d8-125.gguf" +if [[ -z "$EMEL_LM" ]]; then + EMEL_LM="$ARTIFACT_DIR/model-q4_k-emel.gguf" +fi +REFERENCE_LM="$ARTIFACT_DIR/model-q4_k.gguf" +EMEL_VOICE="$ARTIFACT_DIR/voices/NATF0-emel.gguf" +REFERENCE_VOICE="$ARTIFACT_DIR/voices/NATF0.gguf" +CONFIG="$ARTIFACT_DIR/personaplex-config.json" +INFERENCE_CONFIG="$ROOT_DIR/tools/bench/personaplex-inference.json" +for path in "$AUDIO" "$EMEL_RUNNER" "$REFERENCE_DRIVER" "$EMEL_MIMI" \ + "$REFERENCE_MIMI" "$EMEL_LM" "$REFERENCE_LM" "$EMEL_VOICE" \ + "$REFERENCE_VOICE" "$CONFIG" "$INFERENCE_CONFIG"; do + if [[ ! -f "$path" ]]; then + echo "error: missing required artifact: $path" >&2 + exit 1 + fi +done + +python3 "$ROOT_DIR/tools/bench/personaplex_compare.py" \ + --emel-runner "$EMEL_RUNNER" \ + --reference-driver "$REFERENCE_DRIVER" \ + --emel-mimi "$EMEL_MIMI" \ + --reference-mimi "$REFERENCE_MIMI" \ + --emel-lm "$EMEL_LM" \ + --reference-lm "$REFERENCE_LM" \ + --emel-voice "$EMEL_VOICE" \ + --reference-voice "$REFERENCE_VOICE" \ + --config "$CONFIG" \ + --inference-config "$INFERENCE_CONFIG" \ + --audio "$AUDIO" \ + --output-dir "$OUTPUT_DIR" \ + --frames "$FRAMES" \ + --seed "$SEED" \ + --threads "$THREADS" diff --git a/scripts/check_domain_boundaries.sh b/scripts/check_domain_boundaries.sh index 13d6a944..51a2308d 100755 --- a/scripts/check_domain_boundaries.sh +++ b/scripts/check_domain_boundaries.sh @@ -19,24 +19,6 @@ check_no_matches() { rm -f /tmp/emel_domain_boundary_matches.$$ } -check_no_matches_except() { - local label="$1" - local pattern="$2" - local allow_pattern="$3" - shift 3 - - if rg -n "$pattern" "$@" >/tmp/emel_domain_boundary_matches.$$ 2>/dev/null; then - if grep -Ev "$allow_pattern" /tmp/emel_domain_boundary_matches.$$ \ - >/tmp/emel_domain_boundary_filtered.$$; then - echo "error: domain boundary leak: ${label}" >&2 - cat /tmp/emel_domain_boundary_filtered.$$ >&2 - status=1 - fi - rm -f /tmp/emel_domain_boundary_filtered.$$ - fi - rm -f /tmp/emel_domain_boundary_matches.$$ -} - check_absent_path() { local label="$1" shift @@ -74,10 +56,19 @@ check_no_matches "lfm2 model code depends on sibling model-family detail" \ 'emel/model/llama/(detail|any)\.hpp|emel/model/(qwen3|gemma4)/detail\.hpp|emel::model::llama::|emel::model::(qwen3|gemma4)::detail::' \ src/emel/model/lfm2 -check_no_matches_except "legacy text generator root" \ - 'emel/generator|emel::generator|namespace emel::generator|src/emel/generator|tests/generator' \ - '^(src/emel/generator/|tests/text/generator/legacy_compatibility_tests\.cpp:[0-9]+:|scripts/(quality_gates|test_with_coverage)\.sh:[0-9]+:.*src/emel/generator)' \ - src tests tools scripts/quality_gates.sh scripts/test_with_coverage.sh CMakeLists.txt +check_absent_path "legacy text generator root" \ + src/emel/generator tests/text/generator/legacy_compatibility_tests.cpp + +check_no_matches "legacy text generator includes or namespace" \ + 'emel/generator|emel::generator|namespace emel::generator|tests/generator' \ + src tests tools CMakeLists.txt + +check_absent_path "legacy generic machine aliases" \ + src/emel/io/sm.hpp src/emel/text/encoders/sm.hpp + +check_no_matches "legacy top-level machine aliases" \ + 'using[[:space:]]+(ComputeExecutor|DiarizationRequest|Model)[[:space:]]*=' \ + src tests tools CMakeLists.txt check_no_matches "text generator actor internals in maintained generation parity/benchmark lanes" \ 'emel/text/generator/(detail|actions|guards)\.hpp|emel::text::generator::(detail|action|guard)::|emel::text::generator::prefill::guard::|generation_internal_diagnostics' \ @@ -192,6 +183,10 @@ check_no_matches "Moshi/Mimi leaked into generic speech transcriber" \ 'moshi|model/moshi|model::moshi|tokenizer::moshi|codec::mimi|speech/codec/mimi' \ src/emel/speech/transcriber tests/speech/transcriber +check_no_matches "model-family names leaked into generic speech generator" \ + '[Mm]oshi|[Mm]imi|[Pp]ersona[Pp]lex|[Cc]hatterbox' \ + src/emel/speech/generator/*.hpp tests/speech/generator/lifecycle_tests.cpp + check_absent_path "retired generic diarization request owner path" \ src/emel/diarization/request \ tests/diarization/request \ diff --git a/scripts/fuzz_smoke.sh b/scripts/fuzz_smoke.sh index af37deb6..707369c9 100755 --- a/scripts/fuzz_smoke.sh +++ b/scripts/fuzz_smoke.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +# shellcheck source=scripts/build_jobs.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/build_jobs.sh" + for tool in cmake ninja clang clang++; do if ! command -v "$tool" >/dev/null 2>&1; then echo "error: required tool missing: $tool" >&2 @@ -48,11 +51,25 @@ else fi fuzz_cxx_flags="" fuzz_link_flags="" +fuzz_platform_flags=() fuzz_root="$(cd "$(dirname "$fuzz_cc")/.." && pwd)" if [[ -d "$fuzz_root/lib/c++" ]]; then fuzz_cxx_flags="-stdlib=libc++ -I${fuzz_root}/include/c++/v1" fuzz_link_flags="-stdlib=libc++ -L${fuzz_root}/lib/c++ -Wl,-rpath,${fuzz_root}/lib/c++ -lc++ -lc++abi" fi + +if [[ "$(uname -s)" == "Darwin" ]]; then + if ! command -v xcrun >/dev/null 2>&1; then + echo "error: xcrun is required to locate the active macOS SDK" >&2 + exit 1 + fi + fuzz_macos_sysroot="$(xcrun --sdk macosx --show-sdk-path)" + if [[ ! -d "$fuzz_macos_sysroot" ]]; then + echo "error: active macOS SDK does not exist: $fuzz_macos_sysroot" >&2 + exit 1 + fi + fuzz_platform_flags+=("-DCMAKE_OSX_SYSROOT=$fuzz_macos_sysroot") +fi if [[ "$fuzz_cc" == "clang" ]]; then if ! ls /opt/homebrew/opt/llvm/lib/clang/*/lib/darwin/libclang_rt.fuzzer_osx.a >/dev/null 2>&1 && \ ! ls /usr/local/opt/llvm/lib/clang/*/lib/darwin/libclang_rt.fuzzer_osx.a >/dev/null 2>&1; then @@ -71,10 +88,11 @@ cmake -S "$ROOT_DIR" -B "$BUILD_DIR" -G Ninja \ -DCMAKE_CXX_COMPILER="$fuzz_cxx" \ -DCMAKE_CXX_FLAGS="$fuzz_cxx_flags" \ -DCMAKE_EXE_LINKER_FLAGS="$fuzz_link_flags" \ + "${fuzz_platform_flags[@]}" \ -DEMEL_ENABLE_FUZZ=ON \ -DEMEL_ENABLE_TESTS=OFF -cmake --build "$BUILD_DIR" --parallel +cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" run_fuzzer() { local name="$1" diff --git a/scripts/quality_gates.sh b/scripts/quality_gates.sh index 19de3ef7..c8a76c3f 100755 --- a/scripts/quality_gates.sh +++ b/scripts/quality_gates.sh @@ -16,6 +16,11 @@ QUALITY_GATES_BENCH_RUNS="${EMEL_QUALITY_GATES_BENCH_RUNS:-3}" QUALITY_GATES_BENCH_WARMUP_ITERS="${EMEL_QUALITY_GATES_BENCH_WARMUP_ITERS:-10}" QUALITY_GATES_BENCH_WARMUP_RUNS="${EMEL_QUALITY_GATES_BENCH_WARMUP_RUNS:-1}" QUALITY_GATES_BENCH_TOLERANCE="${EMEL_QUALITY_GATES_BENCH_TOLERANCE:-0.30}" +QUALITY_GATES_PERSONAPLEX_FRAMES="${EMEL_QUALITY_GATES_PERSONAPLEX_FRAMES:-40}" +MOSHI_REFERENCE_ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" +PERSONAPLEX_AUDIO_NAME="phase_moshi_440hz_24khz_mono.wav" +DEFAULT_PERSONAPLEX_AUDIO="$MOSHI_REFERENCE_ARTIFACT_DIR/$PERSONAPLEX_AUDIO_NAME" +QUALITY_GATES_PERSONAPLEX_AUDIO="${EMEL_QUALITY_GATES_PERSONAPLEX_AUDIO:-$DEFAULT_PERSONAPLEX_AUDIO}" QUALITY_GATES_GENERATION_BENCH_ITERS="${EMEL_QUALITY_GATES_GENERATION_BENCH_ITERS:-1}" QUALITY_GATES_GENERATION_BENCH_RUNS="${EMEL_QUALITY_GATES_GENERATION_BENCH_RUNS:-1}" QUALITY_GATES_GENERATION_BENCH_WARMUP_ITERS="${EMEL_QUALITY_GATES_GENERATION_BENCH_WARMUP_ITERS:-0}" @@ -234,9 +239,6 @@ run_sml_surface_gate() { is_coverage_excluded_src_file() { local file="$1" case "$file" in - src/emel/generator/*.hpp|src/emel/generator/**/*.hpp) - return 0 - ;; src/emel/*/sm.hpp|src/emel/**/*/sm.hpp) return 0 ;; @@ -329,9 +331,6 @@ infer_test_shard_for_src() { src/emel/model/*|src/emel/model*.hpp|src/emel/gguf/*|src/emel/gbnf/*|src/emel/batch/*) add_test_shard model_and_batch ;; - src/emel/generator/*|src/emel/generator/**/*) - add_test_shard generator_and_runtime - ;; src/emel/text/generator/*|src/emel/embeddings/*|src/emel/logits/*|src/emel/token/*) add_test_shard generator_and_runtime ;; @@ -484,6 +483,7 @@ add_all_benchmark_suites_from_manifest() { kernel_aarch64 \ kernel_x86_64 \ batch_planner \ + parallel_matmul \ sm_any; do add_benchmark_suite_from_manifest "$priority_runner" done @@ -797,8 +797,6 @@ infer_quality_gate_scope() { add_all_benchmark_suites_from_manifest fi ;; - src/emel/generator/*|src/emel/generator/**/*) - ;; docs/templates/*|tools/docsgen/*|snapshots/bench/*|snapshots/embedded_size/*|src/emel/**/sm.hpp) docs_needed=true ;; @@ -844,8 +842,6 @@ infer_quality_gate_scope() { fi case "$file" in - src/emel/generator/*|src/emel/generator/**/*) - ;; src/emel/diarization/*|src/emel/model/sortformer/*|tests/diarization/*|\ tools/bench/diarization*|scripts/bench_diarization_compare.sh|\ scripts/setup_diarization_pytorch_ref_env.sh) @@ -857,6 +853,16 @@ infer_quality_gate_scope() { src/emel/batch/*|tests/batch/*) add_bench_suite batch_planner "scope path=$file" ;; + src/emel/speech/codec/mimi/*|src/emel/speech/generator/*|\ + src/emel/speech/predictor/moshi/*|\ + src/emel/speech/tokenizer/moshi/*|tests/speech/generator/*|\ + tests/speech/codec/mimi*|tests/speech/predictor/*|\ + tests/speech/tokenizer/moshi/*|\ + tools/bench/personaplex*|tools/bench/speech/personaplex*|\ + tools/bench/moshi_gguf_convert.py|scripts/setup_moshi_cpp_reference.sh|\ + scripts/bench_personaplex_compare.sh) + add_bench_suite speech_dialogue_moshi "scope path=$file" + ;; src/emel/io/*|src/emel/io/**/*|tests/io/*|tests/io/**/*) ;; src/emel/kernel/aarch64/*|tests/kernel/aarch64*) @@ -1141,6 +1147,7 @@ run_benchmark_gates() { EMEL_BENCH_RUNS="$QUALITY_GATES_BENCH_RUNS" \ EMEL_BENCH_WARMUP_ITERS="$QUALITY_GATES_BENCH_WARMUP_ITERS" \ EMEL_BENCH_WARMUP_RUNS="$QUALITY_GATES_BENCH_WARMUP_RUNS" \ + EMEL_BENCH_INTERNAL=1 \ BENCH_TOLERANCE="$QUALITY_GATES_BENCH_TOLERANCE" \ "$ROOT_DIR/scripts/bench.sh" --snapshot --compare; then full_status=0 @@ -1210,6 +1217,20 @@ run_benchmark_gates() { continue fi ;; + speech_dialogue_moshi) + # PersonaPlex is an end-to-end dialogue lane rather than a bench_runner + # microbenchmark. It owns the maintained encoder -> tokenizer -> + # predictor -> sampler -> detokenizer -> decoder comparison. + if run_step_allow_fail "bench_snapshot_${suite}" \ + "$ROOT_DIR/scripts/bench_personaplex_compare.sh" \ + --audio "$QUALITY_GATES_PERSONAPLEX_AUDIO" \ + --frames "$QUALITY_GATES_PERSONAPLEX_FRAMES"; then + continue + else + status=$? + continue + fi + ;; speech_codec_mimi) # The generic bench path runs codec_mimi_bench with placeholder # reference rows only; the token-exact Moshi comparison lives in the @@ -1275,6 +1296,11 @@ run_coverage_gate() { } parallel_enabled() { + case "${EMEL_BENCH_STRICT_REGRESSION:-0}" in + 1|true|yes) + return 1 + ;; + esac case "$QUALITY_GATES_PARALLEL" in auto|always|1|true|yes) return 0 diff --git a/scripts/setup_moshi_cpp_reference.sh b/scripts/setup_moshi_cpp_reference.sh index 6dc9c758..2df4b8f6 100755 --- a/scripts/setup_moshi_cpp_reference.sh +++ b/scripts/setup_moshi_cpp_reference.sh @@ -26,6 +26,7 @@ MOSHI_MODEL="$ARTIFACT_DIR/model-q4_k.gguf" MOSHI_MODEL_SHA256="73c4eb247740a48eebe5319ef767669dc8a06fd333b152a6cd6e12bacb64553d" MOSHI_CONFIG="$ARTIFACT_DIR/personaplex-config.json" MOSHI_CONFIG_SHA256="1b215765f6aafc6ef2592dadefcd8ad39c8b56d6eda25242be301b4af36b986a" +MOSHI_INFERENCE_CONFIG="$ROOT_DIR/tools/bench/personaplex-inference.json" MIMI_MODEL="$ARTIFACT_DIR/mimi-e351c8d8-125.gguf" MIMI_MODEL_SHA256="7e0c9ced83cbd035f70b82f1c5602673083fcccec006ea29f48d2e32c60ec697" TOKENIZER_MODEL="$ARTIFACT_DIR/tokenizer_spm_32k_3.model" @@ -36,6 +37,7 @@ REFERENCE_AUDIO="$ARTIFACT_DIR/phase_moshi_440hz_24khz_mono.wav" MOSHI_MODEL_EMEL="$ARTIFACT_DIR/model-q4_k-emel.gguf" MIMI_MODEL_EMEL="$ARTIFACT_DIR/mimi-e351c8d8-125-emel.gguf" +MIMI_MODEL_PERSONAPLEX_EMEL="$ARTIFACT_DIR/mimi-e351c8d8-125-personaplex-emel.gguf" VOICE_MODEL_EMEL="$ARTIFACT_DIR/voices/NATF0-emel.gguf" FETCH_ONLY=false @@ -109,10 +111,15 @@ fetch_pinned() { local url="$1" local path="$2" local expected_sha="$3" - if [[ ! -f "$path" ]]; then - curl -fL "$url" -o "$path" + local actual_sha="" + if [[ -f "$path" ]]; then + actual_sha="$(shasum -a 256 "$path" | awk '{print $1}')" + if [[ "$actual_sha" == "$expected_sha" ]]; then + return 0 + fi + echo "warning: refetching $path (sha256 mismatch)" >&2 fi - local actual_sha + curl -fL --retry 3 "$url" -o "$path" actual_sha="$(shasum -a 256 "$path" | awk '{print $1}')" if [[ "$actual_sha" != "$expected_sha" ]]; then echo "error: sha256 mismatch for $path: $actual_sha != $expected_sha" >&2 @@ -149,6 +156,17 @@ if $FETCH_ONLY; then exit 0 fi +MIMI_FULL_N_Q="$($PYTHON_BIN - "$MOSHI_CONFIG" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")).get("n_q") +if not isinstance(value, int) or value <= 0: + raise SystemExit("Mimi n_q must be a positive integer") +print(value) +PY +)" + convert_if_needed() { local source="$1" local output="$2" @@ -179,8 +197,12 @@ convert_if_needed() { } convert_if_needed "$MOSHI_MODEL" "$MOSHI_MODEL_EMEL" \ - --config "$MOSHI_CONFIG" --tokenizer "$TOKENIZER_MODEL" -convert_if_needed "$MIMI_MODEL" "$MIMI_MODEL_EMEL" --config "$MOSHI_CONFIG" + --config "$MOSHI_CONFIG" --inference-config "$MOSHI_INFERENCE_CONFIG" \ + --tokenizer "$TOKENIZER_MODEL" +convert_if_needed "$MIMI_MODEL" "$MIMI_MODEL_EMEL" \ + --config "$MOSHI_CONFIG" --mimi-n-q "$MIMI_FULL_N_Q" +convert_if_needed "$MIMI_MODEL" "$MIMI_MODEL_PERSONAPLEX_EMEL" \ + --config "$MOSHI_CONFIG" --inference-config "$MOSHI_INFERENCE_CONFIG" convert_if_needed "$VOICE_MODEL" "$VOICE_MODEL_EMEL" echo "EMEL_MOSHI_CPP_SOURCE=$REFERENCE_DIR" @@ -188,6 +210,7 @@ echo "EMEL_MOSHI_REFERENCE_MODEL=$MOSHI_MODEL" echo "EMEL_MOSHI_REFERENCE_MODEL_EMEL=$MOSHI_MODEL_EMEL" echo "EMEL_MIMI_REFERENCE_MODEL=$MIMI_MODEL" echo "EMEL_MIMI_REFERENCE_MODEL_EMEL=$MIMI_MODEL_EMEL" +echo "EMEL_MIMI_PERSONAPLEX_MODEL_EMEL=$MIMI_MODEL_PERSONAPLEX_EMEL" echo "EMEL_MOSHI_TOKENIZER_MODEL=$TOKENIZER_MODEL" echo "EMEL_MOSHI_VOICE_MODEL=$VOICE_MODEL" echo "EMEL_MOSHI_VOICE_MODEL_EMEL=$VOICE_MODEL_EMEL" diff --git a/scripts/setup_personaplex_mlx_reference.sh b/scripts/setup_personaplex_mlx_reference.sh index 070d46ff..13c3b86f 100755 --- a/scripts/setup_personaplex_mlx_reference.sh +++ b/scripts/setup_personaplex_mlx_reference.sh @@ -131,10 +131,20 @@ if [[ ! -f "$MIMI_MODEL_EMEL" || "$MIMI_MODEL" -nt "$MIMI_MODEL_EMEL" || "$CONVERTER" -nt "$MIMI_MODEL_EMEL" ]]; then # Use the interpreter discovered by the version check above: hosts that only # ship python3.12/python3.13 (no python3 shim) must convert with it too. + MIMI_FULL_N_Q="$($PYTHON_BIN - "$MOSHI_CONFIG" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")).get("n_q") +if not isinstance(value, int) or value <= 0: + raise SystemExit("Mimi n_q must be a positive integer") +print(value) +PY +)" "$PYTHON_BIN" "$ROOT_DIR/tools/bench/moshi_gguf_convert.py" \ --source "$MIMI_MODEL" --output "$MIMI_MODEL_EMEL" \ --manifest "${MIMI_MODEL_EMEL%.gguf}.manifest.json" \ - --config "$MOSHI_CONFIG" + --config "$MOSHI_CONFIG" --mimi-n-q "$MIMI_FULL_N_Q" fi cat > "$DRIVER_SHIM" < &dims, const uint32_t n_dims, + uint64_t &data_size_out) noexcept { + constexpr uint64_t k_block_cols = 256u; + constexpr uint64_t k_group_rows = 8u; + constexpr uint64_t k_group_block_bytes = 1152u; + if (n_dims < 2u || n_dims > constants::max_tensor_dims || + (dims[0] % k_block_cols) != 0u) { + return cast_loader_error(error::model_invalid); + } + + uint64_t rows = 1u; + for (uint32_t dim_index = 1u; dim_index < n_dims; ++dim_index) { + if (!multiply_u64(rows, dims[dim_index], rows)) { + return cast_loader_error(error::capacity); + } + } + + const uint64_t block_count = dims[0] / k_block_cols; + const uint64_t group_count = + rows / k_group_rows + static_cast((rows % k_group_rows) != 0u); + uint64_t groups_and_blocks = 0u; + if (!multiply_u64(group_count, block_count, groups_and_blocks) || + !multiply_u64(groups_and_blocks, k_group_block_bytes, data_size_out)) { + return cast_loader_error(error::capacity); + } + + return cast_loader_error(error::none); +} + inline bool pad_to_alignment(const uint64_t value, const uint32_t alignment, uint64_t & out) noexcept { @@ -217,6 +252,9 @@ inline bool valid_gguf_type(const uint32_t type) noexcept { } inline bool valid_ggml_type(const uint32_t type) noexcept { + if (is_emel_packed_q4_k_x8_type(type)) { + return true; + } const ggml_type_layout layout = ggml_layout(type); return layout.block_size != 0u && layout.type_size != 0u; } @@ -339,6 +377,11 @@ inline emel::error::type compute_tensor_data_size(const std::array return cast_loader_error(error::model_invalid); } + if (is_emel_packed_q4_k_x8_type(type)) { + return compute_emel_packed_q4_k_x8_data_size(dims, n_dims, + data_size_out); + } + const ggml_type_layout layout = ggml_layout(type); const uint64_t first_dim = n_dims == 0u ? 1u : dims[0]; if ((first_dim % layout.block_size) != 0u) { diff --git a/src/emel/io/sm.hpp b/src/emel/io/sm.hpp deleted file mode 100644 index e83be647..00000000 --- a/src/emel/io/sm.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -// benchmark: designed - -#include "emel/io/loader/sm.hpp" - -namespace emel::io { - -using sm = emel::io::loader::sm; - -} // namespace emel::io diff --git a/src/emel/kernel/aarch64/actions.hpp b/src/emel/kernel/aarch64/actions.hpp index 85b4855c..267d4f74 100644 --- a/src/emel/kernel/aarch64/actions.hpp +++ b/src/emel/kernel/aarch64/actions.hpp @@ -83,16 +83,44 @@ inline constexpr bool simd_supported_request_v = std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || - std::is_same_v; + std::is_same_v; -inline bool -unary_subop_supported_simd(const event::unary_subop subop) noexcept { - const auto subop_code = static_cast(subop); - return subop_code == static_cast(event::unary_subop::abs) || - subop_code == static_cast(event::unary_subop::neg) || - subop_code == static_cast(event::unary_subop::relu); +#if defined(__aarch64__) || defined(__ARM_NEON) +inline float32x4_t compute_neon_expf(const float32x4_t x) noexcept { + const float32x4_t r = vdupq_n_f32(0x1.8p23f); + const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f)); + const float32x4_t n = vsubq_f32(z, r); + const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n, + vdupq_n_f32(0x1.7f7d1cp-20f)); + const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23); + const float32x4_t k = vreinterpretq_f32_u32( + vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1.0f)))); + const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126.0f)); + const float32x4_t u = vmulq_f32(b, b); + const float32x4_t j = + vfmaq_f32(vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b), + vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), + vdupq_n_f32(0x1.555e66p-3f), b), + vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), + vdupq_n_f32(0x1.0e4020p-7f), b), + u), + u); + const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000u)); + const float32x4_t s1 = + vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000u))); + const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d)); + return vbslq_f32( + vcagtq_f32(n, vdupq_n_f32(192.0f)), vmulq_f32(s1, s1), + vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j))); +} + +inline float32x4_t compute_neon_silu(const float32x4_t x) noexcept { + const float32x4_t neg_x = vsubq_f32(vdupq_n_f32(0.0f), x); + const float32x4_t denominator = + vaddq_f32(vdupq_n_f32(1.0f), compute_neon_expf(neg_x)); + return vdivq_f32(x, denominator); } +#endif inline void execute_neon_unary_abs(const float *src, float *dst, const uint64_t count) noexcept { @@ -149,6 +177,24 @@ inline void execute_neon_unary_relu(const float *src, float *dst, #endif } +inline void execute_neon_unary_silu(const float *src, float *dst, + const uint64_t count) noexcept { +#if defined(__aarch64__) || defined(__ARM_NEON) + uint64_t i = 0; + for (; i + 4 <= count; i += 4) { + vst1q_f32(dst + i, compute_neon_silu(vld1q_f32(src + i))); + } + for (; i < count; ++i) { + const float v = src[i]; + dst[i] = v / (1.0f + std::exp(-v)); + } +#else + (void)src; + (void)dst; + (void)count; +#endif +} + template inline bool execute_neon_image_pointwise_f32( const emel_image_pointwise_f32_request &request) noexcept { @@ -877,12 +923,7 @@ inline bool can_use_neon(const request_type &request, ::emel::kernel::detail::tensor_element_count(request.dst); } - bool unary_supported = true; - if constexpr (std::is_same_v) { - unary_supported = unary_subop_supported_simd(request.subop); - } - - return base_supported && src1_supported && unary_supported && + return base_supported && src1_supported && (quantized_mul_mat || is_dense_contiguous(request.src0)) && is_dense_contiguous(request.dst); #endif @@ -946,17 +987,18 @@ inline bool can_run_neon_mul_mat_q6_vector_q8_rhs_request( request.src0.nb[0] == 1u && row_bytes != 0u && request.src0.nb[1] == row_bytes && request.src0.nb[2] == row_bytes * m && - request.src0.nb[3] == request.src0.nb[2] && - request.src1.nb[0] == 1u && rhs_row_bytes != 0u && - request.src1.nb[1] == rhs_row_bytes && + request.src0.nb[3] == request.src0.nb[2] && request.src1.nb[0] == 1u && + rhs_row_bytes != 0u && request.src1.nb[1] == rhs_row_bytes && request.src1.nb[2] == rhs_row_bytes && request.src1.nb[3] == rhs_row_bytes && is_dense_contiguous(request.dst); } -inline bool can_use_neon_mul_mat_q6_vector_q8_rhs( - const event::op_mul_mat &request, const bool neon_available) noexcept { - return neon_available && can_run_neon_mul_mat_q6_vector_q8_rhs_request(request); +inline bool +can_use_neon_mul_mat_q6_vector_q8_rhs(const event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && + can_run_neon_mul_mat_q6_vector_q8_rhs_request(request); } inline bool neon_q5_0_vector_supported() noexcept { @@ -1147,16 +1189,16 @@ inline bool can_run_neon_mul_mat_q8_0_vector_q8_rhs_request( request.src0.nb[0] == 1u && row_bytes != 0u && request.src0.nb[1] == row_bytes && request.src0.nb[2] == row_bytes * m && - request.src0.nb[3] == request.src0.nb[2] && - request.src1.nb[0] == 1u && rhs_row_bytes != 0u && - request.src1.nb[1] == rhs_row_bytes && + request.src0.nb[3] == request.src0.nb[2] && request.src1.nb[0] == 1u && + rhs_row_bytes != 0u && request.src1.nb[1] == rhs_row_bytes && request.src1.nb[2] == rhs_row_bytes && request.src1.nb[3] == rhs_row_bytes && is_dense_contiguous(request.dst); } -inline bool can_use_neon_mul_mat_q8_0_vector_q8_rhs( - const event::op_mul_mat &request, const bool neon_available) noexcept { +inline bool +can_use_neon_mul_mat_q8_0_vector_q8_rhs(const event::op_mul_mat &request, + const bool neon_available) noexcept { return neon_available && can_run_neon_mul_mat_q8_0_vector_q8_rhs_request(request); } @@ -1292,8 +1334,6 @@ inline bool can_use_neon_mul_mat_q8_0_packed_bl8_full_groups( 0u; } - - inline bool can_use_neon_mul_mat_q8_0_packed_bl8_tail_safe( const event::op_mul_mat &request, const bool neon_available) noexcept { return can_use_neon_mul_mat_q8_0_packed_bl8(request, neon_available) && @@ -1399,6 +1439,35 @@ inline bool can_run_neon_mul_mat_q4_vector_packed_q8_rhs_request( is_dense_contiguous(request.dst); } +inline bool can_run_neon_mul_mat_q4_vector_packed_f32_rhs_request( + const event::op_mul_mat &request, const uint8_t packed_dtype) noexcept { + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + const size_t group_bytes = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_storage_bytes(k); + return k != 0u && m != 0u && block_count != 0u && + block_count <= ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS && + request.src1.ne[0] == 1u && request.src1.ne[1] == k && + request.dst.ne[0] == 1u && request.dst.ne[1] == m && + request.src0.ne[2] == 1u && request.src0.ne[3] == 1u && + request.src1.ne[2] == 1u && request.src1.ne[3] == 1u && + request.dst.ne[2] == 1u && request.dst.ne[3] == 1u && + ::emel::kernel::detail::dtype_code(request.src0.type) == + packed_dtype && + ::emel::kernel::detail::dtype_code(request.src1.type) == + ::emel::kernel::detail::dtype_f32 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + request.src0.nb[0] == 1u && group_bytes != 0u && + request.src0.nb[1] == group_bytes && + request.src0.nb[2] == group_bytes * group_count && + request.src0.nb[3] == request.src0.nb[2] && + is_dense_contiguous(request.src1) && is_dense_contiguous(request.dst); +} + inline bool can_run_neon_mul_mat_q4_vector_q8_rhs_request( const event::op_mul_mat &request) noexcept { const uint64_t k = request.src0.ne[0]; @@ -1425,14 +1494,39 @@ inline bool can_run_neon_mul_mat_q4_vector_q8_rhs_request( request.src0.nb[0] == 1u && row_bytes != 0u && request.src0.nb[1] == row_bytes && request.src0.nb[2] == row_bytes * m && - request.src0.nb[3] == request.src0.nb[2] && - request.src1.nb[0] == 1u && rhs_row_bytes != 0u && - request.src1.nb[1] == rhs_row_bytes && + request.src0.nb[3] == request.src0.nb[2] && request.src1.nb[0] == 1u && + rhs_row_bytes != 0u && request.src1.nb[1] == rhs_row_bytes && request.src1.nb[2] == rhs_row_bytes && request.src1.nb[3] == rhs_row_bytes && is_dense_contiguous(request.dst); } +inline bool can_run_neon_mul_mat_q4_vector_f32_rhs_request( + const event::op_mul_mat &request) noexcept { + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + const size_t row_bytes = ::emel::kernel::detail::quantized_row_storage_bytes( + ::emel::kernel::detail::dtype_q4_k, k); + return k != 0u && m != 0u && block_count != 0u && + block_count <= ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS && + request.src1.ne[0] == 1u && request.src1.ne[1] == k && + request.dst.ne[0] == 1u && request.dst.ne[1] == m && + request.src0.ne[2] == 1u && request.src0.ne[3] == 1u && + request.src1.ne[2] == 1u && request.src1.ne[3] == 1u && + request.dst.ne[2] == 1u && request.dst.ne[3] == 1u && + ::emel::kernel::detail::dtype_code(request.src0.type) == + ::emel::kernel::detail::dtype_q4_k && + ::emel::kernel::detail::dtype_code(request.src1.type) == + ::emel::kernel::detail::dtype_f32 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + request.src0.nb[0] == 1u && row_bytes != 0u && + request.src0.nb[1] == row_bytes && + request.src0.nb[2] == row_bytes * m && + request.src0.nb[3] == request.src0.nb[2] && + is_dense_contiguous(request.src1) && is_dense_contiguous(request.dst); +} inline bool can_run_neon_mul_mat_q6_vector_packed_q8_rhs_request( const event::op_mul_mat &request) noexcept { @@ -1467,7 +1561,6 @@ inline bool can_run_neon_mul_mat_q6_vector_packed_q8_rhs_request( is_dense_contiguous(request.dst); } - inline bool can_run_neon_mul_mat_q6_vector_prepared_q8_rhs_request( const event::op_mul_mat &request) noexcept { const uint64_t k = request.src0.ne[0]; @@ -1522,13 +1615,33 @@ inline bool can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl8( request, ::emel::kernel::detail::dtype_q4_k_x8_bl8); } -inline bool can_use_neon_mul_mat_q4_vector_q8_rhs( +inline bool can_use_neon_mul_mat_q4_vector_packed_f32_rhs_bl4( + const event::op_mul_mat &request, const bool neon_available) noexcept { + return neon_available && neon_q4_vector_packed_supported() && + can_run_neon_mul_mat_q4_vector_packed_f32_rhs_request( + request, ::emel::kernel::detail::dtype_q4_k_x8_bl4); +} + +inline bool can_use_neon_mul_mat_q4_vector_packed_f32_rhs_bl8( const event::op_mul_mat &request, const bool neon_available) noexcept { + return neon_available && neon_q4_vector_packed_supported() && + can_run_neon_mul_mat_q4_vector_packed_f32_rhs_request( + request, ::emel::kernel::detail::dtype_q4_k_x8_bl8); +} + +inline bool +can_use_neon_mul_mat_q4_vector_q8_rhs(const event::op_mul_mat &request, + const bool neon_available) noexcept { return neon_available && neon_q4_vector_q8_rhs_supported() && can_run_neon_mul_mat_q4_vector_q8_rhs_request(request); } - +inline bool +can_use_neon_mul_mat_q4_vector_f32_rhs(const event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && neon_q4_vector_q8_rhs_supported() && + can_run_neon_mul_mat_q4_vector_f32_rhs_request(request); +} inline bool can_use_neon_mul_mat_q6_vector_packed_q8_rhs( const event::op_mul_mat &request, const bool neon_available) noexcept { @@ -1536,7 +1649,6 @@ inline bool can_use_neon_mul_mat_q6_vector_packed_q8_rhs( can_run_neon_mul_mat_q6_vector_packed_q8_rhs_request(request); } - inline bool can_use_neon_mul_mat_q6_vector_prepared_q8_rhs( const event::op_mul_mat &request, const bool neon_available) noexcept { #if defined(__ARM_FEATURE_MATMUL_INT8) @@ -1555,7 +1667,6 @@ inline bool can_use_neon_mul_mat_q6_vector_prepared_q8_rhs_i8mm( can_run_neon_mul_mat_q6_vector_prepared_q8_rhs_request(request); } - inline bool can_run_neon_mul_mat_argmax_q6_vector_prepared_q8_rhs_request( const event::op_mul_mat_argmax &request) noexcept { const uint64_t k = request.src0.ne[0]; @@ -1655,6 +1766,38 @@ inline bool can_run_neon_mul_mat_argmax_q6_vector_packed_q8_rhs_request( ::emel::kernel::aarch64::detail::is_dense_contiguous(request.dst); } +inline bool can_run_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_request( + const event::op_mul_mat_argmax &request, + const uint8_t packed_dtype) noexcept { + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + const size_t group_bytes = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_storage_bytes(k); + return request.index_out != nullptr && k != 0u && m != 0u && + block_count != 0u && + block_count <= ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS && + request.src1.ne[0] == 1u && request.src1.ne[1] == k && + request.dst.ne[0] == 1u && request.dst.ne[1] == 1u && + request.src0.ne[2] == 1u && request.src0.ne[3] == 1u && + request.src1.ne[2] == 1u && request.src1.ne[3] == 1u && + request.dst.ne[2] == 1u && request.dst.ne[3] == 1u && + ::emel::kernel::detail::dtype_code(request.src0.type) == + packed_dtype && + ::emel::kernel::detail::dtype_code(request.src1.type) == + ::emel::kernel::detail::dtype_f32 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + request.src0.nb[0] == 1u && group_bytes != 0u && + request.src0.nb[1] == group_bytes && + request.src0.nb[2] == group_bytes * group_count && + request.src0.nb[3] == request.src0.nb[2] && + ::emel::kernel::aarch64::detail::is_dense_contiguous(request.src1) && + ::emel::kernel::aarch64::detail::is_dense_contiguous(request.dst); +} + inline bool can_use_neon_mul_mat_argmax_q6_vector_packed_q8_rhs( const event::op_mul_mat_argmax &request, const bool neon_available) noexcept { @@ -1662,6 +1805,22 @@ inline bool can_use_neon_mul_mat_argmax_q6_vector_packed_q8_rhs( can_run_neon_mul_mat_argmax_q6_vector_packed_q8_rhs_request(request); } +inline bool can_use_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4( + const event::op_mul_mat_argmax &request, + const bool neon_available) noexcept { + return neon_available && neon_q4_vector_packed_supported() && + can_run_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_request( + request, ::emel::kernel::detail::dtype_q4_k_x8_bl4); +} + +inline bool can_use_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8( + const event::op_mul_mat_argmax &request, + const bool neon_available) noexcept { + return neon_available && neon_q4_vector_packed_supported() && + can_run_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_request( + request, ::emel::kernel::detail::dtype_q4_k_x8_bl8); +} + inline bool can_use_neon_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm( const event::op_mul_mat_argmax &request, const bool neon_available) noexcept { @@ -2630,11 +2789,22 @@ inline int32_t q4_k_min_sum_neon(const uint8_t *mins, } #endif -inline float dot_q4_k_q8_k_block_neon( +inline void +accumulate_q4_k_scaled_sums(const uint16_t lhs_d, const uint16_t lhs_dmin, + const float rhs_d, const int32_t min_sum, + const int32_t dot_sum, float &sum) noexcept { + const float d = ::emel::kernel::detail::quant::fp16_to_fp32(lhs_d) * rhs_d; + const float dmin = + ::emel::kernel::detail::quant::fp16_to_fp32(lhs_dmin) * rhs_d; + sum -= dmin * static_cast(min_sum); + sum += d * static_cast(dot_sum); +} + +inline void accumulate_q4_k_q8_k_block_neon( const ::emel::kernel::detail::quant::block_q4_k &lhs, - const ::emel::kernel::detail::quant::block_q8_k &rhs) noexcept { + const ::emel::kernel::detail::quant::block_q8_k &rhs, float &sum) noexcept { #if !defined(__ARM_FEATURE_DOTPROD) - return ::emel::kernel::detail::dot_q4_k_q8_k_block_scalar(lhs, rhs); + sum += ::emel::kernel::detail::dot_q4_k_q8_k_block_scalar(lhs, rhs); #else uint32_t decoded_words[4] = {}; decode_q4_k_scales_words(lhs, decoded_words); @@ -2649,42 +2819,38 @@ inline float dot_q4_k_q8_k_block_neon( const uint8_t *q4 = lhs.qs.data(); const int8_t *q8 = rhs.qs.data(); - int32x4_t low_acc = vdupq_n_s32(0); - int32x4_t high_acc = vdupq_n_s32(0); + int32_t low_sum = 0; + int32_t high_sum = 0; for (uint64_t group = 0; group < (::emel::kernel::detail::quant::QK_K / 64u); ++group) { - const uint8x16_t q4bits0 = vld1q_u8(q4 + 0u); - const uint8x16_t q4bits1 = vld1q_u8(q4 + 16u); - const int8x16_t q8bytes0 = vld1q_s8(q8 + 0u); - const int8x16_t q8bytes1 = vld1q_s8(q8 + 16u); - const int8x16_t q8bytes2 = vld1q_s8(q8 + 32u); - const int8x16_t q8bytes3 = vld1q_s8(q8 + 48u); - - const int8x16_t q4low0 = vreinterpretq_s8_u8(vandq_u8(q4bits0, m4b)); - const int8x16_t q4low1 = vreinterpretq_s8_u8(vandq_u8(q4bits1, m4b)); - const int8x16_t q4high0 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits0, 4)); - const int8x16_t q4high1 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits1, 4)); - - const int32_t low_scale = static_cast(scales[group * 2u + 0u]); - const int32_t high_scale = static_cast(scales[group * 2u + 1u]); - low_acc = - vmlaq_n_s32(low_acc, vdotq_s32(zero, q4low0, q8bytes0), low_scale); - low_acc = - vmlaq_n_s32(low_acc, vdotq_s32(zero, q4low1, q8bytes1), low_scale); - high_acc = - vmlaq_n_s32(high_acc, vdotq_s32(zero, q4high0, q8bytes2), high_scale); - high_acc = - vmlaq_n_s32(high_acc, vdotq_s32(zero, q4high1, q8bytes3), high_scale); - + const uint8x16x2_t q4bits = load_u8x16x2(q4); q4 += 32u; - q8 += 64u; + + int8x16x2_t q8bytes = vld1q_s8_x2(q8); + q8 += 32u; + const int8x16_t q4low0 = vreinterpretq_s8_u8(vandq_u8(q4bits.val[0], m4b)); + const int8x16_t q4low1 = vreinterpretq_s8_u8(vandq_u8(q4bits.val[1], m4b)); + const int32x4_t low_dot = + vdotq_s32(vdotq_s32(zero, q4low0, q8bytes.val[0]), q4low1, + q8bytes.val[1]); + low_sum += vaddvq_s32(low_dot) * + static_cast(scales[group * 2u + 0u]); + + q8bytes = vld1q_s8_x2(q8); + q8 += 32u; + const int8x16_t q4high0 = + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4)); + const int8x16_t q4high1 = + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4)); + const int32x4_t high_dot = + vdotq_s32(vdotq_s32(zero, q4high0, q8bytes.val[0]), q4high1, + q8bytes.val[1]); + high_sum += vaddvq_s32(high_dot) * + static_cast(scales[group * 2u + 1u]); } - const int32_t isum = vaddvq_s32(vaddq_s32(low_acc, high_acc)); - const float d = ::emel::kernel::detail::quant::fp16_to_fp32(lhs.d) * rhs.d; - const float dmin = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs.dmin) * rhs.d; - return d * static_cast(isum) - dmin * static_cast(min_sum); + accumulate_q4_k_scaled_sums(lhs.d, lhs.dmin, rhs.d, min_sum, + low_sum + high_sum, sum); #endif } @@ -2698,7 +2864,7 @@ dot_q4_k_q8_k_row_neon(const ::emel::kernel::detail::quant::block_q4_k *lhs, #else float sum = 0.0f; for (uint64_t block = 0; block < block_count; ++block) { - sum += dot_q4_k_q8_k_block_neon(lhs[block], rhs[block]); + accumulate_q4_k_q8_k_block_neon(lhs[block], rhs[block], sum); } return sum; #endif @@ -2713,105 +2879,8 @@ dot_q4_k_q8_k_2rows_neon(const ::emel::kernel::detail::quant::block_q4_k *lhs0, out[0] = dot_q4_k_q8_k_row_neon(lhs0, rhs, block_count); out[1] = dot_q4_k_q8_k_row_neon(lhs1, rhs, block_count); #else - const uint8x16_t m4b = vdupq_n_u8(0x0fu); - const int32x4_t zero = vdupq_n_s32(0); - float sum0 = 0.0f; - float sum1 = 0.0f; - - for (uint64_t block = 0; block < block_count; ++block) { - const auto &lhs_block0 = lhs0[block]; - const auto &lhs_block1 = lhs1[block]; - uint32_t decoded_words0[4] = {}; - uint32_t decoded_words1[4] = {}; - decode_q4_k_scales_words(lhs_block0, decoded_words0); - decode_q4_k_scales_words(lhs_block1, decoded_words1); - - const auto *scales0 = reinterpret_cast(decoded_words0); - const auto *scales1 = reinterpret_cast(decoded_words1); - const auto *mins0 = reinterpret_cast(decoded_words0 + 2); - const auto *mins1 = reinterpret_cast(decoded_words1 + 2); - const int16x8_t q8_pair_sums = - vpaddq_s16(vld1q_s16(rhs[block].bsums.data()), - vld1q_s16(rhs[block].bsums.data() + 8)); - const int32_t min_sum0 = q4_k_min_sum_neon(mins0, q8_pair_sums); - const int32_t min_sum1 = q4_k_min_sum_neon(mins1, q8_pair_sums); - - const uint8_t *q40 = lhs_block0.qs.data(); - const uint8_t *q41 = lhs_block1.qs.data(); - const int8_t *q8 = rhs[block].qs.data(); - int32x4_t low_acc0 = vdupq_n_s32(0); - int32x4_t high_acc0 = vdupq_n_s32(0); - int32x4_t low_acc1 = vdupq_n_s32(0); - int32x4_t high_acc1 = vdupq_n_s32(0); - - for (uint64_t group = 0; - group < (::emel::kernel::detail::quant::QK_K / 64u); ++group) { - const uint8x16_t q4bits00 = vld1q_u8(q40 + 0u); - const uint8x16_t q4bits01 = vld1q_u8(q40 + 16u); - const uint8x16_t q4bits10 = vld1q_u8(q41 + 0u); - const uint8x16_t q4bits11 = vld1q_u8(q41 + 16u); - const int8x16_t q8bytes0 = vld1q_s8(q8 + 0u); - const int8x16_t q8bytes1 = vld1q_s8(q8 + 16u); - const int8x16_t q8bytes2 = vld1q_s8(q8 + 32u); - const int8x16_t q8bytes3 = vld1q_s8(q8 + 48u); - - const int8x16_t q4low00 = vreinterpretq_s8_u8(vandq_u8(q4bits00, m4b)); - const int8x16_t q4low01 = vreinterpretq_s8_u8(vandq_u8(q4bits01, m4b)); - const int8x16_t q4high00 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits00, 4)); - const int8x16_t q4high01 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits01, 4)); - const int8x16_t q4low10 = vreinterpretq_s8_u8(vandq_u8(q4bits10, m4b)); - const int8x16_t q4low11 = vreinterpretq_s8_u8(vandq_u8(q4bits11, m4b)); - const int8x16_t q4high10 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits10, 4)); - const int8x16_t q4high11 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits11, 4)); - - const int32_t low_scale0 = static_cast(scales0[group * 2u + 0u]); - const int32_t high_scale0 = - static_cast(scales0[group * 2u + 1u]); - const int32_t low_scale1 = static_cast(scales1[group * 2u + 0u]); - const int32_t high_scale1 = - static_cast(scales1[group * 2u + 1u]); - - low_acc0 = - vmlaq_n_s32(low_acc0, vdotq_s32(zero, q4low00, q8bytes0), low_scale0); - low_acc0 = - vmlaq_n_s32(low_acc0, vdotq_s32(zero, q4low01, q8bytes1), low_scale0); - high_acc0 = vmlaq_n_s32(high_acc0, vdotq_s32(zero, q4high00, q8bytes2), - high_scale0); - high_acc0 = vmlaq_n_s32(high_acc0, vdotq_s32(zero, q4high01, q8bytes3), - high_scale0); - low_acc1 = - vmlaq_n_s32(low_acc1, vdotq_s32(zero, q4low10, q8bytes0), low_scale1); - low_acc1 = - vmlaq_n_s32(low_acc1, vdotq_s32(zero, q4low11, q8bytes1), low_scale1); - high_acc1 = vmlaq_n_s32(high_acc1, vdotq_s32(zero, q4high10, q8bytes2), - high_scale1); - high_acc1 = vmlaq_n_s32(high_acc1, vdotq_s32(zero, q4high11, q8bytes3), - high_scale1); - - q40 += 32u; - q41 += 32u; - q8 += 64u; - } - - const int32_t isum0 = vaddvq_s32(vaddq_s32(low_acc0, high_acc0)); - const int32_t isum1 = vaddvq_s32(vaddq_s32(low_acc1, high_acc1)); - const float rhs_d = rhs[block].d; - const float d0 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block0.d) * rhs_d; - const float d1 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block1.d) * rhs_d; - const float dmin0 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block0.dmin) * rhs_d; - const float dmin1 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block1.dmin) * rhs_d; - sum0 += - d0 * static_cast(isum0) - dmin0 * static_cast(min_sum0); - sum1 += - d1 * static_cast(isum1) - dmin1 * static_cast(min_sum1); - } - - out[0] = sum0; - out[1] = sum1; + out[0] = dot_q4_k_q8_k_row_neon(lhs0, rhs, block_count); + out[1] = dot_q4_k_q8_k_row_neon(lhs1, rhs, block_count); #endif } @@ -2828,177 +2897,40 @@ dot_q4_k_q8_k_4rows_neon(const ::emel::kernel::detail::quant::block_q4_k *lhs0, out[2] = dot_q4_k_q8_k_row_neon(lhs2, rhs, block_count); out[3] = dot_q4_k_q8_k_row_neon(lhs3, rhs, block_count); #else - const uint8x16_t m4b = vdupq_n_u8(0x0fu); - const int32x4_t zero = vdupq_n_s32(0); - float sum0 = 0.0f; - float sum1 = 0.0f; - float sum2 = 0.0f; - float sum3 = 0.0f; + out[0] = dot_q4_k_q8_k_row_neon(lhs0, rhs, block_count); + out[1] = dot_q4_k_q8_k_row_neon(lhs1, rhs, block_count); + out[2] = dot_q4_k_q8_k_row_neon(lhs2, rhs, block_count); + out[3] = dot_q4_k_q8_k_row_neon(lhs3, rhs, block_count); +#endif +} - for (uint64_t block = 0; block < block_count; ++block) { - const auto &lhs_block0 = lhs0[block]; - const auto &lhs_block1 = lhs1[block]; - const auto &lhs_block2 = lhs2[block]; - const auto &lhs_block3 = lhs3[block]; - uint32_t decoded_words0[4] = {}; - uint32_t decoded_words1[4] = {}; - uint32_t decoded_words2[4] = {}; - uint32_t decoded_words3[4] = {}; - decode_q4_k_scales_words(lhs_block0, decoded_words0); - decode_q4_k_scales_words(lhs_block1, decoded_words1); - decode_q4_k_scales_words(lhs_block2, decoded_words2); - decode_q4_k_scales_words(lhs_block3, decoded_words3); - - const auto *scales0 = reinterpret_cast(decoded_words0); - const auto *scales1 = reinterpret_cast(decoded_words1); - const auto *scales2 = reinterpret_cast(decoded_words2); - const auto *scales3 = reinterpret_cast(decoded_words3); - const auto *mins0 = reinterpret_cast(decoded_words0 + 2); - const auto *mins1 = reinterpret_cast(decoded_words1 + 2); - const auto *mins2 = reinterpret_cast(decoded_words2 + 2); - const auto *mins3 = reinterpret_cast(decoded_words3 + 2); - const int16x8_t q8_pair_sums = - vpaddq_s16(vld1q_s16(rhs[block].bsums.data()), - vld1q_s16(rhs[block].bsums.data() + 8)); - const int32_t min_sum0 = q4_k_min_sum_neon(mins0, q8_pair_sums); - const int32_t min_sum1 = q4_k_min_sum_neon(mins1, q8_pair_sums); - const int32_t min_sum2 = q4_k_min_sum_neon(mins2, q8_pair_sums); - const int32_t min_sum3 = q4_k_min_sum_neon(mins3, q8_pair_sums); - - const uint8_t *q40 = lhs_block0.qs.data(); - const uint8_t *q41 = lhs_block1.qs.data(); - const uint8_t *q42 = lhs_block2.qs.data(); - const uint8_t *q43 = lhs_block3.qs.data(); - const int8_t *q8 = rhs[block].qs.data(); - int32x4_t low_acc0 = vdupq_n_s32(0); - int32x4_t high_acc0 = vdupq_n_s32(0); - int32x4_t low_acc1 = vdupq_n_s32(0); - int32x4_t high_acc1 = vdupq_n_s32(0); - int32x4_t low_acc2 = vdupq_n_s32(0); - int32x4_t high_acc2 = vdupq_n_s32(0); - int32x4_t low_acc3 = vdupq_n_s32(0); - int32x4_t high_acc3 = vdupq_n_s32(0); - - for (uint64_t group = 0; - group < (::emel::kernel::detail::quant::QK_K / 64u); ++group) { - const uint8x16_t q4bits00 = vld1q_u8(q40 + 0u); - const uint8x16_t q4bits01 = vld1q_u8(q40 + 16u); - const uint8x16_t q4bits10 = vld1q_u8(q41 + 0u); - const uint8x16_t q4bits11 = vld1q_u8(q41 + 16u); - const uint8x16_t q4bits20 = vld1q_u8(q42 + 0u); - const uint8x16_t q4bits21 = vld1q_u8(q42 + 16u); - const uint8x16_t q4bits30 = vld1q_u8(q43 + 0u); - const uint8x16_t q4bits31 = vld1q_u8(q43 + 16u); - const int8x16_t q8bytes0 = vld1q_s8(q8 + 0u); - const int8x16_t q8bytes1 = vld1q_s8(q8 + 16u); - const int8x16_t q8bytes2 = vld1q_s8(q8 + 32u); - const int8x16_t q8bytes3 = vld1q_s8(q8 + 48u); - - const int8x16_t q4low00 = vreinterpretq_s8_u8(vandq_u8(q4bits00, m4b)); - const int8x16_t q4low01 = vreinterpretq_s8_u8(vandq_u8(q4bits01, m4b)); - const int8x16_t q4high00 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits00, 4)); - const int8x16_t q4high01 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits01, 4)); - const int8x16_t q4low10 = vreinterpretq_s8_u8(vandq_u8(q4bits10, m4b)); - const int8x16_t q4low11 = vreinterpretq_s8_u8(vandq_u8(q4bits11, m4b)); - const int8x16_t q4high10 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits10, 4)); - const int8x16_t q4high11 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits11, 4)); - const int8x16_t q4low20 = vreinterpretq_s8_u8(vandq_u8(q4bits20, m4b)); - const int8x16_t q4low21 = vreinterpretq_s8_u8(vandq_u8(q4bits21, m4b)); - const int8x16_t q4high20 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits20, 4)); - const int8x16_t q4high21 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits21, 4)); - const int8x16_t q4low30 = vreinterpretq_s8_u8(vandq_u8(q4bits30, m4b)); - const int8x16_t q4low31 = vreinterpretq_s8_u8(vandq_u8(q4bits31, m4b)); - const int8x16_t q4high30 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits30, 4)); - const int8x16_t q4high31 = vreinterpretq_s8_u8(vshrq_n_u8(q4bits31, 4)); - - const int32_t low_scale0 = static_cast(scales0[group * 2u + 0u]); - const int32_t high_scale0 = - static_cast(scales0[group * 2u + 1u]); - const int32_t low_scale1 = static_cast(scales1[group * 2u + 0u]); - const int32_t high_scale1 = - static_cast(scales1[group * 2u + 1u]); - const int32_t low_scale2 = static_cast(scales2[group * 2u + 0u]); - const int32_t high_scale2 = - static_cast(scales2[group * 2u + 1u]); - const int32_t low_scale3 = static_cast(scales3[group * 2u + 0u]); - const int32_t high_scale3 = - static_cast(scales3[group * 2u + 1u]); - - low_acc0 = - vmlaq_n_s32(low_acc0, vdotq_s32(zero, q4low00, q8bytes0), low_scale0); - low_acc0 = - vmlaq_n_s32(low_acc0, vdotq_s32(zero, q4low01, q8bytes1), low_scale0); - high_acc0 = vmlaq_n_s32(high_acc0, vdotq_s32(zero, q4high00, q8bytes2), - high_scale0); - high_acc0 = vmlaq_n_s32(high_acc0, vdotq_s32(zero, q4high01, q8bytes3), - high_scale0); - low_acc1 = - vmlaq_n_s32(low_acc1, vdotq_s32(zero, q4low10, q8bytes0), low_scale1); - low_acc1 = - vmlaq_n_s32(low_acc1, vdotq_s32(zero, q4low11, q8bytes1), low_scale1); - high_acc1 = vmlaq_n_s32(high_acc1, vdotq_s32(zero, q4high10, q8bytes2), - high_scale1); - high_acc1 = vmlaq_n_s32(high_acc1, vdotq_s32(zero, q4high11, q8bytes3), - high_scale1); - low_acc2 = - vmlaq_n_s32(low_acc2, vdotq_s32(zero, q4low20, q8bytes0), low_scale2); - low_acc2 = - vmlaq_n_s32(low_acc2, vdotq_s32(zero, q4low21, q8bytes1), low_scale2); - high_acc2 = vmlaq_n_s32(high_acc2, vdotq_s32(zero, q4high20, q8bytes2), - high_scale2); - high_acc2 = vmlaq_n_s32(high_acc2, vdotq_s32(zero, q4high21, q8bytes3), - high_scale2); - low_acc3 = - vmlaq_n_s32(low_acc3, vdotq_s32(zero, q4low30, q8bytes0), low_scale3); - low_acc3 = - vmlaq_n_s32(low_acc3, vdotq_s32(zero, q4low31, q8bytes1), low_scale3); - high_acc3 = vmlaq_n_s32(high_acc3, vdotq_s32(zero, q4high30, q8bytes2), - high_scale3); - high_acc3 = vmlaq_n_s32(high_acc3, vdotq_s32(zero, q4high31, q8bytes3), - high_scale3); - - q40 += 32u; - q41 += 32u; - q42 += 32u; - q43 += 32u; - q8 += 64u; - } +inline void decode_q4_k_x8_6bit_scales_neon(const uint8_t *scales_in, + int16x8_t &mins_out, + int16x8_t &scales_out) noexcept { +#if !(defined(__aarch64__) && defined(__ARM_NEON)) + (void)scales_in; + (void)mins_out; + (void)scales_out; +#else + constexpr uint32_t kmask1 = 0x3f3f3f3fu; + constexpr uint32_t kmask2 = 0x0f0f0f0fu; + constexpr uint32_t kmask3 = 0x03030303u; + std::array sm = {}; + std::memcpy(sm.data(), scales_in, 12u); - const int32_t isum0 = vaddvq_s32(vaddq_s32(low_acc0, high_acc0)); - const int32_t isum1 = vaddvq_s32(vaddq_s32(low_acc1, high_acc1)); - const int32_t isum2 = vaddvq_s32(vaddq_s32(low_acc2, high_acc2)); - const int32_t isum3 = vaddvq_s32(vaddq_s32(low_acc3, high_acc3)); - const float rhs_d = rhs[block].d; - const float d0 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block0.d) * rhs_d; - const float d1 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block1.d) * rhs_d; - const float d2 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block2.d) * rhs_d; - const float d3 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block3.d) * rhs_d; - const float dmin0 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block0.dmin) * rhs_d; - const float dmin1 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block1.dmin) * rhs_d; - const float dmin2 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block2.dmin) * rhs_d; - const float dmin3 = - ::emel::kernel::detail::quant::fp16_to_fp32(lhs_block3.dmin) * rhs_d; - sum0 += - d0 * static_cast(isum0) - dmin0 * static_cast(min_sum0); - sum1 += - d1 * static_cast(isum1) - dmin1 * static_cast(min_sum1); - sum2 += - d2 * static_cast(isum2) - dmin2 * static_cast(min_sum2); - sum3 += - d3 * static_cast(isum3) - dmin3 * static_cast(min_sum3); - } + const uint32_t mins_0_3 = sm[1] & kmask1; + const uint32_t mins_4_7 = + ((sm[2] >> 4u) & kmask2) | (((sm[1] >> 6u) & kmask3) << 4u); + const uint32x2_t mins_u32 = {mins_0_3, mins_4_7}; + mins_out = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins_u32))); - out[0] = sum0; - out[1] = sum1; - out[2] = sum2; - out[3] = sum3; + std::array scales_u32 = {}; + scales_u32[0] = sm[0] & kmask1; + scales_u32[1] = + (sm[2] & kmask2) | (((sm[0] >> 6u) & kmask3) << 4u); + std::array scales_bytes = {}; + std::memcpy(scales_bytes.data(), scales_u32.data(), scales_bytes.size()); + scales_out = vmovl_s8(vld1_s8(scales_bytes.data())); #endif } @@ -3053,12 +2985,9 @@ inline void dot_q4_k_x8_q8_k_group_bl4_neon( std::array q4sb_scales = {}; for (uint64_t half = 0; half < 2u; ++half) { const uint8_t *prepared = - q4_block.scales.data() + - ((sb * 2u + half) * ::emel::kernel::detail::quant::Q4_K_X8_ROWS * - 2u); - q4sb_mins[half] = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(prepared))); - q4sb_scales[half] = vreinterpretq_s16_u16(vmovl_u8( - vld1_u8(prepared + ::emel::kernel::detail::quant::Q4_K_X8_ROWS))); + q4_block.scales.data() + sb * 24u + half * 12u; + decode_q4_k_x8_6bit_scales_neon(prepared, q4sb_mins[half], + q4sb_scales[half]); } std::array q8_qs = {}; @@ -3181,32 +3110,25 @@ inline void dot_q4_k_x8_q8_k_group_bl8_neon( (void)block_count; std::fill(out, out + ::emel::kernel::detail::quant::Q4_K_X8_ROWS, 0.0f); #else + // Keep weights in their x8 interleaved representation for the integer dot + // work. Apply each block's float scale/min terms row-wise in the same order + // as the native Q4_K kernel so fixed-seed behavior remains bit-exact. constexpr uint64_t col_pairs = ::emel::kernel::detail::quant::Q4_K_X8_ROWS / 2u; - std::array - acc_f32 = {}; + std::array sums = {}; const uint8x16_t m4b = vdupq_n_u8(0x0fu); - for (auto &acc : acc_f32) { - acc = vdupq_n_f32(0.0f); - } for (uint64_t block = 0; block < block_count; ++block) { const auto &q4_block = lhs[block]; const auto &q8_block = rhs[block]; - const float32x4_t q4_d_0 = vcvt_f32_f16( - vld1_f16(reinterpret_cast(q4_block.d.data()))); - const float32x4_t q4_d_1 = vcvt_f32_f16( - vld1_f16(reinterpret_cast(q4_block.d.data() + 4u))); - const float32x4_t q8_d = vdupq_n_f32(q8_block.d); - const float32x4_t sb_scale_0 = vmulq_f32(q4_d_0, q8_d); - const float32x4_t sb_scale_1 = vmulq_f32(q4_d_1, q8_d); - const float32x4_t q4_dmin_0 = vcvt_f32_f16( - vld1_f16(reinterpret_cast(q4_block.dmin.data()))); - const float32x4_t q4_dmin_1 = vcvt_f32_f16( - vld1_f16(reinterpret_cast(q4_block.dmin.data() + 4u))); - const float32x4_t sb_min_0 = vmulq_f32(q4_dmin_0, q8_d); - const float32x4_t sb_min_1 = vmulq_f32(q4_dmin_1, q8_d); - + std::array low_acc = { + vdupq_n_s32(0), + vdupq_n_s32(0), + }; + std::array high_acc = { + vdupq_n_s32(0), + vdupq_n_s32(0), + }; std::array bias_acc = { vdupq_n_s32(0), vdupq_n_s32(0), @@ -3222,12 +3144,9 @@ inline void dot_q4_k_x8_q8_k_group_bl8_neon( std::array q4sb_scales = {}; for (uint64_t half = 0; half < 2u; ++half) { const uint8_t *prepared = - q4_block.scales.data() + - ((sb * 2u + half) * ::emel::kernel::detail::quant::Q4_K_X8_ROWS * - 2u); - q4sb_mins[half] = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(prepared))); - q4sb_scales[half] = vreinterpretq_s16_u16(vmovl_u8( - vld1_u8(prepared + ::emel::kernel::detail::quant::Q4_K_X8_ROWS))); + q4_block.scales.data() + sb * 24u + half * 12u; + decode_q4_k_x8_6bit_scales_neon(prepared, q4sb_mins[half], + q4sb_scales[half]); } const uint8_t *q4_base = @@ -3279,25 +3198,18 @@ inline void dot_q4_k_x8_q8_k_group_bl8_neon( q8_qs[7]); } - for (uint64_t i = 0, pair = 0; pair < col_pairs; i += 1u, pair += 2u) { - const int16x4_t group_scales_lo = pair == 0u - ? vget_low_s16(q4sb_scales[0]) - : vget_high_s16(q4sb_scales[0]); - const int16x4_t group_scales_hi = pair == 0u - ? vget_low_s16(q4sb_scales[1]) - : vget_high_s16(q4sb_scales[1]); - const float32x4_t sb_scale = pair == 0u ? sb_scale_0 : sb_scale_1; - - const float32x4_t sumf_0 = vcvtq_f32_s32( - vmulq_s32(vmovl_s16(group_scales_lo), - vpaddq_s32(acc_lo[pair], acc_lo[pair + 1u]))); - acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_0); - - const float32x4_t sumf_1 = vcvtq_f32_s32( - vmulq_s32(vmovl_s16(group_scales_hi), - vpaddq_s32(acc_hi[pair], acc_hi[pair + 1u]))); - acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_1); - } + low_acc[0] = vaddq_s32(low_acc[0], + vmulq_s32(vmovl_s16(vget_low_s16(q4sb_scales[0])), + vpaddq_s32(acc_lo[0], acc_lo[1]))); + high_acc[0] = vaddq_s32(high_acc[0], + vmulq_s32(vmovl_s16(vget_low_s16(q4sb_scales[1])), + vpaddq_s32(acc_hi[0], acc_hi[1]))); + low_acc[1] = vaddq_s32(low_acc[1], + vmulq_s32(vmovl_s16(vget_high_s16(q4sb_scales[0])), + vpaddq_s32(acc_lo[2], acc_lo[3]))); + high_acc[1] = vaddq_s32( + high_acc[1], vmulq_s32(vmovl_s16(vget_high_s16(q4sb_scales[1])), + vpaddq_s32(acc_hi[2], acc_hi[3]))); const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_array[2u * sb + 0u]); const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_array[2u * sb + 1u]); @@ -3311,12 +3223,28 @@ inline void dot_q4_k_x8_q8_k_group_bl8_neon( vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q4sb_mins[1])); } - acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0); - acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_1); + alignas(16) std::array + low_sums = {}; + alignas(16) std::array + high_sums = {}; + alignas(16) std::array + min_sums = {}; + vst1q_s32(low_sums.data(), low_acc[0]); + vst1q_s32(low_sums.data() + 4u, low_acc[1]); + vst1q_s32(high_sums.data(), high_acc[0]); + vst1q_s32(high_sums.data() + 4u, high_acc[1]); + vst1q_s32(min_sums.data(), bias_acc[0]); + vst1q_s32(min_sums.data() + 4u, bias_acc[1]); + + for (uint64_t row = 0; row < ::emel::kernel::detail::quant::Q4_K_X8_ROWS; + ++row) { + accumulate_q4_k_scaled_sums(q4_block.d[row], q4_block.dmin[row], + q8_block.d, min_sums[row], + low_sums[row] + high_sums[row], sums[row]); + } } - vst1q_f32(out, acc_f32[0]); - vst1q_f32(out + 4u, acc_f32[1]); + std::copy(sums.begin(), sums.end(), out); #endif } @@ -3394,12 +3322,9 @@ inline void dot_q4_k_x8_q8_k_group_bl8_x4_neon( std::array q4sb_scales = {}; for (uint64_t half = 0; half < 2u; ++half) { const uint8_t *prepared = - q4_block.scales.data() + - ((sb * 2u + half) * ::emel::kernel::detail::quant::Q4_K_X8_ROWS * - 2u); - q4sb_mins[half] = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(prepared))); - q4sb_scales[half] = vreinterpretq_s16_u16(vmovl_u8( - vld1_u8(prepared + ::emel::kernel::detail::quant::Q4_K_X8_ROWS))); + q4_block.scales.data() + sb * 24u + half * 12u; + decode_q4_k_x8_6bit_scales_neon(prepared, q4sb_mins[half], + q4sb_scales[half]); } const uint8_t *q4_base = @@ -3587,12 +3512,9 @@ inline void dot_q4_k_x8_q8_k_group_bl8_x8_neon( std::array q4sb_scales = {}; for (uint64_t half = 0; half < 2u; ++half) { const uint8_t *prepared = - q4_block.scales.data() + - ((sb * 2u + half) * ::emel::kernel::detail::quant::Q4_K_X8_ROWS * - 2u); - q4sb_mins[half] = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(prepared))); - q4sb_scales[half] = vreinterpretq_s16_u16(vmovl_u8( - vld1_u8(prepared + ::emel::kernel::detail::quant::Q4_K_X8_ROWS))); + q4_block.scales.data() + sb * 24u + half * 12u; + decode_q4_k_x8_6bit_scales_neon(prepared, q4sb_mins[half], + q4sb_scales[half]); } const uint8_t *q4_base = @@ -5668,8 +5590,8 @@ inline void execute_neon_mul_mat_argmax_q6_vector_packed_q8_rhs_unchecked( #endif } -inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_unchecked( - const event::op_mul_mat &request) noexcept { +inline void execute_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_unchecked( + const event::op_mul_mat_argmax &request) noexcept { #if !(defined(__aarch64__) && defined(__ARM_NEON) && \ defined(__ARM_FEATURE_DOTPROD)) (void)request; @@ -5677,15 +5599,117 @@ inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_unchecked( const uint64_t k = request.src0.ne[0]; const uint64_t m = request.src0.ne[1]; const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; - const auto *q8_blocks = - static_cast( - request.src1.data); - const uint8_t *packed = static_cast(request.src0.data); - const size_t group_bytes = request.src0.nb[1]; - const uint64_t group_count = - ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); - float *dst = static_cast(request.dst.data); - for (uint64_t group = 0; group < group_count; ++group) { + std::array<::emel::kernel::detail::quant::block_q8_k, + ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS> + q8_blocks = {}; + ::emel::kernel::detail::quant::quantize_row_q8_k_strided( + static_cast(request.src1.data), 1u, q8_blocks.data(), + static_cast(k)); + + const uint8_t *packed = static_cast(request.src0.data); + const size_t group_bytes = request.src0.nb[1]; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + float best_value = -std::numeric_limits::infinity(); + int32_t best_index = 0; + bool have_best = false; + for (uint64_t group = 0; group < group_count; ++group) { + const auto *group_ptr = + reinterpret_cast( + packed + group * group_bytes); + std::array group_out = + {}; + dot_q4_k_x8_q8_k_group_bl4_neon(group_ptr, q8_blocks.data(), block_count, + group_out.data()); + const uint64_t row_base = + group * ::emel::kernel::detail::quant::Q4_K_X8_ROWS; + const uint64_t rows_in_group = std::min( + static_cast(::emel::kernel::detail::quant::Q4_K_X8_ROWS), + m - row_base); + for (uint64_t row = 0; row < rows_in_group; ++row) { + const float value = group_out[row]; + if (!have_best || value > best_value) { + have_best = true; + best_value = value; + best_index = static_cast(row_base + row); + } + } + } + + static_cast(request.dst.data)[0] = best_value; + *request.index_out = best_index; +#endif +} + +inline void execute_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_unchecked( + const event::op_mul_mat_argmax &request) noexcept { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + (void)request; +#else + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + std::array<::emel::kernel::detail::quant::block_q8_k, + ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS> + q8_blocks = {}; + ::emel::kernel::detail::quant::quantize_row_q8_k_strided( + static_cast(request.src1.data), 1u, q8_blocks.data(), + static_cast(k)); + + const uint8_t *packed = static_cast(request.src0.data); + const size_t group_bytes = request.src0.nb[1]; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + float best_value = -std::numeric_limits::infinity(); + int32_t best_index = 0; + bool have_best = false; + for (uint64_t group = 0; group < group_count; ++group) { + const auto *group_ptr = + reinterpret_cast( + packed + group * group_bytes); + std::array group_out = + {}; + dot_q4_k_x8_q8_k_group_bl8_neon(group_ptr, q8_blocks.data(), block_count, + group_out.data()); + const uint64_t row_base = + group * ::emel::kernel::detail::quant::Q4_K_X8_ROWS; + const uint64_t rows_in_group = std::min( + static_cast(::emel::kernel::detail::quant::Q4_K_X8_ROWS), + m - row_base); + for (uint64_t row = 0; row < rows_in_group; ++row) { + const float value = group_out[row]; + if (!have_best || value > best_value) { + have_best = true; + best_value = value; + best_index = static_cast(row_base + row); + } + } + } + + static_cast(request.dst.data)[0] = best_value; + *request.index_out = best_index; +#endif +} + +inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_unchecked( + const event::op_mul_mat &request) noexcept { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + (void)request; +#else + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + const auto *q8_blocks = + static_cast( + request.src1.data); + const uint8_t *packed = static_cast(request.src0.data); + const size_t group_bytes = request.src0.nb[1]; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + float *dst = static_cast(request.dst.data); + for (uint64_t group = 0; group < group_count; ++group) { const auto *group_ptr = reinterpret_cast( packed + group * group_bytes); @@ -5705,6 +5729,47 @@ inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_unchecked( #endif } +inline void execute_neon_mul_mat_q4_vector_packed_f32_rhs_bl4_unchecked( + const event::op_mul_mat &request) noexcept { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + (void)request; +#else + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + std::array<::emel::kernel::detail::quant::block_q8_k, + ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS> + q8_blocks = {}; + ::emel::kernel::detail::quant::quantize_row_q8_k_strided( + static_cast(request.src1.data), 1u, q8_blocks.data(), + static_cast(k)); + + const uint8_t *packed = static_cast(request.src0.data); + const size_t group_bytes = request.src0.nb[1]; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + float *dst = static_cast(request.dst.data); + for (uint64_t group = 0; group < group_count; ++group) { + const auto *group_ptr = + reinterpret_cast( + packed + group * group_bytes); + std::array group_out = + {}; + dot_q4_k_x8_q8_k_group_bl4_neon(group_ptr, q8_blocks.data(), block_count, + group_out.data()); + const uint64_t row_base = + group * ::emel::kernel::detail::quant::Q4_K_X8_ROWS; + const uint64_t rows_in_group = std::min( + static_cast(::emel::kernel::detail::quant::Q4_K_X8_ROWS), + m - row_base); + for (uint64_t row = 0; row < rows_in_group; ++row) { + dst[row_base + row] = group_out[row]; + } + } +#endif +} + inline void store_batch_major_group_results(const float *group_out, float *dst, const size_t dst_row_stride, @@ -5796,6 +5861,47 @@ inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_unchecked( #endif } +inline void execute_neon_mul_mat_q4_vector_packed_f32_rhs_bl8_unchecked( + const event::op_mul_mat &request) noexcept { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + (void)request; +#else + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + std::array<::emel::kernel::detail::quant::block_q8_k, + ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS> + q8_blocks = {}; + ::emel::kernel::detail::quant::quantize_row_q8_k_strided( + static_cast(request.src1.data), 1u, q8_blocks.data(), + static_cast(k)); + + const uint8_t *packed = static_cast(request.src0.data); + const size_t group_bytes = request.src0.nb[1]; + const uint64_t group_count = + ::emel::kernel::detail::quant::packed_q4_k_x8_group_count(m); + float *dst = static_cast(request.dst.data); + for (uint64_t group = 0; group < group_count; ++group) { + const auto *group_ptr = + reinterpret_cast( + packed + group * group_bytes); + std::array group_out = + {}; + dot_q4_k_x8_q8_k_group_bl8_neon(group_ptr, q8_blocks.data(), block_count, + group_out.data()); + const uint64_t row_base = + group * ::emel::kernel::detail::quant::Q4_K_X8_ROWS; + const uint64_t rows_in_group = std::min( + static_cast(::emel::kernel::detail::quant::Q4_K_X8_ROWS), + m - row_base); + for (uint64_t row = 0; row < rows_in_group; ++row) { + dst[row_base + row] = group_out[row]; + } + } +#endif +} + inline void execute_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4_unchecked( const event::op_mul_mat &request) noexcept { @@ -6707,6 +6813,70 @@ inline void execute_neon_mul_mat_q4_vector_q8_rhs_unchecked( #endif } +inline void execute_neon_mul_mat_q4_vector_f32_rhs_unchecked( + const event::op_mul_mat &request) noexcept { +#if !(defined(__aarch64__) || defined(__ARM_NEON)) + (void)request; +#else + const uint64_t k = request.src0.ne[0]; + const uint64_t m = request.src0.ne[1]; + const uint64_t block_count = k / ::emel::kernel::detail::quant::QK_K; + std::array<::emel::kernel::detail::quant::block_q8_k, + ::emel::kernel::detail::quant::MAX_Q8_K_BLOCKS> + q8_blocks = {}; + ::emel::kernel::detail::quant::quantize_row_q8_k_strided( + static_cast(request.src1.data), 1u, q8_blocks.data(), + static_cast(k)); + + const uint8_t *a = static_cast(request.src0.data); + const size_t row_bytes = request.src0.nb[1]; + float *c = static_cast(request.dst.data); + + uint64_t row = 0u; + for (; row + 4u <= m; row += 4u) { + const auto *row0 = + reinterpret_cast( + a + (row + 0u) * row_bytes); + const auto *row1 = + reinterpret_cast( + a + (row + 1u) * row_bytes); + const auto *row2 = + reinterpret_cast( + a + (row + 2u) * row_bytes); + const auto *row3 = + reinterpret_cast( + a + (row + 3u) * row_bytes); + float out[4] = {}; + dot_q4_k_q8_k_4rows_neon(row0, row1, row2, row3, q8_blocks.data(), + block_count, out); + c[row + 0u] = out[0]; + c[row + 1u] = out[1]; + c[row + 2u] = out[2]; + c[row + 3u] = out[3]; + } + + for (; row + 2u <= m; row += 2u) { + const auto *row0 = + reinterpret_cast( + a + (row + 0u) * row_bytes); + const auto *row1 = + reinterpret_cast( + a + (row + 1u) * row_bytes); + float out[2] = {}; + dot_q4_k_q8_k_2rows_neon(row0, row1, q8_blocks.data(), block_count, out); + c[row + 0u] = out[0]; + c[row + 1u] = out[1]; + } + + for (; row < m; ++row) { + const auto *row_ptr = + reinterpret_cast( + a + row * row_bytes); + c[row] = dot_q4_k_q8_k_row_neon(row_ptr, q8_blocks.data(), block_count); + } +#endif +} + // Exact port of the pinned ggml_vec_dot_f16 aarch64 NEON fp16-arithmetic // path (GGML_F16_STEP=32): four float16x8_t accumulators advanced with // vfmaq_f16, pairwise vaddq_f16 reduce, f32 convert + horizontal add, and an @@ -6765,8 +6935,8 @@ inline float dot_f32_f32_row_neon(const int64_t count, const float *x, return 0.0f; #else const int64_t np = count & ~static_cast(15); - float32x4_t sum[4] = {vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), - vdupq_n_f32(0.0f), vdupq_n_f32(0.0f)}; + float32x4_t sum[4] = {vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), + vdupq_n_f32(0.0f)}; for (int64_t i = 0; i < np; i += 16) { for (int64_t j = 0; j < 4; ++j) { const float32x4_t ax = vld1q_f32(x + i + j * 4); @@ -6785,15 +6955,10 @@ inline float dot_f32_f32_row_neon(const int64_t count, const float *x, #endif } -inline void dot_f32_f32_4rows_neon(const int64_t count, - const float *row0, - const float *row1, - const float *row2, - const float *row3, - const float *y, - float &out0, - float &out1, - float &out2, +inline void dot_f32_f32_4rows_neon(const int64_t count, const float *row0, + const float *row1, const float *row2, + const float *row3, const float *y, + float &out0, float &out1, float &out2, float &out3) noexcept { #if !(defined(__aarch64__) || defined(__ARM_NEON)) (void)count; @@ -6865,16 +7030,10 @@ inline void execute_neon_mul_mat_f32_vector_unchecked( float *dst = static_cast(request.dst.data); uint64_t row = 0u; for (; row + 4u <= m; row += 4u) { - dot_f32_f32_4rows_neon(static_cast(k), - a + (row + 0u) * k, - a + (row + 1u) * k, - a + (row + 2u) * k, - a + (row + 3u) * k, - y, - dst[row + 0u], - dst[row + 1u], - dst[row + 2u], - dst[row + 3u]); + dot_f32_f32_4rows_neon(static_cast(k), a + (row + 0u) * k, + a + (row + 1u) * k, a + (row + 2u) * k, + a + (row + 3u) * k, y, dst[row + 0u], dst[row + 1u], + dst[row + 2u], dst[row + 3u]); } for (; row < m; ++row) { dst[row] = dot_f32_f32_row_neon(static_cast(k), a + row * k, y); @@ -8216,71 +8375,53 @@ inline bool execute_neon_mul_mat_prepared_f32_lhs_4row( #endif } -inline bool execute_neon_unary(const event::op_unary &request) noexcept { +inline void +execute_neon_unary_abs_request(const event::op_unary &request) noexcept { #if defined(__aarch64__) || defined(__ARM_NEON) - const uint8_t subop_code = static_cast(request.subop); - const size_t is_abs = static_cast( - subop_code == static_cast(event::unary_subop::abs)); - const size_t is_neg = static_cast( - subop_code == static_cast(event::unary_subop::neg)); - const size_t is_relu = static_cast( - subop_code == static_cast(event::unary_subop::relu)); - const size_t kernel_index = is_abs * 1u + is_neg * 2u + is_relu * 3u; const uint64_t count = ::emel::kernel::detail::tensor_element_count(request.dst); const float *src = static_cast(request.src0.data); float *dst = static_cast(request.dst.data); - using unary_kernel_t = void (*)(const float *, float *, uint64_t) noexcept; - constexpr unary_kernel_t noop_kernel = - +[](const float *, float *, uint64_t) noexcept {}; - constexpr std::array kernels = { - noop_kernel, - execute_neon_unary_abs, - execute_neon_unary_neg, - execute_neon_unary_relu, - }; - kernels[kernel_index](src, dst, count); - return kernel_index != 0u; + execute_neon_unary_abs(src, dst, count); #else (void)request; - return false; #endif } inline void -execute_neon_unary_abs_request(const event::op_unary &request) noexcept { +execute_neon_unary_neg_request(const event::op_unary &request) noexcept { #if defined(__aarch64__) || defined(__ARM_NEON) const uint64_t count = ::emel::kernel::detail::tensor_element_count(request.dst); const float *src = static_cast(request.src0.data); float *dst = static_cast(request.dst.data); - execute_neon_unary_abs(src, dst, count); + execute_neon_unary_neg(src, dst, count); #else (void)request; #endif } inline void -execute_neon_unary_neg_request(const event::op_unary &request) noexcept { +execute_neon_unary_relu_request(const event::op_unary &request) noexcept { #if defined(__aarch64__) || defined(__ARM_NEON) const uint64_t count = ::emel::kernel::detail::tensor_element_count(request.dst); const float *src = static_cast(request.src0.data); float *dst = static_cast(request.dst.data); - execute_neon_unary_neg(src, dst, count); + execute_neon_unary_relu(src, dst, count); #else (void)request; #endif } inline void -execute_neon_unary_relu_request(const event::op_unary &request) noexcept { +execute_neon_unary_silu_request(const event::op_unary &request) noexcept { #if defined(__aarch64__) || defined(__ARM_NEON) const uint64_t count = ::emel::kernel::detail::tensor_element_count(request.dst); const float *src = static_cast(request.src0.data); float *dst = static_cast(request.dst.data); - execute_neon_unary_relu(src, dst, count); + execute_neon_unary_silu(src, dst, count); #else (void)request; #endif @@ -8298,6 +8439,9 @@ execute_simd_unary_subop_unchecked(const event::op_unary &request) noexcept { if constexpr (subop == event::unary_subop::relu) { execute_neon_unary_relu_request(request); } + if constexpr (subop == event::unary_subop::silu) { + execute_neon_unary_silu_request(request); + } } template @@ -8326,9 +8470,6 @@ inline void execute_simd_unchecked(const request_type &request) noexcept { if constexpr (std::is_same_v) { (void)execute_neon_mul_mat(request); } - if constexpr (std::is_same_v) { - (void)execute_neon_unary(request); - } } template @@ -8357,9 +8498,6 @@ inline bool execute_simd(const request_type &request) noexcept { if constexpr (std::is_same_v) { return execute_neon_mul_mat(request); } - if constexpr (std::is_same_v) { - return execute_neon_unary(request); - } return false; } @@ -8498,6 +8636,19 @@ struct exec_simd_q4_vector_packed_q8_rhs_bl4_matrix_x4_op_mul_mat { } }; +struct exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat { + void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + context &ctx) const noexcept { + ::emel::kernel::aarch64::detail:: + execute_neon_mul_mat_q4_vector_packed_f32_rhs_bl4_unchecked( + ev.request); + ++ctx.optimized_q4_dispatch_count; + ++ctx.optimized_q4_vector_dispatch_count; + ++ctx.optimized_q4_vector_packed_dispatch_count; + detail::mark_done(ev, ctx); + } +}; + struct exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat { void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, context &ctx) const noexcept { @@ -8511,6 +8662,19 @@ struct exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat { } }; +struct exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat { + void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + context &ctx) const noexcept { + ::emel::kernel::aarch64::detail:: + execute_neon_mul_mat_q4_vector_packed_f32_rhs_bl8_unchecked( + ev.request); + ++ctx.optimized_q4_dispatch_count; + ++ctx.optimized_q4_vector_dispatch_count; + ++ctx.optimized_q4_vector_packed_dispatch_count; + detail::mark_done(ev, ctx); + } +}; + struct exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x4_op_mul_mat { void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, context &ctx) const noexcept { @@ -8550,6 +8714,17 @@ struct exec_simd_q4_vector_q8_rhs_op_mul_mat { } }; +struct exec_simd_q4_vector_f32_rhs_op_mul_mat { + void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + context &ctx) const noexcept { + ::emel::kernel::aarch64::detail:: + execute_neon_mul_mat_q4_vector_f32_rhs_unchecked(ev.request); + ++ctx.optimized_q4_dispatch_count; + ++ctx.optimized_q4_vector_dispatch_count; + detail::mark_done(ev, ctx); + } +}; + struct exec_simd_q8_0_vector_op_mul_mat { void operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, context &ctx) const noexcept { @@ -8828,6 +9003,34 @@ struct exec_simd_q6_vector_q8_argmax_prepared_i8mm_op_mul_mat_argmax { } }; +struct exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_argmax { + void operator()( + const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, + context &ctx) const noexcept { + ::emel::kernel::aarch64::detail:: + execute_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_unchecked( + ev.request); + ++ctx.optimized_q4_dispatch_count; + ++ctx.optimized_q4_vector_dispatch_count; + ++ctx.optimized_q4_vector_packed_dispatch_count; + detail::mark_done(ev, ctx); + } +}; + +struct exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_argmax { + void operator()( + const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, + context &ctx) const noexcept { + ::emel::kernel::aarch64::detail:: + execute_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_unchecked( + ev.request); + ++ctx.optimized_q4_dispatch_count; + ++ctx.optimized_q4_vector_dispatch_count; + ++ctx.optimized_q4_vector_packed_dispatch_count; + detail::mark_done(ev, ctx); + } +}; + struct exec_simd_q6_vector_packed_q8_rhs_op_mul_mat_argmax { void operator()( const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, @@ -8901,14 +9104,20 @@ using exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_t = detail::exec_simd_q4_vector_packed_q8_rhs_bl4_op_mul_mat; using exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4_t = detail::exec_simd_q4_vector_packed_q8_rhs_bl4_matrix_x4_op_mul_mat; +using exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4_t = + detail::exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat; using exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_t = detail::exec_simd_q4_vector_packed_q8_rhs_bl8_op_mul_mat; +using exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8_t = + detail::exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat; using exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8_t = detail::exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x8_op_mul_mat; using exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4_t = detail::exec_simd_q4_vector_packed_q8_rhs_bl8_matrix_x4_op_mul_mat; using exec_simd_op_mul_mat_q4_vector_q8_rhs_t = detail::exec_simd_q4_vector_q8_rhs_op_mul_mat; +using exec_simd_op_mul_mat_q4_vector_f32_rhs_t = + detail::exec_simd_q4_vector_f32_rhs_op_mul_mat; using exec_simd_op_mul_mat_q5_0_vector_t = detail::exec_simd_q5_0_vector_op_mul_mat; using exec_simd_op_mul_mat_q4_0_vector_t = @@ -8947,6 +9156,10 @@ using exec_simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_t = detail::exec_simd_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_op_mul_mat; using exec_simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs_t = detail::exec_simd_q6_vector_packed_q8_rhs_op_mul_mat_argmax; +using exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_t = + detail::exec_simd_q4_vector_packed_f32_rhs_bl4_op_mul_mat_argmax; +using exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_t = + detail::exec_simd_q4_vector_packed_f32_rhs_bl8_op_mul_mat_argmax; using exec_simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm_t = detail::exec_simd_q6_vector_prepared_q8_rhs_i8mm_op_mul_mat_argmax; using exec_simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm_t = @@ -8966,6 +9179,8 @@ using exec_simd_op_unary_neg_t = detail::exec_simd_unary_op<::emel::kernel::event::unary_subop::neg>; using exec_simd_op_unary_relu_t = detail::exec_simd_unary_op<::emel::kernel::event::unary_subop::relu>; +using exec_simd_op_unary_silu_t = + detail::exec_simd_unary_op<::emel::kernel::event::unary_subop::silu>; using exec_scalar_op_unary_abs_t = ::emel::kernel::detail::exec_scalar_unary_op< ::emel::kernel::aarch64::event::dispatch_op_unary, context, detail::mark_done_op, ::emel::kernel::event::unary_subop::abs>; @@ -9022,6 +9237,10 @@ using exec_scalar_op_rope_norm_t = ::emel::kernel::detail::exec_scalar_rope_op< using exec_scalar_op_rope_neox_t = ::emel::kernel::detail::exec_scalar_rope_op< ::emel::kernel::aarch64::event::dispatch_op_rope, context, detail::mark_done_op, true>; +using exec_scalar_op_rope_timestep_t = + ::emel::kernel::detail::exec_scalar_timestep_rope_op< + ::emel::kernel::aarch64::event::dispatch_op_rope, context, + detail::mark_done_op>; using exec_scalar_op_im2col_f32_t = ::emel::kernel::detail::exec_scalar_im2col_op< ::emel::kernel::aarch64::event::dispatch_op_im2col, context, @@ -9079,14 +9298,20 @@ inline constexpr exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_t exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4{}; inline constexpr exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4_t exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4{}; +inline constexpr exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4_t + exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4{}; inline constexpr exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_t exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8{}; +inline constexpr exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8_t + exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8{}; inline constexpr exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8_t exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8{}; inline constexpr exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4_t exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4{}; inline constexpr exec_simd_op_mul_mat_q4_vector_q8_rhs_t exec_simd_op_mul_mat_q4_vector_q8_rhs{}; +inline constexpr exec_simd_op_mul_mat_q4_vector_f32_rhs_t + exec_simd_op_mul_mat_q4_vector_f32_rhs{}; inline constexpr exec_simd_op_mul_mat_q5_0_vector_t exec_simd_op_mul_mat_q5_0_vector{}; inline constexpr exec_simd_op_mul_mat_q4_0_vector_t @@ -9125,6 +9350,10 @@ inline constexpr exec_simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4_t exec_simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4{}; inline constexpr exec_simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs_t exec_simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs{}; +inline constexpr exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4_t + exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4{}; +inline constexpr exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8_t + exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8{}; inline constexpr exec_simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm_t exec_simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm{}; inline constexpr exec_simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm_t @@ -9158,6 +9387,7 @@ inline constexpr exec_scalar_op_get_rows_q8_0_t exec_scalar_op_get_rows_q8_0{}; inline constexpr exec_scalar_op_get_rows_q4_k_t exec_scalar_op_get_rows_q4_k{}; inline constexpr exec_scalar_op_rope_norm_t exec_scalar_op_rope_norm{}; inline constexpr exec_scalar_op_rope_neox_t exec_scalar_op_rope_neox{}; +inline constexpr exec_scalar_op_rope_timestep_t exec_scalar_op_rope_timestep{}; inline constexpr exec_scalar_op_im2col_f32_t exec_scalar_op_im2col_f32{}; inline constexpr exec_scalar_op_im2col_f16_t exec_scalar_op_im2col_f16{}; inline constexpr exec_scalar_op_conv_transpose_1d_f32_t diff --git a/src/emel/kernel/aarch64/guards.hpp b/src/emel/kernel/aarch64/guards.hpp index 29191671..c2571456 100644 --- a/src/emel/kernel/aarch64/guards.hpp +++ b/src/emel/kernel/aarch64/guards.hpp @@ -18,9 +18,9 @@ namespace detail { // rows) and the group-aligned row-slice lane view emitted by the parallel // matmul slicer, whose slice is embedded in a wider output (nb[0] covers the // full output row while ne[1]/nb[2] describe only the slice rows). -inline bool is_batch_major_dst_view(const ::emel::kernel::event::tensor_view_mut &dst, - const uint64_t m, - const uint64_t rhs_rows) noexcept { +inline bool +is_batch_major_dst_view(const ::emel::kernel::event::tensor_view_mut &dst, + const uint64_t m, const uint64_t rhs_rows) noexcept { // Keep every byte-stride term in uint64_t: mixing size_t in the products // could truncate on 32-bit hosts and mis-accept a stride contract. constexpr uint64_t k_f32_bytes = sizeof(float); @@ -33,7 +33,8 @@ inline bool is_batch_major_dst_view(const ::emel::kernel::event::tensor_view_mut } inline bool can_use_neon_mul_mat_q8_0_packed_bl8_matrix_x4( - const ::emel::kernel::event::op_mul_mat &request, const bool neon_available) noexcept { + const ::emel::kernel::event::op_mul_mat &request, + const bool neon_available) noexcept { const uint64_t k = request.src0.ne[0]; const uint64_t m = request.src0.ne[1]; const uint64_t rhs_rows = request.src1.ne[0]; @@ -43,8 +44,10 @@ inline bool can_use_neon_mul_mat_q8_0_packed_bl8_matrix_x4( ::emel::kernel::detail::quant::packed_q8_0_x4_group_count(rhs_rows); const size_t group_bytes = ::emel::kernel::detail::quant::packed_q8_0_x4_group_storage_bytes(k); - return neon_available && ::emel::kernel::aarch64::detail::neon_q8_0_packed_bl8_supported() && k != 0u && - m != 0u && rhs_rows == ::emel::kernel::detail::quant::Q8_0_X4_ROWS && + return neon_available && + ::emel::kernel::aarch64::detail::neon_q8_0_packed_bl8_supported() && + k != 0u && m != 0u && + rhs_rows == ::emel::kernel::detail::quant::Q8_0_X4_ROWS && request.src1.ne[1] == k && request.dst.ne[0] == rhs_rows && request.dst.ne[1] == m && request.src0.ne[2] == 1u && request.src0.ne[3] == 1u && request.src1.ne[2] == 1u && @@ -68,7 +71,8 @@ inline bool can_use_neon_mul_mat_q8_0_packed_bl8_matrix_x4( } inline bool can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x4_request( - const ::emel::kernel::event::op_mul_mat &request, const uint8_t packed_dtype) noexcept { + const ::emel::kernel::event::op_mul_mat &request, + const uint8_t packed_dtype) noexcept { const uint64_t k = request.src0.ne[0]; const uint64_t m = request.src0.ne[1]; const uint64_t rhs_rows = request.src1.ne[0]; @@ -103,8 +107,8 @@ inline bool can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x4_request( } inline bool can_run_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4_request( - const ::emel::kernel::event::op_mul_mat &request, const uint8_t packed_dtype, - const size_t group_bytes) noexcept { + const ::emel::kernel::event::op_mul_mat &request, + const uint8_t packed_dtype, const size_t group_bytes) noexcept { const uint64_t k = request.src0.ne[0]; const uint64_t m = request.src0.ne[1]; const uint64_t rhs_rows = request.src1.ne[0]; @@ -137,22 +141,28 @@ inline bool can_run_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4_request( } inline bool can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4( - const ::emel::kernel::event::op_mul_mat &request, const bool neon_available) noexcept { - return neon_available && ::emel::kernel::aarch64::detail::neon_q4_vector_packed_supported() && + const ::emel::kernel::event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && + ::emel::kernel::aarch64::detail::neon_q4_vector_packed_supported() && can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x4_request( request, ::emel::kernel::detail::dtype_q4_k_x8_bl4); } inline bool can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4( - const ::emel::kernel::event::op_mul_mat &request, const bool neon_available) noexcept { - return neon_available && ::emel::kernel::aarch64::detail::neon_q4_vector_packed_supported() && + const ::emel::kernel::event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && + ::emel::kernel::aarch64::detail::neon_q4_vector_packed_supported() && can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x4_request( request, ::emel::kernel::detail::dtype_q4_k_x8_bl8); } inline bool can_use_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4( - const ::emel::kernel::event::op_mul_mat &request, const bool neon_available) noexcept { - return neon_available && ::emel::kernel::aarch64::detail::neon_q6_vector_packed_supported() && + const ::emel::kernel::event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && + ::emel::kernel::aarch64::detail::neon_q6_vector_packed_supported() && can_run_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4_request( request, ::emel::kernel::detail::dtype_q6_k_x8, ::emel::kernel::detail::quant::packed_q6_k_x8_group_storage_bytes( @@ -160,15 +170,17 @@ inline bool can_use_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4( } inline bool can_use_neon_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4( - const ::emel::kernel::event::op_mul_mat &request, const bool neon_available) noexcept { - return neon_available && ::emel::kernel::aarch64::detail::neon_q6_vector_prepared_q8_rhs_i8mm_supported() && + const ::emel::kernel::event::op_mul_mat &request, + const bool neon_available) noexcept { + return neon_available && + ::emel::kernel::aarch64::detail:: + neon_q6_vector_prepared_q8_rhs_i8mm_supported() && can_run_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4_request( request, ::emel::kernel::detail::dtype_q6_k_x8_q8_prepared, ::emel::kernel::detail::quant:: prepared_q6_k_x8_q8_group_storage_bytes(request.src0.ne[0])); } - inline bool can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x8_request( const ::emel::kernel::event::op_mul_mat &request, const uint8_t packed_dtype) noexcept { @@ -182,34 +194,27 @@ inline bool can_run_neon_mul_mat_q4_vector_packed_q8_rhs_matrix_x8_request( const size_t rhs_row_bytes = ::emel::kernel::detail::quantized_row_storage_bytes( ::emel::kernel::detail::dtype_q8_k, k); - return k != 0u && - m != 0u && - rhs_rows == ::emel::kernel::detail::quant::Q4_K_X8_ROWS && - request.src1.ne[1] == k && - request.dst.ne[0] == rhs_rows && - request.dst.ne[1] == m && - request.src0.ne[2] == 1u && - request.src0.ne[3] == 1u && - request.src1.ne[2] == 1u && - request.src1.ne[3] == 1u && - request.dst.ne[2] == 1u && - request.dst.ne[3] == 1u && - ::emel::kernel::detail::dtype_code(request.src0.type) == packed_dtype && - ::emel::kernel::detail::dtype_code(request.src1.type) == - ::emel::kernel::detail::dtype_q8_k_x8 && - ::emel::kernel::detail::dtype_code(request.dst.type) == - ::emel::kernel::detail::dtype_f32 && - request.src0.nb[0] == 1u && - group_bytes != 0u && - request.src0.nb[1] == group_bytes && - request.src0.nb[2] == group_bytes * group_count && - request.src0.nb[3] == request.src0.nb[2] && - request.src1.nb[0] == 1u && - rhs_row_bytes != 0u && - request.src1.nb[1] == rhs_row_bytes && - request.src1.nb[2] == rhs_row_bytes * rhs_rows && - request.src1.nb[3] == request.src1.nb[2] && - is_batch_major_dst_view(request.dst, m, rhs_rows); + return k != 0u && m != 0u && + rhs_rows == ::emel::kernel::detail::quant::Q4_K_X8_ROWS && + request.src1.ne[1] == k && request.dst.ne[0] == rhs_rows && + request.dst.ne[1] == m && request.src0.ne[2] == 1u && + request.src0.ne[3] == 1u && request.src1.ne[2] == 1u && + request.src1.ne[3] == 1u && request.dst.ne[2] == 1u && + request.dst.ne[3] == 1u && + ::emel::kernel::detail::dtype_code(request.src0.type) == + packed_dtype && + ::emel::kernel::detail::dtype_code(request.src1.type) == + ::emel::kernel::detail::dtype_q8_k_x8 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + request.src0.nb[0] == 1u && group_bytes != 0u && + request.src0.nb[1] == group_bytes && + request.src0.nb[2] == group_bytes * group_count && + request.src0.nb[3] == request.src0.nb[2] && request.src1.nb[0] == 1u && + rhs_row_bytes != 0u && request.src1.nb[1] == rhs_row_bytes && + request.src1.nb[2] == rhs_row_bytes * rhs_rows && + request.src1.nb[3] == request.src1.nb[2] && + is_batch_major_dst_view(request.dst, m, rhs_rows); } inline bool @@ -225,35 +230,27 @@ can_run_neon_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8_request( const size_t rhs_row_bytes = ::emel::kernel::detail::quantized_row_storage_bytes( ::emel::kernel::detail::dtype_q8_k, k); - return k != 0u && - m != 0u && - rhs_rows == ::emel::kernel::detail::quant::Q6_K_X8_ROWS && - request.src1.ne[1] == k && - request.dst.ne[0] == rhs_rows && - request.dst.ne[1] == m && - request.src0.ne[2] == 1u && - request.src0.ne[3] == 1u && - request.src1.ne[2] == 1u && - request.src1.ne[3] == 1u && - request.dst.ne[2] == 1u && - request.dst.ne[3] == 1u && - ::emel::kernel::detail::dtype_code(request.src0.type) == - ::emel::kernel::detail::dtype_q6_k_x8_q8_prepared && - ::emel::kernel::detail::dtype_code(request.src1.type) == - ::emel::kernel::detail::dtype_q8_k_x8 && - ::emel::kernel::detail::dtype_code(request.dst.type) == - ::emel::kernel::detail::dtype_f32 && - request.src0.nb[0] == 1u && - group_bytes != 0u && - request.src0.nb[1] == group_bytes && - request.src0.nb[2] == group_bytes * group_count && - request.src0.nb[3] == request.src0.nb[2] && - request.src1.nb[0] == 1u && - rhs_row_bytes != 0u && - request.src1.nb[1] == rhs_row_bytes && - request.src1.nb[2] == rhs_row_bytes * rhs_rows && - request.src1.nb[3] == request.src1.nb[2] && - is_batch_major_dst_view(request.dst, m, rhs_rows); + return k != 0u && m != 0u && + rhs_rows == ::emel::kernel::detail::quant::Q6_K_X8_ROWS && + request.src1.ne[1] == k && request.dst.ne[0] == rhs_rows && + request.dst.ne[1] == m && request.src0.ne[2] == 1u && + request.src0.ne[3] == 1u && request.src1.ne[2] == 1u && + request.src1.ne[3] == 1u && request.dst.ne[2] == 1u && + request.dst.ne[3] == 1u && + ::emel::kernel::detail::dtype_code(request.src0.type) == + ::emel::kernel::detail::dtype_q6_k_x8_q8_prepared && + ::emel::kernel::detail::dtype_code(request.src1.type) == + ::emel::kernel::detail::dtype_q8_k_x8 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + request.src0.nb[0] == 1u && group_bytes != 0u && + request.src0.nb[1] == group_bytes && + request.src0.nb[2] == group_bytes * group_count && + request.src0.nb[3] == request.src0.nb[2] && request.src1.nb[0] == 1u && + rhs_row_bytes != 0u && request.src1.nb[1] == rhs_row_bytes && + request.src1.nb[2] == rhs_row_bytes * rhs_rows && + request.src1.nb[3] == request.src1.nb[2] && + is_batch_major_dst_view(request.dst, m, rhs_rows); } inline bool can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8( @@ -299,6 +296,25 @@ inline bool can_use_neon_conv_transpose_1d_f32( sizeof(float); } +inline bool +can_use_neon_unary_f32(const ::emel::kernel::event::op_unary &request, + const bool neon_available) noexcept { +#if !(defined(__aarch64__) || defined(__ARM_NEON)) + (void)request; + (void)neon_available; + return false; +#else + return neon_available && + ::emel::kernel::detail::can_run_backend_request(request) && + ::emel::kernel::detail::dtype_code(request.src0.type) == + ::emel::kernel::detail::dtype_f32 && + ::emel::kernel::detail::dtype_code(request.dst.type) == + ::emel::kernel::detail::dtype_f32 && + ::emel::kernel::detail::is_dense_contiguous(request.src0) && + ::emel::kernel::detail::is_dense_contiguous(request.dst); +#endif +} + } // namespace detail template struct simd_op { @@ -454,8 +470,7 @@ struct simd_op_mul_mat_q8_0_vector_q8_rhs { return false; } return ::emel::kernel::aarch64::detail:: - can_use_neon_mul_mat_q8_0_vector_q8_rhs(ev.request, - ctx.neon_available); + can_use_neon_mul_mat_q8_0_vector_q8_rhs(ev.request, ctx.neon_available); } }; @@ -477,9 +492,20 @@ struct simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4 { if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { return false; } - return detail:: - can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4( - ev.request, ctx.neon_available); + return detail::can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4( + ev.request, ctx.neon_available); + } +}; + +struct simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4 { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return ::emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_q4_vector_packed_f32_rhs_bl4(ev.request, + ctx.neon_available); } }; @@ -495,15 +521,26 @@ struct simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8 { } }; +struct simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8 { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return ::emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_q4_vector_packed_f32_rhs_bl8(ev.request, + ctx.neon_available); + } +}; + struct simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4 { bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, const action::context &ctx) const noexcept { if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { return false; } - return detail:: - can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4( - ev.request, ctx.neon_available); + return detail::can_use_neon_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4( + ev.request, ctx.neon_available); } }; @@ -525,8 +562,18 @@ struct simd_op_mul_mat_q4_vector_q8_rhs { return false; } return ::emel::kernel::aarch64::detail:: - can_use_neon_mul_mat_q4_vector_q8_rhs(ev.request, - ctx.neon_available); + can_use_neon_mul_mat_q4_vector_q8_rhs(ev.request, ctx.neon_available); + } +}; + +struct simd_op_mul_mat_q4_vector_f32_rhs { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_mul_mat &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return ::emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_q4_vector_f32_rhs(ev.request, ctx.neon_available); } }; @@ -548,8 +595,7 @@ struct simd_op_mul_mat_q6_vector_q8_rhs { return false; } return ::emel::kernel::aarch64::detail:: - can_use_neon_mul_mat_q6_vector_q8_rhs(ev.request, - ctx.neon_available); + can_use_neon_mul_mat_q6_vector_q8_rhs(ev.request, ctx.neon_available); } }; @@ -571,9 +617,8 @@ struct simd_op_mul_mat_q6_vector_packed_q8_rhs_matrix_x4 { if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { return false; } - return detail:: - can_use_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4( - ev.request, ctx.neon_available); + return detail::can_use_neon_mul_mat_q6_vector_packed_q8_rhs_matrix_x4( + ev.request, ctx.neon_available); } }; @@ -638,6 +683,32 @@ struct simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs { } }; +struct simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4 { + bool operator()( + const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return ::emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4( + ev.request, ctx.neon_available); + } +}; + +struct simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8 { + bool operator()( + const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return ::emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8( + ev.request, ctx.neon_available); + } +}; + struct simd_op_mul_mat_argmax_q6_vector_prepared_q8_rhs_i8mm { bool operator()( const ::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax &ev, @@ -682,9 +753,12 @@ struct simd_op_mul_mat_generic { !simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x8{}(ev, ctx) && !simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8_matrix_x4{}(ev, ctx) && !simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8{}(ev, ctx) && + !simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8{}(ev, ctx) && !simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4{}(ev, ctx) && !simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4{}(ev, ctx) && + !simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4{}(ev, ctx) && !simd_op_mul_mat_q4_vector_q8_rhs{}(ev, ctx) && + !simd_op_mul_mat_q4_vector_f32_rhs{}(ev, ctx) && !simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8{}(ev, ctx) && !simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x4{}(ev, @@ -749,6 +823,16 @@ template struct invalid_op { } }; +struct simd_op_unary_f32 { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_unary &ev, + const action::context &ctx) const noexcept { + if (!::emel::kernel::detail::validate_dispatch_request(ev.request)) { + return false; + } + return detail::can_use_neon_unary_f32(ev.request, ctx.neon_available); + } +}; + template <::emel::kernel::event::unary_subop subop> struct unary_subop_is { bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_unary &ev, const action::context &) const noexcept { @@ -759,8 +843,7 @@ template <::emel::kernel::event::unary_subop subop> struct unary_subop_is { template <::emel::kernel::event::unary_subop subop> using simd_op_unary_subop = ::emel::kernel::detail::simd_unary_subop_guard< ::emel::kernel::aarch64::event::dispatch_op_unary, action::context, - simd_op<::emel::kernel::aarch64::event::dispatch_op_unary>, - unary_subop_is>; + simd_op_unary_f32, unary_subop_is>; template <::emel::kernel::event::unary_subop subop> using valid_op_unary_subop = ::emel::kernel::detail::valid_unary_subop_guard< @@ -768,18 +851,29 @@ using valid_op_unary_subop = ::emel::kernel::detail::valid_unary_subop_guard< valid_op<::emel::kernel::aarch64::event::dispatch_op_unary>, unary_subop_is>; +template <::emel::kernel::event::unary_subop subop> +struct valid_scalar_op_unary_subop { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_unary &ev, + const action::context &ctx) const noexcept { + return valid_op_unary_subop{}(ev, ctx) && + !simd_op_unary_f32{}(ev, ctx); + } +}; + using simd_op_unary_abs = simd_op_unary_subop<::emel::kernel::event::unary_subop::abs>; using simd_op_unary_neg = simd_op_unary_subop<::emel::kernel::event::unary_subop::neg>; using simd_op_unary_relu = simd_op_unary_subop<::emel::kernel::event::unary_subop::relu>; +using simd_op_unary_silu = + simd_op_unary_subop<::emel::kernel::event::unary_subop::silu>; using valid_op_unary_abs = - valid_op_unary_subop<::emel::kernel::event::unary_subop::abs>; + valid_scalar_op_unary_subop<::emel::kernel::event::unary_subop::abs>; using valid_op_unary_neg = - valid_op_unary_subop<::emel::kernel::event::unary_subop::neg>; + valid_scalar_op_unary_subop<::emel::kernel::event::unary_subop::neg>; using valid_op_unary_relu = - valid_op_unary_subop<::emel::kernel::event::unary_subop::relu>; + valid_scalar_op_unary_subop<::emel::kernel::event::unary_subop::relu>; using valid_op_unary_exp = valid_op_unary_subop<::emel::kernel::event::unary_subop::exp>; using valid_op_unary_tanh = @@ -789,7 +883,7 @@ using valid_op_unary_elu = using valid_op_unary_gelu = valid_op_unary_subop<::emel::kernel::event::unary_subop::gelu>; using valid_op_unary_silu = - valid_op_unary_subop<::emel::kernel::event::unary_subop::silu>; + valid_scalar_op_unary_subop<::emel::kernel::event::unary_subop::silu>; // Variant predicates for the ops whose dtype/mode choice is modeled as // explicit transition rows (op_unary pattern). @@ -802,15 +896,39 @@ template struct get_rows_src_dtype_is { } }; -template struct rope_mode_is { +template struct rope_mode_is { bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_rope &ev, const action::context &) const noexcept { int32_t mode = 0; return ::emel::kernel::detail::read_op_param_i32( ev.request.op_params.data(), ev.request.op_params_size, 2u, mode) && - mode == (neox_mode ? ::emel::kernel::detail::rope_mode_neox - : ::emel::kernel::detail::rope_mode_norm); + mode == mode_value; + } +}; + +struct rope_timestep_layout_is { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_rope &ev, + const action::context &) const noexcept { + int32_t n_dims = 0; + return ::emel::kernel::detail::read_op_param_i32( + ev.request.op_params.data(), ev.request.op_params_size, 1u, + n_dims) && + n_dims > 0 && + static_cast(n_dims) == ev.request.src0.ne[0] && + ev.request.src0.data != ev.request.dst.data && + ::emel::kernel::detail::is_dense_contiguous(ev.request.src0) && + ::emel::kernel::detail::is_dense_contiguous(ev.request.src1) && + ::emel::kernel::detail::is_dense_contiguous(ev.request.dst); + } +}; + +struct rope_timestep_is { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_rope &ev, + const action::context &ctx) const noexcept { + return rope_mode_is<::emel::kernel::detail::rope_mode_timestep>{}(ev, + ctx) && + rope_timestep_layout_is{}(ev, ctx); } }; @@ -878,11 +996,23 @@ struct valid_op_mul_mat_f16 { using valid_op_rope_norm = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::aarch64::event::dispatch_op_rope, action::context, valid_op<::emel::kernel::aarch64::event::dispatch_op_rope>, - rope_mode_is>; + rope_mode_is<::emel::kernel::detail::rope_mode_norm>>; using valid_op_rope_neox = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::aarch64::event::dispatch_op_rope, action::context, valid_op<::emel::kernel::aarch64::event::dispatch_op_rope>, - rope_mode_is>; + rope_mode_is<::emel::kernel::detail::rope_mode_neox>>; +using valid_op_rope_timestep = ::emel::kernel::detail::valid_variant_guard< + ::emel::kernel::aarch64::event::dispatch_op_rope, action::context, + valid_op<::emel::kernel::aarch64::event::dispatch_op_rope>, + rope_timestep_is>; + +template <> struct invalid_op<::emel::kernel::aarch64::event::dispatch_op_rope> { + bool operator()(const ::emel::kernel::aarch64::event::dispatch_op_rope &ev, + const action::context &ctx) const noexcept { + return !valid_op_rope_norm{}(ev, ctx) && !valid_op_rope_neox{}(ev, ctx) && + !valid_op_rope_timestep{}(ev, ctx); + } +}; using valid_op_im2col_f32 = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::aarch64::event::dispatch_op_im2col, action::context, diff --git a/src/emel/kernel/aarch64/sm.hpp b/src/emel/kernel/aarch64/sm.hpp index b50fead4..865a9cc6 100644 --- a/src/emel/kernel/aarch64/sm.hpp +++ b/src/emel/kernel/aarch64/sm.hpp @@ -412,6 +412,11 @@ struct model { [ guard::simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8{} ] / action::exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl8 + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> + [ guard::simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8{} ] + / action::exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl8 + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> [ guard::simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4_matrix_x4{} ] @@ -422,11 +427,21 @@ struct model { [ guard::simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4{} ] / action::exec_simd_op_mul_mat_q4_vector_packed_q8_rhs_bl4 + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> + [ guard::simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4{} ] + / action::exec_simd_op_mul_mat_q4_vector_packed_f32_rhs_bl4 + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> [ guard::simd_op_mul_mat_q4_vector_q8_rhs{} ] / action::exec_simd_op_mul_mat_q4_vector_q8_rhs + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> + [ guard::simd_op_mul_mat_q4_vector_f32_rhs{} ] + / action::exec_simd_op_mul_mat_q4_vector_f32_rhs + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat> [ guard::simd_op_mul_mat_q6_vector_prepared_q8_rhs_i8mm_matrix_x8{} ] @@ -447,6 +462,16 @@ struct model { [ guard::simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm{} ] / action::exec_simd_op_mul_mat_argmax_q6_vector_q8_argmax_prepared_i8mm + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax> + [ guard::simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8{} ] + / action::exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl8 + + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax> + [ guard::simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4{} ] + / action::exec_simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4 + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax> [ guard::simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs{} ] @@ -742,6 +767,11 @@ struct model { [ guard::valid_op_rope_neox{} ] / action::exec_scalar_op_rope_neox + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_rope> + [ guard::valid_op_rope_timestep{} ] + / action::exec_scalar_op_rope_timestep + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_rope> [ guard::invalid_op_rope{} ] @@ -1142,6 +1172,11 @@ struct model { [ guard::simd_op_unary_relu{} ] / action::exec_simd_op_unary_relu + , sml::state <= sml::state + + sml::event<::emel::kernel::aarch64::event::dispatch_op_unary> + [ guard::simd_op_unary_silu{} ] + / action::exec_simd_op_unary_silu_t{} + , sml::state <= sml::state + sml::event<::emel::kernel::aarch64::event::dispatch_op_unary> [ guard::valid_op_unary_abs{} ] diff --git a/src/emel/kernel/any.hpp b/src/emel/kernel/any.hpp index a62ccbb1..a85d0942 100644 --- a/src/emel/kernel/any.hpp +++ b/src/emel/kernel/any.hpp @@ -27,7 +27,7 @@ constexpr kernel_kind detect_host_kind() noexcept { class any { public: - any() = default; + any() : core_(detect_host_kind()) {} explicit any(const kernel_kind kind) : core_(kind) {} any(const any &) = delete; diff --git a/src/emel/kernel/detail.hpp b/src/emel/kernel/detail.hpp index c7f48aba..c8a82643 100644 --- a/src/emel/kernel/detail.hpp +++ b/src/emel/kernel/detail.hpp @@ -272,7 +272,7 @@ struct block_q6_k { struct block_q4_kx8 { std::array d = {}; std::array dmin = {}; - std::array scales = {}; + std::array scales = {}; std::array qs = {}; }; @@ -319,7 +319,7 @@ static_assert(sizeof(block_q5_k) == static_assert(sizeof(block_q6_k) == sizeof(uint16_t) + (QK_K / 16) + (3 * QK_K / 4)); static_assert(sizeof(block_q4_kx8) == sizeof(uint16_t) * (Q4_K_X8_ROWS * 2u) + - ((QK_K / 32u) * Q4_K_X8_ROWS * 2u) + + K_SCALE_SIZE * Q4_K_X8_ROWS + (QK_K / 2) * Q4_K_X8_ROWS); static_assert(sizeof(block_q6_kx8) == sizeof(uint16_t) * Q6_K_X8_ROWS + (QK_K / 16) * Q6_K_X8_ROWS + @@ -949,19 +949,75 @@ make_block_q4_k_x8(const block_q4_k *rows, static_cast(interleave_block_bytes)); } - for (uint64_t sb = 0; sb < (QK_K / 64u); ++sb) { - for (uint64_t half = 0; half < 2u; ++half) { - const uint64_t scale_index = sb * 2u + half; - uint8_t *packed = out.scales.data() + (scale_index * Q4_K_X8_ROWS * 2u); - for (uint64_t row = 0; row < Q4_K_X8_ROWS; ++row) { - uint8_t scale = 0u; - uint8_t min = 0u; - get_scale_min_k4(static_cast(scale_index), rows[row].scales.data(), - &scale, &min); - packed[row] = min; - packed[row + Q4_K_X8_ROWS] = scale; - } - } + std::array scales = {}; + std::array mins = {}; + for (uint64_t i = 0; i < 4u; ++i) { + for (uint64_t row = 0; row < Q4_K_X8_ROWS; ++row) { + scales[row] = rows[row].scales[i] & 63u; + mins[row] = rows[row].scales[i + 4u] & 63u; + } + + out.scales[i * 12u + 0u] = + static_cast((scales[0] & 63u) + ((scales[4] & 48u) << 2u)); + out.scales[i * 12u + 1u] = + static_cast((scales[1] & 63u) + ((scales[5] & 48u) << 2u)); + out.scales[i * 12u + 2u] = + static_cast((scales[2] & 63u) + ((scales[6] & 48u) << 2u)); + out.scales[i * 12u + 3u] = + static_cast((scales[3] & 63u) + ((scales[7] & 48u) << 2u)); + out.scales[i * 12u + 4u] = + static_cast((mins[0] & 63u) + ((mins[4] & 48u) << 2u)); + out.scales[i * 12u + 5u] = + static_cast((mins[1] & 63u) + ((mins[5] & 48u) << 2u)); + out.scales[i * 12u + 6u] = + static_cast((mins[2] & 63u) + ((mins[6] & 48u) << 2u)); + out.scales[i * 12u + 7u] = + static_cast((mins[3] & 63u) + ((mins[7] & 48u) << 2u)); + out.scales[i * 12u + 8u] = + static_cast((scales[4] & 15u) + ((mins[4] & 15u) << 4u)); + out.scales[i * 12u + 9u] = + static_cast((scales[5] & 15u) + ((mins[5] & 15u) << 4u)); + out.scales[i * 12u + 10u] = + static_cast((scales[6] & 15u) + ((mins[6] & 15u) << 4u)); + out.scales[i * 12u + 11u] = + static_cast((scales[7] & 15u) + ((mins[7] & 15u) << 4u)); + } + + for (uint64_t i = 0; i < 4u; ++i) { + for (uint64_t row = 0; row < Q4_K_X8_ROWS; ++row) { + scales[row] = static_cast( + ((rows[row].scales[i] & 192u) >> 2u) | + (rows[row].scales[i + 8u] & 15u)); + mins[row] = static_cast( + ((rows[row].scales[i + 4u] & 192u) >> 2u) | + ((rows[row].scales[i + 8u] & 240u) >> 4u)); + } + + const uint64_t base = i * 12u + 48u; + out.scales[base + 0u] = + static_cast((scales[0] & 63u) + ((scales[4] & 48u) << 2u)); + out.scales[base + 1u] = + static_cast((scales[1] & 63u) + ((scales[5] & 48u) << 2u)); + out.scales[base + 2u] = + static_cast((scales[2] & 63u) + ((scales[6] & 48u) << 2u)); + out.scales[base + 3u] = + static_cast((scales[3] & 63u) + ((scales[7] & 48u) << 2u)); + out.scales[base + 4u] = + static_cast((mins[0] & 63u) + ((mins[4] & 48u) << 2u)); + out.scales[base + 5u] = + static_cast((mins[1] & 63u) + ((mins[5] & 48u) << 2u)); + out.scales[base + 6u] = + static_cast((mins[2] & 63u) + ((mins[6] & 48u) << 2u)); + out.scales[base + 7u] = + static_cast((mins[3] & 63u) + ((mins[7] & 48u) << 2u)); + out.scales[base + 8u] = + static_cast((scales[4] & 15u) + ((mins[4] & 15u) << 4u)); + out.scales[base + 9u] = + static_cast((scales[5] & 15u) + ((mins[5] & 15u) << 4u)); + out.scales[base + 10u] = + static_cast((scales[6] & 15u) + ((mins[6] & 15u) << 4u)); + out.scales[base + 11u] = + static_cast((scales[7] & 15u) + ((mins[7] & 15u) << 4u)); } return out; @@ -2780,21 +2836,109 @@ inline float dot_q4_k_q8_k_block_scalar(const quant::block_q4_k &lhs, const float d = quant::fp16_to_fp32(lhs.d) * rhs.d; const float dmin = quant::fp16_to_fp32(lhs.dmin) * rhs.d; - float sum = -dmin * static_cast(min_sum); - for (int32_t lane : aux32) { - sum += d * static_cast(lane); + float sumf = -dmin * static_cast(min_sum); + for (const int32_t lane : aux32) { + sumf += d * static_cast(lane); } - return sum; + return sumf; } inline float dot_q4_k_q8_k_row_scalar(const quant::block_q4_k *lhs, const quant::block_q8_k *rhs, const uint64_t block_count) noexcept { - float sum = 0.0f; + constexpr uint32_t kmask1 = 0x3f3f3f3fu; + constexpr uint32_t kmask2 = 0x0f0f0f0fu; + constexpr uint32_t kmask3 = 0x03030303u; + + alignas(64) int8_t aux8[quant::QK_K] = {}; + int16_t aux16[8] = {}; + int32_t aux32[8] = {}; + uint32_t auxs[4] = {}; + float sums[8] = {}; + + const uint8_t *scales = reinterpret_cast(auxs); + const uint8_t *mins = reinterpret_cast(auxs + 2); + + float sumf = 0.0f; for (uint64_t block = 0; block < block_count; ++block) { - sum += dot_q4_k_q8_k_block_scalar(lhs[block], rhs[block]); + const uint8_t *q4 = lhs[block].qs.data(); + const int8_t *q8 = rhs[block].qs.data(); + int8_t *a = aux8; + for (uint64_t group = 0; group < (quant::QK_K / 64u); ++group) { + for (uint64_t lane = 0; lane < 32u; ++lane) { + a[lane] = static_cast(q4[lane] & 0x0fu); + } + a += 32; + for (uint64_t lane = 0; lane < 32u; ++lane) { + a[lane] = static_cast(q4[lane] >> 4u); + } + a += 32; + q4 += 32; + } + + std::memcpy(auxs, lhs[block].scales.data(), lhs[block].scales.size()); + auxs[3] = ((auxs[2] >> 4u) & kmask2) | (((auxs[1] >> 6u) & kmask3) << 4u); + const uint32_t aux = auxs[1] & kmask1; + auxs[1] = (auxs[2] & kmask2) | (((auxs[0] >> 6u) & kmask3) << 4u); + auxs[2] = aux; + auxs[0] &= kmask1; + + int32_t min_sum = 0; + for (uint64_t group = 0; group < (quant::QK_K / 16u); ++group) { + min_sum += static_cast(rhs[block].bsums[group]) * + static_cast(mins[group / 2u]); + } + + std::memset(aux32, 0, sizeof(aux32)); + a = aux8; + int scale_index = 0; + for (uint64_t group = 0; group < (quant::QK_K / 32u); ++group) { + const int32_t scale = scales[scale_index++]; + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux16[lane] = static_cast(q8[lane] * a[lane]); + } + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux32[lane] += scale * static_cast(aux16[lane]); + } + q8 += 8; + a += 8; + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux16[lane] = static_cast(q8[lane] * a[lane]); + } + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux32[lane] += scale * static_cast(aux16[lane]); + } + q8 += 8; + a += 8; + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux16[lane] = static_cast(q8[lane] * a[lane]); + } + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux32[lane] += scale * static_cast(aux16[lane]); + } + q8 += 8; + a += 8; + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux16[lane] = static_cast(q8[lane] * a[lane]); + } + for (uint64_t lane = 0; lane < 8u; ++lane) { + aux32[lane] += scale * static_cast(aux16[lane]); + } + q8 += 8; + a += 8; + } + + const float d = quant::fp16_to_fp32(lhs[block].d) * rhs[block].d; + for (int lane = 0; lane < 8; ++lane) { + sums[lane] += d * static_cast(aux32[lane]); + } + const float dmin = quant::fp16_to_fp32(lhs[block].dmin) * rhs[block].d; + sumf -= dmin * static_cast(min_sum); } - return sum; + for (const float lane : sums) { + sumf += lane; + } + return sumf; } inline float dot_q5_k_q8_k_block_scalar(const quant::block_q5_k &lhs, @@ -3933,10 +4077,48 @@ inline __m256 v_expf_ggml(__m256 x) noexcept { } #endif +#if defined(__aarch64__) && defined(__ARM_NEON) +// Exact port of ggml_v_expf's AArch64 NEON polynomial. +inline float32x4_t v_expf_ggml(float32x4_t x) noexcept { + const float32x4_t r = vdupq_n_f32(0x1.8p23f); + const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f)); + const float32x4_t n = vsubq_f32(z, r); + const float32x4_t b = + vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n, + vdupq_n_f32(0x1.7f7d1cp-20f)); + const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23); + const float32x4_t k = vreinterpretq_f32_u32( + vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1.0f)))); + const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126.0f)); + const float32x4_t u = vmulq_f32(b, b); + const float32x4_t j = vfmaq_f32( + vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b), + vfmaq_f32( + vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), + vdupq_n_f32(0x1.555e66p-3f), b), + vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), + vdupq_n_f32(0x1.0e4020p-7f), b), + u), + u); + if (vpaddd_u64(vreinterpretq_u64_u32(c)) == 0u) { + return vfmaq_f32(k, k, j); + } + const uint32x4_t d = + vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000u)); + const float32x4_t s1 = + vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000u))); + const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d)); + return vbslq_f32( + vcagtq_f32(n, vdupq_n_f32(192.0f)), vmulq_f32(s1, s1), + vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), + vfmaq_f32(k, k, j))); +} +#endif + // Exact port of ggml's soft_max row kernel over a pre-scaled, pre-masked -// row: max (order independent), 8-lane v_expf blocks with per-block -// horizontal reduce into a double sum, libm expf scalar tail, then a -// per-element multiply by (float)(1.0 / sum). +// row: max (order independent), architecture-native v_expf blocks with +// per-block horizontal reduction into a double sum, libm expf scalar tail, +// then a per-element multiply by (float)(1.0 / sum). inline void soft_max_row_ggml(const int64_t count, float *data) noexcept { float max_value = -std::numeric_limits::infinity(); for (int64_t i = 0; i < count; ++i) { @@ -3955,6 +4137,13 @@ inline void soft_max_row_ggml(const int64_t count, float *data) noexcept { val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2)); sum += static_cast(_mm_cvtss_f32(val2)); } +#elif defined(__aarch64__) && defined(__ARM_NEON) + for (; i + 3 < count; i += 4) { + const float32x4_t val = v_expf_ggml( + vsubq_f32(vld1q_f32(data + i), vdupq_n_f32(max_value))); + vst1q_f32(data + i, val); + sum += static_cast(vaddvq_f32(val)); + } #endif for (; i < count; ++i) { const float val = std::exp(data[i] - max_value); @@ -4196,8 +4385,8 @@ inline bool can_run_flash_attn_ext(const request_type &request) noexcept { return explicit_operand_contract && dims_present && src2_valid && shape_match && layout_supported && storage_bound && - masked_total_tokens >= kv_tokens && - std::isfinite(scale) && scale > 0.0f; + masked_total_tokens >= kv_tokens && std::isfinite(scale) && + scale > 0.0f; } //------------------------------------------------------------------------------// @@ -4396,12 +4585,10 @@ inline bool run_rms_norm(const request_type &request) noexcept { for (uint64_t i0 = 0; i0 < cols; ++i0) { float value = 0.0f; std::memcpy(&value, src_row + i0 * src_nb0, sizeof(value)); - sum += static_cast(value) * static_cast(value); + sum += static_cast(value * value); } - const float scale = - 1.0f / - std::sqrt(static_cast(sum / static_cast(cols)) + - eps); + const float mean = static_cast(sum / static_cast(cols)); + const float scale = 1.0f / std::sqrt(mean + eps); for (uint64_t i0 = 0; i0 < cols; ++i0) { float value = 0.0f; std::memcpy(&value, src_row + i0 * src_nb0, sizeof(value)); @@ -4474,6 +4661,7 @@ struct rope_op_params { inline constexpr int32_t rope_mode_norm = 0; inline constexpr int32_t rope_mode_neox = 2; +inline constexpr int32_t rope_mode_timestep = 4; template inline bool read_rope_params(const request_type &request, @@ -4491,13 +4679,14 @@ inline bool read_rope_params(const request_type &request, template inline bool can_run_rope(const request_type &request) noexcept { rope_op_params params = {}; + const bool params_ok = read_rope_params(request, params); const bool same_shape = request.src0.ne[0] == request.dst.ne[0] && request.src0.ne[1] == request.dst.ne[1] && request.src0.ne[2] == request.dst.ne[2] && request.src0.ne[3] == request.dst.ne[3]; - return read_rope_params(request, params) && same_shape && - request.src0.data != nullptr && request.src1.data != nullptr && - request.src0.ne[0] > 0 && dtype_code(request.src0.type) == dtype_f32 && + return params_ok && same_shape && request.src0.data != nullptr && + request.src1.data != nullptr && request.src0.ne[0] > 0 && + dtype_code(request.src0.type) == dtype_f32 && dtype_code(request.dst.type) == dtype_f32 && dtype_code(request.src1.type) == dtype_i32 && request.src2.data == nullptr && @@ -4507,7 +4696,6 @@ inline bool can_run_rope(const request_type &request) noexcept { request.src1.ne[0] == request.src0.ne[2] && params.n_dims > 0 && (params.n_dims % 2) == 0 && static_cast(params.n_dims) <= request.src0.ne[0] && - (params.mode == rope_mode_norm || params.mode == rope_mode_neox) && params.ext_factor == 0.0f && std::isfinite(params.freq_base) && params.freq_base > 0.0f && std::isfinite(params.freq_scale) && params.freq_scale > 0.0f && std::isfinite(params.attn_factor); @@ -4572,6 +4760,61 @@ inline bool run_rope_as(const request_type &request) noexcept { return true; } +// Moshi's timestep embedding rotates adjacent projected pairs and writes the +// real and imaginary results into split halves. The reference graph computes +// each multiplication as its own f32 node, so volatile product temporaries are +// intentional round barriers that prevent contraction into FMA instructions. +template +inline void compute_timestep_rope(const request_type &request) noexcept { + float freq_base = 0.0f; + float freq_scale = 0.0f; + float attn_factor = 0.0f; + std::memcpy(&freq_base, + request.op_params.data() + 5u * sizeof(freq_base), + sizeof(freq_base)); + std::memcpy(&freq_scale, + request.op_params.data() + 6u * sizeof(freq_scale), + sizeof(freq_scale)); + std::memcpy(&attn_factor, + request.op_params.data() + 8u * sizeof(attn_factor), + sizeof(attn_factor)); + + const uint64_t cols = request.src0.ne[0]; + const uint64_t half_dims = cols / 2u; + const float neg_log_period = -std::log(freq_base); + const auto *src_base = static_cast(request.src0.data); + const auto *positions = static_cast(request.src1.data); + auto *dst_base = static_cast(request.dst.data); + + for (uint64_t i3 = 0; i3 < request.src0.ne[3]; ++i3) { + for (uint64_t i2 = 0; i2 < request.src0.ne[2]; ++i2) { + const float position = static_cast(positions[i2]); + for (uint64_t i1 = 0; i1 < request.src0.ne[1]; ++i1) { + const uint64_t row = + (i3 * request.src0.ne[2] + i2) * request.src0.ne[1] + i1; + const float *src_row = src_base + row * cols; + float *dst_row = dst_base + row * cols; + for (uint64_t pair = 0; pair < half_dims; ++pair) { + const float real = src_row[2u * pair]; + const float imaginary = src_row[2u * pair + 1u]; + const float frequency = + std::exp(neg_log_period * static_cast(pair) / + static_cast(half_dims)); + const float argument = position * frequency * freq_scale; + const float cosine = std::cos(argument) * attn_factor; + const float sine = std::sin(argument) * attn_factor; + volatile float real_cosine = real * cosine; + volatile float imaginary_sine = imaginary * sine; + volatile float real_sine = real * sine; + volatile float imaginary_cosine = imaginary * cosine; + dst_row[pair] = real_cosine - imaginary_sine; + dst_row[half_dims + pair] = real_sine + imaginary_cosine; + } + } + } + } +} + // ggml im2col op_params layout: i32 slots {s0, s1, p0, p1, d0, d1, is_2D}. struct im2col_op_params { int32_t s0 = 0; @@ -4638,8 +4881,8 @@ inline bool can_run_im2col(const request_type &request) noexcept { constexpr int64_t k_guard_cap = static_cast(k_max_conv_guard_extent); // Each bound keeps the next product under 2^62, so the chain evaluates // without signed overflow: factors first, then rows, then the batch total. - if (request.src1.ne[2] > k_max_conv_guard_extent || out_length > k_guard_cap || - row_width > k_guard_cap) { + if (request.src1.ne[2] > k_max_conv_guard_extent || + out_length > k_guard_cap || row_width > k_guard_cap) { return false; } if (out_length > 0 && batches > 0 && @@ -4649,8 +4892,7 @@ inline bool can_run_im2col(const request_type &request) noexcept { } const uint8_t src0_type = dtype_code(request.src0.type); return request.src1.data != nullptr && request.dst.data != nullptr && - kernel > 0 && channels > 0 && - length > 0 && out_length > 0 && + kernel > 0 && channels > 0 && length > 0 && out_length > 0 && (src0_type == dtype_f32 || src0_type == dtype_f16) && dtype_code(request.src1.type) == dtype_f32 && (dst_type == dtype_f32 || dst_type == dtype_f16) && @@ -4751,8 +4993,8 @@ inline bool can_run_conv_transpose_1d(const request_type &request) noexcept { // destination; metadata-only tensors must reject here instead of // dereferencing null inside the action. return request.src0.data != nullptr && request.src1.data != nullptr && - request.dst.data != nullptr && - kernel > 0 && out_channels > 0 && in_channels > 0 && length > 0 && + request.dst.data != nullptr && kernel > 0 && out_channels > 0 && + in_channels > 0 && length > 0 && (src0_type == dtype_f32 || src0_type == dtype_f16) && dtype_code(request.src1.type) == dtype_f32 && dtype_code(request.dst.type) == dtype_f32 && request.src0.ne[3] == 1 && @@ -4863,6 +5105,15 @@ struct exec_scalar_rope_op { } }; +template +struct exec_scalar_timestep_rope_op { + void operator()(const dispatch_event_type &ev, + context_type &ctx) const noexcept { + compute_timestep_rope(ev.request); + mark_done_type{}(ev, ctx); + } +}; + template struct exec_scalar_im2col_op { diff --git a/src/emel/kernel/x86_64/actions.hpp b/src/emel/kernel/x86_64/actions.hpp index 61f59609..d45558f5 100644 --- a/src/emel/kernel/x86_64/actions.hpp +++ b/src/emel/kernel/x86_64/actions.hpp @@ -2627,6 +2627,10 @@ using exec_scalar_op_rope_norm_t = ::emel::kernel::detail::exec_scalar_rope_op< using exec_scalar_op_rope_neox_t = ::emel::kernel::detail::exec_scalar_rope_op< ::emel::kernel::x86_64::event::dispatch_op_rope, context, detail::mark_done_op, true>; +using exec_scalar_op_rope_timestep_t = + ::emel::kernel::detail::exec_scalar_timestep_rope_op< + ::emel::kernel::x86_64::event::dispatch_op_rope, context, + detail::mark_done_op>; using exec_scalar_op_im2col_f32_t = ::emel::kernel::detail::exec_scalar_im2col_op< ::emel::kernel::x86_64::event::dispatch_op_im2col, context, @@ -2723,6 +2727,7 @@ inline constexpr exec_scalar_op_get_rows_q8_0_t exec_scalar_op_get_rows_q8_0{}; inline constexpr exec_scalar_op_get_rows_q4_k_t exec_scalar_op_get_rows_q4_k{}; inline constexpr exec_scalar_op_rope_norm_t exec_scalar_op_rope_norm{}; inline constexpr exec_scalar_op_rope_neox_t exec_scalar_op_rope_neox{}; +inline constexpr exec_scalar_op_rope_timestep_t exec_scalar_op_rope_timestep{}; inline constexpr exec_scalar_op_im2col_f32_t exec_scalar_op_im2col_f32{}; inline constexpr exec_scalar_op_im2col_f16_t exec_scalar_op_im2col_f16{}; inline constexpr exec_scalar_op_conv_transpose_1d_f32_t diff --git a/src/emel/kernel/x86_64/guards.hpp b/src/emel/kernel/x86_64/guards.hpp index 2dbe3abc..cb3dac82 100644 --- a/src/emel/kernel/x86_64/guards.hpp +++ b/src/emel/kernel/x86_64/guards.hpp @@ -273,15 +273,39 @@ template struct get_rows_src_dtype_is { } }; -template struct rope_mode_is { +template struct rope_mode_is { bool operator()(const ::emel::kernel::x86_64::event::dispatch_op_rope &ev, const action::context &) const noexcept { int32_t mode = 0; return ::emel::kernel::detail::read_op_param_i32( ev.request.op_params.data(), ev.request.op_params_size, 2u, mode) && - mode == (neox_mode ? ::emel::kernel::detail::rope_mode_neox - : ::emel::kernel::detail::rope_mode_norm); + mode == mode_value; + } +}; + +struct rope_timestep_layout_is { + bool operator()(const ::emel::kernel::x86_64::event::dispatch_op_rope &ev, + const action::context &) const noexcept { + int32_t n_dims = 0; + return ::emel::kernel::detail::read_op_param_i32( + ev.request.op_params.data(), ev.request.op_params_size, 1u, + n_dims) && + n_dims > 0 && + static_cast(n_dims) == ev.request.src0.ne[0] && + ev.request.src0.data != ev.request.dst.data && + ::emel::kernel::detail::is_dense_contiguous(ev.request.src0) && + ::emel::kernel::detail::is_dense_contiguous(ev.request.src1) && + ::emel::kernel::detail::is_dense_contiguous(ev.request.dst); + } +}; + +struct rope_timestep_is { + bool operator()(const ::emel::kernel::x86_64::event::dispatch_op_rope &ev, + const action::context &ctx) const noexcept { + return rope_mode_is<::emel::kernel::detail::rope_mode_timestep>{}(ev, + ctx) && + rope_timestep_layout_is{}(ev, ctx); } }; @@ -338,11 +362,23 @@ using valid_op_mul_mat_f16 = ::emel::kernel::detail::valid_variant_guard< using valid_op_rope_norm = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::x86_64::event::dispatch_op_rope, action::context, valid_op<::emel::kernel::x86_64::event::dispatch_op_rope>, - rope_mode_is>; + rope_mode_is<::emel::kernel::detail::rope_mode_norm>>; using valid_op_rope_neox = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::x86_64::event::dispatch_op_rope, action::context, valid_op<::emel::kernel::x86_64::event::dispatch_op_rope>, - rope_mode_is>; + rope_mode_is<::emel::kernel::detail::rope_mode_neox>>; +using valid_op_rope_timestep = ::emel::kernel::detail::valid_variant_guard< + ::emel::kernel::x86_64::event::dispatch_op_rope, action::context, + valid_op<::emel::kernel::x86_64::event::dispatch_op_rope>, + rope_timestep_is>; + +template <> struct invalid_op<::emel::kernel::x86_64::event::dispatch_op_rope> { + bool operator()(const ::emel::kernel::x86_64::event::dispatch_op_rope &ev, + const action::context &ctx) const noexcept { + return !valid_op_rope_norm{}(ev, ctx) && !valid_op_rope_neox{}(ev, ctx) && + !valid_op_rope_timestep{}(ev, ctx); + } +}; using valid_op_im2col_f32 = ::emel::kernel::detail::valid_variant_guard< ::emel::kernel::x86_64::event::dispatch_op_im2col, action::context, diff --git a/src/emel/kernel/x86_64/sm.hpp b/src/emel/kernel/x86_64/sm.hpp index d4704b6b..0b4d635e 100644 --- a/src/emel/kernel/x86_64/sm.hpp +++ b/src/emel/kernel/x86_64/sm.hpp @@ -647,6 +647,11 @@ struct model { [ guard::valid_op_rope_neox{} ] / action::exec_scalar_op_rope_neox + , sml::state <= sml::state + + sml::event<::emel::kernel::x86_64::event::dispatch_op_rope> + [ guard::valid_op_rope_timestep{} ] + / action::exec_scalar_op_rope_timestep + , sml::state <= sml::state + sml::event<::emel::kernel::x86_64::event::dispatch_op_rope> [ guard::invalid_op_rope{} ] diff --git a/src/emel/logits/sampler/actions.hpp b/src/emel/logits/sampler/actions.hpp index 401fb936..a4c5430c 100644 --- a/src/emel/logits/sampler/actions.hpp +++ b/src/emel/logits/sampler/actions.hpp @@ -1,5 +1,11 @@ #pragma once +#include +#include +#include + +#include "emel/kernel/detail.hpp" + #include "emel/logits/sampler/context.hpp" #include "emel/logits/sampler/events.hpp" @@ -85,6 +91,74 @@ struct advance_sampler_index { } }; +struct scale_temperature_logits { + void operator()(const event::sample_temperature_top_k_runtime &ev, + context &) const noexcept { + const float inverse_temperature = 1.0f / ev.request.temperature; + for (int32_t index = 0; index < ev.request.card; ++index) { + ev.request.logits[static_cast(index)] *= inverse_temperature; + } + } +}; + +struct compute_temperature_probabilities { + void operator()(const event::sample_temperature_top_k_runtime &ev, + context &) const noexcept { + emel::kernel::detail::soft_max_row_ggml(ev.request.card, + ev.request.logits.data()); + } +}; + +struct compute_temperature_top_k { + void operator()(const event::sample_temperature_top_k_runtime &ev, + context &) const noexcept { + for (int32_t index = 0; index < ev.request.card; ++index) { + ev.request.sorted_indices[static_cast(index)] = index; + } + std::sort(ev.request.sorted_indices.begin(), + ev.request.sorted_indices.begin() + ev.request.card, + [&ev](const int32_t lhs, const int32_t rhs) { + return ev.request.logits[static_cast(lhs)] > + ev.request.logits[static_cast(rhs)]; + }); + for (int32_t slot = 0; slot < ev.request.top_k; ++slot) { + const int32_t index = + ev.request.sorted_indices[static_cast(slot)]; + ev.request.top_probabilities[static_cast(slot)] = + ev.request.logits[static_cast(index)]; + ev.request.top_indices[static_cast(slot)] = index; + } + } +}; + +struct select_temperature_top_k { + void operator()(const event::sample_temperature_top_k_runtime &ev, + context &) const noexcept { + constexpr uint32_t multiplier = 16807u; + constexpr uint32_t modulus = 2147483647u; + constexpr float random_max = 2147483647.0f; + ev.request.selected_token_out = -1; + ev.request.selected_score_out = -std::numeric_limits::infinity(); + for (int32_t slot = 0; slot < ev.request.top_k; ++slot) { + ev.request.random_state = static_cast( + (static_cast(ev.request.random_state) * multiplier) % + modulus); + const float uniform = + static_cast(ev.request.random_state) / random_max; + const float divisor = -std::log(uniform); + const float score = + ev.request.top_probabilities[static_cast(slot)] / divisor; + const int32_t replace = + static_cast(score > ev.request.selected_score_out); + ev.request.selected_token_out = + replace * ev.request.top_indices[static_cast(slot)] + + (1 - replace) * ev.request.selected_token_out; + ev.request.selected_score_out = + std::max(ev.request.selected_score_out, score); + } + } +}; + struct publish_done { template void operator()(const runtime_event_type & ev, context &) const noexcept { @@ -121,6 +195,11 @@ inline constexpr prepare_candidates prepare_candidates{}; inline constexpr apply_sampler apply_sampler{}; inline constexpr mark_sampler_error mark_sampler_error{}; inline constexpr advance_sampler_index advance_sampler_index{}; +inline constexpr scale_temperature_logits scale_temperature_logits{}; +inline constexpr compute_temperature_probabilities + compute_temperature_probabilities{}; +inline constexpr compute_temperature_top_k compute_temperature_top_k{}; +inline constexpr select_temperature_top_k select_temperature_top_k{}; inline constexpr publish_done publish_done{}; inline constexpr publish_error publish_error{}; inline constexpr on_unexpected on_unexpected{}; diff --git a/src/emel/logits/sampler/events.hpp b/src/emel/logits/sampler/events.hpp index 1341337e..83ab6201 100644 --- a/src/emel/logits/sampler/events.hpp +++ b/src/emel/logits/sampler/events.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include "emel/callback.hpp" #include "emel/error/error.hpp" @@ -68,6 +69,36 @@ struct sample_preselected { error_out(error_out_ref) {} }; +struct sample_temperature_top_k { + sample_temperature_top_k(std::span logits_ref, const int32_t card_ref, + const float temperature_ref, const int32_t top_k_ref, + std::span sorted_indices_ref, + std::span top_probabilities_ref, + std::span top_indices_ref, + uint32_t &random_state_ref, + int32_t &selected_token_out_ref, + float &selected_score_out_ref, + emel::error::type &error_out_ref) noexcept + : logits(logits_ref), card(card_ref), temperature(temperature_ref), + top_k(top_k_ref), sorted_indices(sorted_indices_ref), + top_probabilities(top_probabilities_ref), top_indices(top_indices_ref), + random_state(random_state_ref), + selected_token_out(selected_token_out_ref), + selected_score_out(selected_score_out_ref), error_out(error_out_ref) {} + + std::span logits; + int32_t card; + float temperature; + int32_t top_k; + std::span sorted_indices; + std::span top_probabilities; + std::span top_indices; + uint32_t &random_state; + int32_t &selected_token_out; + float &selected_score_out; + emel::error::type &error_out; +}; + struct configure_ctx { emel::error::type err = emel::error::cast(error::none); }; @@ -83,6 +114,10 @@ struct sample_preselected_ctx { emel::error::type err = emel::error::cast(error::none); }; +struct sample_temperature_top_k_ctx { + emel::error::type err = emel::error::cast(error::none); +}; + struct configure_runtime { const configure & request; configure_ctx & ctx; @@ -98,6 +133,11 @@ struct sample_preselected_runtime { sample_preselected_ctx & ctx; }; +struct sample_temperature_top_k_runtime { + const sample_temperature_top_k &request; + sample_temperature_top_k_ctx &ctx; +}; + } // namespace emel::logits::sampler::event namespace emel::logits::sampler::events { diff --git a/src/emel/logits/sampler/guards.hpp b/src/emel/logits/sampler/guards.hpp index 590f0bf0..f04fb614 100644 --- a/src/emel/logits/sampler/guards.hpp +++ b/src/emel/logits/sampler/guards.hpp @@ -1,132 +1,182 @@ #pragma once +#include + #include "emel/logits/sampler/context.hpp" #include "emel/logits/sampler/events.hpp" namespace emel::logits::sampler::guard { struct valid_config { - bool operator()(const event::configure_runtime & ev) const noexcept { + bool operator()(const event::configure_runtime &ev) const noexcept { return ev.request.sampler_count > 0; } }; struct invalid_config { - bool operator()(const event::configure_runtime & ev) const noexcept { + bool operator()(const event::configure_runtime &ev) const noexcept { return !valid_config{}(ev); } }; struct request_has_valid_sizes { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return ev.request.vocab_size > 0 && ev.request.candidate_capacity >= ev.request.vocab_size; } }; struct context_has_valid_sampler_table { - bool operator()(const event::sample_logits_runtime &, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &, + const action::context &ctx) const noexcept { return ctx.sampler_count > 0 && ctx.sampler_fns != nullptr; } }; struct valid_request { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { - return request_has_valid_sizes{}(ev) && context_has_valid_sampler_table{}(ev, ctx); + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { + return request_has_valid_sizes{}(ev) && + context_has_valid_sampler_table{}(ev, ctx); } }; struct invalid_request { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { return !valid_request{}(ev, ctx); } }; struct preselected_token_valid { - bool operator()(const event::sample_preselected_runtime & ev) const noexcept { - return ev.request.vocab_size > 0 && - ev.request.selected_token_out >= 0 && + bool operator()(const event::sample_preselected_runtime &ev) const noexcept { + return ev.request.vocab_size > 0 && ev.request.selected_token_out >= 0 && ev.request.selected_token_out < ev.request.vocab_size; } }; struct preselected_token_invalid { - bool operator()(const event::sample_preselected_runtime & ev) const noexcept { + bool operator()(const event::sample_preselected_runtime &ev) const noexcept { return !preselected_token_valid{}(ev); } }; struct has_more_samplers { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { return ev.ctx.sampler_index < ctx.sampler_count; } }; struct no_more_samplers { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { return ev.ctx.sampler_index >= ctx.sampler_count; } }; struct sampler_fn_available { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { return static_cast(ctx.sampler_fns[ev.ctx.sampler_index]); } }; struct sampler_fn_missing { - bool operator()(const event::sample_logits_runtime & ev, const action::context & ctx) const noexcept { + bool operator()(const event::sample_logits_runtime &ev, + const action::context &ctx) const noexcept { return !sampler_fn_available{}(ev, ctx); } }; struct sampler_call_succeeded { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return ev.ctx.sampler_call_error == emel::error::cast(error::none); } }; struct sampler_call_failed { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return ev.ctx.sampler_call_error != emel::error::cast(error::none); } }; struct candidate_count_valid { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { - return ev.ctx.candidate_count > 0 && ev.ctx.candidate_count <= ev.request.vocab_size; + bool operator()(const event::sample_logits_runtime &ev) const noexcept { + return ev.ctx.candidate_count > 0 && + ev.ctx.candidate_count <= ev.request.vocab_size; } }; struct candidate_count_invalid { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return !candidate_count_valid{}(ev); } }; struct sampler_call_succeeded_with_valid_candidate_count { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return sampler_call_succeeded{}(ev) && candidate_count_valid{}(ev); } }; struct sampler_call_succeeded_with_invalid_candidate_count { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return sampler_call_succeeded{}(ev) && candidate_count_invalid{}(ev); } }; struct selected_token_valid { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return ev.request.selected_token_out >= 0 && ev.request.selected_token_out < ev.request.vocab_size; } }; struct selected_token_missing_or_invalid { - bool operator()(const event::sample_logits_runtime & ev) const noexcept { + bool operator()(const event::sample_logits_runtime &ev) const noexcept { return !selected_token_valid{}(ev); } }; -} // namespace emel::logits::sampler::guard +struct temperature_top_k_request_valid { + bool + operator()(const event::sample_temperature_top_k_runtime &ev) const noexcept { + const auto &request = ev.request; + constexpr uint32_t random_modulus = 2147483647u; + return request.card > 0 && std::isfinite(request.temperature) && + request.temperature > 0.0f && request.top_k > 0 && + request.top_k <= request.card && + request.logits.size() >= static_cast(request.card) && + request.sorted_indices.size() >= static_cast(request.card) && + request.top_probabilities.size() >= + static_cast(request.top_k) && + request.top_indices.size() >= static_cast(request.top_k) && + request.random_state % random_modulus != 0u; + } +}; + +struct temperature_top_k_selected_token_valid { + bool + operator()(const event::sample_temperature_top_k_runtime &ev) const noexcept { + return ev.request.selected_token_out >= 0 && + ev.request.selected_token_out < ev.request.card && + std::isfinite(ev.request.selected_score_out); + } +}; + +struct temperature_top_k_selected_token_invalid { + bool + operator()(const event::sample_temperature_top_k_runtime &ev) const noexcept { + return !temperature_top_k_selected_token_valid{}(ev); + } +}; + +struct temperature_top_k_request_invalid { + bool + operator()(const event::sample_temperature_top_k_runtime &ev) const noexcept { + return !temperature_top_k_request_valid{}(ev); + } +}; + +} // namespace emel::logits::sampler::guard diff --git a/src/emel/logits/sampler/sm.hpp b/src/emel/logits/sampler/sm.hpp index a2db721e..1a44f93e 100644 --- a/src/emel/logits/sampler/sm.hpp +++ b/src/emel/logits/sampler/sm.hpp @@ -17,6 +17,11 @@ struct sample_decision {}; struct sample_call {}; struct sample_call_decision {}; struct sample_complete_decision {}; +struct state_temperature_top_k_request_decision {}; +struct state_temperature_top_k_scale {}; +struct state_temperature_top_k_probabilities {}; +struct state_temperature_top_k_rank {}; +struct state_temperature_top_k_select {}; struct done {}; struct errored {}; @@ -88,6 +93,41 @@ struct model { + sml::completion [ guard::selected_token_missing_or_invalid{} ] / action::mark_invalid_request + //------------------------------------------------------------------------------// + // Native temperature/top-k sampling is a typed actor path. Each numeric + // phase is explicit and uses caller-provided, allocation-free workspace. + , sml::state <= sml::state + + sml::event + , sml::state <= + sml::state + + sml::completion + [ guard::temperature_top_k_request_valid{} ] + / action::scale_temperature_logits + , sml::state <= + sml::state + + sml::completion + [ guard::temperature_top_k_request_invalid{} ] + / action::mark_invalid_request + , sml::state <= + sml::state + + sml::completion + / action::compute_temperature_probabilities + , sml::state <= + sml::state + + sml::completion + / action::compute_temperature_top_k + , sml::state <= + sml::state + + sml::completion + / action::select_temperature_top_k + , sml::state <= sml::state + + sml::completion + [ guard::temperature_top_k_selected_token_valid{} ] + , sml::state <= sml::state + + sml::completion + [ guard::temperature_top_k_selected_token_invalid{} ] + / action::mark_invalid_request + //------------------------------------------------------------------------------// , sml::state <= sml::state + sml::completion / action::publish_done @@ -101,6 +141,12 @@ struct model { / action::publish_done , sml::state <= sml::state + sml::completion / action::publish_error + , sml::state <= sml::state + + sml::completion + / action::publish_done + , sml::state <= sml::state + + sml::completion + / action::publish_error //------------------------------------------------------------------------------// , sml::state <= sml::state + sml::unexpected_event @@ -121,6 +167,16 @@ struct model { / action::on_unexpected , sml::state <= sml::state + sml::unexpected_event / action::on_unexpected + , sml::state <= sml::state + + sml::unexpected_event / action::on_unexpected + , sml::state <= sml::state + + sml::unexpected_event / action::on_unexpected + , sml::state <= sml::state + + sml::unexpected_event / action::on_unexpected + , sml::state <= sml::state + + sml::unexpected_event / action::on_unexpected + , sml::state <= sml::state + + sml::unexpected_event / action::on_unexpected , sml::state <= sml::state + sml::unexpected_event / action::on_unexpected , sml::state <= sml::state + sml::unexpected_event @@ -138,30 +194,37 @@ struct sm : public emel::sm { sm() : base_type() {} - bool process_event(const event::configure & ev) { + bool process_event(const event::configure &ev) { event::configure_ctx ctx{}; event::configure_runtime runtime{ev, ctx}; const bool accepted = base_type::process_event(runtime); return accepted && ctx.err == emel::error::cast(error::none); } - bool process_event(const event::sample_logits & ev) { + bool process_event(const event::sample_logits &ev) { event::sample_logits_ctx ctx{}; event::sample_logits_runtime runtime{ev, ctx}; const bool accepted = base_type::process_event(runtime); return accepted && ctx.err == emel::error::cast(error::none); } - bool process_event(const event::sample_preselected & ev) { + bool process_event(const event::sample_preselected &ev) { event::sample_preselected_ctx ctx{}; event::sample_preselected_runtime runtime{ev, ctx}; const bool accepted = base_type::process_event(runtime); return accepted && ctx.err == emel::error::cast(error::none); } - private: + bool process_event(const event::sample_temperature_top_k &ev) { + event::sample_temperature_top_k_ctx ctx{}; + event::sample_temperature_top_k_runtime runtime{ev, ctx}; + const bool accepted = base_type::process_event(runtime); + return accepted && ctx.err == emel::error::cast(error::none); + } + +private: }; using Sampler = sm; -} // namespace emel::logits::sampler +} // namespace emel::logits::sampler diff --git a/src/emel/machines.hpp b/src/emel/machines.hpp index 91faf11b..9ce5cd3c 100644 --- a/src/emel/machines.hpp +++ b/src/emel/machines.hpp @@ -8,36 +8,37 @@ #include "emel/embeddings/generator/sm.hpp" #include "emel/gguf/loader/sm.hpp" #include "emel/graph/processor/sm.hpp" +#include "emel/io/loader/sm.hpp" #include "emel/io/mmap/sm.hpp" #include "emel/io/read/sm.hpp" -#include "emel/io/sm.hpp" #include "emel/io/staged_read/sm.hpp" #include "emel/memory/hybrid/sm.hpp" #include "emel/memory/kv/sm.hpp" #include "emel/memory/recurrent/sm.hpp" +#include "emel/memory/streaming/sm.hpp" #include "emel/model/loader/sm.hpp" #include "emel/model/tensor/sm.hpp" #include "emel/speech/codec/mimi/sm.hpp" +#include "emel/speech/generator/sm.hpp" #include "emel/text/conditioner/sm.hpp" #include "emel/text/detokenizer/sm.hpp" #include "emel/text/encoders/any.hpp" -#include "emel/text/encoders/sm.hpp" #include "emel/text/generator/sm.hpp" #include "emel/text/renderer/sm.hpp" #include "emel/text/tokenizer/sm.hpp" namespace emel { -using ComputeExecutor = emel::graph::processor::sm; using Conditioner = emel::text::conditioner::sm; using Detokenizer = emel::text::detokenizer::sm; using SortformerRequest = emel::diarization::sortformer::request::sm; -using DiarizationRequest = SortformerRequest; using SortformerExecutor = emel::diarization::sortformer::executor::sm; using SortformerPipeline = emel::diarization::sortformer::pipeline::sm; using OmniEmbedEmbeddingsGenerator = emel::embeddings::generator::omniembed::sm; using EmbeddingsGenerator = emel::embeddings::generator::sm; using MimiCodec = emel::speech::codec::mimi::sm; +template +using SpeechGenerator = emel::speech::generator::sm; using EncoderAny = emel::text::encoders::any; using EncoderBpe = emel::text::encoders::bpe::sm; using EncoderSpm = emel::text::encoders::spm::sm; @@ -54,8 +55,8 @@ using IoStagedRead = emel::io::staged_read::sm; using MemoryHybrid = emel::memory::hybrid::sm; using KvCache = emel::memory::kv::sm; using MemoryRecurrent = emel::memory::recurrent::sm; +using MemoryStreaming = emel::memory::streaming::sm; using ModelLoader = emel::model::loader::sm; -using Model = emel::model::loader::sm; using Parser = emel::gguf::loader::sm; using Renderer = emel::text::renderer::sm; using Tokenizer = emel::text::tokenizer::sm; diff --git a/src/emel/memory/streaming/actions.hpp b/src/emel/memory/streaming/actions.hpp new file mode 100644 index 00000000..ac3a7aba --- /dev/null +++ b/src/emel/memory/streaming/actions.hpp @@ -0,0 +1,117 @@ +#pragma once + +#include + +#include "emel/error/error.hpp" +#include "emel/memory/streaming/context.hpp" +#include "emel/memory/streaming/errors.hpp" +#include "emel/memory/streaming/events.hpp" + +namespace emel::memory::streaming::action { + +enum class window_mode : uint8_t { + empty, + filling, + full, +}; + +enum class cursor_mode : uint8_t { + advance, + wrap, +}; + +inline int32_t error_code(const error value) noexcept { + return static_cast(emel::error::cast(value)); +} + +struct effect_initialize { + void operator()(const event::initialize &ev, context &ctx) const noexcept { + ctx.next_logical_position = 0; + ctx.next_physical_position = 0; + ev.error_out = error_code(error::none); + } +}; + +template +struct effect_advance { + void operator()(const event::advance &ev, context &ctx) const noexcept { + const int64_t logical_position = ctx.next_logical_position; + const int32_t physical_position = ctx.next_physical_position; + const int64_t logical_end = logical_position + 1; + int32_t next_physical_position = physical_position + 1; + + if constexpr (resulting_cursor == cursor_mode::wrap) { + next_physical_position = 0; + } + + ev.result.logical_position = logical_position; + ev.result.physical_position = physical_position; + ev.result.window.logical_end = logical_end; + ev.result.window.next_physical_position = next_physical_position; + ev.result.window.capacity = ctx.capacity; + + if constexpr (resulting_window == window_mode::full) { + ev.result.window.logical_begin = + logical_end - static_cast(ctx.capacity); + ev.result.window.physical_begin = next_physical_position; + ev.result.window.valid_positions = ctx.capacity; + } else { + ev.result.window.logical_begin = 0; + ev.result.window.physical_begin = 0; + ev.result.window.valid_positions = static_cast(logical_end); + } + + ctx.next_logical_position = logical_end; + ctx.next_physical_position = next_physical_position; + ev.error_out = error_code(error::none); + } +}; + +struct effect_reset { + void operator()(const event::reset &ev, context &ctx) const noexcept { + ctx.next_logical_position = 0; + ctx.next_physical_position = 0; + ev.error_out = error_code(error::none); + } +}; + +template struct effect_capture_view { + void operator()(const event::capture_view &ev, + const context &ctx) const noexcept { + ev.view_out.logical_end = ctx.next_logical_position; + ev.view_out.next_physical_position = ctx.next_physical_position; + ev.view_out.capacity = ctx.capacity; + + if constexpr (current_window == window_mode::empty) { + ev.view_out.logical_begin = 0; + ev.view_out.physical_begin = 0; + ev.view_out.valid_positions = 0; + } else if constexpr (current_window == window_mode::filling) { + ev.view_out.logical_begin = 0; + ev.view_out.physical_begin = 0; + ev.view_out.valid_positions = + static_cast(ctx.next_logical_position); + } else { + ev.view_out.logical_begin = + ctx.next_logical_position - static_cast(ctx.capacity); + ev.view_out.physical_begin = ctx.next_physical_position; + ev.view_out.valid_positions = ctx.capacity; + } + + ev.error_out = error_code(error::none); + } +}; + +template struct effect_reject { + template + void operator()(const event_type &ev, const context &) const noexcept { + ev.error_out = error_code(error_value); + } +}; + +struct effect_unexpected { + template + void operator()(const event_type &, const context &) const noexcept {} +}; + +} // namespace emel::memory::streaming::action diff --git a/src/emel/memory/streaming/context.hpp b/src/emel/memory/streaming/context.hpp new file mode 100644 index 00000000..0d2c18c6 --- /dev/null +++ b/src/emel/memory/streaming/context.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace emel::memory::streaming { + +struct dependencies { + int32_t capacity; +}; + +namespace action { + +struct context { + explicit context(const dependencies &deps) noexcept + : capacity(deps.capacity) {} + + // Persistent source state is deliberately minimal. Fill/full is represented + // by SML state, and visible-window bounds are derived into event outputs. + const int32_t capacity; + int64_t next_logical_position = 0; + + // This is derivable as next_logical_position % capacity, but retaining the + // cursor removes a 64-bit remainder from every hot streaming advance. The + // transition actions maintain the equivalence invariant. + int32_t next_physical_position = 0; +}; + +} // namespace action + +} // namespace emel::memory::streaming diff --git a/src/emel/memory/streaming/errors.hpp b/src/emel/memory/streaming/errors.hpp new file mode 100644 index 00000000..19df8029 --- /dev/null +++ b/src/emel/memory/streaming/errors.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::memory::streaming { + +enum class error : emel::error::type { + none = 0u, + invalid_configuration = (1u << 0), + uninitialized = (1u << 1), + already_initialized = (1u << 2), + position_overflow = (1u << 3), + internal_error = (1u << 4), +}; + +} // namespace emel::memory::streaming diff --git a/src/emel/memory/streaming/events.hpp b/src/emel/memory/streaming/events.hpp new file mode 100644 index 00000000..2fbec107 --- /dev/null +++ b/src/emel/memory/streaming/events.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include + +namespace emel::memory::streaming { + +// Half-open logical window [logical_begin, logical_end). physical_begin is +// the ring slot holding logical_begin; next_physical_position is the slot the +// next successful advance will overwrite. These values are derived at the +// event boundary and are never mirrored into persistent actor context. +struct window_view { + int64_t logical_begin = 0; + int64_t logical_end = 0; + int32_t physical_begin = 0; + int32_t next_physical_position = 0; + int32_t valid_positions = 0; + int32_t capacity = 0; +}; + +struct advance_result { + int64_t logical_position = -1; + int32_t physical_position = -1; + window_view window = {}; +}; + +namespace event { + +struct initialize { + int32_t &error_out; +}; + +struct advance { + advance_result &result; + int32_t &error_out; +}; + +struct reset { + int32_t &error_out; +}; + +struct capture_view { + window_view &view_out; + int32_t &error_out; +}; + +} // namespace event + +} // namespace emel::memory::streaming diff --git a/src/emel/memory/streaming/guards.hpp b/src/emel/memory/streaming/guards.hpp new file mode 100644 index 00000000..db711f43 --- /dev/null +++ b/src/emel/memory/streaming/guards.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +#include "emel/memory/streaming/context.hpp" + +namespace emel::memory::streaming::guard { + +struct guard_configuration_valid { + bool operator()(const action::context &ctx) const noexcept { + return ctx.capacity > 0; + } +}; + +struct guard_configuration_invalid { + bool operator()(const action::context &ctx) const noexcept { + return !guard_configuration_valid{}(ctx); + } +}; + +struct guard_capacity_one { + bool operator()(const action::context &ctx) const noexcept { + return ctx.capacity == 1; + } +}; + +struct guard_capacity_many { + bool operator()(const action::context &ctx) const noexcept { + return ctx.capacity > 1; + } +}; + +struct guard_filling_remains_partial { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position + 1 < static_cast(ctx.capacity); + } +}; + +struct guard_filling_becomes_full { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position + 1 == static_cast(ctx.capacity); + } +}; + +struct guard_full_position_available_before_wrap { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position < std::numeric_limits::max() && + ctx.next_physical_position >= 0 && + ctx.next_physical_position < ctx.capacity - 1; + } +}; + +struct guard_full_position_available_at_wrap { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position < std::numeric_limits::max() && + ctx.next_physical_position >= 0 && + ctx.next_physical_position == ctx.capacity - 1; + } +}; + +struct guard_full_position_overflow { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position == std::numeric_limits::max(); + } +}; + +struct guard_full_cursor_invalid { + bool operator()(const action::context &ctx) const noexcept { + return ctx.next_logical_position < std::numeric_limits::max() && + (ctx.next_physical_position < 0 || + ctx.next_physical_position >= ctx.capacity); + } +}; + +} // namespace emel::memory::streaming::guard diff --git a/src/emel/memory/streaming/sm.hpp b/src/emel/memory/streaming/sm.hpp new file mode 100644 index 00000000..94025007 --- /dev/null +++ b/src/emel/memory/streaming/sm.hpp @@ -0,0 +1,190 @@ +#pragma once + +// benchmark: designed + +#include + +#include "emel/memory/streaming/actions.hpp" +#include "emel/memory/streaming/context.hpp" +#include "emel/memory/streaming/events.hpp" +#include "emel/memory/streaming/guards.hpp" +#include "emel/sm.hpp" + +namespace emel::memory::streaming { + +struct state_uninitialized {}; +struct state_empty {}; +struct state_filling {}; +struct state_full {}; +struct state_errored {}; + +/* +streaming memory architecture notes (single source of truth) + +state purpose +- state_uninitialized rejects use until the injected capacity is validated. +- state_empty represents an initialized ring with no visible history. +- state_filling represents 0 < visible positions < capacity. +- state_full represents a capacity-sized rolling window. +- state_errored makes an unexpected external event observable until a valid + initialize event recovers the actor. + +control invariants +- one actor instance owns one temporal ring. Independent temporal and depformer + caches compose as independent actors, so stream ids, active bits, ownership + maps, free stacks, and per-slot reference actors are unnecessary here. +- actor state, rather than context flags, owns empty/filling/full behavior. +- context stores only the absolute next logical position and a cached physical + cursor; valid length and window bounds are derived into event outputs. +- every fill, full, wrap, overflow, reset, and error choice is an explicit + guard/transition. Actions execute only the already-selected cursor update. +- reset is O(1): stale payload bytes are excluded by the empty state and are + overwritten on later advances, so the actor never scans or clears storage. + +guard semantics and action side effects +- capacity guards validate the constructor-injected ring geometry. +- filling guards select partial versus first-full publication. +- full guards select cursor increment, cursor wrap, overflow, or invariant + failure without action-local runtime branching. +- successful advance actions publish the write slot and visible window, then + update both cursors. No action allocates or performs I/O. +*/ +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + using effect_advance_filling = + action::effect_advance; + using effect_advance_full = + action::effect_advance; + using effect_advance_full_wrap = + action::effect_advance; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Initialization validates injected capacity. Duplicate initialization + // is explicit and preserves the current ring. + sml::state <= *sml::state + + sml::event [ guard::guard_configuration_valid{} ] + / action::effect_initialize{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_invalid{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_valid{} ] + / action::effect_initialize{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_invalid{} ] + / action::effect_reject{} + + //------------------------------------------------------------------------------// + // Advance from empty or partially filled rings. + , sml::state <= sml::state + + sml::event [ guard::guard_capacity_one{} ] + / effect_advance_full_wrap{} + , sml::state <= sml::state + + sml::event [ guard::guard_capacity_many{} ] + / effect_advance_filling{} + , sml::state <= sml::state + + sml::event [ guard::guard_filling_remains_partial{} ] + / effect_advance_filling{} + , sml::state <= sml::state + + sml::event [ guard::guard_filling_becomes_full{} ] + / effect_advance_full_wrap{} + + //------------------------------------------------------------------------------// + // Full-ring advance explicitly selects increment, wrap, or failure. + , sml::state <= sml::state + + sml::event + [ guard::guard_full_position_available_before_wrap{} ] + / effect_advance_full{} + , sml::state <= sml::state + + sml::event + [ guard::guard_full_position_available_at_wrap{} ] + / effect_advance_full_wrap{} + , sml::state <= sml::state + + sml::event [ guard::guard_full_position_overflow{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event [ guard::guard_full_cursor_invalid{} ] + / action::effect_reject{} + + //------------------------------------------------------------------------------// + // O(1) reset and derived view publication. + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event + / action::effect_capture_view{} + , sml::state <= sml::state + + sml::event + / action::effect_capture_view{} + , sml::state <= sml::state + + sml::event + / action::effect_capture_view{} + + //------------------------------------------------------------------------------// + // Known operations before initialization are explicit failures. + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + + //------------------------------------------------------------------------------// + // Every external event outside the public contract is explicit. + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::process_event; + using base_type::visit_current_states; + + explicit sm(const dependencies &deps) : base_type(std::in_place, deps) {} +}; + +} // namespace emel::memory::streaming diff --git a/src/emel/model/data.hpp b/src/emel/model/data.hpp index 0d85ac69..78a25b98 100644 --- a/src/emel/model/data.hpp +++ b/src/emel/model/data.hpp @@ -273,10 +273,13 @@ struct data { struct moshi_lm_hparams { static constexpr int32_t k_max_delays = 64; + static constexpr int32_t k_max_inference_prompt_tokens = k_max_delays; + static constexpr int32_t k_max_depformer_weight_schedule = k_max_delays; int32_t card = 0; int32_t n_q = 0; int32_t dep_q = 0; + int32_t inference_dep_q = 0; int32_t text_card = 0; int32_t text_padding_id = -1; int32_t dim = 0; @@ -293,8 +296,16 @@ struct data { int32_t depformer_max_period = 0; int32_t depformer_low_rank_embeddings = 0; int32_t extra_heads_num_heads = 0; + int32_t inference_pre_text_silence_frames = 0; + int32_t inference_post_text_silence_frames = 0; uint32_t delay_count = 0; + uint32_t inference_prompt_token_count = 0; + uint32_t depformer_weight_schedule_count = 0; std::array delays = {}; + std::array inference_prompt_tokens = + {}; + std::array + depformer_weight_schedule = {}; bool causal = false; bool cross_attention = false; bool demux_second_stream = false; diff --git a/src/emel/model/moshi/detail.cpp b/src/emel/model/moshi/detail.cpp index aeab6e19..94f25faf 100644 --- a/src/emel/model/moshi/detail.cpp +++ b/src/emel/model/moshi/detail.cpp @@ -1,5 +1,7 @@ #include "emel/model/moshi/detail.hpp" +#include +#include #include #include #include @@ -112,6 +114,12 @@ bool assign_optional_i32(const emel::model::detail::kv_binding &binding, return require_u32_as_i32(binding, key, field); } +bool require_nonnegative_i32(const emel::model::detail::kv_binding &binding, + const std::string_view key, + int32_t &field) noexcept { + return require_u32_as_i32(binding, key, field) && field >= 0; +} + bool require_f32(const emel::model::detail::hparam_loader &loader, const std::string_view key, float &field) noexcept { const auto *entry = emel::model::detail::find_kv_entry(loader.binding, key); @@ -146,6 +154,60 @@ bool require_i32_array(const emel::model::detail::kv_binding &binding, return true; } +bool assign_optional_i32_array(const emel::model::detail::kv_binding &binding, + const std::string_view key, + const std::span dst, + uint32_t &count_out) noexcept { + const auto *entry = emel::model::detail::find_kv_entry(binding, key); + if (entry == nullptr) { + count_out = 0u; + return true; + } + return require_i32_array(binding, key, dst, count_out); +} + +bool validate_lm_inference_hparams( + const emel::model::data::moshi_lm_hparams &lm) noexcept { + if (!lm.depformer_weights_per_step) { + return true; + } + + const uint32_t codebook_count = static_cast(lm.n_q) + 1u; + if (lm.inference_dep_q <= 0 || lm.inference_dep_q > lm.dep_q || + lm.inference_dep_q > lm.n_q || + lm.inference_prompt_token_count != codebook_count || + lm.inference_pre_text_silence_frames < 0 || + lm.inference_post_text_silence_frames < 0) { + return false; + } + + if (lm.inference_prompt_tokens[0] < 0 || + lm.inference_prompt_tokens[0] >= lm.text_card) { + return false; + } + for (uint32_t index = 1u; index < codebook_count; ++index) { + const int32_t token = lm.inference_prompt_tokens[index]; + if (token < 0 || token >= lm.card) { + return false; + } + } + + if (lm.depformer_weight_schedule_count > 0u) { + if (lm.depformer_weight_schedule_count != static_cast(lm.dep_q)) { + return false; + } + for (uint32_t index = 0u; index < lm.depformer_weight_schedule_count; + ++index) { + const int32_t weight_index = lm.depformer_weight_schedule[index]; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + } + } + + return true; +} + bool load_lm_hparams(const emel::model::detail::hparam_loader &loader, emel::model::data &model_out) noexcept { auto &lm = model_out.moshi_lm; @@ -210,6 +272,10 @@ bool load_lm_hparams(const emel::model::detail::hparam_loader &loader, lm.depformer_multi_linear) || !require_bool(binding, "moshi.lm.depformer.weights_per_step", lm.depformer_weights_per_step) || + !assign_optional_i32_array( + binding, "moshi.lm.depformer.weights_per_step_schedule", + std::span{lm.depformer_weight_schedule}, + lm.depformer_weight_schedule_count) || !assign_optional_i32(binding, "moshi.lm.depformer.low_rank_embeddings", lm.depformer_low_rank_embeddings) || !lm.depformer_multi_linear || @@ -217,6 +283,25 @@ bool load_lm_hparams(const emel::model::detail::hparam_loader &loader, return false; } + if (lm.depformer_weights_per_step && + (!require_positive_i32(binding, "moshi.lm.inference.dep_q", + lm.inference_dep_q) || + !require_nonnegative_i32(binding, + "moshi.lm.inference.pre_text_silence_frames", + lm.inference_pre_text_silence_frames) || + !require_nonnegative_i32(binding, + "moshi.lm.inference.post_text_silence_frames", + lm.inference_post_text_silence_frames) || + !require_i32_array(binding, "moshi.lm.inference.prompt_tokens", + std::span{lm.inference_prompt_tokens}, + lm.inference_prompt_token_count))) { + return false; + } + + if (!validate_lm_inference_hparams(lm)) { + return false; + } + model_out.moshi_component_id = emel::model::data::moshi_component::lm; model_out.params.n_embd = lm.dim; model_out.params.n_embd_out = lm.dim; @@ -296,43 +381,24 @@ bool tensor_has_storage( return false; } - uint64_t rows = 1u; + std::array dims = {1u, 1u, 1u, 1u}; for (int32_t dim = 0; dim < tensor.n_dims; ++dim) { if (tensor.dims[static_cast(dim)] <= 0) { return false; } - const uint64_t extent = + dims[static_cast(dim)] = static_cast(tensor.dims[static_cast(dim)]); - if (dim > 0) { - if (extent > UINT64_MAX / rows) { - return false; - } - rows *= extent; - } } - const auto layout = - emel::gguf::loader::detail::ggml_layout( - static_cast(tensor.type)); - if (layout.block_size == 0u || layout.type_size == 0u) { + uint64_t required_bytes = 0u; + const emel::error::type size_err = + emel::gguf::loader::detail::compute_tensor_data_size( + dims, static_cast(tensor.n_dims), + static_cast(tensor.type), required_bytes); + if (size_err != emel::error::cast(emel::gguf::loader::error::none)) { return false; } - const uint64_t cols = static_cast(tensor.dims[0]); - if (cols % layout.block_size != 0u) { - return false; - } - const uint64_t blocks_per_row = cols / layout.block_size; - if (blocks_per_row > UINT64_MAX / rows) { - return false; - } - const uint64_t blocks = blocks_per_row * rows; - if (blocks > UINT64_MAX / layout.type_size) { - return false; - } - const uint64_t required_bytes = - blocks * static_cast(layout.type_size); - return tensor.data_size >= required_bytes; } @@ -353,16 +419,17 @@ bool tensor_has_lm_execution_storage( case 11: // Q3_K case 12: // Q4_K case 14: // Q6_K + case 42: // EMEL Q4_K x8 BL8 return true; default: return false; } } -bool assign_family_view(const emel::model::data &model_data, - const std::string_view prefix, - family_view &family_out, - const bool require_lm_execution_storage = false) noexcept { +bool assign_family_view( + const emel::model::data &model_data, const std::string_view prefix, + family_view &family_out, + const bool require_lm_execution_storage = false) noexcept { family_out = {}; family_out.prefix = prefix; @@ -399,6 +466,20 @@ find_tensor(const emel::model::data &model_data, return nullptr; } +bool bind_exact_tensor(const emel::model::data &model_data, + const std::string_view name, + tensor_view &view_out) noexcept { + const auto *tensor = find_tensor(model_data, name); + if (tensor == nullptr) { + return false; + } + view_out = tensor_view{ + .tensor = tensor, + .name = emel::model::tensor_name_view(model_data, *tensor), + }; + return true; +} + bool has_tensor(const emel::model::data &model_data, const std::string_view name) noexcept { const auto *tensor = find_tensor(model_data, name); @@ -456,8 +537,7 @@ bool has_projection_tensor(const emel::model::data &model_data, return false; } return q8_class ? tensor->type == k_dtype_q8_0 - : tensor->type == k_dtype_f32 || - tensor->type == k_dtype_f16; + : tensor->type == k_dtype_f32 || tensor->type == k_dtype_f16; } bool require_tensor_shape(const emel::model::data &model_data, @@ -480,6 +560,26 @@ bool require_tensor_shape(const emel::model::data &model_data, return true; } +bool require_lm_embedding_tensor_shape( + const emel::model::data &model_data, const std::string_view name, + const std::initializer_list dims) noexcept { + if (!require_tensor_shape(model_data, name, dims)) { + return false; + } + const auto *tensor = find_tensor(model_data, name); + switch (tensor->type) { + case 0: // F32 + case 1: // F16 + case 2: // Q4_0 + case 8: // Q8_0 + case 12: // Q4_K + case 30: // BF16 + return true; + default: + return false; + } +} + bool has_indexed_tensor(const emel::model::data &model_data, const char *format, const int32_t index) noexcept { char buffer[96] = {}; @@ -527,8 +627,7 @@ bool has_depformer_block(const emel::model::data &model_data, block_index)); } -uint32_t -count_tensors_with_storage( +uint32_t count_tensors_with_storage( const emel::model::data &model_data, const bool require_lm_execution_storage = false) noexcept { uint32_t count = 0u; @@ -575,16 +674,122 @@ emel::error::type validate_lm_contract(const emel::model::data &model_data, {dim, text_card + 1}) && require_tensor_shape(model_data, "lm.text_linear.weight", {dim, text_card}) && - require_tensor_shape(model_data, "lm.emb.0.weight", {dim, card + 1}) && + require_lm_embedding_tensor_shape(model_data, "lm.emb.0.weight", + {dim, card + 1}) && has_tensor(model_data, "lm.out_norm.alpha"); if (!shapes_ok) { return emel::error::cast(emel::model::loader::error::model_invalid); } + if (lm.n_q > emel::model::data::moshi_lm_hparams::k_max_delays) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const auto *text_embedding = find_tensor(model_data, "lm.text_emb.weight"); + contract_out.text_embedding = tensor_view{ + .tensor = text_embedding, + .name = emel::model::tensor_name_view(model_data, *text_embedding), + }; + const auto *output_norm = find_tensor(model_data, "lm.out_norm.alpha"); + contract_out.output_norm = tensor_view{ + .tensor = output_norm, + .name = emel::model::tensor_name_view(model_data, *output_norm), + }; + const auto *text_output_projection = + find_tensor(model_data, "lm.text_linear.weight"); + contract_out.text_output_projection = tensor_view{ + .tensor = text_output_projection, + .name = + emel::model::tensor_name_view(model_data, *text_output_projection), + }; + for (int32_t codebook = 0; codebook < lm.n_q; ++codebook) { + char name[96] = {}; + const int written = + std::snprintf(name, sizeof(name), "lm.emb.%d.weight", codebook); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const std::string_view embedding_name{name, static_cast(written)}; + if (!require_lm_embedding_tensor_shape(model_data, embedding_name, + {dim, card + 1})) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const auto *audio_embedding = find_tensor(model_data, embedding_name); + contract_out.audio_embeddings[static_cast(codebook)] = tensor_view{ + .tensor = audio_embedding, + .name = emel::model::tensor_name_view(model_data, *audio_embedding), + }; + } + + if (lm.num_layers > emel::model::data::moshi_lm_hparams::k_max_delays) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } for (int32_t block = 0; block < lm.num_layers; ++block) { if (!has_lm_transformer_block(model_data, block)) { return emel::error::cast(emel::model::loader::error::model_invalid); } + auto &layer = contract_out.temporal_layers[static_cast(block)]; + char name[128] = {}; + int written = std::snprintf(name, sizeof(name), + "lm.transformer.layers.%d.norm1.alpha", block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor(model_data, + std::string_view{name, static_cast(written)}, + layer.norm1)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf( + name, sizeof(name), + "lm.transformer.layers.%d.self_attn.in_projs.0.weight", block); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + (void)bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + layer.split_input_projection); + written = std::snprintf(name, sizeof(name), + "lm.transformer.layers.%d.self_attn.in_proj_weight", + block); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + (void)bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + layer.fused_input_projection); + written = std::snprintf( + name, sizeof(name), + "lm.transformer.layers.%d.self_attn.out_projs.0.weight", block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor(model_data, + std::string_view{name, static_cast(written)}, + layer.output_projection)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf(name, sizeof(name), + "lm.transformer.layers.%d.norm2.alpha", block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor(model_data, + std::string_view{name, static_cast(written)}, + layer.norm2)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf(name, sizeof(name), + "lm.transformer.layers.%d.gating.linear_in.weight", + block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor(model_data, + std::string_view{name, static_cast(written)}, + layer.gating_input)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf(name, sizeof(name), + "lm.transformer.layers.%d.gating.linear_out.weight", + block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor(model_data, + std::string_view{name, static_cast(written)}, + layer.gating_output)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } } if (lm.dep_q > 0) { @@ -607,10 +812,185 @@ emel::error::type validate_lm_contract(const emel::model::data &model_data, return emel::error::cast(emel::model::loader::error::model_invalid); } + std::array + required_weight_indices = {}; + if (lm.depformer_weights_per_step && + lm.depformer_weight_schedule_count > 0u) { + for (uint32_t index = 0u; index < lm.depformer_weight_schedule_count; + ++index) { + const int32_t weight_index = lm.depformer_weight_schedule[index]; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + required_weight_indices[static_cast(weight_index)] = true; + } + } else { + std::fill_n(required_weight_indices.begin(), lm.dep_q, true); + } + + const auto *depformer_text_embedding = + find_tensor(model_data, "lm.depformer_text_emb.weight"); + if (depformer_text_embedding == nullptr || + !require_lm_embedding_tensor_shape(model_data, + "lm.depformer_text_emb.weight", + {lm.depformer_dim, text_card + 1})) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + contract_out.depformer_text_embedding = tensor_view{ + .tensor = depformer_text_embedding, + .name = emel::model::tensor_name_view(model_data, + *depformer_text_embedding), + }; + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + char name[96] = {}; + int written = 0; + if (required_weight_indices[static_cast(codebook)]) { + written = std::snprintf(name, sizeof(name), "lm.depformer_in.%d.weight", + codebook); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const auto *input_projection = find_tensor( + model_data, std::string_view{name, static_cast(written)}); + if (input_projection == nullptr) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + contract_out + .depformer_input_projections[static_cast(codebook)] = + tensor_view{ + .tensor = input_projection, + .name = emel::model::tensor_name_view(model_data, + *input_projection), + }; + } + + written = + std::snprintf(name, sizeof(name), "lm.linears.%d.weight", codebook); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const auto *output_projection = find_tensor( + model_data, std::string_view{name, static_cast(written)}); + if (output_projection == nullptr) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + contract_out.depformer_output_projections[static_cast(codebook)] = + tensor_view{ + .tensor = output_projection, + .name = + emel::model::tensor_name_view(model_data, *output_projection), + }; + + if (codebook > 0) { + written = std::snprintf(name, sizeof(name), + "lm.depformer_emb.%d.weight", codebook - 1); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const auto *audio_embedding = find_tensor( + model_data, std::string_view{name, static_cast(written)}); + if (audio_embedding == nullptr || + !require_lm_embedding_tensor_shape( + model_data, + std::string_view{name, static_cast(written)}, + {lm.depformer_dim, card + 1})) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + contract_out.depformer_audio_embeddings[static_cast( + codebook - 1)] = tensor_view{ + .tensor = audio_embedding, + .name = emel::model::tensor_name_view(model_data, *audio_embedding), + }; + } + } + + if (lm.depformer_num_layers > + emel::model::data::moshi_lm_hparams::k_max_delays) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } for (int32_t block = 0; block < lm.depformer_num_layers; ++block) { if (!has_depformer_block(model_data, block)) { return emel::error::cast(emel::model::loader::error::model_invalid); } + auto &layer = contract_out.depformer_layers[static_cast(block)]; + char name[160] = {}; + int written = std::snprintf(name, sizeof(name), + "lm.depformer.layers.%d.norm1.alpha", block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + layer.norm1)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf(name, sizeof(name), + "lm.depformer.layers.%d.norm2.alpha", block); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + layer.norm2)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + if (!required_weight_indices[static_cast(codebook)]) { + continue; + } + auto &codebook_layer = layer.codebooks[static_cast(codebook)]; + written = + std::snprintf(name, sizeof(name), + "lm.depformer.layers.%d.self_attn.in_projs.%d.weight", + block, codebook); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const bool split_input_bound = bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + codebook_layer.split_input_projection); + written = std::snprintf( + name, sizeof(name), + "lm.depformer.layers.%d.self_attn.in_proj_weight", block); + if (written <= 0 || static_cast(written) >= sizeof(name)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + const bool fused_input_bound = bind_exact_tensor( + model_data, std::string_view{name, static_cast(written)}, + codebook_layer.fused_input_projection); + if (!split_input_bound && !fused_input_bound) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = std::snprintf( + name, sizeof(name), + "lm.depformer.layers.%d.self_attn.out_projs.%d.weight", block, + codebook); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor( + model_data, + std::string_view{name, static_cast(written)}, + codebook_layer.output_projection)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = + std::snprintf(name, sizeof(name), + "lm.depformer.layers.%d.gating.%d.linear_in.weight", + block, codebook); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor( + model_data, + std::string_view{name, static_cast(written)}, + codebook_layer.gating_input)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + written = + std::snprintf(name, sizeof(name), + "lm.depformer.layers.%d.gating.%d.linear_out.weight", + block, codebook); + if (written <= 0 || static_cast(written) >= sizeof(name) || + !bind_exact_tensor( + model_data, + std::string_view{name, static_cast(written)}, + codebook_layer.gating_output)) { + return emel::error::cast(emel::model::loader::error::model_invalid); + } + } } } @@ -672,35 +1052,31 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, "mimi.quantizer.rvq_first.vq.layers.%d." "_codebook.embedding", level); - quantizer_ok = written > 0 && static_cast(written) < sizeof(name) && - require_tensor_shape( - model_data, - std::string_view{name, static_cast(written)}, - {codebook_dim, card}) && - // bind_rvq_split consumes codebooks via tensor_is_float - has_float_tensor_with_elements( - model_data, - std::string_view{name, static_cast(written)}, - static_cast(codebook_dim) * - static_cast(card)); - } - for (int32_t level = 0; - quantizer_ok && level < mimi.n_q - mimi.semantic_n_q; ++level) { + quantizer_ok = + written > 0 && static_cast(written) < sizeof(name) && + require_tensor_shape( + model_data, std::string_view{name, static_cast(written)}, + {codebook_dim, card}) && + // bind_rvq_split consumes codebooks via tensor_is_float + has_float_tensor_with_elements( + model_data, std::string_view{name, static_cast(written)}, + static_cast(codebook_dim) * static_cast(card)); + } + for (int32_t level = 0; quantizer_ok && level < mimi.n_q - mimi.semantic_n_q; + ++level) { char name[96] = {}; const int written = std::snprintf(name, sizeof(name), "mimi.quantizer.rvq_rest.vq.layers.%d." "_codebook.embedding", level); - quantizer_ok = written > 0 && static_cast(written) < sizeof(name) && - require_tensor_shape( - model_data, - std::string_view{name, static_cast(written)}, - {codebook_dim, card}) && - has_float_tensor_with_elements( - model_data, - std::string_view{name, static_cast(written)}, - static_cast(codebook_dim) * - static_cast(card)); + quantizer_ok = + written > 0 && static_cast(written) < sizeof(name) && + require_tensor_shape( + model_data, std::string_view{name, static_cast(written)}, + {codebook_dim, card}) && + has_float_tensor_with_elements( + model_data, std::string_view{name, static_cast(written)}, + static_cast(codebook_dim) * static_cast(card)); } if (!quantizer_ok) { return emel::error::cast(emel::model::loader::error::model_invalid); @@ -743,9 +1119,9 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, for (int32_t layer = 0; transformers_ok && layer < mimi.transformer_num_layers; ++layer) { char base[96] = {}; - const int base_len = std::snprintf( - base, sizeof(base), "mimi.%s.transformer.layers.%d.", - k_transformer_families[family], layer); + const int base_len = + std::snprintf(base, sizeof(base), "mimi.%s.transformer.layers.%d.", + k_transformer_families[family], layer); if (base_len <= 0 || static_cast(base_len) >= sizeof(base)) { transformers_ok = false; break; @@ -791,34 +1167,27 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, has_projection_tensor(model_data, layer_name("self_attn.in_projs.0.weight", name), dim_elements * 3u * dim_elements, proj_q8) && - has_projection_tensor(model_data, - layer_name("self_attn.out_projs.0.weight", - name), - dim_elements * dim_elements, proj_q8) && + has_projection_tensor( + model_data, layer_name("self_attn.out_projs.0.weight", name), + dim_elements * dim_elements, proj_q8) && has_projection_tensor(model_data, layer_name("linear2.weight", name), dim_elements * mlp_elements, proj_q8) && // norms, biases, and layer scales bind through prepare_vector, // which accepts only f32/f16. - has_float_tensor_with_elements(model_data, - layer_name("norm1.weight", name), - dim_elements) && - has_float_tensor_with_elements(model_data, - layer_name("norm1.bias", name), - dim_elements) && - has_float_tensor_with_elements(model_data, - layer_name("norm2.weight", name), - dim_elements) && - has_float_tensor_with_elements(model_data, - layer_name("norm2.bias", name), - dim_elements) && - has_float_tensor_with_elements(model_data, - layer_name("layer_scale_1.scale", - name), - dim_elements) && - has_float_tensor_with_elements(model_data, - layer_name("layer_scale_2.scale", - name), - dim_elements); + has_float_tensor_with_elements( + model_data, layer_name("norm1.weight", name), dim_elements) && + has_float_tensor_with_elements( + model_data, layer_name("norm1.bias", name), dim_elements) && + has_float_tensor_with_elements( + model_data, layer_name("norm2.weight", name), dim_elements) && + has_float_tensor_with_elements( + model_data, layer_name("norm2.bias", name), dim_elements) && + has_float_tensor_with_elements( + model_data, layer_name("layer_scale_1.scale", name), + dim_elements) && + has_float_tensor_with_elements( + model_data, layer_name("layer_scale_2.scale", name), + dim_elements); } } if (!transformers_ok) { @@ -839,8 +1208,8 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, find_tensor(model_data, "mimi.encoder.model.0.conv.conv.weight"); const bool conv_f16 = first_conv != nullptr && first_conv->type == k_dtype_f16; - const uint64_t proj_elements = - static_cast(mimi.dim) * static_cast(mimi.codebook_dim); + const uint64_t proj_elements = static_cast(mimi.dim) * + static_cast(mimi.codebook_dim); const auto *first_rvq_proj = find_tensor(model_data, "mimi.quantizer.rvq_first.input_proj.weight"); const bool rvq_q8 = @@ -867,12 +1236,11 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, for (size_t split = 0; projections_ok && split < 2u; ++split) { char name[96] = {}; const int in_len = - std::snprintf(name, sizeof(name), - "mimi.quantizer.%s.input_proj.weight", k_rvq_splits[split]); + std::snprintf(name, sizeof(name), "mimi.quantizer.%s.input_proj.weight", + k_rvq_splits[split]); projections_ok = in_len > 0 && static_cast(in_len) < sizeof(name) && - rvq_projection_ok( - std::string_view{name, static_cast(in_len)}); + rvq_projection_ok(std::string_view{name, static_cast(in_len)}); if (!projections_ok) { break; } @@ -881,8 +1249,7 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, k_rvq_splits[split]); projections_ok = out_len > 0 && static_cast(out_len) < sizeof(name) && - rvq_projection_ok( - std::string_view{name, static_cast(out_len)}); + rvq_projection_ok(std::string_view{name, static_cast(out_len)}); } if (!projections_ok) { return emel::error::cast(emel::model::loader::error::model_invalid); @@ -897,11 +1264,11 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, // stride otherwise validates and fails codec initialization. plan_codec // also selects the f16 conv operand class from the first encoder conv and // bind_conv requires raw f16 taps on every non-transposed conv. - const auto resolve_conv = [&model_data, conv_f16]( - const std::string_view name, - const int64_t in_channels, - const bool transposed, int64_t &taps_out, - int64_t &out_channels_out) noexcept { + const auto resolve_conv = [&model_data, + conv_f16](const std::string_view name, + const int64_t in_channels, + const bool transposed, int64_t &taps_out, + int64_t &out_channels_out) noexcept { const auto *weight = find_tensor(model_data, name); constexpr int32_t k_dtype_f32 = 0; if (weight == nullptr || !tensor_has_storage(*weight) || @@ -941,17 +1308,18 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, int32_t stride; }; static constexpr seanet_module k_encoder_modules[] = { - {0, 0, 1}, {1, 2, 1}, {3, 0, 4}, {4, 2, 1}, {6, 0, 5}, + {0, 0, 1}, {1, 2, 1}, {3, 0, 4}, {4, 2, 1}, {6, 0, 5}, {7, 2, 1}, {9, 0, 6}, {10, 2, 1}, {12, 0, 8}, {14, 0, 1}, }; static constexpr seanet_module k_decoder_modules[] = { - {0, 0, 1}, {2, 1, 8}, {3, 2, 1}, {5, 1, 6}, {6, 2, 1}, + {0, 0, 1}, {2, 1, 8}, {3, 2, 1}, {5, 1, 6}, {6, 2, 1}, {8, 1, 5}, {9, 2, 1}, {11, 1, 4}, {12, 2, 1}, {14, 0, 1}, }; - const auto walk_seanet = [&resolve_conv](const char *family, - std::span modules, - int64_t channels, - int64_t &channels_out) noexcept { + const auto walk_seanet = [&resolve_conv]( + const char *family, + std::span modules, + int64_t channels, + int64_t &channels_out) noexcept { for (const seanet_module &module : modules) { char name[112] = {}; if (module.kind == 2u) { @@ -999,8 +1367,8 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, int64_t decoder_channels = 0; int64_t down_taps = 0; int64_t down_out = 0; - const auto *upsample = find_tensor( - model_data, "mimi.upsample.convtr.convtr.convtr.weight"); + const auto *upsample = + find_tensor(model_data, "mimi.upsample.convtr.convtr.convtr.weight"); uint64_t upsample_elements = 0u; if (upsample != nullptr && tensor_has_storage(*upsample)) { upsample_elements = 1u; @@ -1010,12 +1378,10 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, } } const bool seanet_ok = - walk_seanet("encoder", - std::span{k_encoder_modules}, + walk_seanet("encoder", std::span{k_encoder_modules}, 1, encoder_channels) && encoder_channels == dim && - walk_seanet("decoder", - std::span{k_decoder_modules}, + walk_seanet("decoder", std::span{k_decoder_modules}, dim, decoder_channels) && decoder_channels == 1 && resolve_conv("mimi.downsample.conv.conv.conv.weight", dim, false, @@ -1047,7 +1413,6 @@ emel::error::type validate_mimi_contract(const emel::model::data &model_data, return emel::error::cast(emel::model::loader::error::model_invalid); } - return emel::error::cast(emel::model::loader::error::none); } diff --git a/src/emel/model/moshi/detail.hpp b/src/emel/model/moshi/detail.hpp index a27e6da9..88b4d304 100644 --- a/src/emel/model/moshi/detail.hpp +++ b/src/emel/model/moshi/detail.hpp @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include @@ -34,6 +36,33 @@ struct family_view { tensor_view first = {}; }; +struct temporal_layer_contract { + tensor_view norm1 = {}; + tensor_view split_input_projection = {}; + tensor_view fused_input_projection = {}; + tensor_view output_projection = {}; + tensor_view norm2 = {}; + tensor_view gating_input = {}; + tensor_view gating_output = {}; +}; + +struct depformer_codebook_layer_contract { + tensor_view split_input_projection = {}; + tensor_view fused_input_projection = {}; + tensor_view output_projection = {}; + tensor_view gating_input = {}; + tensor_view gating_output = {}; +}; + +struct depformer_layer_contract { + tensor_view norm1 = {}; + tensor_view norm2 = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + codebooks = {}; +}; + struct lm_contract { family_view text_emb = {}; family_view audio_emb = {}; @@ -45,6 +74,34 @@ struct lm_contract { family_view depformer = {}; family_view depformer_text_emb = {}; family_view depformer_emb = {}; + tensor_view text_embedding = {}; + tensor_view output_norm = {}; + tensor_view text_output_projection = {}; + tensor_view depformer_text_embedding = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + audio_embeddings = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + depformer_input_projections = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + depformer_audio_embeddings = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + depformer_output_projections = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + temporal_layers = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + depformer_layers = {}; }; struct mimi_contract { diff --git a/src/emel/sm.hpp b/src/emel/sm.hpp index 30653df1..eed2c89c 100644 --- a/src/emel/sm.hpp +++ b/src/emel/sm.hpp @@ -716,6 +716,12 @@ class sm { explicit sm(std::in_place_t, context_args &&... context_args_in) : context_(std::forward(context_args_in)...), state_machine_(context_) {} + template + requires (sizeof...(args) > 0) + explicit sm(std::piecewise_construct_t, context_arg && context_arg_in, + args &&... args_in) + : context_(std::forward(context_arg_in)), + state_machine_(context_, std::forward(args_in)...) {} explicit sm(const context_type & context_in) : context_(context_in), state_machine_(context_) {} explicit sm(context_type && context_in) diff --git a/src/emel/speech/codec/mimi/any.hpp b/src/emel/speech/codec/mimi/any.hpp index 8806b83e..046a38af 100644 --- a/src/emel/speech/codec/mimi/any.hpp +++ b/src/emel/speech/codec/mimi/any.hpp @@ -7,8 +7,6 @@ namespace emel::speech::codec::mimi { -using MimiCodec = sm; - // Public arena-sizing contract for event::initialize: the caller owns the // four arenas, sizes them here before dispatching initialize, and keeps them // alive for the codec's lifetime (the machine never allocates). A result of diff --git a/src/emel/speech/codec/mimi/detail.cpp b/src/emel/speech/codec/mimi/detail.cpp index 17f047bb..ad29145b 100644 --- a/src/emel/speech/codec/mimi/detail.cpp +++ b/src/emel/speech/codec/mimi/detail.cpp @@ -427,16 +427,6 @@ bool dispatch_add(codec_runtime &runtime, const kev::tensor_view &lhs, return runtime.kernel.process_event(ev); } -bool dispatch_sub(codec_runtime &runtime, const kev::tensor_view &lhs, - const kev::tensor_view &rhs, - const kev::tensor_view_mut &dst) noexcept { - kev::op_sub ev{}; - ev.src0 = lhs; - ev.src1 = rhs; - ev.dst = dst; - return runtime.kernel.process_event(ev); -} - bool dispatch_mul_mat(codec_runtime &runtime, const kev::tensor_view &src0, const kev::tensor_view &src1, const kev::tensor_view_mut &dst) noexcept { @@ -1197,15 +1187,13 @@ void compute_rvq_encode(codec_runtime &runtime, const frame_buffer &latent, static_cast(n_q) * static_cast(frames)) { return; } - const uint64_t need = 2u * static_cast(codebook_dim) + 2u + + const uint64_t need = static_cast(codebook_dim) + (static_cast(dim) + 1u) / 2u + 1u; if (workspace.size() < need) { return; } - float *residual = workspace.data(); // [codebook_dim + 1] with trailing 1 - float *chosen = residual + codebook_dim + 1; - auto *staged_f16 = - reinterpret_cast(chosen + codebook_dim + 1); // [dim] halfs + float *residual = workspace.data(); // [codebook_dim] + auto *staged_f16 = reinterpret_cast(residual + codebook_dim); for (int64_t frame = 0; frame < frames; ++frame) { const float *x = latent.data + frame * dim; @@ -1216,7 +1204,7 @@ void compute_rvq_encode(codec_runtime &runtime, const frame_buffer &latent, if (split->level_count <= 0) { continue; } - // project into codebook space; append the bias-fold constant + // project into codebook space bool projected_in = false; if constexpr (proj_q8) { projected_in = dispatch_q8_matvec(runtime, split->input_proj_q8, dim, @@ -1243,43 +1231,39 @@ void compute_rvq_encode(codec_runtime &runtime, const frame_buffer &latent, if (!projected_in) { return; } - residual[codebook_dim] = 1.0f; for (int32_t level = 0; level < split->level_count; ++level) { - int32_t index = -1; - float score = 0.0f; - kev::op_mul_mat_argmax argmax_ev{}; - argmax_ev.src0 = - make_view(split->search_tables[static_cast(level)], - static_cast(codebook_dim) + 1u, - static_cast(entries)); - argmax_ev.src1 = - make_view(residual, 1u, static_cast(codebook_dim) + 1u); - argmax_ev.dst = make_view_mut(&score, 1u, 1u); - argmax_ev.index_out = &index; - if (!runtime.kernel.process_event(argmax_ev) || index < 0) { + const float *codebook = split->codebooks[static_cast(level)]; + if (codebook == nullptr || entries <= 0 || codebook_dim <= 0) { return; } + int32_t index = 0; + float best_distance = 0.0f; + const float *first_entry = codebook; + for (int64_t c = 0; c < codebook_dim; ++c) { + const float diff = first_entry[c] - residual[c]; + best_distance += diff * diff; + } + for (int64_t entry = 1; entry < entries; ++entry) { + const float *candidate = codebook + entry * codebook_dim; + float distance = 0.0f; + for (int64_t c = 0; c < codebook_dim; ++c) { + const float diff = candidate[c] - residual[c]; + distance += diff * diff; + } + if (distance < best_distance) { + best_distance = distance; + index = static_cast(entry); + } + } codes_out[static_cast(code_row * frames + frame)] = index; ++code_row; if (level + 1 < split->level_count) { - kev::op_get_rows rows_ev{}; - rows_ev.src0 = make_view(split->codebooks[static_cast(level)], - static_cast(codebook_dim), - static_cast(entries)); - rows_ev.src1 = make_view(reinterpret_cast(&index), 1u); - rows_ev.src1.type = kev::dtype::i32; - rows_ev.dst = - make_view_mut(chosen, static_cast(codebook_dim), 1u); - if (!runtime.kernel.process_event(rows_ev) || - !dispatch_sub( - runtime, - make_view(residual, static_cast(codebook_dim)), - make_view(chosen, static_cast(codebook_dim)), - make_view_mut(residual, - static_cast(codebook_dim)))) { - return; + const float *chosen = + codebook + static_cast(index) * codebook_dim; + for (int64_t c = 0; c < codebook_dim; ++c) { + residual[c] -= chosen[c]; } } } @@ -1299,9 +1283,14 @@ void compute_rvq_decode(codec_runtime &runtime, std::span codes, const int64_t dim = runtime.dim; const int64_t codebook_dim = quantizer.codebook_dim; const int64_t entries = quantizer.codebook_entries; - const int64_t n_q = runtime.n_q; - if (frames <= 0 || - codes.size() < static_cast(n_q) * static_cast(frames)) { + const int64_t semantic_levels = quantizer.semantic.level_count; + if (frames <= 0 || codes.empty() || + codes.size() % static_cast(frames) != 0u) { + return; + } + const int64_t decode_n_q = + static_cast(codes.size() / static_cast(frames)); + if (decode_n_q < semantic_levels || decode_n_q > runtime.n_q) { return; } const uint64_t need = 2u * static_cast(codebook_dim) + @@ -1321,13 +1310,23 @@ void compute_rvq_decode(codec_runtime &runtime, std::span codes, std::memset(out, 0, static_cast(dim) * sizeof(float)); const rvq_split_weights *splits[2] = {&quantizer.semantic, &quantizer.acoustic}; + const int32_t split_levels[2] = { + static_cast(semantic_levels), + static_cast(decode_n_q - semantic_levels), + }; int64_t code_row = 0; - for (const rvq_split_weights *split : splits) { - if (split->level_count <= 0) { + for (int32_t split_index = 0; split_index < 2; ++split_index) { + const rvq_split_weights *split = splits[static_cast(split_index)]; + const int32_t active_levels = + split_levels[static_cast(split_index)]; + if (active_levels <= 0) { continue; } + if (active_levels > split->level_count) { + return; + } std::memset(summed, 0, static_cast(codebook_dim) * sizeof(float)); - for (int32_t level = 0; level < split->level_count; ++level) { + for (int32_t level = 0; level < active_levels; ++level) { const int32_t index = codes[static_cast(code_row * frames + frame)]; ++code_row; @@ -1385,7 +1384,7 @@ void compute_rvq_decode(codec_runtime &runtime, std::span codes, return; } } - if (code_row != n_q) { + if (code_row != decode_n_q) { return; } } @@ -2099,26 +2098,19 @@ bool bind_rvq_split(arena_cursor_t &cursor, } float *codebook = cursor.take(static_cast(cb_dim) * static_cast(entries)); - float *table = cursor.take((static_cast(cb_dim) + 1u) * - static_cast(entries)); - if (codebook == nullptr || table == nullptr) { + if (codebook == nullptr) { return false; } if constexpr (!dry_run) { for (int64_t entry = 0; entry < entries; ++entry) { - float norm2 = 0.0f; for (int64_t component = 0; component < cb_dim; ++component) { const float value = source_value( *embedding, static_cast(component + entry * cb_dim)); codebook[component + entry * cb_dim] = value; - table[component + entry * (cb_dim + 1)] = value; - norm2 += value * value; } - table[cb_dim + entry * (cb_dim + 1)] = -0.5f * norm2; } } split_out.codebooks[static_cast(level)] = codebook; - split_out.search_tables[static_cast(level)] = table; } return true; } @@ -2317,8 +2309,9 @@ bool validate_codec_contract(const emel::model::data &model_data) noexcept { bool validate_codes_in_range(const codec_runtime &runtime, std::span codes) noexcept { const int32_t entries = runtime.quantizer.codebook_entries; - const size_t count = static_cast(runtime.n_q); - if (codes.size() < count) { + const size_t count = codes.size(); + if (count == 0u || count > static_cast(runtime.n_q) || + count < static_cast(runtime.quantizer.semantic.level_count)) { return false; } for (size_t index = 0; index < count; ++index) { diff --git a/src/emel/speech/codec/mimi/detail.hpp b/src/emel/speech/codec/mimi/detail.hpp index 2170928a..99b9d893 100644 --- a/src/emel/speech/codec/mimi/detail.hpp +++ b/src/emel/speech/codec/mimi/detail.hpp @@ -154,9 +154,9 @@ struct transformer_weights { }; // One RVQ split (rvq_first carries the semantic level, rvq_rest the acoustic -// levels). Codebook search operands are prepared at bind time: the argmax -// score for level codebooks is x . c - |c|^2 / 2, so the prepared table -// appends the bias column and the input vector a trailing 1. +// levels). Codebook rows stay in reference layout; encode scans them with the +// reference squared-distance arithmetic so near-tie argmax behavior does not +// drift from ggml. struct rvq_split_weights { const float *input_proj = nullptr; // [dim, codebook_dim] (conv1x1) const float *output_proj = nullptr; // [codebook_dim, dim] @@ -166,9 +166,7 @@ struct rvq_split_weights { // raw q8_0 row blocks when the model carries quantized projections const uint8_t *input_proj_q8 = nullptr; const uint8_t *output_proj_q8 = nullptr; - // per level: prepared search table [codebook_dim + 1, entries] and raw - // codebook rows [codebook_dim, entries] - std::array search_tables = {}; + // per level: raw codebook rows [codebook_dim, entries] std::array codebooks = {}; int32_t level_count = 0; }; @@ -258,8 +256,8 @@ uint64_t required_frame_floats(const emel::model::data &model_data) noexcept; bool validate_codec_contract(const emel::model::data &model_data) noexcept; // Pure per-request validator for decode codes: every code addresses a valid -// codebook entry. Bounded scan over n_q codes; guards route on this before -// the decode action runs. +// codebook entry, and the request carries a valid codebook prefix. Bounded +// scan; guards route on this before the decode action runs. bool validate_codes_in_range(const codec_runtime &runtime, std::span codes) noexcept; @@ -328,7 +326,7 @@ void compute_rvq_encode(codec_runtime &runtime, const frame_buffer &latent, std::span codes_out, std::span workspace) noexcept; -// codes [n_q, frames] -> latent [dim, frames] +// codes [active_n_q, frames] -> latent [dim, frames] template void compute_rvq_decode(codec_runtime &runtime, std::span codes, int32_t frames, frame_buffer &latent_out, diff --git a/src/emel/speech/codec/mimi/guards.hpp b/src/emel/speech/codec/mimi/guards.hpp index dc7caf9d..e2a3f5e6 100644 --- a/src/emel/speech/codec/mimi/guards.hpp +++ b/src/emel/speech/codec/mimi/guards.hpp @@ -80,9 +80,12 @@ struct guard_decode_request_valid { const action::context &ctx) const noexcept { const auto &request = runtime_ev.request; // Mirror of the encode guard: codes are read and pcm_out is written. + const size_t code_count = request.codes.size(); return request.codes.data() != nullptr && request.pcm_out.data() != nullptr && - request.codes.size() >= static_cast(ctx.runtime.n_q) && + code_count >= static_cast( + ctx.runtime.quantizer.semantic.level_count) && + code_count <= static_cast(ctx.runtime.n_q) && request.pcm_out.size() >= static_cast(ctx.runtime.frame_samples); } diff --git a/src/emel/speech/codec/mimi/quantizer/guards.hpp b/src/emel/speech/codec/mimi/quantizer/guards.hpp index c2e2324a..7031d809 100644 --- a/src/emel/speech/codec/mimi/quantizer/guards.hpp +++ b/src/emel/speech/codec/mimi/quantizer/guards.hpp @@ -64,10 +64,13 @@ struct guard_decode_shape_valid { bool operator()(const event::decode_run &runtime_ev, const action::context &) const noexcept { const auto &request = runtime_ev.request; + const size_t code_count = request.codes.size(); return request.codes.data() != nullptr && request.latent_out.data() != nullptr && request.workspace.data() != nullptr && - request.codes.size() >= static_cast(request.runtime.n_q) && + code_count >= static_cast( + request.runtime.quantizer.semantic.level_count) && + code_count <= static_cast(request.runtime.n_q) && request.latent_out.size() >= static_cast(request.runtime.dim) && request.workspace.size() >= diff --git a/src/emel/speech/codec/mimi/sm.hpp b/src/emel/speech/codec/mimi/sm.hpp index fb028155..30cc3d3c 100644 --- a/src/emel/speech/codec/mimi/sm.hpp +++ b/src/emel/speech/codec/mimi/sm.hpp @@ -475,6 +475,4 @@ struct sm : public emel::sm { } }; -using Codec = sm; - } // namespace emel::speech::codec::mimi diff --git a/src/emel/speech/decoder/whisper/sm.hpp b/src/emel/speech/decoder/whisper/sm.hpp index fe24c67a..52102149 100644 --- a/src/emel/speech/decoder/whisper/sm.hpp +++ b/src/emel/speech/decoder/whisper/sm.hpp @@ -199,6 +199,4 @@ struct sm : public emel::sm { } }; -using Decoder = sm; - } // namespace emel::speech::decoder::whisper diff --git a/src/emel/speech/encoder/whisper/sm.hpp b/src/emel/speech/encoder/whisper/sm.hpp index 0c57f974..3490ca0d 100644 --- a/src/emel/speech/encoder/whisper/sm.hpp +++ b/src/emel/speech/encoder/whisper/sm.hpp @@ -200,6 +200,4 @@ struct sm : public emel::sm { } }; -using Encoder = sm; - } // namespace emel::speech::encoder::whisper diff --git a/src/emel/speech/generator/actions.hpp b/src/emel/speech/generator/actions.hpp new file mode 100644 index 00000000..3bf12f21 --- /dev/null +++ b/src/emel/speech/generator/actions.hpp @@ -0,0 +1,771 @@ +#pragma once + +#include +#include + +#include + +#include "emel/batch/planner/events.hpp" +#include "emel/error/error.hpp" +#include "emel/memory/streaming/errors.hpp" +#include "emel/memory/streaming/events.hpp" +#include "emel/speech/generator/context.hpp" +#include "emel/speech/generator/errors.hpp" +#include "emel/speech/generator/events.hpp" + +namespace emel::speech::generator::action { + +struct frame_plan_capture { + event::frame_ctx &ctx; + + void on_done(const emel::batch::planner::events::plan_done &ev) noexcept { + ctx.child_err = emel::error::cast(emel::batch::planner::error::none); + ctx.plan_step_size = ev.step_sizes[0]; + ctx.plan_output_count = ev.total_outputs; + } + + void on_error(const emel::batch::planner::events::plan_error &ev) noexcept { + ctx.child_err = ev.err; + } +}; + +inline emel::error::type error_code(const error value) noexcept { + return emel::error::cast(value); +} + +template +void effect_store_error(const runtime_event_type &runtime_ev, + const error value) noexcept { + runtime_ev.ctx.err = error_code(value); + runtime_ev.request.error_out = runtime_ev.ctx.err; +} + +template +void effect_dispatch_child(const runtime_event_type &runtime_ev, + actor_type &actor, request_type &request) noexcept { + runtime_ev.ctx.child_err = 0; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = actor.process_event(request); +} + +template struct effect_initialize_temporal_positions { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + int32_t child_err = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.child_accepted = + ctx.collaborators.temporal_positions.process_event( + emel::memory::streaming::event::initialize{child_err}); + runtime_ev.ctx.child_err = static_cast(child_err); + } +}; + +template +struct effect_initialize_secondary_positions { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + int32_t child_err = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.child_accepted = + ctx.collaborators.secondary_positions.process_event( + emel::memory::streaming::event::initialize{child_err}); + runtime_ev.ctx.child_err = static_cast(child_err); + } +}; + +template struct effect_initialize_encoder { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + auto request = ctx.collaborators.encoder_initialize; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.encoder.process_event(request); + } +}; + +template struct effect_initialize_decoder { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + auto request = ctx.collaborators.decoder_initialize; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.decoder.process_event(request); + } +}; + +template struct effect_initialize_predictor { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + auto request = ctx.collaborators.predictor_initialize; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_initialize_conditioning { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + auto request = ctx.collaborators.conditioning_initialize; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_initialize_tokenizer { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + typename dependencies_type::tokenizer_initialize_event request{ + runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template struct effect_publish_initialize_done { + void operator()(const event::initialize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template +struct effect_fail_initialize { + void operator()(const event::initialize_run &runtime_ev, + const context &) const noexcept { + effect_store_error(runtime_ev, error_value); + } +}; + +template struct effect_emit_initialize_done { + void operator()(const event::initialize_run &runtime_ev, + const context &ctx) const noexcept { + runtime_ev.request.on_done(events::initialize_done{ + runtime_ev.request, ctx.collaborators.frame_samples, + ctx.collaborators.codebook_count}); + } +}; + +template struct effect_emit_initialize_error { + void operator()(const event::initialize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::initialize_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template +struct effect_emit_synthesis_initialize_done { + void operator()(const event::initialize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done( + events::initialize_done{runtime_ev.request, 0, 0}); + } +}; + +template +struct effect_initialize_synthesis_conditioner { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.conditioner_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.conditioner, request); + } +}; + +template +struct effect_initialize_synthesis_prefiller { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.prefiller_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.prefiller, request); + } +}; + +template +struct effect_initialize_synthesis_predictor { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.predictor_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.predictor, request); + } +}; + +template struct effect_initialize_synthesis_sampler { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.sampler_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.sampler, request); + } +}; + +template struct effect_initialize_synthesis_decoder { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.decoder_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.decoder, request); + } +}; + +template +struct effect_initialize_synthesis_postprocessor { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + auto request = ctx.collaborators.postprocessor_initialize; + effect_dispatch_child(runtime_ev, ctx.collaborators.postprocessor, request); + } +}; + +template struct effect_condition_voice { + void operator()(const event::condition_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + runtime_ev.ctx.complete = false; + runtime_ev.ctx.remaining = -1; + typename dependencies_type::voice_condition_event request{}; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + request.complete_out = &runtime_ev.ctx.complete; + request.remaining_frames_out = &runtime_ev.ctx.remaining; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_begin_prompt_conditioning { + void operator()(const event::condition_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + auto request = ctx.collaborators.prompt_begin; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_condition_prompt { + void operator()(const event::condition_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + runtime_ev.ctx.complete = false; + runtime_ev.ctx.remaining = -1; + typename dependencies_type::prompt_condition_event request{}; + request.text_token = runtime_ev.request.token; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + request.complete_out = &runtime_ev.ctx.complete; + request.remaining_frames_out = &runtime_ev.ctx.remaining; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_capture_tokenizer_state { + void operator()(const event::condition_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.tokenizer_offset = 0; + typename dependencies_type::capture_tokenizer_state_event request{ + ctx.collaborators.tokenizer_cache_snapshot, + runtime_ev.ctx.tokenizer_offset, runtime_ev.ctx.child_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_restore_tokenizer_state { + void operator()(const event::condition_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + typename dependencies_type::restore_tokenizer_state_event request{ + std::span{ctx.collaborators.tokenizer_cache_snapshot}, + runtime_ev.ctx.tokenizer_offset, runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template struct effect_publish_condition_pending { + void operator()(const event::condition_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.complete_out = false; + runtime_ev.request.remaining_out = runtime_ev.ctx.remaining; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_publish_condition_complete { + void operator()(const event::condition_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.complete_out = true; + runtime_ev.request.remaining_out = 0; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template +struct effect_fail_condition { + void operator()(const event::condition_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.complete_out = false; + runtime_ev.request.remaining_out = -1; + effect_store_error(runtime_ev, error_value); + } +}; + +template struct effect_emit_condition_done { + void operator()(const event::condition_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done(events::condition_done{ + runtime_ev.request, runtime_ev.request.complete_out, + runtime_ev.request.remaining_out}); + } +}; + +template struct effect_emit_condition_error { + void operator()(const event::condition_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::condition_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template struct effect_prepare_generate { + void operator()(const event::generate_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.sample_count_out = 0; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_condition_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::condition_event request{runtime_ev.request}; + effect_dispatch_child(runtime_ev, ctx.collaborators.conditioner, request); + } +}; + +template struct effect_prefill_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::prefill_event request{runtime_ev.request}; + effect_dispatch_child(runtime_ev, ctx.collaborators.prefiller, request); + } +}; + +template struct effect_predict_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::predict_event request{runtime_ev.request}; + effect_dispatch_child(runtime_ev, ctx.collaborators.predictor, request); + } +}; + +template struct effect_sample_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::sample_event request{runtime_ev.request}; + effect_dispatch_child(runtime_ev, ctx.collaborators.sampler, request); + } +}; + +template struct effect_decode_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::decode_event request{ + runtime_ev.request, runtime_ev.ctx.sample_count}; + effect_dispatch_child(runtime_ev, ctx.collaborators.decoder, request); + } +}; + +template struct effect_postprocess_generate { + void operator()(const event::generate_run &runtime_ev, + context &ctx) const noexcept { + typename dependencies_type::postprocess_event request{ + runtime_ev.request, runtime_ev.ctx.sample_count}; + effect_dispatch_child(runtime_ev, ctx.collaborators.postprocessor, request); + } +}; + +template struct effect_publish_generation_done { + void operator()(const event::generate_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.sample_count_out = runtime_ev.ctx.sample_count; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_emit_generation_done { + void operator()(const event::generate_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done(events::generation_done{ + runtime_ev.request, runtime_ev.request.sample_count_out}); + } +}; + +template +struct effect_fail_generate { + void operator()(const event::generate_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.sample_count_out = 0; + effect_store_error(runtime_ev, error_value); + } +}; + +template struct effect_emit_generation_error { + void operator()(const event::generate_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::generation_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template struct effect_encode_stream_frame { + void operator()(const event::stream_frame_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + typename dependencies_type::encode_event request{ + runtime_ev.request.pcm_in, ctx.collaborators.input_codes}; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.encoder.process_event(request); + } +}; + +template struct effect_encode_flush_frame { + void operator()(const event::flush_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + typename dependencies_type::encode_event request{ + std::span{ctx.collaborators.silence_pcm}, + ctx.collaborators.input_codes}; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.encoder.process_event(request); + } +}; + +template +struct effect_tokenize_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + typename dependencies_type::tokenize_event request{ + ctx.collaborators.tokenize_input_codes, ctx.collaborators.model_codes, + runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template +struct effect_predict_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + typename dependencies_type::predict_event request{ + std::span{ctx.collaborators.model_codes}, + ctx.collaborators.prediction_workspace, runtime_ev.ctx.plan_step_size, + runtime_ev.ctx.plan_output_count}; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template +struct effect_plan_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = + emel::error::cast(emel::batch::planner::error::none); + runtime_ev.ctx.plan_step_size = 0; + runtime_ev.ctx.plan_output_count = 0; + frame_plan_capture capture{runtime_ev.ctx}; + const auto on_done = + emel::callback:: + template from( + &capture); + const auto on_error = + emel::callback:: + template from( + &capture); + const emel::batch::planner::event::plan_request request{ + .token_ids = ctx.collaborators.model_codes.data(), + .n_tokens = ctx.collaborators.frame_plan_token_count, + .n_steps = ctx.collaborators.frame_plan_steps, + .mode = ctx.collaborators.frame_plan_mode, + .output_all = ctx.collaborators.frame_plan_output_all, + .on_done = on_done, + .on_error = on_error, + }; + runtime_ev.ctx.child_accepted = + ctx.collaborators.planner.process_event(request); + } +}; + +template +struct effect_execute_prediction_graph { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + typename dependencies_type::graph_event request{ + ctx.collaborators.prediction_workspace, + std::span{ctx.collaborators.model_codes}}; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.graph.process_event(request); + } +}; + +template +struct effect_sample_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + runtime_ev.ctx.predicted_text_token = -1; + std::fill(ctx.collaborators.predicted_codes.begin(), + ctx.collaborators.predicted_codes.end(), -1); + typename dependencies_type::sample_event request{ + ctx.collaborators.prediction_workspace, + std::span{ctx.collaborators.model_codes}, + ctx.collaborators.predicted_codes, runtime_ev.ctx.predicted_text_token}; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.sampler.process_event(request); + } +}; + +template +struct effect_detokenize_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + runtime_ev.ctx.produced = false; + runtime_ev.ctx.text_token = -1; + std::fill(ctx.collaborators.output_codes.begin(), + ctx.collaborators.output_codes.end(), -1); + typename dependencies_type::detokenize_event request{ + runtime_ev.ctx.predicted_text_token, + std::span{ctx.collaborators.predicted_codes}, + runtime_ev.ctx.text_token, + ctx.collaborators.output_codes, + runtime_ev.ctx.produced, + runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template +struct effect_decode_frame { + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + typename dependencies_type::decode_event request{ + std::span{ctx.collaborators.output_codes}, + runtime_ev.request.pcm_out}; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.decoder.process_event(request); + } +}; + +template +void effect_publish_frame_tokens(const runtime_event_type &runtime_ev, + const auto &ctx) noexcept { + std::copy(ctx.collaborators.input_codes.begin(), + ctx.collaborators.input_codes.end(), + runtime_ev.request.encoded_tokens_out.begin()); + std::copy(ctx.collaborators.output_codes.begin(), + ctx.collaborators.output_codes.end(), + runtime_ev.request.generated_tokens_out.begin()); + runtime_ev.request.text_token_out = runtime_ev.ctx.text_token; +} + +template struct effect_publish_stream_frame_produced { + void operator()(const event::stream_frame_run &runtime_ev, + const context &ctx) const noexcept { + effect_publish_frame_tokens(runtime_ev, ctx); + runtime_ev.request.sample_count_out = ctx.collaborators.frame_samples; + runtime_ev.request.produced_out = true; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_publish_stream_frame_pending { + void operator()(const event::stream_frame_run &runtime_ev, + const context &ctx) const noexcept { + effect_publish_frame_tokens(runtime_ev, ctx); + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template +struct effect_fail_stream_frame { + void operator()(const event::stream_frame_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + effect_store_error(runtime_ev, error_value); + } +}; + +template struct effect_emit_stream_frame_done { + void operator()(const event::stream_frame_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done(events::stream_frame_done{ + runtime_ev.request, runtime_ev.request.sample_count_out, + runtime_ev.request.produced_out}); + } +}; + +template struct effect_emit_stream_frame_error { + void operator()(const event::stream_frame_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::stream_frame_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template struct effect_publish_flush_produced { + void operator()(const event::flush_run &runtime_ev, + const context &ctx) const noexcept { + effect_publish_frame_tokens(runtime_ev, ctx); + runtime_ev.request.sample_count_out = ctx.collaborators.frame_samples; + runtime_ev.request.complete_out = false; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_publish_flush_pending { + void operator()(const event::flush_run &runtime_ev, + const context &ctx) const noexcept { + effect_publish_frame_tokens(runtime_ev, ctx); + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.complete_out = false; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_fail_flush { + void operator()(const event::flush_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.complete_out = false; + effect_store_error(runtime_ev, error_value); + } +}; + +template struct effect_emit_flush_done { + void operator()(const event::flush_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done(events::flush_done{ + runtime_ev.request, runtime_ev.request.sample_count_out, + runtime_ev.request.complete_out}); + } +}; + +template struct effect_emit_flush_error { + void operator()(const event::flush_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::flush_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template +struct effect_reject_reset { + void operator()(const event::reset &ev, + const context &) const noexcept { + ev.error_out = error_code(error_value); + } +}; + +template struct effect_unexpected { + static void + effect_reject_origin(const event::initialize_run &runtime_ev, + const context &ctx) noexcept { + effect_fail_initialize{}( + runtime_ev, ctx); + } + + static void + effect_reject_origin(const event::condition_run &runtime_ev, + const context &ctx) noexcept { + effect_fail_condition{}( + runtime_ev, ctx); + } + + static void + effect_reject_origin(const event::generate_run &runtime_ev, + const context &ctx) noexcept { + effect_fail_generate{}(runtime_ev, + ctx); + } + + static void + effect_reject_origin(const event::stream_frame_run &runtime_ev, + const context &ctx) noexcept { + effect_fail_stream_frame{}( + runtime_ev, ctx); + } + + static void + effect_reject_origin(const event::flush_run &runtime_ev, + const context &ctx) noexcept { + effect_fail_flush{}(runtime_ev, + ctx); + } + + static void + effect_reject_origin(const event::reset &ev, + const context &ctx) noexcept { + effect_reject_reset{}(ev, ctx); + } + + template + static void + effect_reject_origin(const event_type &, + const context &) noexcept {} + + template + void operator()(const unexpected_type &unexpected, + const context &ctx) const noexcept { + effect_reject_origin(stateforward::sml::back::get_origin_event(unexpected), + ctx); + } +}; + +} // namespace emel::speech::generator::action diff --git a/src/emel/speech/generator/any.hpp b/src/emel/speech/generator/any.hpp new file mode 100644 index 00000000..c2f68a04 --- /dev/null +++ b/src/emel/speech/generator/any.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "emel/speech/generator/sm.hpp" + +namespace emel::speech::generator { + +using duplex = action::mode::duplex; +using synthesis = action::mode::synthesis; + +} // namespace emel::speech::generator diff --git a/src/emel/speech/generator/context.hpp b/src/emel/speech/generator/context.hpp new file mode 100644 index 00000000..2819c58a --- /dev/null +++ b/src/emel/speech/generator/context.hpp @@ -0,0 +1,98 @@ +#pragma once + +#include + +#include "emel/batch/planner/events.hpp" + +namespace emel::speech::generator::action { + +namespace mode { + +struct duplex {}; +struct synthesis {}; + +} // namespace mode + +template +concept duplex_dependencies = requires(dependencies_type deps) { + requires std::same_as; + typename dependencies_type::voice_condition_event; + typename dependencies_type::prompt_begin_event; + typename dependencies_type::prompt_condition_event; + typename dependencies_type::encode_event; + typename dependencies_type::tokenizer_initialize_event; + typename dependencies_type::tokenize_event; + typename dependencies_type::predict_event; + typename dependencies_type::graph_event; + typename dependencies_type::sample_event; + typename dependencies_type::detokenize_event; + typename dependencies_type::capture_tokenizer_state_event; + typename dependencies_type::restore_tokenizer_state_event; + typename dependencies_type::decode_event; + deps.temporal_positions; + deps.secondary_positions; + deps.planner; + deps.encoder; + deps.tokenizer; + deps.decoder; + deps.predictor; + deps.graph; + deps.sampler; + deps.prediction_workspace; + deps.encoder_initialize; + deps.decoder_initialize; + deps.predictor_initialize; + deps.conditioning_initialize; + deps.prompt_begin; + deps.silence_pcm; + deps.input_codes; + deps.tokenize_input_codes; + deps.model_codes; + deps.predicted_codes; + deps.output_codes; + deps.tokenizer_cache_snapshot; + deps.frame_samples; + deps.codebook_count; + deps.frame_plan_mode; + deps.frame_plan_steps; + deps.frame_plan_token_count; + deps.frame_plan_output_all; +}; + +template +concept synthesis_dependencies = requires(dependencies_type deps) { + requires std::same_as; + typename dependencies_type::condition_event; + typename dependencies_type::prefill_event; + typename dependencies_type::predict_event; + typename dependencies_type::sample_event; + typename dependencies_type::decode_event; + typename dependencies_type::postprocess_event; + deps.conditioner; + deps.prefiller; + deps.predictor; + deps.sampler; + deps.decoder; + deps.postprocessor; + deps.conditioner_initialize; + deps.prefiller_initialize; + deps.predictor_initialize; + deps.sampler_initialize; + deps.decoder_initialize; + deps.postprocessor_initialize; +}; + +template +concept generator_dependencies = duplex_dependencies || + synthesis_dependencies; + +template struct context { + explicit context(const dependencies_type &deps) noexcept + : collaborators(deps) {} + + const dependencies_type collaborators; +}; + +} // namespace emel::speech::generator::action diff --git a/src/emel/speech/generator/errors.hpp b/src/emel/speech/generator/errors.hpp new file mode 100644 index 00000000..9328a93f --- /dev/null +++ b/src/emel/speech/generator/errors.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::speech::generator { + +enum class error : emel::error::type { + none = 0u, + uninitialized = (1u << 0), + already_initialized = (1u << 1), + invalid_request = (1u << 2), + internal_error = (1u << 4), + memory_initialize_failed = (1u << 5), + encoder_initialize_failed = (1u << 6), + decoder_initialize_failed = (1u << 7), + predictor_initialize_failed = (1u << 9), + conditioning_failed = (1u << 10), + encode_failed = (1u << 11), + predict_failed = (1u << 12), + decode_failed = (1u << 13), + conditioner_initialize_failed = (1u << 14), + prefiller_initialize_failed = (1u << 15), + sampler_initialize_failed = (1u << 16), + postprocessor_initialize_failed = (1u << 17), + prefill_failed = (1u << 18), + sample_failed = (1u << 19), + postprocess_failed = (1u << 20), + unsupported_request = (1u << 21), + tokenizer_initialize_failed = (1u << 22), + tokenize_failed = (1u << 23), + detokenize_failed = (1u << 24), + planning_failed = (1u << 25), + graph_failed = (1u << 26), +}; + +} // namespace emel::speech::generator diff --git a/src/emel/speech/generator/events.hpp b/src/emel/speech/generator/events.hpp new file mode 100644 index 00000000..3d400efb --- /dev/null +++ b/src/emel/speech/generator/events.hpp @@ -0,0 +1,244 @@ +#pragma once + +#include +#include +#include + +#include "emel/callback.hpp" +#include "emel/error/error.hpp" + +namespace emel::speech::generator::events { + +struct initialize_done; +struct initialize_error; +struct condition_done; +struct condition_error; +struct generation_done; +struct generation_error; +struct stream_frame_done; +struct stream_frame_error; +struct flush_done; +struct flush_error; + +} // namespace emel::speech::generator::events + +namespace emel::speech::generator::event { + +struct initialize { + explicit initialize(emel::error::type &error_out_ref) noexcept + : error_out(error_out_ref) {} + + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct condition { + condition(const int32_t token_ref, bool &complete_out_ref, + int32_t &remaining_out_ref, + emel::error::type &error_out_ref) noexcept + : token(token_ref), complete_out(complete_out_ref), + remaining_out(remaining_out_ref), error_out(error_out_ref) {} + + int32_t token = -1; + std::string_view text = {}; + std::span reference_pcm = {}; + bool &complete_out; + int32_t &remaining_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct generate { + generate(const std::string_view text_ref, std::span pcm_out_ref, + int32_t &sample_count_out_ref, + emel::error::type &error_out_ref) noexcept + : text(text_ref), pcm_out(pcm_out_ref), + sample_count_out(sample_count_out_ref), error_out(error_out_ref) {} + + std::string_view text = {}; + std::span reference_pcm = {}; + std::span pcm_out = {}; + int32_t &sample_count_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct stream_frame { + stream_frame(std::span pcm_in_ref, std::span pcm_out_ref, + std::span encoded_tokens_out_ref, + std::span generated_tokens_out_ref, + int32_t &text_token_out_ref, int32_t &sample_count_out_ref, + bool &produced_out_ref, + emel::error::type &error_out_ref) noexcept + : pcm_in(pcm_in_ref), pcm_out(pcm_out_ref), + encoded_tokens_out(encoded_tokens_out_ref), + generated_tokens_out(generated_tokens_out_ref), + text_token_out(text_token_out_ref), + sample_count_out(sample_count_out_ref), produced_out(produced_out_ref), + error_out(error_out_ref) {} + + std::span pcm_in = {}; + std::span pcm_out = {}; + std::span encoded_tokens_out = {}; + std::span generated_tokens_out = {}; + int32_t &text_token_out; + int32_t &sample_count_out; + bool &produced_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct flush { + flush(std::span pcm_out_ref, std::span encoded_tokens_out_ref, + std::span generated_tokens_out_ref, + int32_t &text_token_out_ref, int32_t &sample_count_out_ref, + bool &complete_out_ref, emel::error::type &error_out_ref) noexcept + : pcm_out(pcm_out_ref), encoded_tokens_out(encoded_tokens_out_ref), + generated_tokens_out(generated_tokens_out_ref), + text_token_out(text_token_out_ref), + sample_count_out(sample_count_out_ref), complete_out(complete_out_ref), + error_out(error_out_ref) {} + + std::span pcm_out = {}; + std::span encoded_tokens_out = {}; + std::span generated_tokens_out = {}; + int32_t &text_token_out; + int32_t &sample_count_out; + bool &complete_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct reset { + emel::error::type &error_out; +}; + +struct initialize_ctx { + emel::error::type err = {}; + emel::error::type child_err = {}; + bool child_accepted = false; + int32_t tokenizer_err = 0; +}; + +struct condition_ctx { + emel::error::type err = {}; + emel::error::type child_err = {}; + emel::error::type graph_err = {}; + bool child_accepted = false; + bool complete = false; + int32_t remaining = -1; + int32_t tokenizer_err = 0; + int64_t tokenizer_offset = 0; +}; + +struct generate_ctx { + emel::error::type err = {}; + emel::error::type child_err = {}; + bool child_accepted = false; + int32_t sample_count = 0; +}; + +struct frame_ctx { + emel::error::type err = {}; + emel::error::type child_err = {}; + emel::error::type graph_err = {}; + bool child_accepted = false; + bool produced = false; + int32_t tokenizer_err = 0; + int32_t predicted_text_token = -1; + int32_t text_token = -1; + int32_t plan_step_size = 0; + int32_t plan_output_count = 0; +}; + +struct flush_ctx : frame_ctx {}; + +struct initialize_run { + const initialize &request; + initialize_ctx &ctx; +}; + +struct condition_run { + const condition &request; + condition_ctx &ctx; +}; + +struct generate_run { + const generate &request; + generate_ctx &ctx; +}; + +struct stream_frame_run { + const stream_frame &request; + frame_ctx &ctx; +}; + +struct flush_run { + const flush &request; + flush_ctx &ctx; +}; + +} // namespace emel::speech::generator::event + +namespace emel::speech::generator::events { + +struct initialize_done { + const event::initialize &request; + int32_t frame_samples = 0; + int32_t codebook_count = 0; +}; + +struct initialize_error { + const event::initialize &request; + emel::error::type err = {}; +}; + +struct condition_done { + const event::condition &request; + bool complete = false; + int32_t remaining = -1; +}; + +struct condition_error { + const event::condition &request; + emel::error::type err = {}; +}; + +struct generation_done { + const event::generate &request; + int32_t sample_count = 0; +}; + +struct generation_error { + const event::generate &request; + emel::error::type err = {}; +}; + +struct stream_frame_done { + const event::stream_frame &request; + int32_t sample_count = 0; + bool produced = false; +}; + +struct stream_frame_error { + const event::stream_frame &request; + emel::error::type err = {}; +}; + +struct flush_done { + const event::flush &request; + int32_t sample_count = 0; + bool complete = false; +}; + +struct flush_error { + const event::flush &request; + emel::error::type err = {}; +}; + +} // namespace emel::speech::generator::events diff --git a/src/emel/speech/generator/guards.hpp b/src/emel/speech/generator/guards.hpp new file mode 100644 index 00000000..add259da --- /dev/null +++ b/src/emel/speech/generator/guards.hpp @@ -0,0 +1,266 @@ +#pragma once + +#include + +#include "emel/speech/generator/context.hpp" +#include "emel/speech/generator/events.hpp" + +namespace emel::speech::generator::guard { + +template struct guard_initialize_request_valid { + bool + operator()(const event::initialize_run &, + const action::context &ctx) const noexcept { + const auto count = static_cast(ctx.collaborators.codebook_count); + const auto samples = static_cast(ctx.collaborators.frame_samples); + return ctx.collaborators.codebook_count > 0 && + ctx.collaborators.frame_samples > 0 && + ctx.collaborators.input_codes.size() == count && + ctx.collaborators.output_codes.size() == count && + ctx.collaborators.silence_pcm.size() == samples && + ctx.collaborators.frame_plan_token_count > 0 && + ctx.collaborators.frame_plan_steps > 0 && + static_cast(ctx.collaborators.frame_plan_token_count) <= + ctx.collaborators.model_codes.size(); + } +}; + +template struct guard_initialize_request_invalid { + bool + operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_initialize_request_valid{}(runtime_ev, + ctx); + } +}; + +template +struct guard_child_succeeded { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0; + } +}; + +template +struct guard_child_failed { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_child_succeeded{}( + runtime_ev, ctx); + } +}; + +template struct guard_condition_succeeded_pending { + bool operator()(const event::condition_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0 && !runtime_ev.ctx.complete; + } +}; + +template struct guard_condition_succeeded_complete { + bool operator()(const event::condition_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0 && runtime_ev.ctx.complete; + } +}; + +template struct guard_condition_failed { + bool operator()(const event::condition_run &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.ctx.child_accepted || runtime_ev.ctx.child_err != 0 || + runtime_ev.ctx.graph_err != 0; + } +}; + +template struct guard_generate_request_valid { + bool operator()(const event::generate_run &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.request.text.empty() && + !runtime_ev.request.pcm_out.empty(); + } +}; + +template struct guard_generate_request_invalid { + bool + operator()(const event::generate_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_generate_request_valid{}(runtime_ev, ctx); + } +}; + +template struct guard_stream_request_valid { + bool + operator()(const event::stream_frame_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto count = static_cast(ctx.collaborators.codebook_count); + const auto samples = static_cast(ctx.collaborators.frame_samples); + return runtime_ev.request.pcm_in.size() == samples && + runtime_ev.request.pcm_out.size() >= samples && + runtime_ev.request.encoded_tokens_out.size() >= count && + runtime_ev.request.generated_tokens_out.size() >= count; + } +}; + +template struct guard_stream_request_invalid { + bool + operator()(const event::stream_frame_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_stream_request_valid{}(runtime_ev, ctx); + } +}; + +template struct guard_flush_request_valid { + bool + operator()(const event::flush_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto count = static_cast(ctx.collaborators.codebook_count); + const auto samples = static_cast(ctx.collaborators.frame_samples); + return runtime_ev.request.pcm_out.size() >= samples && + runtime_ev.request.encoded_tokens_out.size() >= count && + runtime_ev.request.generated_tokens_out.size() >= count; + } +}; + +template struct guard_flush_request_invalid { + bool + operator()(const event::flush_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_flush_request_valid{}(runtime_ev, ctx); + } +}; + +template +struct guard_frame_produced { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0 && runtime_ev.ctx.produced; + } +}; + +template +struct guard_prediction_succeeded { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0; + } +}; + +template +struct guard_prediction_failed { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_prediction_succeeded{}( + runtime_ev, ctx); + } +}; + +template +struct guard_frame_pending { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0 && !runtime_ev.ctx.produced; + } +}; + +template +struct guard_frame_failed { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.ctx.child_accepted || runtime_ev.ctx.child_err != 0 || + runtime_ev.ctx.graph_err != 0; + } +}; + +template +struct guard_done_callback_present { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_done); + } +}; + +template +struct guard_done_callback_absent { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_done_callback_present{}(runtime_ev, ctx); + } +}; + +template +struct guard_error_callback_present { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_error); + } +}; + +template +struct guard_error_callback_absent { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_error_callback_present{}(runtime_ev, ctx); + } +}; + +template +struct guard_condition_pending_done_callback_present { + bool + operator()(const event::condition_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_condition_succeeded_pending{}(runtime_ev, + ctx) && + guard_done_callback_present{}(runtime_ev, ctx); + } +}; + +template +struct guard_condition_pending_done_callback_absent { + bool + operator()(const event::condition_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_condition_succeeded_pending{}(runtime_ev, + ctx) && + guard_done_callback_absent{}(runtime_ev, ctx); + } +}; + +template +struct guard_condition_complete_done_callback_present { + bool + operator()(const event::condition_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_condition_succeeded_complete{}(runtime_ev, + ctx) && + guard_done_callback_present{}(runtime_ev, ctx); + } +}; + +template +struct guard_condition_complete_done_callback_absent { + bool + operator()(const event::condition_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_condition_succeeded_complete{}(runtime_ev, + ctx) && + guard_done_callback_absent{}(runtime_ev, ctx); + } +}; + +} // namespace emel::speech::generator::guard diff --git a/src/emel/speech/generator/sm.hpp b/src/emel/speech/generator/sm.hpp new file mode 100644 index 00000000..f872ab57 --- /dev/null +++ b/src/emel/speech/generator/sm.hpp @@ -0,0 +1,843 @@ +#pragma once + +// benchmark: designed + +#include + +#include "emel/sm.hpp" +#include "emel/speech/generator/actions.hpp" +#include "emel/speech/generator/context.hpp" +#include "emel/speech/generator/events.hpp" +#include "emel/speech/generator/guards.hpp" + +namespace emel::speech::generator { + +struct state_uninitialized {}; +struct state_initialize_temporal_result {}; +struct state_initialize_secondary_result {}; +struct state_initialize_encoder_result {}; +struct state_initialize_decoder_result {}; +struct state_initialize_predictor_result {}; +struct state_initialize_conditioning_result {}; +struct state_initialize_tokenizer_result {}; +struct state_initialize_synthesis_conditioner_result {}; +struct state_initialize_synthesis_prefiller_result {}; +struct state_initialize_synthesis_predictor_result {}; +struct state_initialize_synthesis_sampler_result {}; +struct state_initialize_synthesis_decoder_result {}; +struct state_initialize_synthesis_postprocessor_result {}; +struct state_initialize_done_channel_decision {}; +struct state_initialize_error_channel_decision {}; + +struct state_condition_voice {}; +struct state_condition_voice_result {}; +struct state_condition_prompt_begin_result {}; +struct state_condition_voice_done_channel_decision {}; +struct state_condition_prompt {}; +struct state_condition_prompt_result {}; +struct state_condition_capture_tokenizer_result {}; +struct state_condition_restore_tokenizer_result {}; +struct state_condition_prompt_done_channel_decision {}; +struct state_condition_error_channel_decision {}; + +struct state_ready {}; +struct state_generate_conditioning {}; +struct state_generate_prefill {}; +struct state_generate_predict {}; +struct state_generate_sample {}; +struct state_generate_decode {}; +struct state_generate_postprocess {}; +struct state_generate_error_channel_decision {}; +struct state_generate_done_channel_decision {}; + +struct state_stream_encode_result {}; +struct state_stream_tokenize_result {}; +struct state_stream_plan_result {}; +struct state_stream_predict_result {}; +struct state_stream_graph_result {}; +struct state_stream_sample_result {}; +struct state_stream_detokenize_result {}; +struct state_stream_decode_result {}; +struct state_stream_done_channel_decision {}; +struct state_stream_error_channel_decision {}; + +struct state_flushing {}; +struct state_flush_encode_result {}; +struct state_flush_tokenize_result {}; +struct state_flush_plan_result {}; +struct state_flush_predict_result {}; +struct state_flush_graph_result {}; +struct state_flush_sample_result {}; +struct state_flush_detokenize_result {}; +struct state_flush_decode_result {}; +struct state_flush_done_channel_decision {}; +struct state_flush_error_channel_decision {}; +struct state_errored {}; + +/* +generic speech generator (single source of truth) + +state purpose +- initialization deterministically initializes injected temporal memory, + encoder, decoder, predictor, conditioning, and tokenizer actors. Model-family + predictors own initialization of their injected execution actors. +- condition states advance reference and prompt conditioning one bounded frame + per public dispatch. +- state_ready accepts offline synthesis or duplex input frames. +- state_flushing advances silence frames through the same encode, predict, and + decode actors without a hidden session loop. + +control invariants +- actor and event types are compile-time injected. Generic headers contain no + model-family contracts, handler tables, route callbacks, or runtime type + selection. +- every child result is interpreted by guards and explicit transitions. +- child calls are synchronous, deterministically ordered, and remain inside + the originating RTC boundary. +- request-local status stays in typed runtime events. Context owns only the + persistent injected composition. +*/ +template struct duplex_model { + auto operator()() const { + namespace sml = stateforward::sml; + + using init_run = event::initialize_run; + using condition_run = event::condition_run; + using generate_run = event::generate_run; + using stream_run = event::stream_frame_run; + using flush_run = event::flush_run; + + using init_done_present = + guard::guard_done_callback_present; + using init_done_absent = + guard::guard_done_callback_absent; + using init_error_present = + guard::guard_error_callback_present; + using init_error_absent = + guard::guard_error_callback_absent; + using condition_error_present = + guard::guard_error_callback_present; + using condition_error_absent = + guard::guard_error_callback_absent; + using generate_error_present = + guard::guard_error_callback_present; + using generate_error_absent = + guard::guard_error_callback_absent; + using stream_done_present = + guard::guard_done_callback_present; + using stream_done_absent = + guard::guard_done_callback_absent; + using stream_error_present = + guard::guard_error_callback_present; + using stream_error_absent = + guard::guard_error_callback_absent; + using flush_done_present = + guard::guard_done_callback_present; + using flush_done_absent = + guard::guard_done_callback_absent; + using flush_error_present = + guard::guard_error_callback_present; + using flush_error_absent = + guard::guard_error_callback_absent; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Injected actor initialization. + sml::state <= *sml::state + + sml::event [ guard::guard_initialize_request_valid{} ] + / action::effect_initialize_temporal_positions{} + , sml::state <= + sml::state + sml::event + [ guard::guard_initialize_request_invalid{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_secondary_positions{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_encoder{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_decoder{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_predictor{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_conditioning{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_tokenizer{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_initialize_done{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ init_done_present{} ] / action::effect_emit_initialize_done{} + , sml::state <= + sml::state + sml::completion + [ init_done_absent{} ] + , sml::state <= + sml::state + sml::completion + [ init_error_present{} ] / action::effect_emit_initialize_error{} + , sml::state <= + sml::state + sml::completion + [ init_error_absent{} ] + + //------------------------------------------------------------------------------// + // Reference and prompt conditioning, one bounded frame per dispatch. + , sml::state <= sml::state + + sml::event / action::effect_condition_voice{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_succeeded_pending{} ] + / action::effect_publish_condition_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_succeeded_complete{} ] + / action::effect_begin_prompt_conditioning{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_failed{} ] + / action::effect_fail_condition{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_condition_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_condition{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_pending_done_callback_present{} ] + / action::effect_emit_condition_done{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_pending_done_callback_absent{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_complete_done_callback_present{} ] + / action::effect_emit_condition_done{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_complete_done_callback_absent{} ] + , sml::state <= sml::state + + sml::event / action::effect_condition_prompt{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_succeeded_pending{} ] + / action::effect_publish_condition_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_succeeded_complete{} ] + / action::effect_capture_tokenizer_state{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_condition_failed{} ] + / action::effect_fail_condition{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_restore_tokenizer_state{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_condition{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_condition_complete{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_condition{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_pending_done_callback_present{} ] + / action::effect_emit_condition_done{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_pending_done_callback_absent{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_complete_done_callback_present{} ] + / action::effect_emit_condition_done{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_condition_complete_done_callback_absent{} ] + , sml::state <= + sml::state + + sml::completion [ condition_error_present{} ] + / action::effect_emit_condition_error{} + , sml::state <= + sml::state + + sml::completion [ condition_error_absent{} ] + + //------------------------------------------------------------------------------// + // Duplex generators expose streaming requests; offline synthesis is a + // distinct, explicitly unsupported request shape for this composition. + , sml::state <= + sml::state + sml::event + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion [ generate_error_present{} ] + / action::effect_emit_generation_error{} + , sml::state <= sml::state + + sml::completion [ generate_error_absent{} ] + + //------------------------------------------------------------------------------// + // Duplex encode -> tokenize -> plan -> predict -> graph -> sample -> detokenize -> decode. + , sml::state <= sml::state + + sml::event [ guard::guard_stream_request_valid{} ] + / action::effect_encode_stream_frame{} + , sml::state <= sml::state + + sml::event [ guard::guard_stream_request_invalid{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_tokenize_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_plan_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_predict_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_execute_prediction_graph{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_sample_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_detokenize_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_frame_produced{} ] + / action::effect_decode_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_frame_pending{} ] + / action::effect_publish_stream_frame_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_frame_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_stream_frame_produced{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ stream_done_present{} ] + / action::effect_emit_stream_frame_done{} + , sml::state <= sml::state + + sml::completion [ stream_done_absent{} ] + , sml::state <= sml::state + + sml::completion [ stream_error_present{} ] + / action::effect_emit_stream_frame_error{} + , sml::state <= sml::state + + sml::completion [ stream_error_absent{} ] + + //------------------------------------------------------------------------------// + // Silence flush uses the same actor pipeline without a hidden loop. + , sml::state <= sml::state + + sml::event [ guard::guard_flush_request_valid{} ] + / action::effect_encode_flush_frame{} + , sml::state <= sml::state + + sml::event [ guard::guard_flush_request_valid{} ] + / action::effect_encode_flush_frame{} + , sml::state <= sml::state + + sml::event [ guard::guard_flush_request_invalid{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::event [ guard::guard_flush_request_invalid{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_tokenize_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_plan_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_predict_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_execute_prediction_graph{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_sample_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_detokenize_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_prediction_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ guard::guard_frame_produced{} ] + / action::effect_decode_frame{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_frame_pending{} ] + / action::effect_publish_flush_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_frame_failed{} ] + / action::effect_fail_flush{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_flush_produced{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ flush_done_present{} ] + / action::effect_emit_flush_done{} + , sml::state <= sml::state + + sml::completion [ flush_done_absent{} ] + , sml::state <= sml::state + + sml::completion [ flush_error_present{} ] + / action::effect_emit_flush_error{} + , sml::state <= sml::state + + sml::completion [ flush_error_absent{} ] + + //------------------------------------------------------------------------------// + // Known invalid lifecycle requests and unexpected external events. + , sml::state <= sml::state + + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + ); + // clang-format on + } +}; + +/* +generic offline synthesis composition + +- the dependency mode selects this graph at compile time; there is no runtime + route, callback table, or model-family test. +- each synthesis stage is a separately injected actor. Child acceptance and + error results are interpreted only by guards and transitions. +- condition, stream_frame, and flush remain distinct public requests and are + explicitly rejected by a synthesis-only composition. +*/ +template struct synthesis_model { + auto operator()() const { + namespace sml = stateforward::sml; + + using init_run = event::initialize_run; + using condition_run = event::condition_run; + using generate_run = event::generate_run; + using stream_run = event::stream_frame_run; + using flush_run = event::flush_run; + using init_done_present = + guard::guard_done_callback_present; + using init_done_absent = + guard::guard_done_callback_absent; + using init_error_present = + guard::guard_error_callback_present; + using init_error_absent = + guard::guard_error_callback_absent; + using generate_done_present = + guard::guard_done_callback_present; + using generate_done_absent = + guard::guard_done_callback_absent; + using generate_error_present = + guard::guard_error_callback_present; + using generate_error_absent = + guard::guard_error_callback_absent; + using condition_error_present = + guard::guard_error_callback_present; + using condition_error_absent = + guard::guard_error_callback_absent; + using stream_error_present = + guard::guard_error_callback_present; + using stream_error_absent = + guard::guard_error_callback_absent; + using flush_error_present = + guard::guard_error_callback_present; + using flush_error_absent = + guard::guard_error_callback_absent; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Injected offline-synthesis actor initialization. + sml::state <= + *sml::state + sml::event + / action::effect_initialize_synthesis_conditioner{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_synthesis_prefiller{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_synthesis_predictor{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_synthesis_sampler{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_synthesis_decoder{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_initialize_synthesis_postprocessor{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_initialize_done{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_initialize{} + , sml::state <= + sml::state + sml::completion + [ init_done_present{} ] + / action::effect_emit_synthesis_initialize_done{} + , sml::state <= + sml::state + sml::completion + [ init_done_absent{} ] + , sml::state <= + sml::state + sml::completion + [ init_error_present{} ] / action::effect_emit_initialize_error{} + , sml::state <= + sml::state + sml::completion + [ init_error_absent{} ] + + //------------------------------------------------------------------------------// + // Offline condition -> prefill -> predict -> sample -> decode -> postprocess. + , sml::state <= sml::state + + sml::event [ guard::guard_generate_request_valid{} ] + / action::effect_condition_generate{} + , sml::state <= sml::state + + sml::event [ guard::guard_generate_request_invalid{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_prefill_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_predict_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_sample_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_decode_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_postprocess_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_succeeded{} ] + / action::effect_publish_generation_done{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_child_failed{} ] + / action::effect_fail_generate{} + , sml::state <= sml::state + + sml::completion [ generate_done_present{} ] + / action::effect_emit_generation_done{} + , sml::state <= sml::state + + sml::completion [ generate_done_absent{} ] + , sml::state <= sml::state + + sml::completion [ generate_error_present{} ] + / action::effect_emit_generation_error{} + , sml::state <= sml::state + + sml::completion [ generate_error_absent{} ] + + //------------------------------------------------------------------------------// + // Public request shapes not implemented by a synthesis-only composition. + , sml::state <= sml::state + + sml::event + / action::effect_fail_condition{} + , sml::state <= sml::state + + sml::completion [ condition_error_present{} ] + / action::effect_emit_condition_error{} + , sml::state <= sml::state + + sml::completion [ condition_error_absent{} ] + , sml::state <= sml::state + + sml::event + / action::effect_fail_stream_frame{} + , sml::state <= sml::state + + sml::completion [ stream_error_present{} ] + / action::effect_emit_stream_frame_error{} + , sml::state <= sml::state + + sml::completion [ stream_error_absent{} ] + , sml::state <= sml::state + + sml::event + / action::effect_fail_flush{} + , sml::state <= sml::state + + sml::completion [ flush_error_present{} ] + / action::effect_emit_flush_error{} + , sml::state <= sml::state + + sml::completion [ flush_error_absent{} ] + + //------------------------------------------------------------------------------// + // Lifecycle errors remain explicit. + , sml::state <= sml::state + + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + sml::event + / action::effect_reject_reset{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + ); + // clang-format on + } +}; + +template +struct model; + +template +struct model + : duplex_model {}; + +template +struct model + : synthesis_model {}; + +template +struct sm : public emel::sm, + action::context> { + using base_type = + emel::sm, action::context>; + using base_type::is; + using base_type::process_event; + using base_type::visit_current_states; + + explicit sm(const dependencies_type &deps) : base_type(std::in_place, deps) {} + + bool process_event(const event::initialize &ev) { + event::initialize_ctx ctx{}; + const bool accepted = + base_type::process_event(event::initialize_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::condition &ev) { + event::condition_ctx ctx{}; + const bool accepted = + base_type::process_event(event::condition_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::generate &ev) { + event::generate_ctx ctx{}; + const bool accepted = + base_type::process_event(event::generate_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::stream_frame &ev) { + event::frame_ctx ctx{}; + const bool accepted = + base_type::process_event(event::stream_frame_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::flush &ev) { + event::flush_ctx ctx{}; + const bool accepted = base_type::process_event(event::flush_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::reset &ev) { + const bool accepted = base_type::process_event(ev); + return accepted && ev.error_out == action::error_code(error::none); + } +}; + +template +sm(const dependencies_type &) -> sm; + +} // namespace emel::speech::generator diff --git a/src/emel/speech/predictor/moshi/actions.hpp b/src/emel/speech/predictor/moshi/actions.hpp new file mode 100644 index 00000000..3a77895c --- /dev/null +++ b/src/emel/speech/predictor/moshi/actions.hpp @@ -0,0 +1,939 @@ +#pragma once + +#include +#include +#include +#include + +#include "emel/kernel/detail.hpp" +#include "emel/memory/events.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/context.hpp" +#include "emel/speech/predictor/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/events.hpp" + +namespace emel::speech::predictor::moshi::action { + +namespace detail_ns { + +inline emel::error::type to_error(const error value) noexcept { + return emel::error::cast(value); +} + +} // namespace detail_ns + +template struct effect_mark_not_initialized { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::not_initialized); + } +}; + +template +struct effect_mark_not_initialized_and_store { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::not_initialized); + *runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +struct effect_mark_bind_failed { + void operator()(const event::initialize_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::bind_failed); + } +}; + +template struct effect_mark_memory_error { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::memory); + } +}; + +struct effect_mark_step_request_invalid { + template + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + ev.ctx.err = detail_ns::to_error(error::request_shape); + } +}; + +struct effect_mark_step_request_invalid_and_store { + template + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + ev.ctx.err = detail_ns::to_error(error::request_shape); + *ev.request.error_out = ev.ctx.err; + } +}; + +template struct effect_mark_graph_runtime_error { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::graph_runtime); + } +}; + +template struct effect_mark_voice_contract_error { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::voice_contract); + } +}; + +template +struct effect_mark_personaplex_prompt_error { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::personaplex_prompt); + } +}; + +struct effect_mark_voice_prompt_pending { + template + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + ev.ctx.err = detail_ns::to_error(error::voice_prompt_pending); + } +}; + +struct effect_bind_contract { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.session = {}; + ctx.session.model = &runtime_ev.request.model; + ctx.session.sequence_id = runtime_ev.request.sequence_id; + ctx.session.n_q = runtime_ev.request.model.moshi_lm.n_q; + ctx.session.dep_q = runtime_ev.request.model.moshi_lm.dep_q; + ctx.session.text_card = runtime_ev.request.model.moshi_lm.text_card; + ctx.session.audio_card = runtime_ev.request.model.moshi_lm.card; + (void)emel::model::moshi::detail::build_execution_contract( + runtime_ev.request.model, ctx.session.contract); + + const auto &lm = runtime_ev.request.model.moshi_lm; + ctx.lmgen = {}; + ctx.lmgen.codebook_count = lm.n_q + 1; + ctx.lmgen.generated_dep_q = lm.dep_q; + ctx.lmgen.cache_row_count = 2; + std::fill(ctx.lmgen.cache.begin(), ctx.lmgen.cache.end(), + ctx.policy.token_ungenerated); + std::fill(ctx.lmgen.initial.begin(), ctx.lmgen.initial.end(), lm.card); + ctx.lmgen.initial[0] = lm.text_card; + for (uint32_t index = 0; index < lm.delay_count; ++index) { + ctx.lmgen.delays[index] = lm.delays[index]; + ctx.lmgen.max_delay = std::max(ctx.lmgen.max_delay, lm.delays[index]); + } + ctx.lmgen.cache_row_count = ctx.lmgen.max_delay + 2 + ctx.lmgen.delay_steps; + } +}; + +struct effect_configure_standard_lmgen { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + ctx.lmgen.delayed_dep_q = ctx.lmgen.generated_dep_q; + ctx.lmgen.needed_tokens = + ctx.lmgen.codebook_count - ctx.lmgen.delayed_dep_q - 1; + } +}; + +template struct effect_initialize_graph { + void operator()(const event::initialize_run &runtime_ev, context &, + graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + event::initialize_graph request{runtime_ev.request}; + request.error_out = &runtime_ev.ctx.graph_error; + runtime_ev.ctx.graph_accepted = graph.process_event(request); + } +}; + +struct effect_configure_personaplex_lmgen { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + ctx.lmgen.delayed_dep_q = ctx.session.model->moshi_lm.inference_dep_q; + ctx.lmgen.needed_tokens = + ctx.lmgen.codebook_count - ctx.lmgen.delayed_dep_q - 1; + ctx.lmgen.cache_row_count = ctx.lmgen.max_delay + 3; + } +}; + +struct effect_bind_voice_contract { + void operator()(const event::load_voice_run &runtime_ev, + context &ctx) const noexcept { + ctx.voice = {}; + ctx.voice.model = &runtime_ev.request.voice_model; + (void)emel::model::moshi::detail::build_execution_contract( + runtime_ev.request.voice_model, ctx.voice.contract); + const auto *embeddings = ctx.voice.contract.voice.embeddings.tensor; + const auto *cache = ctx.voice.contract.voice.cache.tensor; + ctx.voice.embedding_dim = static_cast(embeddings->dims[0]); + ctx.voice.prompt_frame_count = static_cast(embeddings->dims[3]); + ctx.voice.prompt_frame_index = 0; + ctx.voice.cache_row_count = static_cast(cache->dims[0]); + ctx.voice.cache_column_count = static_cast(cache->dims[1]); + ctx.voice.loaded = true; + ctx.voice.ready = false; + } +}; + +struct effect_reserve_memory { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::reserve{ + .max_sequences = runtime_ev.request.max_sequences, + .max_blocks = runtime_ev.request.max_blocks, + .block_tokens = runtime_ev.request.block_tokens, + .error_out = &runtime_ev.ctx.memory_error, + }); + } +}; + +struct effect_allocate_sequence { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::allocate_sequence{ + .seq_id = runtime_ev.request.sequence_id, + .error_out = &runtime_ev.ctx.memory_error, + }); + } +}; + +struct effect_allocate_step_slot { + void operator()(const event::predict_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_block_count = 0; + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::allocate_slots{ + .seq_id = ctx.session.sequence_id, + .token_count = runtime_ev.request.planned_step_size, + .block_count_out = &runtime_ev.ctx.memory_block_count, + .error_out = &runtime_ev.ctx.memory_error, + .copy_block = nullptr, + .copy_block_user_data = nullptr, + }); + } +}; + +struct effect_capture_memory { + template + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::capture_view{ + .snapshot_out = &ev.ctx.memory_snapshot, + .error_out = &ev.ctx.memory_error, + }); + } +}; + +struct effect_allocate_voice_prefill_slot { + void operator()(const event::prefill_voice_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_block_count = 0; + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::allocate_slots{ + .seq_id = ctx.session.sequence_id, + .token_count = 1, + .block_count_out = &runtime_ev.ctx.memory_block_count, + .error_out = &runtime_ev.ctx.memory_error, + .copy_block = nullptr, + .copy_block_user_data = nullptr, + }); + } +}; + +struct effect_capture_voice_prefill_memory { + void operator()(const event::prefill_voice_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::capture_view{ + .snapshot_out = &runtime_ev.ctx.memory_snapshot, + .error_out = &runtime_ev.ctx.memory_error, + }); + } +}; + +struct effect_begin_voice_prefill { + void operator()(const event::prefill_voice_run &runtime_ev, + const context &ctx) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_accepted = false; + runtime_ev.ctx.embedding_frame_ok = false; + runtime_ev.ctx.complete = ctx.voice.ready; + runtime_ev.ctx.remaining_frames = + ctx.voice.prompt_frame_count - ctx.voice.prompt_frame_index; + runtime_ev.ctx.text_token = ctx.session.model->moshi_lm.text_padding_id; + std::fill(runtime_ev.ctx.input_sequence.begin(), + runtime_ev.ctx.input_sequence.end(), ctx.policy.token_zero); + runtime_ev.ctx.input_sequence[0] = + ctx.session.model->moshi_lm.text_padding_id; + std::fill(runtime_ev.ctx.audio_tokens.begin(), + runtime_ev.ctx.audio_tokens.end(), ctx.policy.token_zero); + } +}; + +struct effect_load_voice_embedding_frame_f32 { + void operator()(const event::prefill_voice_run &runtime_ev, + const context &ctx) const noexcept { + const auto *tensor = ctx.voice.contract.voice.embeddings.tensor; + const auto *src = static_cast(tensor->data); + const int32_t dim = ctx.voice.embedding_dim; + const size_t offset = static_cast(ctx.voice.prompt_frame_index) * + static_cast(dim); + for (int32_t index = 0; index < dim; ++index) { + runtime_ev.ctx.embedding_frame[static_cast(index)] = + src[offset + static_cast(index)]; + } + runtime_ev.ctx.embedding_frame_ok = true; + } +}; + +struct effect_load_voice_embedding_frame_f16 { + void operator()(const event::prefill_voice_run &runtime_ev, + const context &ctx) const noexcept { + const auto *tensor = ctx.voice.contract.voice.embeddings.tensor; + const auto *src = static_cast(tensor->data); + const int32_t dim = ctx.voice.embedding_dim; + const size_t offset = static_cast(ctx.voice.prompt_frame_index) * + static_cast(dim); + for (int32_t index = 0; index < dim; ++index) { + runtime_ev.ctx.embedding_frame[static_cast(index)] = + emel::kernel::detail::quant::fp16_to_fp32( + src[offset + static_cast(index)]); + } + runtime_ev.ctx.embedding_frame_ok = true; + } +}; + +struct effect_load_voice_embedding_frame_bf16 { + void operator()(const event::prefill_voice_run &runtime_ev, + const context &ctx) const noexcept { + const auto *tensor = ctx.voice.contract.voice.embeddings.tensor; + const auto *src = static_cast(tensor->data); + const int32_t dim = ctx.voice.embedding_dim; + const size_t offset = static_cast(ctx.voice.prompt_frame_index) * + static_cast(dim); + for (int32_t index = 0; index < dim; ++index) { + runtime_ev.ctx.embedding_frame[static_cast(index)] = + emel::kernel::detail::bf16_to_fp32( + src[offset + static_cast(index)]); + } + runtime_ev.ctx.embedding_frame_ok = true; + } +}; + +template struct effect_run_voice_graph_runtime { + void operator()(const event::prefill_voice_run &runtime_ev, context &ctx, + graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + event::graph_step graph_step{ + *ctx.session.model, + runtime_ev.ctx.memory_snapshot, + std::span{runtime_ev.ctx.input_sequence.data(), + static_cast(ctx.lmgen.codebook_count)}, + std::span{runtime_ev.ctx.audio_tokens.data(), + static_cast(ctx.lmgen.generated_dep_q)}, + runtime_ev.ctx.text_token, + }; + graph_step.sequence_id = ctx.session.sequence_id; + graph_step.input_embedding = + std::span{runtime_ev.ctx.embedding_frame.data(), + static_cast(ctx.voice.embedding_dim)}; + graph_step.forced_text_token = ctx.session.model->moshi_lm.text_padding_id; + graph_step.error_out = &runtime_ev.ctx.graph_error; + runtime_ev.ctx.graph_accepted = graph.process_event(graph_step); + } +}; + +struct effect_advance_voice_prefill { + void operator()(const event::prefill_voice_run &runtime_ev, + context &ctx) const noexcept { + ++ctx.voice.prompt_frame_index; + ++ctx.lmgen.offset; + runtime_ev.ctx.remaining_frames = + ctx.voice.prompt_frame_count - ctx.voice.prompt_frame_index; + runtime_ev.ctx.complete = + ctx.voice.prompt_frame_index >= ctx.voice.prompt_frame_count; + } +}; + +struct effect_copy_voice_cache { + void operator()(const event::prefill_voice_run &runtime_ev, + context &ctx) const noexcept { + const auto *tensor = ctx.voice.contract.voice.cache.tensor; + const auto *src = static_cast(tensor->data); + for (int32_t row = 0; row < ctx.voice.cache_row_count; ++row) { + for (int32_t column = 0; column < ctx.voice.cache_column_count; + ++column) { + const size_t source_index = + static_cast(row) + + static_cast(column) * + static_cast(ctx.voice.cache_row_count); + detail::cache_at(ctx.lmgen, row, column) = src[source_index]; + } + } + ctx.voice.ready = true; + runtime_ev.ctx.complete = true; + runtime_ev.ctx.remaining_frames = 0; + } +}; + +struct effect_bind_personaplex_prompt { + void operator()(const event::begin_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + ctx.voice.pre_text_silence_remaining = + runtime_ev.request.pre_text_silence_frames; + ctx.voice.text_tokens_remaining = runtime_ev.request.text_token_count; + ctx.voice.post_text_silence_remaining = + runtime_ev.request.post_text_silence_frames; + ctx.voice.prompt_started = true; + ctx.voice.prompt_ready = false; + runtime_ev.ctx.remaining_frames = ctx.voice.pre_text_silence_remaining + + ctx.voice.text_tokens_remaining + + ctx.voice.post_text_silence_remaining; + } +}; + +struct effect_bind_empty_personaplex_prompt { + void operator()(const event::begin_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = ctx.session.model->moshi_lm; + ctx.voice.pre_text_silence_remaining = lm.inference_pre_text_silence_frames; + ctx.voice.text_tokens_remaining = 0; + ctx.voice.post_text_silence_remaining = + lm.inference_post_text_silence_frames; + ctx.voice.prompt_started = true; + ctx.voice.prompt_ready = false; + runtime_ev.ctx.remaining_frames = ctx.voice.pre_text_silence_remaining + + ctx.voice.post_text_silence_remaining; + } +}; + +struct effect_allocate_personaplex_prompt_slot { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_block_count = 0; + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::allocate_slots{ + .seq_id = ctx.session.sequence_id, + .token_count = 1, + .block_count_out = &runtime_ev.ctx.memory_block_count, + .error_out = &runtime_ev.ctx.memory_error, + .copy_block = nullptr, + .copy_block_user_data = nullptr, + }); + } +}; + +struct effect_capture_personaplex_prompt_memory { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::capture_view{ + .snapshot_out = &runtime_ev.ctx.memory_snapshot, + .error_out = &runtime_ev.ctx.memory_error, + }); + } +}; + +struct effect_begin_personaplex_prompt_prefill { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const context &ctx) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_accepted = false; + runtime_ev.ctx.complete = false; + runtime_ev.ctx.remaining_frames = ctx.voice.pre_text_silence_remaining + + ctx.voice.text_tokens_remaining + + ctx.voice.post_text_silence_remaining; + runtime_ev.ctx.text_token = ctx.session.model->moshi_lm.text_padding_id; + std::fill(runtime_ev.ctx.prompt_sequence.begin(), + runtime_ev.ctx.prompt_sequence.end(), ctx.policy.token_zero); + std::fill(runtime_ev.ctx.input_sequence.begin(), + runtime_ev.ctx.input_sequence.end(), ctx.policy.token_zero); + std::fill(runtime_ev.ctx.audio_tokens.begin(), + runtime_ev.ctx.audio_tokens.end(), ctx.policy.token_zero); + } +}; + +struct effect_build_personaplex_prompt_silence_frame { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const context &ctx) const noexcept { + for (int32_t index = 0; index < ctx.lmgen.codebook_count; ++index) { + runtime_ev.ctx.prompt_sequence[static_cast(index)] = + ctx.session.model->moshi_lm + .inference_prompt_tokens[static_cast(index)]; + } + } +}; + +struct effect_build_personaplex_prompt_text_frame { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const context &ctx) const noexcept { + effect_build_personaplex_prompt_silence_frame{}(runtime_ev, ctx); + runtime_ev.ctx.prompt_sequence[0] = runtime_ev.request.text_token; + } +}; + +struct effect_write_and_build_personaplex_prompt_input { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + for (int32_t index = 0; index < ctx.lmgen.codebook_count; ++index) { + const int32_t write_row = detail::cache_position( + ctx.lmgen, ctx.lmgen.offset + ctx.lmgen.delays[index]); + detail::cache_at(ctx.lmgen, write_row, index) = + runtime_ev.ctx.prompt_sequence[static_cast(index)]; + } + + const int32_t read_row = + detail::cache_position(ctx.lmgen, ctx.lmgen.offset); + for (int32_t index = 0; index < ctx.lmgen.codebook_count; ++index) { + const bool use_initial = ctx.lmgen.offset <= ctx.lmgen.delays[index]; + const int32_t initial_lane = static_cast(use_initial); + const int32_t cache_lane = static_cast(!use_initial); + runtime_ev.ctx.input_sequence[static_cast(index)] = + (initial_lane * ctx.lmgen.initial[index]) + + (cache_lane * detail::cache_at(ctx.lmgen, read_row, index)); + } + } +}; + +template +struct effect_run_personaplex_prompt_graph_runtime { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &ctx, graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + event::graph_step graph_step{ + *ctx.session.model, + runtime_ev.ctx.memory_snapshot, + std::span{runtime_ev.ctx.input_sequence.data(), + static_cast(ctx.lmgen.codebook_count)}, + std::span{runtime_ev.ctx.audio_tokens.data(), + static_cast(ctx.lmgen.generated_dep_q)}, + runtime_ev.ctx.text_token, + }; + graph_step.sequence_id = ctx.session.sequence_id; + graph_step.error_out = &runtime_ev.ctx.graph_error; + runtime_ev.ctx.graph_accepted = graph.process_event(graph_step); + } +}; + +struct effect_advance_personaplex_prompt_pre_silence { + void operator()(const event::prefill_personaplex_prompt_run &, + context &ctx) const noexcept { + --ctx.voice.pre_text_silence_remaining; + ++ctx.lmgen.offset; + } +}; + +struct effect_advance_personaplex_prompt_text { + void operator()(const event::prefill_personaplex_prompt_run &, + context &ctx) const noexcept { + --ctx.voice.text_tokens_remaining; + ++ctx.lmgen.offset; + } +}; + +struct effect_advance_personaplex_prompt_post_silence { + void operator()(const event::prefill_personaplex_prompt_run &, + context &ctx) const noexcept { + --ctx.voice.post_text_silence_remaining; + ++ctx.lmgen.offset; + } +}; + +struct effect_finish_personaplex_prompt { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &ctx) const noexcept { + ctx.voice.prompt_ready = true; + runtime_ev.ctx.complete = true; + runtime_ev.ctx.remaining_frames = 0; + } +}; + +struct effect_publish_personaplex_prompt_pending { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const context &ctx) const noexcept { + runtime_ev.ctx.complete = false; + runtime_ev.ctx.remaining_frames = ctx.voice.pre_text_silence_remaining + + ctx.voice.text_tokens_remaining + + ctx.voice.post_text_silence_remaining; + } +}; + +struct effect_store_personaplex_prompt_complete_out { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.complete_out = runtime_ev.ctx.complete; + } +}; + +struct effect_store_personaplex_prompt_remaining_out { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.remaining_frames_out = runtime_ev.ctx.remaining_frames; + } +}; + +struct effect_store_voice_complete_out { + void operator()(const event::prefill_voice_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.complete_out = runtime_ev.ctx.complete; + } +}; + +struct effect_store_voice_remaining_out { + void operator()(const event::prefill_voice_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.remaining_frames_out = runtime_ev.ctx.remaining_frames; + } +}; + +struct effect_begin_predict { + void operator()(const event::predict_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + } +}; + +struct effect_publish_predict { + void operator()(const event::predict_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + } +}; + +struct effect_begin_execute { + void operator()(const event::execute_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_accepted = false; + runtime_ev.request.prediction_workspace.sampled_text_token = -1; + std::fill( + runtime_ev.request.prediction_workspace.sampled_audio_tokens.begin(), + runtime_ev.request.prediction_workspace.sampled_audio_tokens.end(), -1); + } +}; + +template struct effect_run_prediction_graph { + void operator()(const event::execute_run &runtime_ev, context &ctx, + graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + event::graph_step graph_step{ + *ctx.session.model, + runtime_ev.request.prediction_workspace.memory, + runtime_ev.request.model_tokens, + std::span{ + runtime_ev.request.prediction_workspace.sampled_audio_tokens.data(), + static_cast(ctx.lmgen.generated_dep_q)}, + runtime_ev.request.prediction_workspace.sampled_text_token, + }; + graph_step.sequence_id = ctx.session.sequence_id; + graph_step.phase = event::graph_phase::prediction; + graph_step.temporal_state = + runtime_ev.request.prediction_workspace.temporal_state; + graph_step.error_out = &runtime_ev.ctx.graph_error; + runtime_ev.ctx.graph_accepted = graph.process_event(graph_step); + } +}; + +struct effect_publish_execute { + void operator()(const event::execute_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + } +}; + +struct effect_store_execute_graph_error_out { + void operator()(const event::execute_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.graph_error_out = runtime_ev.ctx.graph_error; + } +}; + +struct effect_begin_sample { + void operator()(const event::sample_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_accepted = false; + } +}; + +template struct effect_run_sampling_graph { + void operator()(const event::sample_run &runtime_ev, context &ctx, + graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + event::graph_step graph_step{ + *ctx.session.model, + runtime_ev.request.prediction_workspace.memory, + runtime_ev.request.model_tokens, + std::span{ + runtime_ev.request.prediction_workspace.sampled_audio_tokens.data(), + static_cast(ctx.lmgen.generated_dep_q)}, + runtime_ev.request.prediction_workspace.sampled_text_token, + }; + graph_step.sequence_id = ctx.session.sequence_id; + graph_step.phase = event::graph_phase::sampling; + graph_step.temporal_state = + runtime_ev.request.prediction_workspace.temporal_state; + graph_step.error_out = &runtime_ev.ctx.graph_error; + runtime_ev.ctx.graph_accepted = graph.process_event(graph_step); + } +}; + +struct effect_store_sample_graph_error_out { + void operator()(const event::sample_run &runtime_ev, + context &) const noexcept { + *runtime_ev.request.graph_error_out = runtime_ev.ctx.graph_error; + } +}; + +struct effect_publish_sample { + void operator()(const event::sample_run &runtime_ev, + const context &ctx) const noexcept { + std::copy_n( + runtime_ev.request.prediction_workspace.sampled_audio_tokens.begin(), + static_cast(ctx.lmgen.generated_dep_q), + runtime_ev.request.audio_tokens_out.begin()); + runtime_ev.request.text_token_out = + runtime_ev.request.prediction_workspace.sampled_text_token; + runtime_ev.ctx.err = detail_ns::to_error(error::none); + } +}; + +struct effect_capture_tokenizer_state { + void operator()(const event::capture_tokenizer_state &ev, + const context &ctx) const noexcept { + for (int32_t codebook = 0; codebook < ctx.lmgen.codebook_count; + ++codebook) { + for (int32_t row = 0; row < ctx.lmgen.cache_row_count; ++row) { + const size_t destination = + static_cast(row) + + static_cast(codebook) * + static_cast(ctx.lmgen.cache_row_count); + ev.cache_out[destination] = detail::cache_at(ctx.lmgen, row, codebook); + } + } + ev.offset_out = ctx.lmgen.offset; + ev.error_out = detail_ns::to_error(error::none); + } +}; + +template struct effect_reject_capture_tokenizer_state { + void operator()(const event::capture_tokenizer_state &ev, + const context &) const noexcept { + ev.offset_out = 0; + ev.error_out = detail_ns::to_error(error_value); + } +}; + +struct effect_store_graph_error_out { + template + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + *ev.request.graph_error_out = ev.ctx.graph_error; + } +}; + +struct effect_reset_session { + void operator()(const event::reset_run &, context &ctx) const noexcept { + ctx.session = {}; + ctx.voice = {}; + ctx.lmgen = {}; + } +}; + +template struct effect_reset_graph { + void operator()(const event::reset_run &runtime_ev, context &, + graph_actor_type &graph) const noexcept { + runtime_ev.ctx.graph_error = detail_ns::to_error(error::none); + runtime_ev.ctx.graph_accepted = graph.process_event(runtime_ev.request); + } +}; + +struct effect_release_reset_sequence { + void operator()(const event::reset_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.memory_error = static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + runtime_ev.ctx.memory_accepted = + ctx.memory.process_event(emel::memory::event::free_sequence{ + .seq_id = ctx.session.sequence_id, + .error_out = &runtime_ev.ctx.memory_error, + }); + } +}; + +struct effect_mark_reset_graph_failed { + void operator()(const event::reset_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::graph_runtime); + } +}; + +struct effect_mark_reset_memory_failed { + void operator()(const event::reset_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::memory); + } +}; + +template struct effect_store_error_out { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + *runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +struct effect_emit_initialize_done { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.request.on_done(events::initialize_done{ + .request = &runtime_ev.request, + .n_q = ctx.session.n_q, + .dep_q = ctx.session.dep_q, + }); + } +}; + +struct effect_emit_initialize_error { + void operator()(const event::initialize_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::initialize_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_emit_load_voice_done { + void operator()(const event::load_voice_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.request.on_done(events::load_voice_done{ + .request = &runtime_ev.request, + .prompt_frames = ctx.voice.prompt_frame_count, + }); + } +}; + +struct effect_emit_load_voice_error { + void operator()(const event::load_voice_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::load_voice_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_emit_prefill_voice_done { + void operator()(const event::prefill_voice_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_done(events::prefill_voice_done{ + .request = &runtime_ev.request, + .complete = runtime_ev.ctx.complete, + .remaining_frames = runtime_ev.ctx.remaining_frames, + }); + } +}; + +struct effect_emit_prefill_voice_error { + void operator()(const event::prefill_voice_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::prefill_voice_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_emit_begin_personaplex_prompt_done { + void operator()(const event::begin_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_done(events::begin_personaplex_prompt_done{ + .request = &runtime_ev.request, + .remaining_frames = runtime_ev.ctx.remaining_frames, + }); + } +}; + +struct effect_emit_begin_personaplex_prompt_error { + void operator()(const event::begin_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::begin_personaplex_prompt_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_emit_prefill_personaplex_prompt_done { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_done(events::prefill_personaplex_prompt_done{ + .request = &runtime_ev.request, + .complete = runtime_ev.ctx.complete, + .remaining_frames = runtime_ev.ctx.remaining_frames, + }); + } +}; + +struct effect_emit_prefill_personaplex_prompt_error { + void operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::prefill_personaplex_prompt_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_mark_unexpected { + template + void operator()(const unexpected_event_type &ev, context &) const noexcept { + if constexpr (requires { ev.ctx.err; }) { + ev.ctx.err = detail_ns::to_error(error::unexpected_event); + } + } +}; + +struct effect_mark_unexpected_and_store { + template + void operator()(const unexpected_event_type &ev, + context &ctx) const noexcept { + effect_mark_unexpected{}(ev, ctx); + if constexpr (requires { + ev.ctx.err; + ev.request.error_out; + }) { + *ev.request.error_out = ev.ctx.err; + } + } +}; + +} // namespace emel::speech::predictor::moshi::action diff --git a/src/emel/speech/predictor/moshi/any.hpp b/src/emel/speech/predictor/moshi/any.hpp new file mode 100644 index 00000000..eb3d7174 --- /dev/null +++ b/src/emel/speech/predictor/moshi/any.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "emel/speech/predictor/moshi/sm.hpp" + +namespace emel::speech::predictor::moshi { + +using policies = action::policies; + +template +using dependencies = action::dependencies; + +} // namespace emel::speech::predictor::moshi diff --git a/src/emel/speech/predictor/moshi/context.hpp b/src/emel/speech/predictor/moshi/context.hpp new file mode 100644 index 00000000..8e1552d7 --- /dev/null +++ b/src/emel/speech/predictor/moshi/context.hpp @@ -0,0 +1,88 @@ +#pragma once + +#include +#include + +#include "emel/memory/hybrid/sm.hpp" +#include "emel/model/data.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/events.hpp" + +namespace emel::speech::predictor::moshi::action { + +inline constexpr int32_t k_max_codebooks = + static_cast(event::k_max_codebooks); +inline constexpr int32_t k_max_delay_rows = 128; + +struct policies { + int32_t token_zero = 0; + int32_t token_ungenerated = 0; + int32_t prediction_step_size = 0; + int32_t prediction_output_count = 0; +}; + +struct lmgen_state { + int32_t codebook_count = 0; + int32_t generated_dep_q = 0; + int32_t delayed_dep_q = 0; + int32_t needed_tokens = 0; + int32_t max_delay = 0; + int32_t delay_steps = 0; + int32_t cache_row_count = 0; + int64_t offset = 0; + int32_t skip = 0; + std::array delays = {}; + std::array initial = {}; + std::array cache = {}; +}; + +struct runtime { + const emel::model::data *model = nullptr; + emel::model::moshi::detail::execution_contract contract = {}; + int32_t n_q = 0; + int32_t dep_q = 0; + int32_t text_card = 0; + int32_t audio_card = 0; + int32_t sequence_id = 0; +}; + +struct voice_prompt_state { + const emel::model::data *model = nullptr; + emel::model::moshi::detail::execution_contract contract = {}; + int32_t embedding_dim = 0; + int32_t prompt_frame_count = 0; + int32_t prompt_frame_index = 0; + int32_t cache_row_count = 0; + int32_t cache_column_count = 0; + int32_t pre_text_silence_remaining = 0; + int32_t text_tokens_remaining = 0; + int32_t post_text_silence_remaining = 0; + bool loaded = false; + bool ready = false; + bool prompt_started = false; + bool prompt_ready = false; +}; + +template struct dependencies { + emel::memory::hybrid::sm &memory; + graph_actor_type &graph; + policies policy = {}; +}; + +struct context { + explicit context(emel::memory::hybrid::sm &memory_ref, + const policies policy_ref = {}) noexcept + : memory(memory_ref), policy(policy_ref) {} + + template + explicit context(const dependencies &deps) + : context(deps.memory, deps.policy) {} + + runtime session = {}; + voice_prompt_state voice = {}; + lmgen_state lmgen = {}; + emel::memory::hybrid::sm &memory; + const policies policy = {}; +}; + +} // namespace emel::speech::predictor::moshi::action diff --git a/src/emel/speech/predictor/moshi/detail.hpp b/src/emel/speech/predictor/moshi/detail.hpp new file mode 100644 index 00000000..c7f0b30d --- /dev/null +++ b/src/emel/speech/predictor/moshi/detail.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include "emel/speech/predictor/moshi/context.hpp" + +namespace emel::speech::predictor::moshi::detail { + +template +constexpr decltype(auto) +unwrap_runtime_event(const runtime_event_type &ev) noexcept { + if constexpr (requires { ev.event_; }) { + return (ev.event_); + } else { + return (ev); + } +} + +inline int32_t cache_position(const action::lmgen_state &lmgen, + const int64_t offset) noexcept { + const int64_t row_count = lmgen.cache_row_count; + const int64_t position = offset % row_count; + return static_cast(position + ((position < 0) * row_count)); +} + +inline int32_t &cache_at(action::lmgen_state &lmgen, const int32_t row, + const int32_t column) noexcept { + return lmgen + .cache[static_cast(row * action::k_max_codebooks + column)]; +} + +inline int32_t cache_at(const action::lmgen_state &lmgen, const int32_t row, + const int32_t column) noexcept { + return lmgen + .cache[static_cast(row * action::k_max_codebooks + column)]; +} + +} // namespace emel::speech::predictor::moshi::detail diff --git a/src/emel/speech/predictor/moshi/errors.hpp b/src/emel/speech/predictor/moshi/errors.hpp new file mode 100644 index 00000000..0db3677a --- /dev/null +++ b/src/emel/speech/predictor/moshi/errors.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::speech::predictor::moshi { + +enum class error : emel::error::type { + none = 0, + not_initialized = 1, + bind_failed = 2, + memory = 3, + request_shape = 4, + graph_runtime_unavailable = 5, + graph_runtime = 6, + output_waiting = 7, + voice_contract = 8, + voice_prompt_pending = 9, + personaplex_prompt = 10, + unexpected_event = 11, +}; + +} // namespace emel::speech::predictor::moshi diff --git a/src/emel/speech/predictor/moshi/events.hpp b/src/emel/speech/predictor/moshi/events.hpp new file mode 100644 index 00000000..b914e937 --- /dev/null +++ b/src/emel/speech/predictor/moshi/events.hpp @@ -0,0 +1,391 @@ +#pragma once + +#include +#include +#include +#include + +#include "emel/callback.hpp" +#include "emel/error/error.hpp" +#include "emel/memory/view.hpp" +#include "emel/model/data.hpp" +#include "emel/speech/predictor/moshi/errors.hpp" + +namespace emel::speech::predictor::moshi::events { + +struct initialize_done; +struct initialize_error; +struct load_voice_done; +struct load_voice_error; +struct begin_personaplex_prompt_done; +struct begin_personaplex_prompt_error; +struct prefill_voice_done; +struct prefill_voice_error; +struct prefill_personaplex_prompt_done; +struct prefill_personaplex_prompt_error; + +} // namespace emel::speech::predictor::moshi::events + +namespace emel::speech::predictor::moshi::event { + +inline constexpr std::size_t k_max_codebooks = 64; +inline constexpr std::size_t k_max_voice_embedding_dim = 8192; + +struct initialize { + explicit initialize(const emel::model::data &model_ref) noexcept + : model(model_ref) {} + + const emel::model::data &model; + int32_t max_sequences = 0; + int32_t max_blocks = 0; + int32_t block_tokens = 0; + int32_t sequence_id = -1; + int32_t codebook_capacity = 0; + int32_t delay_cache_row_capacity = 0; + bool sampling_enabled = false; + bool sampling_consume_forced_text = false; + float sampling_audio_temperature = 0.0f; + float sampling_text_temperature = 0.0f; + int32_t sampling_audio_top_k = 0; + int32_t sampling_text_top_k = 0; + uint32_t sampling_seed = 0u; + emel::error::type *error_out = nullptr; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct initialize_graph { + explicit initialize_graph(const initialize &request_ref) noexcept + : request(request_ref) {} + + const initialize &request; + emel::error::type *error_out = nullptr; +}; + +struct load_voice { + explicit load_voice(const emel::model::data &voice_model_ref) noexcept + : voice_model(voice_model_ref) {} + + const emel::model::data &voice_model; + emel::error::type *error_out = nullptr; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct begin_personaplex_prompt { + int32_t text_token_count = 0; + int32_t pre_text_silence_frames = 0; + int32_t post_text_silence_frames = 0; + emel::error::type *error_out = nullptr; + emel::callback on_done = + {}; + emel::callback + on_error = {}; +}; + +struct prefill_voice { + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; + bool *complete_out = nullptr; + int32_t *remaining_frames_out = nullptr; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct prefill_personaplex_prompt { + int32_t text_token = -1; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; + bool *complete_out = nullptr; + int32_t *remaining_frames_out = nullptr; + emel::callback + on_done = {}; + emel::callback + on_error = {}; +}; + +struct predict { + struct workspace { + emel::memory::view::snapshot memory = {}; + std::span temporal_state = {}; + std::array sampled_audio_tokens = {}; + int32_t sampled_text_token = -1; + }; + + predict(std::span model_tokens_ref, workspace &workspace_ref, + const int32_t planned_step_size_ref, + const int32_t planned_output_count_ref) noexcept + : model_tokens(model_tokens_ref), prediction_workspace(workspace_ref), + planned_step_size(planned_step_size_ref), + planned_output_count(planned_output_count_ref) {} + + std::span model_tokens = {}; + workspace &prediction_workspace; + int32_t planned_step_size = 0; + int32_t planned_output_count = 0; + emel::error::type *error_out = nullptr; +}; + +struct execute { + execute(predict::workspace &workspace_ref, + std::span model_tokens_ref) noexcept + : prediction_workspace(workspace_ref), model_tokens(model_tokens_ref) {} + + predict::workspace &prediction_workspace; + std::span model_tokens = {}; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct sample { + sample(predict::workspace &workspace_ref, + std::span model_tokens_ref, + std::span audio_tokens_out_ref, + int32_t &text_token_out_ref) noexcept + : prediction_workspace(workspace_ref), model_tokens(model_tokens_ref), + audio_tokens_out(audio_tokens_out_ref), + text_token_out(text_token_out_ref) {} + + predict::workspace &prediction_workspace; + std::span model_tokens = {}; + std::span audio_tokens_out = {}; + int32_t &text_token_out; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct capture_tokenizer_state { + capture_tokenizer_state(std::span cache_out_ref, + int64_t &offset_out_ref, + emel::error::type &error_out_ref) noexcept + : cache_out(cache_out_ref), offset_out(offset_out_ref), + error_out(error_out_ref) {} + + std::span cache_out = {}; + int64_t &offset_out; + emel::error::type &error_out; +}; + +struct reset {}; + +enum class graph_phase : uint8_t { + full, + prediction, + sampling, +}; + +struct graph_step { + graph_step(const emel::model::data &model_ref, + const emel::memory::view::snapshot &memory_snapshot_ref, + std::span input_sequence_ref, + std::span audio_tokens_out_ref, + int32_t &text_token_out_ref) noexcept + : model(model_ref), memory_snapshot(memory_snapshot_ref), + input_sequence(input_sequence_ref), + audio_tokens_out(audio_tokens_out_ref), + text_token_out(text_token_out_ref) {} + + const emel::model::data &model; + const emel::memory::view::snapshot &memory_snapshot; + int32_t sequence_id = 0; + graph_phase phase = graph_phase::full; + std::span input_sequence = {}; + std::span input_embedding = {}; + std::span temporal_state = {}; + int32_t forced_text_token = -1; + std::span audio_tokens_out = {}; + int32_t &text_token_out; + emel::error::type *error_out = nullptr; +}; + +struct initialize_ctx { + emel::error::type err = emel::error::cast(error::none); + emel::error::type graph_error = emel::error::cast(error::none); + bool graph_accepted = false; + bool memory_accepted = false; + int32_t memory_error = static_cast(emel::error::cast(error::none)); +}; + +struct load_voice_ctx { + emel::error::type err = emel::error::cast(error::none); +}; + +struct begin_personaplex_prompt_ctx { + emel::error::type err = emel::error::cast(error::none); + int32_t remaining_frames = 0; +}; + +struct prefill_voice_ctx { + explicit prefill_voice_ctx( + emel::memory::view::snapshot &memory_snapshot_ref) noexcept + : memory_snapshot(memory_snapshot_ref) {} + + emel::error::type err = emel::error::cast(error::none); + bool memory_accepted = false; + int32_t memory_error = static_cast(emel::error::cast(error::none)); + int32_t memory_block_count = 0; + bool embedding_frame_ok = false; + bool graph_accepted = false; + emel::error::type graph_error = emel::error::cast(error::none); + bool complete = false; + int32_t remaining_frames = 0; + int32_t text_token = 0; + std::array embedding_frame = {}; + std::array input_sequence = {}; + std::array audio_tokens = {}; + emel::memory::view::snapshot &memory_snapshot; +}; + +struct prefill_personaplex_prompt_ctx { + explicit prefill_personaplex_prompt_ctx( + emel::memory::view::snapshot &memory_snapshot_ref) noexcept + : memory_snapshot(memory_snapshot_ref) {} + + emel::error::type err = emel::error::cast(error::none); + bool memory_accepted = false; + int32_t memory_error = static_cast(emel::error::cast(error::none)); + int32_t memory_block_count = 0; + bool graph_accepted = false; + emel::error::type graph_error = emel::error::cast(error::none); + bool complete = false; + int32_t remaining_frames = 0; + int32_t text_token = 0; + std::array prompt_sequence = {}; + std::array input_sequence = {}; + std::array audio_tokens = {}; + emel::memory::view::snapshot &memory_snapshot; +}; + +struct predict_ctx { + explicit predict_ctx( + emel::memory::view::snapshot &memory_snapshot_ref) noexcept + : memory_snapshot(memory_snapshot_ref) {} + + emel::error::type err = emel::error::cast(error::none); + bool memory_accepted = false; + int32_t memory_error = static_cast(emel::error::cast(error::none)); + int32_t memory_block_count = 0; + emel::memory::view::snapshot &memory_snapshot; +}; + +struct execute_ctx { + emel::error::type err = emel::error::cast(error::none); + bool graph_accepted = false; + emel::error::type graph_error = emel::error::cast(error::none); +}; + +struct sample_ctx { + emel::error::type err = emel::error::cast(error::none); + bool graph_accepted = false; + emel::error::type graph_error = emel::error::cast(error::none); +}; + +struct initialize_run { + const initialize &request; + initialize_ctx &ctx; +}; + +struct load_voice_run { + const load_voice &request; + load_voice_ctx &ctx; +}; + +struct begin_personaplex_prompt_run { + const begin_personaplex_prompt &request; + begin_personaplex_prompt_ctx &ctx; +}; + +struct prefill_voice_run { + const prefill_voice &request; + prefill_voice_ctx &ctx; +}; + +struct prefill_personaplex_prompt_run { + const prefill_personaplex_prompt &request; + prefill_personaplex_prompt_ctx &ctx; +}; + +struct predict_run { + const predict &request; + predict_ctx &ctx; +}; + +struct execute_run { + const execute &request; + execute_ctx &ctx; +}; + +struct sample_run { + const sample &request; + sample_ctx &ctx; +}; + +struct reset_run { + const reset &request; + struct context { + emel::error::type err = emel::error::cast(error::none); + bool graph_accepted = false; + emel::error::type graph_error = emel::error::cast(error::none); + bool memory_accepted = false; + int32_t memory_error = static_cast(emel::error::cast(error::none)); + } &ctx; +}; + +} // namespace emel::speech::predictor::moshi::event + +namespace emel::speech::predictor::moshi::events { + +struct initialize_done { + const event::initialize *request = nullptr; + int32_t n_q = 0; + int32_t dep_q = 0; +}; + +struct initialize_error { + const event::initialize *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +struct load_voice_done { + const event::load_voice *request = nullptr; + int32_t prompt_frames = 0; +}; + +struct load_voice_error { + const event::load_voice *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +struct begin_personaplex_prompt_done { + const event::begin_personaplex_prompt *request = nullptr; + int32_t remaining_frames = 0; +}; + +struct begin_personaplex_prompt_error { + const event::begin_personaplex_prompt *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +struct prefill_voice_done { + const event::prefill_voice *request = nullptr; + bool complete = false; + int32_t remaining_frames = 0; +}; + +struct prefill_voice_error { + const event::prefill_voice *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +struct prefill_personaplex_prompt_done { + const event::prefill_personaplex_prompt *request = nullptr; + bool complete = false; + int32_t remaining_frames = 0; +}; + +struct prefill_personaplex_prompt_error { + const event::prefill_personaplex_prompt *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +} // namespace emel::speech::predictor::moshi::events diff --git a/src/emel/speech/predictor/moshi/executor/actions.hpp b/src/emel/speech/predictor/moshi/executor/actions.hpp new file mode 100644 index 00000000..6e59db81 --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/actions.hpp @@ -0,0 +1,1992 @@ +#pragma once + +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/executor/context.hpp" +#include "emel/speech/predictor/moshi/executor/detail.hpp" +#include "emel/speech/predictor/moshi/executor/events.hpp" + +#include +#include +#include +#include + +namespace emel::speech::predictor::moshi::executor::action { + +namespace detail_ns { + +inline emel::error::type to_error(const error value) noexcept { + return emel::error::cast(value); +} + +} // namespace detail_ns + +struct effect_bind_contract { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.session = {}; + ctx.sampling = {}; + ctx.session.model = &runtime_ev.request.model; + ctx.session.codebook_count = runtime_ev.request.model.moshi_lm.n_q + 1; + ctx.session.dep_q = runtime_ev.request.model.moshi_lm.dep_q; + ctx.session.text_card = runtime_ev.request.model.moshi_lm.text_card; + ctx.session.audio_card = runtime_ev.request.model.moshi_lm.card; + ctx.session.hidden_dim = runtime_ev.request.model.moshi_lm.dim; + ctx.sampling.enabled = runtime_ev.request.sampling_enabled; + ctx.sampling.consume_forced_text = + runtime_ev.request.sampling_consume_forced_text; + ctx.sampling.audio_temperature = + runtime_ev.request.sampling_audio_temperature; + ctx.sampling.text_temperature = + runtime_ev.request.sampling_text_temperature; + (void)emel::model::moshi::detail::build_execution_contract( + runtime_ev.request.model, ctx.session.contract); + } +}; + +struct effect_bind_temporal_split_projection_layout { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + for (int32_t layer = 0; layer < ctx.session.model->moshi_lm.num_layers; + ++layer) { + ctx.session.temporal_input_projections[static_cast(layer)] = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .split_input_projection.tensor; + } + } +}; + +struct effect_bind_temporal_fused_projection_layout { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + for (int32_t layer = 0; layer < ctx.session.model->moshi_lm.num_layers; + ++layer) { + ctx.session.temporal_input_projections[static_cast(layer)] = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .fused_input_projection.tensor; + } + } +}; + +struct effect_bind_depformer_split_projection_layout { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + const auto &lm = ctx.session.model->moshi_lm; + for (int32_t layer = 0; layer < lm.depformer_num_layers; ++layer) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + ctx.session.depformer_input_projections[static_cast(layer)] + [static_cast(codebook)] = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(codebook)] + .split_input_projection.tensor; + } + } + } +}; + +struct effect_bind_depformer_fused_projection_layout { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + const auto &lm = ctx.session.model->moshi_lm; + for (int32_t layer = 0; layer < lm.depformer_num_layers; ++layer) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + ctx.session.depformer_input_projections[static_cast(layer)] + [static_cast(codebook)] = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(codebook)] + .fused_input_projection.tensor; + } + } + } +}; + +struct effect_bind_nonzero_sampling_seed { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.sampling.random_state = ((runtime_ev.request.sampling_seed - 1u) % + (ctx.policy.sampling_modulus - 1u)) + + 1u; + } +}; + +struct effect_bind_zero_sampling_seed { + void operator()(const event::initialize_run &, context &ctx) const noexcept { + ctx.sampling.random_state = ctx.policy.zero_seed_state; + } +}; + +struct effect_bind_requested_text_sampling_top_k { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.sampling.text_top_k = runtime_ev.request.sampling_text_top_k; + } +}; + +struct effect_bind_full_card_text_sampling_top_k { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.sampling.text_top_k = runtime_ev.request.model.moshi_lm.text_card; + } +}; + +struct effect_bind_requested_audio_sampling_top_k { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.sampling.audio_top_k = runtime_ev.request.sampling_audio_top_k; + } +}; + +struct effect_bind_full_card_audio_sampling_top_k { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + ctx.sampling.audio_top_k = runtime_ev.request.model.moshi_lm.card; + } +}; + +struct effect_mark_bind_failed { + void operator()(const event::initialize_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::bind_failed); + } +}; + +struct effect_mark_not_initialized { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::not_initialized); + } +}; + +struct effect_mark_model_mismatch { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::model_mismatch); + } +}; + +struct effect_mark_request_shape { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::request_shape); + } +}; + +struct effect_mark_graph_execution_unsupported { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.err = + detail_ns::to_error(error::graph_execution_unsupported); + } +}; + +struct effect_begin_input_embedding { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const int32_t hidden_dim = ctx.session.hidden_dim; + (void)model; + runtime_ev.ctx.err = detail_ns::to_error(error::none); + runtime_ev.ctx.input_embedding_ok = false; + runtime_ev.ctx.input_text_embedding_ok = false; + runtime_ev.ctx.input_audio_embedding_ok = false; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.temporal_position_accepted = false; + runtime_ev.ctx.temporal_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.temporal_position = {}; + runtime_ev.ctx.depformer_input_projection_bound = false; + runtime_ev.ctx.depformer_input_projection_ok = false; + runtime_ev.ctx.temporal_layer_norm_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm_ok = false; + runtime_ev.ctx.temporal_layer_projection_ok = false; + runtime_ev.ctx.temporal_layer_rope_ok = false; + runtime_ev.ctx.temporal_layer_cache_write_ok = false; + runtime_ev.ctx.temporal_layer_attention_ok = false; + runtime_ev.ctx.temporal_layer_out_projection_ok = false; + runtime_ev.ctx.temporal_layer_residual_ok = false; + runtime_ev.ctx.temporal_layer_norm2_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm2_ok = false; + runtime_ev.ctx.temporal_layer_gating_in_ok = false; + runtime_ev.ctx.temporal_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.temporal_layer_silu_gate_ok = false; + runtime_ev.ctx.temporal_layer_gating_out_ok = false; + runtime_ev.ctx.temporal_layer_ff_residual_ok = false; + runtime_ev.ctx.temporal_out_norm_rms_ok = false; + runtime_ev.ctx.temporal_out_norm_ok = false; + runtime_ev.ctx.text_logits_ok = false; + runtime_ev.ctx.depformer_kv_bound = false; + runtime_ev.ctx.depformer_position_accepted = false; + runtime_ev.ctx.depformer_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.depformer_position = {}; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + runtime_ev.ctx.depformer_layer_attention_ok = false; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.depformer_layer_residual_ok = false; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + runtime_ev.ctx.input_audio_codebook_index = 0; + runtime_ev.ctx.temporal_layer_index = 0; + runtime_ev.ctx.temporal_rope_position = -1; + runtime_ev.ctx.depformer_codebook_index = 0; + runtime_ev.ctx.depformer_weight_index = -1; + runtime_ev.ctx.depformer_layer_index = 0; + runtime_ev.ctx.embedding_dim = 0; + runtime_ev.ctx.embedding_view = {}; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.hidden.data(), static_cast(hidden_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.transformer_out.data(), + static_cast(hidden_dim), 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim), 0.0f); + std::fill_n(runtime_ev.ctx.qkv.data(), static_cast(hidden_dim) * 3u, + 0.0f); + std::fill_n(runtime_ev.ctx.attention.data(), + static_cast(hidden_dim), 0.0f); + std::fill_n(runtime_ev.ctx.projection.data(), + static_cast(hidden_dim), 0.0f); + std::fill(runtime_ev.ctx.gating_projection.begin(), + runtime_ev.ctx.gating_projection.end(), 0.0f); + std::fill(runtime_ev.ctx.gating_silu.begin(), + runtime_ev.ctx.gating_silu.end(), 0.0f); + std::fill(runtime_ev.ctx.gated.begin(), runtime_ev.ctx.gated.end(), 0.0f); + } +}; + +struct effect_use_depformer_scheduled_weight { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + runtime_ev.ctx.depformer_weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + } +}; + +struct effect_use_depformer_codebook_weight { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.depformer_weight_index = + runtime_ev.ctx.depformer_codebook_index; + } +}; + +struct effect_apply_external_input_embedding { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.input_embedding_ok = false; + for (int32_t index = 0; index < hidden_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] = + runtime_ev.request.input_embedding[static_cast(index)]; + } + runtime_ev.ctx.input_embedding_ok = true; + } +}; + +struct effect_bind_input_text_embedding { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const auto *text_emb = ctx.session.contract.lm.text_embedding.tensor; + runtime_ev.ctx.embedding_view = {}; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.embedding_dim = ctx.session.hidden_dim; + runtime_ev.ctx.row_index = runtime_ev.request.input_sequence[0]; + runtime_ev.ctx.input_text_embedding_ok = false; + runtime_ev.ctx.embedding_view_bound = + detail::bind_tensor_view(*text_emb, runtime_ev.ctx.embedding_view); + } +}; + +struct effect_run_embedding_row_fetch { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t embedding_dim = runtime_ev.ctx.embedding_dim; + runtime_ev.ctx.embedding_row_ok = false; + std::fill_n(runtime_ev.ctx.row.data(), static_cast(embedding_dim), + 0.0f); + emel::kernel::event::op_get_rows rows_ev{ + .src0 = runtime_ev.ctx.embedding_view, + .src1 = detail::make_i32_src(&runtime_ev.ctx.row_index, 1u), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(embedding_dim)), + }; + runtime_ev.ctx.embedding_row_ok = ctx.kernel.process_event(rows_ev); + } +}; + +struct effect_apply_input_text_embedding_row { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const int32_t embedding_dim = runtime_ev.ctx.embedding_dim; + runtime_ev.ctx.input_text_embedding_ok = false; + for (int32_t index = 0; index < embedding_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.row[static_cast(index)]; + } + runtime_ev.ctx.input_text_embedding_ok = true; + } +}; + +struct effect_bind_input_audio_embedding { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t index = runtime_ev.ctx.input_audio_codebook_index; + const auto *audio_emb = + ctx.session.contract.lm.audio_embeddings[static_cast(index)] + .tensor; + runtime_ev.ctx.embedding_view = {}; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.embedding_dim = ctx.session.hidden_dim; + runtime_ev.ctx.row_index = + runtime_ev.request.input_sequence[static_cast(index + 1)]; + runtime_ev.ctx.input_audio_embedding_ok = false; + runtime_ev.ctx.embedding_view_bound = + detail::bind_tensor_view(*audio_emb, runtime_ev.ctx.embedding_view); + } +}; + +struct effect_apply_input_audio_embedding_row { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const int32_t embedding_dim = runtime_ev.ctx.embedding_dim; + runtime_ev.ctx.input_audio_embedding_ok = false; + for (int32_t index = 0; index < embedding_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.row[static_cast(index)]; + } + runtime_ev.ctx.input_audio_embedding_ok = true; + } +}; + +struct effect_skip_zero_input_audio_embedding { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.input_audio_embedding_ok = true; + } +}; + +struct effect_advance_input_audio_codebook { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + ++runtime_ev.ctx.input_audio_codebook_index; + runtime_ev.ctx.input_audio_embedding_ok = false; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.embedding_dim = 0; + runtime_ev.ctx.embedding_view = {}; + } +}; + +struct effect_finish_input_embedding { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.input_embedding_ok = true; + } +}; + +struct effect_run_temporal_layer_norm_rms { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_norm_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm_ok = false; + + std::fill_n(runtime_ev.ctx.row.data(), static_cast(hidden_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim), 0.0f); + + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + }; + detail::set_op_param_f32(rms_ev, 0u, ctx.policy.rms_norm_epsilon); + runtime_ev.ctx.temporal_layer_norm_rms_ok = + ctx.kernel.process_event(rms_ev); + } +}; + +struct effect_run_temporal_layer_norm_scale { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + runtime_ev.ctx.temporal_layer_norm_ok = false; + + const auto *norm1 = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .norm1.tensor; + const auto *alpha = static_cast(norm1->data); + + emel::kernel::event::op_mul mul_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + .src1 = detail::make_f32_src(alpha, static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim)), + }; + runtime_ev.ctx.temporal_layer_norm_ok = ctx.kernel.process_event(mul_ev); + } +}; + +struct effect_bind_temporal_layer_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + runtime_ev.ctx.temporal_layer_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.qkv.data(), static_cast(hidden_dim) * 3u, + 0.0f); + + const auto *projection = + ctx.session.temporal_input_projections[static_cast(layer)]; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_temporal_layer_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_projection_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.qkv.data(), 1u, + static_cast(hidden_dim) * 3u), + }; + runtime_ev.ctx.temporal_layer_projection_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +template struct effect_run_temporal_layer_rope { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t head_dim = hidden_dim / lm.num_heads; + runtime_ev.ctx.temporal_layer_rope_ok = false; + runtime_ev.ctx.temporal_rope_position = + static_cast(runtime_ev.ctx.temporal_position.logical_position); + + const uint64_t head_dim_u64 = static_cast(head_dim); + const uint64_t heads_u64 = static_cast(lm.num_heads); + const uint64_t hidden_dim_u64 = static_cast(hidden_dim); + const uint64_t element_bytes = sizeof(float); + const float *source = + runtime_ev.ctx.qkv.data() + projection_part * hidden_dim_u64; + emel::kernel::event::op_rope rope_ev{ + .src0 = + { + .data = source, + .type = emel::kernel::event::dtype::f32, + .ne = {head_dim_u64, heads_u64, 1u, 1u}, + .nb = {element_bytes, element_bytes * head_dim_u64, + element_bytes * hidden_dim_u64, + element_bytes * hidden_dim_u64}, + }, + .src1 = + { + .data = &runtime_ev.ctx.temporal_rope_position, + .type = emel::kernel::event::dtype::i32, + .ne = {1u, 1u, 1u, 1u}, + .nb = {sizeof(int32_t), sizeof(int32_t), sizeof(int32_t), + sizeof(int32_t)}, + }, + .dst = + { + .data = runtime_ev.ctx.rope.data(), + .type = emel::kernel::event::dtype::f32, + .ne = {head_dim_u64, heads_u64, 1u, 1u}, + .nb = {element_bytes, element_bytes * head_dim_u64, + element_bytes * hidden_dim_u64, + element_bytes * hidden_dim_u64}, + }, + }; + const int32_t rope_mode = emel::kernel::detail::rope_mode_timestep; + const float freq_base = static_cast(lm.max_period); + const float freq_scale = 1.0f; + const float ext_factor = 0.0f; + const float attn_factor = 1.0f; + std::memcpy(rope_ev.op_params.data() + 1u * sizeof(int32_t), &head_dim, + sizeof(head_dim)); + std::memcpy(rope_ev.op_params.data() + 2u * sizeof(int32_t), &rope_mode, + sizeof(rope_mode)); + std::memcpy(rope_ev.op_params.data() + 5u * sizeof(float), &freq_base, + sizeof(freq_base)); + std::memcpy(rope_ev.op_params.data() + 6u * sizeof(float), &freq_scale, + sizeof(freq_scale)); + std::memcpy(rope_ev.op_params.data() + 7u * sizeof(float), &ext_factor, + sizeof(ext_factor)); + std::memcpy(rope_ev.op_params.data() + 8u * sizeof(float), &attn_factor, + sizeof(attn_factor)); + rope_ev.op_params_size = 9u * sizeof(float); + runtime_ev.ctx.temporal_layer_rope_ok = ctx.kernel.process_event(rope_ev); + } +}; + +using effect_run_temporal_layer_query_rope = effect_run_temporal_layer_rope<0u>; +using effect_run_temporal_layer_key_rope = effect_run_temporal_layer_rope<1u>; + +template struct effect_copy_temporal_layer_rope { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const size_t hidden_dim = static_cast(ctx.session.hidden_dim); + std::memcpy(runtime_ev.ctx.qkv.data() + projection_part * hidden_dim, + runtime_ev.ctx.rope.data(), hidden_dim * sizeof(float)); + } +}; + +using effect_copy_temporal_layer_query_rope = + effect_copy_temporal_layer_rope<0u>; +using effect_copy_temporal_layer_key_rope = effect_copy_temporal_layer_rope<1u>; + +struct effect_write_temporal_layer_kv_cache { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_cache_write_ok = false; + auto &view = runtime_ev.ctx.temporal_kv; + const size_t layer_offset = view.layer_cache_offsets[static_cast( + runtime_ev.ctx.temporal_layer_index)]; + const size_t physical_position = + static_cast(runtime_ev.ctx.temporal_position.physical_position); + const size_t dim = static_cast(hidden_dim); + const size_t begin = layer_offset + physical_position * dim; + const float *key = runtime_ev.ctx.qkv.data() + dim; + const float *value = runtime_ev.ctx.qkv.data() + dim * 2u; + + for (size_t index = 0; index < dim; ++index) { + view.key_cache[begin + index] = + emel::kernel::detail::fp32_to_bf16(key[index]); + view.value_cache[begin + index] = + emel::kernel::detail::fp32_to_bf16(value[index]); + } + runtime_ev.ctx.temporal_layer_cache_write_ok = true; + } +}; + +struct effect_run_temporal_layer_attention { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const auto &view = runtime_ev.ctx.temporal_kv; + const auto &window = runtime_ev.ctx.temporal_position.window; + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t head_dim = hidden_dim / lm.num_heads; + const int32_t capacity = view.position_capacity; + const int32_t valid_positions = window.valid_positions; + const float scale = 1.0f / std::sqrt(static_cast(head_dim)); + const size_t layer_offset = view.layer_cache_offsets[static_cast( + runtime_ev.ctx.temporal_layer_index)]; + runtime_ev.ctx.temporal_layer_attention_ok = false; + std::fill_n(runtime_ev.ctx.attention.data(), + static_cast(hidden_dim), 0.0f); + + for (int32_t head = 0; head < lm.num_heads; ++head) { + const int32_t head_offset = head * head_dim; + const float *q_head = + runtime_ev.ctx.qkv.data() + static_cast(head_offset); + for (int32_t dim = 0; dim < head_dim; ++dim) { + runtime_ev.ctx.q_bf16[static_cast(dim)] = + emel::kernel::detail::fp32_to_bf16(q_head[dim]); + } + for (int32_t physical = 0; physical < capacity; ++physical) { + runtime_ev.ctx.attention_scores[static_cast(physical)] = + -std::numeric_limits::infinity(); + } + for (int32_t position = 0; position < valid_positions; ++position) { + const int32_t unwrapped = window.physical_begin + position; + const int32_t physical = + unwrapped - static_cast(unwrapped >= capacity) * capacity; + const size_t cache_begin = + layer_offset + + static_cast(physical) * static_cast(hidden_dim) + + static_cast(head_offset); + runtime_ev.ctx.attention_scores[static_cast(physical)] = + emel::kernel::detail::vec_dot_bf16_ggml( + head_dim, view.key_cache.data() + cache_begin, + runtime_ev.ctx.q_bf16.data()) * + scale; + } + + emel::kernel::detail::soft_max_row_ggml( + capacity, runtime_ev.ctx.attention_scores.data()); + for (int32_t physical = 0; physical < capacity; ++physical) { + runtime_ev.ctx.attention_weights_bf16[static_cast(physical)] = + emel::kernel::detail::fp32_to_bf16( + runtime_ev.ctx.attention_scores[static_cast(physical)]); + } + + float *attention_head = + runtime_ev.ctx.attention.data() + static_cast(head_offset); + for (int32_t dim = 0; dim < head_dim; ++dim) { + double sum = 0.0; + for (int32_t position = 0; position < valid_positions; ++position) { + const int32_t unwrapped = window.physical_begin + position; + const int32_t physical = + unwrapped - + static_cast(unwrapped >= capacity) * capacity; + const size_t value_index = + layer_offset + + static_cast(physical) * static_cast(hidden_dim) + + static_cast(head_offset + dim); + sum += static_cast( + emel::kernel::detail::bf16_to_fp32( + view.value_cache[value_index]) * + emel::kernel::detail::bf16_to_fp32( + runtime_ev.ctx + .attention_weights_bf16[static_cast(physical)])); + } + attention_head[dim] = static_cast(sum); + } + } + runtime_ev.ctx.temporal_layer_attention_ok = true; + } +}; + +struct effect_bind_temporal_layer_out_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + runtime_ev.ctx.temporal_layer_out_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.projection.data(), + static_cast(hidden_dim), 0.0f); + + const auto *projection = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .output_projection.tensor; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_temporal_layer_out_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_out_projection_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.attention.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.projection.data(), 1u, + static_cast(hidden_dim)), + }; + runtime_ev.ctx.temporal_layer_out_projection_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_apply_temporal_layer_attention_residual { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_residual_ok = false; + for (int32_t index = 0; index < hidden_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.projection[static_cast(index)]; + } + runtime_ev.ctx.temporal_layer_residual_ok = true; + } +}; + +struct effect_run_temporal_layer_norm2_rms { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_norm2_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm2_ok = false; + + std::fill_n(runtime_ev.ctx.row.data(), static_cast(hidden_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim), 0.0f); + + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + }; + detail::set_op_param_f32(rms_ev, 0u, ctx.policy.rms_norm_epsilon); + runtime_ev.ctx.temporal_layer_norm2_rms_ok = + ctx.kernel.process_event(rms_ev); + } +}; + +struct effect_run_temporal_layer_norm2_scale { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + runtime_ev.ctx.temporal_layer_norm2_ok = false; + + const auto *norm2 = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .norm2.tensor; + const auto *alpha = static_cast(norm2->data); + + emel::kernel::event::op_mul mul_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + .src1 = detail::make_f32_src(alpha, static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim)), + }; + runtime_ev.ctx.temporal_layer_norm2_ok = ctx.kernel.process_event(mul_ev); + } +}; + +struct effect_bind_temporal_layer_gating_in { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + const auto *linear_in = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + runtime_ev.ctx.temporal_layer_gating_in_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.gating_projection.data(), + static_cast(projection_dim), 0.0f); + + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*linear_in, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_temporal_layer_gating_in { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + const auto *linear_in = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + runtime_ev.ctx.temporal_layer_gating_in_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.gating_projection.data(), 1u, + projection_dim), + }; + runtime_ev.ctx.temporal_layer_gating_in_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_run_temporal_layer_silu_gate_silu { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto *linear_in = ctx.session.contract.lm + .temporal_layers[static_cast( + runtime_ev.ctx.temporal_layer_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + const uint64_t gate_dim = projection_dim / 2u; + runtime_ev.ctx.temporal_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.temporal_layer_silu_gate_ok = false; + std::fill_n(runtime_ev.ctx.gating_silu.data(), + static_cast(gate_dim), 0.0f); + std::fill_n(runtime_ev.ctx.gated.data(), static_cast(gate_dim), + 0.0f); + + emel::kernel::event::op_unary silu_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.gating_projection.data(), + gate_dim), + .dst = + detail::make_f32_dst(runtime_ev.ctx.gating_silu.data(), gate_dim), + .subop = emel::kernel::event::unary_subop::silu, + }; + runtime_ev.ctx.temporal_layer_silu_gate_silu_ok = + ctx.kernel.process_event(silu_ev); + } +}; + +struct effect_run_temporal_layer_silu_gate_mul { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto *linear_in = ctx.session.contract.lm + .temporal_layers[static_cast( + runtime_ev.ctx.temporal_layer_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + const uint64_t gate_dim = projection_dim / 2u; + const float *right = runtime_ev.ctx.gating_projection.data() + gate_dim; + runtime_ev.ctx.temporal_layer_silu_gate_ok = false; + + emel::kernel::event::op_mul gate_ev{ + .src0 = + detail::make_f32_src(runtime_ev.ctx.gating_silu.data(), gate_dim), + .src1 = detail::make_f32_src(right, gate_dim), + .dst = detail::make_f32_dst(runtime_ev.ctx.gated.data(), gate_dim), + }; + runtime_ev.ctx.temporal_layer_silu_gate_ok = + ctx.kernel.process_event(gate_ev); + } +}; + +struct effect_bind_temporal_layer_gating_out { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + const auto &layer_contract = + ctx.session.contract.lm.temporal_layers[static_cast(layer)]; + const auto *linear_in = layer_contract.gating_input.tensor; + const auto *linear_out = layer_contract.gating_output.tensor; + const uint64_t gate_dim = static_cast(linear_in->dims[1]) / 2u; + (void)gate_dim; + runtime_ev.ctx.temporal_layer_gating_out_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.projection.data(), + static_cast(hidden_dim), 0.0f); + + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*linear_out, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_temporal_layer_gating_out { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + const auto *linear_in = + ctx.session.contract.lm.temporal_layers[static_cast(layer)] + .gating_input.tensor; + const uint64_t gate_dim = static_cast(linear_in->dims[1]) / 2u; + runtime_ev.ctx.temporal_layer_gating_out_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.gated.data(), 1u, gate_dim), + .dst = detail::make_f32_dst(runtime_ev.ctx.projection.data(), 1u, + static_cast(hidden_dim)), + }; + runtime_ev.ctx.temporal_layer_gating_out_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_apply_temporal_layer_ff_residual { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_layer_ff_residual_ok = false; + for (int32_t index = 0; index < hidden_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.projection[static_cast(index)]; + } + runtime_ev.ctx.temporal_layer_ff_residual_ok = true; + } +}; + +struct effect_advance_temporal_layer { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + ++runtime_ev.ctx.temporal_layer_index; + runtime_ev.ctx.temporal_layer_norm_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm_ok = false; + runtime_ev.ctx.temporal_layer_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.temporal_layer_rope_ok = false; + runtime_ev.ctx.temporal_layer_cache_write_ok = false; + runtime_ev.ctx.temporal_layer_attention_ok = false; + runtime_ev.ctx.temporal_layer_out_projection_ok = false; + runtime_ev.ctx.temporal_layer_residual_ok = false; + runtime_ev.ctx.temporal_layer_norm2_rms_ok = false; + runtime_ev.ctx.temporal_layer_norm2_ok = false; + runtime_ev.ctx.temporal_layer_gating_in_ok = false; + runtime_ev.ctx.temporal_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.temporal_layer_silu_gate_ok = false; + runtime_ev.ctx.temporal_layer_gating_out_ok = false; + runtime_ev.ctx.temporal_layer_ff_residual_ok = false; + } +}; + +struct effect_run_temporal_out_norm_rms { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.temporal_out_norm_rms_ok = false; + runtime_ev.ctx.temporal_out_norm_ok = false; + + std::fill_n(runtime_ev.ctx.row.data(), static_cast(hidden_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim), 0.0f); + + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + }; + detail::set_op_param_f32(rms_ev, 0u, ctx.policy.rms_norm_epsilon); + runtime_ev.ctx.temporal_out_norm_rms_ok = ctx.kernel.process_event(rms_ev); + } +}; + +struct effect_run_temporal_out_norm_scale { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const auto *out_norm = ctx.session.contract.lm.output_norm.tensor; + const auto *alpha = static_cast(out_norm->data); + runtime_ev.ctx.temporal_out_norm_ok = false; + + emel::kernel::event::op_mul mul_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.row.data(), + static_cast(hidden_dim)), + .src1 = detail::make_f32_src(alpha, static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim)), + }; + runtime_ev.ctx.temporal_out_norm_ok = ctx.kernel.process_event(mul_ev); + } +}; + +struct effect_publish_temporal_out_norm { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + std::copy_n(runtime_ev.ctx.normalized.data(), + static_cast(hidden_dim), + runtime_ev.ctx.transformer_out.data()); + } +}; + +struct effect_publish_prediction_state { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + std::copy_n(runtime_ev.ctx.normalized.data(), + static_cast(ctx.session.hidden_dim), + runtime_ev.request.temporal_state.begin()); + } +}; + +struct effect_restore_prediction_state_and_bind_text_logits { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.err = detail_ns::to_error(error::none); + std::copy_n(runtime_ev.request.temporal_state.begin(), + static_cast(hidden_dim), + runtime_ev.ctx.normalized.data()); + std::copy_n(runtime_ev.request.temporal_state.begin(), + static_cast(hidden_dim), + runtime_ev.ctx.transformer_out.data()); + const auto *text_linear = + ctx.session.contract.lm.text_output_projection.tensor; + runtime_ev.ctx.text_logits_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*text_linear, runtime_ev.ctx.projection_view); + } +}; + +struct effect_bind_text_token_logits { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto *text_linear = + ctx.session.contract.lm.text_output_projection.tensor; + runtime_ev.ctx.text_logits_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*text_linear, runtime_ev.ctx.projection_view); + } +}; + +struct effect_select_text_token { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + runtime_ev.ctx.text_logits_ok = false; + emel::kernel::event::op_mul_mat_argmax logits_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(&runtime_ev.ctx.best_score, 1u, 1u), + .index_out = &runtime_ev.ctx.best_index, + }; + runtime_ev.ctx.text_logits_ok = ctx.kernel.process_event(logits_ev); + runtime_ev.request.text_token_out = runtime_ev.ctx.best_index; + } +}; + +struct effect_compute_text_token_logits { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t text_card = ctx.session.text_card; + runtime_ev.ctx.text_logits_ok = false; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + emel::kernel::event::op_mul_mat logits_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.logits.data(), 1u, + static_cast(text_card)), + }; + runtime_ev.ctx.text_logits_ok = ctx.kernel.process_event(logits_ev); + } +}; + +struct effect_select_text_sampling_token { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.sampler_error = + emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k sample_ev{ + runtime_ev.ctx.logits, ctx.session.text_card, + ctx.sampling.text_temperature, ctx.sampling.text_top_k, + runtime_ev.ctx.sampling_indices, runtime_ev.ctx.top_scores, + runtime_ev.ctx.top_indices, ctx.sampling.random_state, + runtime_ev.ctx.best_index, runtime_ev.ctx.best_score, + runtime_ev.ctx.sampler_error}; + runtime_ev.ctx.sampler_accepted = ctx.sampler->process_event(sample_ev); + } +}; + +struct effect_publish_sampled_text_token { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.request.text_token_out = runtime_ev.ctx.best_index; + } +}; + +struct effect_publish_forced_text_token { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + runtime_ev.ctx.text_logits_ok = true; + runtime_ev.ctx.best_index = runtime_ev.request.forced_text_token; + runtime_ev.request.text_token_out = runtime_ev.request.forced_text_token; + } +}; + +struct effect_bind_depformer_kv { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.depformer_kv = ctx.depformer_kv; + runtime_ev.ctx.depformer_kv_bound = true; + } +}; + +struct effect_reset_depformer_positions { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.depformer_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.depformer_position_accepted = + ctx.depformer_positions->process_event( + emel::memory::streaming::event::reset{ + .error_out = runtime_ev.ctx.depformer_position_error}); + } +}; + +struct effect_advance_depformer_position { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.depformer_position = {}; + runtime_ev.ctx.depformer_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.depformer_position_accepted = + ctx.depformer_positions->process_event( + emel::memory::streaming::event::advance{ + .result = runtime_ev.ctx.depformer_position, + .error_out = runtime_ev.ctx.depformer_position_error}); + } +}; + +struct effect_bind_depformer_text_input_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.depformer_input_projection_bound = false; + runtime_ev.ctx.depformer_input_projection_ok = false; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.depformer_layer_index = 0; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + runtime_ev.ctx.depformer_layer_attention_ok = false; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.depformer_layer_residual_ok = false; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + std::fill_n(runtime_ev.ctx.hidden.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.projection.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.qkv.data(), static_cast(dep_dim) * 3u, + 0.0f); + std::fill_n(runtime_ev.ctx.attention.data(), static_cast(dep_dim), + 0.0f); + + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + const auto *input_projection = + ctx.session.contract.lm + .depformer_input_projections[static_cast(weight_index)] + .tensor; + runtime_ev.ctx.projection_view = {}; + runtime_ev.ctx.depformer_input_projection_bound = detail::bind_tensor_view( + *input_projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_bind_depformer_audio_input_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.depformer_input_projection_bound = false; + runtime_ev.ctx.depformer_input_projection_ok = false; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.depformer_layer_index = 0; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + runtime_ev.ctx.depformer_layer_attention_ok = false; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.depformer_layer_residual_ok = false; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + std::fill_n(runtime_ev.ctx.hidden.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.projection.data(), static_cast(dep_dim), + 0.0f); + std::fill_n(runtime_ev.ctx.qkv.data(), static_cast(dep_dim) * 3u, + 0.0f); + std::fill_n(runtime_ev.ctx.attention.data(), static_cast(dep_dim), + 0.0f); + + const auto *input_projection = + ctx.session.contract.lm + .depformer_input_projections[static_cast(weight_index)] + .tensor; + runtime_ev.ctx.projection_view = {}; + runtime_ev.ctx.depformer_input_projection_bound = detail::bind_tensor_view( + *input_projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_depformer_input_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t dep_dim = lm.depformer_dim; + runtime_ev.ctx.depformer_input_projection_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.transformer_out.data(), 1u, + static_cast(hidden_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.hidden.data(), 1u, + static_cast(dep_dim)), + }; + runtime_ev.ctx.depformer_input_projection_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_bind_depformer_text_input_embedding { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + const auto *text_emb = + ctx.session.contract.lm.depformer_text_embedding.tensor; + runtime_ev.ctx.embedding_view = {}; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.embedding_dim = dep_dim; + runtime_ev.ctx.row_index = runtime_ev.request.text_token_out; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.embedding_view_bound = + detail::bind_tensor_view(*text_emb, runtime_ev.ctx.embedding_view); + } +}; + +struct effect_bind_depformer_audio_input_embedding { + void operator()(const event::step_run &runtime_ev, + const context &ctx) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + + const auto *audio_emb = + ctx.session.contract.lm + .depformer_audio_embeddings[static_cast(codebook - 1)] + .tensor; + runtime_ev.ctx.embedding_view = {}; + runtime_ev.ctx.embedding_view_bound = false; + runtime_ev.ctx.embedding_row_ok = false; + runtime_ev.ctx.embedding_dim = dep_dim; + runtime_ev.ctx.row_index = + runtime_ev.request.audio_tokens_out[static_cast(codebook - 1)]; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.embedding_view_bound = + detail::bind_tensor_view(*audio_emb, runtime_ev.ctx.embedding_view); + } +}; + +struct effect_apply_depformer_input_embedding_row { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const int32_t embedding_dim = runtime_ev.ctx.embedding_dim; + runtime_ev.ctx.depformer_input_ok = false; + for (int32_t index = 0; index < embedding_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.row[static_cast(index)]; + } + runtime_ev.ctx.depformer_input_ok = true; + } +}; + +struct effect_run_depformer_layer_norm_rms { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const int32_t dep_dim = model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + + std::fill_n(runtime_ev.ctx.row.data(), static_cast(dep_dim), 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), static_cast(dep_dim), + 0.0f); + + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(dep_dim)), + }; + detail::set_op_param_f32(rms_ev, 0u, ctx.policy.rms_norm_epsilon); + runtime_ev.ctx.depformer_layer_norm_rms_ok = + ctx.kernel.process_event(rms_ev); + } +}; + +struct effect_run_depformer_layer_norm_scale { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const int32_t dep_dim = model.moshi_lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + runtime_ev.ctx.depformer_layer_norm_ok = false; + + const auto *norm1 = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .norm1.tensor; + const auto *alpha = static_cast(norm1->data); + + emel::kernel::event::op_mul mul_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.row.data(), + static_cast(dep_dim)), + .src1 = detail::make_f32_src(alpha, static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.normalized.data(), + static_cast(dep_dim)), + }; + runtime_ev.ctx.depformer_layer_norm_ok = ctx.kernel.process_event(mul_ev); + } +}; + +struct effect_bind_depformer_layer_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.qkv.data(), static_cast(dep_dim) * 3u, + 0.0f); + + const auto *projection = + ctx.session.depformer_input_projections[static_cast( + layer)][static_cast(weight_index)]; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_depformer_layer_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_projection_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.qkv.data(), 1u, + static_cast(dep_dim) * 3u), + }; + runtime_ev.ctx.depformer_layer_projection_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_write_depformer_layer_kv_cache { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + auto &view = runtime_ev.ctx.depformer_kv; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + + const int32_t physical_position = + runtime_ev.ctx.depformer_position.physical_position; + + const size_t layer_offset = view.layer_cache_offsets[static_cast( + runtime_ev.ctx.depformer_layer_index)]; + const size_t begin = layer_offset + static_cast(physical_position) * + static_cast(dep_dim); + const float *key = runtime_ev.ctx.qkv.data() + static_cast(dep_dim); + const float *value = + runtime_ev.ctx.qkv.data() + static_cast(dep_dim) * 2u; + + for (int32_t index = 0; index < dep_dim; ++index) { + view.key_cache[begin + static_cast(index)] = + emel::kernel::detail::fp32_to_bf16(key[index]); + view.value_cache[begin + static_cast(index)] = + emel::kernel::detail::fp32_to_bf16(value[index]); + } + runtime_ev.ctx.depformer_layer_cache_write_ok = true; + } +}; + +struct effect_run_depformer_layer_attention { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const auto &view = runtime_ev.ctx.depformer_kv; + const auto &window = runtime_ev.ctx.depformer_position.window; + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t head_dim = dep_dim / lm.depformer_num_heads; + const int32_t capacity = view.position_capacity; + const int32_t valid_positions = window.valid_positions; + const float scale = 1.0f / std::sqrt(static_cast(head_dim)); + const size_t layer_offset = view.layer_cache_offsets[static_cast( + runtime_ev.ctx.depformer_layer_index)]; + runtime_ev.ctx.depformer_layer_attention_ok = false; + std::fill_n(runtime_ev.ctx.attention.data(), static_cast(dep_dim), + 0.0f); + + for (int32_t head = 0; head < lm.depformer_num_heads; ++head) { + const int32_t head_offset = head * head_dim; + const float *q_head = + runtime_ev.ctx.qkv.data() + static_cast(head_offset); + for (int32_t dim = 0; dim < head_dim; ++dim) { + runtime_ev.ctx.q_bf16[static_cast(dim)] = + emel::kernel::detail::fp32_to_bf16(q_head[dim]); + } + for (int32_t physical = 0; physical < capacity; ++physical) { + runtime_ev.ctx.attention_scores[static_cast(physical)] = + -std::numeric_limits::infinity(); + } + for (int32_t position = 0; position < valid_positions; ++position) { + const int32_t unwrapped = window.physical_begin + position; + const int32_t physical = + unwrapped - static_cast(unwrapped >= capacity) * capacity; + const size_t cache_begin = + layer_offset + + static_cast(physical) * static_cast(dep_dim) + + static_cast(head_offset); + runtime_ev.ctx.attention_scores[static_cast(physical)] = + emel::kernel::detail::vec_dot_bf16_ggml( + head_dim, view.key_cache.data() + cache_begin, + runtime_ev.ctx.q_bf16.data()) * + scale; + } + + emel::kernel::detail::soft_max_row_ggml( + capacity, runtime_ev.ctx.attention_scores.data()); + for (int32_t physical = 0; physical < capacity; ++physical) { + runtime_ev.ctx.attention_weights_bf16[static_cast(physical)] = + emel::kernel::detail::fp32_to_bf16( + runtime_ev.ctx.attention_scores[static_cast(physical)]); + } + + float *attention_head = + runtime_ev.ctx.attention.data() + static_cast(head_offset); + for (int32_t dim = 0; dim < head_dim; ++dim) { + double sum = 0.0; + for (int32_t position = 0; position < valid_positions; ++position) { + const int32_t unwrapped = window.physical_begin + position; + const int32_t physical = + unwrapped - + static_cast(unwrapped >= capacity) * capacity; + const size_t value_index = + layer_offset + + static_cast(physical) * static_cast(dep_dim) + + static_cast(head_offset + dim); + sum += static_cast( + emel::kernel::detail::bf16_to_fp32( + view.value_cache[value_index]) * + emel::kernel::detail::bf16_to_fp32( + runtime_ev.ctx + .attention_weights_bf16[static_cast(physical)])); + } + attention_head[dim] = static_cast(sum); + } + } + runtime_ev.ctx.depformer_layer_attention_ok = true; + } +}; + +struct effect_bind_depformer_layer_out_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.projection.data(), static_cast(dep_dim), + 0.0f); + + const auto *projection = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(weight_index)] + .output_projection.tensor; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*projection, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_depformer_layer_out_projection { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.attention.data(), 1u, + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.projection.data(), 1u, + static_cast(dep_dim)), + }; + runtime_ev.ctx.depformer_layer_out_projection_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_apply_depformer_layer_attention_residual { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_residual_ok = false; + for (int32_t index = 0; index < dep_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.projection[static_cast(index)]; + } + runtime_ev.ctx.depformer_layer_residual_ok = true; + } +}; + +struct effect_run_depformer_layer_norm2_rms { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const int32_t dep_dim = model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + + std::fill_n(runtime_ev.ctx.row.data(), static_cast(dep_dim), 0.0f); + std::fill_n(runtime_ev.ctx.normalized.data(), static_cast(dep_dim), + 0.0f); + + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.row.data(), + static_cast(dep_dim)), + }; + detail::set_op_param_f32(rms_ev, 0u, ctx.policy.rms_norm_epsilon); + runtime_ev.ctx.depformer_layer_norm2_rms_ok = + ctx.kernel.process_event(rms_ev); + } +}; + +struct effect_run_depformer_layer_norm2_scale { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const int32_t dep_dim = model.moshi_lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + + const auto *norm2 = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .norm2.tensor; + const auto *alpha = static_cast(norm2->data); + + emel::kernel::event::op_mul mul_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.row.data(), + static_cast(dep_dim)), + .src1 = detail::make_f32_src(alpha, static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.normalized.data(), + static_cast(dep_dim)), + }; + runtime_ev.ctx.depformer_layer_norm2_ok = ctx.kernel.process_event(mul_ev); + } +}; + +struct effect_bind_depformer_layer_gating_in { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + const auto *linear_in = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(weight_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.gating_projection.data(), + static_cast(projection_dim), 0.0f); + + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*linear_in, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_depformer_layer_gating_in { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + const auto *linear_in = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(weight_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.normalized.data(), 1u, + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.gating_projection.data(), 1u, + projection_dim), + }; + runtime_ev.ctx.depformer_layer_gating_in_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_run_depformer_layer_silu_gate_silu { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto *linear_in = ctx.session.contract.lm + .depformer_layers[static_cast( + runtime_ev.ctx.depformer_layer_index)] + .codebooks[static_cast( + runtime_ev.ctx.depformer_weight_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + const uint64_t gate_dim = projection_dim / 2u; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + std::fill_n(runtime_ev.ctx.gating_silu.data(), + static_cast(gate_dim), 0.0f); + std::fill_n(runtime_ev.ctx.gated.data(), static_cast(gate_dim), + 0.0f); + + emel::kernel::event::op_unary silu_ev{ + .src0 = detail::make_f32_src(runtime_ev.ctx.gating_projection.data(), + gate_dim), + .dst = + detail::make_f32_dst(runtime_ev.ctx.gating_silu.data(), gate_dim), + .subop = emel::kernel::event::unary_subop::silu, + }; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = + ctx.kernel.process_event(silu_ev); + } +}; + +struct effect_run_depformer_layer_silu_gate_mul { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto *linear_in = ctx.session.contract.lm + .depformer_layers[static_cast( + runtime_ev.ctx.depformer_layer_index)] + .codebooks[static_cast( + runtime_ev.ctx.depformer_weight_index)] + .gating_input.tensor; + const uint64_t projection_dim = static_cast(linear_in->dims[1]); + const uint64_t gate_dim = projection_dim / 2u; + const float *right = runtime_ev.ctx.gating_projection.data() + gate_dim; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + + emel::kernel::event::op_mul gate_ev{ + .src0 = + detail::make_f32_src(runtime_ev.ctx.gating_silu.data(), gate_dim), + .src1 = detail::make_f32_src(right, gate_dim), + .dst = detail::make_f32_dst(runtime_ev.ctx.gated.data(), gate_dim), + }; + runtime_ev.ctx.depformer_layer_silu_gate_ok = + ctx.kernel.process_event(gate_ev); + } +}; + +struct effect_bind_depformer_layer_gating_out { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + const auto &codebook_layer = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(weight_index)]; + const auto *linear_in = codebook_layer.gating_input.tensor; + const auto *linear_out = codebook_layer.gating_output.tensor; + const uint64_t gate_dim = static_cast(linear_in->dims[1]) / 2u; + (void)gate_dim; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + std::fill_n(runtime_ev.ctx.projection.data(), static_cast(dep_dim), + 0.0f); + + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*linear_out, runtime_ev.ctx.projection_view); + } +}; + +struct effect_run_depformer_layer_gating_out { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + const auto *linear_in = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(weight_index)] + .gating_input.tensor; + const uint64_t gate_dim = static_cast(linear_in->dims[1]) / 2u; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + emel::kernel::event::op_mul_mat projection_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.gated.data(), 1u, gate_dim), + .dst = detail::make_f32_dst(runtime_ev.ctx.projection.data(), 1u, + static_cast(dep_dim)), + }; + runtime_ev.ctx.depformer_layer_gating_out_ok = + ctx.kernel.process_event(projection_ev); + } +}; + +struct effect_apply_depformer_layer_ff_residual { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + for (int32_t index = 0; index < dep_dim; ++index) { + runtime_ev.ctx.hidden[static_cast(index)] += + runtime_ev.ctx.projection[static_cast(index)]; + } + runtime_ev.ctx.depformer_layer_ff_residual_ok = true; + } +}; + +struct effect_advance_depformer_layer { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + ++runtime_ev.ctx.depformer_layer_index; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + runtime_ev.ctx.depformer_layer_attention_ok = false; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.depformer_layer_residual_ok = false; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + } +}; + +struct effect_bind_depformer_token_logits { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const auto *linear = + ctx.session.contract.lm + .depformer_output_projections[static_cast(codebook)] + .tensor; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.projection_view = {}; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + runtime_ev.ctx.projection_view_bound = + detail::bind_tensor_view(*linear, runtime_ev.ctx.projection_view); + } +}; + +struct effect_select_depformer_token { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + runtime_ev.ctx.depformer_logits_ok = false; + emel::kernel::event::op_mul_mat_argmax logits_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), 1u, + static_cast(dep_dim)), + .dst = detail::make_f32_dst(&runtime_ev.ctx.best_score, 1u, 1u), + .index_out = &runtime_ev.ctx.best_index, + }; + runtime_ev.ctx.depformer_logits_ok = ctx.kernel.process_event(logits_ev); + } +}; + +struct effect_compute_depformer_token_logits { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t audio_card = ctx.session.audio_card; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.best_index = -1; + runtime_ev.ctx.best_score = 0.0f; + emel::kernel::event::op_mul_mat logits_ev{ + .src0 = runtime_ev.ctx.projection_view, + .src1 = detail::make_f32_src(runtime_ev.ctx.hidden.data(), 1u, + static_cast(dep_dim)), + .dst = detail::make_f32_dst(runtime_ev.ctx.logits.data(), 1u, + static_cast(audio_card)), + }; + runtime_ev.ctx.depformer_logits_ok = ctx.kernel.process_event(logits_ev); + } +}; + +struct effect_select_depformer_sampling_token { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.sampler_error = + emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k sample_ev{ + runtime_ev.ctx.logits, ctx.session.audio_card, + ctx.sampling.audio_temperature, ctx.sampling.audio_top_k, + runtime_ev.ctx.sampling_indices, runtime_ev.ctx.top_scores, + runtime_ev.ctx.top_indices, ctx.sampling.random_state, + runtime_ev.ctx.best_index, runtime_ev.ctx.best_score, + runtime_ev.ctx.sampler_error}; + runtime_ev.ctx.sampler_accepted = ctx.sampler->process_event(sample_ev); + } +}; + +struct effect_publish_depformer_token { + void operator()(const event::step_run &runtime_ev, context &) const noexcept { + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + runtime_ev.request.audio_tokens_out[static_cast(codebook)] = + runtime_ev.ctx.best_index; + } +}; + +struct effect_advance_depformer_codebook { + void operator()(const event::step_run &runtime_ev, + const context &) const noexcept { + ++runtime_ev.ctx.depformer_codebook_index; + runtime_ev.ctx.depformer_input_ok = false; + runtime_ev.ctx.depformer_layer_index = 0; + runtime_ev.ctx.depformer_layer_norm_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm_ok = false; + runtime_ev.ctx.depformer_layer_projection_ok = false; + runtime_ev.ctx.projection_view_bound = false; + runtime_ev.ctx.depformer_layer_cache_write_ok = false; + runtime_ev.ctx.depformer_layer_attention_ok = false; + runtime_ev.ctx.depformer_layer_out_projection_ok = false; + runtime_ev.ctx.depformer_layer_residual_ok = false; + runtime_ev.ctx.depformer_layer_norm2_rms_ok = false; + runtime_ev.ctx.depformer_layer_norm2_ok = false; + runtime_ev.ctx.depformer_layer_gating_in_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_silu_ok = false; + runtime_ev.ctx.depformer_layer_silu_gate_ok = false; + runtime_ev.ctx.depformer_layer_gating_out_ok = false; + runtime_ev.ctx.depformer_layer_ff_residual_ok = false; + runtime_ev.ctx.depformer_logits_ok = false; + runtime_ev.ctx.depformer_weight_index = -1; + } +}; + +struct effect_bind_temporal_kv { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.temporal_kv = ctx.temporal_kv; + runtime_ev.ctx.temporal_kv_bound = true; + } +}; + +struct effect_advance_temporal_position { + void operator()(const event::step_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.temporal_position = {}; + runtime_ev.ctx.temporal_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.temporal_position_accepted = + ctx.temporal_positions->process_event( + emel::memory::streaming::event::advance{ + .result = runtime_ev.ctx.temporal_position, + .error_out = runtime_ev.ctx.temporal_position_error}); + } +}; + +template struct effect_store_error_out { + void operator()(const runtime_event_type &runtime_ev, + context &) const noexcept { + *runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +struct effect_emit_initialize_done { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.request.on_done(events::initialize_done{ + .request = &runtime_ev.request, + .n_q = ctx.session.codebook_count - 1, + .dep_q = ctx.session.dep_q, + }); + } +}; + +struct effect_emit_initialize_error { + void operator()(const event::initialize_run &runtime_ev, + context &) const noexcept { + runtime_ev.request.on_error(events::initialize_error{ + .request = &runtime_ev.request, + .err = runtime_ev.ctx.err, + }); + } +}; + +struct effect_reset_session { + void operator()(const event::reset_run &, context &ctx) const noexcept { + ctx.session = {}; + } +}; + +struct effect_reset_temporal_positions { + void operator()(const event::reset_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.temporal_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.temporal_position_accepted = + ctx.temporal_positions->process_event( + emel::memory::streaming::event::reset{ + .error_out = runtime_ev.ctx.temporal_position_error}); + } +}; + +struct effect_reset_bound_depformer_positions { + void operator()(const event::reset_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.depformer_position_error = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + runtime_ev.ctx.depformer_position_accepted = + ctx.depformer_positions->process_event( + emel::memory::streaming::event::reset{ + .error_out = runtime_ev.ctx.depformer_position_error}); + } +}; + +struct effect_mark_reset_failed { + void operator()(const event::reset_run &runtime_ev, + context &) const noexcept { + runtime_ev.ctx.err = detail_ns::to_error(error::reset_failed); + } +}; + +struct effect_mark_unexpected { + template + void operator()(const unexpected_event_type &ev, context &) const noexcept { + if constexpr (requires { ev.ctx.err; }) { + ev.ctx.err = detail_ns::to_error(error::unexpected_event); + } + } +}; + +struct effect_mark_unexpected_and_store { + template + void operator()(const unexpected_event_type &ev, + context &ctx) const noexcept { + effect_mark_unexpected{}(ev, ctx); + if constexpr (requires { + ev.ctx.err; + ev.request.error_out; + }) { + *ev.request.error_out = ev.ctx.err; + } + } +}; + +} // namespace emel::speech::predictor::moshi::executor::action diff --git a/src/emel/speech/predictor/moshi/executor/any.hpp b/src/emel/speech/predictor/moshi/executor/any.hpp new file mode 100644 index 00000000..5c21a728 --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/any.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "emel/speech/predictor/moshi/executor/sm.hpp" + +namespace emel::speech::predictor::moshi::executor { + +using kv_views = action::kv_views; +using dependencies = action::dependencies; +using policies = action::policies; +using capacities = action::capacities; + +} // namespace emel::speech::predictor::moshi::executor diff --git a/src/emel/speech/predictor/moshi/executor/context.hpp b/src/emel/speech/predictor/moshi/executor/context.hpp new file mode 100644 index 00000000..197f9fca --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/context.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include + +#include "emel/kernel/sm.hpp" +#include "emel/logits/sampler/sm.hpp" +#include "emel/memory/streaming/sm.hpp" +#include "emel/memory/view.hpp" +#include "emel/model/data.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/executor/detail.hpp" + +namespace emel::speech::predictor::moshi::executor::action { + +struct kv_views { + detail::temporal_kv_view temporal = {}; + detail::depformer_kv_view depformer = {}; + emel::memory::streaming::sm *temporal_positions = nullptr; + emel::memory::streaming::sm *depformer_positions = nullptr; +}; + +struct policies { + float rms_norm_epsilon = 0.0f; + uint32_t zero_seed_state = 0u; + uint32_t sampling_modulus = 0u; + int32_t token_zero = 0; +}; + +struct capacities { + uint64_t hidden_dim = 0u; + uint64_t temporal_context = 0u; + uint64_t depformer_context = 0u; + uint64_t sampling_card = 0u; + uint64_t sampling_top_k = 0u; +}; + +struct dependencies { + kv_views kv = {}; + emel::kernel::sm &kernel; + emel::logits::sampler::sm *sampler = nullptr; + policies policy = {}; + capacities capacity = {}; +}; + +struct runtime { + const emel::model::data *model = nullptr; + emel::model::moshi::detail::execution_contract contract = {}; + int32_t codebook_count = 0; + int32_t dep_q = 0; + int32_t text_card = 0; + int32_t audio_card = 0; + int32_t hidden_dim = 0; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)> + temporal_input_projections = {}; + std::array( + emel::model::data::moshi_lm_hparams::k_max_delays)>, + static_cast( + emel::model::data::moshi_lm_hparams::k_max_delays)> + depformer_input_projections = {}; +}; + +struct sampling_config { + bool enabled = false; + bool consume_forced_text = false; + float audio_temperature = 0.0f; + float text_temperature = 0.0f; + int32_t audio_top_k = 0; + int32_t text_top_k = 0; + uint32_t random_state = 1u; +}; + +struct context { + explicit context(const dependencies &deps) + : temporal_kv(deps.kv.temporal), depformer_kv(deps.kv.depformer), + temporal_positions(deps.kv.temporal_positions), + depformer_positions(deps.kv.depformer_positions), kernel(deps.kernel), + sampler(deps.sampler), policy(deps.policy), capacity(deps.capacity) {} + + runtime session = {}; + sampling_config sampling = {}; + const detail::temporal_kv_view temporal_kv; + const detail::depformer_kv_view depformer_kv; + emel::memory::streaming::sm *temporal_positions = nullptr; + emel::memory::streaming::sm *depformer_positions = nullptr; + emel::kernel::sm &kernel; + emel::logits::sampler::sm *sampler = nullptr; + const policies policy; + const capacities capacity; +}; + +} // namespace emel::speech::predictor::moshi::executor::action diff --git a/src/emel/speech/predictor/moshi/executor/detail.hpp b/src/emel/speech/predictor/moshi/executor/detail.hpp new file mode 100644 index 00000000..796a1233 --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/detail.hpp @@ -0,0 +1,195 @@ +#pragma once + +#include +#include +#include + +#include "emel/kernel/detail.hpp" +#include "emel/kernel/events.hpp" +#include "emel/memory/view.hpp" +#include "emel/model/data.hpp" + +namespace emel::speech::predictor::moshi::executor::detail { + +using dtype = emel::kernel::event::dtype; +using tensor_record = emel::model::data::tensor_record; +using tensor_view = emel::kernel::event::tensor_view; +using tensor_view_mut = emel::kernel::event::tensor_view_mut; + +inline constexpr uint64_t k_max_hidden_dim = 8192u; +inline constexpr uint64_t k_max_qkv_dim = k_max_hidden_dim * 3u; +inline constexpr uint64_t k_max_gating_projection_dim = k_max_hidden_dim * 8u; +inline constexpr uint64_t k_max_gating_dim = k_max_gating_projection_dim / 2u; +inline constexpr uint64_t k_max_temporal_context = 8192u; +inline constexpr uint64_t k_max_depformer_context = 512u; +inline constexpr uint64_t k_max_sampling_card = 65536u; +inline constexpr uint64_t k_max_sampling_top_k = 1024u; + +struct streaming_kv_view { + std::span key_cache = {}; + std::span value_cache = {}; + std::span layer_cache_offsets = {}; + int32_t layer_count = 0; + int32_t position_capacity = 0; + int32_t kv_dim = 0; +}; + +using temporal_kv_view = streaming_kv_view; +using depformer_kv_view = streaming_kv_view; + +template +void fill_default_nb(tensor_type &tensor) noexcept { + const uint64_t elem_size = emel::kernel::detail::dtype_size_bytes( + emel::kernel::detail::dtype_code(tensor.type)); + tensor.nb[0] = elem_size; + tensor.nb[1] = tensor.nb[0] * tensor.ne[0]; + tensor.nb[2] = tensor.nb[1] * tensor.ne[1]; + tensor.nb[3] = tensor.nb[2] * tensor.ne[2]; +} + +inline bool supported_get_rows_dtype(const dtype type) noexcept { + const auto code = emel::kernel::detail::dtype_code(type); + return code == emel::kernel::detail::dtype_f32 || + code == emel::kernel::detail::dtype_f16 || + code == emel::kernel::detail::dtype_bf16 || + code == emel::kernel::detail::dtype_q4_0 || + code == emel::kernel::detail::dtype_q8_0 || + code == emel::kernel::detail::dtype_q4_k; +} + +inline bool supported_argmax_dtype(const dtype type) noexcept { + const auto code = emel::kernel::detail::dtype_code(type); + return code == emel::kernel::detail::dtype_f32 || + emel::kernel::detail::is_native_quantized_dtype(code) || + emel::kernel::detail::is_packed_q4_vector_dtype(code); +} + +inline bool supported_mul_mat_dtype(const dtype type) noexcept { + const auto code = emel::kernel::detail::dtype_code(type); + return code == emel::kernel::detail::dtype_f32 || + code == emel::kernel::detail::dtype_f16 || + emel::kernel::detail::is_native_quantized_dtype(code) || + emel::kernel::detail::is_packed_q4_vector_dtype(code); +} + +inline bool is_quantized_row_tensor(const dtype type) noexcept { + const auto code = emel::kernel::detail::dtype_code(type); + return emel::kernel::detail::is_native_quantized_dtype(code) || + emel::kernel::detail::is_packed_q4_vector_dtype(code); +} + +inline bool bind_tensor_view(const tensor_record &tensor, + tensor_view &view) noexcept { + if (tensor.data == nullptr || tensor.n_dims <= 0 || tensor.dims[0] <= 0) { + return false; + } + + view = {}; + view.data = tensor.data; + view.type = static_cast(tensor.type); + view.ne = { + static_cast(tensor.dims[0]), + tensor.n_dims > 1 ? static_cast(tensor.dims[1]) : 1u, + tensor.n_dims > 2 ? static_cast(tensor.dims[2]) : 1u, + tensor.n_dims > 3 ? static_cast(tensor.dims[3]) : 1u, + }; + + if (is_quantized_row_tensor(view.type)) { + const uint8_t code = emel::kernel::detail::dtype_code(view.type); + if (emel::kernel::detail::is_packed_q4_vector_dtype(code)) { + const uint64_t group_count = + emel::kernel::detail::quant::packed_q4_k_x8_group_count(view.ne[1]); + const size_t group_bytes = + emel::kernel::detail::quant::packed_q4_k_x8_group_storage_bytes( + view.ne[0]); + if (group_count == 0u || group_bytes == 0u) { + return false; + } + view.nb[0] = 1u; + view.nb[1] = group_bytes; + view.nb[2] = group_bytes * group_count; + view.nb[3] = view.nb[2]; + return true; + } + const size_t row_bytes = + emel::kernel::detail::quantized_row_storage_bytes(code, view.ne[0]); + if (row_bytes == 0u) { + return false; + } + view.nb[0] = 1u; + view.nb[1] = row_bytes; + view.nb[2] = row_bytes * view.ne[1]; + view.nb[3] = view.nb[2] * view.ne[2]; + return true; + } + + fill_default_nb(view); + return view.nb[0] != 0u; +} + +inline tensor_view make_f32_src(const float *data, const uint64_t ne0, + const uint64_t ne1 = 1u) noexcept { + tensor_view view{}; + view.data = data; + view.type = dtype::f32; + view.ne = {ne0, ne1, 1u, 1u}; + fill_default_nb(view); + return view; +} + +inline tensor_view make_i32_src(const int32_t *data, + const uint64_t ne0) noexcept { + tensor_view view{}; + view.data = data; + view.type = dtype::i32; + view.ne = {ne0, 1u, 1u, 1u}; + fill_default_nb(view); + return view; +} + +inline tensor_view_mut make_f32_dst(float *data, const uint64_t ne0, + const uint64_t ne1 = 1u) noexcept { + tensor_view_mut view{}; + view.data = data; + view.type = dtype::f32; + view.ne = {ne0, ne1, 1u, 1u}; + fill_default_nb(view); + return view; +} + +template +inline void set_op_param_f32(event_type &ev, const uint32_t slot, + const float value) noexcept { + const auto offset = slot * static_cast(sizeof(float)); + if (offset + sizeof(float) <= ev.op_params.size()) { + auto *bytes = ev.op_params.data() + offset; + std::memcpy(bytes, &value, sizeof(float)); + ev.op_params_size = offset + static_cast(sizeof(float)); + } +} + +template +inline void set_op_param_i32(event_type &ev, const uint32_t slot, + const int32_t value) noexcept { + const auto offset = slot * static_cast(sizeof(int32_t)); + if (offset + sizeof(int32_t) <= ev.op_params.size()) { + auto *bytes = ev.op_params.data() + offset; + std::memcpy(bytes, &value, sizeof(int32_t)); + ev.op_params_size = offset + static_cast(sizeof(int32_t)); + } +} + +inline bool tensor_shape(const tensor_record *tensor, const int64_t ne0, + const int64_t ne1 = 1) noexcept { + return tensor != nullptr && tensor->data != nullptr && tensor->n_dims >= 1 && + tensor->dims[0] == ne0 && + ((tensor->n_dims == 1 && ne1 == 1) || + (tensor->n_dims >= 2 && tensor->dims[1] == ne1)); +} + +inline bool token_in_embedding_range(const int32_t token, const int32_t card, + const int32_t token_zero) noexcept { + return token >= token_zero && token <= card; +} + +} // namespace emel::speech::predictor::moshi::executor::detail diff --git a/src/emel/speech/predictor/moshi/executor/errors.hpp b/src/emel/speech/predictor/moshi/executor/errors.hpp new file mode 100644 index 00000000..663c73fe --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/errors.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::speech::predictor::moshi::executor { + +enum class error : emel::error::type { + none = 0, + not_initialized = 1, + bind_failed = 2, + request_shape = 3, + model_mismatch = 4, + graph_execution_unsupported = 5, + unexpected_event = 6, + reset_failed = 7, +}; + +} // namespace emel::speech::predictor::moshi::executor diff --git a/src/emel/speech/predictor/moshi/executor/events.hpp b/src/emel/speech/predictor/moshi/executor/events.hpp new file mode 100644 index 00000000..737c118c --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/events.hpp @@ -0,0 +1,177 @@ +#pragma once + +#include + +#include "emel/callback.hpp" +#include "emel/error/error.hpp" +#include "emel/memory/streaming/events.hpp" +#include "emel/model/data.hpp" +#include "emel/speech/predictor/moshi/events.hpp" +#include "emel/speech/predictor/moshi/executor/detail.hpp" +#include "emel/speech/predictor/moshi/executor/errors.hpp" + +namespace emel::speech::predictor::moshi::executor::events { + +struct initialize_done; +struct initialize_error; + +} // namespace emel::speech::predictor::moshi::executor::events + +namespace emel::speech::predictor::moshi::executor::event { + +struct initialize { + explicit initialize(const emel::model::data &model_ref) noexcept + : model(model_ref) {} + + const emel::model::data &model; + bool sampling_enabled = false; + bool sampling_consume_forced_text = false; + float sampling_audio_temperature = 0.0f; + float sampling_text_temperature = 0.0f; + int32_t sampling_audio_top_k = 0; + int32_t sampling_text_top_k = 0; + uint32_t sampling_seed = 0u; + emel::error::type *error_out = nullptr; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct initialize_ctx { + emel::error::type err = emel::error::cast(error::none); +}; + +struct step_ctx { + emel::error::type err = emel::error::cast(error::none); + bool input_embedding_ok = false; + bool input_text_embedding_ok = false; + bool input_audio_embedding_ok = false; + bool embedding_view_bound = false; + bool embedding_row_ok = false; + bool projection_view_bound = false; + bool temporal_kv_bound = false; + bool temporal_position_accepted = false; + bool temporal_layer_norm_rms_ok = false; + bool temporal_layer_norm_ok = false; + bool temporal_layer_projection_ok = false; + bool temporal_layer_rope_ok = false; + bool temporal_layer_cache_write_ok = false; + bool temporal_layer_attention_ok = false; + bool temporal_layer_out_projection_ok = false; + bool temporal_layer_residual_ok = false; + bool temporal_layer_norm2_rms_ok = false; + bool temporal_layer_norm2_ok = false; + bool temporal_layer_gating_in_ok = false; + bool temporal_layer_silu_gate_silu_ok = false; + bool temporal_layer_silu_gate_ok = false; + bool temporal_layer_gating_out_ok = false; + bool temporal_layer_ff_residual_ok = false; + bool temporal_out_norm_rms_ok = false; + bool temporal_out_norm_ok = false; + bool text_logits_ok = false; + bool depformer_kv_bound = false; + bool depformer_position_accepted = false; + bool depformer_input_ok = false; + bool depformer_input_projection_bound = false; + bool depformer_input_projection_ok = false; + bool depformer_layer_norm_rms_ok = false; + bool depformer_layer_norm_ok = false; + bool depformer_layer_projection_ok = false; + bool depformer_layer_cache_write_ok = false; + bool depformer_layer_attention_ok = false; + bool depformer_layer_out_projection_ok = false; + bool depformer_layer_residual_ok = false; + bool depformer_layer_norm2_rms_ok = false; + bool depformer_layer_norm2_ok = false; + bool depformer_layer_gating_in_ok = false; + bool depformer_layer_silu_gate_silu_ok = false; + bool depformer_layer_silu_gate_ok = false; + bool depformer_layer_gating_out_ok = false; + bool depformer_layer_ff_residual_ok = false; + bool depformer_logits_ok = false; + bool sampler_accepted = false; + int32_t row_index = 0; + int32_t embedding_dim = 0; + int32_t best_index = -1; + int32_t input_audio_codebook_index = 0; + int32_t temporal_layer_index = 0; + int32_t temporal_rope_position = -1; + int32_t temporal_position_error = 0; + int32_t depformer_codebook_index = 0; + int32_t depformer_weight_index = -1; + int32_t depformer_layer_index = 0; + int32_t depformer_position_error = 0; + emel::error::type sampler_error = {}; + float best_score = 0.0f; + detail::temporal_kv_view temporal_kv = {}; + detail::depformer_kv_view depformer_kv = {}; + emel::memory::streaming::advance_result temporal_position = {}; + emel::memory::streaming::advance_result depformer_position = {}; + detail::tensor_view embedding_view = {}; + detail::tensor_view projection_view = {}; + alignas(64) std::array logits = {}; + alignas(64) std::array top_scores = {}; + alignas( + 64) std::array top_indices = {}; + alignas(64) + std::array sampling_indices = {}; + alignas(64) std::array row = {}; + alignas(64) std::array hidden = {}; + alignas(64) std::array transformer_out = {}; + alignas(64) std::array normalized = {}; + alignas(64) std::array attention = {}; + alignas(64) std::array projection = {}; + alignas(64) std::array qkv = {}; + alignas(64) std::array rope = {}; + alignas(64) + std::array gating_projection = + {}; + alignas(64) std::array gating_silu = {}; + alignas(64) std::array gated = {}; + alignas(64) + std::array attention_scores = {}; + alignas(64) std::array q_bf16 = {}; + alignas(64) std::array< + uint16_t, detail::k_max_temporal_context> attention_weights_bf16 = {}; +}; + +struct initialize_run { + const initialize &request; + initialize_ctx &ctx; +}; + +struct step_run { + const emel::speech::predictor::moshi::event::graph_step &request; + step_ctx &ctx; +}; + +struct reset {}; + +struct reset_ctx { + emel::error::type err = emel::error::cast(error::none); + bool temporal_position_accepted = false; + bool depformer_position_accepted = false; + int32_t temporal_position_error = 0; + int32_t depformer_position_error = 0; +}; + +struct reset_run { + const reset &request; + reset_ctx &ctx; +}; + +} // namespace emel::speech::predictor::moshi::executor::event + +namespace emel::speech::predictor::moshi::executor::events { + +struct initialize_done { + const event::initialize *request = nullptr; + int32_t n_q = 0; + int32_t dep_q = 0; +}; + +struct initialize_error { + const event::initialize *request = nullptr; + emel::error::type err = emel::error::cast(error::none); +}; + +} // namespace emel::speech::predictor::moshi::executor::events diff --git a/src/emel/speech/predictor/moshi/executor/guards.hpp b/src/emel/speech/predictor/moshi/executor/guards.hpp new file mode 100644 index 00000000..29a99c70 --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/guards.hpp @@ -0,0 +1,2442 @@ +#pragma once + +#include +#include +#include + +#include "emel/model/loader/errors.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/executor/context.hpp" +#include "emel/speech/predictor/moshi/executor/detail.hpp" +#include "emel/speech/predictor/moshi/executor/events.hpp" + +namespace emel::speech::predictor::moshi::executor::guard { + +inline bool guard_cache_span_valid(const size_t layer_offset, + const size_t first_position, + const size_t position_count, + const size_t dim, + const size_t key_cache_size, + const size_t value_cache_size) noexcept { + if (dim == 0u || first_position > std::numeric_limits::max() / dim || + position_count > std::numeric_limits::max() / dim) { + return false; + } + const size_t first_element = first_position * dim; + const size_t element_count = position_count * dim; + if (layer_offset > std::numeric_limits::max() - first_element) { + return false; + } + const size_t begin = layer_offset + first_element; + return begin <= key_cache_size && begin <= value_cache_size && + element_count <= key_cache_size - begin && + element_count <= value_cache_size - begin; +} + +struct guard_bind_contract_valid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const bool sampling_config_valid = + !runtime_ev.request.sampling_enabled || + (std::isfinite(runtime_ev.request.sampling_audio_temperature) && + runtime_ev.request.sampling_audio_temperature > 0.0f && + std::isfinite(runtime_ev.request.sampling_text_temperature) && + runtime_ev.request.sampling_text_temperature > 0.0f && + runtime_ev.request.sampling_audio_top_k > 0 && + runtime_ev.request.sampling_text_top_k > 0); + return emel::model::moshi::detail::validate_execution_contract(model) == + emel::error::cast(emel::model::loader::error::none) && + model.moshi_component_id == emel::model::data::moshi_component::lm && + model.moshi_lm.n_q > 0 && model.moshi_lm.dep_q > 0 && + model.moshi_lm.dep_q <= model.moshi_lm.n_q && + model.moshi_lm.text_card > 0 && model.moshi_lm.card > 0 && + model.moshi_lm.dim > 0 && + (!runtime_ev.request.sampling_enabled || ctx.sampler != nullptr) && + sampling_config_valid && ctx.policy.rms_norm_epsilon > 0.0f && + ctx.policy.sampling_modulus > 1u && + ctx.policy.zero_seed_state % ctx.policy.sampling_modulus != 0u && + ctx.policy.token_zero < 0 && ctx.capacity.hidden_dim > 0u && + ctx.capacity.hidden_dim >= + static_cast(model.moshi_lm.dim) && + ctx.capacity.hidden_dim >= + static_cast(model.moshi_lm.depformer_dim) && + ctx.capacity.hidden_dim <= detail::k_max_hidden_dim && + ctx.capacity.temporal_context > 0u && + ctx.capacity.temporal_context <= detail::k_max_temporal_context && + ctx.capacity.depformer_context > 0u && + ctx.capacity.depformer_context <= detail::k_max_depformer_context && + ctx.capacity.sampling_card > 0u && + ctx.capacity.sampling_card <= detail::k_max_sampling_card && + ctx.capacity.sampling_top_k > 0u && + ctx.capacity.sampling_top_k <= detail::k_max_sampling_top_k && + (!runtime_ev.request.sampling_enabled || + (ctx.capacity.sampling_card >= + static_cast(model.moshi_lm.text_card) && + ctx.capacity.sampling_card >= + static_cast(model.moshi_lm.card) && + ctx.capacity.sampling_top_k >= + static_cast( + std::min(runtime_ev.request.sampling_text_top_k, + model.moshi_lm.text_card)) && + ctx.capacity.sampling_top_k >= + static_cast( + std::min(runtime_ev.request.sampling_audio_top_k, + model.moshi_lm.card)))); + } +}; + +struct guard_bind_contract_invalid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_bind_contract_valid{}(runtime_ev, ctx); + } +}; + +struct guard_bound_root_operands_supported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const auto *text_emb = ctx.session.contract.lm.text_embedding.tensor; + const auto *output_norm = ctx.session.contract.lm.output_norm.tensor; + const auto *text_output_projection = + ctx.session.contract.lm.text_output_projection.tensor; + bool supported = + detail::tensor_shape(text_emb, hidden_dim, lm.text_card + 1) && + detail::supported_get_rows_dtype( + static_cast(text_emb->type)) && + detail::tensor_shape(output_norm, hidden_dim) && + static_cast(output_norm->type) == detail::dtype::f32 && + detail::tensor_shape(text_output_projection, hidden_dim, + lm.text_card) && + detail::supported_argmax_dtype( + static_cast(text_output_projection->type)); + for (int32_t index = 0; index < lm.n_q; ++index) { + const auto *audio_emb = + ctx.session.contract.lm.audio_embeddings[static_cast(index)] + .tensor; + supported = supported && + detail::tensor_shape(audio_emb, hidden_dim, lm.card + 1) && + detail::supported_get_rows_dtype( + static_cast(audio_emb->type)); + } + const int32_t dep_dim = lm.depformer_dim; + const auto *depformer_text_embedding = + ctx.session.contract.lm.depformer_text_embedding.tensor; + supported = supported && dep_dim > 0 && + detail::tensor_shape(depformer_text_embedding, dep_dim, + lm.text_card + 1) && + detail::supported_get_rows_dtype( + static_cast(depformer_text_embedding->type)); + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + int32_t weight_index = codebook; + if (lm.depformer_weight_schedule_count > 0u) { + if (static_cast(codebook) >= + lm.depformer_weight_schedule_count) { + supported = false; + } else { + weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + } + } + if (weight_index < 0 || weight_index >= lm.dep_q) { + supported = false; + } else { + const auto *input_projection = + ctx.session.contract.lm + .depformer_input_projections[static_cast(weight_index)] + .tensor; + supported = + supported && + detail::tensor_shape(input_projection, hidden_dim, dep_dim) && + detail::supported_mul_mat_dtype( + static_cast(input_projection->type)); + } + + const auto *output_projection = + ctx.session.contract.lm + .depformer_output_projections[static_cast(codebook)] + .tensor; + supported = supported && + detail::tensor_shape(output_projection, dep_dim, lm.card) && + detail::supported_argmax_dtype( + static_cast(output_projection->type)); + if (codebook > 0) { + const auto *audio_embedding = + ctx.session.contract.lm + .depformer_audio_embeddings[static_cast(codebook - 1)] + .tensor; + supported = + supported && + detail::tensor_shape(audio_embedding, dep_dim, lm.card + 1) && + detail::supported_get_rows_dtype( + static_cast(audio_embedding->type)); + } + } + for (int32_t layer_index = 0; layer_index < lm.num_layers; ++layer_index) { + const auto &layer = + ctx.session.contract.lm + .temporal_layers[static_cast(layer_index)]; + const auto *norm1 = layer.norm1.tensor; + const auto *output_projection = layer.output_projection.tensor; + const auto *norm2 = layer.norm2.tensor; + const auto *gating_input = layer.gating_input.tensor; + const auto *gating_output = layer.gating_output.tensor; + int64_t gating_projection_dim = 0; + if (gating_input != nullptr && gating_input->n_dims > 1) { + gating_projection_dim = gating_input->dims[1]; + } + const int64_t gating_dim = gating_projection_dim / 2; + supported = + supported && detail::tensor_shape(norm1, hidden_dim) && + static_cast(norm1->type) == detail::dtype::f32 && + detail::tensor_shape(output_projection, hidden_dim, hidden_dim) && + detail::supported_mul_mat_dtype( + static_cast(output_projection->type)) && + detail::tensor_shape(norm2, hidden_dim) && + static_cast(norm2->type) == detail::dtype::f32 && + gating_projection_dim > 0 && (gating_projection_dim % 2) == 0 && + static_cast(gating_projection_dim) <= + ctx.capacity.hidden_dim * 8u && + detail::tensor_shape(gating_input, hidden_dim, + gating_projection_dim) && + detail::supported_mul_mat_dtype( + static_cast(gating_input->type)) && + gating_dim > 0 && + static_cast(gating_dim) <= ctx.capacity.hidden_dim * 4u && + detail::tensor_shape(gating_output, gating_dim, hidden_dim) && + detail::supported_mul_mat_dtype( + static_cast(gating_output->type)); + } + for (int32_t layer_index = 0; layer_index < lm.depformer_num_layers; + ++layer_index) { + const auto &layer = + ctx.session.contract.lm + .depformer_layers[static_cast(layer_index)]; + const auto *norm1 = layer.norm1.tensor; + const auto *norm2 = layer.norm2.tensor; + supported = + supported && detail::tensor_shape(norm1, dep_dim) && + static_cast(norm1->type) == detail::dtype::f32 && + detail::tensor_shape(norm2, dep_dim) && + static_cast(norm2->type) == detail::dtype::f32; + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + int32_t weight_index = codebook; + if (lm.depformer_weight_schedule_count > 0u) { + if (static_cast(codebook) >= + lm.depformer_weight_schedule_count) { + supported = false; + } else { + weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + } + } + if (weight_index < 0 || weight_index >= lm.dep_q) { + supported = false; + continue; + } + const auto &codebook_layer = + layer.codebooks[static_cast(weight_index)]; + const auto *output_projection = codebook_layer.output_projection.tensor; + const auto *gating_input = codebook_layer.gating_input.tensor; + const auto *gating_output = codebook_layer.gating_output.tensor; + int64_t gating_projection_dim = 0; + if (gating_input != nullptr && gating_input->n_dims > 1) { + gating_projection_dim = gating_input->dims[1]; + } + const int64_t gating_dim = gating_projection_dim / 2; + supported = + supported && + detail::tensor_shape(output_projection, dep_dim, dep_dim) && + detail::supported_mul_mat_dtype( + static_cast(output_projection->type)) && + gating_projection_dim > 0 && (gating_projection_dim % 2) == 0 && + static_cast(gating_projection_dim) <= + ctx.capacity.hidden_dim * 8u && + detail::tensor_shape(gating_input, dep_dim, + gating_projection_dim) && + detail::supported_mul_mat_dtype( + static_cast(gating_input->type)) && + gating_dim > 0 && + static_cast(gating_dim) <= ctx.capacity.hidden_dim * 4u && + detail::tensor_shape(gating_output, gating_dim, dep_dim) && + detail::supported_mul_mat_dtype( + static_cast(gating_output->type)); + } + } + return supported; + } +}; + +struct guard_bound_root_operands_unsupported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_bound_root_operands_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_split_projection_layout_supported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + bool supported = lm.num_layers > 0; + for (int32_t layer_index = 0; layer_index < lm.num_layers; ++layer_index) { + const auto *projection = + ctx.session.contract.lm + .temporal_layers[static_cast(layer_index)] + .split_input_projection.tensor; + supported = supported && + detail::tensor_shape(projection, lm.dim, lm.dim * 3) && + detail::supported_mul_mat_dtype( + static_cast(projection->type)); + } + return supported; + } +}; + +struct guard_temporal_fused_projection_layout_supported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + if (guard_temporal_split_projection_layout_supported{}(runtime_ev, ctx)) { + return false; + } + const auto &lm = runtime_ev.request.model.moshi_lm; + bool supported = lm.num_layers > 0; + for (int32_t layer_index = 0; layer_index < lm.num_layers; ++layer_index) { + const auto *projection = + ctx.session.contract.lm + .temporal_layers[static_cast(layer_index)] + .fused_input_projection.tensor; + supported = supported && + detail::tensor_shape(projection, lm.dim, lm.dim * 3) && + detail::supported_mul_mat_dtype( + static_cast(projection->type)); + } + return supported; + } +}; + +struct guard_temporal_projection_layout_unsupported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_split_projection_layout_supported{}(runtime_ev, + ctx) && + !guard_temporal_fused_projection_layout_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_split_projection_layout_supported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + bool supported = lm.depformer_num_layers > 0 && lm.dep_q > 0; + for (int32_t layer_index = 0; layer_index < lm.depformer_num_layers; + ++layer_index) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + int32_t weight_index = codebook; + if (lm.depformer_weight_schedule_count > 0u) { + if (static_cast(codebook) >= + lm.depformer_weight_schedule_count) { + supported = false; + } else { + weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + } + } + if (weight_index < 0 || weight_index >= lm.dep_q) { + supported = false; + continue; + } + const auto *projection = + ctx.session.contract.lm + .depformer_layers[static_cast(layer_index)] + .codebooks[static_cast(weight_index)] + .split_input_projection.tensor; + supported = supported && + detail::tensor_shape(projection, lm.depformer_dim, + lm.depformer_dim * 3) && + detail::supported_mul_mat_dtype( + static_cast(projection->type)); + } + } + return supported; + } +}; + +struct guard_depformer_fused_projection_layout_supported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + if (guard_depformer_split_projection_layout_supported{}(runtime_ev, ctx)) { + return false; + } + const auto &lm = runtime_ev.request.model.moshi_lm; + bool supported = lm.depformer_num_layers > 0 && lm.dep_q > 0; + for (int32_t layer_index = 0; layer_index < lm.depformer_num_layers; + ++layer_index) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + int32_t weight_index = codebook; + if (lm.depformer_weight_schedule_count > 0u) { + if (static_cast(codebook) >= + lm.depformer_weight_schedule_count) { + supported = false; + } else { + weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + } + } + if (weight_index < 0 || weight_index >= lm.dep_q) { + supported = false; + continue; + } + const auto *projection = + ctx.session.contract.lm + .depformer_layers[static_cast(layer_index)] + .codebooks[static_cast(weight_index)] + .fused_input_projection.tensor; + supported = supported && + detail::tensor_shape(projection, lm.depformer_dim, + lm.depformer_dim * 3) && + detail::supported_mul_mat_dtype( + static_cast(projection->type)); + } + } + return supported; + } +}; + +struct guard_depformer_projection_layout_unsupported { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_split_projection_layout_supported{}(runtime_ev, + ctx) && + !guard_depformer_fused_projection_layout_supported{}(runtime_ev, + ctx); + } +}; + +struct guard_sampling_seed_nonzero { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_seed != 0u; + } +}; + +struct guard_sampling_seed_zero { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_seed == 0u; + } +}; + +struct guard_text_sampling_top_k_within_card { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_text_top_k <= + runtime_ev.request.model.moshi_lm.text_card; + } +}; + +struct guard_text_sampling_top_k_exceeds_card { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_text_top_k > + runtime_ev.request.model.moshi_lm.text_card; + } +}; + +struct guard_audio_sampling_top_k_within_card { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_audio_top_k <= + runtime_ev.request.model.moshi_lm.card; + } +}; + +struct guard_audio_sampling_top_k_exceeds_card { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.sampling_audio_top_k > + runtime_ev.request.model.moshi_lm.card; + } +}; + +struct guard_step_model_matches { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.session.model == &runtime_ev.request.model; + } +}; + +struct guard_step_model_mismatch { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_step_model_matches{}(runtime_ev, ctx); + } +}; + +struct guard_step_shape_valid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const bool full = runtime_ev.request.phase == + emel::speech::predictor::moshi::event::graph_phase::full; + const bool prediction = + runtime_ev.request.phase == + emel::speech::predictor::moshi::event::graph_phase::prediction; + return (full || prediction) && + runtime_ev.request.input_sequence.data() != nullptr && + runtime_ev.request.input_sequence.size() == + static_cast(ctx.session.codebook_count) && + runtime_ev.request.audio_tokens_out.data() != nullptr && + runtime_ev.request.audio_tokens_out.size() >= + static_cast(ctx.session.dep_q) && + (full || runtime_ev.request.temporal_state.size() >= + static_cast(ctx.session.hidden_dim)); + } +}; + +struct guard_sampling_step_valid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.request.phase == + emel::speech::predictor::moshi::event::graph_phase::sampling && + guard_step_model_matches{}(runtime_ev, ctx) && + runtime_ev.request.input_sequence.data() != nullptr && + runtime_ev.request.input_sequence.size() == + static_cast(ctx.session.codebook_count) && + runtime_ev.request.audio_tokens_out.data() != nullptr && + runtime_ev.request.audio_tokens_out.size() >= + static_cast(ctx.session.dep_q) && + runtime_ev.request.temporal_state.size() >= + static_cast(ctx.session.hidden_dim); + } +}; + +struct guard_sampling_step_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_sampling_step_valid{}(runtime_ev, ctx); + } +}; + +struct guard_full_graph_phase { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.phase == + emel::speech::predictor::moshi::event::graph_phase::full; + } +}; + +struct guard_prediction_graph_phase { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.phase == + emel::speech::predictor::moshi::event::graph_phase::prediction; + } +}; + +struct guard_step_shape_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_step_shape_valid{}(runtime_ev, ctx); + } +}; + +struct guard_external_input_embedding_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.request.input_embedding.data() != nullptr && + runtime_ev.request.input_embedding.size() == + static_cast(ctx.session.hidden_dim) && + ctx.session.hidden_dim > 0 && + static_cast(ctx.session.hidden_dim) <= + ctx.capacity.hidden_dim; + } +}; + +struct guard_token_input_embedding_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + bool tokens_ok = + runtime_ev.request.input_sequence[0] != ctx.policy.token_zero && + detail::token_in_embedding_range(runtime_ev.request.input_sequence[0], + lm.text_card, ctx.policy.token_zero); + for (int32_t index = 0; index < lm.n_q; ++index) { + tokens_ok = + tokens_ok && + detail::token_in_embedding_range( + runtime_ev.request.input_sequence[static_cast(index + 1)], + lm.card, ctx.policy.token_zero); + } + return runtime_ev.request.input_embedding.data() == nullptr && + !lm.demux_second_stream && hidden_dim > 0 && + static_cast(hidden_dim) <= ctx.capacity.hidden_dim && + tokens_ok; + } +}; + +struct guard_input_embedding_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_external_input_embedding_supported{}(runtime_ev, ctx) || + guard_token_input_embedding_supported{}(runtime_ev, ctx); + } +}; + +struct guard_input_embedding_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_input_embedding_supported{}(runtime_ev, ctx); + } +}; + +struct guard_token_input_embedding_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_external_input_embedding_supported{}(runtime_ev, ctx) && + !guard_token_input_embedding_supported{}(runtime_ev, ctx); + } +}; + +struct guard_input_text_embedding_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.input_text_embedding_ok; + } +}; + +struct guard_input_text_embedding_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_input_text_embedding_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_embedding_view_bound { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.embedding_view_bound; + } +}; + +struct guard_embedding_view_bind_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_embedding_view_bound{}(runtime_ev, ctx); + } +}; + +struct guard_embedding_row_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.embedding_row_ok; + } +}; + +struct guard_embedding_row_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_embedding_row_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_projection_view_bound { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.projection_view_bound; + } +}; + +struct guard_projection_view_bind_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_projection_view_bound{}(runtime_ev, ctx); + } +}; + +struct guard_current_input_audio_token_present { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t index = runtime_ev.ctx.input_audio_codebook_index; + if (index < 0 || index >= lm.n_q) { + return false; + } + const int32_t token = + runtime_ev.request.input_sequence[static_cast(index + 1)]; + return token != ctx.policy.token_zero && + detail::token_in_embedding_range(token, lm.card, + ctx.policy.token_zero) && + guard_input_embedding_supported{}(runtime_ev, ctx); + } +}; + +struct guard_current_input_audio_token_zero { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t index = runtime_ev.ctx.input_audio_codebook_index; + if (index < 0 || index >= lm.n_q) { + return false; + } + return runtime_ev.request.input_sequence[static_cast(index + 1)] == + ctx.policy.token_zero && + guard_input_embedding_supported{}(runtime_ev, ctx); + } +}; + +struct guard_current_input_audio_token_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_current_input_audio_token_present{}(runtime_ev, ctx) && + !guard_current_input_audio_token_zero{}(runtime_ev, ctx); + } +}; + +struct guard_input_audio_embedding_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.input_audio_embedding_ok; + } +}; + +struct guard_input_audio_embedding_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_input_audio_embedding_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_more_input_audio_codebooks { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.input_audio_embedding_ok && + runtime_ev.ctx.input_audio_codebook_index + 1 < + runtime_ev.request.model.moshi_lm.n_q; + } +}; + +struct guard_input_audio_codebooks_complete { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.ctx.input_audio_embedding_ok && + !guard_more_input_audio_codebooks{}(runtime_ev, ctx); + } +}; + +struct guard_input_embedding_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.input_embedding_ok; + } +}; + +struct guard_input_embedding_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_input_embedding_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_kv_binding_present { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + (void)runtime_ev; + return !ctx.temporal_kv.key_cache.empty() && + !ctx.temporal_kv.value_cache.empty() && + !ctx.temporal_kv.layer_cache_offsets.empty() && + ctx.temporal_positions != nullptr; + } +}; + +struct guard_temporal_kv_binding_missing { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_kv_binding_present{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_kv_bound { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &view = runtime_ev.ctx.temporal_kv; + return runtime_ev.ctx.temporal_kv_bound && !view.key_cache.empty() && + !view.value_cache.empty() && !view.layer_cache_offsets.empty() && + view.layer_count > 0 && view.position_capacity > 0 && + view.kv_dim > 0; + } +}; + +struct guard_temporal_position_advance_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &result = runtime_ev.ctx.temporal_position; + const auto &window = result.window; + return runtime_ev.ctx.temporal_position_accepted && + runtime_ev.ctx.temporal_position_error == + static_cast( + emel::error::cast(emel::memory::streaming::error::none)) && + result.logical_position >= 0 && result.physical_position >= 0 && + result.physical_position < window.capacity && window.capacity > 0 && + window.logical_end == result.logical_position + 1 && + window.valid_positions > 0 && + window.valid_positions <= window.capacity; + } +}; + +struct guard_temporal_position_advance_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_position_advance_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_kv_bind_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_kv_bound{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return layer >= 0 && layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0 && + static_cast(hidden_dim) <= ctx.capacity.hidden_dim; + } +}; + +struct guard_temporal_layer_norm_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm_rms_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_norm_rms_ok; + } +}; + +struct guard_temporal_layer_norm_rms_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm_rms_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_norm_ok; + } +}; + +struct guard_temporal_layer_norm_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_projection_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return layer >= 0 && layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0 && + static_cast(hidden_dim) <= ctx.capacity.hidden_dim && + static_cast(hidden_dim) * 3u <= + ctx.capacity.hidden_dim * 3u; + } +}; + +struct guard_temporal_layer_projection_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_projection_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_projection_ok; + } +}; + +struct guard_temporal_layer_projection_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_projection_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_rope_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const auto &position = runtime_ev.ctx.temporal_position; + const int32_t head_dim = lm.num_heads > 0 ? hidden_dim / lm.num_heads : 0; + return runtime_ev.ctx.temporal_layer_projection_ok && hidden_dim > 0 && + lm.num_heads > 0 && lm.max_period > 0 && + hidden_dim % lm.num_heads == 0 && head_dim > 0 && + (head_dim % 2) == 0 && + static_cast(head_dim) <= ctx.capacity.hidden_dim && + position.logical_position >= 0 && + position.logical_position <= std::numeric_limits::max() && + position.physical_position >= 0 && + position.physical_position < + runtime_ev.ctx.temporal_kv.position_capacity; + } +}; + +struct guard_temporal_layer_rope_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_rope_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_rope_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_rope_ok; + } +}; + +struct guard_temporal_layer_rope_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_rope_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_cache_write_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &view = runtime_ev.ctx.temporal_kv; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + if (!runtime_ev.ctx.temporal_layer_rope_ok || hidden_dim <= 0 || + view.kv_dim != hidden_dim || layer < 0 || layer >= view.layer_count || + view.layer_cache_offsets.empty() || + static_cast(layer) >= view.layer_cache_offsets.size() || + runtime_ev.ctx.temporal_position.physical_position < 0) { + return false; + } + const size_t physical_position = + static_cast(runtime_ev.ctx.temporal_position.physical_position); + const size_t dim = static_cast(hidden_dim); + return physical_position < static_cast(view.position_capacity) && + guard_cache_span_valid( + view.layer_cache_offsets[static_cast(layer)], + physical_position, 1u, dim, view.key_cache.size(), + view.value_cache.size()); + } +}; + +struct guard_temporal_layer_cache_write_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_cache_write_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_cache_write_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_cache_write_ok; + } +}; + +struct guard_temporal_layer_cache_write_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_cache_write_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_attention_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &view = runtime_ev.ctx.temporal_kv; + const auto &window = runtime_ev.ctx.temporal_position.window; + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t capacity = view.position_capacity; + const int32_t head_dim = lm.num_heads > 0 ? hidden_dim / lm.num_heads : 0; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + if (!runtime_ev.ctx.temporal_layer_cache_write_ok || hidden_dim <= 0 || + lm.num_heads <= 0 || hidden_dim % lm.num_heads != 0 || head_dim <= 0 || + static_cast(head_dim) > ctx.capacity.hidden_dim || + window.valid_positions <= 0 || capacity <= 0 || + static_cast(capacity) > ctx.capacity.temporal_context || + window.capacity != capacity || window.valid_positions > capacity || + window.logical_begin < 0 || + window.logical_end - window.logical_begin != window.valid_positions || + window.physical_begin < 0 || window.physical_begin >= capacity || + view.kv_dim != hidden_dim || layer < 0 || layer >= view.layer_count || + view.layer_cache_offsets.empty() || + static_cast(layer) >= view.layer_cache_offsets.size()) { + return false; + } + const size_t layer_offset = + view.layer_cache_offsets[static_cast(layer)]; + const size_t dim = static_cast(hidden_dim); + return guard_cache_span_valid( + layer_offset, 0u, static_cast(capacity), dim, + view.key_cache.size(), view.value_cache.size()); + } +}; + +struct guard_temporal_layer_attention_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_attention_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_attention_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_attention_ok; + } +}; + +struct guard_temporal_layer_attention_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_attention_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_out_projection_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return runtime_ev.ctx.temporal_layer_attention_ok && layer >= 0 && + layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0; + } +}; + +struct guard_temporal_layer_out_projection_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_out_projection_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_out_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_out_projection_ok; + } +}; + +struct guard_temporal_layer_out_projection_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_out_projection_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_residual_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_residual_ok; + } +}; + +struct guard_temporal_layer_residual_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_residual_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm2_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return runtime_ev.ctx.temporal_layer_residual_ok && layer >= 0 && + layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0 && + static_cast(hidden_dim) <= ctx.capacity.hidden_dim; + } +}; + +struct guard_temporal_layer_norm2_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm2_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm2_rms_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_norm2_rms_ok; + } +}; + +struct guard_temporal_layer_norm2_rms_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm2_rms_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_norm2_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_norm2_ok; + } +}; + +struct guard_temporal_layer_norm2_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_norm2_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_gating_in_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return runtime_ev.ctx.temporal_layer_norm2_ok && layer >= 0 && + layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0; + } +}; + +struct guard_temporal_layer_gating_in_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_gating_in_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_gating_in_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_gating_in_ok; + } +}; + +struct guard_temporal_layer_gating_in_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_gating_in_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_silu_gate_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_gating_in_ok && + runtime_ev.ctx.temporal_layer_index >= 0 && + runtime_ev.ctx.temporal_layer_index < + runtime_ev.request.model.moshi_lm.num_layers; + } +}; + +struct guard_temporal_layer_silu_gate_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_silu_gate_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_silu_gate_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_silu_gate_ok; + } +}; + +struct guard_temporal_layer_silu_gate_silu_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_silu_gate_silu_ok; + } +}; + +struct guard_temporal_layer_silu_gate_silu_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_silu_gate_silu_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_silu_gate_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_silu_gate_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_gating_out_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t layer = runtime_ev.ctx.temporal_layer_index; + return runtime_ev.ctx.temporal_layer_silu_gate_ok && layer >= 0 && + layer < runtime_ev.request.model.moshi_lm.num_layers && + hidden_dim > 0; + } +}; + +struct guard_temporal_layer_gating_out_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_gating_out_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_gating_out_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_gating_out_ok; + } +}; + +struct guard_temporal_layer_gating_out_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_gating_out_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_layer_ff_residual_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_ff_residual_ok; + } +}; + +struct guard_temporal_layer_ff_residual_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_layer_ff_residual_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_more_temporal_layers { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_layer_ff_residual_ok && + runtime_ev.ctx.temporal_layer_index + 1 < + runtime_ev.request.model.moshi_lm.num_layers; + } +}; + +struct guard_temporal_layers_complete { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.ctx.temporal_layer_ff_residual_ok && + !guard_more_temporal_layers{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_out_norm_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t hidden_dim = ctx.session.hidden_dim; + return guard_temporal_layers_complete{}(runtime_ev, ctx) && + hidden_dim > 0 && + static_cast(hidden_dim) <= ctx.capacity.hidden_dim; + } +}; + +struct guard_temporal_out_norm_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_out_norm_supported{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_out_norm_rms_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_out_norm_rms_ok; + } +}; + +struct guard_temporal_out_norm_rms_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_out_norm_rms_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_temporal_out_norm_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_out_norm_ok; + } +}; + +struct guard_temporal_out_norm_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_temporal_out_norm_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + return runtime_ev.ctx.temporal_out_norm_ok && hidden_dim > 0 && + lm.text_card > 0; + } +}; + +struct guard_text_logits_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_text_logits_supported{}(runtime_ev, ctx); + } +}; + +struct guard_forced_text_token_valid { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.forced_text_token >= 0 && + runtime_ev.request.forced_text_token < + runtime_ev.request.model.moshi_lm.text_card; + } +}; + +struct guard_forced_text_token_absent { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.forced_text_token < 0; + } +}; + +struct guard_forced_text_token_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.forced_text_token >= + runtime_ev.request.model.moshi_lm.text_card; + } +}; + +struct guard_text_sampling_config_valid { + bool operator()(const event::step_run &, + const action::context &ctx) const noexcept { + return ctx.sampling.enabled && ctx.sampler != nullptr && + ctx.sampling.text_temperature > 0.0f && + ctx.sampling.text_top_k > 0 && + ctx.sampling.text_top_k <= ctx.session.text_card && + static_cast(ctx.session.text_card) <= + ctx.capacity.sampling_card && + static_cast(ctx.sampling.text_top_k) <= + ctx.capacity.sampling_top_k; + } +}; + +struct guard_text_sampling_config_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.sampling.enabled && ctx.sampling.text_temperature > 0.0f && + !guard_text_sampling_config_valid{}(runtime_ev, ctx); + } +}; + +struct guard_forced_text_token_valid_and_sampling_consume { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_forced_text_token_valid{}(runtime_ev, ctx) && + ctx.sampling.consume_forced_text && + guard_text_sampling_config_valid{}(runtime_ev, ctx); + } +}; + +struct guard_forced_text_token_valid_without_sampling_consume { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_forced_text_token_valid{}(runtime_ev, ctx) && + (!ctx.sampling.consume_forced_text || !ctx.sampling.enabled || + ctx.sampling.text_temperature <= 0.0f); + } +}; + +struct guard_forced_text_sampling_config_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_forced_text_token_valid{}(runtime_ev, ctx) && + ctx.sampling.consume_forced_text && + guard_text_sampling_config_invalid{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_projection_bound_and_no_forced_token { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_projection_view_bound{}(runtime_ev, ctx) && + guard_forced_text_token_absent{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_projection_bound_and_no_forced_token_argmax { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_text_logits_projection_bound_and_no_forced_token{}(runtime_ev, + ctx) && + (!ctx.sampling.enabled || ctx.sampling.text_temperature <= 0.0f); + } +}; + +struct guard_text_logits_projection_bound_and_no_forced_token_sampling { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_text_logits_projection_bound_and_no_forced_token{}(runtime_ev, + ctx) && + guard_text_sampling_config_valid{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_text_logits_projection_bound_and_no_forced_token{}(runtime_ev, + ctx) && + guard_text_sampling_config_invalid{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_matmul_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.text_logits_ok; + } +}; + +struct guard_text_logits_matmul_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_text_logits_matmul_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_text_logits_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.text_logits_ok && runtime_ev.ctx.best_index >= 0; + } +}; + +struct guard_text_logits_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_text_logits_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_text_sampling_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.sampler_accepted && + runtime_ev.ctx.sampler_error == + emel::error::cast(emel::logits::sampler::error::none); + } +}; + +struct guard_text_sampling_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_text_sampling_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_sampled_text_token_ready { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_text_sampling_succeeded{}(runtime_ev, ctx) && + guard_forced_text_token_absent{}(runtime_ev, ctx); + } +}; + +struct guard_forced_text_sampling_consumed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_text_sampling_succeeded{}(runtime_ev, ctx) && + guard_forced_text_token_valid_and_sampling_consume{}(runtime_ev, + ctx); + } +}; + +struct guard_depformer_kv_binding_present { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + (void)runtime_ev; + return !ctx.depformer_kv.key_cache.empty() && + !ctx.depformer_kv.value_cache.empty() && + !ctx.depformer_kv.layer_cache_offsets.empty() && + ctx.depformer_positions != nullptr; + } +}; + +struct guard_depformer_kv_binding_missing { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_kv_binding_present{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_kv_bound { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &view = runtime_ev.ctx.depformer_kv; + return runtime_ev.ctx.depformer_kv_bound && !view.key_cache.empty() && + !view.value_cache.empty() && !view.layer_cache_offsets.empty() && + view.layer_count > 0 && view.position_capacity > 0 && + view.kv_dim > 0; + } +}; + +struct guard_depformer_position_reset_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_position_accepted && + runtime_ev.ctx.depformer_position_error == + static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + } +}; + +struct guard_depformer_position_reset_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_position_reset_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_position_advance_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &result = runtime_ev.ctx.depformer_position; + const auto &window = result.window; + return runtime_ev.ctx.depformer_position_accepted && + runtime_ev.ctx.depformer_position_error == + static_cast( + emel::error::cast(emel::memory::streaming::error::none)) && + result.logical_position >= 0 && result.physical_position >= 0 && + result.physical_position < window.capacity && window.capacity > 0 && + window.logical_end == result.logical_position + 1 && + window.valid_positions > 0 && + window.valid_positions <= window.capacity; + } +}; + +struct guard_depformer_position_advance_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_position_advance_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_kv_bind_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_kv_bound{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_scheduled_weight_present { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + if (lm.depformer_weight_schedule_count == 0u || codebook < 0 || + static_cast(codebook) >= lm.depformer_weight_schedule_count) { + return false; + } + const int32_t weight_index = + lm.depformer_weight_schedule[static_cast(codebook)]; + return weight_index >= 0 && weight_index < lm.dep_q; + } +}; + +struct guard_depformer_scheduled_weight_absent { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.model.moshi_lm.depformer_weight_schedule_count == + 0u; + } +}; + +struct guard_depformer_scheduled_weight_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_scheduled_weight_absent{}(runtime_ev, ctx) && + !guard_depformer_scheduled_weight_present{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_text_input_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + if (codebook != 0) { + return false; + } + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t dep_dim = lm.depformer_dim; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_kv_bound && runtime_ev.ctx.text_logits_ok && + codebook == 0 && hidden_dim > 0 && dep_dim > 0 && + static_cast(dep_dim) <= ctx.capacity.hidden_dim && + detail::token_in_embedding_range(runtime_ev.request.text_token_out, + lm.text_card, + ctx.policy.token_zero); + } +}; + +struct guard_depformer_audio_input_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + if (codebook <= 0 || codebook >= lm.dep_q) { + return false; + } + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t dep_dim = lm.depformer_dim; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_kv_bound && runtime_ev.ctx.text_logits_ok && + codebook > 0 && codebook < lm.dep_q && hidden_dim > 0 && + dep_dim > 0 && + static_cast(dep_dim) <= ctx.capacity.hidden_dim && + detail::token_in_embedding_range( + runtime_ev.request + .audio_tokens_out[static_cast(codebook - 1)], + lm.card, ctx.policy.token_zero); + } +}; + +struct guard_depformer_input_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_text_input_supported{}(runtime_ev, ctx) && + !guard_depformer_audio_input_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_input_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_input_ok; + } +}; + +struct guard_depformer_input_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_input_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_input_projection_bound { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_input_projection_bound; + } +}; + +struct guard_depformer_input_projection_bind_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_input_projection_bound{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_input_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_input_projection_ok; + } +}; + +struct guard_depformer_input_projection_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_input_projection_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_text_input_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_depformer_input_projection_succeeded{}(runtime_ev, ctx) && + guard_depformer_text_input_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_audio_input_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_depformer_input_projection_succeeded{}(runtime_ev, ctx) && + guard_depformer_audio_input_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_input_projection_embedding_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_depformer_input_projection_succeeded{}(runtime_ev, ctx) && + !guard_depformer_text_input_supported{}(runtime_ev, ctx) && + !guard_depformer_audio_input_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + return runtime_ev.ctx.depformer_input_ok && layer >= 0 && + layer < lm.depformer_num_layers && dep_dim > 0 && + static_cast(dep_dim) <= ctx.capacity.hidden_dim; + } +}; + +struct guard_depformer_layer_norm_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm_rms_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_norm_rms_ok; + } +}; + +struct guard_depformer_layer_norm_rms_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm_rms_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_norm_ok; + } +}; + +struct guard_depformer_layer_norm_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_projection_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = model.moshi_lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_layer_norm_ok && layer >= 0 && + layer < model.moshi_lm.depformer_num_layers && codebook >= 0 && + codebook < model.moshi_lm.dep_q && dep_dim > 0 && + static_cast(dep_dim) * 3u <= ctx.capacity.hidden_dim * 3u; + } +}; + +struct guard_depformer_layer_projection_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_projection_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_projection_ok; + } +}; + +struct guard_depformer_layer_projection_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_projection_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_cache_write_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &view = runtime_ev.ctx.depformer_kv; + const int32_t dep_dim = runtime_ev.request.model.moshi_lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t physical = + runtime_ev.ctx.depformer_position.physical_position; + if (!runtime_ev.ctx.depformer_layer_projection_ok || dep_dim <= 0 || + view.kv_dim != dep_dim || layer < 0 || layer >= view.layer_count || + view.layer_cache_offsets.empty() || + static_cast(layer) >= view.layer_cache_offsets.size() || + view.position_capacity <= 0 || physical < 0 || + physical >= view.position_capacity) { + return false; + } + return guard_cache_span_valid( + view.layer_cache_offsets[static_cast(layer)], + static_cast(physical), 1u, static_cast(dep_dim), + view.key_cache.size(), view.value_cache.size()); + } +}; + +struct guard_depformer_layer_cache_write_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_cache_write_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_cache_write_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_cache_write_ok; + } +}; + +struct guard_depformer_layer_cache_write_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_cache_write_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_attention_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const auto &view = runtime_ev.ctx.depformer_kv; + const int32_t dep_dim = lm.depformer_dim; + const int32_t head_dim = + lm.depformer_num_heads > 0 ? dep_dim / lm.depformer_num_heads : 0; + const auto &window = runtime_ev.ctx.depformer_position.window; + const int32_t valid_positions = window.valid_positions; + const int32_t capacity = view.position_capacity; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + if (!runtime_ev.ctx.depformer_layer_cache_write_ok || dep_dim <= 0 || + lm.depformer_num_heads <= 0 || dep_dim % lm.depformer_num_heads != 0 || + head_dim <= 0 || + static_cast(head_dim) > ctx.capacity.hidden_dim || + valid_positions <= 0 || capacity <= 0 || + static_cast(capacity) > ctx.capacity.depformer_context || + valid_positions > capacity || window.capacity != capacity || + window.logical_begin < 0 || + window.logical_end - window.logical_begin != valid_positions || + window.physical_begin < 0 || window.physical_begin >= capacity || + view.kv_dim != dep_dim || layer < 0 || layer >= view.layer_count || + view.layer_cache_offsets.empty() || + static_cast(layer) >= view.layer_cache_offsets.size()) { + return false; + } + const size_t dim = static_cast(dep_dim); + return guard_cache_span_valid( + view.layer_cache_offsets[static_cast(layer)], 0u, + static_cast(capacity), dim, view.key_cache.size(), + view.value_cache.size()); + } +}; + +struct guard_depformer_layer_attention_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_attention_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_attention_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_attention_ok; + } +}; + +struct guard_depformer_layer_attention_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_attention_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_out_projection_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_layer_attention_ok && layer >= 0 && + layer < model.moshi_lm.depformer_num_layers && codebook >= 0 && + codebook < model.moshi_lm.dep_q && dep_dim > 0; + } +}; + +struct guard_depformer_layer_out_projection_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_out_projection_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_out_projection_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_out_projection_ok; + } +}; + +struct guard_depformer_layer_out_projection_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_out_projection_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_residual_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_residual_ok; + } +}; + +struct guard_depformer_layer_residual_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_residual_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm2_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + return runtime_ev.ctx.depformer_layer_residual_ok && layer >= 0 && + layer < lm.depformer_num_layers && dep_dim > 0 && + static_cast(dep_dim) <= ctx.capacity.hidden_dim; + } +}; + +struct guard_depformer_layer_norm2_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm2_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm2_rms_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_norm2_rms_ok; + } +}; + +struct guard_depformer_layer_norm2_rms_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm2_rms_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_norm2_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_norm2_ok; + } +}; + +struct guard_depformer_layer_norm2_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_norm2_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_gating_in_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_layer_norm2_ok && layer >= 0 && + layer < model.moshi_lm.depformer_num_layers && codebook >= 0 && + codebook < model.moshi_lm.dep_q && dep_dim > 0; + } +}; + +struct guard_depformer_layer_gating_in_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_gating_in_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_gating_in_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_gating_in_ok; + } +}; + +struct guard_depformer_layer_gating_in_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_gating_in_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_silu_gate_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q || codebook < 0 || + codebook >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_layer_gating_in_ok && + runtime_ev.ctx.depformer_layer_index >= 0 && + runtime_ev.ctx.depformer_layer_index < + runtime_ev.request.model.moshi_lm.depformer_num_layers; + } +}; + +struct guard_depformer_layer_silu_gate_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_silu_gate_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_silu_gate_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_silu_gate_ok; + } +}; + +struct guard_depformer_layer_silu_gate_silu_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_silu_gate_silu_ok; + } +}; + +struct guard_depformer_layer_silu_gate_silu_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_silu_gate_silu_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_silu_gate_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_silu_gate_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_gating_out_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t layer = runtime_ev.ctx.depformer_layer_index; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + const int32_t weight_index = runtime_ev.ctx.depformer_weight_index; + if (weight_index < 0 || weight_index >= lm.dep_q) { + return false; + } + return runtime_ev.ctx.depformer_layer_silu_gate_ok && layer >= 0 && + layer < model.moshi_lm.depformer_num_layers && codebook >= 0 && + codebook < model.moshi_lm.dep_q && dep_dim > 0; + } +}; + +struct guard_depformer_layer_gating_out_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_gating_out_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_gating_out_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_gating_out_ok; + } +}; + +struct guard_depformer_layer_gating_out_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_gating_out_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_layer_ff_residual_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_ff_residual_ok; + } +}; + +struct guard_depformer_layer_ff_residual_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_layer_ff_residual_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_more_depformer_layers { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_layer_ff_residual_ok && + runtime_ev.ctx.depformer_layer_index + 1 < + runtime_ev.request.model.moshi_lm.depformer_num_layers; + } +}; + +struct guard_depformer_layers_complete { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.ctx.depformer_layer_ff_residual_ok && + !guard_more_depformer_layers{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_supported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + const int32_t dep_dim = lm.depformer_dim; + const int32_t codebook = runtime_ev.ctx.depformer_codebook_index; + return guard_depformer_layers_complete{}(runtime_ev, ctx) && + codebook >= 0 && codebook < lm.dep_q && dep_dim > 0 && lm.card > 0; + } +}; + +struct guard_depformer_logits_unsupported { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_logits_supported{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_logits_ok && + runtime_ev.ctx.best_index >= 0 && + runtime_ev.ctx.best_index < runtime_ev.request.model.moshi_lm.card; + } +}; + +struct guard_depformer_logits_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_logits_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_sampling_config_valid { + bool operator()(const event::step_run &, + const action::context &ctx) const noexcept { + return ctx.sampling.enabled && ctx.sampler != nullptr && + ctx.sampling.audio_temperature > 0.0f && + ctx.sampling.audio_top_k > 0 && + ctx.sampling.audio_top_k <= ctx.session.audio_card && + static_cast(ctx.session.audio_card) <= + ctx.capacity.sampling_card && + static_cast(ctx.sampling.audio_top_k) <= + ctx.capacity.sampling_top_k; + } +}; + +struct guard_depformer_sampling_config_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.sampling.enabled && ctx.sampling.audio_temperature > 0.0f && + !guard_depformer_sampling_config_valid{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_sampling_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.sampler_accepted && + runtime_ev.ctx.sampler_error == + emel::error::cast(emel::logits::sampler::error::none); + } +}; + +struct guard_depformer_sampling_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_sampling_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_argmax { + bool operator()(const event::step_run &, + const action::context &ctx) const noexcept { + return !ctx.sampling.enabled || ctx.sampling.audio_temperature <= 0.0f; + } +}; + +struct guard_depformer_logits_sampling { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_depformer_sampling_config_valid{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_sampling_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_depformer_sampling_config_invalid{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_projection_bound_argmax { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_projection_view_bound{}(runtime_ev, ctx) && + guard_depformer_logits_argmax{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_projection_bound_sampling { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_projection_view_bound{}(runtime_ev, ctx) && + guard_depformer_logits_sampling{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_projection_bound_sampling_invalid { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_projection_view_bound{}(runtime_ev, ctx) && + guard_depformer_logits_sampling_invalid{}(runtime_ev, ctx); + } +}; + +struct guard_depformer_logits_matmul_succeeded { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_logits_ok; + } +}; + +struct guard_depformer_logits_matmul_failed { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_depformer_logits_matmul_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_reset_temporal_positions_present { + bool operator()(const event::reset_run &, + const action::context &ctx) const noexcept { + return ctx.temporal_positions != nullptr; + } +}; + +struct guard_reset_temporal_positions_missing { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_temporal_positions_present{}(runtime_ev, ctx); + } +}; + +struct guard_reset_temporal_positions_succeeded { + bool operator()(const event::reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.temporal_position_accepted && + runtime_ev.ctx.temporal_position_error == + static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + } +}; + +struct guard_reset_temporal_positions_failed { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_temporal_positions_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_reset_depformer_positions_present { + bool operator()(const event::reset_run &, + const action::context &ctx) const noexcept { + return ctx.depformer_positions != nullptr; + } +}; + +struct guard_reset_depformer_positions_missing { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_depformer_positions_present{}(runtime_ev, ctx); + } +}; + +struct guard_reset_depformer_positions_succeeded { + bool operator()(const event::reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_position_accepted && + runtime_ev.ctx.depformer_position_error == + static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + } +}; + +struct guard_reset_depformer_positions_failed { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_depformer_positions_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_more_depformer_codebooks { + bool operator()(const event::step_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.depformer_logits_ok && + runtime_ev.ctx.depformer_codebook_index + 1 < + runtime_ev.request.model.moshi_lm.dep_q; + } +}; + +struct guard_depformer_codebooks_complete { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.ctx.depformer_logits_ok && + !guard_more_depformer_codebooks{}(runtime_ev, ctx); + } +}; + +template struct guard_has_error_out { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.error_out != nullptr; + } +}; + +template struct guard_no_error_out { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_error_out{}(runtime_ev, ctx); + } +}; + +struct guard_has_initialize_done_callback { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_done); + } +}; + +struct guard_no_initialize_done_callback { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_initialize_done_callback{}(runtime_ev, ctx); + } +}; + +struct guard_has_initialize_error_callback { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_error); + } +}; + +struct guard_no_initialize_error_callback { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_initialize_error_callback{}(runtime_ev, ctx); + } +}; + +struct guard_unexpected_error_out_present { + template + bool operator()(const event_type &ev, + const action::context &) const noexcept { + if constexpr (requires { + ev.ctx.err; + ev.request.error_out; + }) { + return ev.request.error_out != nullptr; + } else { + return false; + } + } +}; + +struct guard_unexpected_error_out_absent { + template + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_unexpected_error_out_present{}(ev, ctx); + } +}; + +} // namespace emel::speech::predictor::moshi::executor::guard diff --git a/src/emel/speech/predictor/moshi/executor/sm.hpp b/src/emel/speech/predictor/moshi/executor/sm.hpp new file mode 100644 index 00000000..e136f446 --- /dev/null +++ b/src/emel/speech/predictor/moshi/executor/sm.hpp @@ -0,0 +1,1156 @@ +#pragma once +// benchmark: designed + +#include + +#include "emel/sm.hpp" +#include "emel/speech/predictor/moshi/executor/actions.hpp" +#include "emel/speech/predictor/moshi/executor/context.hpp" +#include "emel/speech/predictor/moshi/executor/events.hpp" +#include "emel/speech/predictor/moshi/executor/guards.hpp" + +namespace emel::speech::predictor::moshi::executor { + +struct state_uninitialized {}; +struct state_reset_temporal_positions_decision {}; +struct state_reset_temporal_positions_result_decision {}; +struct state_reset_depformer_positions_decision {}; +struct state_reset_depformer_positions_result_decision {}; +struct state_reset_failed {}; +struct state_bind_contract_decision {}; +struct state_input_embedding_contract_decision {}; +struct state_temporal_projection_layout_decision {}; +struct state_depformer_projection_layout_decision {}; +struct state_sampling_seed_decision {}; +struct state_text_sampling_top_k_decision {}; +struct state_audio_sampling_top_k_decision {}; +struct state_init_error_out_decision {}; +struct state_init_callback_decision {}; +struct state_init_failed_error_out_decision {}; +struct state_init_failed_callback_decision {}; +struct state_ready {}; +struct state_step_model_decision {}; +struct state_step_shape_decision {}; +struct state_step_execution_decision {}; +struct state_input_embedding {}; +struct state_input_text_embedding_bind {}; +struct state_input_text_embedding_bind_result_decision {}; +struct state_input_text_embedding_row {}; +struct state_input_text_embedding_row_result_decision {}; +struct state_input_text_embedding_apply {}; +struct state_input_text_embedding_result_decision {}; +struct state_input_audio_token_decision {}; +struct state_input_audio_embedding {}; +struct state_input_audio_embedding_bind_result_decision {}; +struct state_input_audio_embedding_row {}; +struct state_input_audio_embedding_row_result_decision {}; +struct state_input_audio_embedding_apply {}; +struct state_input_audio_embedding_result_decision {}; +struct state_input_audio_codebook_advance {}; +struct state_input_embedding_result_decision {}; +struct state_bind_temporal_kv {}; +struct state_temporal_kv_result_decision {}; +struct state_temporal_position_result_decision {}; +struct state_temporal_position_valid_decision {}; +struct state_temporal_layer_norm {}; +struct state_temporal_layer_norm_rms_result_decision {}; +struct state_temporal_layer_norm_scale {}; +struct state_temporal_layer_norm_result_decision {}; +struct state_temporal_layer_projection {}; +struct state_temporal_layer_projection_bind_result_decision {}; +struct state_temporal_layer_projection_run {}; +struct state_temporal_layer_projection_result_decision {}; +struct state_temporal_layer_rope {}; +struct state_temporal_layer_rope_result_decision {}; +struct state_temporal_layer_key_rope {}; +struct state_temporal_layer_key_rope_result_decision {}; +struct state_temporal_layer_cache_write {}; +struct state_temporal_layer_cache_write_result_decision {}; +struct state_temporal_layer_attention {}; +struct state_temporal_layer_attention_result_decision {}; +struct state_temporal_layer_out_projection {}; +struct state_temporal_layer_out_projection_bind_result_decision {}; +struct state_temporal_layer_out_projection_run {}; +struct state_temporal_layer_out_projection_result_decision {}; +struct state_temporal_layer_residual {}; +struct state_temporal_layer_residual_result_decision {}; +struct state_temporal_layer_norm2 {}; +struct state_temporal_layer_norm2_rms_result_decision {}; +struct state_temporal_layer_norm2_scale {}; +struct state_temporal_layer_norm2_result_decision {}; +struct state_temporal_layer_gating_in {}; +struct state_temporal_layer_gating_in_bind_result_decision {}; +struct state_temporal_layer_gating_in_run {}; +struct state_temporal_layer_gating_in_result_decision {}; +struct state_temporal_layer_silu_gate {}; +struct state_temporal_layer_silu_gate_silu_result_decision {}; +struct state_temporal_layer_silu_gate_mul {}; +struct state_temporal_layer_silu_gate_result_decision {}; +struct state_temporal_layer_gating_out {}; +struct state_temporal_layer_gating_out_bind_result_decision {}; +struct state_temporal_layer_gating_out_run {}; +struct state_temporal_layer_gating_out_result_decision {}; +struct state_temporal_layer_ff_residual {}; +struct state_temporal_layer_ff_residual_result_decision {}; +struct state_temporal_layer_advance {}; +struct state_temporal_out_norm {}; +struct state_temporal_out_norm_rms_result_decision {}; +struct state_temporal_out_norm_scale {}; +struct state_temporal_out_norm_result_decision {}; +struct state_text_logits_phase_decision {}; +struct state_sampling_ready {}; +struct state_text_logits {}; +struct state_text_logits_bind_result_decision {}; +struct state_text_logits_run {}; +struct state_text_logits_sample_projection {}; +struct state_text_logits_sample_projection_result_decision {}; +struct state_text_logits_sample_select {}; +struct state_text_logits_result_decision {}; +struct state_bind_depformer_kv {}; +struct state_depformer_kv_result_decision {}; +struct state_depformer_position_reset_result_decision {}; +struct state_depformer_position_advance_result_decision {}; +struct state_depformer_weight_route_decision {}; +struct state_depformer_weight_decision {}; +struct state_depformer_input {}; +struct state_depformer_input_projection_bind_result_decision {}; +struct state_depformer_input_projection {}; +struct state_depformer_input_projection_result_decision {}; +struct state_depformer_input_embedding_bind {}; +struct state_depformer_input_embedding_bind_result_decision {}; +struct state_depformer_input_embedding_row {}; +struct state_depformer_input_embedding_row_result_decision {}; +struct state_depformer_input_embedding_apply {}; +struct state_depformer_input_result_decision {}; +struct state_depformer_layer_norm {}; +struct state_depformer_layer_norm_rms_result_decision {}; +struct state_depformer_layer_norm_scale {}; +struct state_depformer_layer_norm_result_decision {}; +struct state_depformer_layer_projection {}; +struct state_depformer_layer_projection_bind_result_decision {}; +struct state_depformer_layer_projection_run {}; +struct state_depformer_layer_projection_result_decision {}; +struct state_depformer_layer_cache_write {}; +struct state_depformer_layer_cache_write_result_decision {}; +struct state_depformer_layer_attention {}; +struct state_depformer_layer_attention_result_decision {}; +struct state_depformer_layer_out_projection {}; +struct state_depformer_layer_out_projection_bind_result_decision {}; +struct state_depformer_layer_out_projection_run {}; +struct state_depformer_layer_out_projection_result_decision {}; +struct state_depformer_layer_residual {}; +struct state_depformer_layer_residual_result_decision {}; +struct state_depformer_layer_norm2 {}; +struct state_depformer_layer_norm2_rms_result_decision {}; +struct state_depformer_layer_norm2_scale {}; +struct state_depformer_layer_norm2_result_decision {}; +struct state_depformer_layer_gating_in {}; +struct state_depformer_layer_gating_in_bind_result_decision {}; +struct state_depformer_layer_gating_in_run {}; +struct state_depformer_layer_gating_in_result_decision {}; +struct state_depformer_layer_silu_gate {}; +struct state_depformer_layer_silu_gate_silu_result_decision {}; +struct state_depformer_layer_silu_gate_mul {}; +struct state_depformer_layer_silu_gate_result_decision {}; +struct state_depformer_layer_gating_out {}; +struct state_depformer_layer_gating_out_bind_result_decision {}; +struct state_depformer_layer_gating_out_run {}; +struct state_depformer_layer_gating_out_result_decision {}; +struct state_depformer_layer_ff_residual {}; +struct state_depformer_layer_ff_residual_result_decision {}; +struct state_depformer_layer_advance {}; +struct state_depformer_logits {}; +struct state_depformer_logits_bind_result_decision {}; +struct state_depformer_logits_run {}; +struct state_depformer_logits_sample_projection {}; +struct state_depformer_logits_sample_projection_result_decision {}; +struct state_depformer_logits_sample_select {}; +struct state_depformer_token_publish {}; +struct state_depformer_logits_result_decision {}; +struct state_depformer_codebook_advance {}; +struct state_step_error_out_decision {}; +struct state_uninit_step_error_out_decision {}; + +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + using reset_run = event::reset_run; + using init_run = event::initialize_run; + using step_run = event::step_run; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Bind the Moshi LM execution contract. + sml::state <= *sml::state + + sml::event + , sml::state <= + sml::state + + sml::completion [ guard::guard_bind_contract_valid{} ] + / action::effect_bind_contract{} + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_contract_invalid{} ] + / action::effect_mark_bind_failed{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_bound_root_operands_supported{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_bound_root_operands_unsupported{} ] + / action::effect_mark_bind_failed{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_temporal_split_projection_layout_supported{} ] + / action::effect_bind_temporal_split_projection_layout{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_temporal_fused_projection_layout_supported{} ] + / action::effect_bind_temporal_fused_projection_layout{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_temporal_projection_layout_unsupported{} ] + / action::effect_mark_bind_failed{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_split_projection_layout_supported{} ] + / action::effect_bind_depformer_split_projection_layout{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_fused_projection_layout_supported{} ] + / action::effect_bind_depformer_fused_projection_layout{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_projection_layout_unsupported{} ] + / action::effect_mark_bind_failed{} + , sml::state <= sml::state + + sml::completion [ guard::guard_sampling_seed_nonzero{} ] + / action::effect_bind_nonzero_sampling_seed{} + , sml::state <= sml::state + + sml::completion [ guard::guard_sampling_seed_zero{} ] + / action::effect_bind_zero_sampling_seed{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_sampling_top_k_within_card{} ] + / action::effect_bind_requested_text_sampling_top_k{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_sampling_top_k_exceeds_card{} ] + / action::effect_bind_full_card_text_sampling_top_k{} + , sml::state <= sml::state + + sml::completion [ guard::guard_audio_sampling_top_k_within_card{} ] + / action::effect_bind_requested_audio_sampling_top_k{} + , sml::state <= sml::state + + sml::completion [ guard::guard_audio_sampling_top_k_exceeds_card{} ] + / action::effect_bind_full_card_audio_sampling_top_k{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_initialize_done_callback{} ] + / action::effect_emit_initialize_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_initialize_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_initialize_error_callback{} ] + / action::effect_emit_initialize_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_initialize_error_callback{} ] + + //------------------------------------------------------------------------------// + // Graph-step request validation and Moshi LM execution. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_step_model_matches{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_step_model_mismatch{} ] + / action::effect_mark_model_mismatch{} + , sml::state <= sml::state + + sml::completion [ guard::guard_step_shape_valid{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_step_shape_invalid{} ] + / action::effect_mark_request_shape{} + , sml::state <= sml::state + + sml::completion [ guard::guard_input_embedding_supported{} ] + / action::effect_begin_input_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_input_embedding_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_external_input_embedding_supported{} ] + / action::effect_apply_external_input_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_token_input_embedding_supported{} ] + / action::effect_bind_input_text_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_token_input_embedding_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bound{} ] + / action::effect_run_embedding_row_fetch{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_succeeded{} ] + / action::effect_apply_input_text_embedding_row{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_input_text_embedding_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_input_text_embedding_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_current_input_audio_token_present{} ] + / action::effect_bind_input_audio_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_current_input_audio_token_zero{} ] + / action::effect_skip_zero_input_audio_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_current_input_audio_token_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bound{} ] + / action::effect_run_embedding_row_fetch{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_succeeded{} ] + / action::effect_apply_input_audio_embedding_row{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_input_audio_embedding_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_input_audio_embedding_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_more_input_audio_codebooks{} ] + / action::effect_advance_input_audio_codebook{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_input_audio_codebooks_complete{} ] + / action::effect_finish_input_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_input_embedding_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_kv_binding_present{} ] + / action::effect_bind_temporal_kv{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_kv_binding_missing{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_kv_bound{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_kv_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion / action::effect_advance_temporal_position{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_temporal_position_advance_succeeded{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_temporal_position_advance_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_temporal_layer_norm_supported{} ] + / action::effect_run_temporal_layer_norm_rms{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_temporal_layer_norm_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_rms_succeeded{} ] + / action::effect_run_temporal_layer_norm_scale{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_rms_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_projection_supported{} ] + / action::effect_bind_temporal_layer_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_projection_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_temporal_layer_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_projection_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_projection_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_supported{} ] + / action::effect_run_temporal_layer_query_rope{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_succeeded{} ] + / action::effect_copy_temporal_layer_query_rope{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + / action::effect_run_temporal_layer_key_rope{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_succeeded{} ] + / action::effect_copy_temporal_layer_key_rope{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_rope_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_cache_write_supported{} ] + / action::effect_write_temporal_layer_kv_cache{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_cache_write_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_cache_write_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_cache_write_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_supported{} ] + / action::effect_run_temporal_layer_attention{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_out_projection_supported{} ] + / action::effect_bind_temporal_layer_out_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_out_projection_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_temporal_layer_out_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_out_projection_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_out_projection_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + / action::effect_apply_temporal_layer_attention_residual{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_residual_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_residual_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_supported{} ] + / action::effect_run_temporal_layer_norm2_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_rms_succeeded{} ] + / action::effect_run_temporal_layer_norm2_scale{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_rms_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm2_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_in_supported{} ] + / action::effect_bind_temporal_layer_gating_in{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_in_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_temporal_layer_gating_in{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_in_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_in_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_supported{} ] + / action::effect_run_temporal_layer_silu_gate_silu{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_silu_succeeded{} ] + / action::effect_run_temporal_layer_silu_gate_mul{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_silu_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_silu_gate_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_out_supported{} ] + / action::effect_bind_temporal_layer_gating_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_out_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_temporal_layer_gating_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_out_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_gating_out_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + / action::effect_apply_temporal_layer_ff_residual{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_ff_residual_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_ff_residual_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_more_temporal_layers{} ] + / action::effect_advance_temporal_layer{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layers_complete{} ] + / action::effect_run_temporal_out_norm_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_out_norm_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_supported{} ] + / action::effect_run_temporal_layer_norm_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_norm_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_out_norm_rms_succeeded{} ] + / action::effect_run_temporal_out_norm_scale{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_out_norm_rms_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_out_norm_succeeded{} ] + / action::effect_publish_temporal_out_norm{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_out_norm_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_full_graph_phase{} ] + , sml::state <= + sml::state + + sml::completion [ guard::guard_prediction_graph_phase{} ] + / action::effect_publish_prediction_state{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_supported{} ] + / action::effect_bind_text_token_logits{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::event [ guard::guard_sampling_step_valid{} ] + / action::effect_restore_prediction_state_and_bind_text_logits{} + , sml::state <= sml::state + + sml::event [ guard::guard_sampling_step_invalid{} ] + / action::effect_mark_request_shape{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_forced_text_token_valid_and_sampling_consume{} ] + / action::effect_compute_text_token_logits{} + , sml::state <= sml::state + + sml::completion [ guard::guard_forced_text_token_valid_without_sampling_consume{} ] + / action::effect_publish_forced_text_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_forced_text_sampling_config_invalid{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_forced_text_token_invalid{} ] + / action::effect_mark_request_shape{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_argmax{} ] + / action::effect_select_text_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_sampling{} ] + / action::effect_compute_text_token_logits{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_matmul_succeeded{} ] + / action::effect_select_text_sampling_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_matmul_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_sampled_text_token_ready{} ] + / action::effect_publish_sampled_text_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_forced_text_sampling_consumed{} ] + / action::effect_publish_forced_text_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_sampling_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_text_logits_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_kv_binding_present{} ] + / action::effect_bind_depformer_kv{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_kv_binding_missing{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_kv_bound{} ] + / action::effect_reset_depformer_positions{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_kv_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_position_reset_succeeded{} ] + / action::effect_advance_depformer_position{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_position_reset_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_position_advance_succeeded{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_depformer_position_advance_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_depformer_scheduled_weight_present{} ] + / action::effect_use_depformer_scheduled_weight{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_depformer_scheduled_weight_absent{} ] + / action::effect_use_depformer_codebook_weight{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_depformer_scheduled_weight_invalid{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_text_input_supported{} ] + / action::effect_bind_depformer_text_input_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_audio_input_supported{} ] + / action::effect_bind_depformer_audio_input_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_projection_bound{} ] + / action::effect_run_depformer_input_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_projection_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_text_input_projection_succeeded{} ] + / action::effect_bind_depformer_text_input_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_audio_input_projection_succeeded{} ] + / action::effect_bind_depformer_audio_input_embedding{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_projection_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_projection_embedding_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bound{} ] + / action::effect_run_embedding_row_fetch{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_succeeded{} ] + / action::effect_apply_depformer_input_embedding_row{} + , sml::state <= sml::state + + sml::completion [ guard::guard_embedding_row_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_input_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_supported{} ] + / action::effect_run_depformer_layer_norm_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_rms_succeeded{} ] + / action::effect_run_depformer_layer_norm_scale{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_rms_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_projection_supported{} ] + / action::effect_bind_depformer_layer_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_projection_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_depformer_layer_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_projection_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_projection_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_cache_write_supported{} ] + / action::effect_write_depformer_layer_kv_cache{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_cache_write_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_cache_write_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_cache_write_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_attention_supported{} ] + / action::effect_run_depformer_layer_attention{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_attention_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_attention_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_attention_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_out_projection_supported{} ] + / action::effect_bind_depformer_layer_out_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_out_projection_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_depformer_layer_out_projection{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_out_projection_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_out_projection_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + / action::effect_apply_depformer_layer_attention_residual{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_residual_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_residual_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_supported{} ] + / action::effect_run_depformer_layer_norm2_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_rms_succeeded{} ] + / action::effect_run_depformer_layer_norm2_scale{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_rms_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm2_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_in_supported{} ] + / action::effect_bind_depformer_layer_gating_in{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_in_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_depformer_layer_gating_in{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_in_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_in_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_supported{} ] + / action::effect_run_depformer_layer_silu_gate_silu{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_silu_succeeded{} ] + / action::effect_run_depformer_layer_silu_gate_mul{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_silu_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_silu_gate_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_out_supported{} ] + / action::effect_bind_depformer_layer_gating_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_out_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound{} ] + / action::effect_run_depformer_layer_gating_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_out_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_gating_out_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + / action::effect_apply_depformer_layer_ff_residual{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_ff_residual_succeeded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_ff_residual_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_more_depformer_layers{} ] + / action::effect_advance_depformer_layer{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_supported{} ] + / action::effect_bind_depformer_token_logits{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_projection_bound_argmax{} ] + / action::effect_select_depformer_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_projection_bound_sampling{} ] + / action::effect_compute_depformer_token_logits{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_projection_bound_sampling_invalid{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bind_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_matmul_succeeded{} ] + / action::effect_select_depformer_sampling_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_matmul_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_supported{} ] + / action::effect_run_depformer_layer_norm_rms{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_layer_norm_unsupported{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_succeeded{} ] + / action::effect_publish_depformer_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_logits_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_sampling_succeeded{} ] + / action::effect_publish_depformer_token{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_sampling_failed{} ] + / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_more_depformer_codebooks{} ] + / action::effect_advance_depformer_codebook{} + , sml::state <= sml::state + + sml::completion [ guard::guard_depformer_codebooks_complete{} ] + , sml::state <= + sml::state + + sml::completion / action::effect_advance_depformer_position{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + + //------------------------------------------------------------------------------// + // Requests before initialization answer explicitly. + , sml::state <= sml::state + + sml::event + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + + //------------------------------------------------------------------------------// + // Reset. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::event + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_temporal_positions_present{} ] + / action::effect_reset_temporal_positions{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_temporal_positions_missing{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_temporal_positions_succeeded{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_temporal_positions_failed{} ] + / action::effect_mark_reset_failed{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_depformer_positions_present{} ] + / action::effect_reset_bound_depformer_positions{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_depformer_positions_missing{} ] + / action::effect_reset_session{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_depformer_positions_succeeded{} ] + / action::effect_reset_session{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_reset_depformer_positions_failed{} ] + / action::effect_mark_reset_failed{} + , sml::state <= sml::state + + sml::completion / action::effect_reset_session{} + + //------------------------------------------------------------------------------// + // Unexpected events. + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::visit_current_states; + + explicit sm(const action::dependencies &deps) + : base_type(std::in_place, deps) {} + + bool process_event(const event::initialize &ev) { + event::initialize_ctx ctx{}; + event::initialize_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event( + const emel::speech::predictor::moshi::event::initialize_graph &ev) { + event::initialize request{ev.request.model}; + request.sampling_enabled = ev.request.sampling_enabled; + request.sampling_consume_forced_text = + ev.request.sampling_consume_forced_text; + request.sampling_audio_temperature = ev.request.sampling_audio_temperature; + request.sampling_text_temperature = ev.request.sampling_text_temperature; + request.sampling_audio_top_k = ev.request.sampling_audio_top_k; + request.sampling_text_top_k = ev.request.sampling_text_top_k; + request.sampling_seed = ev.request.sampling_seed; + request.error_out = ev.error_out; + return process_event(request); + } + + bool + process_event(const emel::speech::predictor::moshi::event::graph_step &ev) { + event::step_run runtime_ev{ev, step_workspace_}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && + step_workspace_.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::reset &ev) { + event::reset_ctx ctx{}; + event::reset_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const emel::speech::predictor::moshi::event::reset &) { + return process_event(event::reset{}); + } + +private: + // Actor-owned numeric scratch is initialized once and reused synchronously. + // Entry actions reset scalar outcomes and only the active tensor extents. + event::step_ctx step_workspace_ = {}; +}; + +} // namespace emel::speech::predictor::moshi::executor diff --git a/src/emel/speech/predictor/moshi/guards.hpp b/src/emel/speech/predictor/moshi/guards.hpp new file mode 100644 index 00000000..5e05ced5 --- /dev/null +++ b/src/emel/speech/predictor/moshi/guards.hpp @@ -0,0 +1,693 @@ +#pragma once + +#include "emel/kernel/detail.hpp" +#include "emel/memory/hybrid/errors.hpp" +#include "emel/memory/view.hpp" +#include "emel/model/loader/errors.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/context.hpp" +#include "emel/speech/predictor/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/events.hpp" + +namespace emel::speech::predictor::moshi::guard { + +struct guard_bind_contract_valid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto &model = runtime_ev.request.model; + const auto &lm = model.moshi_lm; + if (lm.delay_count > lm.delays.size()) { + return false; + } + if (lm.n_q < 0 || lm.n_q >= action::k_max_codebooks) { + return false; + } + int32_t max_delay = 0; + for (uint32_t index = 0; index < lm.delay_count; ++index) { + if (lm.delays[index] < 0) { + return false; + } + if (lm.delays[index] > max_delay) { + max_delay = lm.delays[index]; + } + } + const int32_t codebook_count = lm.n_q + 1; + const bool personaplex = lm.depformer_weights_per_step; + const int32_t delayed_dep_q = personaplex ? lm.inference_dep_q : lm.dep_q; + const int32_t needed_tokens = codebook_count - delayed_dep_q - 1; + const int64_t row_count = + static_cast(max_delay) + 2 + static_cast(personaplex); + const int64_t reserved_token_capacity = + static_cast(runtime_ev.request.max_blocks) * + static_cast(runtime_ev.request.block_tokens); + const bool inference_contract = + !personaplex || + (lm.inference_dep_q > 0 && lm.inference_dep_q <= lm.dep_q && + lm.inference_prompt_token_count == + static_cast(codebook_count) && + lm.inference_pre_text_silence_frames >= 0 && + lm.inference_post_text_silence_frames >= 0); + return emel::model::moshi::detail::validate_execution_contract(model) == + emel::error::cast(emel::model::loader::error::none) && + codebook_count > 0 && codebook_count <= action::k_max_codebooks && + lm.dep_q > 0 && lm.dep_q < codebook_count && delayed_dep_q > 0 && + delayed_dep_q <= lm.dep_q && needed_tokens >= 0 && row_count > 0 && + row_count <= action::k_max_delay_rows && inference_contract && + lm.delay_count >= static_cast(codebook_count) && + runtime_ev.request.max_sequences > 0 && + runtime_ev.request.max_sequences <= + emel::memory::view::MAX_SEQUENCES && + runtime_ev.request.max_blocks > 0 && + runtime_ev.request.max_blocks <= emel::memory::view::MAX_BLOCKS && + runtime_ev.request.block_tokens > 0 && reserved_token_capacity > 0 && + reserved_token_capacity <= static_cast(lm.context) && + runtime_ev.request.sequence_id >= 0 && + runtime_ev.request.sequence_id < runtime_ev.request.max_sequences && + runtime_ev.request.codebook_capacity >= codebook_count && + runtime_ev.request.codebook_capacity <= action::k_max_codebooks && + runtime_ev.request.delay_cache_row_capacity >= row_count && + runtime_ev.request.delay_cache_row_capacity <= + action::k_max_delay_rows && + ctx.policy.token_ungenerated < ctx.policy.token_zero && + ctx.policy.prediction_step_size > 0 && + ctx.policy.prediction_output_count > 0; + } +}; + +struct guard_bind_contract_invalid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_bind_contract_valid{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_lmgen { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.model.moshi_lm.depformer_weights_per_step; + } +}; + +struct guard_standard_lmgen { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_personaplex_lmgen{}(runtime_ev, ctx); + } +}; + +struct guard_voice_contract_valid { + bool operator()(const event::load_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + emel::model::moshi::detail::execution_contract contract = {}; + const auto &voice_model = runtime_ev.request.voice_model; + const auto result = emel::model::moshi::detail::build_execution_contract( + voice_model, contract); + const auto *embeddings = contract.voice.embeddings.tensor; + const auto *cache = contract.voice.cache.tensor; + const auto embedding_type = + embeddings != nullptr ? static_cast(embeddings->type) : 0u; + const auto cache_type = + cache != nullptr ? static_cast(cache->type) : 0u; + const bool embedding_type_ok = + embedding_type == emel::kernel::detail::dtype_f32 || + embedding_type == emel::kernel::detail::dtype_f16 || + embedding_type == emel::kernel::detail::dtype_bf16; + return ctx.session.model != nullptr && + ctx.session.model->moshi_lm.depformer_weights_per_step && + ctx.lmgen.delayed_dep_q > 0 && + ctx.lmgen.delayed_dep_q == + ctx.session.model->moshi_lm.inference_dep_q && + result == emel::error::cast(emel::model::loader::error::none) && + contract.component == emel::model::data::moshi_component::voice && + embeddings != nullptr && cache != nullptr && + embeddings->n_dims == 4 && + embeddings->dims[0] == ctx.session.model->moshi_lm.dim && + embeddings->dims[1] == 1 && embeddings->dims[2] == 1 && + embeddings->dims[3] > 0 && + embeddings->dims[0] <= + static_cast(event::k_max_voice_embedding_dim) && + embedding_type_ok && cache->n_dims == 2 && + cache->dims[0] == ctx.lmgen.cache_row_count && + cache->dims[1] == ctx.lmgen.codebook_count && + cache_type == emel::kernel::detail::dtype_i32; + } +}; + +struct guard_voice_contract_invalid { + bool operator()(const event::load_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_voice_contract_valid{}(runtime_ev, ctx); + } +}; + +struct guard_memory_accepted { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + return ev.ctx.memory_accepted && + ev.ctx.memory_error == static_cast(emel::error::cast( + emel::memory::hybrid::error::none)); + } +}; + +struct guard_graph_initialize_succeeded { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.graph_accepted && + runtime_ev.ctx.graph_error == + action::detail_ns::to_error(error::none); + } +}; + +struct guard_graph_initialize_failed { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_graph_initialize_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_memory_rejected { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_memory_accepted{}(runtime_ev, ctx); + } +}; + +struct guard_predict_request_valid { + bool operator()(const event::predict_run &runtime_ev, + const action::context &ctx) const noexcept { + const bool voice_ready = + !ctx.voice.loaded || (ctx.voice.ready && ctx.voice.prompt_ready); + return ctx.session.model != nullptr && voice_ready && + runtime_ev.request.model_tokens.size() == + static_cast(ctx.lmgen.codebook_count) && + runtime_ev.request.planned_step_size == + ctx.policy.prediction_step_size && + runtime_ev.request.planned_output_count == + ctx.policy.prediction_output_count; + } +}; + +struct guard_execute_request_valid { + bool operator()(const event::execute_run &runtime_ev, + const action::context &ctx) const noexcept { + const bool voice_ready = + !ctx.voice.loaded || (ctx.voice.ready && ctx.voice.prompt_ready); + return ctx.session.model != nullptr && voice_ready && + runtime_ev.request.model_tokens.size() == + static_cast(ctx.lmgen.codebook_count); + } +}; + +struct guard_execute_request_invalid { + bool operator()(const event::execute_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_execute_request_valid{}(runtime_ev, ctx); + } +}; + +struct guard_sample_request_valid { + bool operator()(const event::sample_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.session.model != nullptr && + runtime_ev.request.model_tokens.size() == + static_cast(ctx.lmgen.codebook_count) && + runtime_ev.request.audio_tokens_out.size() >= + static_cast(ctx.lmgen.generated_dep_q); + } +}; + +struct guard_sample_request_invalid { + bool operator()(const event::sample_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_sample_request_valid{}(runtime_ev, ctx); + } +}; + +struct guard_predict_blocked_by_voice_prompt { + bool operator()(const event::predict_run &, + const action::context &ctx) const noexcept { + return ctx.session.model != nullptr && ctx.voice.loaded && + (!ctx.voice.ready || !ctx.voice.prompt_ready); + } +}; + +struct guard_predict_request_shape_invalid { + bool operator()(const event::predict_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_predict_request_valid{}(runtime_ev, ctx) && + !guard_predict_blocked_by_voice_prompt{}(runtime_ev, ctx); + } +}; + +struct guard_capture_tokenizer_state_valid { + bool operator()(const event::capture_tokenizer_state &ev, + const action::context &ctx) const noexcept { + return ctx.session.model != nullptr && + ev.cache_out.size() == + static_cast(ctx.lmgen.cache_row_count * + ctx.lmgen.codebook_count); + } +}; + +struct guard_capture_tokenizer_state_invalid { + bool operator()(const event::capture_tokenizer_state &ev, + const action::context &ctx) const noexcept { + return !guard_capture_tokenizer_state_valid{}(ev, ctx); + } +}; + +struct guard_voice_prefill_request_valid { + bool operator()(const event::prefill_voice_run &, + const action::context &ctx) const noexcept { + return ctx.session.model != nullptr && ctx.voice.loaded && + !ctx.voice.ready && ctx.voice.prompt_frame_index >= 0 && + ctx.voice.prompt_frame_index < ctx.voice.prompt_frame_count && + ctx.voice.embedding_dim > 0 && + ctx.voice.embedding_dim <= + static_cast(event::k_max_voice_embedding_dim); + } +}; + +struct guard_voice_prefill_request_invalid { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_voice_prefill_request_valid{}(runtime_ev, ctx); + } +}; + +struct guard_voice_embedding_frame_f32 { + bool operator()(const event::prefill_voice_run &, + const action::context &ctx) const noexcept { + return ctx.voice.contract.voice.embeddings.tensor != nullptr && + static_cast( + ctx.voice.contract.voice.embeddings.tensor->type) == + emel::kernel::detail::dtype_f32; + } +}; + +struct guard_voice_embedding_frame_f16 { + bool operator()(const event::prefill_voice_run &, + const action::context &ctx) const noexcept { + return ctx.voice.contract.voice.embeddings.tensor != nullptr && + static_cast( + ctx.voice.contract.voice.embeddings.tensor->type) == + emel::kernel::detail::dtype_f16; + } +}; + +struct guard_voice_embedding_frame_bf16 { + bool operator()(const event::prefill_voice_run &, + const action::context &ctx) const noexcept { + return ctx.voice.contract.voice.embeddings.tensor != nullptr && + static_cast( + ctx.voice.contract.voice.embeddings.tensor->type) == + emel::kernel::detail::dtype_bf16; + } +}; + +struct guard_voice_embedding_frame_loaded { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.embedding_frame_ok; + } +}; + +struct guard_voice_embedding_frame_failed { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_voice_embedding_frame_loaded{}(runtime_ev, ctx); + } +}; + +struct guard_voice_prefill_complete { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.complete; + } +}; + +struct guard_voice_prefill_pending { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_voice_prefill_complete{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_prompt_begin_valid { + bool operator()(const event::begin_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + const int64_t frame_count = + static_cast(runtime_ev.request.pre_text_silence_frames) + + static_cast(runtime_ev.request.text_token_count) + + static_cast(runtime_ev.request.post_text_silence_frames); + return ctx.session.model != nullptr && ctx.voice.loaded && + ctx.voice.ready && !ctx.voice.prompt_ready && + !ctx.voice.prompt_started && + runtime_ev.request.text_token_count >= 0 && + runtime_ev.request.pre_text_silence_frames >= 0 && + runtime_ev.request.post_text_silence_frames >= 0 && + frame_count >= 0 && frame_count <= action::k_max_delay_rows && + ctx.session.model->moshi_lm.inference_prompt_token_count == + static_cast(ctx.lmgen.codebook_count); + } +}; + +struct guard_personaplex_prompt_begin_nonempty_valid { + bool operator()(const event::begin_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + const int64_t frame_count = + static_cast(runtime_ev.request.pre_text_silence_frames) + + static_cast(runtime_ev.request.text_token_count) + + static_cast(runtime_ev.request.post_text_silence_frames); + return guard_personaplex_prompt_begin_valid{}(runtime_ev, ctx) && + frame_count > 0; + } +}; + +struct guard_personaplex_prompt_begin_empty_valid { + bool operator()(const event::begin_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + const int64_t frame_count = + static_cast(runtime_ev.request.pre_text_silence_frames) + + static_cast(runtime_ev.request.text_token_count) + + static_cast(runtime_ev.request.post_text_silence_frames); + const auto *model = ctx.session.model; + const int64_t default_frame_count = + model == nullptr + ? -1 + : static_cast( + model->moshi_lm.inference_pre_text_silence_frames) + + static_cast( + model->moshi_lm.inference_post_text_silence_frames); + return guard_personaplex_prompt_begin_valid{}(runtime_ev, ctx) && + frame_count == 0 && default_frame_count > 0 && + default_frame_count <= action::k_max_delay_rows; + } +}; + +struct guard_personaplex_prompt_begin_invalid { + bool operator()(const event::begin_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_personaplex_prompt_begin_nonempty_valid{}(runtime_ev, ctx) && + !guard_personaplex_prompt_begin_empty_valid{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_prompt_current_phase_valid { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.voice.pre_text_silence_remaining > 0 || + (ctx.voice.pre_text_silence_remaining == 0 && + ctx.voice.text_tokens_remaining > 0 && + runtime_ev.request.text_token >= 0 && + runtime_ev.request.text_token < ctx.session.text_card) || + (ctx.voice.pre_text_silence_remaining == 0 && + ctx.voice.text_tokens_remaining == 0 && + ctx.voice.post_text_silence_remaining > 0); + } +}; + +struct guard_personaplex_prompt_prefill_request_valid { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + const int32_t frame_count = ctx.voice.pre_text_silence_remaining + + ctx.voice.text_tokens_remaining + + ctx.voice.post_text_silence_remaining; + return ctx.session.model != nullptr && ctx.voice.loaded && + ctx.voice.ready && ctx.voice.prompt_started && + !ctx.voice.prompt_ready && frame_count > 0 && + guard_personaplex_prompt_current_phase_valid{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_prompt_prefill_request_invalid { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_personaplex_prompt_prefill_request_valid{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_prompt_pre_silence_pending { + bool operator()(const event::prefill_personaplex_prompt_run &, + const action::context &ctx) const noexcept { + return ctx.voice.pre_text_silence_remaining > 0; + } +}; + +struct guard_personaplex_prompt_text_pending { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return ctx.voice.pre_text_silence_remaining == 0 && + ctx.voice.text_tokens_remaining > 0 && + runtime_ev.request.text_token >= 0 && + runtime_ev.request.text_token < ctx.session.text_card; + } +}; + +struct guard_personaplex_prompt_post_silence_pending { + bool operator()(const event::prefill_personaplex_prompt_run &, + const action::context &ctx) const noexcept { + return ctx.voice.pre_text_silence_remaining == 0 && + ctx.voice.text_tokens_remaining == 0 && + ctx.voice.post_text_silence_remaining > 0; + } +}; + +struct guard_personaplex_prompt_phase_invalid { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_personaplex_prompt_current_phase_valid{}(runtime_ev, ctx); + } +}; + +struct guard_reset_graph_succeeded { + bool operator()(const event::reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.graph_accepted; + } +}; + +struct guard_reset_graph_failed { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_graph_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_reset_memory_succeeded { + bool operator()(const event::reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.memory_accepted && + runtime_ev.ctx.memory_error == + static_cast( + emel::error::cast(emel::memory::hybrid::error::none)); + } +}; + +struct guard_reset_memory_failed { + bool operator()(const event::reset_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_reset_memory_succeeded{}(runtime_ev, ctx); + } +}; + +struct guard_personaplex_prompt_complete { + bool operator()(const event::prefill_personaplex_prompt_run &, + const action::context &ctx) const noexcept { + return ctx.voice.pre_text_silence_remaining == 0 && + ctx.voice.text_tokens_remaining == 0 && + ctx.voice.post_text_silence_remaining == 0; + } +}; + +struct guard_personaplex_prompt_pending { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_personaplex_prompt_complete{}(runtime_ev, ctx); + } +}; + +struct guard_has_personaplex_prompt_complete_out { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.complete_out != nullptr; + } +}; + +struct guard_no_personaplex_prompt_complete_out { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_personaplex_prompt_complete_out{}(runtime_ev, ctx); + } +}; + +struct guard_has_personaplex_prompt_remaining_out { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.remaining_frames_out != nullptr; + } +}; + +struct guard_no_personaplex_prompt_remaining_out { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_personaplex_prompt_remaining_out{}(runtime_ev, ctx); + } +}; + +struct guard_has_voice_complete_out { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.complete_out != nullptr; + } +}; + +struct guard_no_voice_complete_out { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_voice_complete_out{}(runtime_ev, ctx); + } +}; + +struct guard_has_voice_remaining_out { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.remaining_frames_out != nullptr; + } +}; + +struct guard_no_voice_remaining_out { + bool operator()(const event::prefill_voice_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_voice_remaining_out{}(runtime_ev, ctx); + } +}; + +struct guard_graph_step_accepted { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + return ev.ctx.graph_accepted && + ev.ctx.graph_error == action::detail_ns::to_error(error::none); + } +}; + +struct guard_graph_step_rejected { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_graph_step_accepted{}(runtime_ev, ctx); + } +}; + +struct guard_graph_step_accepted_personaplex_prompt_pre_silence { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_graph_step_accepted{}(runtime_ev, ctx) && + guard_personaplex_prompt_pre_silence_pending{}(runtime_ev, ctx); + } +}; + +struct guard_graph_step_accepted_personaplex_prompt_text { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_graph_step_accepted{}(runtime_ev, ctx) && + guard_personaplex_prompt_text_pending{}(runtime_ev, ctx); + } +}; + +struct guard_graph_step_accepted_personaplex_prompt_post_silence { + bool operator()(const event::prefill_personaplex_prompt_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_graph_step_accepted{}(runtime_ev, ctx) && + guard_personaplex_prompt_post_silence_pending{}(runtime_ev, ctx); + } +}; + +struct guard_has_graph_error_out { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + const auto &ev = detail::unwrap_runtime_event(runtime_ev); + return ev.request.graph_error_out != nullptr; + } +}; + +struct guard_no_graph_error_out { + template + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_graph_error_out{}(runtime_ev, ctx); + } +}; + +template struct guard_has_error_out { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.error_out != nullptr; + } +}; + +template struct guard_no_error_out { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_error_out{}(runtime_ev, ctx); + } +}; + +template struct guard_has_done_callback { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_done); + } +}; + +template struct guard_no_done_callback { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_done_callback{}(runtime_ev, ctx); + } +}; + +template struct guard_has_error_callback { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_error); + } +}; + +template struct guard_no_error_callback { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_has_error_callback{}(runtime_ev, ctx); + } +}; + +struct guard_unexpected_error_out_present { + template + bool operator()(const event_type &ev, + const action::context &) const noexcept { + if constexpr (requires { + ev.ctx.err; + ev.request.error_out; + }) { + return ev.request.error_out != nullptr; + } else { + return false; + } + } +}; + +struct guard_unexpected_error_out_absent { + template + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_unexpected_error_out_present{}(ev, ctx); + } +}; + +} // namespace emel::speech::predictor::moshi::guard diff --git a/src/emel/speech/predictor/moshi/sm.hpp b/src/emel/speech/predictor/moshi/sm.hpp new file mode 100644 index 00000000..c19256b6 --- /dev/null +++ b/src/emel/speech/predictor/moshi/sm.hpp @@ -0,0 +1,870 @@ +#pragma once +// benchmark: designed + +#include + +#include + +#include "emel/sm.hpp" +#include "emel/speech/predictor/moshi/actions.hpp" +#include "emel/speech/predictor/moshi/context.hpp" +#include "emel/speech/predictor/moshi/events.hpp" +#include "emel/speech/predictor/moshi/guards.hpp" + +namespace emel::speech::predictor::moshi { + +struct state_uninitialized {}; +struct state_reset_graph_result_decision {}; +struct state_reset_memory_result_decision {}; +struct state_reset_failed {}; +struct state_bind_contract_decision {}; +struct state_binding {}; +struct state_lmgen_mode_decision {}; +struct state_configure_lmgen {}; +struct state_initialize_graph {}; +struct state_initialize_graph_decision {}; +struct state_reserve_memory {}; +struct state_reserve_memory_decision {}; +struct state_allocate_sequence {}; +struct state_allocate_sequence_decision {}; +struct state_init_error_out_decision {}; +struct state_init_callback_decision {}; +struct state_init_failed_error_out_decision {}; +struct state_init_failed_callback_decision {}; +struct state_session_ready {}; +struct state_voice_request_decision {}; +struct state_voice_error_out_decision {}; +struct state_voice_callback_decision {}; +struct state_voice_failed_error_out_decision {}; +struct state_voice_failed_callback_decision {}; +struct state_prefill_voice_request_decision {}; +struct state_prefill_voice_allocate_slot {}; +struct state_prefill_voice_allocate_slot_decision {}; +struct state_prefill_voice_capture_memory {}; +struct state_prefill_voice_capture_memory_decision {}; +struct state_prefill_voice_begin {}; +struct state_prefill_voice_embedding_frame {}; +struct state_prefill_voice_embedding_frame_decision {}; +struct state_prefill_voice_graph_runtime_decision {}; +struct state_prefill_voice_running_graph {}; +struct state_prefill_voice_graph_error_out_decision {}; +struct state_prefill_voice_graph_result_decision {}; +struct state_prefill_voice_advance {}; +struct state_prefill_voice_complete_decision {}; +struct state_prefill_voice_output_decision {}; +struct state_prefill_voice_remaining_out_decision {}; +struct state_prefill_voice_error_out_decision {}; +struct state_prefill_voice_callback_decision {}; +struct state_prefill_voice_failed_error_out_decision {}; +struct state_prefill_voice_failed_callback_decision {}; +struct state_personaplex_prompt_begin_decision {}; +struct state_personaplex_prompt_begin_error_out_decision {}; +struct state_personaplex_prompt_begin_callback_decision {}; +struct state_personaplex_prompt_begin_failed_error_out_decision {}; +struct state_personaplex_prompt_begin_failed_callback_decision {}; +struct state_prefill_personaplex_prompt_request_decision {}; +struct state_prefill_personaplex_prompt_allocate_slot {}; +struct state_prefill_personaplex_prompt_allocate_slot_decision {}; +struct state_prefill_personaplex_prompt_capture_memory {}; +struct state_prefill_personaplex_prompt_capture_memory_decision {}; +struct state_prefill_personaplex_prompt_begin {}; +struct state_prefill_personaplex_prompt_phase_decision {}; +struct state_prefill_personaplex_prompt_frame {}; +struct state_prefill_personaplex_prompt_write_input {}; +struct state_prefill_personaplex_prompt_graph_runtime_decision {}; +struct state_prefill_personaplex_prompt_running_graph {}; +struct state_prefill_personaplex_prompt_graph_error_out_decision {}; +struct state_prefill_personaplex_prompt_graph_result_decision {}; +struct state_prefill_personaplex_prompt_advance {}; +struct state_prefill_personaplex_prompt_complete_decision {}; +struct state_prefill_personaplex_prompt_output_decision {}; +struct state_prefill_personaplex_prompt_remaining_out_decision {}; +struct state_prefill_personaplex_prompt_error_out_decision {}; +struct state_prefill_personaplex_prompt_callback_decision {}; +struct state_prefill_personaplex_prompt_failed_error_out_decision {}; +struct state_prefill_personaplex_prompt_failed_callback_decision {}; +struct state_predict_request_decision {}; +struct state_predict_allocate_slot {}; +struct state_predict_allocate_slot_decision {}; +struct state_predict_capture_memory {}; +struct state_predict_capture_memory_decision {}; +struct state_predict_begin {}; +struct state_predict_error_out_decision {}; +struct state_predict_failed_error_out_decision {}; +struct state_prediction_ready {}; +struct state_execute_request_decision {}; +struct state_execute_begin {}; +struct state_execute_graph_runtime_decision {}; +struct state_execute_running_graph {}; +struct state_execute_graph_error_out_decision {}; +struct state_execute_graph_result_decision {}; +struct state_execute_error_out_decision {}; +struct state_execute_failed_error_out_decision {}; +struct state_execution_ready {}; +struct state_sample_request_decision {}; +struct state_sample_begin {}; +struct state_sample_running_graph {}; +struct state_sample_graph_error_out_decision {}; +struct state_sample_graph_result_decision {}; +struct state_sample_error_out_decision {}; +struct state_sample_failed_error_out_decision {}; +struct state_sample_graph_failed_error_out_decision {}; +struct state_uninit_voice_error_out_decision {}; +struct state_uninit_voice_callback_decision {}; +struct state_uninit_prefill_voice_error_out_decision {}; +struct state_uninit_prefill_voice_callback_decision {}; +struct state_uninit_begin_personaplex_prompt_error_out_decision {}; +struct state_uninit_begin_personaplex_prompt_callback_decision {}; +struct state_uninit_prefill_personaplex_prompt_error_out_decision {}; +struct state_uninit_prefill_personaplex_prompt_callback_decision {}; + +template struct model { + auto operator()() const { + namespace sml = stateforward::sml; + using init_run = event::initialize_run; + using begin_prompt_run = event::begin_personaplex_prompt_run; + using load_voice_run = event::load_voice_run; + using prefill_prompt_run = event::prefill_personaplex_prompt_run; + using prefill_voice_run = event::prefill_voice_run; + using predict_run = event::predict_run; + using execute_run = event::execute_run; + using sample_run = event::sample_run; + using reset_run = event::reset_run; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Initialization: Moshi contract, then Cortext-owned memory reservation. + sml::state <= *sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_contract_valid{} ] + / action::effect_bind_contract{} + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_contract_invalid{} ] + / action::effect_mark_bind_failed{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_lmgen{} ] + / action::effect_configure_personaplex_lmgen{} + , sml::state <= sml::state + + sml::completion [ guard::guard_standard_lmgen{} ] + / action::effect_configure_standard_lmgen{} + , sml::state <= sml::state + + sml::completion + / action::effect_initialize_graph{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_initialize_succeeded{} ] + / action::effect_reserve_memory{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_graph_initialize_failed{} ] + / action::effect_mark_graph_runtime_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + / action::effect_allocate_sequence{} + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_done_callback{} ] + / action::effect_emit_initialize_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_initialize_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + //------------------------------------------------------------------------------// + // PersonaPlex voice prompt: bind NATF0/voice GGUF first, then consume + // one prompt embedding frame per top-level RTC prefill dispatch. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_contract_valid{} ] + / action::effect_bind_voice_contract{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_contract_invalid{} ] + / action::effect_mark_voice_contract_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_done_callback{} ] + / action::effect_emit_load_voice_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_load_voice_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_prefill_request_valid{} ] + / action::effect_allocate_voice_prefill_slot{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_prefill_request_invalid{} ] + / action::effect_mark_voice_contract_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + / action::effect_capture_voice_prefill_memory{} + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + / action::effect_begin_voice_prefill{} + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_embedding_frame_f32{} ] + / action::effect_load_voice_embedding_frame_f32{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_embedding_frame_f16{} ] + / action::effect_load_voice_embedding_frame_f16{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_embedding_frame_bf16{} ] + / action::effect_load_voice_embedding_frame_bf16{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_embedding_frame_loaded{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_embedding_frame_failed{} ] + / action::effect_mark_voice_contract_error{} + , sml::state <= sml::state + + sml::completion + / action::effect_run_voice_graph_runtime{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_has_graph_error_out{} ] + / action::effect_store_graph_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_graph_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_accepted{} ] + / action::effect_advance_voice_prefill{} + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_rejected{} ] + / action::effect_mark_graph_runtime_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_prefill_complete{} ] + / action::effect_copy_voice_cache{} + , sml::state <= sml::state + + sml::completion [ guard::guard_voice_prefill_pending{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_voice_complete_out{} ] + / action::effect_store_voice_complete_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_voice_complete_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_voice_remaining_out{} ] + / action::effect_store_voice_remaining_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_voice_remaining_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_done_callback{} ] + / action::effect_emit_prefill_voice_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_prefill_voice_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + //------------------------------------------------------------------------------// + // PersonaPlex system prompt: after voice embeddings/cache are loaded, consume + // the reference silence/text/silence prompt frames one RTC dispatch at a time. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_begin_nonempty_valid{} ] + / action::effect_bind_personaplex_prompt{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_begin_empty_valid{} ] + / action::effect_bind_empty_personaplex_prompt{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_begin_invalid{} ] + / action::effect_mark_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_done_callback{} ] + / action::effect_emit_begin_personaplex_prompt_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_begin_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_prefill_request_valid{} ] + / action::effect_allocate_personaplex_prompt_slot{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_prefill_request_invalid{} ] + / action::effect_mark_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + / action::effect_capture_personaplex_prompt_memory{} + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_accepted{} ] + / action::effect_begin_personaplex_prompt_prefill{} + , sml::state <= sml::state + + sml::completion [ guard::guard_memory_rejected{} ] + / action::effect_mark_memory_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_pre_silence_pending{} ] + / action::effect_build_personaplex_prompt_silence_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_text_pending{} ] + / action::effect_build_personaplex_prompt_text_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_post_silence_pending{} ] + / action::effect_build_personaplex_prompt_silence_frame{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_phase_invalid{} ] + / action::effect_mark_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion + / action::effect_write_and_build_personaplex_prompt_input{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + / action::effect_run_personaplex_prompt_graph_runtime{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_has_graph_error_out{} ] + / action::effect_store_graph_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_graph_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_accepted_personaplex_prompt_pre_silence{} ] + / action::effect_advance_personaplex_prompt_pre_silence{} + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_accepted_personaplex_prompt_text{} ] + / action::effect_advance_personaplex_prompt_text{} + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_accepted_personaplex_prompt_post_silence{} ] + / action::effect_advance_personaplex_prompt_post_silence{} + , sml::state <= sml::state + + sml::completion [ guard::guard_graph_step_rejected{} ] + / action::effect_mark_graph_runtime_error{} + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_complete{} ] + / action::effect_finish_personaplex_prompt{} + , sml::state <= sml::state + + sml::completion [ guard::guard_personaplex_prompt_pending{} ] + / action::effect_publish_personaplex_prompt_pending{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_personaplex_prompt_complete_out{} ] + / action::effect_store_personaplex_prompt_complete_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_personaplex_prompt_complete_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_personaplex_prompt_remaining_out{} ] + / action::effect_store_personaplex_prompt_remaining_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_personaplex_prompt_remaining_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_done_callback{} ] + / action::effect_emit_prefill_personaplex_prompt_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_prefill_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + //------------------------------------------------------------------------------// + // Model-token prediction prepares memory for one typed graph execution. Delay + // tokenization and detokenization are owned by the caller's tokenizer. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_predict_request_valid{} ] + / action::effect_allocate_step_slot{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_predict_blocked_by_voice_prompt{} ] + / action::effect_mark_voice_prompt_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_predict_request_shape_invalid{} ] + / action::effect_mark_step_request_invalid{} + , sml::state <= + sml::state + sml::completion + , sml::state <= + sml::state + sml::completion + [ guard::guard_memory_accepted{} ] / action::effect_capture_memory{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_memory_rejected{} ] / action::effect_mark_memory_error{} + , sml::state <= + sml::state + sml::completion + , sml::state <= + sml::state + sml::completion + [ guard::guard_memory_accepted{} ] / action::effect_begin_predict{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_memory_rejected{} ] / action::effect_mark_memory_error{} + , sml::state <= + sml::state + sml::completion + / action::effect_publish_predict{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + + // Typed graph execution follows prediction-memory preparation. + , sml::state <= sml::state + + sml::event + , sml::state <= sml::state + + sml::completion [ guard::guard_execute_request_valid{} ] + / action::effect_begin_execute{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_execute_request_invalid{} ] + / action::effect_mark_step_request_invalid{} + , sml::state <= + sml::state + sml::completion + , sml::state <= + sml::state + sml::completion + / action::effect_run_prediction_graph{} + , sml::state <= + sml::state + sml::completion + , sml::state <= + sml::state + sml::completion + [ guard::guard_has_graph_error_out{} ] / action::effect_store_execute_graph_error_out{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_no_graph_error_out{} ] + , sml::state <= + sml::state + sml::completion + [ guard::guard_graph_step_accepted{} ] / action::effect_publish_execute{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_graph_step_rejected{} ] + / action::effect_mark_graph_runtime_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + + // Sample publication is a distinct typed request after graph execution. + , sml::state <= sml::state + + sml::event + , sml::state <= + sml::state + sml::completion + [ guard::guard_sample_request_valid{} ] / action::effect_begin_sample{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_sample_request_invalid{} ] + / action::effect_mark_step_request_invalid{} + , sml::state <= sml::state + + sml::completion + / action::effect_run_sampling_graph{} + , sml::state <= + sml::state + sml::completion + , sml::state <= + sml::state + sml::completion + [ guard::guard_has_graph_error_out{} ] / action::effect_store_sample_graph_error_out{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_no_graph_error_out{} ] + , sml::state <= + sml::state + sml::completion + [ guard::guard_graph_step_accepted{} ] / action::effect_publish_sample{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_graph_step_rejected{} ] + / action::effect_mark_graph_runtime_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= + sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_no_error_out{} ] + + // Predictor-owned prompt cache handoff into the injected tokenizer. + , sml::state <= sml::state + + sml::event + [ guard::guard_capture_tokenizer_state_valid{} ] + / action::effect_capture_tokenizer_state{} + , sml::state <= sml::state + + sml::event + [ guard::guard_capture_tokenizer_state_invalid{} ] + / action::effect_reject_capture_tokenizer_state{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_capture_tokenizer_state{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_capture_tokenizer_state{} + + //------------------------------------------------------------------------------// + // Requests before initialization answer with explicit errors. + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_not_initialized_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_not_initialized_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_not_initialized_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_capture_tokenizer_state{} + , sml::state <= sml::state + + sml::event + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_load_voice_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::event + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_prefill_voice_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::event + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_begin_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::event + / action::effect_mark_not_initialized{} + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_out{} ] + / action::effect_store_error_out{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_out{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_has_error_callback{} ] + / action::effect_emit_prefill_personaplex_prompt_error{} + , sml::state <= sml::state + + sml::completion [ guard::guard_no_error_callback{} ] + + //------------------------------------------------------------------------------// + // Known out-of-order prediction protocol requests. + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_step_request_invalid_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_step_request_invalid{} + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_step_request_invalid_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_step_request_invalid{} + , sml::state <= sml::state + + sml::event [ guard::guard_has_error_out{} ] + / action::effect_mark_step_request_invalid_and_store{} + , sml::state <= sml::state + + sml::event [ guard::guard_no_error_out{} ] + / action::effect_mark_step_request_invalid{} + + //------------------------------------------------------------------------------// + // Reset. + , sml::state <= sml::state + + sml::event / action::effect_reset_graph{} + , sml::state <= sml::state + + sml::event / action::effect_reset_graph{} + , sml::state <= sml::state + + sml::event / action::effect_reset_graph{} + , sml::state <= + sml::state + + sml::completion [ guard::guard_reset_graph_succeeded{} ] + / action::effect_release_reset_sequence{} + , sml::state <= sml::state + + sml::completion [ guard::guard_reset_graph_failed{} ] + / action::effect_mark_reset_graph_failed{} + , sml::state <= sml::state + + sml::completion [ guard::guard_reset_memory_succeeded{} ] + / action::effect_reset_session{} + , sml::state <= sml::state + + sml::completion [ guard::guard_reset_memory_failed{} ] + / action::effect_mark_reset_memory_failed{} + , sml::state <= sml::state + + sml::completion / action::effect_reset_session{} + + //------------------------------------------------------------------------------// + // Unexpected events. + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + ); + // clang-format on + } +}; + +template +struct sm : public emel::sm, action::context> { + using base_type = emel::sm, action::context>; + using base_type::is; + using base_type::visit_current_states; + + explicit sm(const action::dependencies &deps) + : base_type(std::piecewise_construct, deps, deps.graph), + memory_snapshot_(std::make_unique()) {} + + sm(const sm &) = delete; + sm &operator=(const sm &) = delete; + sm(sm &&) = delete; + sm &operator=(sm &&) = delete; + + bool process_event(const event::initialize &ev) { + event::initialize_ctx ctx{}; + event::initialize_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::load_voice &ev) { + event::load_voice_ctx ctx{}; + event::load_voice_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::begin_personaplex_prompt &ev) { + event::begin_personaplex_prompt_ctx ctx{}; + event::begin_personaplex_prompt_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::prefill_voice &ev) { + event::prefill_voice_ctx ctx{*memory_snapshot_}; + event::prefill_voice_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::prefill_personaplex_prompt &ev) { + event::prefill_personaplex_prompt_ctx ctx{*memory_snapshot_}; + event::prefill_personaplex_prompt_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::predict &ev) { + event::predict_ctx ctx{ev.prediction_workspace.memory}; + event::predict_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::execute &ev) { + event::execute_ctx ctx{}; + event::execute_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::sample &ev) { + event::sample_ctx ctx{}; + event::sample_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::capture_tokenizer_state &ev) { + const bool accepted = base_type::process_event(ev); + return accepted && ev.error_out == action::detail_ns::to_error(error::none); + } + + bool process_event(const event::reset &ev) { + event::reset_run::context ctx{}; + event::reset_run runtime_ev{ev, ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && ctx.err == action::detail_ns::to_error(error::none); + } + +private: + std::unique_ptr memory_snapshot_ = {}; +}; + +template +sm(const action::dependencies &) -> sm; + +} // namespace emel::speech::predictor::moshi diff --git a/src/emel/speech/tokenizer/moshi/actions.hpp b/src/emel/speech/tokenizer/moshi/actions.hpp new file mode 100644 index 00000000..55c48d4c --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/actions.hpp @@ -0,0 +1,210 @@ +#pragma once + +#include +#include +#include + +#include "emel/error/error.hpp" +#include "emel/speech/tokenizer/moshi/context.hpp" +#include "emel/speech/tokenizer/moshi/detail.hpp" +#include "emel/speech/tokenizer/moshi/errors.hpp" +#include "emel/speech/tokenizer/moshi/events.hpp" + +namespace emel::speech::tokenizer::moshi::action { + +inline int32_t error_code(const error value) noexcept { + return static_cast(emel::error::cast(value)); +} + +struct effect_initialize { + void operator()(const event::initialize &ev, context &ctx) const noexcept { + std::fill(ctx.config.cache.begin(), ctx.config.cache.end(), + ctx.config.token_ungenerated); + ctx.offset = 0; + ev.error_out = error_code(error::none); + } +}; + +struct effect_reset { + void operator()(const event::reset &ev, context &ctx) const noexcept { + std::fill(ctx.config.cache.begin(), ctx.config.cache.end(), + ctx.config.token_ungenerated); + ctx.offset = 0; + ev.error_out = error_code(error::none); + } +}; + +struct effect_restore_column_major_cache { + void operator()(const event::restore_cache &ev, context &ctx) const noexcept { + for (int32_t row = 0; row < ctx.config.cache_rows; ++row) { + for (int32_t codebook = 0; codebook < ctx.config.codebooks; ++codebook) { + const size_t source = static_cast(row) + + static_cast(codebook) * + static_cast(ctx.config.cache_rows); + detail::cache_at(ctx, row, codebook) = ev.column_major_cache[source]; + } + } + ctx.offset = ev.offset; + ev.error_out = error_code(error::none); + } +}; + +struct effect_advance { + void operator()(const event::advance &ev, context &ctx) const noexcept { + ++ctx.offset; + ev.error_out = error_code(error::none); + } +}; + +struct effect_tokenize_full { + void operator()(const event::tokenize &ev, context &ctx) const noexcept { + for (int32_t codebook = 0; codebook < ctx.config.codebooks; ++codebook) { + const int32_t row = detail::compute_cache_row( + ctx, ctx.offset + ctx.config.delays[static_cast(codebook)]); + detail::cache_at(ctx, row, codebook) = + ev.audio_tokens[static_cast(codebook)]; + } + detail::compute_model_tokens(ev, ctx); + ev.error_out = error_code(error::none); + } +}; + +struct effect_tokenize_tail { + void operator()(const event::tokenize &ev, context &ctx) const noexcept { + const int32_t first_codebook = ctx.config.delayed_audio_codebooks + 1; + const int32_t needed = ctx.config.codebooks - first_codebook; + for (int32_t tail = 0; tail < needed; ++tail) { + const int32_t codebook = first_codebook + tail; + const int32_t row = detail::compute_cache_row( + ctx, ctx.offset + ctx.config.delays[static_cast(codebook)]); + detail::cache_at(ctx, row, codebook) = + ev.audio_tokens[static_cast(tail)]; + } + detail::compute_model_tokens(ev, ctx); + ev.error_out = error_code(error::none); + } +}; + +struct effect_tokenize_empty { + void operator()(const event::tokenize &ev, + const context &ctx) const noexcept { + detail::compute_model_tokens(ev, ctx); + ev.error_out = error_code(error::none); + } +}; + +struct effect_begin_detokenize { + void operator()(const event::detokenize_run &runtime_ev, + const context &ctx) const noexcept { + runtime_ev.ctx.source_offset = ctx.offset; + runtime_ev.request.text_token_out = ctx.config.token_zero; + std::fill(runtime_ev.request.audio_tokens_out.begin(), + runtime_ev.request.audio_tokens_out.end(), ctx.config.token_zero); + runtime_ev.request.produced_out = false; + runtime_ev.request.error_out = error_code(error::none); + } +}; + +enum class cache_write_mode : uint8_t { + replace, + preserve_provided, +}; + +enum class audio_write_mode : uint8_t { + zero, + generated, +}; + +template +struct effect_commit_generated { + void operator()(const event::detokenize_run &runtime_ev, + context &ctx) const noexcept { + ctx.offset = runtime_ev.ctx.source_offset + 1; + const int32_t row = detail::compute_cache_row(ctx, ctx.offset); + detail::cache_at(ctx, row, 0) = runtime_ev.request.text_token; + + for (int32_t audio_codebook = 0; + audio_codebook < ctx.config.generated_audio_codebooks; + ++audio_codebook) { + const int32_t codebook = audio_codebook + 1; + int32_t token = ctx.config.token_zero; + if constexpr (audio_mode == audio_write_mode::generated) { + const int32_t masked = static_cast( + ctx.config.initial_delay_frames > 0 && + runtime_ev.ctx.source_offset < + static_cast( + ctx.config.delays[static_cast(codebook)]) + + static_cast(ctx.config.initial_delay_frames)); + token = masked * ctx.config.token_zero + + (1 - masked) * + runtime_ev.request + .audio_tokens[static_cast(audio_codebook)]; + } + + if constexpr (cache_mode == cache_write_mode::preserve_provided) { + const int32_t current = detail::cache_at(ctx, row, codebook); + const int32_t missing = + static_cast(current == ctx.config.token_ungenerated); + token = missing * token + (1 - missing) * current; + } + detail::cache_at(ctx, row, codebook) = token; + } + runtime_ev.request.error_out = error_code(error::none); + } +}; + +struct effect_collect_output { + void operator()(const event::detokenize_run &runtime_ev, + const context &ctx) const noexcept { + int32_t row = detail::compute_cache_row( + ctx, ctx.offset - ctx.config.maximum_delay + ctx.config.delays[0]); + runtime_ev.request.text_token_out = detail::cache_at(ctx, row, 0); + for (int32_t audio_codebook = 0; + audio_codebook < ctx.config.delayed_audio_codebooks; + ++audio_codebook) { + const int32_t codebook = audio_codebook + 1; + row = detail::compute_cache_row( + ctx, ctx.offset - ctx.config.maximum_delay + + ctx.config.delays[static_cast(codebook)]); + runtime_ev.request.audio_tokens_out[static_cast(audio_codebook)] = + detail::cache_at(ctx, row, codebook); + } + } +}; + +struct effect_publish_output { + void operator()(const event::detokenize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.produced_out = true; + runtime_ev.request.error_out = error_code(error::none); + } +}; + +struct effect_publish_no_output { + void operator()(const event::detokenize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.produced_out = false; + runtime_ev.request.error_out = error_code(error::none); + } +}; + +template struct effect_reject { + template + void operator()(const event_type &ev, const context &) const noexcept { + ev.error_out = error_code(error_value); + } +}; + +template struct effect_reject_detokenize { + void operator()(const event::detokenize_run &runtime_ev, + const context &) const noexcept { + runtime_ev.request.error_out = error_code(error_value); + } +}; + +struct effect_unexpected { + template + void operator()(const event_type &, const context &) const noexcept {} +}; + +} // namespace emel::speech::tokenizer::moshi::action diff --git a/src/emel/speech/tokenizer/moshi/any.hpp b/src/emel/speech/tokenizer/moshi/any.hpp new file mode 100644 index 00000000..81d682d6 --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/any.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "emel/speech/tokenizer/moshi/sm.hpp" diff --git a/src/emel/speech/tokenizer/moshi/context.hpp b/src/emel/speech/tokenizer/moshi/context.hpp new file mode 100644 index 00000000..209cd881 --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/context.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +namespace emel::speech::tokenizer::moshi { + +struct dependencies { + std::span delays = {}; + std::span cache = {}; + int32_t codebooks = 0; + int32_t generated_audio_codebooks = 0; + int32_t delayed_audio_codebooks = 0; + int32_t cache_rows = 0; + int32_t maximum_delay = 0; + int32_t initial_delay_frames = 0; + int32_t text_initial_token = 0; + int32_t audio_initial_token = 0; + int32_t token_zero = 0; + int32_t token_ungenerated = 0; +}; + +namespace action { + +struct context { + explicit context(const dependencies &deps) noexcept : config(deps) {} + + const dependencies config; + int64_t offset = 0; +}; + +} // namespace action + +} // namespace emel::speech::tokenizer::moshi diff --git a/src/emel/speech/tokenizer/moshi/detail.hpp b/src/emel/speech/tokenizer/moshi/detail.hpp new file mode 100644 index 00000000..b3dc5e31 --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/detail.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +#include "emel/speech/tokenizer/moshi/context.hpp" +#include "emel/speech/tokenizer/moshi/events.hpp" + +namespace emel::speech::tokenizer::moshi::detail { + +inline int32_t compute_cache_row(const action::context &ctx, + const int64_t offset) noexcept { + const int64_t rows = ctx.config.cache_rows; + const int64_t position = offset % rows; + return static_cast(position + ((position < 0) * rows)); +} + +inline int32_t &cache_at(action::context &ctx, const int32_t row, + const int32_t codebook) noexcept { + return ctx.config.cache[static_cast(row) * + static_cast(ctx.config.codebooks) + + static_cast(codebook)]; +} + +inline int32_t cache_at(const action::context &ctx, const int32_t row, + const int32_t codebook) noexcept { + return ctx.config.cache[static_cast(row) * + static_cast(ctx.config.codebooks) + + static_cast(codebook)]; +} + +inline void compute_model_tokens(const event::tokenize &ev, + const action::context &ctx) noexcept { + const int32_t row = compute_cache_row(ctx, ctx.offset); + const int32_t text_initial_lane = static_cast( + ctx.offset <= static_cast(ctx.config.delays[0])); + const int32_t text_cache_lane = 1 - text_initial_lane; + ev.model_tokens_out[0] = text_initial_lane * ctx.config.text_initial_token + + text_cache_lane * cache_at(ctx, row, 0); + + for (int32_t codebook = 1; codebook < ctx.config.codebooks; ++codebook) { + const int32_t initial_lane = static_cast( + ctx.offset <= static_cast(ctx.config.delays[codebook])); + const int32_t cache_lane = 1 - initial_lane; + ev.model_tokens_out[static_cast(codebook)] = + initial_lane * ctx.config.audio_initial_token + + cache_lane * cache_at(ctx, row, codebook); + } +} + +} // namespace emel::speech::tokenizer::moshi::detail diff --git a/src/emel/speech/tokenizer/moshi/errors.hpp b/src/emel/speech/tokenizer/moshi/errors.hpp new file mode 100644 index 00000000..eb3521ef --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/errors.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::speech::tokenizer::moshi { + +enum class error : emel::error::type { + none = 0u, + invalid_configuration = (1u << 0), + uninitialized = (1u << 1), + already_initialized = (1u << 2), + request_shape = (1u << 3), + phase_order = (1u << 4), + position_overflow = (1u << 5), + internal_error = (1u << 6), +}; + +} // namespace emel::speech::tokenizer::moshi diff --git a/src/emel/speech/tokenizer/moshi/events.hpp b/src/emel/speech/tokenizer/moshi/events.hpp new file mode 100644 index 00000000..0cf0d026 --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/events.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +namespace emel::speech::tokenizer::moshi::event { + +struct initialize { + int32_t &error_out; +}; + +struct tokenize { + std::span audio_tokens = {}; + std::span model_tokens_out = {}; + int32_t &error_out; +}; + +struct detokenize { + int32_t text_token; + std::span audio_tokens; + int32_t &text_token_out; + std::span audio_tokens_out; + bool &produced_out; + int32_t &error_out; +}; + +struct restore_cache { + std::span column_major_cache; + int64_t offset; + int32_t &error_out; +}; + +struct advance { + int32_t &error_out; +}; + +struct reset { + int32_t &error_out; +}; + +struct detokenize_ctx { + int64_t source_offset = 0; +}; + +struct detokenize_run { + const detokenize &request; + detokenize_ctx &ctx; +}; + +} // namespace emel::speech::tokenizer::moshi::event diff --git a/src/emel/speech/tokenizer/moshi/guards.hpp b/src/emel/speech/tokenizer/moshi/guards.hpp new file mode 100644 index 00000000..c066888c --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/guards.hpp @@ -0,0 +1,343 @@ +#pragma once + +#include +#include +#include + +#include "emel/speech/tokenizer/moshi/context.hpp" +#include "emel/speech/tokenizer/moshi/events.hpp" + +namespace emel::speech::tokenizer::moshi::guard { + +struct guard_configuration_valid { + bool operator()(const event::initialize &, + const action::context &ctx) const noexcept { + const auto &config = ctx.config; + if (config.codebooks <= 1 || config.generated_audio_codebooks <= 0 || + config.generated_audio_codebooks >= config.codebooks || + config.delayed_audio_codebooks <= 0 || + config.delayed_audio_codebooks > config.generated_audio_codebooks || + config.cache_rows <= 0 || config.maximum_delay < 0 || + config.initial_delay_frames < 0 || config.text_initial_token <= 0 || + config.audio_initial_token <= 0 || config.token_zero >= 0 || + config.token_ungenerated >= 0 || + config.token_zero == config.token_ungenerated || + config.delays.size() < static_cast(config.codebooks)) { + return false; + } + const int32_t needed = + config.codebooks - config.delayed_audio_codebooks - 1; + if (needed < 0 || static_cast(config.cache_rows) < + static_cast(config.maximum_delay) + 2) { + return false; + } + const uint64_t elements = static_cast(config.cache_rows) * + static_cast(config.codebooks); + if (elements > config.cache.size()) { + return false; + } + int32_t observed_maximum = 0; + for (int32_t codebook = 0; codebook < config.codebooks; ++codebook) { + const int32_t delay = config.delays[static_cast(codebook)]; + if (delay < 0 || delay > config.maximum_delay) { + return false; + } + if (delay > observed_maximum) { + observed_maximum = delay; + } + } + return observed_maximum == config.maximum_delay; + } +}; + +struct guard_configuration_invalid { + bool operator()(const event::initialize &ev, + const action::context &ctx) const noexcept { + return !guard_configuration_valid{}(ev, ctx); + } +}; + +struct guard_tokenize_output_valid { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return ev.model_tokens_out.size() == + static_cast(ctx.config.codebooks); + } +}; + +struct guard_tokenize_position_available { + bool operator()(const event::tokenize &, + const action::context &ctx) const noexcept { + return ctx.offset <= std::numeric_limits::max() - + static_cast(ctx.config.maximum_delay); + } +}; + +struct guard_tokenize_full_shape { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return guard_tokenize_output_valid{}(ev, ctx) && + ev.audio_tokens.size() == static_cast(ctx.config.codebooks); + } +}; + +struct guard_tokenize_tail_shape { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + const int32_t needed = + ctx.config.codebooks - ctx.config.delayed_audio_codebooks - 1; + return guard_tokenize_output_valid{}(ev, ctx) && needed > 0 && + ev.audio_tokens.size() == static_cast(needed); + } +}; + +struct guard_tokenize_empty_shape { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + const int32_t needed = + ctx.config.codebooks - ctx.config.delayed_audio_codebooks - 1; + return guard_tokenize_output_valid{}(ev, ctx) && needed == 0 && + ev.audio_tokens.empty(); + } +}; + +struct guard_tokenize_tokens_valid { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + bool valid = true; + for (const int32_t token : ev.audio_tokens) { + if (token < 0 || token >= ctx.config.audio_initial_token) { + valid = false; + } + } + return valid; + } +}; + +struct guard_tokenize_shape_valid { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return (guard_tokenize_full_shape{}(ev, ctx) || + guard_tokenize_tail_shape{}(ev, ctx) || + guard_tokenize_empty_shape{}(ev, ctx)) && + guard_tokenize_tokens_valid{}(ev, ctx); + } +}; + +struct guard_tokenize_full { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return guard_tokenize_full_shape{}(ev, ctx) && + guard_tokenize_tokens_valid{}(ev, ctx) && + guard_tokenize_position_available{}(ev, ctx); + } +}; + +struct guard_tokenize_tail { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return guard_tokenize_tail_shape{}(ev, ctx) && + guard_tokenize_tokens_valid{}(ev, ctx) && + guard_tokenize_position_available{}(ev, ctx); + } +}; + +struct guard_tokenize_empty { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return guard_tokenize_empty_shape{}(ev, ctx) && + guard_tokenize_tokens_valid{}(ev, ctx) && + guard_tokenize_position_available{}(ev, ctx); + } +}; + +struct guard_tokenize_invalid { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return !guard_tokenize_shape_valid{}(ev, ctx); + } +}; + +struct guard_tokenize_position_overflow { + bool operator()(const event::tokenize &ev, + const action::context &ctx) const noexcept { + return guard_tokenize_shape_valid{}(ev, ctx) && + !guard_tokenize_position_available{}(ev, ctx); + } +}; + +struct guard_restore_valid { + bool operator()(const event::restore_cache &ev, + const action::context &ctx) const noexcept { + const uint64_t elements = static_cast(ctx.config.cache_rows) * + static_cast(ctx.config.codebooks); + bool valid = + elements == ev.column_major_cache.size() && ev.offset >= 0 && + ev.offset <= std::numeric_limits::max() - + static_cast(ctx.config.maximum_delay); + for (size_t index = 0; index < ev.column_major_cache.size(); ++index) { + const int32_t token = ev.column_major_cache[index]; + const bool sentinel = token == ctx.config.token_zero || + token == ctx.config.token_ungenerated; + const size_t codebook = + index / static_cast(ctx.config.cache_rows); + const int32_t upper_bound = codebook == 0u + ? ctx.config.text_initial_token + : ctx.config.audio_initial_token; + if (!sentinel && (token < 0 || token >= upper_bound)) { + valid = false; + } + } + return valid; + } +}; + +struct guard_advance_position_available { + bool operator()(const event::advance &, + const action::context &ctx) const noexcept { + return ctx.offset < std::numeric_limits::max(); + } +}; + +struct guard_advance_position_overflow { + bool operator()(const event::advance &ev, + const action::context &ctx) const noexcept { + return !guard_advance_position_available{}(ev, ctx); + } +}; + +struct guard_restore_invalid { + bool operator()(const event::restore_cache &ev, + const action::context &ctx) const noexcept { + return !guard_restore_valid{}(ev, ctx); + } +}; + +struct guard_detokenize_shape_valid { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return runtime_ev.request.audio_tokens.size() == + static_cast(ctx.config.generated_audio_codebooks) && + runtime_ev.request.audio_tokens_out.size() == + static_cast(ctx.config.delayed_audio_codebooks); + } +}; + +struct guard_detokenize_tokens_valid { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + bool valid = runtime_ev.request.text_token >= 0 && + runtime_ev.request.text_token < ctx.config.text_initial_token; + for (const int32_t token : runtime_ev.request.audio_tokens) { + if (token < 0 || token >= ctx.config.audio_initial_token) { + valid = false; + } + } + return valid; + } +}; + +struct guard_detokenize_request_valid { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_detokenize_shape_valid{}(runtime_ev, ctx) && + guard_detokenize_tokens_valid{}(runtime_ev, ctx); + } +}; + +struct guard_detokenize_request_invalid { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_detokenize_request_valid{}(runtime_ev, ctx); + } +}; + +struct guard_position_available { + bool operator()(const event::detokenize_run &, + const action::context &ctx) const noexcept { + return ctx.offset < std::numeric_limits::max(); + } +}; + +struct guard_position_overflow { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_detokenize_request_valid{}(runtime_ev, ctx) && + !guard_position_available{}(runtime_ev, ctx); + } +}; + +struct guard_replace_generated_audio { + bool operator()(const event::detokenize_run &, + const action::context &ctx) const noexcept { + return ctx.config.initial_delay_frames > 0 && + ctx.offset < ctx.config.initial_delay_frames; + } +}; + +struct guard_use_generated_audio { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_replace_generated_audio{}(runtime_ev, ctx); + } +}; + +struct guard_detokenize_valid_replace { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_detokenize_request_valid{}(runtime_ev, ctx) && + guard_position_available{}(runtime_ev, ctx) && + guard_replace_generated_audio{}(runtime_ev, ctx); + } +}; + +struct guard_detokenize_valid_generated { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return guard_detokenize_request_valid{}(runtime_ev, ctx) && + guard_position_available{}(runtime_ev, ctx) && + guard_use_generated_audio{}(runtime_ev, ctx); + } +}; + +struct guard_before_output_delay { + bool operator()(const event::detokenize_run &, + const action::context &ctx) const noexcept { + return ctx.offset <= ctx.config.maximum_delay; + } +}; + +struct guard_past_output_delay { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_before_output_delay{}(runtime_ev, ctx); + } +}; + +struct guard_output_incomplete { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + bool incomplete = + runtime_ev.request.text_token_out == ctx.config.token_zero || + runtime_ev.request.text_token_out == ctx.config.token_ungenerated; + for (int32_t codebook = 0; codebook < ctx.config.delayed_audio_codebooks; + ++codebook) { + const int32_t token = + runtime_ev.request.audio_tokens_out[static_cast(codebook)]; + if (token == ctx.config.token_zero || + token == ctx.config.token_ungenerated) { + incomplete = true; + } + } + return incomplete; + } +}; + +struct guard_output_complete { + bool operator()(const event::detokenize_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_output_incomplete{}(runtime_ev, ctx); + } +}; + +} // namespace emel::speech::tokenizer::moshi::guard diff --git a/src/emel/speech/tokenizer/moshi/sm.hpp b/src/emel/speech/tokenizer/moshi/sm.hpp new file mode 100644 index 00000000..35b81005 --- /dev/null +++ b/src/emel/speech/tokenizer/moshi/sm.hpp @@ -0,0 +1,305 @@ +#pragma once + +// benchmark: designed + +#include + +#include "emel/sm.hpp" +#include "emel/speech/tokenizer/moshi/actions.hpp" +#include "emel/speech/tokenizer/moshi/context.hpp" +#include "emel/speech/tokenizer/moshi/events.hpp" +#include "emel/speech/tokenizer/moshi/guards.hpp" + +namespace emel::speech::tokenizer::moshi { + +struct state_uninitialized {}; +struct state_ready {}; +struct state_prepared_full {}; +struct state_prepared_tail {}; +struct state_prepared_generated {}; +struct state_commit_full_zero {}; +struct state_commit_full_generated {}; +struct state_commit_generated_zero {}; +struct state_commit_generated {}; +struct state_output_decision {}; +struct state_output_validation {}; +struct state_errored {}; + +/* +Moshi tokenizer architecture notes (single source of truth) + +state purpose +- state_ready accepts the first half of a frame and tokenizes the caller's + audio codebooks into the delay-aligned LM input sequence. +- state_prepared_full records that all codebooks were provided, + state_prepared_tail records that PersonaPlex tail lanes were provided, and + state_prepared_generated records generator-owned input. The state graph + therefore owns cache-write policy without a persistent flag. +- commit states select initial-delay replacement and provided-cache + preservation before executing fixed data-plane cache writes. + +control invariants +- delay geometry, token sentinels, and cache storage are injected; the actor + owns no global limits and allocates nothing during construction or dispatch. +- tokenize and detokenize must alternate. Phase-order, shape, overflow, + initialization, and restore failures are explicit transitions. +- context contains only the persistent frame offset and injected storage. + Per-dispatch source offsets travel through the typed detokenize_run event. + +guard semantics and action side effects +- configuration guards validate every injected extent and delay before the + cache is initialized. +- tokenize guards select full, tail, or empty input shapes; their actions write + delayed inputs and publish one model-token frame. +- detokenize guards select generated versus zero audio and overflow outcomes; + commit actions advance once, update the ring, and output actions reverse the + codebook delay pattern. +*/ +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + using effect_commit_full_zero = action::effect_commit_generated< + action::cache_write_mode::preserve_provided, + action::audio_write_mode::zero>; + using effect_commit_full_generated = action::effect_commit_generated< + action::cache_write_mode::preserve_provided, + action::audio_write_mode::generated>; + using effect_commit_zero = + action::effect_commit_generated; + using effect_commit_generated = + action::effect_commit_generated; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Initialization and recovery validate only injected geometry. + sml::state <= *sml::state + + sml::event [ guard::guard_configuration_valid{} ] + / action::effect_initialize{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_invalid{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_valid{} ] + / action::effect_initialize{} + , sml::state <= sml::state + + sml::event [ guard::guard_configuration_invalid{} ] + / action::effect_reject{} + + //------------------------------------------------------------------------------// + // Tokenization writes delayed caller input and records the cache policy + // for the matching detokenize dispatch in state. + , sml::state <= sml::state + + sml::event [ guard::guard_tokenize_full{} ] + / action::effect_tokenize_full{} + , sml::state <= sml::state + + sml::event [ guard::guard_tokenize_tail{} ] + / action::effect_tokenize_tail{} + , sml::state <= sml::state + + sml::event [ guard::guard_tokenize_empty{} ] + / action::effect_tokenize_empty{} + , sml::state <= sml::state + + sml::event [ guard::guard_tokenize_invalid{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + [ guard::guard_tokenize_position_overflow{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + + //------------------------------------------------------------------------------// + // Detokenization explicitly selects full-input preservation and initial + // delay replacement before the fixed commit action executes. + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_replace{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_generated{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_replace{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_generated{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_replace{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_valid_generated{} ] + / action::effect_begin_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_request_invalid{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_request_invalid{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event + [ guard::guard_detokenize_request_invalid{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event [ guard::guard_position_overflow{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event [ guard::guard_position_overflow{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event [ guard::guard_position_overflow{} ] + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::completion / effect_commit_full_zero{} + , sml::state <= sml::state + + sml::completion / effect_commit_full_generated{} + , sml::state <= sml::state + + sml::completion / effect_commit_zero{} + , sml::state <= sml::state + + sml::completion / effect_commit_generated{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_before_output_delay{} ] / action::effect_publish_no_output{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_past_output_delay{} ] / action::effect_collect_output{} + , sml::state <= sml::state + + sml::completion [ guard::guard_output_complete{} ] + / action::effect_publish_output{} + , sml::state <= sml::state + + sml::completion [ guard::guard_output_incomplete{} ] + / action::effect_publish_no_output{} + + //------------------------------------------------------------------------------// + // Cache restore and sequence reset preserve injected geometry. + , sml::state <= sml::state + + sml::event [ guard::guard_restore_valid{} ] + / action::effect_restore_column_major_cache{} + , sml::state <= sml::state + + sml::event [ guard::guard_restore_invalid{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event + [ guard::guard_advance_position_available{} ] / action::effect_advance{} + , sml::state <= sml::state + + sml::event + [ guard::guard_advance_position_overflow{} ] + / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + , sml::state <= sml::state + + sml::event / action::effect_reset{} + + //------------------------------------------------------------------------------// + // Known operations outside their legal phase are explicit. + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + , sml::state <= sml::state + + sml::event + / action::effect_reject_detokenize{} + , sml::state <= sml::state + + sml::event + / action::effect_reject{} + , sml::state <= sml::state + + sml::event / action::effect_reject{} + + //------------------------------------------------------------------------------// + // Unknown external events make the actor observably errored. + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::process_event; + using base_type::visit_current_states; + + explicit sm(const dependencies &deps) : base_type(std::in_place, deps) {} + + bool process_event(const event::detokenize &ev) { + event::detokenize_ctx ctx{}; + event::detokenize_run runtime_ev{ev, ctx}; + return base_type::process_event(runtime_ev); + } +}; + +} // namespace emel::speech::tokenizer::moshi diff --git a/src/emel/text/encoders/sm.hpp b/src/emel/text/encoders/sm.hpp deleted file mode 100644 index fc9e3f6f..00000000 --- a/src/emel/text/encoders/sm.hpp +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -/* -design doc: docs/designs/text/encoders/encoder.design.md - --- - title: text/encoders architecture design - status: rolling - --- - - # text/encoders architecture design - - this document defines the text-domain encoder actor cluster that maps - preprocessed text fragments to token ids. this is distinct from model - encoders (vision/audio/etc) and lives under the text domain for clarity. - - ## role - - text/encoders is the algorithmic encoder cluster used by the tokenizer codec. - - it encodes raw text fragments (or preprocessed fragments) into token ids. - - it is model/vocab-aware but does not perform special-token parsing. - - ## scope - - text-domain tokenization only (BPE/SPM/WPM/UGM/RWKV/PLAMO2/fallback). - - excludes model encoders used for multimodal inputs. - - ## public interface (current contract) - - `event::encode`: - - inputs: `vocab`, `text`, `preprocessed`, `token_ids`, - `token_count_out`, `error_out`, optional sync callbacks. - - outputs: `token_count_out` and `error_out`. - - callbacks (`dispatch_done`/`dispatch_error`) are invoked synchronously and are - not stored. - - ## composition - - per-algorithm SMs: - - `bpe`, `spm`, `wpm`, `ugm`, `rwkv`, `plamo2`, `fallback`. - - each encoder SM uses the same single-step encode flow: - `initialized` -> `encoding` -> `encode_decision` -> (`done` | `errored`), - with explicit unexpected-event handling. - - `text/encoders::any` (sm_any) selects the active encoder kind and dispatches - `event::encode`. - - ## invariants - - no allocations during dispatch. - - no special-token parsing; caller handles special-token policy. - - bounded work per encode request. - - ## error mapping - - invalid requests or capacity errors -> - `emel::text::encoders::error::to_emel(emel::text::encoders::error::code::invalid_argument)`. - - kernel/data errors propagate via `error_out`. - - ## status - - implemented under `src/emel/text/encoders/...` with namespace - `emel::text::encoders`. - - ## open questions - - should `text/encoder` expose a unified `sm` alias or require `any` - everywhere? - - how should byte-fallback support be represented (encoder vs tokenizer - helper)? -*/ - -// benchmark: designed - -#include "emel/text/encoders/bpe/sm.hpp" -#include "emel/text/encoders/fallback/sm.hpp" -#include "emel/text/encoders/plamo2/sm.hpp" -#include "emel/text/encoders/rwkv/sm.hpp" -#include "emel/text/encoders/spm/sm.hpp" -#include "emel/text/encoders/ugm/sm.hpp" -#include "emel/text/encoders/wpm/sm.hpp" - -namespace emel::text::encoders { - -using sm = emel::text::encoders::bpe::sm; -using Encoder = sm; - -} // namespace emel::text::encoders diff --git a/src/emel/text/generator/decode_wavefront/context.hpp b/src/emel/text/generator/decode_wavefront/context.hpp index b755ffc5..d6f301ae 100644 --- a/src/emel/text/generator/decode_wavefront/context.hpp +++ b/src/emel/text/generator/decode_wavefront/context.hpp @@ -6,10 +6,10 @@ namespace emel::text::generator::decode_wavefront::action { using lane_pool = - emel::policy::thread_pool_scheduler; + emel::policy::fork_join_lane_pool; struct context { - lane_pool * pool = nullptr; + lane_pool *pool = nullptr; }; -} // namespace emel::text::generator::decode_wavefront::action +} // namespace emel::text::generator::decode_wavefront::action diff --git a/tests/diarization/sortformer/pipeline/lifecycle_tests.cpp b/tests/diarization/sortformer/pipeline/lifecycle_tests.cpp index c63bdab8..b6efb308 100644 --- a/tests/diarization/sortformer/pipeline/lifecycle_tests.cpp +++ b/tests/diarization/sortformer/pipeline/lifecycle_tests.cpp @@ -18,6 +18,7 @@ #include "emel/diarization/sortformer/modules/detail.hpp" #include "emel/diarization/sortformer/transformer/detail.hpp" #include "emel/model/data.hpp" +#include "emel/model/sortformer/detail.hpp" namespace { diff --git a/tests/diarization/sortformer/request/lifecycle_tests.cpp b/tests/diarization/sortformer/request/lifecycle_tests.cpp index 956c818a..8ed9de85 100644 --- a/tests/diarization/sortformer/request/lifecycle_tests.cpp +++ b/tests/diarization/sortformer/request/lifecycle_tests.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -161,10 +160,6 @@ diarization_request::event::prepare make_request( } // namespace -TEST_CASE("sortformer request keeps top-level compatibility alias") { - CHECK((std::is_same_v)); -} - TEST_CASE("diarization request extracts deterministic Sortformer features") { auto model = std::make_unique(); const auto contract = make_contract(model->model); diff --git a/tests/gguf/loader/lifecycle_tests.cpp b/tests/gguf/loader/lifecycle_tests.cpp index 38769d29..ee4279af 100644 --- a/tests/gguf/loader/lifecycle_tests.cpp +++ b/tests/gguf/loader/lifecycle_tests.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -639,3 +640,17 @@ TEST_CASE("gguf loader explicit error guard classification") { parse_ctx.err = 0x7fff; CHECK(emel::gguf::loader::guard::parse_error_unknown{}(parse_runtime, ctx)); } + +TEST_CASE("gguf packed q4_k row group ceiling does not overflow") { + const std::array dims{ + 256u, + std::numeric_limits::max(), + 1u, + 1u, + }; + uint64_t data_size = 0u; + + CHECK(emel::gguf::loader::detail::compute_emel_packed_q4_k_x8_data_size( + dims, 2u, data_size) == + emel::error::cast(emel::gguf::loader::error::capacity)); +} diff --git a/tests/io/loader/lifecycle_tests.cpp b/tests/io/loader/lifecycle_tests.cpp index 168aba77..89a82d92 100644 --- a/tests/io/loader/lifecycle_tests.cpp +++ b/tests/io/loader/lifecycle_tests.cpp @@ -139,11 +139,9 @@ uint32_t occurrence_count(const std::string_view source, TEST_CASE("io loader exposes canonical machine aliases") { emel::io::loader::sm loader{}; - emel::io::sm io_loader{}; emel::IoLoader top_level_loader{}; CHECK(loader.is(stateforward::sml::state)); - CHECK(io_loader.is(stateforward::sml::state)); CHECK(top_level_loader.is( stateforward::sml::state)); } diff --git a/tests/kernel/aarch64_tests.cpp b/tests/kernel/aarch64_tests.cpp index b0347ad8..9972bb48 100644 --- a/tests/kernel/aarch64_tests.cpp +++ b/tests/kernel/aarch64_tests.cpp @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -55,7 +57,9 @@ std::filesystem::path repo_root() { #ifdef EMEL_TEST_REPO_ROOT return std::filesystem::path{EMEL_TEST_REPO_ROOT}; #else - return std::filesystem::path{__FILE__}.parent_path().parent_path() + return std::filesystem::path{__FILE__} + .parent_path() + .parent_path() .parent_path(); #endif } @@ -69,15 +73,14 @@ std::string read_text_file(const std::filesystem::path &path) { // Mirrors the generator's parallel matmul lane slicing: a group-aligned row // slice of src0/dst embedded in the full batch-major output, sharing src1. -inline emel::kernel::event::op_mul_mat make_row_sliced_lane_event( - const emel::kernel::event::op_mul_mat & ev, - const uint64_t group_rows, - const uint64_t row_begin, - const uint64_t row_count) { +inline emel::kernel::event::op_mul_mat +make_row_sliced_lane_event(const emel::kernel::event::op_mul_mat &ev, + const uint64_t group_rows, const uint64_t row_begin, + const uint64_t row_count) { emel::kernel::event::op_mul_mat sliced = ev; const uint64_t slice_groups = (row_count + group_rows - 1u) / group_rows; sliced.src0.data = static_cast(ev.src0.data) + - (row_begin / group_rows) * ev.src0.nb[1]; + (row_begin / group_rows) * ev.src0.nb[1]; sliced.src0.ne[1] = row_count; sliced.src0.nb[2] = ev.src0.nb[1] * slice_groups; sliced.src0.nb[3] = sliced.src0.nb[2]; @@ -89,7 +92,68 @@ inline emel::kernel::event::op_mul_mat make_row_sliced_lane_event( return sliced; } -} // namespace +float compute_ggml_q4_k_q8_k_row( + const emel::kernel::detail::quant::block_q4_k *lhs, + const emel::kernel::detail::quant::block_q8_k *rhs, + const size_t block_count) { + constexpr uint32_t kmask1 = 0x3f3f3f3fu; + constexpr uint32_t kmask2 = 0x0f0f0f0fu; + constexpr uint32_t kmask3 = 0x03030303u; + float sum = 0.0f; + + for (size_t block = 0; block < block_count; ++block) { + uint32_t decoded_words[4] = {}; + std::memcpy(decoded_words, lhs[block].scales.data(), + lhs[block].scales.size()); + decoded_words[3] = ((decoded_words[2] >> 4u) & kmask2) | + (((decoded_words[1] >> 6u) & kmask3) << 4u); + const uint32_t decoded_aux = decoded_words[1] & kmask1; + decoded_words[1] = (decoded_words[2] & kmask2) | + (((decoded_words[0] >> 6u) & kmask3) << 4u); + decoded_words[2] = decoded_aux; + decoded_words[0] &= kmask1; + + const auto *scales = reinterpret_cast(decoded_words); + const auto *mins = reinterpret_cast(decoded_words + 2); + int32_t min_sum = 0; + for (size_t index = 0; index < 8u; ++index) { + const int32_t pair_sum = + static_cast(rhs[block].bsums[index * 2u]) + + static_cast(rhs[block].bsums[index * 2u + 1u]); + min_sum += pair_sum * static_cast(mins[index]); + } + + const uint8_t *q4 = lhs[block].qs.data(); + const int8_t *q8 = rhs[block].qs.data(); + int32_t low_sum = 0; + int32_t high_sum = 0; + for (size_t group = 0; group < 4u; ++group) { + int32_t low_dot = 0; + int32_t high_dot = 0; + for (size_t index = 0; index < 32u; ++index) { + low_dot += static_cast(q4[index] & 0x0fu) * + static_cast(q8[index]); + high_dot += static_cast(q4[index] >> 4u) * + static_cast(q8[index + 32u]); + } + low_sum += low_dot * static_cast(scales[group * 2u]); + high_sum += high_dot * static_cast(scales[group * 2u + 1u]); + q4 += 32u; + q8 += 64u; + } + + const float d = + emel::kernel::detail::quant::fp16_to_fp32(lhs[block].d) * rhs[block].d; + const float dmin = + emel::kernel::detail::quant::fp16_to_fp32(lhs[block].dmin) * + rhs[block].d; + sum -= dmin * static_cast(min_sum); + sum += d * static_cast(low_sum + high_sum); + } + return sum; +} + +} // namespace TEST_CASE("kernel_aarch64_numeric_paths") { float lhs[4] = {1.0f, 2.0f, 3.0f, 4.0f}; @@ -1375,6 +1439,56 @@ TEST_CASE("kernel_aarch64_q4_k_row_neon_matches_scalar") { #endif } +TEST_CASE("kernel_aarch64_q4_k_row_matches_ggml_accumulation_order") { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + CHECK(true); +#else + using emel::kernel::detail::quant::block_q4_k; + using emel::kernel::detail::quant::block_q8_k; + using emel::kernel::detail::quant::QK_K; + + constexpr size_t block_count = 64u; + std::array q4_blocks = {}; + std::array q8_blocks = {}; + for (size_t block = 0; block < block_count; ++block) { + auto &q4 = q4_blocks[block]; + auto &q8 = q8_blocks[block]; + q4.d = emel::kernel::detail::quant::fp32_to_fp16( + 0.0009765625f * static_cast((block % 31u) + 1u)); + q4.dmin = emel::kernel::detail::quant::fp32_to_fp16( + 0.00048828125f * static_cast((block % 29u) + 1u)); + q8.d = 0.000244140625f * static_cast((block % 23u) + 1u); + for (size_t index = 0; index < q4.scales.size(); ++index) { + q4.scales[index] = + static_cast((block * 37u + index * 19u + 0x5au) & 0xffu); + } + for (size_t index = 0; index < q4.qs.size(); ++index) { + q4.qs[index] = + static_cast((block * 17u + index * 29u + 0x96u) & 0xffu); + } + for (size_t index = 0; index < q8.qs.size(); ++index) { + q8.qs[index] = static_cast( + static_cast((block * 13u + index * 7u) % 255u) - 127); + } + for (size_t group = 0; group < q8.bsums.size(); ++group) { + int32_t group_sum = 0; + for (size_t index = 0; index < 16u; ++index) { + group_sum += static_cast(q8.qs[group * 16u + index]); + } + q8.bsums[group] = static_cast(group_sum); + } + } + + const float expected = compute_ggml_q4_k_q8_k_row( + q4_blocks.data(), q8_blocks.data(), block_count); + const float actual = emel::kernel::aarch64::detail::dot_q4_k_q8_k_row_neon( + q4_blocks.data(), q8_blocks.data(), block_count); + + CHECK(actual == expected); +#endif +} + TEST_CASE("kernel_aarch64_q4_k_2rows_neon_matches_scalar") { #if !(defined(__aarch64__) || defined(__ARM_NEON)) CHECK(true); @@ -1897,7 +2011,8 @@ TEST_CASE("kernel_aarch64_f16_mul_mat_without_neon_takes_shared_route") { } } -TEST_CASE("kernel_aarch64_f32_vector_gemv_route_is_explicit_and_bit_identical") { +TEST_CASE( + "kernel_aarch64_f32_vector_gemv_route_is_explicit_and_bit_identical") { // k covers four 16-wide f32 steps plus a 6-element scalar tail so the // vfmaq body and the leftover loop of the ported ggml_vec_dot_f32 both // execute. Integer-valued inputs keep every product and partial sum @@ -2172,14 +2287,28 @@ TEST_CASE("kernel_aarch64_q4_k_uses_neon_dispatch_when_dotprod_is_available") { #if defined(__ARM_FEATURE_DOTPROD) CHECK(emel::kernel::aarch64::detail::can_use_neon(shared_ev, true)); + CHECK(emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_q4_vector_f32_rhs(shared_ev, true)); #else CHECK_FALSE(emel::kernel::aarch64::detail::can_use_neon(shared_ev, true)); + CHECK_FALSE(emel::kernel::aarch64::detail:: + can_use_neon_mul_mat_q4_vector_f32_rhs(shared_ev, true)); #endif aarch64_sm machine{}; CHECK(machine.process_event(shared_ev)); CHECK(shared_out[0] == doctest::Approx(scalar_out[0]).epsilon(1.0e-6f)); CHECK(shared_out[0] == doctest::Approx(-256.0f).epsilon(1.0e-6f)); +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD) + CHECK(machine.optimized_q4_dispatch_count() == 1u); + CHECK(machine.optimized_q4_vector_dispatch_count() == 1u); + CHECK(machine.shared_q4_dispatch_count() == 0u); +#else + CHECK(machine.optimized_q4_dispatch_count() == 0u); + CHECK(machine.optimized_q4_vector_dispatch_count() == 0u); + CHECK(machine.shared_q4_dispatch_count() == 1u); +#endif } TEST_CASE("kernel_aarch64_supported_quantized_dispatch_is_alloc_free") { @@ -2545,7 +2674,9 @@ TEST_CASE( std::array native_out = {}; std::array packed_bl4_out = {}; + std::array packed_bl4_f32_out = {}; std::array packed_bl8_out = {}; + std::array packed_bl8_f32_out = {}; const emel::kernel::event::op_mul_mat native_ev{ .src0 = make_quantized_src(native_rows.data(), dtype::q4_k, QK_K, Q4_K_X8_ROWS), @@ -2564,20 +2695,84 @@ TEST_CASE( .src1 = make_q8_k_vector_src(q8_input.data(), QK_K), .dst = make_dst(packed_bl8_out.data(), dtype::f32, 1u, Q4_K_X8_ROWS), }; + const emel::kernel::event::op_mul_mat packed_bl4_f32_ev{ + .src0 = make_packed_q4_k_x8_bl4_src(packed_bl4_storage.data(), QK_K, + Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, QK_K), + .dst = + make_dst(packed_bl4_f32_out.data(), dtype::f32, 1u, Q4_K_X8_ROWS), + }; + const emel::kernel::event::op_mul_mat packed_bl8_f32_ev{ + .src0 = make_packed_q4_k_x8_bl8_src(packed_bl8_storage.data(), QK_K, + Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, QK_K), + .dst = + make_dst(packed_bl8_f32_out.data(), dtype::f32, 1u, Q4_K_X8_ROWS), + }; + float argmax_score = 0.0f; + int32_t argmax_index = -1; + float bl4_argmax_score = 0.0f; + int32_t bl4_argmax_index = -1; + const emel::kernel::event::op_mul_mat_argmax packed_bl4_f32_argmax_ev{ + .src0 = make_packed_q4_k_x8_bl4_src(packed_bl4_storage.data(), QK_K, + Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, QK_K), + .dst = make_dst(&bl4_argmax_score, dtype::f32, 1u, 1u), + .index_out = &bl4_argmax_index, + }; + const emel::kernel::event::op_mul_mat_argmax packed_bl8_f32_argmax_ev{ + .src0 = make_packed_q4_k_x8_bl8_src(packed_bl8_storage.data(), QK_K, + Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, QK_K), + .dst = make_dst(&argmax_score, dtype::f32, 1u, 1u), + .index_out = &argmax_index, + }; aarch64_sm native_machine{}; aarch64_sm packed_bl4_machine{}; + aarch64_sm packed_bl4_f32_machine{}; + aarch64_sm packed_bl4_f32_argmax_machine{}; aarch64_sm packed_bl8_machine{}; + aarch64_sm packed_bl8_f32_machine{}; + aarch64_sm packed_bl8_f32_argmax_machine{}; CHECK(native_machine.process_event(native_ev)); CHECK(packed_bl4_machine.process_event(packed_bl4_ev)); + CHECK(packed_bl4_f32_machine.process_event(packed_bl4_f32_ev)); + CHECK(packed_bl4_f32_argmax_machine.process_event( + packed_bl4_f32_argmax_ev)); CHECK(packed_bl8_machine.process_event(packed_bl8_ev)); + CHECK(packed_bl8_f32_machine.process_event(packed_bl8_f32_ev)); + CHECK(packed_bl8_f32_argmax_machine.process_event( + packed_bl8_f32_argmax_ev)); + + int32_t expected_argmax_index = 0; + float expected_argmax_score = native_out[0]; + for (size_t row = 1; row < native_out.size(); ++row) { + if (native_out[row] > expected_argmax_score) { + expected_argmax_score = native_out[row]; + expected_argmax_index = static_cast(row); + } + } for (size_t row = 0; row < native_out.size(); ++row) { CHECK(packed_bl4_out[row] == doctest::Approx(native_out[row]).epsilon(1.0e-5f)); + CHECK(packed_bl4_f32_out[row] == + doctest::Approx(native_out[row]).epsilon(1.0e-5f)); CHECK(packed_bl8_out[row] == doctest::Approx(native_out[row]).epsilon(1.0e-5f)); + CHECK(packed_bl8_f32_out[row] == + doctest::Approx(native_out[row]).epsilon(1.0e-5f)); + CHECK(std::bit_cast(packed_bl8_out[row]) == + std::bit_cast(native_out[row])); + CHECK(std::bit_cast(packed_bl8_f32_out[row]) == + std::bit_cast(native_out[row])); } + CHECK(bl4_argmax_index == expected_argmax_index); + CHECK(bl4_argmax_score == + doctest::Approx(expected_argmax_score).epsilon(1.0e-5f)); + CHECK(argmax_index == expected_argmax_index); + CHECK(argmax_score == doctest::Approx(expected_argmax_score).epsilon(1.0e-5f)); CHECK(packed_bl4_machine.optimized_q4_dispatch_count() == 1u); CHECK(packed_bl4_machine.optimized_q4_vector_dispatch_count() == 1u); @@ -2585,17 +2780,108 @@ TEST_CASE( CHECK(packed_bl4_machine.optimized_q4_vector_packed_q8_rhs_dispatch_count() == 1u); CHECK(packed_bl4_machine.shared_q4_dispatch_count() == 0u); + CHECK(packed_bl4_f32_machine.optimized_q4_dispatch_count() == 1u); + CHECK(packed_bl4_f32_machine.optimized_q4_vector_dispatch_count() == 1u); + CHECK(packed_bl4_f32_machine.optimized_q4_vector_packed_dispatch_count() == + 1u); + CHECK(packed_bl4_f32_machine.shared_q4_dispatch_count() == 0u); + CHECK(packed_bl4_f32_argmax_machine.optimized_q4_dispatch_count() == 1u); + CHECK(packed_bl4_f32_argmax_machine.optimized_q4_vector_dispatch_count() == + 1u); + CHECK(packed_bl4_f32_argmax_machine + .optimized_q4_vector_packed_dispatch_count() == 1u); + CHECK(packed_bl4_f32_argmax_machine.shared_q4_dispatch_count() == 0u); CHECK(packed_bl8_machine.optimized_q4_dispatch_count() == 1u); CHECK(packed_bl8_machine.optimized_q4_vector_dispatch_count() == 1u); CHECK(packed_bl8_machine.optimized_q4_vector_packed_dispatch_count() == 1u); CHECK(packed_bl8_machine.optimized_q4_vector_packed_q8_rhs_dispatch_count() == 1u); CHECK(packed_bl8_machine.shared_q4_dispatch_count() == 0u); + CHECK(packed_bl8_f32_machine.optimized_q4_dispatch_count() == 1u); + CHECK(packed_bl8_f32_machine.optimized_q4_vector_dispatch_count() == 1u); + CHECK(packed_bl8_f32_machine.optimized_q4_vector_packed_dispatch_count() == + 1u); + CHECK(packed_bl8_f32_machine + .optimized_q4_vector_packed_q8_rhs_dispatch_count() == 0u); + CHECK(packed_bl8_f32_machine.shared_q4_dispatch_count() == 0u); + CHECK(packed_bl8_f32_argmax_machine.optimized_q4_dispatch_count() == 1u); + CHECK(packed_bl8_f32_argmax_machine.optimized_q4_vector_dispatch_count() == + 1u); + CHECK(packed_bl8_f32_argmax_machine + .optimized_q4_vector_packed_dispatch_count() == 1u); + CHECK(packed_bl8_f32_argmax_machine.shared_q4_dispatch_count() == 0u); #endif } -TEST_CASE( - "kernel_aarch64_raw_quantized_prepared_rhs_routes_are_explicit_and_numeric_match") { +TEST_CASE("kernel_aarch64_q4_packed_bl8_preserves_native_block_order_bits") { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + CHECK(true); +#else + using emel::kernel::detail::quant::block_q4_k; + using emel::kernel::detail::quant::block_q4_kx8; + using emel::kernel::detail::quant::Q4_K_X8_ROWS; + using emel::kernel::detail::quant::QK_K; + + constexpr uint64_t block_count = 4u; + constexpr uint64_t cols = block_count * QK_K; + std::array native_rows = {}; + for (size_t row = 0; row < Q4_K_X8_ROWS; ++row) { + for (size_t block = 0; block < block_count; ++block) { + auto &value = native_rows[row * block_count + block]; + value.d = static_cast(0x3000u + row * 31u + block * 7u); + value.dmin = static_cast(0x2800u + row * 19u + block * 11u); + for (size_t index = 0; index < value.scales.size(); ++index) { + value.scales[index] = static_cast( + (row * 37u + block * 53u + index * 17u + 9u) & 0xffu); + } + for (size_t index = 0; index < value.qs.size(); ++index) { + value.qs[index] = static_cast( + (row * 29u + block * 43u + index * 13u + 5u) & 0xffu); + } + } + } + + const std::array input = [] { + std::array values = {}; + for (size_t index = 0; index < values.size(); ++index) { + const int32_t centered = static_cast(index % 61u) - 30; + values[index] = static_cast(centered) * 0.03125f; + } + return values; + }(); + + std::array packed = {}; + REQUIRE(emel::kernel::detail::quant::pack_q4_k_rows_x8_bl8( + native_rows.data(), Q4_K_X8_ROWS, cols, packed.data())); + + std::array native_out = {}; + std::array packed_out = {}; + const emel::kernel::event::op_mul_mat native_ev{ + .src0 = make_quantized_src(native_rows.data(), dtype::q4_k, cols, + Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, cols), + .dst = make_dst(native_out.data(), dtype::f32, 1u, Q4_K_X8_ROWS), + }; + const emel::kernel::event::op_mul_mat packed_ev{ + .src0 = make_packed_q4_k_x8_bl8_src(packed.data(), cols, Q4_K_X8_ROWS), + .src1 = make_src(input.data(), dtype::f32, 1u, cols), + .dst = make_dst(packed_out.data(), dtype::f32, 1u, Q4_K_X8_ROWS), + }; + + aarch64_sm native_machine{}; + aarch64_sm packed_machine{}; + REQUIRE(native_machine.process_event(native_ev)); + REQUIRE(packed_machine.process_event(packed_ev)); + for (size_t row = 0; row < native_out.size(); ++row) { + CHECK(std::bit_cast(packed_out[row]) == + std::bit_cast(native_out[row])); + } +#endif +} + +TEST_CASE("kernel_aarch64_raw_quantized_prepared_rhs_routes_are_explicit_and_" + "numeric_match") { #if !(defined(__aarch64__) && defined(__ARM_NEON) && \ defined(__ARM_FEATURE_DOTPROD)) CHECK(true); @@ -3470,10 +3756,10 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_multi_block_matrix_x8_route_matches_" #endif } -TEST_CASE("kernel_aarch64_q4_k_packed_bl8_routes_do_not_compile_bad_i8mm_tile") { - const std::string actions_source = - read_text_file(repo_root() / "src" / "emel" / "kernel" / "aarch64" / - "actions.hpp"); +TEST_CASE( + "kernel_aarch64_q4_k_packed_bl8_routes_do_not_compile_bad_i8mm_tile") { + const std::string actions_source = read_text_file( + repo_root() / "src" / "emel" / "kernel" / "aarch64" / "actions.hpp"); CHECK(actions_source.find("dot_q4_k_x8_q8_k_group_bl8_i8mm_x4") == std::string::npos); } @@ -3896,11 +4182,12 @@ TEST_CASE("kernel_aarch64_q6_k_prepared_q8_rhs_i8mm_multi_block_matrix_x4_" #endif } -TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_supported") { - using emel::kernel::detail::quant::Q4_K_X8_ROWS; - using emel::kernel::detail::quant::QK_K; +TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_" + "supported") { using emel::kernel::detail::quant::block_q4_k; using emel::kernel::detail::quant::block_q8_k; + using emel::kernel::detail::quant::Q4_K_X8_ROWS; + using emel::kernel::detail::quant::QK_K; constexpr uint64_t row_count = 16u; constexpr uint64_t slice_rows = Q4_K_X8_ROWS; @@ -3908,13 +4195,15 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_su constexpr uint64_t col_count = QK_K; constexpr uint64_t block_count = col_count / QK_K; - std::vector native_rows(static_cast(row_count * block_count)); + std::vector native_rows( + static_cast(row_count * block_count)); for (size_t row = 0; row < static_cast(row_count); ++row) { - auto & q4 = native_rows[row]; + auto &q4 = native_rows[row]; q4.d = 0x3c00u; q4.dmin = static_cast(0x3600u + row * 0x0020u); for (size_t idx = 0; idx < q4.scales.size(); ++idx) { - q4.scales[idx] = static_cast((idx * (13u + row)) ^ (0x66u - row)); + q4.scales[idx] = + static_cast((idx * (13u + row)) ^ (0x66u - row)); } for (size_t idx = 0; idx < q4.qs.size(); ++idx) { q4.qs[idx] = static_cast((idx * (7u + row)) ^ (0xa5u + row)); @@ -3934,15 +4223,15 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_su std::vector rhs_q8(static_cast(rhs_rows * block_count)); for (size_t row = 0; row < static_cast(rhs_rows); ++row) { emel::kernel::detail::quant::quantize_row_q8_k_strided( - rhs_dense.data() + row * static_cast(col_count), - 1u, + rhs_dense.data() + row * static_cast(col_count), 1u, rhs_q8.data() + row * static_cast(block_count), static_cast(col_count)); } std::vector reference(static_cast(rhs_rows * row_count), 0.0f); for (size_t rhs_row = 0; rhs_row < static_cast(rhs_rows); ++rhs_row) { - for (size_t lhs_row = 0; lhs_row < static_cast(row_count); ++lhs_row) { + for (size_t lhs_row = 0; lhs_row < static_cast(row_count); + ++lhs_row) { reference[rhs_row * static_cast(row_count) + lhs_row] = emel::kernel::detail::dot_q4_k_q8_k_row_scalar( native_rows.data() + lhs_row * static_cast(block_count), @@ -3953,24 +4242,29 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_su std::vector packed_bl8( sizeof(emel::kernel::detail::quant::block_q4_kx8) * - emel::kernel::detail::quant::packed_q4_k_x8_group_count(row_count) * block_count); + emel::kernel::detail::quant::packed_q4_k_x8_group_count(row_count) * + block_count); REQUIRE(emel::kernel::detail::quant::pack_q4_k_rows_x8_bl8( native_rows.data(), row_count, col_count, packed_bl8.data())); - std::vector packed_out(static_cast(rhs_rows * row_count), 0.0f); + std::vector packed_out(static_cast(rhs_rows * row_count), + 0.0f); aarch64_sm machine{}; const emel::kernel::event::op_mul_mat full_ev{ - .src0 = make_packed_q4_k_x8_bl8_src(packed_bl8.data(), col_count, row_count), + .src0 = + make_packed_q4_k_x8_bl8_src(packed_bl8.data(), col_count, row_count), .src1 = make_q8_k_x8_rhs_src(rhs_q8.data(), rhs_rows, col_count), - .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, row_count), + .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, + row_count), }; const auto lane0 = make_row_sliced_lane_event(full_ev, Q4_K_X8_ROWS, 0u, slice_rows); - const auto lane1 = - make_row_sliced_lane_event(full_ev, Q4_K_X8_ROWS, slice_rows, row_count - slice_rows); + const auto lane1 = make_row_sliced_lane_event( + full_ev, Q4_K_X8_ROWS, slice_rows, row_count - slice_rows); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD) CHECK(machine.process_event(lane0)); CHECK(machine.process_event(lane1)); for (size_t idx = 0; idx < packed_out.size(); ++idx) { @@ -3983,12 +4277,12 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x8_row_sliced_lane_views_are_su #endif } -TEST_CASE( - "kernel_aarch64_q6_k_prepared_q8_rhs_i8mm_matrix_x8_row_sliced_lane_views_are_supported") { - using emel::kernel::detail::quant::Q6_K_X8_ROWS; - using emel::kernel::detail::quant::QK_K; +TEST_CASE("kernel_aarch64_q6_k_prepared_q8_rhs_i8mm_matrix_x8_row_sliced_lane_" + "views_are_supported") { using emel::kernel::detail::quant::block_q6_k; using emel::kernel::detail::quant::block_q8_k; + using emel::kernel::detail::quant::Q6_K_X8_ROWS; + using emel::kernel::detail::quant::QK_K; constexpr uint64_t row_count = 16u; constexpr uint64_t slice_rows = Q6_K_X8_ROWS; @@ -3996,12 +4290,14 @@ TEST_CASE( constexpr uint64_t col_count = QK_K; constexpr uint64_t block_count = col_count / QK_K; - std::vector native_rows(static_cast(row_count * block_count)); + std::vector native_rows( + static_cast(row_count * block_count)); for (size_t row = 0; row < static_cast(row_count); ++row) { - auto & q6 = native_rows[row]; + auto &q6 = native_rows[row]; q6.d = 0x3c00u; for (size_t idx = 0; idx < q6.scales.size(); ++idx) { - const int32_t scale_value = static_cast((idx + row * 5u) % 15u) - 7; + const int32_t scale_value = + static_cast((idx + row * 5u) % 15u) - 7; q6.scales[idx] = static_cast(scale_value); } for (size_t idx = 0; idx < q6.ql.size(); ++idx) { @@ -4025,15 +4321,15 @@ TEST_CASE( std::vector rhs_q8(static_cast(rhs_rows * block_count)); for (size_t row = 0; row < static_cast(rhs_rows); ++row) { emel::kernel::detail::quant::quantize_row_q8_k_strided( - rhs_dense.data() + row * static_cast(col_count), - 1u, + rhs_dense.data() + row * static_cast(col_count), 1u, rhs_q8.data() + row * static_cast(block_count), static_cast(col_count)); } std::vector reference(static_cast(rhs_rows * row_count), 0.0f); for (size_t rhs_row = 0; rhs_row < static_cast(rhs_rows); ++rhs_row) { - for (size_t lhs_row = 0; lhs_row < static_cast(row_count); ++lhs_row) { + for (size_t lhs_row = 0; lhs_row < static_cast(row_count); + ++lhs_row) { reference[rhs_row * static_cast(row_count) + lhs_row] = emel::kernel::detail::dot_q6_k_q8_k_row_scalar( native_rows.data() + lhs_row * static_cast(block_count), @@ -4043,41 +4339,49 @@ TEST_CASE( } std::vector prepared_storage( - emel::kernel::detail::quant::prepared_q6_k_x8_q8_group_storage_bytes(col_count) * + emel::kernel::detail::quant::prepared_q6_k_x8_q8_group_storage_bytes( + col_count) * emel::kernel::detail::quant::packed_q6_k_x8_group_count(row_count)); REQUIRE(emel::kernel::detail::quant::pack_q6_k_rows_x8_q8_prepared( native_rows.data(), row_count, col_count, prepared_storage.data())); - std::vector prepared_out(static_cast(rhs_rows * row_count), 0.0f); + std::vector prepared_out(static_cast(rhs_rows * row_count), + 0.0f); aarch64_sm machine{}; const emel::kernel::event::op_mul_mat full_ev{ - .src0 = make_prepared_q6_k_x8_q8_src(prepared_storage.data(), col_count, row_count), + .src0 = make_prepared_q6_k_x8_q8_src(prepared_storage.data(), col_count, + row_count), .src1 = make_q8_k_x8_rhs_src(rhs_q8.data(), rhs_rows, col_count), - .dst = make_batch_major_dst(prepared_out.data(), dtype::f32, rhs_rows, row_count), + .dst = make_batch_major_dst(prepared_out.data(), dtype::f32, rhs_rows, + row_count), }; const auto lane0 = make_row_sliced_lane_event(full_ev, Q6_K_X8_ROWS, 0u, slice_rows); - const auto lane1 = - make_row_sliced_lane_event(full_ev, Q6_K_X8_ROWS, slice_rows, row_count - slice_rows); + const auto lane1 = make_row_sliced_lane_event( + full_ev, Q6_K_X8_ROWS, slice_rows, row_count - slice_rows); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_MATMUL_INT8) CHECK(machine.process_event(lane0)); CHECK(machine.process_event(lane1)); for (size_t idx = 0; idx < prepared_out.size(); ++idx) { - CHECK(prepared_out[idx] == doctest::Approx(reference[idx]).epsilon(1.0e-5f)); + CHECK(prepared_out[idx] == + doctest::Approx(reference[idx]).epsilon(1.0e-5f)); } - CHECK(machine.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count() == 2u); + CHECK(machine.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count() == + 2u); #else CHECK_FALSE(machine.process_event(lane0)); CHECK_FALSE(machine.process_event(lane1)); #endif } -TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_supported") { +TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_" + "supported") { + using emel::kernel::detail::quant::block_q8_0; using emel::kernel::detail::quant::Q8_0_X4_ROWS; using emel::kernel::detail::quant::QK8_0; - using emel::kernel::detail::quant::block_q8_0; constexpr uint64_t row_count = 8u; constexpr uint64_t slice_rows = Q8_0_X4_ROWS; @@ -4085,15 +4389,18 @@ TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_su constexpr uint64_t col_count = QK8_0 * 8u; constexpr uint64_t block_count = col_count / QK8_0; - std::vector native_rows(static_cast(row_count * block_count)); + std::vector native_rows( + static_cast(row_count * block_count)); for (size_t row = 0; row < static_cast(row_count); ++row) { for (size_t block = 0; block < static_cast(block_count); ++block) { - auto & cell = native_rows[row * static_cast(block_count) + block]; + auto &cell = native_rows[row * static_cast(block_count) + block]; cell.d = emel::kernel::detail::quant::fp32_to_fp16( - 0.0078125f * static_cast(((row + 3u) * (block + 5u)) % 23u + 1u)); + 0.0078125f * + static_cast(((row + 3u) * (block + 5u)) % 23u + 1u)); for (size_t idx = 0; idx < cell.qs.size(); ++idx) { const int32_t centered = - static_cast(((row + 7u) * 17u + (block + 11u) * 13u + idx * 5u) % 255u) - + static_cast( + ((row + 7u) * 17u + (block + 11u) * 13u + idx * 5u) % 255u) - 127; cell.qs[idx] = static_cast(std::clamp(centered, -127, 127)); } @@ -4113,15 +4420,15 @@ TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_su std::vector rhs_q8(static_cast(rhs_rows * block_count)); for (size_t row = 0; row < static_cast(rhs_rows); ++row) { emel::kernel::detail::quant::quantize_row_q8_0_strided( - rhs_dense.data() + row * static_cast(col_count), - 1u, + rhs_dense.data() + row * static_cast(col_count), 1u, rhs_q8.data() + row * static_cast(block_count), static_cast(col_count)); } std::vector reference(static_cast(rhs_rows * row_count), 0.0f); for (size_t rhs_row = 0; rhs_row < static_cast(rhs_rows); ++rhs_row) { - for (size_t lhs_row = 0; lhs_row < static_cast(row_count); ++lhs_row) { + for (size_t lhs_row = 0; lhs_row < static_cast(row_count); + ++lhs_row) { reference[rhs_row * static_cast(row_count) + lhs_row] = emel::kernel::detail::dot_q8_0_q8_0_row_scalar( native_rows.data() + lhs_row * static_cast(block_count), @@ -4132,7 +4439,8 @@ TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_su std::vector packed_bl8( sizeof(emel::kernel::detail::quant::block_q8_0x4) * - emel::kernel::detail::quant::packed_q8_0_x4_group_count(row_count) * block_count); + emel::kernel::detail::quant::packed_q8_0_x4_group_count(row_count) * + block_count); std::vector rhs_packed_bl8( sizeof(emel::kernel::detail::quant::block_q8_0x4) * block_count); REQUIRE(emel::kernel::detail::quant::pack_q8_0_rows_x4_bl8( @@ -4140,20 +4448,25 @@ TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_su REQUIRE(emel::kernel::detail::quant::pack_q8_0_rows_x4_bl8( rhs_q8.data(), rhs_rows, col_count, rhs_packed_bl8.data())); - std::vector packed_out(static_cast(rhs_rows * row_count), 0.0f); + std::vector packed_out(static_cast(rhs_rows * row_count), + 0.0f); aarch64_sm machine{}; const emel::kernel::event::op_mul_mat full_ev{ - .src0 = make_packed_q8_0_x4_bl8_src(packed_bl8.data(), col_count, row_count), - .src1 = make_packed_q8_0_x4_bl8_rhs_src(rhs_packed_bl8.data(), rhs_rows, col_count), - .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, row_count), + .src0 = + make_packed_q8_0_x4_bl8_src(packed_bl8.data(), col_count, row_count), + .src1 = make_packed_q8_0_x4_bl8_rhs_src(rhs_packed_bl8.data(), rhs_rows, + col_count), + .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, + row_count), }; const auto lane0 = make_row_sliced_lane_event(full_ev, Q8_0_X4_ROWS, 0u, slice_rows); - const auto lane1 = - make_row_sliced_lane_event(full_ev, Q8_0_X4_ROWS, slice_rows, row_count - slice_rows); + const auto lane1 = make_row_sliced_lane_event( + full_ev, Q8_0_X4_ROWS, slice_rows, row_count - slice_rows); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_MATMUL_INT8) CHECK(machine.process_event(lane0)); CHECK(machine.process_event(lane1)); for (size_t idx = 0; idx < packed_out.size(); ++idx) { @@ -4166,12 +4479,13 @@ TEST_CASE("kernel_aarch64_q8_0_packed_bl8_matrix_x4_row_sliced_lane_views_are_su #endif } -TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_supported") { +TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_" + "supported") { + using emel::kernel::detail::quant::block_q4_k; + using emel::kernel::detail::quant::block_q8_k; using emel::kernel::detail::quant::Q4_K_X8_ROWS; using emel::kernel::detail::quant::Q8_0_X4_ROWS; using emel::kernel::detail::quant::QK_K; - using emel::kernel::detail::quant::block_q4_k; - using emel::kernel::detail::quant::block_q8_k; constexpr uint64_t row_count = 16u; constexpr uint64_t slice_rows = Q4_K_X8_ROWS; @@ -4179,13 +4493,15 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_su constexpr uint64_t col_count = QK_K; constexpr uint64_t block_count = col_count / QK_K; - std::vector native_rows(static_cast(row_count * block_count)); + std::vector native_rows( + static_cast(row_count * block_count)); for (size_t row = 0; row < static_cast(row_count); ++row) { - auto & q4 = native_rows[row]; + auto &q4 = native_rows[row]; q4.d = 0x3c00u; q4.dmin = static_cast(0x3800u + row * 0x0040u); for (size_t idx = 0; idx < q4.scales.size(); ++idx) { - q4.scales[idx] = static_cast((idx * (11u + row)) ^ (0x5au + row)); + q4.scales[idx] = + static_cast((idx * (11u + row)) ^ (0x5au + row)); } for (size_t idx = 0; idx < q4.qs.size(); ++idx) { q4.qs[idx] = static_cast((idx * (9u + row)) ^ (0x96u - row)); @@ -4205,15 +4521,15 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_su std::vector rhs_q8(static_cast(rhs_rows * block_count)); for (size_t row = 0; row < static_cast(rhs_rows); ++row) { emel::kernel::detail::quant::quantize_row_q8_k_strided( - rhs_dense.data() + row * static_cast(col_count), - 1u, + rhs_dense.data() + row * static_cast(col_count), 1u, rhs_q8.data() + row * static_cast(block_count), static_cast(col_count)); } std::vector reference(static_cast(rhs_rows * row_count), 0.0f); for (size_t rhs_row = 0; rhs_row < static_cast(rhs_rows); ++rhs_row) { - for (size_t lhs_row = 0; lhs_row < static_cast(row_count); ++lhs_row) { + for (size_t lhs_row = 0; lhs_row < static_cast(row_count); + ++lhs_row) { reference[rhs_row * static_cast(row_count) + lhs_row] = emel::kernel::detail::dot_q4_k_q8_k_row_scalar( native_rows.data() + lhs_row * static_cast(block_count), @@ -4224,24 +4540,29 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_su std::vector packed_bl8( sizeof(emel::kernel::detail::quant::block_q4_kx8) * - emel::kernel::detail::quant::packed_q4_k_x8_group_count(row_count) * block_count); + emel::kernel::detail::quant::packed_q4_k_x8_group_count(row_count) * + block_count); REQUIRE(emel::kernel::detail::quant::pack_q4_k_rows_x8_bl8( native_rows.data(), row_count, col_count, packed_bl8.data())); - std::vector packed_out(static_cast(rhs_rows * row_count), 0.0f); + std::vector packed_out(static_cast(rhs_rows * row_count), + 0.0f); aarch64_sm machine{}; const emel::kernel::event::op_mul_mat full_ev{ - .src0 = make_packed_q4_k_x8_bl8_src(packed_bl8.data(), col_count, row_count), + .src0 = + make_packed_q4_k_x8_bl8_src(packed_bl8.data(), col_count, row_count), .src1 = make_q8_k_x4_rhs_src(rhs_q8.data(), rhs_rows, col_count), - .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, row_count), + .dst = make_batch_major_dst(packed_out.data(), dtype::f32, rhs_rows, + row_count), }; const auto lane0 = make_row_sliced_lane_event(full_ev, Q4_K_X8_ROWS, 0u, slice_rows); - const auto lane1 = - make_row_sliced_lane_event(full_ev, Q4_K_X8_ROWS, slice_rows, row_count - slice_rows); + const auto lane1 = make_row_sliced_lane_event( + full_ev, Q4_K_X8_ROWS, slice_rows, row_count - slice_rows); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD) CHECK(machine.process_event(lane0)); CHECK(machine.process_event(lane1)); for (size_t idx = 0; idx < packed_out.size(); ++idx) { @@ -4254,13 +4575,13 @@ TEST_CASE("kernel_aarch64_q4_k_packed_bl8_matrix_x4_row_sliced_lane_views_are_su #endif } -TEST_CASE( - "kernel_aarch64_q6_k_prepared_q8_rhs_i8mm_matrix_x4_row_sliced_lane_views_are_supported") { +TEST_CASE("kernel_aarch64_q6_k_prepared_q8_rhs_i8mm_matrix_x4_row_sliced_lane_" + "views_are_supported") { + using emel::kernel::detail::quant::block_q6_k; + using emel::kernel::detail::quant::block_q8_k; using emel::kernel::detail::quant::Q6_K_X8_ROWS; using emel::kernel::detail::quant::Q8_0_X4_ROWS; using emel::kernel::detail::quant::QK_K; - using emel::kernel::detail::quant::block_q6_k; - using emel::kernel::detail::quant::block_q8_k; constexpr uint64_t row_count = 16u; constexpr uint64_t slice_rows = Q6_K_X8_ROWS; @@ -4268,12 +4589,14 @@ TEST_CASE( constexpr uint64_t col_count = QK_K; constexpr uint64_t block_count = col_count / QK_K; - std::vector native_rows(static_cast(row_count * block_count)); + std::vector native_rows( + static_cast(row_count * block_count)); for (size_t row = 0; row < static_cast(row_count); ++row) { - auto & q6 = native_rows[row]; + auto &q6 = native_rows[row]; q6.d = 0x3c00u; for (size_t idx = 0; idx < q6.scales.size(); ++idx) { - const int32_t scale_value = static_cast((idx + row * 5u) % 15u) - 7; + const int32_t scale_value = + static_cast((idx + row * 5u) % 15u) - 7; q6.scales[idx] = static_cast(scale_value); } for (size_t idx = 0; idx < q6.ql.size(); ++idx) { @@ -4297,15 +4620,15 @@ TEST_CASE( std::vector rhs_q8(static_cast(rhs_rows * block_count)); for (size_t row = 0; row < static_cast(rhs_rows); ++row) { emel::kernel::detail::quant::quantize_row_q8_k_strided( - rhs_dense.data() + row * static_cast(col_count), - 1u, + rhs_dense.data() + row * static_cast(col_count), 1u, rhs_q8.data() + row * static_cast(block_count), static_cast(col_count)); } std::vector reference(static_cast(rhs_rows * row_count), 0.0f); for (size_t rhs_row = 0; rhs_row < static_cast(rhs_rows); ++rhs_row) { - for (size_t lhs_row = 0; lhs_row < static_cast(row_count); ++lhs_row) { + for (size_t lhs_row = 0; lhs_row < static_cast(row_count); + ++lhs_row) { reference[rhs_row * static_cast(row_count) + lhs_row] = emel::kernel::detail::dot_q6_k_q8_k_row_scalar( native_rows.data() + lhs_row * static_cast(block_count), @@ -4315,31 +4638,38 @@ TEST_CASE( } std::vector prepared_storage( - emel::kernel::detail::quant::prepared_q6_k_x8_q8_group_storage_bytes(col_count) * + emel::kernel::detail::quant::prepared_q6_k_x8_q8_group_storage_bytes( + col_count) * emel::kernel::detail::quant::packed_q6_k_x8_group_count(row_count)); REQUIRE(emel::kernel::detail::quant::pack_q6_k_rows_x8_q8_prepared( native_rows.data(), row_count, col_count, prepared_storage.data())); - std::vector prepared_out(static_cast(rhs_rows * row_count), 0.0f); + std::vector prepared_out(static_cast(rhs_rows * row_count), + 0.0f); aarch64_sm machine{}; const emel::kernel::event::op_mul_mat full_ev{ - .src0 = make_prepared_q6_k_x8_q8_src(prepared_storage.data(), col_count, row_count), + .src0 = make_prepared_q6_k_x8_q8_src(prepared_storage.data(), col_count, + row_count), .src1 = make_q8_k_x4_rhs_src(rhs_q8.data(), rhs_rows, col_count), - .dst = make_batch_major_dst(prepared_out.data(), dtype::f32, rhs_rows, row_count), + .dst = make_batch_major_dst(prepared_out.data(), dtype::f32, rhs_rows, + row_count), }; const auto lane0 = make_row_sliced_lane_event(full_ev, Q6_K_X8_ROWS, 0u, slice_rows); - const auto lane1 = - make_row_sliced_lane_event(full_ev, Q6_K_X8_ROWS, slice_rows, row_count - slice_rows); + const auto lane1 = make_row_sliced_lane_event( + full_ev, Q6_K_X8_ROWS, slice_rows, row_count - slice_rows); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_MATMUL_INT8) CHECK(machine.process_event(lane0)); CHECK(machine.process_event(lane1)); for (size_t idx = 0; idx < prepared_out.size(); ++idx) { - CHECK(prepared_out[idx] == doctest::Approx(reference[idx]).epsilon(1.0e-5f)); + CHECK(prepared_out[idx] == + doctest::Approx(reference[idx]).epsilon(1.0e-5f)); } - CHECK(machine.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count() == 2u); + CHECK(machine.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count() == + 2u); #else CHECK_FALSE(machine.process_event(lane0)); CHECK_FALSE(machine.process_event(lane1)); @@ -4610,8 +4940,8 @@ TEST_CASE("kernel_aarch64_detail_branch_paths") { using emel::kernel::detail::quant::block_q4_k; using emel::kernel::detail::quant::block_q6_k; - using emel::kernel::detail::quant::block_q8_k; using emel::kernel::detail::quant::block_q8_0; + using emel::kernel::detail::quant::block_q8_k; using emel::kernel::detail::quant::QK8_0; std::array q4_k_rows = {}; std::array q6_k_rows = {}; @@ -4772,25 +5102,28 @@ TEST_CASE("kernel_aarch64_detail_branch_paths") { .src1 = make_src(lhs, dtype::f32, 1u, 4u), .dst = make_dst(vector_out.data(), dtype::f32, 1u, 1u), }; - CHECK(emel::kernel::aarch64::detail:: - can_run_neon_mul_mat_f32_vector_request(f32_ev)); + CHECK(emel::kernel::aarch64::detail::can_run_neon_mul_mat_f32_vector_request( + f32_ev)); { auto bad = f32_ev; bad.src1.ne[0] = 2u; - CHECK_FALSE(emel::kernel::aarch64::detail:: - can_run_neon_mul_mat_f32_vector_request(bad)); + CHECK_FALSE( + emel::kernel::aarch64::detail::can_run_neon_mul_mat_f32_vector_request( + bad)); } { auto bad = f32_ev; bad.src0.nb[1] = bad.src0.nb[1] + sizeof(float); - CHECK_FALSE(emel::kernel::aarch64::detail:: - can_run_neon_mul_mat_f32_vector_request(bad)); + CHECK_FALSE( + emel::kernel::aarch64::detail::can_run_neon_mul_mat_f32_vector_request( + bad)); } const auto reject_f32 = [&](const auto mutate) { auto bad = f32_ev; mutate(bad); - CHECK_FALSE(emel::kernel::aarch64::detail:: - can_run_neon_mul_mat_f32_vector_request(bad)); + CHECK_FALSE( + emel::kernel::aarch64::detail::can_run_neon_mul_mat_f32_vector_request( + bad)); }; reject_f32([](auto &bad) { bad.src0.ne[0] = 0u; }); reject_f32([](auto &bad) { bad.src0.ne[1] = 0u; }); @@ -4807,8 +5140,10 @@ TEST_CASE("kernel_aarch64_detail_branch_paths") { reject_f32([](auto &bad) { bad.src1.type = dtype::q4_k; }); reject_f32([](auto &bad) { bad.dst.type = dtype::q4_k; }); reject_f32([](auto &bad) { bad.src0.nb[0] = sizeof(float) * 2u; }); - reject_f32([](auto &bad) { bad.src0.nb[2] = bad.src0.nb[2] + sizeof(float); }); - reject_f32([](auto &bad) { bad.src0.nb[3] = bad.src0.nb[3] + sizeof(float); }); + reject_f32( + [](auto &bad) { bad.src0.nb[2] = bad.src0.nb[2] + sizeof(float); }); + reject_f32( + [](auto &bad) { bad.src0.nb[3] = bad.src0.nb[3] + sizeof(float); }); emel::kernel::event::op_unary unary_ev{ .src0 = make_src(lhs, dtype::f32, 4), @@ -4816,9 +5151,11 @@ TEST_CASE("kernel_aarch64_detail_branch_paths") { .subop = emel::kernel::event::unary_subop::relu, }; #if defined(__aarch64__) || defined(__ARM_NEON) - CHECK(emel::kernel::aarch64::detail::can_use_neon(unary_ev, true)); + CHECK(emel::kernel::aarch64::guard::detail::can_use_neon_unary_f32(unary_ev, + true)); #else - CHECK_FALSE(emel::kernel::aarch64::detail::can_use_neon(unary_ev, true)); + CHECK_FALSE(emel::kernel::aarch64::guard::detail::can_use_neon_unary_f32( + unary_ev, true)); #endif unary_ev.subop = emel::kernel::event::unary_subop::exp; CHECK_FALSE(emel::kernel::aarch64::detail::can_use_neon(unary_ev, true)); @@ -4865,8 +5202,8 @@ TEST_CASE("kernel_aarch64_q4_k_vector_q8_rhs_tail_rows_match_scalar") { std::array out = {}; const emel::kernel::event::op_mul_mat ev{ - .src0 = make_quantized_src(native_rows.data(), dtype::q4_k, QK_K, - row_count), + .src0 = + make_quantized_src(native_rows.data(), dtype::q4_k, QK_K, row_count), .src1 = make_q8_k_vector_src(rhs_q8.data(), QK_K), .dst = make_dst(out.data(), dtype::f32, 1u, row_count), }; @@ -4958,7 +5295,8 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { CHECK(emel::kernel::aarch64::detail::execute_neon_sqr(sqr_ev)); CHECK(emel::kernel::aarch64::detail::execute_neon_sqrt(sqrt_ev)); CHECK(emel::kernel::aarch64::detail::execute_neon_mul_mat(mul_mat_ev)); - CHECK(emel::kernel::aarch64::detail::execute_neon_unary(unary_ev)); + emel::kernel::aarch64::detail::execute_simd_unary_subop_unchecked< + emel::kernel::event::unary_subop::relu>(unary_ev); #else CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_dup(dup_ev)); CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_add(add_ev)); @@ -4968,7 +5306,6 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_sqr(sqr_ev)); CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_sqrt(sqrt_ev)); CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_mul_mat(mul_mat_ev)); - CHECK_FALSE(emel::kernel::aarch64::detail::execute_neon_unary(unary_ev)); #endif const bool simd_dup = emel::kernel::aarch64::detail::execute_simd(dup_ev); @@ -4980,7 +5317,6 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { const bool simd_sqrt = emel::kernel::aarch64::detail::execute_simd(sqrt_ev); const bool simd_mul_mat = emel::kernel::aarch64::detail::execute_simd(mul_mat_ev); - const bool simd_unary = emel::kernel::aarch64::detail::execute_simd(unary_ev); (void)simd_dup; (void)simd_add; (void)simd_sub; @@ -4989,7 +5325,6 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { (void)simd_sqr; (void)simd_sqrt; (void)simd_mul_mat; - (void)simd_unary; #if defined(__aarch64__) || defined(__ARM_NEON) CHECK(simd_dup); @@ -5000,7 +5335,6 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { CHECK(simd_sqr); CHECK(simd_sqrt); CHECK(simd_mul_mat); - CHECK(simd_unary); #else CHECK_FALSE(simd_dup); CHECK_FALSE(simd_add); @@ -5010,7 +5344,6 @@ TEST_CASE("kernel_aarch64_detail_helper_edge_paths") { CHECK_FALSE(simd_sqr); CHECK_FALSE(simd_sqrt); CHECK_FALSE(simd_mul_mat); - CHECK_FALSE(simd_unary); #endif unary_ev.subop = emel::kernel::event::unary_subop::exp; @@ -5089,6 +5422,54 @@ TEST_CASE("kernel_aarch64_unary_subop_scalar_paths") { CHECK_FALSE(scalar_machine.process_event(unary_ev)); } +TEST_CASE("kernel_aarch64_neon_silu_matches_moshi_reference_activation") { +#if !(defined(__aarch64__) || defined(__ARM_NEON)) + return; +#else + const float src[5] = { + std::bit_cast(0x3a9dd9b3u), + std::bit_cast(0xbef68af7u), + std::bit_cast(0xbdc5f39au), + std::bit_cast(0xbf218bc1u), + 0.5f, + }; + float dst[5] = {}; + const emel::kernel::event::op_unary unary_ev{ + .src0 = make_src(src, dtype::f32, 5), + .dst = make_dst(dst, dtype::f32, 5), + .subop = emel::kernel::event::unary_subop::silu, + }; + aarch64_sm machine{emel::kernel::aarch64::action::context{true, {}, 0}}; + + REQUIRE(machine.process_event(unary_ev)); + CHECK(std::bit_cast(dst[0]) == 0x3a1df208u); + CHECK(std::bit_cast(dst[1]) == 0xbe3c4e19u); + CHECK(std::bit_cast(dst[2]) == 0xbd3c6475u); + CHECK(std::bit_cast(dst[3]) == 0xbe606749u); + CHECK(dst[4] == doctest::Approx(0.5f / (1.0f + std::exp(-0.5f)))); +#endif +} + +TEST_CASE("kernel_aarch64_simd_guards_reject_invalid_unary_and_argmax") { + const emel::kernel::event::op_unary invalid_unary{}; + const emel::kernel::event::op_mul_mat_argmax invalid_argmax{}; + emel::kernel::aarch64::event::dispatch_ctx dispatch_context{}; + const emel::kernel::aarch64::event::dispatch_op_unary unary_dispatch{ + invalid_unary, dispatch_context}; + const emel::kernel::aarch64::event::dispatch_op_mul_mat_argmax + argmax_dispatch{invalid_argmax, dispatch_context}; + const emel::kernel::aarch64::action::context guard_context{true, {}, 0}; + + CHECK_FALSE(emel::kernel::aarch64::guard::simd_op_unary_f32{}(unary_dispatch, + guard_context)); + CHECK_FALSE(emel::kernel::aarch64::guard:: + simd_op_mul_mat_argmax_q6_vector_packed_q8_rhs{}( + argmax_dispatch, guard_context)); + CHECK_FALSE(emel::kernel::aarch64::guard:: + simd_op_mul_mat_argmax_q4_vector_packed_f32_rhs_bl4{}( + argmax_dispatch, guard_context)); +} + TEST_CASE("kernel_aarch64_flash_attn_ext_reuses_persistent_workspace") { #if !(defined(__aarch64__) || defined(__ARM_NEON)) return; @@ -5164,7 +5545,6 @@ TEST_CASE("kernel_aarch64_flash_attn_ext_uses_optimized_backend_path") { #endif } - TEST_CASE( "kernel_aarch64_flash_attn_ext_matches_shared_workspace_on_long_kv_spans") { #if !(defined(__aarch64__) || defined(__ARM_NEON)) diff --git a/tests/kernel/lifecycle_tests.cpp b/tests/kernel/lifecycle_tests.cpp index f36a706a..05517641 100644 --- a/tests/kernel/lifecycle_tests.cpp +++ b/tests/kernel/lifecycle_tests.cpp @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -42,6 +44,8 @@ using emel::kernel::test::make_quantized_src; using emel::kernel::test::make_q8_k_vector_src; using emel::kernel::test::make_smoke_op_event; using emel::kernel::test::make_src; +using emel::kernel::test::set_op_param_f32; +using emel::kernel::test::set_op_param_i32; using emel::kernel::test::to_fp16_storage; using emel::kernel::test::within_flash_online_f16_tolerance; @@ -80,6 +84,40 @@ void check_backend_op_paths(machine_type & machine, } // namespace +TEST_CASE("kernel timestep rope preserves ggml f32 round points") { + const auto check_machine = [](auto & machine) { + std::array src = {}; + std::array dst = {}; + src[88] = std::bit_cast(UINT32_C(0x3f91261c)); + src[89] = std::bit_cast(UINT32_C(0x3e9d87b5)); + int32_t position = 9; + + emel::kernel::event::op_rope rope_ev{ + .src0 = make_src(src.data(), dtype::f32, 128, 1, 1), + .src1 = make_src(&position, dtype::i32, 1), + .dst = make_dst(dst.data(), dtype::f32, 128, 1, 1), + }; + set_op_param_i32(rope_ev, 1u, 128); + set_op_param_i32(rope_ev, 2u, + emel::kernel::detail::rope_mode_timestep); + set_op_param_f32(rope_ev, 5u, 10000.0f); + set_op_param_f32(rope_ev, 6u, 1.0f); + set_op_param_f32(rope_ev, 7u, 0.0f); + set_op_param_f32(rope_ev, 8u, 1.0f); + + REQUIRE(machine.process_event(rope_ev)); + CHECK(std::bit_cast(dst[44]) == UINT32_C(0x3f908001)); + CHECK(std::bit_cast(dst[108]) == UINT32_C(0x3ea6cd3b)); + }; + + x86_64_sm x86_machine{ + emel::kernel::x86_64::action::context{false, {}, 0}}; + aarch64_sm arm_machine{ + emel::kernel::aarch64::action::context{false, {}, 0}}; + check_machine(x86_machine); + check_machine(arm_machine); +} + TEST_CASE("kernel_mul_mat_accepts_quantized_qk_weights") { using emel::kernel::detail::quant::QK_K; using emel::kernel::detail::quant::block_q2_k; @@ -415,6 +453,8 @@ TEST_CASE("kernel_backends_accept_dispatch_event") { kernel_sm kernel_machine{}; emel::kernel::any any_machine{}; + CHECK(kernel_machine.kind() == emel::kernel::detect_host_kind()); + CHECK(any_machine.kind() == emel::kernel::detect_host_kind()); CHECK(x86_64_machine.process_event(event)); CHECK(aarch64_machine.process_event(event)); CHECK(kernel_machine.process_event(event)); @@ -705,6 +745,80 @@ TEST_CASE("kernel_detail_stride_paths_and_scalar_helpers") { CHECK_FALSE(emel::kernel::detail::execute_scalar(unsupported_ev)); } +TEST_CASE("kernel_soft_max_row_matches_ggml_arm64_neon_bits") { +#if defined(__aarch64__) && defined(__ARM_NEON) + std::array values = { + 0.25f, -0.5f, 1.25f, -3.0f, 2.5f, 0.0f, -1.5f, 0.75f, + 3.25f, -2.25f, 1.0f, 0.125f, -0.875f, 2.0f, -4.0f, 0.5f, + }; + constexpr std::array expected = { + 0x3caece97u, 0x3c25255eu, 0x3d6d965au, 0x3a58e566u, + 0x3e4f50b8u, 0x3c8823cbu, 0x3b7303e3u, 0x3d101aa1u, + 0x3edb717au, 0x3ae595a1u, 0x3d390880u, 0x3c9a4441u, + 0x3be3018fu, 0x3dfb7c76u, 0x399f954fu, 0x3ce074e5u, + }; + + emel::kernel::detail::soft_max_row_ggml( + static_cast(values.size()), values.data()); + + for (size_t index = 0; index < values.size(); ++index) { + CHECK(std::bit_cast(values[index]) == expected[index]); + } +#endif +} + +TEST_CASE("kernel_rms_norm_matches_ggml_float_square_order") { + std::array src = {}; + constexpr float eps = 1.0e-8f; + float scale = 0.0f; + uint32_t fixture_seed = 0u; + for (uint32_t seed = 1u; seed <= 4096u; ++seed) { + uint32_t state = seed; + for (size_t index = 0; index < src.size(); ++index) { + state = state * 1664525u + 1013904223u; + const int32_t centered = static_cast(state & 0xffffu) - 32768; + const int exponent = static_cast((state >> 16u) % 18u) - 12; + src[index] = + std::ldexp(static_cast(centered) / 32768.0f, exponent); + } + + double ggml_sum = 0.0; + double widened_sum = 0.0; + for (const float value : src) { + ggml_sum += static_cast(value * value); + widened_sum += static_cast(value) * static_cast(value); + } + const float mean = static_cast(ggml_sum / src.size()); + const float widened_mean = static_cast(widened_sum / src.size()); + scale = 1.0f / std::sqrt(mean + eps); + const float widened_scale = 1.0f / std::sqrt(widened_mean + eps); + if (std::bit_cast(scale) != + std::bit_cast(widened_scale)) { + fixture_seed = seed; + break; + } + } + REQUIRE(fixture_seed != 0u); + std::array expected = {}; + for (size_t index = 0; index < src.size(); ++index) { + expected[index] = src[index] * scale; + } + + std::array actual = {}; + emel::kernel::event::op_rms_norm rms_ev{ + .src0 = make_src(src.data(), dtype::f32, src.size()), + .dst = make_dst(actual.data(), dtype::f32, actual.size()), + }; + std::memcpy(rms_ev.op_params.data(), &eps, sizeof(eps)); + rms_ev.op_params_size = sizeof(eps); + REQUIRE(emel::kernel::detail::run_rms_norm(rms_ev)); + + for (size_t index = 0; index < actual.size(); ++index) { + CHECK(std::bit_cast(actual[index]) == + std::bit_cast(expected[index])); + } +} + TEST_CASE("kernel_backends_reject_quantized_dispatch_dtypes") { float src0[4] = {1.0f, 2.0f, 3.0f, 4.0f}; float src1[4] = {4.0f, 3.0f, 2.0f, 1.0f}; diff --git a/tests/kernel/test_helpers.hpp b/tests/kernel/test_helpers.hpp index 5f030e1c..d2d6a6e9 100644 --- a/tests/kernel/test_helpers.hpp +++ b/tests/kernel/test_helpers.hpp @@ -26,6 +26,24 @@ inline bool within_flash_online_f16_tolerance(const float actual, return std::fabs(actual - expected) <= k_flash_online_f16_abs_tolerance; } +template +inline void set_op_param_i32(event_type &ev, const uint32_t slot, + const int32_t value) { + std::memcpy(ev.op_params.data() + slot * sizeof(int32_t), &value, + sizeof(value)); + ev.op_params_size = + std::max(ev.op_params_size, (slot + 1u) * sizeof(int32_t)); +} + +template +inline void set_op_param_f32(event_type &ev, const uint32_t slot, + const float value) { + std::memcpy(ev.op_params.data() + slot * sizeof(float), &value, + sizeof(value)); + ev.op_params_size = + std::max(ev.op_params_size, (slot + 1u) * sizeof(float)); +} + template inline void fill_default_nb(tensor_type & tensor) { const auto elem_size = diff --git a/tests/kernel/x86_64_tests.cpp b/tests/kernel/x86_64_tests.cpp index adf28609..20d34ccb 100644 --- a/tests/kernel/x86_64_tests.cpp +++ b/tests/kernel/x86_64_tests.cpp @@ -27,6 +27,8 @@ using emel::kernel::test::make_dst; using emel::kernel::test::make_flash_attn_ext_event; using emel::kernel::test::make_quantized_src; using emel::kernel::test::make_src; +using emel::kernel::test::set_op_param_f32; +using emel::kernel::test::set_op_param_i32; using emel::kernel::test::to_fp16_storage; using emel::kernel::test::within_flash_online_f16_tolerance; @@ -1852,27 +1854,6 @@ TEST_CASE("kernel_x86_64_simd_action_exec_marks_done") { emel::kernel::x86_64::error::none))); } -namespace { - -template -void set_op_param_i32(event_type &ev, const uint32_t slot, - const int32_t value) { - std::memcpy(ev.op_params.data() + slot * sizeof(int32_t), &value, - sizeof(value)); - ev.op_params_size = - std::max(ev.op_params_size, (slot + 1u) * sizeof(int32_t)); -} - -template -void set_op_param_f32(event_type &ev, const uint32_t slot, const float value) { - std::memcpy(ev.op_params.data() + slot * sizeof(float), &value, - sizeof(value)); - ev.op_params_size = - std::max(ev.op_params_size, (slot + 1u) * sizeof(float)); -} - -} // namespace - TEST_CASE("kernel_x86_64_get_rows_gathers_f32_and_q8_0_rows") { x86_64_sm machine{emel::kernel::x86_64::action::context{false, {}, 0}}; diff --git a/tests/logits/sampler/pipeline_tests.cpp b/tests/logits/sampler/pipeline_tests.cpp index 1a941bf5..8bd0fd6f 100644 --- a/tests/logits/sampler/pipeline_tests.cpp +++ b/tests/logits/sampler/pipeline_tests.cpp @@ -1,5 +1,8 @@ #include "doctest/doctest.h" +#include +#include + #include "emel/emel.h" #include "emel/error/error.hpp" #include "emel/gbnf/detail.hpp" @@ -8,10 +11,8 @@ namespace { -emel::error::type sampler_shift_scores(int32_t &, - float & candidate_scores, - int32_t & candidate_count, - int32_t &) { +emel::error::type sampler_shift_scores(int32_t &, float &candidate_scores, + int32_t &candidate_count, int32_t &) { for (int32_t i = 0; i < candidate_count; ++i) { (&candidate_scores)[i] = -1.0f; } @@ -19,10 +20,10 @@ emel::error::type sampler_shift_scores(int32_t &, return emel::error::cast(emel::logits::sampler::error::none); } -emel::error::type sampler_select_argmax(int32_t & candidate_ids, - float & candidate_scores, - int32_t & candidate_count, - int32_t & selected_token_out) { +emel::error::type sampler_select_argmax(int32_t &candidate_ids, + float &candidate_scores, + int32_t &candidate_count, + int32_t &selected_token_out) { int32_t best_idx = 0; float best_score = (&candidate_scores)[0]; for (int32_t i = 1; i < candidate_count; ++i) { @@ -35,46 +36,38 @@ emel::error::type sampler_select_argmax(int32_t & candidate_ids, return emel::error::cast(emel::logits::sampler::error::none); } -emel::error::type sampler_select_fixed(int32_t &, - float &, - int32_t &, - int32_t & selected_token_out) { +emel::error::type sampler_select_fixed(int32_t &, float &, int32_t &, + int32_t &selected_token_out) { selected_token_out = 7; return emel::error::cast(emel::logits::sampler::error::none); } -emel::error::type sampler_select_first(int32_t & candidate_ids, - float &, - int32_t &, - int32_t & selected_token_out) { +emel::error::type sampler_select_first(int32_t &candidate_ids, float &, + int32_t &, int32_t &selected_token_out) { selected_token_out = (&candidate_ids)[0]; return emel::error::cast(emel::logits::sampler::error::none); } -emel::error::type sampler_error(int32_t &, - float &, - int32_t &, - int32_t &) { +emel::error::type sampler_error(int32_t &, float &, int32_t &, int32_t &) { return emel::error::cast(emel::logits::sampler::error::backend_error); } -emel::error::type sampler_set_invalid_candidate_count(int32_t &, - float &, - int32_t & candidate_count, +emel::error::type sampler_set_invalid_candidate_count(int32_t &, float &, + int32_t &candidate_count, int32_t &) { candidate_count = 0; return emel::error::cast(emel::logits::sampler::error::none); } -bool configure_sampler_chain(emel::logits::sampler::sm & machine, - emel::logits::sampler::fn & sampler_fns, - int32_t sampler_count, - emel::error::type & err) { - emel::logits::sampler::event::configure request{sampler_fns, sampler_count, err}; +bool configure_sampler_chain(emel::logits::sampler::sm &machine, + emel::logits::sampler::fn &sampler_fns, + int32_t sampler_count, emel::error::type &err) { + emel::logits::sampler::event::configure request{sampler_fns, sampler_count, + err}; return machine.process_event(request); } -} // namespace +} // namespace TEST_CASE("sampler pipeline selects token via selector sampler") { float logits[3] = {0.0f, 0.1f, 0.2f}; @@ -98,7 +91,8 @@ TEST_CASE("sampler pipeline selects token via selector sampler") { CHECK(selected == 1); } -TEST_CASE("sampler pipeline reports invalid request when no samplers are configured") { +TEST_CASE("sampler pipeline reports invalid request when no samplers are " + "configured") { emel::logits::sampler::sm machine{}; float logits[2] = {1.0f, 2.0f}; @@ -111,17 +105,20 @@ TEST_CASE("sampler pipeline reports invalid request when no samplers are configu logits[0], 2, ids[0], scores[0], 2, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == -1); } TEST_CASE("sampler configure rejects invalid sampler count") { emel::logits::sampler::sm machine{}; - emel::logits::sampler::fn sampler = emel::logits::sampler::fn::from(); + emel::logits::sampler::fn sampler = + emel::logits::sampler::fn::from(); emel::error::type err = emel::error::cast(emel::logits::sampler::error::none); CHECK_FALSE(configure_sampler_chain(machine, sampler, 0, err)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); } TEST_CASE("sampler configure can replace the sampler chain") { @@ -156,11 +153,13 @@ TEST_CASE("sampler configure can replace the sampler chain") { CHECK(selected == 0); } -TEST_CASE("sampler pipeline accepts valid preselected token without sampler chain") { +TEST_CASE( + "sampler pipeline accepts valid preselected token without sampler chain") { emel::logits::sampler::sm machine{}; int32_t selected = 2; - emel::error::type err = emel::error::cast(emel::logits::sampler::error::backend_error); + emel::error::type err = + emel::error::cast(emel::logits::sampler::error::backend_error); emel::logits::sampler::event::sample_preselected request{3, selected, err}; @@ -178,7 +177,8 @@ TEST_CASE("sampler pipeline rejects invalid preselected token") { emel::logits::sampler::event::sample_preselected request{3, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == 7); } @@ -195,17 +195,20 @@ TEST_CASE("sampler pipeline reports invalid arguments") { logits[0], 2, ids[0], scores[0], 1, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == -1); } TEST_CASE("sampler pipeline reports invalid context sampler table") { emel::logits::sampler::sm machine{}; - emel::logits::sampler::fn sampler = emel::logits::sampler::fn::from(); + emel::logits::sampler::fn sampler = + emel::logits::sampler::fn::from(); emel::error::type err = emel::error::cast(emel::logits::sampler::error::none); CHECK_FALSE(configure_sampler_chain(machine, sampler, 0, err)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); } TEST_CASE("sampler pipeline reports missing sampler function") { @@ -223,7 +226,8 @@ TEST_CASE("sampler pipeline reports missing sampler function") { logits[0], 1, ids[0], scores[0], 1, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == -1); } @@ -248,7 +252,8 @@ TEST_CASE("sampler pipeline reports sampler errors") { CHECK(selected == -1); } -TEST_CASE("sampler pipeline reports invalid request when no selector sampler sets a token") { +TEST_CASE("sampler pipeline reports invalid request when no selector sampler " + "sets a token") { float logits[2] = {0.0f, 0.1f}; int32_t ids[2] = {}; float scores[2] = {}; @@ -265,11 +270,13 @@ TEST_CASE("sampler pipeline reports invalid request when no selector sampler set logits[0], 2, ids[0], scores[0], 2, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == -1); } -TEST_CASE("sampler pipeline reports invalid request when sampler sets invalid candidate_count") { +TEST_CASE("sampler pipeline reports invalid request when sampler sets invalid " + "candidate_count") { float logits[3] = {0.0f, 0.1f, 0.2f}; int32_t ids[3] = {}; float scores[3] = {}; @@ -286,11 +293,13 @@ TEST_CASE("sampler pipeline reports invalid request when sampler sets invalid ca logits[0], 3, ids[0], scores[0], 3, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == -1); } -TEST_CASE("sampler pipeline reports invalid request when selector picks out-of-range token") { +TEST_CASE("sampler pipeline reports invalid request when selector picks " + "out-of-range token") { float logits[3] = {0.0f, 0.1f, 0.2f}; int32_t ids[3] = {}; float scores[3] = {}; @@ -307,7 +316,8 @@ TEST_CASE("sampler pipeline reports invalid request when selector picks out-of-r logits[0], 3, ids[0], scores[0], 3, selected, err}; CHECK(!machine.process_event(request)); - CHECK(err == emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); CHECK(selected == 7); } @@ -362,3 +372,97 @@ TEST_CASE("sampler pipeline propagates gbnf sampler errors") { CHECK(err == emel::error::cast(emel::gbnf::sampler::error::invalid_request)); CHECK(selected == -1); } + +TEST_CASE("sampler rejects invalid typed temperature top-k workspace") { + std::array logits = {1.0f, 2.0f}; + std::array sorted_indices = {}; + std::array top_probabilities = {}; + std::array top_indices = {}; + uint32_t random_state = 1234u; + int32_t selected = -1; + float score = 0.0f; + emel::error::type err = emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k request{ + logits, + 2, + 0.8f, + 1, + sorted_indices, + top_probabilities, + top_indices, + random_state, + selected, + score, + err}; + emel::logits::sampler::sm machine{}; + + CHECK_FALSE(machine.process_event(request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(selected == -1); +} + +TEST_CASE("sampler rejects Park-Miller zero-state seeds") { + std::array logits = {1.0f, 2.0f}; + std::array sorted_indices = {}; + std::array top_probabilities = {}; + std::array top_indices = {}; + uint32_t random_state = 2147483647u; + int32_t selected = -1; + float score = 0.0f; + emel::error::type err = emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k request{ + logits, + 2, + 0.8f, + 1, + sorted_indices, + top_probabilities, + top_indices, + random_state, + selected, + score, + err}; + emel::logits::sampler::sm machine{}; + + CHECK_FALSE(machine.process_event(request)); + CHECK(err == + emel::error::cast(emel::logits::sampler::error::invalid_request)); + CHECK(selected == -1); +} + +TEST_CASE("sampler rejects non-finite temperature and unselectable logits") { + const auto rejected = [](std::array logits, + const float temperature) { + std::array sorted_indices{}; + std::array top_probabilities{}; + std::array top_indices{}; + uint32_t random_state = 1234u; + int32_t selected = -1; + float score = 0.0f; + emel::error::type err = + emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k request{ + logits, + 2, + temperature, + 1, + sorted_indices, + top_probabilities, + top_indices, + random_state, + selected, + score, + err}; + emel::logits::sampler::sm machine{}; + return !machine.process_event(request) && + err == emel::error::cast( + emel::logits::sampler::error::invalid_request) && + selected == -1; + }; + CHECK(rejected({1.0f, 2.0f}, std::numeric_limits::infinity())); + CHECK(rejected({std::numeric_limits::quiet_NaN(), 2.0f}, 0.8f)); + CHECK(rejected({-std::numeric_limits::infinity(), + -std::numeric_limits::infinity()}, + 0.8f)); +} diff --git a/tests/memory/streaming/lifecycle_tests.cpp b/tests/memory/streaming/lifecycle_tests.cpp new file mode 100644 index 00000000..7b2f3f9f --- /dev/null +++ b/tests/memory/streaming/lifecycle_tests.cpp @@ -0,0 +1,232 @@ +#include +#include + +#include + +#include "../../allocation_tracker.hpp" +#include "emel/error/error.hpp" +#include "emel/memory/streaming/guards.hpp" +#include "emel/memory/streaming/sm.hpp" + +namespace { + +namespace streaming = emel::memory::streaming; +namespace sml = stateforward::sml; + +int32_t error_code(const streaming::error value) noexcept { + return static_cast(emel::error::cast(value)); +} + +struct fixture { + explicit fixture(const int32_t capacity) + : machine(streaming::dependencies{.capacity = capacity}) {} + + bool initialize() { + err = error_code(streaming::error::none); + return machine.process_event( + streaming::event::initialize{.error_out = err}); + } + + bool advance() { + err = error_code(streaming::error::none); + return machine.process_event( + streaming::event::advance{.result = result, .error_out = err}); + } + + bool reset() { + err = error_code(streaming::error::none); + return machine.process_event(streaming::event::reset{.error_out = err}); + } + + bool capture() { + err = error_code(streaming::error::none); + return machine.process_event( + streaming::event::capture_view{.view_out = view, .error_out = err}); + } + + streaming::sm machine; + streaming::advance_result result = {}; + streaming::window_view view = {}; + int32_t err = error_code(streaming::error::none); +}; + +void check_window(const streaming::window_view &view, + const int64_t logical_begin, const int64_t logical_end, + const int32_t physical_begin, const int32_t next_physical, + const int32_t valid_positions, const int32_t capacity) { + CHECK(view.logical_begin == logical_begin); + CHECK(view.logical_end == logical_end); + CHECK(view.physical_begin == physical_begin); + CHECK(view.next_physical_position == next_physical); + CHECK(view.valid_positions == valid_positions); + CHECK(view.capacity == capacity); +} + +void check_advance(const fixture &state, const int64_t logical_position, + const int32_t physical_position, const int64_t logical_begin, + const int64_t logical_end, const int32_t physical_begin, + const int32_t next_physical, const int32_t valid_positions, + const int32_t capacity) { + CHECK(state.result.logical_position == logical_position); + CHECK(state.result.physical_position == physical_position); + check_window(state.result.window, logical_begin, logical_end, physical_begin, + next_physical, valid_positions, capacity); +} + +} // namespace + +TEST_CASE("memory streaming rejects invalid injected capacity") { + fixture state{0}; + + CHECK(state.initialize()); + CHECK(state.err == error_code(streaming::error::invalid_configuration)); + CHECK(state.machine.is(sml::state)); +} + +TEST_CASE("memory streaming fills then wraps a fixed ring") { + fixture state{3}; + + REQUIRE(state.initialize()); + CHECK(state.machine.is(sml::state)); + REQUIRE(state.capture()); + check_window(state.view, 0, 0, 0, 0, 0, 3); + + REQUIRE(state.advance()); + check_advance(state, 0, 0, 0, 1, 0, 1, 1, 3); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.advance()); + check_advance(state, 1, 1, 0, 2, 0, 2, 2, 3); + CHECK(state.machine.is(sml::state)); + REQUIRE(state.capture()); + check_window(state.view, 0, 2, 0, 2, 2, 3); + + REQUIRE(state.advance()); + check_advance(state, 2, 2, 0, 3, 0, 0, 3, 3); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.advance()); + check_advance(state, 3, 0, 1, 4, 1, 1, 3, 3); + + REQUIRE(state.advance()); + check_advance(state, 4, 1, 2, 5, 2, 2, 3, 3); + + REQUIRE(state.advance()); + check_advance(state, 5, 2, 3, 6, 0, 0, 3, 3); + + REQUIRE(state.capture()); + check_window(state.view, 3, 6, 0, 0, 3, 3); +} + +TEST_CASE("memory streaming capacity one stays full and overwrites one slot") { + fixture state{1}; + + REQUIRE(state.initialize()); + REQUIRE(state.advance()); + check_advance(state, 0, 0, 0, 1, 0, 0, 1, 1); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.advance()); + check_advance(state, 1, 0, 1, 2, 0, 0, 1, 1); +} + +TEST_CASE( + "memory streaming reset invalidates history without clearing storage") { + fixture state{2}; + + REQUIRE(state.initialize()); + REQUIRE(state.advance()); + REQUIRE(state.advance()); + REQUIRE(state.advance()); + REQUIRE(state.reset()); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.capture()); + check_window(state.view, 0, 0, 0, 0, 0, 2); + + REQUIRE(state.advance()); + check_advance(state, 0, 0, 0, 1, 0, 1, 1, 2); +} + +TEST_CASE( + "memory streaming models uninitialized and duplicate initialize errors") { + fixture state{4}; + + CHECK(state.advance()); + CHECK(state.err == error_code(streaming::error::uninitialized)); + CHECK(state.reset()); + CHECK(state.err == error_code(streaming::error::uninitialized)); + CHECK(state.capture()); + CHECK(state.err == error_code(streaming::error::uninitialized)); + + REQUIRE(state.initialize()); + CHECK(state.initialize()); + CHECK(state.err == error_code(streaming::error::already_initialized)); + CHECK(state.machine.is(sml::state)); +} + +TEST_CASE("memory streaming dispatch remains allocation free") { + fixture state{8}; + REQUIRE(state.initialize()); + + emel::test::allocation::allocation_scope allocations; + for (int32_t index = 0; index < 64; ++index) { + REQUIRE(state.advance()); + } + REQUIRE(state.capture()); + REQUIRE(state.reset()); + + CHECK(allocations.allocations() == 0u); +} + +TEST_CASE( + "memory streaming makes unexpected events observable and recoverable") { + struct unknown_event {}; + + fixture state{3}; + REQUIRE(state.initialize()); + CHECK(state.machine.process_event(unknown_event{})); + CHECK(state.machine.is(sml::state)); + + CHECK(state.capture()); + CHECK(state.err == error_code(streaming::error::internal_error)); + REQUIRE(state.initialize()); + CHECK(state.machine.is(sml::state)); +} + +TEST_CASE("memory streaming full-state invariant failures are explicit") { + streaming::action::context ctx{streaming::dependencies{.capacity = 3}}; + stateforward::sml::sm machine{ + ctx}; + machine.set_current_states(sml::state); + + streaming::advance_result result = {}; + int32_t err = error_code(streaming::error::none); + + ctx.next_logical_position = std::numeric_limits::max(); + ctx.next_physical_position = 0; + CHECK(machine.process_event( + streaming::event::advance{.result = result, .error_out = err})); + CHECK(err == error_code(streaming::error::position_overflow)); + + ctx.next_logical_position = 3; + ctx.next_physical_position = -1; + err = error_code(streaming::error::none); + CHECK(machine.process_event( + streaming::event::advance{.result = result, .error_out = err})); + CHECK(err == error_code(streaming::error::internal_error)); + + ctx.next_physical_position = std::numeric_limits::max(); + err = error_code(streaming::error::none); + CHECK(machine.process_event( + streaming::event::advance{.result = result, .error_out = err})); + CHECK(err == error_code(streaming::error::internal_error)); + + const streaming::guard::guard_full_cursor_invalid cursor_invalid{}; + ctx.next_logical_position = std::numeric_limits::max(); + ctx.next_physical_position = 0; + CHECK_FALSE(cursor_invalid(ctx)); + + ctx.next_logical_position = 3; + CHECK_FALSE(cursor_invalid(ctx)); +} diff --git a/tests/model/moshi/binding_tests.cpp b/tests/model/moshi/binding_tests.cpp index 1eaeaf08..e24d5495 100644 --- a/tests/model/moshi/binding_tests.cpp +++ b/tests/model/moshi/binding_tests.cpp @@ -297,6 +297,14 @@ kv_store make_lm_kv(const lm_kv_options &options) { } store.i32_array("moshi.lm.delays", std::span{delays}); } + put_u32("moshi.lm.inference.dep_q", 2u); + put_u32("moshi.lm.inference.pre_text_silence_frames", 1u); + put_u32("moshi.lm.inference.post_text_silence_frames", 1u); + if (options.skip_key != "moshi.lm.inference.prompt_tokens") { + const std::array prompt_tokens = {3, 4, 5, 6, 7}; + store.i32_array("moshi.lm.inference.prompt_tokens", + std::span{prompt_tokens}); + } put_u32("moshi.lm.depformer.dim", 16u); put_u32("moshi.lm.depformer.num_heads", 2u); put_u32("moshi.lm.depformer.num_layers", 2u); @@ -406,7 +414,23 @@ void configure_q4_k_lm_contract(emel::model::data &model) { model.params.n_vocab = k_text_card; set_q4_k_matrix(model, "lm.text_emb.weight", k_dim, k_text_card + 1); set_q4_k_matrix(model, "lm.text_linear.weight", k_dim, k_text_card); - set_q4_k_matrix(model, "lm.emb.0.weight", k_dim, k_audio_card + 1); + for (int32_t codebook = 0; codebook < model.moshi_lm.n_q; ++codebook) { + set_q4_k_matrix(model, "lm.emb." + std::to_string(codebook) + ".weight", + k_dim, k_audio_card + 1); + } + set_matrix_tensor(model, "lm.depformer_text_emb.weight", + emel::kernel::detail::dtype_f32, + model.moshi_lm.depformer_dim, k_text_card + 1, + dense_storage_bytes(model.moshi_lm.depformer_dim, + k_text_card + 1, sizeof(uint32_t))); + for (int32_t codebook = 0; codebook < model.moshi_lm.dep_q - 1; ++codebook) { + set_matrix_tensor( + model, "lm.depformer_emb." + std::to_string(codebook) + ".weight", + emel::kernel::detail::dtype_f32, model.moshi_lm.depformer_dim, + k_audio_card + 1, + dense_storage_bytes(model.moshi_lm.depformer_dim, k_audio_card + 1, + sizeof(uint32_t))); + } } } // namespace @@ -468,11 +492,220 @@ TEST_CASE("enriched moshi lm fixture loads hparams, vocab, and contract") { REQUIRE(moshi::build_execution_contract(model, contract) == k_none); CHECK(contract.component == emel::model::data::moshi_component::lm); CHECK(contract.lm.audio_emb.tensor_count == 4u); + REQUIRE(contract.lm.text_embedding.tensor != nullptr); + CHECK(contract.lm.text_embedding.name == "lm.text_emb.weight"); + REQUIRE(contract.lm.output_norm.tensor != nullptr); + CHECK(contract.lm.output_norm.name == "lm.out_norm.alpha"); + REQUIRE(contract.lm.text_output_projection.tensor != nullptr); + CHECK(contract.lm.text_output_projection.name == "lm.text_linear.weight"); + for (int32_t codebook = 0; codebook < model.moshi_lm.n_q; ++codebook) { + const auto &embedding = + contract.lm.audio_embeddings[static_cast(codebook)]; + REQUIRE(embedding.tensor != nullptr); + CHECK(embedding.name == "lm.emb." + std::to_string(codebook) + ".weight"); + } + REQUIRE(contract.lm.depformer_text_embedding.tensor != nullptr); + CHECK(contract.lm.depformer_text_embedding.name == + "lm.depformer_text_emb.weight"); + for (int32_t codebook = 0; codebook < model.moshi_lm.dep_q; ++codebook) { + const auto &input_projection = + contract.lm.depformer_input_projections[static_cast(codebook)]; + REQUIRE(input_projection.tensor != nullptr); + CHECK(input_projection.name == + "lm.depformer_in." + std::to_string(codebook) + ".weight"); + const auto &output_projection = + contract.lm.depformer_output_projections[static_cast(codebook)]; + REQUIRE(output_projection.tensor != nullptr); + CHECK(output_projection.name == + "lm.linears." + std::to_string(codebook) + ".weight"); + if (codebook > 0) { + const auto &audio_embedding = + contract.lm + .depformer_audio_embeddings[static_cast(codebook - 1)]; + REQUIRE(audio_embedding.tensor != nullptr); + CHECK(audio_embedding.name == + "lm.depformer_emb." + std::to_string(codebook - 1) + ".weight"); + } + } CHECK(contract.lm.transformer.tensor_count > 0u); + for (int32_t layer_index = 0; layer_index < model.moshi_lm.num_layers; + ++layer_index) { + const auto &layer = + contract.lm.temporal_layers[static_cast(layer_index)]; + REQUIRE(layer.norm1.tensor != nullptr); + REQUIRE((layer.split_input_projection.tensor != nullptr || + layer.fused_input_projection.tensor != nullptr)); + REQUIRE(layer.output_projection.tensor != nullptr); + REQUIRE(layer.norm2.tensor != nullptr); + REQUIRE(layer.gating_input.tensor != nullptr); + REQUIRE(layer.gating_output.tensor != nullptr); + } + for (int32_t layer_index = 0; + layer_index < model.moshi_lm.depformer_num_layers; ++layer_index) { + const auto &layer = + contract.lm.depformer_layers[static_cast(layer_index)]; + REQUIRE(layer.norm1.tensor != nullptr); + REQUIRE(layer.norm2.tensor != nullptr); + for (int32_t codebook = 0; codebook < model.moshi_lm.dep_q; ++codebook) { + const auto &codebook_layer = + layer.codebooks[static_cast(codebook)]; + REQUIRE((codebook_layer.split_input_projection.tensor != nullptr || + codebook_layer.fused_input_projection.tensor != nullptr)); + REQUIRE(codebook_layer.output_projection.tensor != nullptr); + REQUIRE(codebook_layer.gating_input.tensor != nullptr); + REQUIRE(codebook_layer.gating_output.tensor != nullptr); + } + } CHECK(contract.lm.depformer.tensor_count > 0u); CHECK(contract.lm.linears.tensor_count == 4u); } +TEST_CASE("moshi lm contract requires every split depformer input projection") { + auto loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (loaded.model == nullptr) { + return; + } + + auto model = std::make_unique(*loaded.model); + const auto hide_tensor = [&model](const std::string_view target, + const size_t changed_character) { + for (uint32_t index = 0; index < model->n_tensors; ++index) { + auto &tensor = model->tensors[index]; + const std::string_view name{ + model->name_storage.data() + tensor.name_offset, tensor.name_length}; + if (name == target) { + model->name_storage[tensor.name_offset + changed_character] = '9'; + return true; + } + } + return false; + }; + + constexpr std::string_view split_name = + "lm.depformer.layers.0.self_attn.in_projs.1.weight"; + REQUIRE(hide_tensor(split_name, split_name.find("in_projs.1") + 9u)); + constexpr std::string_view fused_name = + "lm.depformer.layers.0.self_attn.in_proj_weight"; + (void)hide_tensor(fused_name, fused_name.size() - 1u); + CHECK(moshi::validate_execution_contract(*model) != k_none); +} + +TEST_CASE("moshi lm contract validates every audio embedding") { + auto loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (loaded.model == nullptr) { + return; + } + + auto model = std::make_unique(*loaded.model); + auto *embedding = find_mutable_tensor(*model, "lm.emb.1.weight"); + REQUIRE(embedding != nullptr); + + SUBCASE("shape") { embedding->dims[0] += 1; } + SUBCASE("execution dtype") { embedding->type = 10; } + + CHECK(moshi::validate_execution_contract(*model) != k_none); +} + +TEST_CASE("moshi lm contract validates depformer embeddings") { + auto loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (loaded.model == nullptr) { + return; + } + auto model = std::make_unique(*loaded.model); + SUBCASE("text shape") { + auto *tensor = find_mutable_tensor(*model, "lm.depformer_text_emb.weight"); + REQUIRE(tensor != nullptr); + tensor->dims[0] += 1; + } + SUBCASE("audio dtype") { + auto *tensor = find_mutable_tensor(*model, "lm.depformer_emb.0.weight"); + REQUIRE(tensor != nullptr); + tensor->type = 10; + } + CHECK(moshi::validate_execution_contract(*model) != k_none); +} + +TEST_CASE("moshi lm contract binds only scheduled depformer weight sets") { + auto loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (loaded.model == nullptr) { + return; + } + + auto model = std::make_unique(*loaded.model); + model->moshi_lm.depformer_weights_per_step = true; + model->moshi_lm.depformer_weight_schedule_count = 4u; + model->moshi_lm.depformer_weight_schedule[0] = 0; + model->moshi_lm.depformer_weight_schedule[1] = 1; + model->moshi_lm.depformer_weight_schedule[2] = 1; + model->moshi_lm.depformer_weight_schedule[3] = 2; + + for (uint32_t index = 0; index < model->n_tensors; ++index) { + auto &tensor = model->tensors[index]; + const std::string_view name{model->name_storage.data() + tensor.name_offset, + tensor.name_length}; + const std::size_t codebook = name.find(".3."); + if ((name.starts_with("lm.depformer_in.") || + name.starts_with("lm.depformer.layers.")) && + codebook != std::string_view::npos) { + model->name_storage[tensor.name_offset + codebook + 1u] = '9'; + } + } + + CHECK(moshi::validate_execution_contract(*model) == k_none); +} + +TEST_CASE("moshi lm contract rejects invalid scheduled tensor bindings") { + auto loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (loaded.model == nullptr) { + return; + } + + auto model = std::make_unique(*loaded.model); + const auto hide_tensor = [&model](const std::string_view target, + const size_t changed_character) { + for (uint32_t index = 0; index < model->n_tensors; ++index) { + auto &tensor = model->tensors[index]; + const std::string_view name{ + model->name_storage.data() + tensor.name_offset, tensor.name_length}; + if (name == target) { + model->name_storage[tensor.name_offset + changed_character] = '9'; + return true; + } + } + return false; + }; + + SUBCASE("schedule index must be in range") { + model->moshi_lm.depformer_weights_per_step = true; + model->moshi_lm.depformer_weight_schedule_count = 1u; + model->moshi_lm.depformer_weight_schedule[0] = model->moshi_lm.dep_q; + } + SUBCASE("text embedding is required") { + constexpr std::string_view name = "lm.depformer_text_emb.weight"; + REQUIRE(hide_tensor(name, name.size() - 1u)); + } + SUBCASE("scheduled input projection is required") { + constexpr std::string_view name = "lm.depformer_in.1.weight"; + REQUIRE(hide_tensor(name, name.find(".1.") + 1u)); + } + SUBCASE("codebook output projection is required") { + constexpr std::string_view name = "lm.linears.1.weight"; + REQUIRE(hide_tensor(name, name.find(".1.") + 1u)); + } + SUBCASE("unscheduled codebook output projection is still required") { + model->moshi_lm.depformer_weights_per_step = true; + model->moshi_lm.depformer_weight_schedule_count = 4u; + model->moshi_lm.depformer_weight_schedule[0] = 0; + model->moshi_lm.depformer_weight_schedule[1] = 1; + model->moshi_lm.depformer_weight_schedule[2] = 1; + model->moshi_lm.depformer_weight_schedule[3] = 2; + constexpr std::string_view name = "lm.linears.3.weight"; + REQUIRE(hide_tensor(name, name.find(".3.") + 1u)); + } + + CHECK(moshi::validate_execution_contract(*model) != k_none); +} + TEST_CASE("enriched mimi fixture loads hparams and contract") { auto loaded = load_fixture_or_skip("mimi-tiny.gguf"); if (loaded.model == nullptr) { @@ -483,13 +716,13 @@ TEST_CASE("enriched mimi fixture loads hparams and contract") { CHECK(model.moshi_component_id == emel::model::data::moshi_component::mimi); CHECK(model.mimi.sample_rate == 24000); CHECK(model.mimi.frame_rate == doctest::Approx(12.5f)); - CHECK(model.mimi.n_q == 4); + CHECK(model.mimi.n_q == 2); CHECK(model.mimi.card == 32); CHECK(model.mimi.dim == 16); CHECK(model.mimi.codebook_dim == 8); CHECK(model.mimi.semantic_n_q == 1); CHECK(model.mimi.transformer_num_layers == 2); - CHECK(model.params.n_features == 4); + CHECK(model.params.n_features == 2); moshi::execution_contract contract = {}; REQUIRE(moshi::build_execution_contract(model, contract) == k_none); @@ -582,6 +815,31 @@ TEST_CASE("moshi lm hparams require the full metadata contract") { CHECK_FALSE(load_lm_hparams(make_lm_kv({.dep_q = 5}), *model)); } + SUBCASE("missing inference keys fail for personaplex lmgen") { + CHECK_FALSE(load_lm_hparams( + make_lm_kv({.skip_key = "moshi.lm.inference.dep_q"}), *model)); + CHECK_FALSE(load_lm_hparams( + make_lm_kv({.skip_key = "moshi.lm.inference.prompt_tokens"}), *model)); + } + + SUBCASE("depformer weight schedule metadata loads when present") { + auto store = make_lm_kv({}); + const std::array schedule = {0, 1, 1, 2}; + store.i32_array("moshi.lm.depformer.weights_per_step_schedule", + std::span{schedule}); + CHECK(load_lm_hparams(store, *model)); + CHECK(model->moshi_lm.depformer_weight_schedule_count == schedule.size()); + CHECK(model->moshi_lm.depformer_weight_schedule[2] == 1); + } + + SUBCASE("depformer weight schedule count must match dep_q") { + auto store = make_lm_kv({}); + const std::array schedule = {0, 1, 2}; + store.i32_array("moshi.lm.depformer.weights_per_step_schedule", + std::span{schedule}); + CHECK_FALSE(load_lm_hparams(store, *model)); + } + SUBCASE("missing depformer keys fail when dep_q is positive") { CHECK_FALSE(load_lm_hparams( make_lm_kv({.skip_key = "moshi.lm.depformer.dim"}), *model)); @@ -1499,9 +1757,10 @@ TEST_CASE("mimi contract validation requires every acoustic rvq codebook") { // bind_rvq_split consumes every acoustic level 0..n_q-semantic_n_q-1 with // the full {codebook_dim, card} shape; corrupt the intermediate one (level - // 1 of 0..2 in the tiny fixture) so a last-index-only or existence-only - // probe would still pass while initialization fails. + // 1 of 0..2 after declaring the full tiny raw count) so a last-index-only + // or existence-only probe would still pass while initialization fails. auto &model = *loaded.model; + model.mimi.n_q = 4; bool corrupted = false; for (uint32_t idx = 0; idx < model.n_tensors; ++idx) { auto &tensor = model.tensors[idx]; diff --git a/tests/models/README.md b/tests/models/README.md index 2d863436..c78e0b2e 100644 --- a/tests/models/README.md +++ b/tests/models/README.md @@ -349,21 +349,22 @@ same-host ARM comparison matrix in Phase `59.1.1`. intermediate). - `moshi-tiny-lm-raw.gguf` is intentionally metadata-less (zero GGUF KV entries, moshi.cpp cache style) and is used to prove emel rejects unconverted moshi GGUFs. + - SHA256: - - `moshi-tiny-config.json`: `9808917cb929a06319033c86083b2589dcf5f8ee8314db0e78319c13cf61b58c` + - `moshi-tiny-config.json`: `7e6fbcacaa7ce6279fb8731cb01456f371781dc6b17c6e2308212d08b7f8ec0e` - `moshi-tokenizer-tiny.model`: `5ed25a383b9e3034941550f8b67d360d4178792a88f3f06b35ffba4417bc58e5` - `moshi-tiny-lm-raw.gguf`: `06e5824e6b586956147e388caf27c6dd52053cdb2ab2a1e4190ddb64bd0dab23` - - `moshi-tiny-lm.gguf`: `c83570ad540fed7a7390178046c161cf719dbf1b080c1421dec0a0af91c4f1bf` - - `mimi-tiny.gguf`: `92873fc4a6127f10a6798c03d3f0ce805ab2fa5544ca4d0b1efce996f72f0821` + - `moshi-tiny-lm.gguf`: `28cc6cd14d7fe8a2954b887e89d1b0d36dcb7fec0c295e21df51e1e15fa2cd8c` + - `mimi-tiny.gguf`: `518693f7b8d261fb41b2a018518dc1d64dbb505d7f869e996605a83d66d4af0f` (v1.28 M2 update: carries the full tiny-scale mimi_v0_1 SEANet topology — base 4 channels doubling to 64, kernel = 2×stride stage convs, resnet blocks, depthwise [4,1,16] upsample — so the codec compute layer can run complete encode/decode chains against it) - - `mimi-tiny-f16.gguf`: `011485af3e45a02b3e58b85f65d333681fa5adae8324738981f66dfe5556c5e3` + - `mimi-tiny-f16.gguf`: `3880b627a5b352a2ce1cf0a3a26ceca190a1ffff4773dda8213afc071dca75c1` (f16 operand-class variant: conv/projection weights stored f16 like the real Mimi artifact, driving the codec machines' guard_conv_f16 rows and the f16 im2col/mul_mat and raw-f16 RVQ projection paths) - - `mimi-tiny-q8.gguf`: `f0fc70d70556d23e89cae440b63f3dfa2a4385393f229f1fc3f908f75b1ee869` + - `mimi-tiny-q8.gguf`: `f5919a3eb852772c67a6a913ccc2ee960d162f79949d8a96dcf5d847c3c44412` (q8_0 operand-class variant: dim/codebook_dim widened to one q8 block of 32 and the transformer + RVQ projections quantized by the converter's `--quantize q8_0` path, driving guard_proj_q8 / guard_class_q8 rows and @@ -372,7 +373,7 @@ same-host ARM comparison matrix in Phase `59.1.1`. - Modeled contract (tiny scale, not real weights): PersonaPlex-shaped `lm` component (dim `32`, `2` layers/`2` heads, `n_q=4`, `dep_q=4`, `card=32`, `text_card=64`, depformer dim `16`/`2` layers), Mimi-shaped `mimi` component (dim `16`, - codebook_dim `8`, `2`-layer codec transformers), and a PersonaPlex `voice` + active `n_q=2`, codebook_dim `8`, `2`-layer codec transformers), and a PersonaPlex `voice` prompt component (`voice.embeddings` + `voice.cache`). - Loader/runtime boundary note: these fixtures prove converter output, moshi architecture registration, hparam binding, and execution-contract validation only; diff --git a/tests/models/mimi-tiny-f16.gguf b/tests/models/mimi-tiny-f16.gguf index d8fe4c45..097efa0d 100644 --- a/tests/models/mimi-tiny-f16.gguf +++ b/tests/models/mimi-tiny-f16.gguf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:011485af3e45a02b3e58b85f65d333681fa5adae8324738981f66dfe5556c5e3 +oid sha256:3880b627a5b352a2ce1cf0a3a26ceca190a1ffff4773dda8213afc071dca75c1 size 327872 diff --git a/tests/models/mimi-tiny-q8.gguf b/tests/models/mimi-tiny-q8.gguf index e2ac4458..5770b121 100644 --- a/tests/models/mimi-tiny-q8.gguf +++ b/tests/models/mimi-tiny-q8.gguf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f0fc70d70556d23e89cae440b63f3dfa2a4385393f229f1fc3f908f75b1ee869 +oid sha256:f5919a3eb852772c67a6a913ccc2ee960d162f79949d8a96dcf5d847c3c44412 size 514976 diff --git a/tests/models/mimi-tiny.gguf b/tests/models/mimi-tiny.gguf index f354dd8c..2bd7319d 100644 --- a/tests/models/mimi-tiny.gguf +++ b/tests/models/mimi-tiny.gguf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:92873fc4a6127f10a6798c03d3f0ce805ab2fa5544ca4d0b1efce996f72f0821 +oid sha256:518693f7b8d261fb41b2a018518dc1d64dbb505d7f869e996605a83d66d4af0f size 443232 diff --git a/tests/models/moshi-tiny-config.json b/tests/models/moshi-tiny-config.json index da6d7f54..b9ef478f 100644 --- a/tests/models/moshi-tiny-config.json +++ b/tests/models/moshi-tiny-config.json @@ -31,6 +31,16 @@ "depformer_gating": "silu", "depformer_pos_emb": "none", "depformer_weights_per_step": true, + "inference_dep_q": 2, + "inference_prompt_tokens": [ + 3, + 4, + 5, + 6, + 7 + ], + "inference_pre_text_silence_frames": 1, + "inference_post_text_silence_frames": 1, "cross_attention": false, "model_type": "personaplex", "tokenizer_name": "moshi-tokenizer-tiny.model" diff --git a/tests/models/moshi-tiny-lm.gguf b/tests/models/moshi-tiny-lm.gguf index 001dde9b..88be114d 100644 --- a/tests/models/moshi-tiny-lm.gguf +++ b/tests/models/moshi-tiny-lm.gguf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c83570ad540fed7a7390178046c161cf719dbf1b080c1421dec0a0af91c4f1bf -size 251744 +oid sha256:28cc6cd14d7fe8a2954b887e89d1b0d36dcb7fec0c295e21df51e1e15fa2cd8c +size 251968 diff --git a/tests/speech/codec/mimi_detail_tests.cpp b/tests/speech/codec/mimi_detail_tests.cpp index 64a06ea1..ce1c2eac 100644 --- a/tests/speech/codec/mimi_detail_tests.cpp +++ b/tests/speech/codec/mimi_detail_tests.cpp @@ -232,7 +232,7 @@ TEST_CASE("mimi codec binds and encodes deterministic frames") { bound_codec codec_state{}; REQUIRE(bind_or_fail(*loaded.model, codec_state)); CHECK(codec_state.runtime.frame_samples == 1920); - CHECK(codec_state.runtime.n_q == 4); + CHECK(codec_state.runtime.n_q == 2); CHECK(codec_state.runtime.dim == 16); const auto pcm = deterministic_pcm(codec_state.runtime.frame_samples); @@ -277,6 +277,23 @@ TEST_CASE("mimi codec decodes codes back to a full frame of audio") { std::span{pcm})); CHECK(all_finite(pcm.data(), pcm.size())); + codec::reset_streaming_state(codec_state.runtime, codec_state.streaming); + const size_t partial_count = + std::min(codes.size() - 1u, + static_cast( + codec_state.runtime.quantizer.semantic.level_count + 1)); + REQUIRE(partial_count > 0u); + REQUIRE(partial_count < codes.size()); + std::vector partial_pcm( + static_cast(codec_state.runtime.frame_samples), 0.0f); + CHECK(codec::validate_codes_in_range( + codec_state.runtime, + std::span{codes.data(), partial_count})); + REQUIRE(decode_frame(codec_state, + std::span{codes.data(), partial_count}, + std::span{partial_pcm})); + CHECK(all_finite(partial_pcm.data(), partial_pcm.size())); + // out-of-range codes are rejected by the pure request validator the // quantizer/facade guards route on (the compute path never sees them) std::vector bad_codes = codes; diff --git a/tests/speech/codec/mimi_lifecycle_tests.cpp b/tests/speech/codec/mimi_lifecycle_tests.cpp index e730df29..8cc75a91 100644 --- a/tests/speech/codec/mimi_lifecycle_tests.cpp +++ b/tests/speech/codec/mimi_lifecycle_tests.cpp @@ -228,7 +228,7 @@ TEST_CASE("mimi codec facade initializes, encodes, and decodes frames") { CHECK(err == emel::error::cast(mimi::error::none)); check_session_ready(machine); CHECK(g_frame_samples == 1920); - CHECK(g_n_q == 4); + CHECK(g_n_q == 2); const auto pcm = deterministic_pcm(g_frame_samples); std::vector codes(static_cast(g_n_q), -1); @@ -256,6 +256,28 @@ TEST_CASE("mimi codec facade initializes, encodes, and decodes frames") { check_session_ready(machine); CHECK(g_decode_done_count == 1u); + REQUIRE(machine.process_event(mimi::event::reset_stream{})); + const size_t partial_count = + std::min(codes.size() - 1u, + static_cast(loaded.model->mimi.semantic_n_q + 1)); + REQUIRE(partial_count > 0u); + REQUIRE(partial_count < codes.size()); + std::vector partial_decoded(static_cast(g_frame_samples), + 0.0f); + mimi::event::decode_frame partial_decode{ + std::span{codes.data(), partial_count}, + std::span{partial_decoded}}; + partial_decode.error_out = &err; + partial_decode.on_done = emel::callback::from<&on_decode_frame_done>(); + REQUIRE(machine.process_event(partial_decode)); + CHECK(err == emel::error::cast(mimi::error::none)); + check_session_ready(machine); + CHECK(g_decode_done_count == 2u); + for (const float sample : partial_decoded) { + CHECK(std::isfinite(sample)); + } + // stream reset replays the exact same codes for the same first frame std::vector replay(codes.size(), -1); REQUIRE(machine.process_event(mimi::event::reset_stream{})); diff --git a/tests/speech/generator/lifecycle_tests.cpp b/tests/speech/generator/lifecycle_tests.cpp new file mode 100644 index 00000000..a4838580 --- /dev/null +++ b/tests/speech/generator/lifecycle_tests.cpp @@ -0,0 +1,1114 @@ +#include "doctest/doctest.h" + +#include +#include +#include +#include +#include + +#include "../../allocation_tracker.hpp" +#include "emel/batch/planner/sm.hpp" +#include "emel/error/error.hpp" +#include "emel/memory/streaming/sm.hpp" +#include "emel/speech/generator/sm.hpp" + +namespace { + +namespace generator = emel::speech::generator; +namespace sml = stateforward::sml; + +struct fake_initialize { + emel::error::type *error_out = nullptr; +}; + +struct fake_voice_condition { + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; + bool *complete_out = nullptr; + int32_t *remaining_frames_out = nullptr; +}; + +struct fake_prompt_begin { + int32_t text_token_count = 0; + emel::error::type *error_out = nullptr; +}; + +struct fake_prompt_condition { + int32_t text_token = -1; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; + bool *complete_out = nullptr; + int32_t *remaining_frames_out = nullptr; +}; + +struct fake_encode { + fake_encode(std::span pcm_ref, + std::span tokens_out_ref) noexcept + : pcm(pcm_ref), tokens_out(tokens_out_ref) {} + + std::span pcm; + std::span tokens_out; + emel::error::type *error_out = nullptr; +}; + +struct fake_prediction_workspace {}; + +struct fake_predict { + fake_predict(std::span tokens_ref, + fake_prediction_workspace &workspace_ref, + const int32_t planned_step_size_ref, + const int32_t planned_output_count_ref) noexcept + : tokens(tokens_ref), workspace(workspace_ref), + planned_step_size(planned_step_size_ref), + planned_output_count(planned_output_count_ref) {} + + std::span tokens; + fake_prediction_workspace &workspace; + int32_t planned_step_size = 0; + int32_t planned_output_count = 0; + emel::error::type *error_out = nullptr; +}; + +struct fake_execute { + fake_execute(fake_prediction_workspace &workspace_ref, + std::span tokens_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref) {} + + fake_prediction_workspace &workspace; + std::span tokens; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_sample { + fake_sample(fake_prediction_workspace &workspace_ref, + std::span tokens_ref, + std::span tokens_out_ref, + int32_t &text_token_out_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref), + tokens_out(tokens_out_ref), text_token_out(text_token_out_ref) {} + + fake_prediction_workspace &workspace; + std::span tokens; + std::span tokens_out; + int32_t &text_token_out; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_tokenizer_initialize { + int32_t &error_out; +}; + +struct fake_tokenize { + std::span audio_tokens = {}; + std::span model_tokens_out = {}; + int32_t &error_out; +}; + +struct fake_detokenize { + int32_t text_token = -1; + std::span audio_tokens = {}; + int32_t &text_token_out; + std::span audio_tokens_out = {}; + bool &produced_out; + int32_t &error_out; +}; + +struct fake_capture_tokenizer_state { + fake_capture_tokenizer_state(std::span cache_out_ref, + int64_t &offset_out_ref, + emel::error::type &error_out_ref) noexcept + : cache_out(cache_out_ref), offset_out(offset_out_ref), + error_out(error_out_ref) {} + + std::span cache_out = {}; + int64_t &offset_out; + emel::error::type &error_out; +}; + +struct fake_restore_tokenizer_state { + std::span cache = {}; + int64_t offset = 0; + int32_t &error_out; +}; + +struct fake_decode { + fake_decode(std::span tokens_ref, + std::span pcm_out_ref) noexcept + : tokens(tokens_ref), pcm_out(pcm_out_ref) {} + + std::span tokens; + std::span pcm_out; + emel::error::type *error_out = nullptr; +}; + +struct fake_encoder_actor { + int32_t initialize_calls = 0; + int32_t encode_calls = 0; + bool initialize_accepted = true; + bool encode_accepted = true; + emel::error::type initialize_error = 0; + emel::error::type encode_error = 0; + + bool process_event(const fake_initialize &request) noexcept { + ++initialize_calls; + *request.error_out = initialize_error; + return initialize_accepted; + } + + bool process_event(const fake_encode &request) noexcept { + ++encode_calls; + request.tokens_out[0] = 11; + request.tokens_out[1] = 12; + *request.error_out = encode_error; + return encode_accepted; + } +}; + +struct fake_decoder_actor { + int32_t initialize_calls = 0; + int32_t decode_calls = 0; + bool initialize_accepted = true; + bool decode_accepted = true; + emel::error::type initialize_error = 0; + emel::error::type decode_error = 0; + + bool process_event(const fake_initialize &request) noexcept { + ++initialize_calls; + *request.error_out = initialize_error; + return initialize_accepted; + } + + bool process_event(const fake_decode &request) noexcept { + ++decode_calls; + std::fill(request.pcm_out.begin(), request.pcm_out.end(), 0.25f); + *request.error_out = decode_error; + return decode_accepted; + } +}; + +struct fake_tokenizer_actor { + int32_t initialize_calls = 0; + int32_t tokenize_calls = 0; + int32_t detokenize_calls = 0; + int32_t restore_calls = 0; + bool produce = true; + + bool process_event(const fake_tokenizer_initialize &request) noexcept { + ++initialize_calls; + request.error_out = 0; + return true; + } + + bool process_event(const fake_tokenize &request) noexcept { + ++tokenize_calls; + std::copy(request.audio_tokens.begin(), request.audio_tokens.end(), + request.model_tokens_out.begin()); + request.error_out = 0; + return true; + } + + bool process_event(const fake_detokenize &request) noexcept { + ++detokenize_calls; + std::copy(request.audio_tokens.begin(), request.audio_tokens.end(), + request.audio_tokens_out.begin()); + request.text_token_out = request.text_token; + request.produced_out = produce; + request.error_out = 0; + return true; + } + + bool process_event(const fake_restore_tokenizer_state &request) noexcept { + ++restore_calls; + request.error_out = 0; + return true; + } +}; + +struct fake_predictor_actor { + int32_t initialize_calls = 0; + int32_t conditioning_initialize_calls = 0; + int32_t voice_condition_calls = 0; + int32_t prompt_begin_calls = 0; + int32_t prompt_begin_text_token_count = 0; + int32_t prompt_condition_calls = 0; + int32_t predict_calls = 0; + int32_t execute_calls = 0; + int32_t sample_calls = 0; + int32_t last_plan_step_size = 0; + int32_t last_plan_output_count = 0; + int32_t rejected_initialize_call = 0; + bool voice_accepted = true; + bool prompt_begin_accepted = true; + bool prompt_accepted = true; + bool predict_accepted = true; + bool execute_accepted = true; + bool sample_accepted = true; + emel::error::type initialize_error = 0; + emel::error::type voice_error = 0; + emel::error::type voice_graph_error = 0; + emel::error::type prompt_begin_error = 0; + emel::error::type prompt_error = 0; + emel::error::type prompt_graph_error = 0; + emel::error::type predict_error = 0; + emel::error::type execute_error = 0; + emel::error::type execute_graph_error = 0; + emel::error::type sample_error = 0; + bool voice_complete = true; + bool prompt_complete = true; + + bool process_event(const fake_initialize &request) noexcept { + ++initialize_calls; + *request.error_out = initialize_error; + return rejected_initialize_call != initialize_calls; + } + + bool process_event(const fake_prompt_begin &request) noexcept { + ++prompt_begin_calls; + prompt_begin_text_token_count = request.text_token_count; + *request.error_out = prompt_begin_error; + return prompt_begin_accepted; + } + + bool process_event(const fake_voice_condition &request) noexcept { + ++voice_condition_calls; + *request.error_out = voice_error; + *request.graph_error_out = voice_graph_error; + *request.complete_out = voice_complete; + *request.remaining_frames_out = voice_complete ? 0 : 1; + return voice_accepted; + } + + bool process_event(const fake_prompt_condition &request) noexcept { + ++prompt_condition_calls; + *request.error_out = prompt_error; + *request.graph_error_out = prompt_graph_error; + *request.complete_out = prompt_complete; + *request.remaining_frames_out = prompt_complete ? 0 : 1; + return prompt_accepted; + } + + bool process_event(const fake_predict &request) noexcept { + ++predict_calls; + last_plan_step_size = request.planned_step_size; + last_plan_output_count = request.planned_output_count; + *request.error_out = predict_error; + return predict_accepted; + } + + bool process_event(const fake_execute &request) noexcept { + ++execute_calls; + *request.error_out = execute_error; + *request.graph_error_out = execute_graph_error; + return execute_accepted; + } + + bool process_event(const fake_sample &request) noexcept { + ++sample_calls; + request.tokens_out[0] = 111; + request.tokens_out[1] = 112; + request.text_token_out = 42; + *request.error_out = sample_error; + *request.graph_error_out = 0; + return sample_accepted; + } + + bool process_event(const fake_capture_tokenizer_state &request) noexcept { + std::fill(request.cache_out.begin(), request.cache_out.end(), 0); + request.offset_out = 0; + request.error_out = 0; + return true; + } +}; + +struct fake_dependencies { + using generator_mode = generator::action::mode::duplex; + using voice_condition_event = fake_voice_condition; + using prompt_begin_event = fake_prompt_begin; + using prompt_condition_event = fake_prompt_condition; + using encode_event = fake_encode; + using tokenizer_initialize_event = fake_tokenizer_initialize; + using tokenize_event = fake_tokenize; + using predict_event = fake_predict; + using graph_event = fake_execute; + using sample_event = fake_sample; + using detokenize_event = fake_detokenize; + using capture_tokenizer_state_event = fake_capture_tokenizer_state; + using restore_tokenizer_state_event = fake_restore_tokenizer_state; + using decode_event = fake_decode; + + emel::memory::streaming::sm &temporal_positions; + emel::memory::streaming::sm &secondary_positions; + emel::batch::planner::sm &planner; + fake_encoder_actor &encoder; + fake_tokenizer_actor &tokenizer; + fake_decoder_actor &decoder; + fake_predictor_actor &predictor; + fake_predictor_actor &graph; + fake_predictor_actor &sampler; + fake_prediction_workspace &prediction_workspace; + fake_initialize encoder_initialize = {}; + fake_initialize decoder_initialize = {}; + fake_initialize predictor_initialize = {}; + fake_initialize conditioning_initialize = {}; + fake_prompt_begin prompt_begin{.text_token_count = 1}; + std::span silence_pcm = {}; + std::span input_codes = {}; + std::span tokenize_input_codes = {}; + std::span model_codes = {}; + std::span predicted_codes = {}; + std::span output_codes = {}; + std::span tokenizer_cache_snapshot = {}; + int32_t frame_samples = 0; + int32_t codebook_count = 0; + emel::batch::planner::event::plan_mode frame_plan_mode = + emel::batch::planner::event::plan_mode::simple; + int32_t frame_plan_steps = 0; + int32_t frame_plan_token_count = 0; + bool frame_plan_output_all = false; +}; + +struct fixture { + std::array silence{}; + std::array input_codes{}; + std::array model_codes{}; + std::array predicted_codes{}; + std::array output_codes{}; + std::array tokenizer_cache_snapshot{}; + emel::memory::streaming::sm temporal_positions{ + emel::memory::streaming::dependencies{.capacity = 16}}; + emel::memory::streaming::sm secondary_positions{ + emel::memory::streaming::dependencies{.capacity = 16}}; + emel::batch::planner::sm planner{}; + fake_encoder_actor encoder{}; + fake_tokenizer_actor tokenizer{}; + fake_decoder_actor decoder{}; + fake_predictor_actor predictor{}; + fake_prediction_workspace prediction_workspace{}; + fake_dependencies dependencies; + generator::sm machine; + + explicit fixture(const int32_t frame_samples = 4, + const int32_t codebook_count = 2, + const emel::batch::planner::event::plan_mode + frame_plan_mode = + emel::batch::planner::event::plan_mode::simple) + : dependencies{ + .temporal_positions = temporal_positions, + .secondary_positions = secondary_positions, + .planner = planner, + .encoder = encoder, + .tokenizer = tokenizer, + .decoder = decoder, + .predictor = predictor, + .graph = predictor, + .sampler = predictor, + .prediction_workspace = prediction_workspace, + .silence_pcm = std::span{silence}, + .input_codes = std::span{input_codes}, + .tokenize_input_codes = std::span{input_codes}, + .model_codes = std::span{model_codes}, + .predicted_codes = std::span{predicted_codes}, + .output_codes = std::span{output_codes}, + .tokenizer_cache_snapshot = std::span{tokenizer_cache_snapshot}, + .frame_samples = frame_samples, + .codebook_count = codebook_count, + .frame_plan_mode = frame_plan_mode, + .frame_plan_steps = 1, + .frame_plan_token_count = 1, + .frame_plan_output_all = true, + }, + machine{dependencies} {} + + bool initialize() { + emel::error::type err = + generator::action::error_code(generator::error::none); + return machine.process_event(generator::event::initialize{err}); + } + + bool condition() { + bool complete = false; + int32_t remaining = -1; + emel::error::type err = + generator::action::error_code(generator::error::none); + return machine.process_event( + generator::event::condition{7, complete, remaining, err}); + } +}; + +struct callback_probe { + int32_t done_calls = 0; + int32_t error_calls = 0; + emel::error::type last_error = 0; + + void initialize_done(const generator::events::initialize_done &) noexcept { + ++done_calls; + } + void + initialize_error(const generator::events::initialize_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } + void condition_done(const generator::events::condition_done &) noexcept { + ++done_calls; + } + void condition_error(const generator::events::condition_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } + void + generation_error(const generator::events::generation_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } + void generation_done(const generator::events::generation_done &) noexcept { + ++done_calls; + } + void stream_done(const generator::events::stream_frame_done &) noexcept { + ++done_calls; + } + void stream_error(const generator::events::stream_frame_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } + void flush_done(const generator::events::flush_done &) noexcept { + ++done_calls; + } + void flush_error(const generator::events::flush_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } +}; + +void advance_to_ready(fixture &test) { + REQUIRE(test.initialize()); + REQUIRE(test.condition()); + REQUIRE(test.condition()); +} + +struct fake_synthesis_stage { + explicit fake_synthesis_stage( + const generator::event::generate &request_ref) noexcept + : request(request_ref) {} + fake_synthesis_stage(const generator::event::generate &request_ref, + int32_t &sample_count_out_ref) noexcept + : request(request_ref), sample_count_out(&sample_count_out_ref) {} + + const generator::event::generate &request; + int32_t *sample_count_out = nullptr; + emel::error::type *error_out = nullptr; +}; + +struct fake_synthesis_actor { + int32_t initialize_calls = 0; + int32_t stage_calls = 0; + bool write_pcm = false; + bool postprocess_pcm = false; + + bool process_event(const fake_initialize &request) noexcept { + ++initialize_calls; + *request.error_out = 0; + return true; + } + + bool process_event(const fake_synthesis_stage &request) noexcept { + ++stage_calls; + if (write_pcm) { + std::fill(request.request.pcm_out.begin(), request.request.pcm_out.end(), + 0.5f); + *request.sample_count_out = + static_cast(request.request.pcm_out.size()); + } + if (postprocess_pcm) { + for (float &sample : request.request.pcm_out) { + sample *= 0.5f; + } + } + *request.error_out = 0; + return true; + } +}; + +struct fake_synthesis_dependencies { + using generator_mode = generator::action::mode::synthesis; + using condition_event = fake_synthesis_stage; + using prefill_event = fake_synthesis_stage; + using predict_event = fake_synthesis_stage; + using sample_event = fake_synthesis_stage; + using decode_event = fake_synthesis_stage; + using postprocess_event = fake_synthesis_stage; + + fake_synthesis_actor &conditioner; + fake_synthesis_actor &prefiller; + fake_synthesis_actor &predictor; + fake_synthesis_actor &sampler; + fake_synthesis_actor &decoder; + fake_synthesis_actor &postprocessor; + fake_initialize conditioner_initialize = {}; + fake_initialize prefiller_initialize = {}; + fake_initialize predictor_initialize = {}; + fake_initialize sampler_initialize = {}; + fake_initialize decoder_initialize = {}; + fake_initialize postprocessor_initialize = {}; +}; + +struct synthesis_fixture { + fake_synthesis_actor conditioner{}; + fake_synthesis_actor prefiller{}; + fake_synthesis_actor predictor{}; + fake_synthesis_actor sampler{}; + fake_synthesis_actor decoder{.write_pcm = true}; + fake_synthesis_actor postprocessor{.postprocess_pcm = true}; + fake_synthesis_dependencies dependencies{ + .conditioner = conditioner, + .prefiller = prefiller, + .predictor = predictor, + .sampler = sampler, + .decoder = decoder, + .postprocessor = postprocessor, + }; + generator::sm machine{dependencies}; +}; + +} // namespace + +TEST_CASE("speech_generator_initializes_injected_actor_composition") { + auto test = std::make_unique(); + + REQUIRE(test->initialize()); + CHECK(test->machine.is(sml::state)); + CHECK(test->encoder.initialize_calls == 1); + CHECK(test->decoder.initialize_calls == 1); + CHECK(test->predictor.initialize_calls == 2); +} + +TEST_CASE("speech_generator_synthesizes_with_model_neutral_actor_composition") { + auto test = std::make_unique(); + emel::error::type err = -1; + REQUIRE(test->machine.process_event(generator::event::initialize{err})); + CHECK(test->machine.is(sml::state)); + CHECK(test->conditioner.initialize_calls == 1); + CHECK(test->prefiller.initialize_calls == 1); + CHECK(test->predictor.initialize_calls == 1); + CHECK(test->sampler.initialize_calls == 1); + CHECK(test->decoder.initialize_calls == 1); + CHECK(test->postprocessor.initialize_calls == 1); + + callback_probe probe{}; + std::array pcm_out{}; + int32_t sample_count = 0; + generator::event::generate request{"a model-neutral synthesis request", + std::span{pcm_out}, sample_count, + err}; + request.on_done = + decltype(request.on_done)::from(&probe); + + REQUIRE(test->machine.process_event(request)); + CHECK(test->machine.is(sml::state)); + CHECK(test->conditioner.stage_calls == 1); + CHECK(test->prefiller.stage_calls == 1); + CHECK(test->predictor.stage_calls == 1); + CHECK(test->sampler.stage_calls == 1); + CHECK(test->decoder.stage_calls == 1); + CHECK(test->postprocessor.stage_calls == 1); + CHECK(sample_count == 8); + CHECK(pcm_out[0] == doctest::Approx(0.25f)); + CHECK(probe.done_calls == 1); + CHECK(err == generator::action::error_code(generator::error::none)); +} + +TEST_CASE("speech_generator_reports_initialize_outcomes_through_callbacks") { + SUBCASE("success") { + auto test = std::make_unique(); + callback_probe probe{}; + emel::error::type err = -1; + generator::event::initialize request{err}; + request.on_done = decltype(request.on_done)::from< + callback_probe, &callback_probe::initialize_done>(&probe); + + REQUIRE(test->machine.process_event(request)); + CHECK(probe.done_calls == 1); + CHECK(err == generator::action::error_code(generator::error::none)); + } + + SUBCASE("invalid composition") { + auto test = std::make_unique(0, 2); + callback_probe probe{}; + emel::error::type err = 0; + generator::event::initialize request{err}; + request.on_error = decltype(request.on_error)::from< + callback_probe, &callback_probe::initialize_error>(&probe); + + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.error_calls == 1); + CHECK(probe.last_error == + generator::action::error_code(generator::error::invalid_request)); + CHECK(test->machine.is(sml::state)); + } + + SUBCASE("child initialization failure") { + auto test = std::make_unique(); + test->encoder.initialize_error = 9; + callback_probe probe{}; + emel::error::type err = 0; + generator::event::initialize request{err}; + request.on_error = decltype(request.on_error)::from< + callback_probe, &callback_probe::initialize_error>(&probe); + + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.error_calls == 1); + CHECK(probe.last_error == generator::action::error_code( + generator::error::encoder_initialize_failed)); + } +} + +TEST_CASE("speech_generator_conditions_without_model_family_contracts") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + + REQUIRE(test->condition()); + CHECK(test->machine.is(sml::state)); + CHECK(test->predictor.voice_condition_calls == 1); + CHECK(test->predictor.prompt_begin_calls == 1); + CHECK(test->predictor.prompt_begin_text_token_count == 1); + + REQUIRE(test->condition()); + CHECK(test->machine.is(sml::state)); + CHECK(test->predictor.prompt_condition_calls == 1); +} + +TEST_CASE("speech_generator_models_condition_pending_and_error_results") { + SUBCASE("bounded pending frames") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + callback_probe probe{}; + bool complete = true; + int32_t remaining = -1; + emel::error::type err = 0; + generator::event::condition request{7, complete, remaining, err}; + request.on_done = decltype(request.on_done)::from< + callback_probe, &callback_probe::condition_done>(&probe); + + test->predictor.voice_complete = false; + REQUIRE(test->machine.process_event(request)); + CHECK_FALSE(complete); + CHECK(remaining == 1); + CHECK(probe.done_calls == 1); + CHECK(test->machine.is(sml::state)); + + test->predictor.voice_complete = true; + REQUIRE(test->machine.process_event(request)); + CHECK(probe.done_calls == 2); + CHECK(test->machine.is(sml::state)); + + test->predictor.prompt_complete = false; + REQUIRE(test->machine.process_event(request)); + CHECK_FALSE(complete); + CHECK(probe.done_calls == 3); + + test->predictor.prompt_complete = true; + REQUIRE(test->machine.process_event(request)); + CHECK(complete); + CHECK(probe.done_calls == 4); + CHECK(test->machine.is(sml::state)); + } + + SUBCASE("graph failure") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + test->predictor.voice_graph_error = 7; + callback_probe probe{}; + bool complete = true; + int32_t remaining = 0; + emel::error::type err = 0; + generator::event::condition request{7, complete, remaining, err}; + request.on_error = decltype(request.on_error)::from< + callback_probe, &callback_probe::condition_error>(&probe); + + CHECK_FALSE(test->machine.process_event(request)); + CHECK_FALSE(complete); + CHECK(remaining == -1); + CHECK(probe.error_calls == 1); + CHECK(probe.last_error == + generator::action::error_code(generator::error::conditioning_failed)); + CHECK(test->machine.is(sml::state)); + } +} + +TEST_CASE("speech_generator_keeps_offline_synthesis_failure_explicit") { + auto test = std::make_unique(); + advance_to_ready(*test); + callback_probe probe{}; + std::array pcm_out{}; + int32_t sample_count = -1; + emel::error::type err = 0; + generator::event::generate request{"hello", std::span{pcm_out}, + sample_count, err}; + request.on_error = decltype(request.on_error)::from< + callback_probe, &callback_probe::generation_error>(&probe); + + CHECK_FALSE(test->machine.process_event(request)); + CHECK(sample_count == 0); + CHECK(probe.error_calls == 1); + CHECK(probe.last_error == + generator::action::error_code(generator::error::unsupported_request)); + CHECK(test->machine.is(sml::state)); +} + +TEST_CASE("speech_generator_streams_through_injected_actor_pipeline") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + REQUIRE(test->condition()); + REQUIRE(test->condition()); + + std::array pcm_in{}; + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool produced = false; + emel::error::type err = generator::action::error_code(generator::error::none); + generator::event::stream_frame request{std::span{pcm_in}, + std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + produced, + err}; + + REQUIRE(test->machine.process_event(request)); + CHECK(test->machine.is(sml::state)); + CHECK(encoded == std::array{11, 12}); + CHECK(generated == std::array{111, 112}); + CHECK(text_token == 42); + CHECK(test->predictor.predict_calls == 1); + CHECK(test->predictor.execute_calls == 1); + CHECK(test->predictor.sample_calls == 1); + CHECK(test->predictor.last_plan_step_size == 1); + CHECK(test->predictor.last_plan_output_count == 1); + CHECK(sample_count == 4); + CHECK(produced); + CHECK(pcm_out[0] == doctest::Approx(0.25f)); +} + +TEST_CASE("speech_generator_rejects_stream_before_initialization") { + auto test = std::make_unique(); + std::array pcm_in{}; + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = 77; + int32_t sample_count = 77; + bool produced = true; + emel::error::type err = generator::action::error_code(generator::error::none); + generator::event::stream_frame request{std::span{pcm_in}, + std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + produced, + err}; + CHECK_FALSE(test->machine.process_event(request)); + CHECK(err == generator::action::error_code(generator::error::internal_error)); + CHECK(sample_count == 0); + CHECK_FALSE(produced); +} + +TEST_CASE("speech_generator_rejects_other_public_requests_outside_lifecycle") { + const auto internal_error = + generator::action::error_code(generator::error::internal_error); + + SUBCASE("second initialization") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + emel::error::type err = + generator::action::error_code(generator::error::none); + CHECK_FALSE(test->machine.process_event(generator::event::initialize{err})); + CHECK(err == internal_error); + } + + SUBCASE("conditioning before initialization") { + auto test = std::make_unique(); + bool complete = true; + int32_t remaining = 77; + emel::error::type err = + generator::action::error_code(generator::error::none); + CHECK_FALSE(test->machine.process_event( + generator::event::condition{7, complete, remaining, err})); + CHECK(err == internal_error); + CHECK_FALSE(complete); + CHECK(remaining == -1); + } + + SUBCASE("generation before initialization") { + auto test = std::make_unique(); + std::array pcm_out{}; + int32_t sample_count = 77; + emel::error::type err = + generator::action::error_code(generator::error::none); + CHECK_FALSE(test->machine.process_event(generator::event::generate{ + "hello", std::span{pcm_out}, sample_count, err})); + CHECK(err == internal_error); + CHECK(sample_count == 0); + } + + SUBCASE("flush before initialization") { + auto test = std::make_unique(); + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = 77; + int32_t sample_count = 77; + bool complete = true; + emel::error::type err = + generator::action::error_code(generator::error::none); + CHECK_FALSE(test->machine.process_event(generator::event::flush{ + std::span{pcm_out}, std::span{encoded}, + std::span{generated}, text_token, sample_count, complete, + err})); + CHECK(err == internal_error); + CHECK(sample_count == 0); + CHECK_FALSE(complete); + } + + SUBCASE("reset while conditioning") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + emel::error::type err = + generator::action::error_code(generator::error::none); + CHECK_FALSE(test->machine.process_event(generator::event::reset{err})); + CHECK(err == internal_error); + } +} + +TEST_CASE("speech_generator_reports_frame_planning_failure") { + auto test = std::make_unique( + 4, 2, static_cast(99)); + advance_to_ready(*test); + std::array pcm_in{}; + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool produced = false; + emel::error::type err = 0; + generator::event::stream_frame request{std::span{pcm_in}, + std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + produced, + err}; + + CHECK_FALSE(test->machine.process_event(request)); + CHECK(err == + generator::action::error_code(generator::error::planning_failed)); + CHECK(test->predictor.predict_calls == 0); +} + +TEST_CASE("speech_generator_streams_pending_frames_and_reports_failures") { + auto test = std::make_unique(); + advance_to_ready(*test); + std::array pcm_in{}; + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = -1; + bool produced = true; + emel::error::type err = 0; + callback_probe probe{}; + generator::event::stream_frame request{std::span{pcm_in}, + std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + produced, + err}; + + SUBCASE("pending") { + test->tokenizer.produce = false; + request.on_done = + decltype(request.on_done)::from(&probe); + REQUIRE(test->machine.process_event(request)); + CHECK_FALSE(produced); + CHECK(sample_count == 0); + CHECK(test->decoder.decode_calls == 0); + CHECK(probe.done_calls == 1); + } + + SUBCASE("invalid request") { + request.pcm_in = request.pcm_in.first(3); + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::invalid_request)); + } + + SUBCASE("encode failure") { + test->encoder.encode_error = 3; + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::encode_failed)); + } + + SUBCASE("graph failure") { + test->predictor.execute_graph_error = 4; + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::graph_failed)); + } + + SUBCASE("sample failure") { + test->predictor.sample_error = 4; + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::sample_failed)); + } + + SUBCASE("decode failure") { + test->decoder.decode_accepted = false; + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::decode_failed)); + } +} + +TEST_CASE("speech_generator_flushes_silence_through_same_pipeline") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + REQUIRE(test->condition()); + REQUIRE(test->condition()); + + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool complete = true; + emel::error::type err = generator::action::error_code(generator::error::none); + generator::event::flush request{std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + complete, + err}; + + REQUIRE(test->machine.process_event(request)); + CHECK(test->machine.is(sml::state)); + CHECK(sample_count == 4); + CHECK_FALSE(complete); + CHECK(encoded == std::array{11, 12}); + CHECK(generated == std::array{111, 112}); +} + +TEST_CASE("speech_generator_flushes_pending_frames_and_reports_failures") { + auto test = std::make_unique(); + advance_to_ready(*test); + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = -1; + bool complete = true; + emel::error::type err = 0; + callback_probe probe{}; + generator::event::flush request{std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + complete, + err}; + + SUBCASE("pending") { + test->tokenizer.produce = false; + request.on_done = + decltype(request.on_done)::from(&probe); + REQUIRE(test->machine.process_event(request)); + CHECK(sample_count == 0); + CHECK_FALSE(complete); + CHECK(probe.done_calls == 1); + } + + SUBCASE("invalid request") { + request.generated_tokens_out = request.generated_tokens_out.first(1); + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::invalid_request)); + } + + SUBCASE("predict failure") { + test->predictor.predict_accepted = false; + request.on_error = + decltype(request.on_error)::from(&probe); + CHECK_FALSE(test->machine.process_event(request)); + CHECK(probe.last_error == + generator::action::error_code(generator::error::predict_failed)); + } +} + +TEST_CASE("speech_generator_duplex_dispatch_is_allocation_free") { + auto test = std::make_unique(); + REQUIRE(test->initialize()); + REQUIRE(test->condition()); + REQUIRE(test->condition()); + + std::array pcm_in{}; + std::array pcm_out{}; + std::array encoded{}; + std::array generated{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool produced = false; + emel::error::type err = generator::action::error_code(generator::error::none); + generator::event::stream_frame request{std::span{pcm_in}, + std::span{pcm_out}, + std::span{encoded}, + std::span{generated}, + text_token, + sample_count, + produced, + err}; + size_t allocation_count = 0; + bool accepted = true; + { + emel::test::allocation::allocation_scope allocations; + for (int32_t iteration = 0; iteration < 32; ++iteration) { + accepted = test->machine.process_event(request) && accepted; + } + allocation_count = allocations.allocations(); + } + + CHECK(accepted); + CHECK(allocation_count == 0u); +} diff --git a/tests/speech/predictor/moshi_executor_detail_tests.cpp b/tests/speech/predictor/moshi_executor_detail_tests.cpp new file mode 100644 index 00000000..e61ade51 --- /dev/null +++ b/tests/speech/predictor/moshi_executor_detail_tests.cpp @@ -0,0 +1,94 @@ +#include +#include + +#include + +#include "emel/kernel/detail.hpp" +#include "emel/speech/predictor/moshi/executor/detail.hpp" + +namespace { + +namespace detail = emel::speech::predictor::moshi::executor::detail; + +detail::dtype dtype(const uint8_t code) { + return static_cast(code); +} + +} // namespace + +TEST_CASE("speech Moshi executor binds supported dense and quantized views") { + std::array dense_data = {}; + emel::model::data::tensor_record tensor{}; + tensor.data = dense_data.data(); + tensor.n_dims = 2; + tensor.dims[0] = 4; + tensor.dims[1] = 4; + tensor.type = emel::kernel::detail::dtype_f32; + detail::tensor_view view{}; + + REQUIRE(detail::bind_tensor_view(tensor, view)); + CHECK(view.data == dense_data.data()); + CHECK(view.ne[0] == 4u); + CHECK(view.ne[1] == 4u); + CHECK(view.nb[0] == sizeof(float)); + + std::array quantized_data = {}; + tensor.data = quantized_data.data(); + tensor.dims[0] = 32; + tensor.dims[1] = 2; + tensor.type = emel::kernel::detail::dtype_q4_0; + REQUIRE(detail::bind_tensor_view(tensor, view)); + CHECK(view.nb[0] == 1u); + CHECK(view.nb[1] > 0u); + + tensor.type = emel::kernel::detail::dtype_q4_k_x8_bl4; + tensor.dims[0] = 256; + tensor.dims[1] = 8; + REQUIRE(detail::bind_tensor_view(tensor, view)); + CHECK(view.nb[1] > 0u); + CHECK(view.nb[2] > 0u); + + tensor.dims[0] = 255; + CHECK_FALSE(detail::bind_tensor_view(tensor, view)); + + tensor.type = emel::kernel::detail::dtype_q4_0; + tensor.dims[0] = 1; + tensor.dims[1] = 2; + CHECK_FALSE(detail::bind_tensor_view(tensor, view)); + + tensor.data = nullptr; + CHECK_FALSE(detail::bind_tensor_view(tensor, view)); +} + +TEST_CASE("speech Moshi executor classifies supported operand dtypes") { + CHECK( + detail::supported_get_rows_dtype(dtype(emel::kernel::detail::dtype_f32))); + CHECK( + detail::supported_get_rows_dtype(dtype(emel::kernel::detail::dtype_f16))); + CHECK(detail::supported_get_rows_dtype( + dtype(emel::kernel::detail::dtype_bf16))); + CHECK(detail::supported_get_rows_dtype( + dtype(emel::kernel::detail::dtype_q4_0))); + CHECK(detail::supported_get_rows_dtype( + dtype(emel::kernel::detail::dtype_q8_0))); + CHECK(detail::supported_get_rows_dtype( + dtype(emel::kernel::detail::dtype_q4_k))); + CHECK_FALSE( + detail::supported_get_rows_dtype(dtype(emel::kernel::detail::dtype_i32))); + + CHECK( + detail::supported_argmax_dtype(dtype(emel::kernel::detail::dtype_q4_k))); + CHECK(detail::supported_argmax_dtype( + dtype(emel::kernel::detail::dtype_q4_k_x8_bl4))); + CHECK_FALSE( + detail::supported_argmax_dtype(dtype(emel::kernel::detail::dtype_i32))); + + CHECK( + detail::supported_mul_mat_dtype(dtype(emel::kernel::detail::dtype_f16))); + CHECK( + detail::supported_mul_mat_dtype(dtype(emel::kernel::detail::dtype_q8_0))); + CHECK(detail::supported_mul_mat_dtype( + dtype(emel::kernel::detail::dtype_q4_k_x8_bl8))); + CHECK_FALSE( + detail::supported_mul_mat_dtype(dtype(emel::kernel::detail::dtype_i32))); +} diff --git a/tests/speech/predictor/moshi_executor_streaming_tests.cpp b/tests/speech/predictor/moshi_executor_streaming_tests.cpp new file mode 100644 index 00000000..3a05f84f --- /dev/null +++ b/tests/speech/predictor/moshi_executor_streaming_tests.cpp @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include + +#include + +#include "emel/error/error.hpp" +#include "emel/memory/streaming/sm.hpp" +#include "emel/speech/predictor/moshi/any.hpp" +#include "emel/speech/predictor/moshi/executor/any.hpp" +#include "moshi_fixture.hpp" + +namespace { + +namespace memory_streaming = emel::memory::streaming; +namespace moshi = emel::speech::predictor::moshi; +namespace executor = emel::speech::predictor::moshi::executor; +using emel::speech::predictor::moshi::test::load_fixture_or_skip; + +struct cache_storage { + cache_storage(const int32_t layer_count, const int32_t capacity, + const int32_t dimension) + : key(static_cast(layer_count) * static_cast(capacity) * + static_cast(dimension)), + value(key.size()), offsets(static_cast(layer_count)) { + const size_t per_layer = + static_cast(capacity) * static_cast(dimension); + for (int32_t layer = 0; layer < layer_count; ++layer) { + offsets[static_cast(layer)] = + static_cast(layer) * per_layer; + } + } + + std::vector key; + std::vector value; + std::vector offsets; +}; + +memory_streaming::window_view capture(memory_streaming::sm &machine) { + memory_streaming::window_view view = {}; + int32_t err = + static_cast(emel::error::cast(memory_streaming::error::none)); + REQUIRE(machine.process_event(memory_streaming::event::capture_view{ + .view_out = view, .error_out = err})); + REQUIRE(err == static_cast( + emel::error::cast(memory_streaming::error::none))); + return view; +} + +} // namespace + +TEST_CASE( + "speech Moshi executor delegates ring positions to streaming actors") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + const auto &lm = fixture.model->moshi_lm; + + cache_storage temporal_cache{lm.num_layers, lm.context, lm.dim}; + cache_storage depformer_cache{lm.depformer_num_layers, lm.depformer_context, + lm.depformer_dim}; + memory_streaming::sm temporal_positions{ + memory_streaming::dependencies{.capacity = lm.context}}; + memory_streaming::sm depformer_positions{ + memory_streaming::dependencies{.capacity = lm.depformer_context}}; + int32_t memory_err = + static_cast(emel::error::cast(memory_streaming::error::none)); + REQUIRE(temporal_positions.process_event( + memory_streaming::event::initialize{.error_out = memory_err})); + REQUIRE(memory_err == static_cast( + emel::error::cast(memory_streaming::error::none))); + REQUIRE(depformer_positions.process_event( + memory_streaming::event::initialize{.error_out = memory_err})); + REQUIRE(memory_err == static_cast( + emel::error::cast(memory_streaming::error::none))); + + emel::kernel::sm kernel{}; + executor::sm machine{ + executor::dependencies{ + .kv = + executor::kv_views{ + .temporal = + executor::detail::temporal_kv_view{ + .key_cache = temporal_cache.key, + .value_cache = temporal_cache.value, + .layer_cache_offsets = temporal_cache.offsets, + .layer_count = lm.num_layers, + .position_capacity = lm.context, + .kv_dim = lm.dim, + }, + .depformer = + executor::detail::depformer_kv_view{ + .key_cache = depformer_cache.key, + .value_cache = depformer_cache.value, + .layer_cache_offsets = depformer_cache.offsets, + .layer_count = lm.depformer_num_layers, + .position_capacity = lm.depformer_context, + .kv_dim = lm.depformer_dim, + }, + .temporal_positions = &temporal_positions, + .depformer_positions = &depformer_positions, + }, + .kernel = kernel, + .policy = + executor::action::policies{ + .rms_norm_epsilon = 1.0e-8f, + .zero_seed_state = 123459876u, + .sampling_modulus = 2147483647u, + .token_zero = -1, + }, + .capacity = + executor::action::capacities{ + .hidden_dim = executor::detail::k_max_hidden_dim, + .temporal_context = executor::detail::k_max_temporal_context, + .depformer_context = + executor::detail::k_max_depformer_context, + .sampling_card = executor::detail::k_max_sampling_card, + .sampling_top_k = executor::detail::k_max_sampling_top_k, + }, + }}; + + emel::error::type err = emel::error::cast(executor::error::none); + executor::event::initialize initialize{*fixture.model}; + initialize.error_out = &err; + REQUIRE(machine.process_event(initialize)); + + std::array input = {}; + std::array output = {}; + std::vector temporal_state(static_cast(lm.dim)); + input.fill(-1); + input[0] = lm.text_padding_id; + emel::memory::view::snapshot unused_snapshot = {}; + int32_t text_token = -1; + moshi::event::graph_step step{ + *fixture.model, unused_snapshot, + std::span{input.data(), static_cast(lm.n_q + 1)}, + std::span{output.data(), static_cast(lm.dep_q)}, + text_token}; + step.error_out = &err; + step.temporal_state = temporal_state; + step.phase = moshi::event::graph_phase::prediction; + + REQUIRE(machine.process_event(step)); + CHECK(capture(temporal_positions).logical_end == 1); + CHECK(capture(depformer_positions).logical_end == 0); + CHECK(machine.is(stateforward::sml::state)); + + step.phase = moshi::event::graph_phase::sampling; + REQUIRE(machine.process_event(step)); + CHECK(capture(depformer_positions).logical_end == lm.dep_q); + CHECK(machine.is(stateforward::sml::state)); + + step.phase = moshi::event::graph_phase::prediction; + REQUIRE(machine.process_event(step)); + CHECK(capture(temporal_positions).logical_end == 2); + step.phase = moshi::event::graph_phase::sampling; + REQUIRE(machine.process_event(step)); + CHECK(capture(depformer_positions).logical_end == lm.dep_q); + + REQUIRE(machine.process_event(executor::event::reset{})); + CHECK(capture(temporal_positions).logical_end == 0); + CHECK(capture(depformer_positions).logical_end == 0); + CHECK(machine.is(stateforward::sml::state)); + + REQUIRE(machine.process_event(initialize)); + step.phase = moshi::event::graph_phase::prediction; + REQUIRE(machine.process_event(step)); + CHECK(capture(temporal_positions).logical_end == 1); +} diff --git a/tests/speech/predictor/moshi_fixture.hpp b/tests/speech/predictor/moshi_fixture.hpp new file mode 100644 index 00000000..492c5f6c --- /dev/null +++ b/tests/speech/predictor/moshi_fixture.hpp @@ -0,0 +1,136 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "emel/gguf/loader/detail.hpp" +#include "emel/gguf/loader/events.hpp" +#include "emel/gguf/loader/sm.hpp" +#include "emel/model/detail.hpp" + +namespace emel::speech::predictor::moshi::test { + +struct loaded_fixture { + std::vector file_bytes = {}; + std::vector kv_arena = {}; + std::vector kv_entries = {}; + std::unique_ptr model = {}; + + emel::model::detail::kv_binding binding() const { + return emel::model::detail::kv_binding{ + .arena = std::span{kv_arena}, + .entries = std::span{kv_entries}, + }; + } +}; + +inline std::filesystem::path repo_root() { +#ifdef EMEL_TEST_REPO_ROOT + return std::filesystem::path{EMEL_TEST_REPO_ROOT}; +#else + return std::filesystem::current_path(); +#endif +} + +inline std::filesystem::path fixture_path(const std::string_view name) { + return repo_root() / "tests" / "models" / std::string{name}; +} + +inline std::vector +read_binary_file(const std::filesystem::path &path) { + std::ifstream file{path, std::ios::binary}; + REQUIRE(file.good()); + file.seekg(0, std::ios::end); + const std::streamoff size = file.tellg(); + REQUIRE(size > 0); + std::vector bytes(static_cast(size)); + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(bytes.data()), size); + REQUIRE(file.good()); + return bytes; +} + +inline void noop_probe_done(const emel::gguf::loader::events::probe_done &) {} +inline void noop_probe_error(const emel::gguf::loader::events::probe_error &) {} +inline void noop_bind_done(const emel::gguf::loader::events::bind_done &) {} +inline void noop_bind_error(const emel::gguf::loader::events::bind_error &) {} +inline void noop_parse_done(const emel::gguf::loader::events::parse_done &) {} +inline void noop_parse_error(const emel::gguf::loader::events::parse_error &) {} + +inline void +materialize_tensor_names_from_file(emel::model::data &model, + const std::vector &file) { + model.name_bytes_used = 0; + for (uint32_t index = 0; index < model.n_tensors; ++index) { + auto &tensor = model.tensors[index]; + const size_t source_offset = static_cast(tensor.name_offset); + const size_t length = static_cast(tensor.name_length); + REQUIRE(source_offset + length <= file.size()); + REQUIRE(static_cast(model.name_bytes_used) + length <= + model.name_storage.size()); + std::copy_n(file.data() + source_offset, length, + model.name_storage.data() + model.name_bytes_used); + tensor.name_offset = model.name_bytes_used; + model.name_bytes_used += static_cast(length); + } +} + +inline loaded_fixture load_fixture_or_skip(const std::string_view name) { + const auto path = fixture_path(name); + if (!std::filesystem::exists(path)) { + MESSAGE("skipping Moshi fixture test because fixture is missing: " + << path.string()); + return {}; + } + + loaded_fixture loaded{}; + loaded.model = std::make_unique(); + loaded.file_bytes = read_binary_file(path); + + emel::gguf::loader::sm loader{}; + emel::gguf::loader::requirements requirements{}; + REQUIRE(loader.process_event(emel::gguf::loader::event::probe{ + std::span{loaded.file_bytes}, + requirements, + emel::gguf::loader::event::probe_done_fn::from<&noop_probe_done>(), + emel::gguf::loader::event::probe_error_fn::from<&noop_probe_error>(), + })); + + const uint64_t arena_bytes = + emel::gguf::loader::detail::required_kv_arena_bytes(requirements); + REQUIRE(arena_bytes != UINT64_MAX); + loaded.kv_arena.resize(static_cast(arena_bytes)); + loaded.kv_entries.resize(requirements.kv_count); + loaded.model->n_tensors = requirements.tensor_count; + + REQUIRE(loader.process_event(emel::gguf::loader::event::bind_storage{ + std::span{loaded.kv_arena}, + std::span{loaded.kv_entries}, + std::span{loaded.model->tensors.data(), + loaded.model->n_tensors}, + emel::gguf::loader::event::bind_done_fn::from<&noop_bind_done>(), + emel::gguf::loader::event::bind_error_fn::from<&noop_bind_error>(), + })); + REQUIRE(loader.process_event(emel::gguf::loader::event::parse{ + std::span{loaded.file_bytes}, + emel::gguf::loader::event::parse_done_fn::from<&noop_parse_done>(), + emel::gguf::loader::event::parse_error_fn::from<&noop_parse_error>(), + })); + REQUIRE(emel::model::detail::load_hparams_from_gguf(loaded.binding(), + *loaded.model)); + loaded.model->weights_data = loaded.file_bytes.data(); + loaded.model->weights_size = loaded.file_bytes.size(); + materialize_tensor_names_from_file(*loaded.model, loaded.file_bytes); + return loaded; +} + +} // namespace emel::speech::predictor::moshi::test diff --git a/tests/speech/predictor/moshi_lifecycle_tests.cpp b/tests/speech/predictor/moshi_lifecycle_tests.cpp new file mode 100644 index 00000000..c93ce6cf --- /dev/null +++ b/tests/speech/predictor/moshi_lifecycle_tests.cpp @@ -0,0 +1,2794 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../../memory/recording_kv_actor.hpp" +#include "emel/logits/sampler/sm.hpp" +#include "emel/memory/streaming/sm.hpp" +#include "emel/model/moshi/detail.hpp" +#include "emel/speech/predictor/moshi/any.hpp" +#include "emel/speech/predictor/moshi/executor/any.hpp" +#include "moshi_fixture.hpp" + +namespace { + +namespace moshi = emel::speech::predictor::moshi; +namespace moshi_executor = emel::speech::predictor::moshi::executor; +namespace memory_streaming = emel::memory::streaming; +namespace model_moshi = emel::model::moshi::detail; +using emel::speech::predictor::moshi::test::load_fixture_or_skip; +using emel::speech::predictor::moshi::test::read_binary_file; +using emel::speech::predictor::moshi::test::repo_root; + +inline constexpr emel::error::type k_no_error = + emel::error::cast(moshi::error::none); + +moshi_executor::dependencies make_executor_dependencies( + emel::kernel::sm &kernel, const moshi_executor::kv_views &kv = {}, + emel::logits::sampler::sm *sampler = nullptr) noexcept { + return moshi_executor::dependencies{ + .kv = kv, + .kernel = kernel, + .sampler = sampler, + .policy = + moshi_executor::action::policies{ + .rms_norm_epsilon = 1.0e-8f, + .zero_seed_state = 123459876u, + .sampling_modulus = 2147483647u, + .token_zero = -1, + }, + .capacity = + moshi_executor::action::capacities{ + .hidden_dim = moshi_executor::detail::k_max_hidden_dim, + .temporal_context = + moshi_executor::detail::k_max_temporal_context, + .depformer_context = + moshi_executor::detail::k_max_depformer_context, + .sampling_card = moshi_executor::detail::k_max_sampling_card, + .sampling_top_k = moshi_executor::detail::k_max_sampling_top_k, + }, + }; +} + +template +moshi::action::dependencies +make_predictor_dependencies(graph_actor_type &graph, + emel::memory::hybrid::sm &memory) noexcept { + return moshi::action::dependencies{ + .memory = memory, + .graph = graph, + .policy = + moshi::action::policies{ + .token_zero = -1, + .token_ungenerated = -2, + .prediction_step_size = 1, + .prediction_output_count = 1, + }, + }; +} + +void configure_predictor_initialize(moshi::event::initialize &request) { + int32_t maximum_delay = 0; + for (uint32_t index = 0; index < request.model.moshi_lm.delay_count; + ++index) { + maximum_delay = + std::max(maximum_delay, request.model.moshi_lm.delays[index]); + } + request.max_sequences = 1; + request.block_tokens = 4; + request.max_blocks = request.model.moshi_lm.context / request.block_tokens; + request.sequence_id = 0; + request.codebook_capacity = request.model.moshi_lm.n_q + 1; + request.delay_cache_row_capacity = maximum_delay + 3; +} + +struct recording_graph_executor { + bool initialized = false; + int32_t reset_count = 0; + int32_t call_count = 0; + int32_t external_embedding_count = 0; + int32_t first_sequence_length = -1; + int32_t second_sequence_length = -1; + int32_t last_sequence_length = -1; + int32_t first_forced_text_token = -1; + int32_t last_forced_text_token = -1; + float first_embedding_sample = 0.0f; + std::array first_input = {}; + std::array last_input = {}; + + bool process_event(const moshi::event::initialize_graph &request) noexcept { + if (initialized) { + *request.error_out = emel::error::cast(moshi::error::graph_runtime); + return false; + } + initialized = true; + *request.error_out = k_no_error; + return true; + } + bool process_event(const moshi::event::reset &) noexcept { + initialized = false; + ++reset_count; + return true; + } + bool process_event(const moshi::event::graph_step &step); +}; + +struct rejecting_graph_executor { + bool process_event(const moshi::event::initialize_graph &request) noexcept { + *request.error_out = k_no_error; + return true; + } + bool process_event(const moshi::event::graph_step &) noexcept { + return false; + } +}; + +struct temporal_kv_probe { + int32_t call_count = 0; + int32_t sequence_id = -1; + int32_t sequence_length = -1; + std::vector key_cache = {}; + std::vector value_cache = {}; + std::vector layer_offsets = {}; +}; + +struct depformer_kv_probe { + int32_t call_count = 0; + int32_t sequence_id = -1; + int32_t sequence_length = -1; + std::vector key_cache = {}; + std::vector value_cache = {}; + std::vector layer_offsets = {}; +}; + +struct moshi_callback_probe { + int32_t done_count = 0; + int32_t error_count = 0; + int32_t n_q = -1; + int32_t dep_q = -1; + int32_t prompt_frames = -1; + int32_t remaining_frames = -1; + bool complete = false; + bool produced = false; + emel::error::type err = k_no_error; + const void *request = nullptr; + + void on_generator_initialize_done( + const moshi::events::initialize_done &done) noexcept { + ++done_count; + request = done.request; + n_q = done.n_q; + dep_q = done.dep_q; + } + + void on_generator_initialize_error( + const moshi::events::initialize_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } + + void on_generator_load_voice_done( + const moshi::events::load_voice_done &done) noexcept { + ++done_count; + request = done.request; + prompt_frames = done.prompt_frames; + } + + void on_generator_load_voice_error( + const moshi::events::load_voice_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } + + void on_generator_prefill_voice_done( + const moshi::events::prefill_voice_done &done) noexcept { + ++done_count; + request = done.request; + complete = done.complete; + remaining_frames = done.remaining_frames; + } + + void on_generator_prefill_voice_error( + const moshi::events::prefill_voice_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } + + void on_generator_begin_prompt_done( + const moshi::events::begin_personaplex_prompt_done &done) noexcept { + ++done_count; + request = done.request; + remaining_frames = done.remaining_frames; + } + + void on_generator_begin_prompt_error( + const moshi::events::begin_personaplex_prompt_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } + + void on_generator_prefill_prompt_done( + const moshi::events::prefill_personaplex_prompt_done &done) noexcept { + ++done_count; + request = done.request; + complete = done.complete; + remaining_frames = done.remaining_frames; + } + + void on_generator_prefill_prompt_error( + const moshi::events::prefill_personaplex_prompt_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } + + void on_executor_initialize_done( + const moshi_executor::events::initialize_done &done) noexcept { + ++done_count; + request = done.request; + n_q = done.n_q; + dep_q = done.dep_q; + } + + void on_executor_initialize_error( + const moshi_executor::events::initialize_error &error) noexcept { + ++error_count; + request = error.request; + err = error.err; + } +}; + +bool recording_graph_executor::process_event( + const moshi::event::graph_step &step) { + auto &executor = *this; + ++executor.call_count; + if (!step.input_embedding.empty()) { + ++executor.external_embedding_count; + if (executor.external_embedding_count == 1) { + executor.first_forced_text_token = step.forced_text_token; + executor.first_embedding_sample = step.input_embedding[0]; + } + } + std::fill(executor.last_input.begin(), executor.last_input.end(), -99); + std::copy(step.input_sequence.begin(), step.input_sequence.end(), + executor.last_input.begin()); + executor.last_sequence_length = step.memory_snapshot.sequence_length(0); + executor.last_forced_text_token = step.forced_text_token; + if (executor.call_count == 1) { + std::fill(executor.first_input.begin(), executor.first_input.end(), -99); + std::copy(step.input_sequence.begin(), step.input_sequence.end(), + executor.first_input.begin()); + executor.first_sequence_length = step.memory_snapshot.sequence_length(0); + } + if (executor.call_count == 2) { + executor.second_sequence_length = step.memory_snapshot.sequence_length(0); + } + if (step.error_out != nullptr) { + *step.error_out = k_no_error; + } + step.text_token_out = 100 + executor.call_count; + for (size_t index = 0; index < step.audio_tokens_out.size(); ++index) { + step.audio_tokens_out[index] = static_cast( + 1000 + executor.call_count * 10 + static_cast(index)); + } + return true; +} + +bool temporal_kv_probe_bind(void *cache_ptr, const emel::model::data &model, + const emel::memory::view::snapshot &snapshot, + const int32_t sequence_id, + moshi_executor::detail::temporal_kv_view &view) { + auto &probe = *static_cast(cache_ptr); + ++probe.call_count; + probe.sequence_id = sequence_id; + probe.sequence_length = snapshot.sequence_length(sequence_id); + const int32_t position_capacity = model.moshi_lm.context; + const int32_t kv_dim = model.moshi_lm.dim; + const int32_t layer_count = model.moshi_lm.num_layers; + REQUIRE(position_capacity > 0); + REQUIRE(kv_dim > 0); + REQUIRE(layer_count > 0); + const size_t per_layer = + static_cast(position_capacity) * static_cast(kv_dim); + probe.layer_offsets.resize(static_cast(layer_count)); + for (int32_t layer = 0; layer < layer_count; ++layer) { + probe.layer_offsets[static_cast(layer)] = + static_cast(layer) * per_layer; + } + probe.key_cache.assign(static_cast(layer_count) * per_layer, 0u); + probe.value_cache.assign(probe.key_cache.size(), 0u); + view.key_cache = + std::span{probe.key_cache.data(), probe.key_cache.size()}; + view.value_cache = + std::span{probe.value_cache.data(), probe.value_cache.size()}; + view.layer_cache_offsets = probe.layer_offsets; + view.layer_count = layer_count; + view.position_capacity = position_capacity; + view.kv_dim = kv_dim; + return true; +} + +bool depformer_kv_probe_bind(void *cache_ptr, const emel::model::data &model, + const emel::memory::view::snapshot &snapshot, + const int32_t sequence_id, + moshi_executor::detail::depformer_kv_view &view) { + auto &probe = *static_cast(cache_ptr); + ++probe.call_count; + probe.sequence_id = sequence_id; + probe.sequence_length = snapshot.sequence_length(sequence_id); + const int32_t position_capacity = model.moshi_lm.depformer_context; + const int32_t kv_dim = model.moshi_lm.depformer_dim; + const int32_t layer_count = model.moshi_lm.depformer_num_layers; + REQUIRE(position_capacity > 0); + REQUIRE(kv_dim > 0); + REQUIRE(layer_count > 0); + const size_t per_layer = + static_cast(position_capacity) * static_cast(kv_dim); + if (probe.key_cache.empty()) { + probe.layer_offsets.resize(static_cast(layer_count)); + for (int32_t layer = 0; layer < layer_count; ++layer) { + probe.layer_offsets[static_cast(layer)] = + static_cast(layer) * per_layer; + } + probe.key_cache.assign(static_cast(layer_count) * per_layer, 0u); + probe.value_cache.assign(probe.key_cache.size(), 0u); + } + view.key_cache = + std::span{probe.key_cache.data(), probe.key_cache.size()}; + view.value_cache = + std::span{probe.value_cache.data(), probe.value_cache.size()}; + view.layer_cache_offsets = probe.layer_offsets; + view.layer_count = layer_count; + view.position_capacity = position_capacity; + view.kv_dim = kv_dim; + return true; +} + +moshi_executor::kv_views +prepare_kv_views(const emel::model::data &model, temporal_kv_probe *temporal, + depformer_kv_probe *depformer, + memory_streaming::sm *temporal_positions = nullptr, + memory_streaming::sm *depformer_positions = nullptr) { + moshi_executor::kv_views views{ + .temporal_positions = temporal_positions, + .depformer_positions = depformer_positions, + }; + emel::memory::view::snapshot snapshot{}; + if (temporal != nullptr) { + REQUIRE( + temporal_kv_probe_bind(temporal, model, snapshot, 0, views.temporal)); + } + if (depformer != nullptr) { + REQUIRE(depformer_kv_probe_bind(depformer, model, snapshot, 0, + views.depformer)); + } + return views; +} + +void initialize_streaming_position(memory_streaming::sm &position) { + int32_t err = + static_cast(emel::error::cast(memory_streaming::error::none)); + REQUIRE(position.process_event( + memory_streaming::event::initialize{.error_out = err})); + REQUIRE(err == static_cast( + emel::error::cast(memory_streaming::error::none))); +} + +} // namespace + +TEST_CASE("speech_moshi_predictor_rejects_predict_before_initialize") { + recording_graph_executor graph{}; + emel::memory::hybrid::sm memory{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + std::array input = {0, 0, 0, 0}; + moshi::event::predict::workspace workspace{}; + emel::error::type err = k_no_error; + + moshi::event::predict predict{std::span{input}, workspace, 1, + 1}; + predict.error_out = &err; + + CHECK_FALSE(generator.process_event(predict)); + CHECK(err == emel::error::cast(moshi::error::not_initialized)); +} + +TEST_CASE("speech_moshi_generator_rejects_non_lm_model_contract") { + auto model = std::make_unique(); + model->moshi_component_id = emel::model::data::moshi_component::mimi; + recording_graph_executor graph{}; + emel::memory::hybrid::sm memory{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*model}; + init.error_out = &err; + + CHECK_FALSE(generator.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); +} + +TEST_CASE("speech_moshi_generator_initializes_lm_with_injected_kv") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + REQUIRE(model_moshi::validate_execution_contract(*fixture.model) == + emel::error::cast(emel::model::loader::error::none)); + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_sequences = 2; + init.max_blocks = fixture.model->moshi_lm.context / init.block_tokens; + init.block_tokens = 4; + init.error_out = &err; + + REQUIRE(generator.process_event(init)); + CHECK(err == k_no_error); + CHECK(kv.reserve_count == 1); + CHECK(kv.allocate_sequence_count == 1); + + REQUIRE(generator.process_event(moshi::event::reset{})); + CHECK(graph.reset_count == 1); + CHECK(kv.free_sequence_count == 1); + err = k_no_error; + REQUIRE(generator.process_event(init)); + CHECK(err == k_no_error); + CHECK(kv.allocate_sequence_count == 2); +} + +TEST_CASE("speech_moshi_predictor_rejects_missing_initialize_capacity") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + recording_graph_executor graph{}; + emel::memory::hybrid::sm memory{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + init.error_out = &err; + + CHECK_FALSE(generator.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); +} + +TEST_CASE("speech_moshi_predictor_rejects_missing_token_policy") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + recording_graph_executor graph{}; + emel::memory::hybrid::sm memory{}; + moshi::sm generator{ + moshi::action::dependencies{.memory = memory, .graph = graph}}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.error_out = &err; + + CHECK_FALSE(generator.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); +} + +TEST_CASE("speech_moshi_predictor_bounds_delay_count_before_scanning") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + auto model = std::make_unique(*fixture.model); + model->moshi_lm.delay_count = + static_cast(model->moshi_lm.delays.size() + 1u); + moshi::event::initialize request{*model}; + moshi::event::initialize_ctx request_ctx{}; + moshi::event::initialize_run runtime{request, request_ctx}; + emel::memory::hybrid::sm memory{}; + moshi::action::context ctx{memory}; + CHECK_FALSE(moshi::guard::guard_bind_contract_valid{}(runtime, ctx)); +} + +TEST_CASE("speech_moshi_predictor_rejects_negative_model_delays") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + auto model = std::make_unique(*fixture.model); + REQUIRE(model->moshi_lm.delay_count > 1u); + model->moshi_lm.delays[1] = -1; + recording_graph_executor graph{}; + emel::memory::hybrid::sm memory{}; + moshi::sm predictor{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize request{*model}; + configure_predictor_initialize(request); + request.error_out = &err; + CHECK_FALSE(predictor.process_event(request)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); +} + +TEST_CASE("speech_moshi_generator_personaplex_uses_model_inference_codebooks") { + auto model = std::make_unique(); + model->moshi_lm.n_q = 16; + model->moshi_lm.dep_q = 16; + model->moshi_lm.inference_dep_q = 8; + model->moshi_lm.text_padding_id = 3; + model->moshi_lm.depformer_weights_per_step = true; + model->moshi_lm.delay_count = 17; + model->moshi_lm.inference_prompt_token_count = 17; + model->moshi_lm.inference_prompt_tokens[0] = 3; + model->moshi_lm.delays[0] = 0; + for (uint32_t index = 1; index < model->moshi_lm.delay_count; ++index) { + model->moshi_lm.delays[index] = 1; + model->moshi_lm.inference_prompt_tokens[index] = + static_cast(index); + } + + moshi::event::initialize init{*model}; + moshi::event::initialize_ctx init_ctx{}; + moshi::event::initialize_run run{init, init_ctx}; + emel::memory::hybrid::sm memory{}; + moshi::action::context ctx{memory}; + + moshi::action::effect_bind_contract{}(run, ctx); + CHECK(moshi::guard::guard_personaplex_lmgen{}(run, ctx)); + moshi::action::effect_configure_personaplex_lmgen{}(run, ctx); + + CHECK(ctx.lmgen.generated_dep_q == 16); + CHECK(ctx.lmgen.delayed_dep_q == model->moshi_lm.inference_dep_q); + CHECK(ctx.lmgen.needed_tokens == 8); + CHECK(ctx.lmgen.cache_row_count == 4); +} + +TEST_CASE( + "speech_moshi_generator_loads_personaplex_voice_and_prefills_prompt") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + auto voice = load_fixture_or_skip("moshi-tiny-voice.gguf"); + if (fixture.model == nullptr || voice.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_blocks = fixture.model->moshi_lm.context / init.block_tokens; + init.block_tokens = 4; + init.error_out = &err; + REQUIRE(generator.process_event(init)); + CHECK(err == k_no_error); + + const int32_t prompt_frames = 5; + moshi::event::load_voice load{*voice.model}; + load.error_out = &err; + REQUIRE(generator.process_event(load)); + CHECK(err == k_no_error); + + std::array model_tokens = {}; + moshi::event::predict::workspace workspace{}; + moshi::event::predict blocked_predict{ + std::span{ + model_tokens.data(), + static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + blocked_predict.error_out = &err; + CHECK_FALSE(generator.process_event(blocked_predict)); + CHECK(err == emel::error::cast(moshi::error::voice_prompt_pending)); + + for (int32_t index = 0; index < prompt_frames; ++index) { + bool complete = true; + int32_t remaining = -1; + emel::error::type graph_err = emel::error::cast(moshi::error::none); + err = k_no_error; + moshi::event::prefill_voice prefill{}; + prefill.error_out = &err; + prefill.graph_error_out = &graph_err; + prefill.complete_out = &complete; + prefill.remaining_frames_out = &remaining; + REQUIRE(generator.process_event(prefill)); + CHECK(err == k_no_error); + CHECK(graph_err == k_no_error); + CHECK(complete == (index + 1 == prompt_frames)); + CHECK(remaining == prompt_frames - index - 1); + } + + CHECK(graph.external_embedding_count == prompt_frames); + CHECK(graph.first_forced_text_token == + fixture.model->moshi_lm.text_padding_id); + CHECK(graph.first_embedding_sample != 0.0f); + CHECK(kv.allocate_slots_count >= prompt_frames); + CHECK(kv.capture_view_count >= prompt_frames); +} + +TEST_CASE("speech_moshi_generator_loads_personaplex_voice_embedding_dtypes") { + auto snapshot = std::make_unique(); + moshi::event::prefill_voice prefill{}; + moshi::event::prefill_voice_ctx prefill_ctx{*snapshot}; + moshi::event::prefill_voice_run prefill_run{prefill, prefill_ctx}; + emel::memory::hybrid::sm memory{}; + moshi::action::context ctx{memory}; + emel::model::data::tensor_record tensor{}; + std::array data = {0x3c00u, 0x4000u, 0x4200u, 0x4400u}; + tensor.data = data.data(); + tensor.dims[0] = 2; + tensor.dims[3] = 2; + ctx.voice.contract.voice.embeddings.tensor = &tensor; + ctx.voice.embedding_dim = 2; + ctx.voice.prompt_frame_index = 1; + + tensor.type = emel::kernel::detail::dtype_f16; + moshi::action::effect_load_voice_embedding_frame_f16{}(prefill_run, ctx); + CHECK(prefill_ctx.embedding_frame_ok); + CHECK(prefill_ctx.embedding_frame[0] != 0.0f); + CHECK(prefill_ctx.embedding_frame[1] != 0.0f); + + prefill_ctx.embedding_frame_ok = false; + prefill_ctx.embedding_frame[0] = 0.0f; + prefill_ctx.embedding_frame[1] = 0.0f; + tensor.type = emel::kernel::detail::dtype_bf16; + moshi::action::effect_load_voice_embedding_frame_bf16{}(prefill_run, ctx); + CHECK(prefill_ctx.embedding_frame_ok); + CHECK(prefill_ctx.embedding_frame[0] != 0.0f); + CHECK(prefill_ctx.embedding_frame[1] != 0.0f); +} + +TEST_CASE("speech_moshi_generator_prefills_personaplex_system_prompt_before_" + "generation") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + auto voice = load_fixture_or_skip("moshi-tiny-voice.gguf"); + if (fixture.model == nullptr || voice.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_blocks = fixture.model->moshi_lm.context / init.block_tokens; + init.block_tokens = 4; + init.error_out = &err; + REQUIRE(generator.process_event(init)); + CHECK(err == k_no_error); + + moshi::event::load_voice load{*voice.model}; + load.error_out = &err; + REQUIRE(generator.process_event(load)); + CHECK(err == k_no_error); + + const int32_t prompt_frames = 5; + for (int32_t index = 0; index < prompt_frames; ++index) { + bool complete = false; + int32_t remaining = -1; + moshi::event::prefill_voice prefill{}; + prefill.error_out = &err; + prefill.complete_out = &complete; + prefill.remaining_frames_out = &remaining; + REQUIRE(generator.process_event(prefill)); + CHECK(err == k_no_error); + CHECK(complete == (index + 1 == prompt_frames)); + CHECK(remaining == prompt_frames - index - 1); + } + + const int32_t needed_tokens = (fixture.model->moshi_lm.n_q + 1) - + fixture.model->moshi_lm.inference_dep_q - 1; + REQUIRE(needed_tokens >= 0); + std::array input = {}; + std::array output = {}; + moshi::event::predict::workspace workspace{}; + std::fill(input.begin(), input.end(), 0); + int32_t text_token = -1; + err = k_no_error; + moshi::event::predict blocked_predict{ + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + blocked_predict.error_out = &err; + CHECK_FALSE(generator.process_event(blocked_predict)); + CHECK(err == emel::error::cast(moshi::error::voice_prompt_pending)); + + err = k_no_error; + moshi::event::begin_personaplex_prompt begin{}; + begin.text_token_count = 2; + begin.pre_text_silence_frames = 1; + begin.post_text_silence_frames = 1; + begin.error_out = &err; + REQUIRE(generator.process_event(begin)); + CHECK(err == k_no_error); + + const std::array prompt_text_tokens = {-1, 42, 43, -1}; + const std::array expected_text_inputs = {3, 42, 43, 3}; + for (int32_t index = 0; index < 4; ++index) { + if (index == 1) { + const int32_t allocations_before = kv.allocate_slots_count; + const int32_t graph_calls_before = graph.call_count; + moshi::event::prefill_personaplex_prompt invalid_prefill{}; + invalid_prefill.text_token = fixture.model->moshi_lm.text_card; + invalid_prefill.error_out = &err; + err = k_no_error; + CHECK_FALSE(generator.process_event(invalid_prefill)); + CHECK(err == emel::error::cast(moshi::error::personaplex_prompt)); + CHECK(kv.allocate_slots_count == allocations_before); + CHECK(graph.call_count == graph_calls_before); + } + bool complete = true; + int32_t remaining = -1; + emel::error::type graph_err = emel::error::cast(moshi::error::none); + err = k_no_error; + moshi::event::prefill_personaplex_prompt prefill{}; + prefill.text_token = prompt_text_tokens[static_cast(index)]; + prefill.error_out = &err; + prefill.graph_error_out = &graph_err; + prefill.complete_out = &complete; + prefill.remaining_frames_out = &remaining; + REQUIRE(generator.process_event(prefill)); + CHECK(err == k_no_error); + CHECK(graph_err == k_no_error); + CHECK(complete == (index == 3)); + CHECK(remaining == 3 - index); + CHECK(graph.last_input[0] == + expected_text_inputs[static_cast(index)]); + CHECK(graph.last_forced_text_token == -1); + } + + CHECK(graph.external_embedding_count == prompt_frames); + CHECK(graph.call_count == prompt_frames + 4); + CHECK(kv.allocate_slots_count >= prompt_frames + 4); + CHECK(kv.capture_view_count >= prompt_frames + 4); + + text_token = -1; + err = k_no_error; + emel::error::type graph_err = emel::error::cast(moshi::error::graph_runtime); + moshi::event::predict first_predict{ + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + first_predict.error_out = &err; + REQUIRE(generator.process_event(first_predict)); + CHECK(generator.is(stateforward::sml::state)); + moshi::event::execute first_execute{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}}; + first_execute.error_out = &err; + first_execute.graph_error_out = &graph_err; + REQUIRE(generator.process_event(first_execute)); + CHECK(generator.is(stateforward::sml::state)); + moshi::event::sample undersized_sample{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), static_cast( + fixture.model->moshi_lm.dep_q - 1)}, + text_token}; + undersized_sample.error_out = &err; + CHECK_FALSE(generator.process_event(undersized_sample)); + CHECK(err == emel::error::cast(moshi::error::request_shape)); + CHECK(generator.is(stateforward::sml::state)); + err = k_no_error; + moshi::event::sample first_sample{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + first_sample.error_out = &err; + first_sample.graph_error_out = &graph_err; + REQUIRE(generator.process_event(first_sample)); + CHECK(generator.is(stateforward::sml::state)); + CHECK(err == k_no_error); + CHECK(graph_err == k_no_error); + CHECK(graph.call_count == prompt_frames + 6); +} + +TEST_CASE("speech_moshi_predictor_rejects_memory_geometry_beyond_temporal_kv") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_blocks = fixture.model->moshi_lm.context; + init.block_tokens = 2; + init.error_out = &err; + + CHECK_FALSE(generator.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); + CHECK(kv.reserve_count == 0); + CHECK_FALSE(graph.initialized); +} + +TEST_CASE("speech_moshi_predictor_rejects_memory_actor_capacity_before_graph") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + emel::memory::hybrid::sm memory{}; + recording_graph_executor graph{}; + moshi::sm predictor{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_sequences = emel::memory::view::MAX_SEQUENCES + 1; + init.error_out = &err; + CHECK_FALSE(predictor.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); + CHECK_FALSE(graph.initialized); +} + +TEST_CASE( + "speech_moshi_predictor_bounds_codebooks_before_delay_slot_addition") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + auto model = std::make_unique(*fixture.model); + model->moshi_lm.n_q = std::numeric_limits::max(); + emel::memory::hybrid::sm memory{}; + recording_graph_executor graph{}; + moshi::sm predictor{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*model}; + init.max_sequences = 1; + init.max_blocks = 1; + init.block_tokens = 1; + init.codebook_capacity = 1; + init.delay_cache_row_capacity = 1; + init.error_out = &err; + CHECK_FALSE(predictor.process_event(init)); + CHECK(err == emel::error::cast(moshi::error::bind_failed)); + CHECK_FALSE(graph.initialized); +} + +TEST_CASE( + "speech_moshi_executor_init_honors_shared_depformer_weight_schedule") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + auto &lm = fixture.model->moshi_lm; + REQUIRE(lm.dep_q >= 2); + + emel::kernel::sm kernel{}; + moshi_executor::action::context ctx{make_executor_dependencies(kernel)}; + moshi_executor::event::initialize request{*fixture.model}; + moshi_executor::event::initialize_ctx request_ctx{}; + moshi_executor::event::initialize_run runtime_ev{request, request_ctx}; + moshi_executor::action::effect_bind_contract{}(runtime_ev, ctx); + REQUIRE(moshi_executor::guard::guard_bound_root_operands_supported{}( + runtime_ev, ctx)); + + lm.depformer_weight_schedule_count = static_cast(lm.dep_q); + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + lm.depformer_weight_schedule[static_cast(codebook)] = codebook; + } + lm.depformer_weight_schedule[static_cast(lm.dep_q - 1)] = + lm.dep_q - 2; + ctx.session.contract.lm + .depformer_input_projections[static_cast(lm.dep_q - 1)] + .tensor = nullptr; + + CHECK(moshi_executor::guard::guard_bound_root_operands_supported{}(runtime_ev, + ctx)); + CHECK_FALSE( + moshi_executor::guard::guard_temporal_fused_projection_layout_supported{}( + runtime_ev, ctx)); + CHECK_FALSE( + moshi_executor::guard:: + guard_depformer_fused_projection_layout_supported{}(runtime_ev, ctx)); + for (int32_t layer = 0; layer < lm.depformer_num_layers; ++layer) { + auto &unused = + ctx.session.contract.lm.depformer_layers[static_cast(layer)] + .codebooks[static_cast(lm.dep_q - 1)]; + unused.output_projection.tensor = nullptr; + unused.gating_input.tensor = nullptr; + unused.gating_output.tensor = nullptr; + unused.split_input_projection.tensor = nullptr; + unused.fused_input_projection.tensor = nullptr; + } + CHECK(moshi_executor::guard::guard_bound_root_operands_supported{}(runtime_ev, + ctx)); + CHECK_FALSE( + moshi_executor::guard::guard_depformer_projection_layout_unsupported{}( + runtime_ev, ctx)); + + lm.depformer_weight_schedule_count = 1u; + CHECK_FALSE(moshi_executor::guard::guard_bound_root_operands_supported{}( + runtime_ev, ctx)); + CHECK(moshi_executor::guard::guard_depformer_projection_layout_unsupported{}( + runtime_ev, ctx)); + + lm.depformer_weight_schedule_count = static_cast(lm.dep_q); + lm.depformer_weight_schedule[0] = lm.dep_q; + CHECK_FALSE(moshi_executor::guard::guard_bound_root_operands_supported{}( + runtime_ev, ctx)); + CHECK(moshi_executor::guard::guard_depformer_projection_layout_unsupported{}( + runtime_ev, ctx)); +} + +TEST_CASE("speech_moshi_predictor_rejects_tokenizer_capture_outside_ready") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm predictor{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.error_out = &err; + REQUIRE(predictor.process_event(init)); + std::array tokens{}; + moshi::event::predict::workspace workspace{}; + moshi::event::predict predict{ + std::span{ + tokens.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + predict.error_out = &err; + REQUIRE(predictor.process_event(predict)); + std::array + cache{}; + int64_t offset = -1; + err = k_no_error; + CHECK_FALSE(predictor.process_event(moshi::event::capture_tokenizer_state{ + std::span{cache}, offset, err})); + CHECK(err == emel::error::cast(moshi::error::request_shape)); + CHECK(offset == 0); +} + +TEST_CASE("speech_moshi_generator_accepts_personaplex_voice_without_system_" + "prompt") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + auto voice = load_fixture_or_skip("moshi-tiny-voice.gguf"); + if (fixture.model == nullptr || voice.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + recording_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.max_blocks = fixture.model->moshi_lm.context / init.block_tokens; + init.block_tokens = 4; + init.error_out = &err; + REQUIRE(generator.process_event(init)); + CHECK(err == k_no_error); + + moshi::event::load_voice load{*voice.model}; + load.error_out = &err; + REQUIRE(generator.process_event(load)); + CHECK(err == k_no_error); + + const int32_t prompt_frames = 5; + for (int32_t index = 0; index < prompt_frames; ++index) { + bool complete = false; + moshi::event::prefill_voice prefill{}; + prefill.error_out = &err; + prefill.complete_out = &complete; + REQUIRE(generator.process_event(prefill)); + CHECK(err == k_no_error); + CHECK(complete == (index + 1 == prompt_frames)); + } + + moshi::event::begin_personaplex_prompt begin{}; + begin.error_out = &err; + err = k_no_error; + REQUIRE(generator.process_event(begin)); + CHECK(err == k_no_error); + + const int32_t needed_tokens = (fixture.model->moshi_lm.n_q + 1) - + fixture.model->moshi_lm.inference_dep_q - 1; + REQUIRE(needed_tokens >= 0); + std::array input = {}; + std::array output = {}; + moshi::event::predict::workspace workspace{}; + std::fill(input.begin(), input.end(), 0); + int32_t text_token = -1; + emel::error::type graph_err = emel::error::cast(moshi::error::graph_runtime); + moshi::event::predict predict{ + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + predict.error_out = &err; + + err = k_no_error; + CHECK_FALSE(generator.process_event(predict)); + CHECK(err == emel::error::cast(moshi::error::voice_prompt_pending)); + + const int32_t personaplex_prompt_frames = + fixture.model->moshi_lm.inference_pre_text_silence_frames + + fixture.model->moshi_lm.inference_post_text_silence_frames; + for (int32_t index = 0; index < personaplex_prompt_frames; ++index) { + bool complete = true; + int32_t remaining = -1; + graph_err = emel::error::cast(moshi::error::graph_runtime); + err = k_no_error; + moshi::event::prefill_personaplex_prompt prefill{}; + prefill.text_token = -1; + prefill.error_out = &err; + prefill.graph_error_out = &graph_err; + prefill.complete_out = &complete; + prefill.remaining_frames_out = &remaining; + + REQUIRE(generator.process_event(prefill)); + CHECK(err == k_no_error); + CHECK(graph_err == k_no_error); + CHECK(complete == (index + 1 == personaplex_prompt_frames)); + CHECK(remaining == personaplex_prompt_frames - index - 1); + CHECK(graph.last_input[0] == fixture.model->moshi_lm.text_padding_id); + CHECK(graph.last_forced_text_token == -1); + } + + err = k_no_error; + graph_err = emel::error::cast(moshi::error::graph_runtime); + REQUIRE(generator.process_event(predict)); + moshi::event::execute execute{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}}; + execute.error_out = &err; + execute.graph_error_out = &graph_err; + REQUIRE(generator.process_event(execute)); + moshi::event::sample sample{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + sample.error_out = &err; + sample.graph_error_out = &graph_err; + REQUIRE(generator.process_event(sample)); + CHECK(err == k_no_error); + CHECK(graph_err == k_no_error); + CHECK(graph.call_count == prompt_frames + personaplex_prompt_frames + 2); +} + +TEST_CASE("speech_moshi_predictor_reports_rejected_graph_event") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + rejecting_graph_executor graph{}; + moshi::sm generator{make_predictor_dependencies(graph, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.error_out = &err; + REQUIRE(generator.process_event(init)); + + std::array input = {}; + moshi::event::predict::workspace workspace{}; + moshi::event::execute premature_execute{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}}; + premature_execute.error_out = &err; + CHECK_FALSE(generator.process_event(premature_execute)); + CHECK(err == emel::error::cast(moshi::error::request_shape)); + + std::array premature_output = {}; + int32_t premature_text_token = -1; + err = k_no_error; + moshi::event::sample premature_sample{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{premature_output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + premature_text_token}; + premature_sample.error_out = &err; + CHECK_FALSE(generator.process_event(premature_sample)); + CHECK(err == emel::error::cast(moshi::error::request_shape)); + + err = k_no_error; + moshi::event::predict predict{ + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + predict.error_out = &err; + + REQUIRE(generator.process_event(predict)); + moshi::event::execute execute{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}}; + execute.error_out = &err; + + CHECK_FALSE(generator.process_event(execute)); + CHECK(err == emel::error::cast(moshi::error::graph_runtime)); + CHECK(kv.allocate_slots_count == 1); + CHECK(kv.capture_view_count == 1); +} + +TEST_CASE("speech_moshi_executor_initializes_lm_fixture") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + emel::logits::sampler::sm sampler{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, {}, &sampler)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + + REQUIRE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::none)); +} + +TEST_CASE("speech_moshi_executor_rejects_missing_numeric_policy") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + moshi_executor::sm executor{moshi_executor::dependencies{.kernel = kernel}}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + + CHECK_FALSE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::bind_failed)); +} + +TEST_CASE("speech_moshi_executor_rejects_missing_capacity") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + auto dependencies = make_executor_dependencies(kernel); + dependencies.capacity.hidden_dim = 0u; + moshi_executor::sm executor{dependencies}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + + CHECK_FALSE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::bind_failed)); +} + +TEST_CASE("speech_moshi_executor_rejects_undersized_hidden_capacity") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + emel::kernel::sm kernel{}; + auto dependencies = make_executor_dependencies(kernel); + dependencies.capacity.hidden_dim = + static_cast(fixture.model->moshi_lm.dim - 1); + moshi_executor::sm executor{dependencies}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + CHECK_FALSE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::bind_failed)); +} + +TEST_CASE("speech_moshi_executor_rejects_undersized_sampling_capacity") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + const auto rejected = [&fixture](const uint64_t sampling_card, + const uint64_t sampling_top_k) { + emel::kernel::sm kernel{}; + emel::logits::sampler::sm sampler{}; + auto dependencies = make_executor_dependencies(kernel, {}, &sampler); + dependencies.capacity.sampling_card = sampling_card; + dependencies.capacity.sampling_top_k = sampling_top_k; + moshi_executor::sm executor{dependencies}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.sampling_enabled = true; + init.sampling_audio_temperature = 0.8f; + init.sampling_text_temperature = 0.7f; + init.sampling_audio_top_k = 2; + init.sampling_text_top_k = 2; + init.sampling_seed = 1234u; + init.error_out = &err; + return !executor.process_event(init) && + err == emel::error::cast(moshi_executor::error::bind_failed); + }; + + CHECK(rejected(static_cast(fixture.model->moshi_lm.text_card - 1), + 2u)); + CHECK(rejected(moshi_executor::detail::k_max_sampling_card, 1u)); +} + +TEST_CASE("speech_moshi_executor_models_zero_seed_and_top_k_clamp") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + emel::logits::sampler::sm sampler{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, {}, &sampler)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.sampling_enabled = true; + init.sampling_audio_temperature = 0.8f; + init.sampling_text_temperature = 0.7f; + init.sampling_audio_top_k = 1; + init.sampling_text_top_k = fixture.model->moshi_lm.text_card + 1; + init.sampling_seed = 0u; + init.error_out = &err; + + REQUIRE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::none)); +} + +TEST_CASE("speech_moshi_executor_normalizes_nonzero_sampling_seed") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + auto dependencies = make_executor_dependencies(kernel); + moshi_executor::action::context ctx{dependencies}; + moshi_executor::event::initialize init{*fixture.model}; + init.sampling_seed = 2147483647u; + moshi_executor::event::initialize_ctx init_ctx{}; + moshi_executor::event::initialize_run init_run{init, init_ctx}; + + moshi_executor::action::effect_bind_nonzero_sampling_seed{}(init_run, ctx); + CHECK(ctx.sampling.random_state > 0u); + CHECK(ctx.sampling.random_state < 2147483647u); +} + +TEST_CASE("speech_moshi_executor_rejects_invalid_sampling_configuration") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + const auto rejected = + [&fixture](const float audio_temperature, const float text_temperature, + const int32_t audio_top_k, const int32_t text_top_k) { + emel::kernel::sm kernel{}; + emel::logits::sampler::sm sampler{}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, {}, &sampler)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.sampling_enabled = true; + init.sampling_audio_temperature = audio_temperature; + init.sampling_text_temperature = text_temperature; + init.sampling_audio_top_k = audio_top_k; + init.sampling_text_top_k = text_top_k; + init.sampling_seed = 1234u; + init.error_out = &err; + return !executor.process_event(init) && + err == emel::error::cast(moshi_executor::error::bind_failed); + }; + + CHECK(rejected(std::numeric_limits::quiet_NaN(), 0.7f, 1, 1)); + CHECK(rejected(0.8f, std::numeric_limits::quiet_NaN(), 1, 1)); + CHECK(rejected(0.8f, 0.7f, 0, 1)); + CHECK(rejected(0.8f, 0.7f, 1, 0)); +} + +TEST_CASE("speech_moshi_machines_report_unexpected_events_after_init") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel)}; + emel::error::type executor_err = k_no_error; + moshi_executor::event::initialize executor_init{*fixture.model}; + executor_init.error_out = &executor_err; + REQUIRE(executor.process_event(executor_init)); + executor_err = k_no_error; + CHECK_FALSE(executor.process_event(executor_init)); + CHECK(executor_err == + emel::error::cast(moshi_executor::error::unexpected_event)); + + emel::memory::hybrid::sm memory{}; + recording_graph_executor graph{}; + moshi::sm predictor{make_predictor_dependencies(graph, memory)}; + emel::error::type predictor_err = k_no_error; + moshi::event::initialize predictor_init{*fixture.model}; + configure_predictor_initialize(predictor_init); + predictor_init.error_out = &predictor_err; + REQUIRE(predictor.process_event(predictor_init)); + predictor_err = k_no_error; + CHECK_FALSE(predictor.process_event(predictor_init)); + CHECK(predictor_err == emel::error::cast(moshi::error::unexpected_event)); +} + +TEST_CASE("speech_moshi_executor_requires_sampler_for_sampling") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.sampling_enabled = true; + init.sampling_audio_temperature = 0.8f; + init.sampling_text_temperature = 0.7f; + init.sampling_audio_top_k = 250; + init.sampling_text_top_k = 25; + init.sampling_seed = 1234u; + init.error_out = &err; + + CHECK_FALSE(executor.process_event(init)); + CHECK(err == emel::error::cast(moshi_executor::error::bind_failed)); +} + +TEST_CASE("speech_moshi_executor_sampling_matches_reference_probability_path") { + std::array logits = {}; + uint32_t logit_state = 1153427707u; + for (float &logit : logits) { + logit_state = static_cast( + (static_cast(logit_state) * 16807u) % 2147483647u); + logit = (static_cast(logit_state) / 2147483647.0f - 0.5f) * 16.0f; + } + std::array top_probabilities = {}; + std::array top_indices = {}; + std::array sorted_indices = {}; + uint32_t random_state = 1234u; + int32_t best_index = -1; + float best_score = 0.0f; + + emel::logits::sampler::sm sampler{}; + emel::error::type err = emel::error::cast(emel::logits::sampler::error::none); + emel::logits::sampler::event::sample_temperature_top_k request{ + logits, + 2048, + 0.8f, + 250, + sorted_indices, + top_probabilities, + top_indices, + random_state, + best_index, + best_score, + err}; + + REQUIRE(sampler.process_event(request)); + CHECK(err == emel::error::cast(emel::logits::sampler::error::none)); + + CHECK(best_index == 1582); + CHECK(random_state == 1237704734u); +} + +TEST_CASE("speech_moshi_executor_rejects_graph_step_before_initialize") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel)}; + std::array input = {0, 1, 2, 3}; + std::array output = {}; + int32_t text_token = -1; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + emel::memory::view::snapshot snapshot{}; + + moshi::event::graph_step step{*fixture.model, snapshot, + std::span{input}, + std::span{output}, text_token}; + step.error_out = &err; + + CHECK_FALSE(executor.process_event(step)); + CHECK(err == emel::error::cast(moshi_executor::error::not_initialized)); +} + +TEST_CASE("speech_moshi_executor_runtime_choice_uses_explicit_transitions") { + const auto bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "executor" / "sm.hpp"); + const std::string source{bytes.begin(), bytes.end()}; + const auto action_bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "executor" / "actions.hpp"); + const std::string actions_source{action_bytes.begin(), action_bytes.end()}; + const auto detail_bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "executor" / "detail.hpp"); + const std::string detail_source{detail_bytes.begin(), detail_bytes.end()}; + const auto predictor_guard_bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "guards.hpp"); + const std::string predictor_guards_source{predictor_guard_bytes.begin(), + predictor_guard_bytes.end()}; + + CHECK(source.find("infer_debug_phase") == std::string::npos); + CHECK(source.find("debug_phase_out != nullptr") == std::string::npos); + CHECK(source.find("effect_store_debug_phase_out") == std::string::npos); + CHECK(source.find("event::step_ctx ctx{}") == std::string::npos); + CHECK(source.find("step_workspace_") != std::string::npos); + CHECK(source.find("if (ctx.depformer_logits_ok)") == std::string::npos); + CHECK(actions_source.find("if (runtime_ev.ctx.temporal_out_norm_ok)") == + std::string::npos); + CHECK(actions_source.find("add_scaled_embedding") == std::string::npos); + CHECK(actions_source.find("if (!ctx.kernel.process_event(rows_ev))") == + std::string::npos); + CHECK(actions_source.find("if (!ctx.kernel.process_event(projection_ev))") == + std::string::npos); + CHECK(actions_source.find("if (!detail::bind_tensor_view") == + std::string::npos); + CHECK(actions_source.find("if (runtime_ev.ctx.depformer_kv_bound") == + std::string::npos); + CHECK(actions_source.find("(void)detail::bind_tensor_view") == + std::string::npos); + CHECK(actions_source.find("effect_bind_input_text_embedding") != + std::string::npos); + CHECK(actions_source.find("effect_run_embedding_row_fetch") != + std::string::npos); + CHECK(actions_source.find("effect_apply_input_text_embedding_row") != + std::string::npos); + CHECK(actions_source.find("effect_bind_input_audio_embedding") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_text_input_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_audio_input_projection") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_input_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_text_input_embedding") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_audio_input_embedding") != + std::string::npos); + CHECK(actions_source.find("effect_bind_temporal_layer_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_temporal_layer_out_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_layer_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_layer_out_projection") != + std::string::npos); + CHECK(actions_source.find("effect_bind_temporal_layer_gating_in") != + std::string::npos); + CHECK(actions_source.find("effect_bind_temporal_layer_gating_out") != + std::string::npos); + CHECK(actions_source.find("effect_bind_text_token_logits") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_layer_gating_in") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_layer_gating_out") != + std::string::npos); + CHECK(actions_source.find("effect_bind_depformer_token_logits") != + std::string::npos); + CHECK(source.find("state_sampling_seed_decision") != std::string::npos); + CHECK(source.find("state_text_sampling_top_k_decision") != std::string::npos); + CHECK(source.find("state_audio_sampling_top_k_decision") != + std::string::npos); + CHECK(source.find("guard_sampling_seed_zero") != std::string::npos); + CHECK(actions_source.find("effect_bind_zero_sampling_seed") != + std::string::npos); + CHECK(detail_source.find("random_state == 0u") == std::string::npos); + CHECK(predictor_guards_source.find("const int64_t row_count") != + std::string::npos); + CHECK(predictor_guards_source.find("const int64_t frame_count") != + std::string::npos); + CHECK(predictor_guards_source.find("const int64_t default_frame_count") != + std::string::npos); + CHECK(predictor_guards_source.find("embeddings->dims[0] <=") != + std::string::npos); + CHECK(actions_source.find("effect_publish_temporal_out_norm") != + std::string::npos); + CHECK(actions_source.find("struct effect_run_temporal_layer_norm {") == + std::string::npos); + CHECK(actions_source.find("struct effect_run_temporal_layer_norm2 {") == + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_norm_rms") != + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_norm_scale") != + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_norm2_rms") != + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_norm2_scale") != + std::string::npos); + CHECK(actions_source.find("struct effect_run_temporal_out_norm {") == + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_out_norm_rms") != + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_out_norm_scale") != + std::string::npos); + CHECK(actions_source.find("effect_apply_temporal_layer_silu_gate") == + std::string::npos); + CHECK(actions_source.find("effect_apply_depformer_layer_silu_gate") == + std::string::npos); + CHECK(actions_source.find("if (!ctx.kernel.process_event(silu_ev))") == + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_silu_gate_silu") != + std::string::npos); + CHECK(actions_source.find("effect_run_temporal_layer_silu_gate_mul") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_silu_gate_silu") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_silu_gate_mul") != + std::string::npos); + CHECK(actions_source.find("struct effect_run_depformer_layer_norm {") == + std::string::npos); + CHECK(actions_source.find("struct effect_run_depformer_layer_norm2 {") == + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_norm_rms") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_norm_scale") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_norm2_rms") != + std::string::npos); + CHECK(actions_source.find("effect_run_depformer_layer_norm2_scale") != + std::string::npos); + CHECK(source.find("state_temporal_layer_norm_rms_result_decision") != + std::string::npos); + CHECK(source.find("state_input_text_embedding_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_input_audio_embedding_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_input_projection_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_input_embedding_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_layer_projection_bind_result_decision") != + std::string::npos); + CHECK( + source.find("state_temporal_layer_out_projection_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_projection_bind_result_decision") != + std::string::npos); + CHECK(source.find( + "state_depformer_layer_out_projection_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_layer_gating_in_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_layer_gating_out_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_text_logits_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_gating_in_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_gating_out_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_logits_bind_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_layer_norm2_rms_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_out_norm_rms_result_decision") != + std::string::npos); + CHECK(source.find("state_temporal_layer_silu_gate_silu_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_silu_gate_silu_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_norm_rms_result_decision") != + std::string::npos); + CHECK(source.find("state_depformer_layer_norm2_rms_result_decision") != + std::string::npos); + CHECK(source.find("guard_temporal_layer_norm_rms_succeeded") != + std::string::npos); + CHECK(source.find("guard_embedding_view_bound") != std::string::npos); + CHECK(source.find("guard_embedding_row_succeeded") != std::string::npos); + CHECK(source.find("guard_projection_view_bound") != std::string::npos); + CHECK(source.find("guard_depformer_input_projection_bound") != + std::string::npos); + CHECK(source.find("guard_depformer_text_input_projection_succeeded") != + std::string::npos); + CHECK(source.find("guard_depformer_audio_input_projection_succeeded") != + std::string::npos); + CHECK(source.find("guard_temporal_layer_norm2_rms_succeeded") != + std::string::npos); + CHECK(source.find("guard_temporal_out_norm_rms_succeeded") != + std::string::npos); + CHECK(source.find("guard_temporal_layer_silu_gate_silu_succeeded") != + std::string::npos); + CHECK(source.find("guard_depformer_layer_silu_gate_silu_succeeded") != + std::string::npos); + CHECK(source.find("guard_depformer_layer_norm_rms_succeeded") != + std::string::npos); + CHECK(source.find("guard_depformer_layer_norm2_rms_succeeded") != + std::string::npos); +} + +TEST_CASE("speech_moshi_generator_graph_outputs_use_explicit_transitions") { + const auto action_bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "actions.hpp"); + const std::string actions_source{action_bytes.begin(), action_bytes.end()}; + const auto sm_bytes = + read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / + "moshi" / "sm.hpp"); + const std::string sm_source{sm_bytes.begin(), sm_bytes.end()}; + + CHECK(actions_source.find( + "if (runtime_ev.request.graph_error_out != nullptr)") == + std::string::npos); + CHECK(actions_source.find("graph_debug_phase_out") == std::string::npos); + CHECK(actions_source.find("effect_store_graph_error_out") != + std::string::npos); + CHECK(actions_source.find("effect_store_graph_debug_phase_out") == + std::string::npos); + CHECK(sm_source.find("state_execute_graph_error_out_decision") != + std::string::npos); + CHECK(sm_source.find("state_sample_error_out_decision") != std::string::npos); + CHECK(sm_source.find("state_graph_debug_phase_out_decision") == + std::string::npos); +} + +TEST_CASE("speech_moshi_generator_and_executor_cover_explicit_error_guards") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::memory::hybrid::sm memory{}; + moshi::action::context generator_ctx{memory}; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + moshi::event::initialize_ctx init_ctx{}; + moshi::event::initialize_run init_run{init, init_ctx}; + generator_ctx.lmgen.codebook_count = fixture.model->moshi_lm.n_q + 1; + generator_ctx.lmgen.generated_dep_q = fixture.model->moshi_lm.dep_q; + moshi::action::effect_configure_standard_lmgen{}(init_run, generator_ctx); + CHECK(generator_ctx.lmgen.delayed_dep_q == fixture.model->moshi_lm.dep_q); + + moshi::event::predict::workspace generator_workspace{}; + moshi::event::predict_ctx generator_predict_ctx{generator_workspace.memory}; + auto generator_snapshot = std::make_unique(); + std::array generator_input = {}; + moshi::event::predict generator_predict{ + std::span{generator_input.data(), 1}, generator_workspace, + 1, 1}; + moshi::event::predict_run generator_predict_run{generator_predict, + generator_predict_ctx}; + moshi::action::effect_mark_not_initialized{}( + generator_predict_run, generator_ctx); + CHECK(generator_predict_ctx.err == + emel::error::cast(moshi::error::not_initialized)); + moshi::action::effect_mark_memory_error{}( + generator_predict_run, generator_ctx); + CHECK(generator_predict_ctx.err == emel::error::cast(moshi::error::memory)); + moshi::action::effect_mark_step_request_invalid{}(generator_predict_run, + generator_ctx); + CHECK(generator_predict_ctx.err == + emel::error::cast(moshi::error::request_shape)); + moshi::action::effect_mark_voice_contract_error{}( + generator_predict_run, generator_ctx); + CHECK(generator_predict_ctx.err == + emel::error::cast(moshi::error::voice_contract)); + moshi::action::effect_mark_personaplex_prompt_error< + moshi::event::predict_run>{}(generator_predict_run, generator_ctx); + CHECK(generator_predict_ctx.err == + emel::error::cast(moshi::error::personaplex_prompt)); + emel::error::type unexpected_err = k_no_error; + generator_predict.error_out = &unexpected_err; + moshi::action::effect_mark_unexpected{}(generator_predict_run, generator_ctx); + CHECK(generator_predict_ctx.err == + emel::error::cast(moshi::error::unexpected_event)); + moshi::action::effect_mark_unexpected_and_store{}(generator_predict_run, + generator_ctx); + CHECK(unexpected_err == emel::error::cast(moshi::error::unexpected_event)); + CHECK(moshi::guard::guard_memory_rejected{}(generator_predict_run, + generator_ctx)); + CHECK(moshi::guard::guard_standard_lmgen{}(init_run, generator_ctx) != + moshi::guard::guard_personaplex_lmgen{}(init_run, generator_ctx)); + CHECK(moshi::guard::guard_graph_initialize_failed{}(init_run, generator_ctx)); + CHECK(moshi::guard::guard_predict_request_shape_invalid{}( + generator_predict_run, generator_ctx)); + + moshi::event::begin_personaplex_prompt begin_prompt{}; + moshi::event::begin_personaplex_prompt_ctx begin_prompt_ctx{}; + moshi::event::begin_personaplex_prompt_run begin_prompt_run{begin_prompt, + begin_prompt_ctx}; + CHECK(moshi::guard::guard_personaplex_prompt_begin_invalid{}(begin_prompt_run, + generator_ctx)); + + auto prompt_snapshot = std::make_unique(); + moshi::event::prefill_personaplex_prompt prompt_prefill{}; + moshi::event::prefill_personaplex_prompt_ctx prompt_prefill_ctx{ + *prompt_snapshot}; + moshi::event::prefill_personaplex_prompt_run prompt_prefill_run{ + prompt_prefill, prompt_prefill_ctx}; + CHECK(moshi::guard::guard_personaplex_prompt_prefill_request_invalid{}( + prompt_prefill_run, generator_ctx)); + CHECK(moshi::guard::guard_personaplex_prompt_phase_invalid{}( + prompt_prefill_run, generator_ctx)); + CHECK(moshi::guard::guard_no_personaplex_prompt_complete_out{}( + prompt_prefill_run, generator_ctx)); + CHECK(moshi::guard::guard_no_personaplex_prompt_remaining_out{}( + prompt_prefill_run, generator_ctx)); + + generator_ctx.session.model = fixture.model.get(); + generator_ctx.session.text_card = fixture.model->moshi_lm.text_card; + generator_ctx.voice.loaded = true; + generator_ctx.voice.ready = true; + generator_ctx.voice.prompt_started = true; + generator_ctx.voice.prompt_ready = false; + generator_ctx.voice.text_tokens_remaining = 1; + prompt_prefill.text_token = fixture.model->moshi_lm.text_card; + CHECK(moshi::guard::guard_personaplex_prompt_phase_invalid{}( + prompt_prefill_run, generator_ctx)); + + moshi_callback_probe callback_probe{}; + init.on_done = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_initialize_done>(&callback_probe); + init.on_error = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_initialize_error>( + &callback_probe); + generator_ctx.session.n_q = fixture.model->moshi_lm.n_q; + generator_ctx.session.dep_q = fixture.model->moshi_lm.dep_q; + init_ctx.err = emel::error::cast(moshi::error::bind_failed); + moshi::action::effect_emit_initialize_done{}(init_run, generator_ctx); + CHECK(callback_probe.n_q == fixture.model->moshi_lm.n_q); + CHECK(callback_probe.dep_q == fixture.model->moshi_lm.dep_q); + CHECK(callback_probe.request == &init); + moshi::action::effect_emit_initialize_error{}(init_run, generator_ctx); + CHECK(callback_probe.err == emel::error::cast(moshi::error::bind_failed)); + + moshi::event::load_voice load_voice{*fixture.model}; + moshi::event::load_voice_ctx load_voice_ctx{}; + moshi::event::load_voice_run load_voice_run{load_voice, load_voice_ctx}; + CHECK(moshi::guard::guard_voice_contract_invalid{}(load_voice_run, + generator_ctx)); + load_voice.on_done = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_load_voice_done>(&callback_probe); + load_voice.on_error = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_load_voice_error>( + &callback_probe); + generator_ctx.voice.prompt_frame_count = 7; + load_voice_ctx.err = emel::error::cast(moshi::error::voice_contract); + moshi::action::effect_emit_load_voice_done{}(load_voice_run, generator_ctx); + CHECK(callback_probe.prompt_frames == 7); + CHECK(callback_probe.request == &load_voice); + moshi::action::effect_emit_load_voice_error{}(load_voice_run, generator_ctx); + CHECK(callback_probe.err == emel::error::cast(moshi::error::voice_contract)); + + moshi::event::prefill_voice prefill_voice{}; + emel::error::type generator_graph_err = k_no_error; + bool voice_complete = false; + int32_t voice_remaining = -1; + prefill_voice.graph_error_out = &generator_graph_err; + prefill_voice.complete_out = &voice_complete; + prefill_voice.remaining_frames_out = &voice_remaining; + prefill_voice.on_done = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_prefill_voice_done>( + &callback_probe); + prefill_voice.on_error = + emel::callback::from< + moshi_callback_probe, + &moshi_callback_probe::on_generator_prefill_voice_error>( + &callback_probe); + moshi::event::prefill_voice_ctx prefill_voice_ctx{*generator_snapshot}; + prefill_voice_ctx.graph_error = + emel::error::cast(moshi::error::graph_runtime); + prefill_voice_ctx.complete = true; + prefill_voice_ctx.remaining_frames = 3; + prefill_voice_ctx.err = + emel::error::cast(moshi::error::graph_runtime_unavailable); + moshi::event::prefill_voice_run prefill_voice_run{prefill_voice, + prefill_voice_ctx}; + moshi::action::effect_store_graph_error_out{}(prefill_voice_run, + generator_ctx); + moshi::action::effect_store_voice_complete_out{}(prefill_voice_run, + generator_ctx); + moshi::action::effect_store_voice_remaining_out{}(prefill_voice_run, + generator_ctx); + CHECK(generator_graph_err == emel::error::cast(moshi::error::graph_runtime)); + CHECK(voice_complete); + CHECK(voice_remaining == 3); + moshi::action::effect_emit_prefill_voice_done{}(prefill_voice_run, + generator_ctx); + CHECK(callback_probe.complete); + CHECK(callback_probe.remaining_frames == 3); + CHECK(callback_probe.request == &prefill_voice); + moshi::action::effect_emit_prefill_voice_error{}(prefill_voice_run, + generator_ctx); + CHECK(callback_probe.err == + emel::error::cast(moshi::error::graph_runtime_unavailable)); + + begin_prompt.on_done = emel::callback::from( + &callback_probe); + begin_prompt.on_error = emel::callback::from( + &callback_probe); + begin_prompt_ctx.remaining_frames = 5; + begin_prompt_ctx.err = emel::error::cast(moshi::error::personaplex_prompt); + moshi::action::effect_emit_begin_personaplex_prompt_done{}(begin_prompt_run, + generator_ctx); + CHECK(callback_probe.remaining_frames == 5); + CHECK(callback_probe.request == &begin_prompt); + moshi::action::effect_emit_begin_personaplex_prompt_error{}(begin_prompt_run, + generator_ctx); + CHECK(callback_probe.err == + emel::error::cast(moshi::error::personaplex_prompt)); + + bool prompt_complete = false; + int32_t prompt_remaining = -1; + prompt_prefill.complete_out = &prompt_complete; + prompt_prefill.remaining_frames_out = &prompt_remaining; + prompt_prefill.on_done = emel::callback::from( + &callback_probe); + prompt_prefill.on_error = emel::callback::from( + &callback_probe); + prompt_prefill_ctx.complete = true; + prompt_prefill_ctx.remaining_frames = 2; + prompt_prefill_ctx.err = + emel::error::cast(moshi::error::graph_runtime_unavailable); + moshi::action::effect_store_personaplex_prompt_complete_out{}( + prompt_prefill_run, generator_ctx); + moshi::action::effect_store_personaplex_prompt_remaining_out{}( + prompt_prefill_run, generator_ctx); + CHECK(prompt_complete); + CHECK(prompt_remaining == 2); + moshi::action::effect_emit_prefill_personaplex_prompt_done{}( + prompt_prefill_run, generator_ctx); + CHECK(callback_probe.complete); + CHECK(callback_probe.remaining_frames == 2); + CHECK(callback_probe.request == &prompt_prefill); + moshi::action::effect_emit_prefill_personaplex_prompt_error{}( + prompt_prefill_run, generator_ctx); + CHECK(callback_probe.err == + emel::error::cast(moshi::error::graph_runtime_unavailable)); + + const auto &lm = fixture.model->moshi_lm; + emel::memory::view::snapshot snapshot{}; + snapshot.max_sequences = 4; + snapshot.sequence_active[0] = 1; + snapshot.sequence_length_values[0] = 1; + snapshot.sequence_kv_block_count[0] = 1; + snapshot.sequence_kv_blocks[0][0] = 0; + + emel::kernel::sm guard_kernel{}; + moshi_executor::action::context executor_ctx{ + make_executor_dependencies(guard_kernel)}; + std::array input = {}; + std::array output = {}; + std::array embedding = {}; + input.fill(executor_ctx.policy.token_zero); + input[0] = lm.text_padding_id; + input[1] = 0; + int32_t text_token = -1; + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{input.data(), static_cast(lm.n_q + 1)}, + std::span{output.data(), static_cast(lm.dep_q)}, + text_token}; + step.input_embedding = + std::span{embedding.data(), static_cast(lm.dim)}; + moshi_executor::event::step_ctx step_ctx{}; + moshi_executor::event::step_run step_run{step, step_ctx}; + executor_ctx.session.model = fixture.model.get(); + executor_ctx.session.codebook_count = lm.n_q + 1; + executor_ctx.session.dep_q = lm.dep_q; + executor_ctx.session.text_card = lm.text_card; + executor_ctx.session.audio_card = lm.card; + executor_ctx.session.hidden_dim = lm.dim; + REQUIRE(emel::model::moshi::detail::build_execution_contract( + *fixture.model, executor_ctx.session.contract) == k_no_error); + + moshi_executor::event::initialize executor_init{*fixture.model}; + moshi_executor::event::initialize_ctx executor_init_ctx{}; + moshi_executor::event::initialize_run executor_init_run{executor_init, + executor_init_ctx}; + CHECK(moshi_executor::guard::guard_bound_root_operands_supported{}( + executor_init_run, executor_ctx)); + const auto *bound_text_embedding = + executor_ctx.session.contract.lm.text_embedding.tensor; + executor_ctx.session.contract.lm.text_embedding.tensor = nullptr; + CHECK(moshi_executor::guard::guard_bound_root_operands_unsupported{}( + executor_init_run, executor_ctx)); + executor_ctx.session.contract.lm.text_embedding.tensor = bound_text_embedding; + for (int32_t layer_index = 0; layer_index < lm.num_layers; ++layer_index) { + auto &layer = executor_ctx.session.contract.lm + .temporal_layers[static_cast(layer_index)]; + const auto *projection = layer.split_input_projection.tensor != nullptr + ? layer.split_input_projection.tensor + : layer.fused_input_projection.tensor; + REQUIRE(projection != nullptr); + layer.split_input_projection.tensor = projection; + layer.fused_input_projection.tensor = projection; + } + CHECK( + moshi_executor::guard::guard_temporal_split_projection_layout_supported{}( + executor_init_run, executor_ctx)); + moshi_executor::action::effect_bind_temporal_split_projection_layout{}( + executor_init_run, executor_ctx); + CHECK(executor_ctx.session.temporal_input_projections[0] == + executor_ctx.session.contract.lm.temporal_layers[0] + .split_input_projection.tensor); + for (int32_t layer_index = 0; layer_index < lm.num_layers; ++layer_index) { + executor_ctx.session.contract.lm + .temporal_layers[static_cast(layer_index)] + .split_input_projection.tensor = nullptr; + } + CHECK( + moshi_executor::guard::guard_temporal_fused_projection_layout_supported{}( + executor_init_run, executor_ctx)); + moshi_executor::action::effect_bind_temporal_fused_projection_layout{}( + executor_init_run, executor_ctx); + CHECK_FALSE( + moshi_executor::guard::guard_temporal_projection_layout_unsupported{}( + executor_init_run, executor_ctx)); + for (int32_t layer_index = 0; layer_index < lm.depformer_num_layers; + ++layer_index) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + auto &codebook_layer = + executor_ctx.session.contract.lm + .depformer_layers[static_cast(layer_index)] + .codebooks[static_cast(codebook)]; + const auto *projection = + codebook_layer.split_input_projection.tensor != nullptr + ? codebook_layer.split_input_projection.tensor + : codebook_layer.fused_input_projection.tensor; + REQUIRE(projection != nullptr); + codebook_layer.split_input_projection.tensor = projection; + codebook_layer.fused_input_projection.tensor = projection; + } + } + CHECK(moshi_executor::guard:: + guard_depformer_split_projection_layout_supported{}( + executor_init_run, executor_ctx)); + moshi_executor::action::effect_bind_depformer_split_projection_layout{}( + executor_init_run, executor_ctx); + CHECK(executor_ctx.session.depformer_input_projections[0][0] == + executor_ctx.session.contract.lm.depformer_layers[0] + .codebooks[0] + .split_input_projection.tensor); + for (int32_t layer_index = 0; layer_index < lm.depformer_num_layers; + ++layer_index) { + for (int32_t codebook = 0; codebook < lm.dep_q; ++codebook) { + executor_ctx.session.contract.lm + .depformer_layers[static_cast(layer_index)] + .codebooks[static_cast(codebook)] + .split_input_projection.tensor = nullptr; + } + } + CHECK(moshi_executor::guard:: + guard_depformer_fused_projection_layout_supported{}( + executor_init_run, executor_ctx)); + moshi_executor::action::effect_bind_depformer_fused_projection_layout{}( + executor_init_run, executor_ctx); + CHECK_FALSE( + moshi_executor::guard::guard_depformer_projection_layout_unsupported{}( + executor_init_run, executor_ctx)); + executor_init.on_done = + emel::callback:: + from( + &callback_probe); + executor_init.on_error = + emel::callback:: + from( + &callback_probe); + executor_init_ctx.err = emel::error::cast(moshi_executor::error::bind_failed); + moshi_executor::action::effect_emit_initialize_done{}(executor_init_run, + executor_ctx); + CHECK(callback_probe.n_q == lm.n_q); + CHECK(callback_probe.dep_q == lm.dep_q); + CHECK(callback_probe.request == &executor_init); + moshi_executor::action::effect_emit_initialize_error{}(executor_init_run, + executor_ctx); + CHECK(callback_probe.err == + emel::error::cast(moshi_executor::error::bind_failed)); + executor_init_ctx.err = k_no_error; + moshi_executor::action::effect_mark_bind_failed{}(executor_init_run, + executor_ctx); + CHECK(executor_init_ctx.err == + emel::error::cast(moshi_executor::error::bind_failed)); + + emel::error::type executor_step_err = k_no_error; + step.error_out = &executor_step_err; + moshi_executor::action::effect_mark_model_mismatch{}(step_run, executor_ctx); + CHECK(step_ctx.err == + emel::error::cast(moshi_executor::error::model_mismatch)); + moshi_executor::action::effect_mark_request_shape{}(step_run, executor_ctx); + CHECK(step_ctx.err == + emel::error::cast(moshi_executor::error::request_shape)); + moshi_executor::action::effect_store_error_out< + moshi_executor::event::step_run>{}(step_run, executor_ctx); + CHECK(executor_step_err == + emel::error::cast(moshi_executor::error::request_shape)); + moshi_executor::action::effect_mark_unexpected_and_store{}(step_run, + executor_ctx); + CHECK(step_ctx.err == + emel::error::cast(moshi_executor::error::unexpected_event)); + CHECK(executor_step_err == + emel::error::cast(moshi_executor::error::unexpected_event)); + + CHECK(moshi_executor::guard::guard_step_model_matches{}(step_run, + executor_ctx)); + executor_ctx.session.model = nullptr; + CHECK(moshi_executor::guard::guard_step_model_mismatch{}(step_run, + executor_ctx)); + executor_ctx.session.model = fixture.model.get(); + CHECK( + moshi_executor::guard::guard_step_shape_valid{}(step_run, executor_ctx)); + executor_ctx.session.codebook_count += 1; + CHECK(moshi_executor::guard::guard_step_shape_invalid{}(step_run, + executor_ctx)); + executor_ctx.session.codebook_count = lm.n_q + 1; + + CHECK(moshi_executor::guard::guard_external_input_embedding_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_input_embedding_supported{}(step_run, + executor_ctx)); + step.input_embedding = {}; + CHECK_FALSE(moshi_executor::guard::guard_input_embedding_unsupported{}( + step_run, executor_ctx)); + CHECK_FALSE(moshi_executor::guard::guard_token_input_embedding_unsupported{}( + step_run, executor_ctx)); + step.input_embedding = + std::span{embedding.data(), static_cast(lm.dim)}; + + CHECK_FALSE(moshi_executor::guard::guard_input_text_embedding_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_input_text_embedding_failed{}( + step_run, executor_ctx)); + step_ctx.input_text_embedding_ok = true; + CHECK(moshi_executor::guard::guard_input_text_embedding_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_embedding_view_bind_failed{}( + step_run, executor_ctx)); + step_ctx.embedding_view_bound = true; + CHECK(moshi_executor::guard::guard_embedding_view_bound{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_embedding_row_failed{}(step_run, + executor_ctx)); + step_ctx.embedding_row_ok = true; + CHECK(moshi_executor::guard::guard_embedding_row_succeeded{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_projection_view_bind_failed{}( + step_run, executor_ctx)); + step_ctx.projection_view_bound = true; + CHECK(moshi_executor::guard::guard_projection_view_bound{}(step_run, + executor_ctx)); + + step_ctx.input_audio_codebook_index = -1; + CHECK_FALSE(moshi_executor::guard::guard_current_input_audio_token_present{}( + step_run, executor_ctx)); + CHECK_FALSE(moshi_executor::guard::guard_current_input_audio_token_zero{}( + step_run, executor_ctx)); + step_ctx.input_audio_codebook_index = 0; + CHECK(moshi_executor::guard::guard_current_input_audio_token_present{}( + step_run, executor_ctx)); + input[1] = executor_ctx.policy.token_zero; + CHECK(moshi_executor::guard::guard_current_input_audio_token_zero{}( + step_run, executor_ctx)); + step.input_embedding = {}; + CHECK_FALSE( + moshi_executor::guard::guard_current_input_audio_token_unsupported{}( + step_run, executor_ctx)); + step.input_embedding = + std::span{embedding.data(), static_cast(lm.dim)}; + + CHECK(moshi_executor::guard::guard_input_audio_embedding_failed{}( + step_run, executor_ctx)); + step_ctx.input_audio_embedding_ok = true; + CHECK(moshi_executor::guard::guard_input_audio_embedding_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_more_input_audio_codebooks{}( + step_run, executor_ctx)); + step_ctx.input_audio_codebook_index = lm.n_q - 1; + CHECK(moshi_executor::guard::guard_input_audio_codebooks_complete{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_input_embedding_failed{}(step_run, + executor_ctx)); + step_ctx.input_embedding_ok = true; + CHECK(moshi_executor::guard::guard_input_embedding_succeeded{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_kv_binding_missing{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_kv_bind_failed{}(step_run, + executor_ctx)); + + std::array key_cache = {}; + std::array value_cache = {}; + std::array offsets = {0}; + step_ctx.temporal_kv.key_cache = std::span{key_cache}; + step_ctx.temporal_kv.value_cache = std::span{value_cache}; + step_ctx.temporal_kv.layer_cache_offsets = std::span{offsets}; + step_ctx.temporal_kv.layer_count = 1; + step_ctx.temporal_kv.position_capacity = 1; + step_ctx.temporal_kv.kv_dim = lm.dim; + step_ctx.temporal_kv_bound = true; + CHECK( + moshi_executor::guard::guard_temporal_kv_bound{}(step_run, executor_ctx)); + + CHECK_FALSE(moshi_executor::guard::guard_temporal_layer_norm_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_norm_rms_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_norm_rms_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_norm_rms_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_norm_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_norm_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_norm_succeeded{}( + step_run, executor_ctx)); + CHECK_FALSE( + moshi_executor::guard::guard_temporal_layer_projection_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_projection_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_projection_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_projection_succeeded{}( + step_run, executor_ctx)); + step_ctx.temporal_position.logical_position = 0; + step_ctx.temporal_position.physical_position = 0; + CHECK_FALSE(moshi_executor::guard::guard_temporal_layer_rope_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_rope_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_rope_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_rope_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_cache_write_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_cache_write_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_cache_write_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_cache_write_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_attention_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_attention_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_attention_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_attention_succeeded{}( + step_run, executor_ctx)); + CHECK_FALSE( + moshi_executor::guard::guard_temporal_layer_out_projection_unsupported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_out_projection_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_out_projection_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_out_projection_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_residual_failed{}( + step_run, executor_ctx)); + step_ctx.temporal_layer_residual_ok = true; + CHECK(moshi_executor::guard::guard_temporal_layer_residual_succeeded{}( + step_run, executor_ctx)); +} + +TEST_CASE("speech_moshi_executor_embeds_input_before_unsupported_transformer") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + REQUIRE(executor.process_event(init)); + + std::array input = {}; + std::array output = {}; + input.fill(-1); + input[0] = fixture.model->moshi_lm.text_padding_id; + int32_t text_token = -1; + emel::memory::view::snapshot snapshot{}; + err = emel::error::cast(moshi_executor::error::none); + + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + step.error_out = &err; + + CHECK_FALSE(executor.process_event(step)); + CHECK(err == + emel::error::cast(moshi_executor::error::graph_execution_unsupported)); + CHECK(text_token == -1); +} + +TEST_CASE( + "speech_moshi_executor_writes_temporal_kv_and_text_before_depformer") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + temporal_kv_probe probe{}; + memory_streaming::sm temporal_positions{memory_streaming::dependencies{ + .capacity = fixture.model->moshi_lm.context}}; + initialize_streaming_position(temporal_positions); + const auto views = + prepare_kv_views(*fixture.model, &probe, nullptr, &temporal_positions); + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + REQUIRE(executor.process_event(init)); + + std::array input = {}; + std::array output = {}; + input.fill(-1); + input[0] = fixture.model->moshi_lm.text_padding_id; + int32_t text_token = -1; + emel::memory::view::snapshot snapshot{}; + snapshot.max_sequences = 8; + snapshot.sequence_active[7] = 1; + snapshot.sequence_length_values[7] = 3; + snapshot.sequence_kv_block_count[7] = 1; + snapshot.sequence_kv_blocks[7][0] = 0; + err = emel::error::cast(moshi_executor::error::none); + + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + step.sequence_id = 7; + step.error_out = &err; + + CHECK_FALSE(executor.process_event(step)); + CHECK(err == + emel::error::cast(moshi_executor::error::graph_execution_unsupported)); + CHECK(probe.call_count == 1); + const size_t write_offset = 0; + const size_t write_end = + write_offset + static_cast(fixture.model->moshi_lm.dim); + REQUIRE(write_end <= probe.key_cache.size()); + CHECK(std::any_of( + probe.key_cache.begin() + static_cast(write_offset), + probe.key_cache.begin() + static_cast(write_end), + [](const uint16_t value) { return value != 0u; })); + CHECK(std::any_of( + probe.value_cache.begin() + static_cast(write_offset), + probe.value_cache.begin() + static_cast(write_end), + [](const uint16_t value) { return value != 0u; })); + const int32_t last_layer = fixture.model->moshi_lm.num_layers - 1; + REQUIRE(last_layer >= 0); + const size_t last_layer_write_offset = + probe.layer_offsets[static_cast(last_layer)] + write_offset; + const size_t last_layer_write_end = + last_layer_write_offset + + static_cast(fixture.model->moshi_lm.dim); + REQUIRE(last_layer_write_end <= probe.key_cache.size()); + CHECK(std::any_of(probe.key_cache.begin() + + static_cast(last_layer_write_offset), + probe.key_cache.begin() + + static_cast(last_layer_write_end), + [](const uint16_t value) { return value != 0u; })); + CHECK(std::any_of(probe.value_cache.begin() + + static_cast(last_layer_write_offset), + probe.value_cache.begin() + + static_cast(last_layer_write_end), + [](const uint16_t value) { return value != 0u; })); + CHECK(text_token >= 0); + CHECK(text_token < fixture.model->moshi_lm.text_card); +} + +TEST_CASE("speech_moshi_executor_accepts_personaplex_voice_embedding_step") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + temporal_kv_probe temporal_probe{}; + depformer_kv_probe depformer_probe{}; + memory_streaming::sm temporal_positions{memory_streaming::dependencies{ + .capacity = fixture.model->moshi_lm.context}}; + memory_streaming::sm depformer_positions{memory_streaming::dependencies{ + .capacity = fixture.model->moshi_lm.depformer_context}}; + initialize_streaming_position(temporal_positions); + initialize_streaming_position(depformer_positions); + emel::kernel::sm kernel{}; + const auto views = + prepare_kv_views(*fixture.model, &temporal_probe, &depformer_probe, + &temporal_positions, &depformer_positions); + moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + REQUIRE(executor.process_event(init)); + + std::array embedding = {}; + for (int32_t index = 0; index < fixture.model->moshi_lm.dim; ++index) { + embedding[static_cast(index)] = + 0.001f * static_cast(index + 1); + } + std::array input = {}; + std::array output = {}; + input.fill(-1); + input[0] = fixture.model->moshi_lm.text_padding_id; + int32_t text_token = -1; + emel::memory::view::snapshot snapshot{}; + snapshot.max_sequences = 8; + snapshot.sequence_active[3] = 1; + snapshot.sequence_length_values[3] = 1; + snapshot.sequence_kv_block_count[3] = 1; + snapshot.sequence_kv_blocks[3][0] = 0; + + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + step.sequence_id = 3; + step.input_embedding = std::span{ + embedding.data(), static_cast(fixture.model->moshi_lm.dim)}; + step.forced_text_token = fixture.model->moshi_lm.text_padding_id; + step.error_out = &err; + + REQUIRE(executor.process_event(step)); + CHECK(err == emel::error::cast(moshi_executor::error::none)); + CHECK(text_token == fixture.model->moshi_lm.text_padding_id); + CHECK(temporal_probe.call_count == 1); + CHECK(depformer_probe.call_count == 1); + for (int32_t index = 0; index < fixture.model->moshi_lm.dep_q; ++index) { + CHECK(output[static_cast(index)] >= 0); + CHECK(output[static_cast(index)] < fixture.model->moshi_lm.card); + } +} + +TEST_CASE("speech_moshi_executor_generates_audio_tokens_with_injected_kv") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + temporal_kv_probe temporal_probe{}; + depformer_kv_probe depformer_probe{}; + memory_streaming::sm temporal_positions{memory_streaming::dependencies{ + .capacity = fixture.model->moshi_lm.context}}; + memory_streaming::sm depformer_positions{memory_streaming::dependencies{ + .capacity = fixture.model->moshi_lm.depformer_context}}; + initialize_streaming_position(temporal_positions); + initialize_streaming_position(depformer_positions); + emel::kernel::sm kernel{}; + const auto views = + prepare_kv_views(*fixture.model, &temporal_probe, &depformer_probe, + &temporal_positions, &depformer_positions); + moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + emel::error::type err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize init{*fixture.model}; + init.error_out = &err; + REQUIRE(executor.process_event(init)); + + std::array input = {}; + std::array output = {}; + input.fill(-1); + input[0] = fixture.model->moshi_lm.text_padding_id; + int32_t text_token = -1; + emel::memory::view::snapshot snapshot{}; + snapshot.max_sequences = 8; + snapshot.sequence_active[7] = 1; + snapshot.sequence_length_values[7] = 3; + snapshot.sequence_kv_block_count[7] = 1; + snapshot.sequence_kv_blocks[7][0] = 0; + err = emel::error::cast(moshi_executor::error::none); + + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + std::span{output.data(), + static_cast(fixture.model->moshi_lm.dep_q)}, + text_token}; + step.sequence_id = 7; + step.error_out = &err; + + CHECK(executor.process_event(step)); + CHECK(err == emel::error::cast(moshi_executor::error::none)); + CHECK(temporal_probe.call_count == 1); + CHECK(depformer_probe.call_count == 1); + memory_streaming::window_view depformer_window = {}; + int32_t memory_err = + static_cast(emel::error::cast(memory_streaming::error::none)); + REQUIRE( + depformer_positions.process_event(memory_streaming::event::capture_view{ + .view_out = depformer_window, .error_out = memory_err})); + CHECK(depformer_window.logical_end == fixture.model->moshi_lm.dep_q); + CHECK(text_token >= 0); + CHECK(text_token < fixture.model->moshi_lm.text_card); + for (int32_t index = 0; index < fixture.model->moshi_lm.dep_q; ++index) { + CHECK(output[static_cast(index)] >= 0); + CHECK(output[static_cast(index)] < fixture.model->moshi_lm.card); + } + const int32_t last_layer = fixture.model->moshi_lm.depformer_num_layers - 1; + REQUIRE(last_layer >= 0); + const size_t last_layer_offset = + depformer_probe.layer_offsets[static_cast(last_layer)]; + const size_t last_layer_end = + last_layer_offset + + static_cast(fixture.model->moshi_lm.depformer_context) * + static_cast(fixture.model->moshi_lm.depformer_dim); + REQUIRE(last_layer_end <= depformer_probe.key_cache.size()); + CHECK(std::any_of(depformer_probe.key_cache.begin() + + static_cast(last_layer_offset), + depformer_probe.key_cache.begin() + + static_cast(last_layer_end), + [](const uint16_t value) { return value != 0u; })); + CHECK(std::any_of(depformer_probe.value_cache.begin() + + static_cast(last_layer_offset), + depformer_probe.value_cache.begin() + + static_cast(last_layer_end), + [](const uint16_t value) { return value != 0u; })); +} + +TEST_CASE( + "speech_moshi_executor_guards_reject_invalid_outputs_and_cache_metadata") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + const auto &lm = fixture.model->moshi_lm; + + emel::kernel::sm kernel{}; + moshi_executor::action::context executor_ctx{ + make_executor_dependencies(kernel)}; + executor_ctx.session.hidden_dim = lm.dim; + executor_ctx.session.text_card = lm.text_card; + + emel::memory::view::snapshot snapshot{}; + std::array input = {}; + std::array output = {}; + int32_t text_token = -1; + moshi::event::graph_step step{ + *fixture.model, snapshot, + std::span{input.data(), static_cast(lm.n_q + 1)}, + std::span{output.data(), static_cast(lm.dep_q)}, + text_token}; + auto step_ctx = std::make_unique(); + moshi_executor::event::step_run step_run{step, *step_ctx}; + + step.forced_text_token = lm.text_card; + CHECK_FALSE(moshi_executor::guard::guard_forced_text_token_valid{}( + step_run, executor_ctx)); + CHECK_FALSE(moshi_executor::guard::guard_forced_text_token_absent{}( + step_run, executor_ctx)); + + float tensor_data = 0.0f; + emel::model::data::tensor_record malformed_matrix{}; + malformed_matrix.data = &tensor_data; + malformed_matrix.n_dims = 1; + malformed_matrix.dims[0] = lm.dim; + CHECK_FALSE(moshi_executor::detail::tensor_shape(&malformed_matrix, lm.dim, + lm.dim * 3)); + + std::array key_cache = {}; + std::array value_cache = + {}; + std::array offsets = {std::numeric_limits::max()}; + step_ctx->temporal_layer_rope_ok = true; + step_ctx->temporal_layer_index = 0; + step_ctx->temporal_position.physical_position = 0; + step_ctx->temporal_kv = moshi_executor::detail::temporal_kv_view{ + .key_cache = key_cache, + .value_cache = value_cache, + .layer_cache_offsets = offsets, + .layer_count = 1, + .position_capacity = 1, + .kv_dim = lm.dim, + }; + CHECK_FALSE( + moshi_executor::guard::guard_temporal_layer_cache_write_supported{}( + step_run, executor_ctx)); + + step_ctx->depformer_layer_projection_ok = true; + step_ctx->depformer_layer_index = 0; + step_ctx->depformer_position.physical_position = 0; + step_ctx->depformer_kv = moshi_executor::detail::depformer_kv_view{ + .key_cache = key_cache, + .value_cache = value_cache, + .layer_cache_offsets = offsets, + .layer_count = 1, + .position_capacity = 1, + .kv_dim = lm.depformer_dim, + }; + CHECK_FALSE( + moshi_executor::guard::guard_depformer_layer_cache_write_supported{}( + step_run, executor_ctx)); + + CHECK_FALSE(moshi_executor::guard::guard_cache_span_valid( + 0u, std::numeric_limits::max(), 1u, 2u, key_cache.size(), + value_cache.size())); + CHECK_FALSE(moshi_executor::guard::guard_cache_span_valid( + std::numeric_limits::max(), 1u, 1u, 1u, key_cache.size(), + value_cache.size())); + CHECK(moshi_executor::guard::guard_sampling_step_invalid{}(step_run, + executor_ctx) != + moshi_executor::guard::guard_sampling_step_valid{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_position_advance_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_position_advance_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_norm2_unsupported{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_norm2_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_norm2_rms_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_norm2_rms_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_norm2_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_norm2_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_gating_in_unsupported{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_gating_in_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_gating_in_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_gating_in_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_silu_gate_unsupported{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_silu_gate_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_silu_gate_silu_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_silu_gate_silu_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_silu_gate_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_silu_gate_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_gating_out_unsupported{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_gating_out_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_gating_out_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_gating_out_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_layer_ff_residual_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_layer_ff_residual_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_out_norm_unsupported{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_out_norm_supported{}( + step_run, executor_ctx)); + step_ctx->temporal_layer_index = lm.num_layers; + step_ctx->temporal_layer_ff_residual_ok = true; + CHECK(moshi_executor::guard::guard_temporal_out_norm_supported{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_out_norm_rms_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_temporal_out_norm_rms_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_temporal_out_norm_failed{}(step_run, + executor_ctx) != + moshi_executor::guard::guard_temporal_out_norm_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_text_logits_unsupported{}(step_run, + executor_ctx) != + moshi_executor::guard::guard_text_logits_supported{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_text_logits_matmul_failed{}( + step_run, executor_ctx) != + moshi_executor::guard::guard_text_logits_matmul_succeeded{}( + step_run, executor_ctx)); + CHECK(moshi_executor::guard::guard_text_logits_failed{}(step_run, + executor_ctx) != + moshi_executor::guard::guard_text_logits_succeeded{}(step_run, + executor_ctx)); + CHECK(moshi_executor::guard::guard_text_sampling_failed{}(step_run, + executor_ctx) != + moshi_executor::guard::guard_text_sampling_succeeded{}(step_run, + executor_ctx)); +} + +TEST_CASE("speech_moshi_executor_sampling_rng_is_actor_owned") { + auto first_fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + auto second_fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (first_fixture.model == nullptr || second_fixture.model == nullptr) { + return; + } + + temporal_kv_probe first_temporal{}; + temporal_kv_probe second_temporal{}; + depformer_kv_probe first_depformer{}; + depformer_kv_probe second_depformer{}; + memory_streaming::sm first_temporal_positions{memory_streaming::dependencies{ + .capacity = first_fixture.model->moshi_lm.context}}; + memory_streaming::sm first_depformer_positions{memory_streaming::dependencies{ + .capacity = first_fixture.model->moshi_lm.depformer_context}}; + memory_streaming::sm second_temporal_positions{memory_streaming::dependencies{ + .capacity = second_fixture.model->moshi_lm.context}}; + memory_streaming::sm second_depformer_positions{ + memory_streaming::dependencies{ + .capacity = second_fixture.model->moshi_lm.depformer_context}}; + initialize_streaming_position(first_temporal_positions); + initialize_streaming_position(first_depformer_positions); + initialize_streaming_position(second_temporal_positions); + initialize_streaming_position(second_depformer_positions); + emel::kernel::sm first_kernel{}; + emel::kernel::sm second_kernel{}; + emel::logits::sampler::sm first_sampler{}; + emel::logits::sampler::sm second_sampler{}; + const auto first_views = + prepare_kv_views(*first_fixture.model, &first_temporal, &first_depformer, + &first_temporal_positions, &first_depformer_positions); + const auto second_views = prepare_kv_views( + *second_fixture.model, &second_temporal, &second_depformer, + &second_temporal_positions, &second_depformer_positions); + moshi_executor::sm first{ + make_executor_dependencies(first_kernel, first_views, &first_sampler)}; + moshi_executor::sm second{ + make_executor_dependencies(second_kernel, second_views, &second_sampler)}; + emel::error::type first_err = emel::error::cast(moshi_executor::error::none); + emel::error::type second_err = emel::error::cast(moshi_executor::error::none); + moshi_executor::event::initialize first_init{*first_fixture.model}; + moshi_executor::event::initialize second_init{*second_fixture.model}; + for (auto *init : {&first_init, &second_init}) { + init->sampling_enabled = true; + init->sampling_audio_temperature = 0.8f; + init->sampling_text_temperature = 0.7f; + init->sampling_audio_top_k = 250; + init->sampling_text_top_k = 25; + init->sampling_seed = 1234u; + } + first_init.error_out = &first_err; + second_init.error_out = &second_err; + REQUIRE(first.process_event(first_init)); + REQUIRE(second.process_event(second_init)); + + std::array input = {}; + std::array first_output = {}; + std::array second_output = {}; + input.fill(-1); + input[0] = first_fixture.model->moshi_lm.text_padding_id; + int32_t first_text_token = -1; + int32_t second_text_token = -1; + emel::memory::view::snapshot snapshot{}; + snapshot.max_sequences = 1; + snapshot.sequence_active[0] = 1; + snapshot.sequence_length_values[0] = 1; + snapshot.sequence_kv_block_count[0] = 1; + snapshot.sequence_kv_blocks[0][0] = 0; + const auto input_span = std::span{ + input.data(), static_cast(first_fixture.model->moshi_lm.n_q + 1)}; + moshi::event::graph_step first_step{ + *first_fixture.model, snapshot, input_span, + std::span{ + first_output.data(), + static_cast(first_fixture.model->moshi_lm.dep_q)}, + first_text_token}; + moshi::event::graph_step second_step{ + *second_fixture.model, snapshot, input_span, + std::span{ + second_output.data(), + static_cast(second_fixture.model->moshi_lm.dep_q)}, + second_text_token}; + first_step.error_out = &first_err; + second_step.error_out = &second_err; + + REQUIRE(first.process_event(first_step)); + REQUIRE(second.process_event(second_step)); + CHECK(first_text_token == second_text_token); + CHECK(std::equal(first_output.begin(), + first_output.begin() + first_fixture.model->moshi_lm.dep_q, + second_output.begin())); +} + +TEST_CASE( + "speech_moshi_predictor_uses_emel_executor_binding_without_fake_tokens") { + auto fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture.model == nullptr) { + return; + } + + emel::memory::test::recording_kv_actor kv{}; + emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; + emel::kernel::sm kernel{}; + moshi_executor::sm executor{make_executor_dependencies(kernel)}; + + moshi::sm generator{make_predictor_dependencies(executor, memory)}; + emel::error::type err = k_no_error; + moshi::event::initialize init{*fixture.model}; + configure_predictor_initialize(init); + init.error_out = &err; + REQUIRE(generator.process_event(init)); + + std::array input = {}; + std::array temporal_state = + {}; + moshi::event::predict::workspace workspace{}; + workspace.temporal_state = std::span{ + temporal_state.data(), static_cast(fixture.model->moshi_lm.dim)}; + moshi::event::predict predict{ + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}, + workspace, 1, 1}; + predict.error_out = &err; + REQUIRE(generator.process_event(predict)); + moshi::event::execute execute{ + workspace, + std::span{ + input.data(), static_cast(fixture.model->moshi_lm.n_q + 1)}}; + execute.error_out = &err; + + CHECK_FALSE(generator.process_event(execute)); + CHECK(err == emel::error::cast(moshi::error::graph_runtime)); + CHECK(kv.allocate_slots_count == 1); + CHECK(kv.capture_view_count == 1); + CHECK(workspace.sampled_text_token == -1); + CHECK(workspace.sampled_audio_tokens[0] == -1); +} diff --git a/tests/speech/tokenizer/moshi/lifecycle_tests.cpp b/tests/speech/tokenizer/moshi/lifecycle_tests.cpp new file mode 100644 index 00000000..385a3874 --- /dev/null +++ b/tests/speech/tokenizer/moshi/lifecycle_tests.cpp @@ -0,0 +1,467 @@ +#include +#include +#include +#include + +#include + +#include "../../../allocation_tracker.hpp" +#include "emel/error/error.hpp" +#include "emel/speech/tokenizer/moshi/sm.hpp" + +namespace { + +namespace moshi = emel::speech::tokenizer::moshi; +namespace sml = stateforward::sml; + +int32_t error_code(const moshi::error value) noexcept { + return static_cast(emel::error::cast(value)); +} + +struct standard_fixture { + static constexpr int32_t codebooks = 3; + static constexpr int32_t generated_codebooks = 2; + static constexpr int32_t delayed_codebooks = 2; + static constexpr int32_t rows = 4; + + explicit standard_fixture(const int32_t initial_delay_frames = 0) + : machine(moshi::dependencies{ + .delays = std::span{delays}, + .cache = std::span{cache}, + .codebooks = codebooks, + .generated_audio_codebooks = generated_codebooks, + .delayed_audio_codebooks = delayed_codebooks, + .cache_rows = rows, + .maximum_delay = 2, + .initial_delay_frames = initial_delay_frames, + .text_initial_token = 100, + .audio_initial_token = 200, + .token_zero = -1, + .token_ungenerated = -2, + }) {} + + bool initialize() { + err = error_code(moshi::error::none); + return machine.process_event(moshi::event::initialize{.error_out = err}); + } + + bool tokenize(const std::span input = {}) { + err = error_code(moshi::error::none); + return machine.process_event(moshi::event::tokenize{ + .audio_tokens = input, + .model_tokens_out = std::span{model_tokens}, + .error_out = err, + }); + } + + bool detokenize(const int32_t text, + const std::span generated) { + err = error_code(moshi::error::none); + return machine.process_event(moshi::event::detokenize{ + .text_token = text, + .audio_tokens = generated, + .text_token_out = text_out, + .audio_tokens_out = std::span{audio_out}, + .produced_out = produced, + .error_out = err, + }); + } + + std::array delays = {0, 1, 2}; + std::array cache = {}; + std::array model_tokens = {}; + std::array audio_out = {}; + int32_t text_out = -1; + bool produced = false; + int32_t err = error_code(moshi::error::none); + moshi::sm machine; +}; + +} // namespace + +TEST_CASE("speech Moshi tokenizer delays and restores codebook alignment") { + standard_fixture state; + REQUIRE(state.initialize()); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{100, 200, 200}); + const std::array first_audio = {11, 12}; + REQUIRE(state.detokenize(10, first_audio)); + CHECK_FALSE(state.produced); + + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{10, 200, 200}); + const std::array second_audio = {21, 22}; + REQUIRE(state.detokenize(20, second_audio)); + CHECK_FALSE(state.produced); + + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{20, 21, 200}); + const std::array third_audio = {31, 32}; + REQUIRE(state.detokenize(30, third_audio)); + CHECK(state.produced); + CHECK(state.text_out == 10); + CHECK(state.audio_out == std::array{21, 32}); +} + +TEST_CASE("speech Moshi tokenizer preserves complete provided frames") { + standard_fixture state; + REQUIRE(state.initialize()); + + const std::array first_input = {50, 51, 52}; + const std::array second_input = {60, 61, 62}; + const std::array third_input = {70, 71, 72}; + const std::array generated = {91, 92}; + + REQUIRE(state.tokenize(first_input)); + REQUIRE(state.detokenize(90, generated)); + REQUIRE(state.tokenize(second_input)); + REQUIRE(state.detokenize(90, generated)); + REQUIRE(state.tokenize(third_input)); + REQUIRE(state.detokenize(90, generated)); + + CHECK(state.produced); + CHECK(state.text_out == 60); + CHECK(state.audio_out == std::array{61, 62}); +} + +TEST_CASE("speech Moshi tokenizer accepts the PersonaPlex tail shape") { + std::array delays = {0, 1, 1, 1, 1}; + std::array cache = {}; + moshi::sm machine{moshi::dependencies{ + .delays = std::span{delays}, + .cache = std::span{cache}, + .codebooks = 5, + .generated_audio_codebooks = 4, + .delayed_audio_codebooks = 2, + .cache_rows = 4, + .maximum_delay = 1, + .initial_delay_frames = 0, + .text_initial_token = 100, + .audio_initial_token = 200, + .token_zero = -1, + .token_ungenerated = -2, + }}; + int32_t err = error_code(moshi::error::none); + REQUIRE(machine.process_event(moshi::event::initialize{.error_out = err})); + + std::array tail = {101, 102}; + std::array model_tokens = {}; + REQUIRE(machine.process_event(moshi::event::tokenize{ + .audio_tokens = std::span{tail}, + .model_tokens_out = std::span{model_tokens}, + .error_out = err, + })); + CHECK(model_tokens == std::array{100, 200, 200, 200, 200}); + + std::array generated = {11, 12, 13, 14}; + std::array output = {}; + int32_t text_out = -1; + bool produced = false; + REQUIRE(machine.process_event(moshi::event::detokenize{ + .text_token = 10, + .audio_tokens = std::span{generated}, + .text_token_out = text_out, + .audio_tokens_out = std::span{output}, + .produced_out = produced, + .error_out = err, + })); + CHECK_FALSE(produced); + + const std::array next_tail = {111, 112}; + REQUIRE(machine.process_event(moshi::event::tokenize{ + .audio_tokens = std::span{next_tail}, + .model_tokens_out = std::span{model_tokens}, + .error_out = err, + })); + REQUIRE(machine.process_event(moshi::event::detokenize{ + .text_token = 20, + .audio_tokens = std::span{generated}, + .text_token_out = text_out, + .audio_tokens_out = std::span{output}, + .produced_out = produced, + .error_out = err, + })); + const std::array third_tail = {121, 122}; + REQUIRE(machine.process_event(moshi::event::tokenize{ + .audio_tokens = std::span{third_tail}, + .model_tokens_out = std::span{model_tokens}, + .error_out = err, + })); + CHECK(model_tokens[3] == next_tail[0]); + CHECK(model_tokens[4] == next_tail[1]); +} + +TEST_CASE("speech Moshi tokenizer rejects invalid public audio tokens before " + "caching") { + standard_fixture state; + REQUIRE(state.initialize()); + + const std::array invalid_input = {50, 200, 52}; + CHECK(state.tokenize(invalid_input)); + CHECK(state.err == error_code(moshi::error::request_shape)); + CHECK(state.machine.is(sml::state)); + + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{100, 200, 200}); +} + +TEST_CASE("speech Moshi tokenizer rejects invalid generated tokens before " + "commit") { + standard_fixture state; + REQUIRE(state.initialize()); + + REQUIRE(state.tokenize()); + const std::array invalid_audio = {11, 200}; + CHECK(state.detokenize(10, invalid_audio)); + CHECK(state.err == error_code(moshi::error::request_shape)); + CHECK(state.machine.is(sml::state)); + + const std::array valid_audio = {11, 12}; + CHECK(state.detokenize(100, valid_audio)); + CHECK(state.err == error_code(moshi::error::request_shape)); + CHECK(state.machine.is(sml::state)); +} + +TEST_CASE("speech Moshi tokenizer masks initial delayed audio explicitly") { + standard_fixture state{2}; + REQUIRE(state.initialize()); + const std::array generated = {11, 12}; + + REQUIRE(state.tokenize()); + REQUIRE(state.detokenize(10, generated)); + CHECK_FALSE(state.produced); + REQUIRE(state.tokenize()); + REQUIRE(state.detokenize(20, generated)); + CHECK_FALSE(state.produced); + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{20, -1, 200}); + REQUIRE(state.detokenize(30, generated)); + CHECK_FALSE(state.produced); +} + +TEST_CASE("speech Moshi tokenizer restores column-major voice cache") { + standard_fixture state; + REQUIRE(state.initialize()); + const std::array generated = {41, 42}; + REQUIRE(state.tokenize()); + REQUIRE(state.detokenize(40, generated)); + REQUIRE(state.tokenize()); + REQUIRE(state.detokenize(40, generated)); + + const std::array column_major = { + 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33, + }; + REQUIRE(state.machine.process_event(moshi::event::restore_cache{ + .column_major_cache = std::span{column_major}, + .offset = 2, + .error_out = state.err, + })); + + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{12, 22, 200}); +} + +TEST_CASE("speech Moshi tokenizer rejects invalid restored cache tokens") { + standard_fixture state; + REQUIRE(state.initialize()); + std::array column_major{}; + column_major[0] = 100; + CHECK(state.machine.process_event(moshi::event::restore_cache{ + .column_major_cache = std::span{column_major}, + .offset = 0, + .error_out = state.err, + })); + CHECK(state.err == error_code(moshi::error::request_shape)); +} + +TEST_CASE( + "speech Moshi tokenizer does not publish restored ungenerated output") { + standard_fixture state; + REQUIRE(state.initialize()); + + std::array column_major{}; + column_major.fill(-2); + REQUIRE(state.machine.process_event(moshi::event::restore_cache{ + .column_major_cache = std::span{column_major}, + .offset = 3, + .error_out = state.err, + })); + + REQUIRE(state.tokenize()); + const std::array generated = {41, 42}; + REQUIRE(state.detokenize(40, generated)); + CHECK_FALSE(state.produced); +} + +TEST_CASE("speech Moshi tokenizer advances a provided prompt frame") { + standard_fixture state; + REQUIRE(state.initialize()); + const std::array prompt = {50, 51, 52}; + REQUIRE(state.tokenize(prompt)); + REQUIRE(state.machine.process_event( + moshi::event::advance{.error_out = state.err})); + REQUIRE(state.tokenize()); + CHECK(state.model_tokens == std::array{-2, 200, 200}); +} + +TEST_CASE("speech Moshi tokenizer models lifecycle and shape failures") { + standard_fixture state; + std::array generated = {11, 12}; + + CHECK(state.tokenize()); + CHECK(state.err == error_code(moshi::error::uninitialized)); + REQUIRE(state.initialize()); + CHECK(state.initialize()); + CHECK(state.err == error_code(moshi::error::already_initialized)); + + std::array invalid_input = {1}; + CHECK(state.tokenize(invalid_input)); + CHECK(state.err == error_code(moshi::error::request_shape)); + REQUIRE(state.tokenize()); + + std::array invalid_generated = {1}; + CHECK(state.detokenize(10, invalid_generated)); + CHECK(state.err == error_code(moshi::error::request_shape)); + REQUIRE(state.detokenize(10, generated)); + + CHECK( + state.machine.process_event(moshi::event::reset{.error_out = state.err})); + CHECK(state.machine.is(sml::state)); +} + +TEST_CASE("speech Moshi tokenizer rejects invalid injected geometry") { + std::array delays = {0}; + std::array cache = {}; + moshi::sm machine{moshi::dependencies{ + .delays = std::span{delays}, + .cache = std::span{cache}, + .codebooks = 1, + .generated_audio_codebooks = 0, + .delayed_audio_codebooks = 0, + .cache_rows = 1, + .maximum_delay = 0, + .initial_delay_frames = 0, + .text_initial_token = 0, + .audio_initial_token = 0, + .token_zero = -1, + .token_ungenerated = -2, + }}; + int32_t err = error_code(moshi::error::none); + CHECK(machine.process_event(moshi::event::initialize{.error_out = err})); + CHECK(err == error_code(moshi::error::invalid_configuration)); + CHECK(machine.is(sml::state)); +} + +TEST_CASE("speech Moshi tokenizer rejects invalid token bounds and sentinels") { + const auto rejected = [](const int32_t text_initial_token, + const int32_t token_zero, + const int32_t token_ungenerated) { + std::array delays = {0, 1, 2}; + std::array cache{}; + moshi::sm machine{moshi::dependencies{ + .delays = std::span{delays}, + .cache = std::span{cache}, + .codebooks = 3, + .generated_audio_codebooks = 2, + .delayed_audio_codebooks = 2, + .cache_rows = 4, + .maximum_delay = 2, + .initial_delay_frames = 0, + .text_initial_token = text_initial_token, + .audio_initial_token = 200, + .token_zero = token_zero, + .token_ungenerated = token_ungenerated, + }}; + int32_t err = error_code(moshi::error::none); + return !machine.process_event(moshi::event::initialize{.error_out = err}) || + err == error_code(moshi::error::invalid_configuration); + }; + CHECK(rejected(0, -1, -2)); + CHECK(rejected(100, 0, -2)); + CHECK(rejected(100, -1, 0)); +} + +TEST_CASE("speech Moshi tokenizer dispatch remains allocation free") { + standard_fixture state; + REQUIRE(state.initialize()); + const std::array generated = {11, 12}; + + emel::test::allocation::allocation_scope allocations; + for (int32_t frame = 0; frame < 32; ++frame) { + REQUIRE(state.tokenize()); + REQUIRE(state.detokenize(frame, generated)); + } + CHECK(allocations.allocations() == 0u); +} + +TEST_CASE("speech Moshi tokenizer makes position overflow explicit") { + standard_fixture state; + REQUIRE(state.initialize()); + REQUIRE(state.tokenize()); + + moshi::action::context context{moshi::dependencies{ + .delays = std::span{state.delays}, + .cache = std::span{state.cache}, + .codebooks = standard_fixture::codebooks, + .generated_audio_codebooks = standard_fixture::generated_codebooks, + .delayed_audio_codebooks = standard_fixture::delayed_codebooks, + .cache_rows = standard_fixture::rows, + .maximum_delay = 2, + .initial_delay_frames = 0, + .text_initial_token = 100, + .audio_initial_token = 200, + .token_zero = -1, + .token_ungenerated = -2, + }}; + stateforward::sml::sm machine{ + context}; + machine.set_current_states(sml::state); + context.offset = std::numeric_limits::max(); + + std::array generated = {11, 12}; + std::array output = {}; + int32_t text_out = -1; + bool produced = false; + int32_t err = error_code(moshi::error::none); + moshi::event::detokenize_ctx detokenize_ctx{}; + CHECK(machine.process_event(moshi::event::detokenize_run{ + .request = + moshi::event::detokenize{ + .text_token = 10, + .audio_tokens = std::span{generated}, + .text_token_out = text_out, + .audio_tokens_out = std::span{output}, + .produced_out = produced, + .error_out = err, + }, + .ctx = detokenize_ctx, + })); + CHECK(err == error_code(moshi::error::position_overflow)); + + machine.set_current_states(sml::state); + std::array model_tokens = {}; + err = error_code(moshi::error::none); + CHECK(machine.process_event(moshi::event::tokenize{ + .audio_tokens = {}, + .model_tokens_out = std::span{model_tokens}, + .error_out = err, + })); + CHECK(err == error_code(moshi::error::position_overflow)); +} + +TEST_CASE("speech Moshi tokenizer exposes unexpected events and recovers") { + struct unknown_event {}; + + standard_fixture state; + REQUIRE(state.initialize()); + CHECK(state.machine.process_event(unknown_event{})); + CHECK(state.machine.is(sml::state)); + + CHECK(state.tokenize()); + CHECK(state.err == error_code(moshi::error::internal_error)); + REQUIRE( + state.machine.process_event(moshi::event::reset{.error_out = state.err})); + CHECK(state.machine.is(sml::state)); +} diff --git a/tests/text/generator/detail_tests.cpp b/tests/text/generator/detail_tests.cpp index 9d68266f..1c7e9e2c 100644 --- a/tests/text/generator/detail_tests.cpp +++ b/tests/text/generator/detail_tests.cpp @@ -4098,16 +4098,30 @@ TEST_CASE("generator_detail_nonflash_chunk4_prefill_matches_scalar_q8_k_on_" REQUIRE(chunk_fixture->backend.bound_logits.size() == scalar_fixture->backend.bound_logits.size()); float max_delta = 0.0f; + float max_reference_magnitude = 0.0f; + size_t max_delta_index = 0u; for (size_t idx = 0; idx < scalar_fixture->backend.bound_logits.size(); ++idx) { - max_delta = std::max(max_delta, - std::fabs(chunk_fixture->backend.bound_logits[idx] - - scalar_fixture->backend.bound_logits[idx])); + max_reference_magnitude = + std::max(max_reference_magnitude, + std::fabs(scalar_fixture->backend.bound_logits[idx])); + const float delta = std::fabs(chunk_fixture->backend.bound_logits[idx] - + scalar_fixture->backend.bound_logits[idx]); + if (delta > max_delta) { + max_delta = delta; + max_delta_index = idx; + } CHECK(chunk_fixture->backend.bound_logits[idx] == doctest::Approx(scalar_fixture->backend.bound_logits[idx]) .epsilon(1.0e-5f)); } - CHECK(max_delta <= 1.0e-4f); + INFO("max delta index=" + << max_delta_index + << " chunk=" << chunk_fixture->backend.bound_logits[max_delta_index] + << " scalar=" << scalar_fixture->backend.bound_logits[max_delta_index]); + const float max_allowed_delta = + std::max(1.0e-4f, max_reference_magnitude * 1.0e-6f); + CHECK(max_delta <= max_allowed_delta); #endif } @@ -4170,16 +4184,30 @@ TEST_CASE("generator_detail_nonflash_chunk8_prefill_matches_scalar_q8_k_on_" REQUIRE(chunk_fixture->backend.bound_logits.size() == scalar_fixture->backend.bound_logits.size()); float max_delta = 0.0f; + float max_reference_magnitude = 0.0f; + size_t max_delta_index = 0u; for (size_t idx = 0; idx < scalar_fixture->backend.bound_logits.size(); ++idx) { - max_delta = std::max(max_delta, - std::fabs(chunk_fixture->backend.bound_logits[idx] - - scalar_fixture->backend.bound_logits[idx])); + max_reference_magnitude = + std::max(max_reference_magnitude, + std::fabs(scalar_fixture->backend.bound_logits[idx])); + const float delta = std::fabs(chunk_fixture->backend.bound_logits[idx] - + scalar_fixture->backend.bound_logits[idx]); + if (delta > max_delta) { + max_delta = delta; + max_delta_index = idx; + } CHECK(chunk_fixture->backend.bound_logits[idx] == doctest::Approx(scalar_fixture->backend.bound_logits[idx]) .epsilon(1.0e-5f)); } - CHECK(max_delta <= 1.0e-4f); + INFO("max delta index=" + << max_delta_index + << " chunk=" << chunk_fixture->backend.bound_logits[max_delta_index] + << " scalar=" << scalar_fixture->backend.bound_logits[max_delta_index]); + const float max_allowed_delta = + std::max(1.0e-4f, max_reference_magnitude * 1.0e-6f); + CHECK(max_delta <= max_allowed_delta); #endif } diff --git a/tests/text/generator/legacy_compatibility_tests.cpp b/tests/text/generator/legacy_compatibility_tests.cpp deleted file mode 100644 index c4deeb26..00000000 --- a/tests/text/generator/legacy_compatibility_tests.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include - -#include "emel/generator/actions.hpp" -#include "emel/generator/context.hpp" -#include "emel/generator/detail.hpp" -#include "emel/generator/errors.hpp" -#include "emel/generator/events.hpp" -#include "emel/generator/guards.hpp" -#include "emel/generator/initializer/actions.hpp" -#include "emel/generator/initializer/context.hpp" -#include "emel/generator/initializer/detail.hpp" -#include "emel/generator/initializer/guards.hpp" -#include "emel/generator/initializer/sm.hpp" -#include "emel/generator/prefill/actions.hpp" -#include "emel/generator/prefill/context.hpp" -#include "emel/generator/prefill/detail.hpp" -#include "emel/generator/prefill/guards.hpp" -#include "emel/generator/prefill/sm.hpp" -#include "emel/generator/sm.hpp" -#include "emel/text/generator/sm.hpp" - -namespace { - -static_assert(std::is_same_v); -static_assert(std::is_same_v); -static_assert(std::is_same_v); -static_assert(std::is_same_v); -static_assert(std::is_same_v); -static_assert(std::is_same_v); - -} // namespace - -TEST_CASE("legacy generator headers forward to text generator surface") { - CHECK((std::is_same_v)); -} diff --git a/tools/bench/CMakeLists.txt b/tools/bench/CMakeLists.txt index 494b2513..4266ca9b 100644 --- a/tools/bench/CMakeLists.txt +++ b/tools/bench/CMakeLists.txt @@ -33,7 +33,8 @@ if(EMEL_BENCH_SUITE_FILTER STREQUAL "batch_planner" OR # subtree, which the moshi reference driver links against; an MLX-only # compare (EMEL_BENCH_SKIP_MOSHI_REFERENCE) builds no such driver. (EMEL_BENCH_SUITE_FILTER STREQUAL "speech_codec_mimi" AND - NOT EMEL_BENCH_SKIP_MOSHI_REFERENCE)) + NOT EMEL_BENCH_SKIP_MOSHI_REFERENCE) OR + EMEL_BENCH_SUITE_FILTER STREQUAL "speech_dialogue_moshi") set(EMEL_BENCH_NEEDS_LLAMA ON) endif() @@ -283,6 +284,9 @@ if(EMEL_BENCH_NEEDS_LLAMA) set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + if(EMEL_BENCH_SUITE_FILTER STREQUAL "speech_dialogue_moshi") + set(GGML_METAL OFF CACHE BOOL "CPU-only PersonaPlex reference lane" FORCE) + endif() set(LLAMA_BUILD_NUMBER 0 CACHE STRING "" FORCE) set(LLAMA_INSTALL_VERSION 0.0.0 CACHE STRING "" FORCE) add_subdirectory(${reference_impl_SOURCE_DIR}/ggml ggml) @@ -455,6 +459,7 @@ add_bench_runner_suite(tokenizer text/tokenizer/tokenizer_bench.cpp EMEL_BENCH_E if(NOT EMEL_BENCH_SUITE_FILTER STREQUAL "" AND NOT EMEL_BENCH_SUITE_FILTER STREQUAL "whisper_compare" AND + NOT EMEL_BENCH_SUITE_FILTER STREQUAL "speech_dialogue_moshi" AND BENCH_RUNNER_COMPILE_DEFINITIONS STREQUAL "") message(FATAL_ERROR "Unknown EMEL_BENCH_SUITE_FILTER='${EMEL_BENCH_SUITE_FILTER}'") endif() @@ -480,6 +485,10 @@ add_executable(mimi_emel_parity_runner ${CMAKE_CURRENT_SOURCE_DIR}/speech/mimi_emel_parity_runner.cpp ) +add_executable(personaplex_emel_runner + ${CMAKE_CURRENT_SOURCE_DIR}/speech/personaplex_emel_runner.cpp +) + target_link_libraries(mimi_emel_parity_runner PRIVATE emel_core @@ -494,6 +503,20 @@ target_include_directories(mimi_emel_parity_runner ${EMEL_ROOT}/include ) +target_link_libraries(personaplex_emel_runner + PRIVATE + emel_core + emel +) + +target_include_directories(personaplex_emel_runner + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${EMEL_ROOT} + ${EMEL_ROOT}/src + ${EMEL_ROOT}/include +) + # Reference-lane Mimi driver: moshi.cpp lib sources + ggml + SentencePiece. # Never linked into bench_runner or the EMEL lane; invoked as a subprocess # via reference_backends/moshi_cpp_mimi.json (two-lane isolation). @@ -641,6 +664,26 @@ add_test( COMMAND diarization_compare_tests ) +add_executable(personaplex_compare_tests + ${CMAKE_CURRENT_SOURCE_DIR}/personaplex_compare_tests.cpp +) + +target_include_directories(personaplex_compare_tests + PRIVATE + ${DOCTEST_INCLUDE_DIR} +) + +target_compile_definitions(personaplex_compare_tests + PRIVATE + BENCH_REPO_ROOT="${EMEL_ROOT}" + PERSONAPLEX_COMPARE_SCRIPT_PATH="${CMAKE_CURRENT_SOURCE_DIR}/personaplex_compare.py" +) + +add_test( + NAME personaplex_compare_tests + COMMAND personaplex_compare_tests +) + add_executable(whisper_benchmark_tests ${CMAKE_CURRENT_SOURCE_DIR}/whisper_benchmark_tests.cpp ) diff --git a/tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc b/tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc new file mode 100644 index 00000000..087136e7 Binary files /dev/null and b/tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc differ diff --git a/tools/bench/__pycache__/moshi_make_tiny_fixture.cpython-314.pyc b/tools/bench/__pycache__/moshi_make_tiny_fixture.cpython-314.pyc new file mode 100644 index 00000000..6c4c62b8 Binary files /dev/null and b/tools/bench/__pycache__/moshi_make_tiny_fixture.cpython-314.pyc differ diff --git a/tools/bench/bench_dependency_manifest.cpp b/tools/bench/bench_dependency_manifest.cpp index 0bcf6232..2389331f 100644 --- a/tools/bench/bench_dependency_manifest.cpp +++ b/tools/bench/bench_dependency_manifest.cpp @@ -380,6 +380,10 @@ constexpr std::array k_records{ kind::script, "tools/bench/moshi_gguf_convert.py", "emel_lane_converter"}, + dependency_record{"speech_codec_mimi", + kind::config, + "tools/bench/personaplex-inference.json", + "inference_contract"}, dependency_record{"speech_codec_mimi", kind::config, "tools/bench/moshi_reference_ref.txt", @@ -396,6 +400,10 @@ constexpr std::array k_records{ kind::source, "tools/bench/speech/lm_moshi_bench.cpp", "suite_source"}, + dependency_record{"speech_lm_moshi", + kind::source, + "tools/bench/speech/personaplex_emel_runner.cpp", + "e2e_emel_lane_runner"}, dependency_record{"speech_lm_moshi", kind::source, "src/emel/model/moshi", @@ -412,6 +420,10 @@ constexpr std::array k_records{ kind::script, "tools/bench/moshi_gguf_convert.py", "emel_lane_converter"}, + dependency_record{"speech_lm_moshi", + kind::config, + "tools/bench/personaplex-inference.json", + "inference_contract"}, // MLX-only inputs map to the MLX suite: quality_gates.sh runs the // personaplex-mlx compare lane only for speech_codec_mimi_mlx, so routing // these through the generic mimi suite would gate MLX regressions with diff --git a/tools/bench/bench_runner_tests.cpp b/tools/bench/bench_runner_tests.cpp index b037cf5a..cff754b8 100644 --- a/tools/bench/bench_runner_tests.cpp +++ b/tools/bench/bench_runner_tests.cpp @@ -1151,6 +1151,7 @@ TEST_CASE( bool has_speech_lm_script = false; bool has_speech_lm_setup = false; bool has_speech_lm_moshi_binding = false; + bool has_personaplex_emel_runner = false; for (const auto &record : speech_lm_records) { has_speech_lm_script = has_speech_lm_script || record.path == "scripts/bench_moshi_lm_compare.sh"; @@ -1160,10 +1161,15 @@ TEST_CASE( has_speech_lm_moshi_binding = has_speech_lm_moshi_binding || record.path == std::string_view{"src/emel/model/moshi"}; + has_personaplex_emel_runner = + has_personaplex_emel_runner || + record.path == std::string_view{ + "tools/bench/speech/personaplex_emel_runner.cpp"}; } CHECK(has_speech_lm_script); CHECK(has_speech_lm_setup); CHECK(has_speech_lm_moshi_binding); + CHECK(has_personaplex_emel_runner); CHECK(manifest::records_for("missing_runner").empty()); } @@ -1185,6 +1191,10 @@ TEST_CASE( CHECK(rendered.find("record runner=speech_lm_moshi kind=source " "path=tools/bench/speech/lm_moshi_bench.cpp") != std::string::npos); + CHECK(rendered.find( + "record runner=speech_lm_moshi kind=source " + "path=tools/bench/speech/personaplex_emel_runner.cpp") != + std::string::npos); const std::filesystem::path manifest_path = std::filesystem::temp_directory_path() / "emel-bench-runner-tests" / diff --git a/tools/bench/dependency_manifest.txt b/tools/bench/dependency_manifest.txt index cd61e9cc..9d7df6f6 100644 --- a/tools/bench/dependency_manifest.txt +++ b/tools/bench/dependency_manifest.txt @@ -102,14 +102,17 @@ record runner=speech_codec_mimi kind=fixture path=tests/models/mimi-tiny.gguf re record runner=speech_codec_mimi kind=script path=tools/bench/mimi_compare.py reason=operator_compare record runner=speech_codec_mimi kind=script path=scripts/bench_mimi_compare.sh reason=compare_wrapper record runner=speech_codec_mimi kind=script path=tools/bench/moshi_gguf_convert.py reason=emel_lane_converter +record runner=speech_codec_mimi kind=config path=tools/bench/personaplex-inference.json reason=inference_contract record runner=speech_codec_mimi kind=config path=tools/bench/moshi_reference_ref.txt reason=reference_pin record runner=speech_codec_mimi kind=config path=tools/bench/reference_backends/moshi_cpp_mimi.json reason=reference_backend_descriptor record runner=speech_codec_mimi kind=script path=scripts/setup_moshi_cpp_reference.sh reason=reference_setup record runner=speech_lm_moshi kind=source path=tools/bench/speech/lm_moshi_bench.cpp reason=suite_source +record runner=speech_lm_moshi kind=source path=tools/bench/speech/personaplex_emel_runner.cpp reason=e2e_emel_lane_runner record runner=speech_lm_moshi kind=source path=src/emel/model/moshi reason=model_binding record runner=speech_lm_moshi kind=script path=scripts/bench_moshi_lm_compare.sh reason=compare_wrapper record runner=speech_lm_moshi kind=script path=scripts/setup_moshi_cpp_reference.sh reason=reference_setup record runner=speech_lm_moshi kind=script path=tools/bench/moshi_gguf_convert.py reason=emel_lane_converter +record runner=speech_lm_moshi kind=config path=tools/bench/personaplex-inference.json reason=inference_contract record runner=speech_codec_mimi_mlx kind=script path=tools/bench/mimi_compare.py reason=operator_compare record runner=speech_codec_mimi_mlx kind=script path=scripts/bench_mimi_compare.sh reason=compare_wrapper record runner=speech_codec_mimi_mlx kind=script path=scripts/setup_personaplex_mlx_reference.sh reason=reference_setup diff --git a/tools/bench/moshi_gguf_convert.py b/tools/bench/moshi_gguf_convert.py index 3060c479..54560b7d 100644 --- a/tools/bench/moshi_gguf_convert.py +++ b/tools/bench/moshi_gguf_convert.py @@ -13,9 +13,11 @@ - CRC-mangled tensor names restored to their canonical long names - for `lm`: the SentencePiece text tokenizer embedded as standard `tokenizer.ggml.*` keys so emel's generic vocab loader consumes it + - optionally for `lm`: selected Q4_K mat-vec weights rewritten to EMEL's + packed aarch64 CPU layout, exposed in GGUF as tensor dtype metadata -Tensor payload bytes are streamed through unchanged (same order, same -alignment, same relative offsets). +Tensor payload bytes are streamed through unchanged unless an explicit rewrite +option is selected. stdlib-only by design, matching tools/bench/whisper_normalize_model.py. """ @@ -26,6 +28,7 @@ import hashlib import json import math +import re import struct import zlib from dataclasses import dataclass @@ -45,9 +48,24 @@ # tensor dtype codes (ggml enumeration) GGML_DTYPE_F32 = 0 GGML_DTYPE_F16 = 1 +GGML_DTYPE_Q4_0 = 2 +GGML_DTYPE_Q4_1 = 3 +GGML_DTYPE_Q5_0 = 6 +GGML_DTYPE_Q5_1 = 7 GGML_DTYPE_Q8_0 = 8 +GGML_DTYPE_Q2_K = 10 +GGML_DTYPE_Q3_K = 11 +GGML_DTYPE_Q4_K = 12 +GGML_DTYPE_Q5_K = 13 +GGML_DTYPE_Q6_K = 14 +GGML_DTYPE_Q8_K = 15 +EMEL_DTYPE_Q4_K_X8_BL8 = 42 Q8_0_BLOCK = 32 Q8_0_BLOCK_BYTES = 34 # fp16 scale + 32 int8 +QK_K = 256 +Q4_K_BLOCK_BYTES = 144 +Q4_K_X8_ROWS = 8 +Q4_K_X8_BLOCK_BYTES = 1152 # weight classes safe to quantize in a mimi component: the transformer and # RVQ projections (uniform row length, k % 32 == 0, consumed as mat-vecs by @@ -130,6 +148,13 @@ class TensorInfo: offset: int +@dataclass(frozen=True) +class TensorPayloadPlan: + source: TensorInfo + mode: str + padding: int + + class Reader: def __init__(self, data: bytes) -> None: self.data = data @@ -309,8 +334,22 @@ def require_elements(infos: list[TensorInfo], name: str, f"elements), the runtime bind consumes {elements}") -def load_lm_config(path: Path) -> dict: +def inject_inference_config(config: dict, + inference_config_path: Path | None) -> None: + if inference_config_path is None: + return + inference = json.loads( + inference_config_path.read_text(encoding="utf-8")) + if not isinstance(inference, dict): + raise ValueError( + f"{inference_config_path}: inference config must be an object") + config["inference"] = inference + + +def load_lm_config(path: Path, + inference_config_path: Path | None = None) -> dict: config = json.loads(path.read_text(encoding="utf-8")) + inject_inference_config(config, inference_config_path) required = [ "card", "n_q", "dep_q", "text_card", "existing_text_padding_id", "dim", "num_heads", "num_layers", "hidden_scale", "causal", "context", @@ -326,10 +365,6 @@ def load_lm_config(path: Path) -> dict: raise ValueError( f"{path}: unsupported {key}={config[key]!r} " f"(only {SUPPORTED_LM_VALUES[key]!r} is supported)") - if config.get("depformer_weights_per_step_schedule"): - raise ValueError( - f"{path}: depformer_weights_per_step_schedule is not supported") - if config["dep_q"] > 0: dep_required = [ "depformer_dim", "depformer_num_heads", "depformer_num_layers", @@ -347,6 +382,24 @@ def load_lm_config(path: Path) -> dict: f"(only {SUPPORTED_LM_VALUES[key]!r} is supported)") if not config["depformer_multi_linear"]: raise ValueError(f"{path}: depformer_multi_linear=false is unsupported") + schedule = config.get("depformer_weights_per_step_schedule") + if schedule is not None: + if not isinstance(schedule, list): + raise ValueError( + f"{path}: depformer_weights_per_step_schedule must be a list") + if len(schedule) != config["dep_q"]: + raise ValueError( + f"{path}: depformer_weights_per_step_schedule has " + f"{len(schedule)} entries, expected dep_q={config['dep_q']}") + if not all(isinstance(value, int) for value in schedule): + raise ValueError( + f"{path}: depformer_weights_per_step_schedule must contain only " + "integers") + for index, value in enumerate(schedule): + if value < 0 or value >= config["dep_q"]: + raise ValueError( + f"{path}: depformer_weights_per_step_schedule[{index}]=" + f"{value} must be within [0, dep_q={config['dep_q']})") delays = config["delays"] if len(delays) != config["n_q"] + 1: @@ -393,9 +446,109 @@ def load_lm_config(path: Path) -> dict: raise ValueError( f"{path}: depformer_dim={config['depformer_dim']} is not divisible " f"by depformer_num_heads={config['depformer_num_heads']}") + normalize_lm_inference_config(config, path) return config +def normalize_lm_inference_config(config: dict, path: Path) -> None: + """Normalize explicit LMGen inference knobs into emitted GGUF metadata.""" + if not config.get("depformer_weights_per_step", False): + return + + inference = config.get("inference", {}) + + def get_value(key: str): + flat_key = f"inference_{key}" + if flat_key in config: + return config[flat_key] + return inference.get(key) + + inference_dep_q = get_value("dep_q") + if inference_dep_q is None: + raise ValueError( + f"{path}: depformer_weights_per_step=true requires " + "inference_dep_q metadata") + + prompt_tokens = get_value("prompt_tokens") + if prompt_tokens is None: + raise ValueError( + f"{path}: depformer_weights_per_step=true requires " + "inference_prompt_tokens metadata") + + pre_silence = get_value("pre_text_silence_frames") + if pre_silence is None: + raise ValueError( + f"{path}: depformer_weights_per_step=true requires " + "inference_pre_text_silence_frames metadata") + + post_silence = get_value("post_text_silence_frames") + if post_silence is None: + raise ValueError( + f"{path}: depformer_weights_per_step=true requires " + "inference_post_text_silence_frames metadata") + + if not isinstance(inference_dep_q, int): + raise ValueError(f"{path}: inference_dep_q must be an integer") + if inference_dep_q <= 0 or inference_dep_q > config["dep_q"]: + raise ValueError( + f"{path}: inference_dep_q={inference_dep_q} must be within " + f"[1, dep_q={config['dep_q']}]") + if not isinstance(pre_silence, int) or pre_silence < 0: + raise ValueError( + f"{path}: inference_pre_text_silence_frames must be a non-negative " + "integer") + if not isinstance(post_silence, int) or post_silence < 0: + raise ValueError( + f"{path}: inference_post_text_silence_frames must be a non-negative " + "integer") + if not isinstance(prompt_tokens, list): + raise ValueError(f"{path}: inference_prompt_tokens must be a list") + if len(prompt_tokens) != config["n_q"] + 1: + raise ValueError( + f"{path}: inference_prompt_tokens has {len(prompt_tokens)} entries, " + f"expected n_q+1 = {config['n_q'] + 1}") + if any(not isinstance(token, int) for token in prompt_tokens): + raise ValueError( + f"{path}: inference_prompt_tokens must contain only integers") + if not 0 <= prompt_tokens[0] < config["text_card"]: + raise ValueError( + f"{path}: inference_prompt_tokens[0]={prompt_tokens[0]} must be " + f"within [0, text_card={config['text_card']})") + for index, token in enumerate(prompt_tokens[1:], start=1): + if not 0 <= token < config["card"]: + raise ValueError( + f"{path}: inference_prompt_tokens[{index}]={token} must be within " + f"[0, card={config['card']})") + + config["inference_dep_q"] = inference_dep_q + config["inference_prompt_tokens"] = list(prompt_tokens) + config["inference_pre_text_silence_frames"] = pre_silence + config["inference_post_text_silence_frames"] = post_silence + + +def resolve_mimi_n_q(config_path: Path | None, + inference_config_path: Path | None = None) -> int: + if config_path is None: + raise ValueError("Mimi conversion requires --config or --mimi-n-q") + config = json.loads(config_path.read_text(encoding="utf-8")) + inject_inference_config(config, inference_config_path) + if "n_q" not in config: + raise ValueError(f"{config_path}: mimi conversion config requires n_q") + if not isinstance(config["n_q"], int): + raise ValueError(f"{config_path}: n_q must be an integer") + if config.get("depformer_weights_per_step", False): + normalize_lm_inference_config(config, config_path) + return config["inference_dep_q"] + return config["n_q"] + + +def depformer_weight_count(config: dict) -> int: + schedule = config.get("depformer_weights_per_step_schedule") + if schedule: + return max(schedule) + 1 + return config["dep_q"] if config.get("depformer_multi_linear") else 1 + + def cross_check_lm(infos: list[TensorInfo], config: dict) -> None: dim = config["dim"] text_card = config["text_card"] @@ -443,8 +596,24 @@ def require_block(prefix: str, layer: int, with_gating: bool) -> None: f"{layer_count} layer tensors present)") require_block("lm.transformer", layer, with_gating=True) if config["dep_q"] > 0: - if count_prefixed(infos, "lm.depformer_in.") != config["dep_q"]: - raise ValueError("lm.depformer_in.* count does not match config dep_q") + schedule = config.get("depformer_weights_per_step_schedule") + if schedule: + required_depformer_indices = set(schedule) + elif config.get("depformer_multi_linear"): + required_depformer_indices = set(range(config["dep_q"])) + else: + required_depformer_indices = {0} + required_depformer_names = { + f"lm.depformer_in.{index}.weight" + for index in required_depformer_indices + } + present_depformer_names = { + info.name for info in infos if info.name.startswith("lm.depformer_in.") + } + if present_depformer_names != required_depformer_names: + raise ValueError( + "lm.depformer_in.* names do not match scheduled weight indices " + f"{sorted(required_depformer_indices)}") if count_prefixed(infos, "lm.linears.") != config["dep_q"]: raise ValueError("lm.linears.* count does not match config dep_q") # The C++ contract also requires the depformer text-embedding family for @@ -932,6 +1101,14 @@ def append_lm_kv(kv: KvWriter, config: dict) -> None: kv.boolean("moshi.lm.demux_second_stream", config.get("demux_second_stream", False)) kv.i32_array("moshi.lm.delays", list(config["delays"])) + if config.get("depformer_weights_per_step", False): + kv.u32("moshi.lm.inference.dep_q", config["inference_dep_q"]) + kv.u32("moshi.lm.inference.pre_text_silence_frames", + config["inference_pre_text_silence_frames"]) + kv.u32("moshi.lm.inference.post_text_silence_frames", + config["inference_post_text_silence_frames"]) + kv.i32_array("moshi.lm.inference.prompt_tokens", + list(config["inference_prompt_tokens"])) kv.u32("moshi.lm.extra_heads.num_heads", config.get("extra_heads_num_heads", 0)) if "model_type" in config: @@ -950,6 +1127,9 @@ def append_lm_kv(kv: KvWriter, config: dict) -> None: config["depformer_multi_linear"]) kv.boolean("moshi.lm.depformer.weights_per_step", config["depformer_weights_per_step"]) + if config.get("depformer_weights_per_step_schedule") is not None: + kv.i32_array("moshi.lm.depformer.weights_per_step_schedule", + list(config["depformer_weights_per_step_schedule"])) kv.u32("moshi.lm.depformer.low_rank_embeddings", config.get("depformer_low_rank_embeddings") or 0) @@ -995,11 +1175,196 @@ def tensor_data_bytes(info: TensorInfo) -> int: return elements * 4 if info.dtype == GGML_DTYPE_F16: return elements * 2 + if info.dtype == GGML_DTYPE_Q4_0: + return elements // 32 * 18 + if info.dtype == GGML_DTYPE_Q4_1: + return elements // 32 * 20 + if info.dtype == GGML_DTYPE_Q5_0: + return elements // 32 * 22 + if info.dtype == GGML_DTYPE_Q5_1: + return elements // 32 * 24 if info.dtype == GGML_DTYPE_Q8_0: return elements // Q8_0_BLOCK * Q8_0_BLOCK_BYTES + if info.dtype == GGML_DTYPE_Q2_K: + return elements // QK_K * 84 + if info.dtype == GGML_DTYPE_Q3_K: + return elements // QK_K * 110 + if info.dtype == GGML_DTYPE_Q4_K: + return elements // QK_K * Q4_K_BLOCK_BYTES + if info.dtype == GGML_DTYPE_Q5_K: + return elements // QK_K * 176 + if info.dtype == GGML_DTYPE_Q6_K: + return elements // QK_K * 210 + if info.dtype == GGML_DTYPE_Q8_K: + return elements // QK_K * 292 + if info.dtype == EMEL_DTYPE_Q4_K_X8_BL8: + if len(info.dims) < 2: + raise ValueError( + f"packed q4_k_x8_bl8 tensor {info.name!r} has dims {info.dims}") + cols, rows = info.dims[0], info.dims[1] + groups = (rows + Q4_K_X8_ROWS - 1) // Q4_K_X8_ROWS + return groups * (cols // QK_K) * Q4_K_X8_BLOCK_BYTES raise ValueError(f"unsupported dtype {info.dtype} for {info.name}") +def get_scale_min_k4(index: int, scales: bytes) -> tuple[int, int]: + if index < 4: + return scales[index] & 63, scales[index + 4] & 63 + scale = (scales[index + 4] & 0x0F) | ((scales[index - 4] >> 6) << 4) + minimum = (scales[index + 4] >> 4) | ((scales[index] >> 6) << 4) + return scale, minimum + + +def write_q4_kx8_scale_chunk(out: bytearray, base: int, scales: list[int], + mins: list[int]) -> None: + out[base + 0] = (scales[0] & 63) + ((scales[4] & 48) << 2) + out[base + 1] = (scales[1] & 63) + ((scales[5] & 48) << 2) + out[base + 2] = (scales[2] & 63) + ((scales[6] & 48) << 2) + out[base + 3] = (scales[3] & 63) + ((scales[7] & 48) << 2) + out[base + 4] = (mins[0] & 63) + ((mins[4] & 48) << 2) + out[base + 5] = (mins[1] & 63) + ((mins[5] & 48) << 2) + out[base + 6] = (mins[2] & 63) + ((mins[6] & 48) << 2) + out[base + 7] = (mins[3] & 63) + ((mins[7] & 48) << 2) + out[base + 8] = (scales[4] & 15) + ((mins[4] & 15) << 4) + out[base + 9] = (scales[5] & 15) + ((mins[5] & 15) << 4) + out[base + 10] = (scales[6] & 15) + ((mins[6] & 15) << 4) + out[base + 11] = (scales[7] & 15) + ((mins[7] & 15) << 4) + + +def pack_q4_k_rows_x8_bl8(raw: bytes, rows: int, cols: int) -> bytes: + if cols <= 0 or rows <= 0 or cols % QK_K != 0: + raise ValueError( + f"cannot pack q4_k matrix with rows={rows}, cols={cols}") + block_count = cols // QK_K + expected = rows * block_count * Q4_K_BLOCK_BYTES + if len(raw) != expected: + raise ValueError( + f"q4_k payload has {len(raw)} bytes, expected {expected}") + + zero_block = bytes(Q4_K_BLOCK_BYTES) + out = bytearray() + group_count = (rows + Q4_K_X8_ROWS - 1) // Q4_K_X8_ROWS + for group in range(group_count): + row_base = group * Q4_K_X8_ROWS + for block in range(block_count): + blocks: list[bytes] = [] + for lane in range(Q4_K_X8_ROWS): + logical_row = row_base + lane + if logical_row < rows: + offset = (logical_row * block_count + block) * Q4_K_BLOCK_BYTES + blocks.append(raw[offset:offset + Q4_K_BLOCK_BYTES]) + else: + blocks.append(zero_block) + + for lane in range(Q4_K_X8_ROWS): + out.extend(blocks[lane][0:2]) + for lane in range(Q4_K_X8_ROWS): + out.extend(blocks[lane][2:4]) + + packed_scales = bytearray(12 * Q4_K_X8_ROWS) + for scale_index in range(4): + scales: list[int] = [] + mins: list[int] = [] + for lane in range(Q4_K_X8_ROWS): + scales_bytes = blocks[lane][4:4 + 12] + scales.append(scales_bytes[scale_index] & 63) + mins.append(scales_bytes[scale_index + 4] & 63) + write_q4_kx8_scale_chunk(packed_scales, scale_index * 12, scales, + mins) + for scale_index in range(4): + scales = [] + mins = [] + for lane in range(Q4_K_X8_ROWS): + scales_bytes = blocks[lane][4:4 + 12] + scales.append(((scales_bytes[scale_index] & 192) >> 2) | + (scales_bytes[scale_index + 8] & 15)) + mins.append(((scales_bytes[scale_index + 4] & 192) >> 2) | + ((scales_bytes[scale_index + 8] & 240) >> 4)) + write_q4_kx8_scale_chunk(packed_scales, 48 + scale_index * 12, + scales, mins) + out.extend(packed_scales) + + packed_qs = bytearray((QK_K // 2) * Q4_K_X8_ROWS) + interleave = 8 + end = (QK_K * 4) // interleave + for index in range(end): + src_row = index % Q4_K_X8_ROWS + src_offset = (index // Q4_K_X8_ROWS) * interleave + dst_offset = index * interleave + qs = blocks[src_row][16:16 + QK_K // 2] + packed_qs[dst_offset:dst_offset + interleave] = \ + qs[src_offset:src_offset + interleave] + out.extend(packed_qs) + + return bytes(out) + + +LM_Q4_K_PACKABLE_PATTERNS = ( + re.compile(r"^lm\.text_linear\.weight$"), + re.compile(r"^lm\.depformer_in\.\d+\.weight$"), + re.compile(r"^lm\.linears\.\d+\.weight$"), + re.compile( + r"^lm\.transformer\.layers\.\d+\.self_attn\.in_proj_weight$"), + re.compile( + r"^lm\.transformer\.layers\.\d+\.self_attn\.in_projs\.0\.weight$"), + re.compile( + r"^lm\.transformer\.layers\.\d+\.self_attn\.out_projs\.0\.weight$"), + re.compile( + r"^lm\.transformer\.layers\.\d+\.gating\.linear_in\.weight$"), + re.compile( + r"^lm\.transformer\.layers\.\d+\.gating\.linear_out\.weight$"), + re.compile( + r"^lm\.depformer\.layers\.\d+\.self_attn\.in_proj_weight$"), + re.compile( + r"^lm\.depformer\.layers\.\d+\.self_attn\.in_projs\.\d+\.weight$"), + re.compile( + r"^lm\.depformer\.layers\.\d+\.self_attn\.out_projs\.\d+\.weight$"), + re.compile( + r"^lm\.depformer\.layers\.\d+\.gating\.\d+\.linear_in\.weight$"), + re.compile( + r"^lm\.depformer\.layers\.\d+\.gating\.\d+\.linear_out\.weight$"), +) + + +def is_lm_q4_k_packable(info: TensorInfo) -> bool: + return (info.dtype == GGML_DTYPE_Q4_K and len(info.dims) == 2 and + info.dims[0] % QK_K == 0 and info.dims[1] > 0 and + any(pattern.match(info.name) for pattern in + LM_Q4_K_PACKABLE_PATTERNS)) + + +def plan_lm_q4_k_packing( + infos: list[TensorInfo], + pack_lm_q4_k: str) -> tuple[list[TensorInfo], list[TensorPayloadPlan], int]: + if pack_lm_q4_k != "q4_k_x8_bl8": + raise ValueError(f"unsupported --pack-lm-q4-k type: {pack_lm_q4_k}") + + new_infos: list[TensorInfo] = [] + plans: list[TensorPayloadPlan] = [] + offset = 0 + packed = 0 + for info in infos: + dtype = info.dtype + mode = "raw" + if is_lm_q4_k_packable(info): + dtype = EMEL_DTYPE_Q4_K_X8_BL8 + mode = "q4_k_x8_bl8" + packed += 1 + new_info = TensorInfo( + name=info.name, dims=info.dims, dtype=dtype, offset=offset) + size = tensor_data_bytes(new_info) + offset += size + pad = (-offset) % GGUF_ALIGNMENT + offset += pad + new_infos.append(new_info) + plans.append(TensorPayloadPlan(source=info, mode=mode, padding=pad)) + + if packed == 0: + raise ValueError( + "--pack-lm-q4-k selected but no matching LM Q4_K tensors were found") + return new_infos, plans, packed + + def is_mimi_quantizable(info: TensorInfo) -> bool: # effective row length: conv1x1 projections store [1, in, out], linear # projections store [in, out]; blocks must not straddle rows. 1-D tensors @@ -1123,14 +1488,79 @@ def write_enriched(source: Path, output: Path, infos: list[TensorInfo], dst.write(chunk) +def copy_exact(src, dst, size: int) -> None: + remaining = size + while remaining > 0: + chunk = src.read(min(remaining, 8 * 1024 * 1024)) + if not chunk: + raise ValueError("unexpected EOF while copying GGUF tensor payload") + dst.write(chunk) + remaining -= len(chunk) + + +def write_rewritten_from_source(output: Path, infos: list[TensorInfo], + plans: list[TensorPayloadPlan], source: Path, + data_start: int, kv: KvWriter) -> None: + if len(infos) != len(plans): + raise ValueError("tensor payload plan count does not match tensor infos") + out = bytearray() + out.extend(GGUF_MAGIC) + out.extend(struct.pack(" dict: + quantize: str | None = None, + pack_lm_q4_k: str | None = None, + inference_config_path: Path | None = None, + mimi_n_q: int | None = None) -> dict: infos, data_start = read_raw_gguf(source) component = detect_component(infos) if quantize is not None and component != "mimi": raise ValueError("--quantize currently supports mimi components only") + if pack_lm_q4_k is not None and component != "lm": + raise ValueError("--pack-lm-q4-k supports lm components only") + if mimi_n_q is not None and component != "mimi": + raise ValueError("--mimi-n-q supports Mimi components only") + if mimi_n_q is not None and mimi_n_q <= 0: + raise ValueError("--mimi-n-q must be a positive integer") + if quantize is not None and pack_lm_q4_k is not None: + raise ValueError("--quantize and --pack-lm-q4-k are component-specific") kv = KvWriter() kv.string("general.architecture", "moshi") @@ -1142,8 +1572,8 @@ def convert(source: Path, output: Path, config_path: Path | None, raise ValueError("--config is required for lm components") if tokenizer_path is None: raise ValueError("--tokenizer is required for lm components") - config = load_lm_config(config_path) - num_weights = config["dep_q"] if config.get("depformer_multi_linear") else 1 + config = load_lm_config(config_path, inference_config_path) + num_weights = depformer_weight_count(config) candidates = transformer_layer_candidates( "lm.transformer.", config["num_layers"], 1) candidates += transformer_layer_candidates( @@ -1152,13 +1582,14 @@ def convert(source: Path, output: Path, config_path: Path | None, cross_check_lm(infos, config) append_lm_kv(kv, config) append_tokenizer_kv(kv, parse_sentencepiece_model(tokenizer_path)) + if pack_lm_q4_k is not None: + kv.string("moshi.lm.tensor_packing", pack_lm_q4_k) elif component == "mimi": preset = dict(MIMI_PRESET) if mimi_preset_path is not None: preset.update(json.loads(mimi_preset_path.read_text(encoding="utf-8"))) - n_q = 16 - if config_path is not None: - n_q = json.loads(config_path.read_text(encoding="utf-8"))["n_q"] + n_q = (mimi_n_q if mimi_n_q is not None else + resolve_mimi_n_q(config_path, inference_config_path)) layers = preset["transformer_num_layers"] candidates = transformer_layer_candidates( "mimi.encoder_transformer.transformer.", layers, 1) @@ -1176,10 +1607,14 @@ def convert(source: Path, output: Path, config_path: Path | None, kv.string("moshi.voice.format", VOICE_FORMAT) quantized_count = 0 + packed_count = 0 if quantize is not None: infos, payloads, quantized_count = quantize_mimi_tensors( source, infos, data_start, quantize) write_rewritten(output, infos, payloads, kv) + elif pack_lm_q4_k is not None: + infos, plans, packed_count = plan_lm_q4_k_packing(infos, pack_lm_q4_k) + write_rewritten_from_source(output, infos, plans, source, data_start, kv) else: write_enriched(source, output, infos, data_start, kv) manifest = { @@ -1193,6 +1628,14 @@ def convert(source: Path, output: Path, config_path: Path | None, if quantize is not None: manifest["quantization"] = quantize manifest["quantized_tensors"] = quantized_count + if pack_lm_q4_k is not None: + manifest["lm_tensor_packing"] = pack_lm_q4_k + manifest["packed_tensors"] = packed_count + if inference_config_path is not None: + manifest["inference_config"] = str(inference_config_path) + manifest["inference_config_sha256"] = sha256_file(inference_config_path) + if mimi_n_q is not None: + manifest["mimi_n_q"] = mimi_n_q return manifest @@ -1204,7 +1647,14 @@ def parse_args() -> argparse.Namespace: help="enriched emel moshi GGUF to write") parser.add_argument("--config", type=Path, default=None, help="moshi config JSON (personaplex-config.json); " - "required for lm components") + "required for lm components and for mimi unless " + "--mimi-n-q is injected") + parser.add_argument("--inference-config", type=Path, default=None, + help="injected inference-only JSON overlay for prompt " + "tokens, public codebooks, and silence framing") + parser.add_argument("--mimi-n-q", type=int, default=None, + help="explicit injected Mimi codebook count; used to " + "keep full-codec and PersonaPlex artifacts separate") parser.add_argument("--tokenizer", type=Path, default=None, help="SentencePiece .model file; required for lm " "components") @@ -1217,13 +1667,18 @@ def parse_args() -> argparse.Namespace: help="quantize the mimi projection weight classes " "(transformer + RVQ projections) to the given " "block format; requires numpy") + parser.add_argument("--pack-lm-q4-k", choices=("q4_k_x8_bl8",), + default=None, + help="rewrite selected LM Q4_K mat-vec/logit weights " + "to EMEL packed CPU tensor dtype metadata") return parser.parse_args() def main() -> int: args = parse_args() manifest = convert(args.source, args.output, args.config, args.tokenizer, - args.mimi_preset, args.quantize) + args.mimi_preset, args.quantize, args.pack_lm_q4_k, + args.inference_config, args.mimi_n_q) if args.manifest is not None: args.manifest.parent.mkdir(parents=True, exist_ok=True) args.manifest.write_text(json.dumps(manifest, indent=2) + "\n", diff --git a/tools/bench/moshi_make_tiny_fixture.py b/tools/bench/moshi_make_tiny_fixture.py index 75c0c70c..ea793cc8 100644 --- a/tools/bench/moshi_make_tiny_fixture.py +++ b/tools/bench/moshi_make_tiny_fixture.py @@ -8,7 +8,7 @@ exercises the loader-rejection path and the converter round-trip) - moshi-tiny-lm.gguf enriched LM fixture (via moshi_gguf_convert) - - mimi-tiny.gguf enriched Mimi codec fixture + - mimi-tiny.gguf enriched PersonaPlex-active Mimi codec fixture - moshi-tiny-voice.gguf enriched PersonaPlex voice-prompt fixture All payloads are deterministic (fixed LCG seed); regenerating the fixtures is @@ -58,6 +58,10 @@ "depformer_gating": "silu", "depformer_pos_emb": "none", "depformer_weights_per_step": True, + "inference_dep_q": 2, + "inference_prompt_tokens": [3, 4, 5, 6, 7], + "inference_pre_text_silence_frames": 1, + "inference_post_text_silence_frames": 1, "cross_attention": False, "model_type": "personaplex", "tokenizer_name": "moshi-tokenizer-tiny.model", @@ -212,6 +216,8 @@ def mimi_tensors(rng: Lcg, preset=TINY_MIMI_PRESET, tuple[str, tuple[int, ...], int, bytes]]: dim = preset["dim"] card = preset["card"] + # The raw synthetic cache carries the full tiny LM codebook count. The + # converter declares the active PersonaPlex inference count in GGUF metadata. n_q = TINY_CONFIG["n_q"] def t(name, *dims): diff --git a/tools/bench/personaplex-inference.json b/tools/bench/personaplex-inference.json new file mode 100644 index 00000000..c1c42155 --- /dev/null +++ b/tools/bench/personaplex-inference.json @@ -0,0 +1,30 @@ +{ + "dep_q": 8, + "prompt_tokens": [ + 3, + 948, + 243, + 1178, + 546, + 1736, + 1030, + 1978, + 2008, + 430, + 1268, + 381, + 1611, + 1095, + 1495, + 56, + 472 + ], + "pre_text_silence_frames": 6, + "post_text_silence_frames": 6, + "max_blocks": 256, + "block_tokens": 4, + "prompt_text_token": -1, + "frame_samples": 1920, + "audio_top_k": 250, + "text_top_k": 25 +} diff --git a/tools/bench/personaplex_compare.py b/tools/bench/personaplex_compare.py new file mode 100755 index 00000000..842c917e --- /dev/null +++ b/tools/bench/personaplex_compare.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""CPU-only fixed-seed PersonaPlex E2E comparison (personaplex_compare/v1).""" + +import argparse +import hashlib +import json +import math +import re +import struct +import subprocess +import sys +import time +import wave +from pathlib import Path + + +def run_lane(command: list[str], stdout_path: Path, stderr_path: Path) -> float: + started = time.monotonic() + with stdout_path.open("w") as stdout, stderr_path.open("w") as stderr: + result = subprocess.run(command, stdout=stdout, stderr=stderr, check=False) + elapsed = time.monotonic() - started + if result.returncode != 0: + raise RuntimeError( + f"lane failed with status {result.returncode}: {' '.join(command)}\n" + f"stderr: {stderr_path}") + return elapsed + + +def parse_codes(line: str) -> list[int]: + return [int(value) for value in line.split("codes=", 1)[1].strip().split(",")] + + +def parse_emel_log(path: Path) -> tuple[list[list[int]], list[list[int]], list[int]]: + input_frames = [] + output_frames = [] + text_tokens = [] + pattern = re.compile(r"text=(-?\d+).*codes=([0-9,-]+)") + for line in path.read_text().splitlines(): + if line.startswith("EMEL_INPUT "): + input_frames.append(parse_codes(line)) + elif line.startswith("EMEL_OUTPUT "): + match = pattern.search(line) + if match is None: + raise ValueError(f"malformed EMEL graph line: {line}") + text_tokens.append(int(match.group(1))) + output_frames.append([int(value) for value in match.group(2).split(",")]) + return input_frames, output_frames, text_tokens + + +def parse_reference_log(path: Path) -> tuple[list[list[int]], list[list[int]], list[int]]: + input_frames = [] + output_frames = [] + text_tokens = [] + pattern = re.compile(r"text=(-?\d+).*codes=([0-9,-]+)") + for line in path.read_text().splitlines(): + if line.startswith("input "): + input_frames.append(parse_codes(line)) + elif line.startswith("output "): + match = pattern.search(line) + if match is None: + raise ValueError(f"malformed reference output line: {line}") + text_tokens.append(int(match.group(1))) + output_frames.append([int(value) for value in match.group(2).split(",")]) + return input_frames, output_frames, text_tokens + + +def match_fraction(left: list[list[int]], right: list[list[int]], streams: int) -> float: + if len(left) != len(right) or not left: + return 0.0 + matched = 0 + total = 0 + for left_frame, right_frame in zip(left, right): + if len(left_frame) < streams or len(right_frame) < streams: + return 0.0 + matched += sum(a == b for a, b in + zip(left_frame[:streams], right_frame[:streams])) + total += streams + return matched / total if total else 0.0 + + +def common_token_prefix(left: list[list[int]], right: list[list[int]]) -> int: + count = 0 + for left_frame, right_frame in zip(left, right): + for left_token, right_token in zip(left_frame, right_frame): + if left_token != right_token: + return count + count += 1 + if len(left_frame) != len(right_frame): + return count + return count + + +def append_text_parity_reason(reasons: list[str], emel_text: list[int], + ref_text: list[int], text_match: float) -> None: + if len(emel_text) != len(ref_text): + reasons.append("text frame count mismatch") + elif text_match != 1.0: + reasons.append( + f"text token match is {text_match:.6f}, expected 1.0") + + +def read_wav(path: Path) -> tuple[int, list[float]]: + with wave.open(str(path), "rb") as source: + if (source.getnchannels(), source.getsampwidth()) != (1, 2): + raise ValueError(f"expected mono s16 WAV: {path}") + sample_rate = source.getframerate() + raw = source.readframes(source.getnframes()) + samples = struct.unpack(f"<{len(raw) // 2}h", raw) + return sample_rate, [sample / 32768.0 for sample in samples] + + +def audio_metrics(samples: list[float], sample_rate: int) -> dict[str, object]: + energy = sum(sample * sample for sample in samples) + rms = math.sqrt(energy / len(samples)) if samples else 0.0 + peak = max((abs(sample) for sample in samples), default=0.0) + window = max(1, int(sample_rate * 0.08)) + frame_rms = [] + for begin in range(0, len(samples), window): + frame = samples[begin:begin + window] + frame_energy = sum(sample * sample for sample in frame) + frame_rms.append(math.sqrt(frame_energy / len(frame))) + active = sum(value >= 0.01 for value in frame_rms) + return { + "samples": len(samples), + "seconds": len(samples) / sample_rate, + "peak": peak, + "rms": rms, + "active_ratio": active / len(frame_rms) if frame_rms else 0.0, + "frame_rms": frame_rms, + } + + +def correlation(left: list[float], right: list[float]) -> float: + if len(left) != len(right) or len(left) < 2: + return 0.0 + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + numerator = sum((a - left_mean) * (b - right_mean) + for a, b in zip(left, right)) + left_energy = sum((value - left_mean) ** 2 for value in left) + right_energy = sum((value - right_mean) ** 2 for value in right) + denominator = math.sqrt(left_energy * right_energy) + return numerator / denominator if denominator else 0.0 + + +def best_energy_correlation(left: list[float], right: list[float], max_lag: int) -> tuple[float, int]: + best = (-2.0, 0) + for lag in range(-max_lag, max_lag + 1): + if lag < 0: + current = correlation(left[-lag:], right[:len(right) + lag]) + elif lag > 0: + current = correlation(left[:len(left) - lag], right[lag:]) + else: + current = correlation(left, right) + if current > best[0]: + best = (current, lag) + return best + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while block := source.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--emel-runner", required=True) + parser.add_argument("--reference-driver", required=True) + parser.add_argument("--emel-mimi", required=True) + parser.add_argument("--reference-mimi", required=True) + parser.add_argument("--emel-lm", required=True) + parser.add_argument("--reference-lm", required=True) + parser.add_argument("--emel-voice", required=True) + parser.add_argument("--reference-voice", required=True) + parser.add_argument("--config", required=True) + parser.add_argument("--inference-config", required=True) + parser.add_argument("--audio", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--frames", type=int, default=125) + parser.add_argument("--seed", type=int, default=1234) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--audio-temperature", type=float, default=0.8) + parser.add_argument("--text-temperature", type=float, default=0.7) + parser.add_argument("--min-output-match", type=float, default=0.15) + parser.add_argument("--min-first-four-match", type=float, default=0.30) + parser.add_argument("--min-energy-correlation", type=float, default=0.30) + parser.add_argument("--max-active-ratio-delta", type=float, default=0.35) + args = parser.parse_args() + if args.frames <= 0 or args.seed <= 0 or args.threads <= 0: + parser.error("frames, seed, and threads must be positive") + + inference_path = Path(args.inference_config) + inference = json.loads(inference_path.read_text(encoding="utf-8")) + required_inference = ( + "dep_q", "max_blocks", "block_tokens", "prompt_text_token", + "frame_samples", "audio_top_k", "text_top_k") + if any(not isinstance(inference.get(key), int) + for key in required_inference): + parser.error("inference config integer contract is incomplete") + public_n_q = inference["dep_q"] + max_blocks = inference["max_blocks"] + block_tokens = inference["block_tokens"] + prompt_text_token = inference["prompt_text_token"] + frame_samples = inference["frame_samples"] + audio_top_k = inference["audio_top_k"] + text_top_k = inference["text_top_k"] + if (public_n_q <= 0 or max_blocks <= 0 or block_tokens <= 0 or + frame_samples <= 0 or audio_top_k <= 0 or text_top_k <= 0): + parser.error( + "inference dep_q, max_blocks, block_tokens, frame_samples, and top-k " + "limits must be positive") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + emel_wav = output_dir / "emel_personaplex.wav" + reference_wav = output_dir / "moshi_cpp_personaplex.wav" + emel_stdout = output_dir / "emel.stdout.log" + emel_stderr = output_dir / "emel.stderr.log" + reference_stdout = output_dir / "moshi_cpp.stdout.log" + reference_stderr = output_dir / "moshi_cpp.stderr.log" + + emel_elapsed = run_lane([ + args.emel_runner, + args.emel_mimi, + args.emel_lm, + args.emel_voice, + args.audio, + str(emel_wav), + str(args.frames), + str(args.seed), + str(args.audio_temperature), + str(args.text_temperature), + str(audio_top_k), + str(text_top_k), + str(max_blocks), + str(block_tokens), + str(prompt_text_token), + ], emel_stdout, emel_stderr) + reference_elapsed = run_lane([ + args.reference_driver, + "personaplex", + "--config", args.config, + "--lm", args.reference_lm, + "--mimi", args.reference_mimi, + "--voice", args.reference_voice, + "--audio", args.audio, + "--out", str(reference_wav), + "--frames", str(args.frames), + "--threads", str(args.threads), + "--seed", str(args.seed), + "--n-q", str(public_n_q), + "--audio-temperature", str(args.audio_temperature), + "--text-temperature", str(args.text_temperature), + ], reference_stdout, reference_stderr) + + emel_input, emel_output, emel_text = parse_emel_log(emel_stderr) + ref_input, ref_output, ref_text = parse_reference_log(reference_stdout) + input_rate, input_samples = read_wav(Path(args.audio)) + if input_rate != 24000: + raise RuntimeError("input WAV sample rate must be 24 kHz") + audio_input_frames = ( + len(input_samples) + frame_samples - 1) // frame_samples + emel_wav_input = emel_input[:audio_input_frames] + ref_public_input = [frame[:public_n_q] + for frame in ref_input[:audio_input_frames]] + input_match = match_fraction( + emel_wav_input, ref_public_input, public_n_q) + output_match = match_fraction(emel_output, ref_output, public_n_q) + first_four_match = match_fraction( + emel_output, ref_output, min(4, public_n_q)) + text_match = (sum(a == b for a, b in zip(emel_text, ref_text)) / + len(emel_text) if len(emel_text) == len(ref_text) and emel_text + else 0.0) + + emel_rate, emel_samples = read_wav(emel_wav) + reference_rate, reference_samples = read_wav(reference_wav) + if emel_rate != reference_rate: + raise RuntimeError("output WAV sample-rate mismatch") + emel_audio = audio_metrics(emel_samples, emel_rate) + reference_audio = audio_metrics(reference_samples, reference_rate) + emel_log_energy = [math.log(max(value, 1.0e-8)) + for value in emel_audio.pop("frame_rms")] + reference_log_energy = [math.log(max(value, 1.0e-8)) + for value in reference_audio.pop("frame_rms")] + energy_correlation, energy_lag = best_energy_correlation( + emel_log_energy, reference_log_energy, 25) + active_ratio_delta = abs( + float(emel_audio["active_ratio"]) - float(reference_audio["active_ratio"])) + + reasons = [] + if len(emel_input) != args.frames or len(ref_input) != args.frames: + reasons.append("input frame count mismatch") + if len(emel_output) != args.frames or len(ref_output) != args.frames: + reasons.append("output frame count mismatch") + append_text_parity_reason(reasons, emel_text, ref_text, text_match) + if input_match != 1.0: + reasons.append(f"same-WAV input token match is {input_match:.6f}, expected 1.0") + if output_match < args.min_output_match: + reasons.append( + f"output token match {output_match:.6f} below {args.min_output_match:.6f}") + if first_four_match < args.min_first_four_match: + reasons.append( + f"first-four token match {first_four_match:.6f} below " + f"{args.min_first_four_match:.6f}") + if energy_correlation < args.min_energy_correlation: + reasons.append( + f"energy correlation {energy_correlation:.6f} below " + f"{args.min_energy_correlation:.6f}") + if active_ratio_delta > args.max_active_ratio_delta: + reasons.append( + f"active ratio delta {active_ratio_delta:.6f} above " + f"{args.max_active_ratio_delta:.6f}") + if float(emel_audio["rms"]) <= 0.0 or float(reference_audio["rms"]) <= 0.0: + reasons.append("one or both output WAVs are silent") + + report = { + "surface": "personaplex_compare/v1", + "status": "pass" if not reasons else "fail", + "reasons": reasons, + "cpu_threads": args.threads, + "seed": args.seed, + "frames": args.frames, + "inference": { + "path": str(inference_path.resolve()), + "sha256": sha256(inference_path), + "public_n_q": public_n_q, + "max_blocks": max_blocks, + "block_tokens": block_tokens, + "prompt_text_token": prompt_text_token, + "frame_samples": frame_samples, + }, + "models": { + "emel_mimi": { + "path": str(Path(args.emel_mimi).resolve()), + "sha256": sha256(Path(args.emel_mimi)), + }, + "reference_mimi": { + "path": str(Path(args.reference_mimi).resolve()), + "sha256": sha256(Path(args.reference_mimi)), + }, + "emel_lm": { + "path": str(Path(args.emel_lm).resolve()), + "sha256": sha256(Path(args.emel_lm)), + }, + "reference_lm": { + "path": str(Path(args.reference_lm).resolve()), + "sha256": sha256(Path(args.reference_lm)), + }, + }, + "input": { + "path": str(Path(args.audio).resolve()), + "sha256": sha256(Path(args.audio)), + "emel_frames": len(emel_input), + "reference_frames": len(ref_input), + "audio_frames": audio_input_frames, + "compared_public_tokens": audio_input_frames * public_n_q, + "public_codebook_match_fraction": input_match, + }, + "tokens": { + "emel_output_frames": len(emel_output), + "reference_output_frames": len(ref_output), + "common_prefix_tokens": common_token_prefix(emel_output, ref_output), + "public_codebook_match_fraction": output_match, + "first_four_codebook_match_fraction": first_four_match, + "text_token_match_fraction": text_match, + }, + "audio": { + "emel": emel_audio, + "reference": reference_audio, + "best_log_energy_correlation": energy_correlation, + "best_energy_lag_frames_80ms": energy_lag, + "active_ratio_delta": active_ratio_delta, + "emel_wav": str(emel_wav.resolve()), + "reference_wav": str(reference_wav.resolve()), + "emel_sha256": sha256(emel_wav), + "reference_sha256": sha256(reference_wav), + }, + "timing": { + "emel_wall_seconds": emel_elapsed, + "reference_wall_seconds": reference_elapsed, + "reference_over_emel": reference_elapsed / emel_elapsed, + }, + } + report_path = output_dir / "personaplex_compare.json" + report_path.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + print(f"report: {report_path.resolve()}") + return 0 if not reasons else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, RuntimeError, ValueError) as error: + sys.exit(f"error: {error}") diff --git a/tools/bench/personaplex_compare_tests.cpp b/tools/bench/personaplex_compare_tests.cpp new file mode 100644 index 00000000..1ae782ce --- /dev/null +++ b/tools/bench/personaplex_compare_tests.cpp @@ -0,0 +1,207 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN + +#include +#include +#include +#include + +#include + +#if !defined(_WIN32) +#include +#endif + +namespace { + +std::filesystem::path repo_root() { +#ifdef BENCH_REPO_ROOT + return BENCH_REPO_ROOT; +#else + return std::filesystem::current_path(); +#endif +} + +std::filesystem::path compare_script_path() { +#ifdef PERSONAPLEX_COMPARE_SCRIPT_PATH + return PERSONAPLEX_COMPARE_SCRIPT_PATH; +#else + return repo_root() / "tools" / "bench" / "personaplex_compare.py"; +#endif +} + +std::string read_file(const std::filesystem::path &path) { + std::ifstream input(path, std::ios::binary); + if (!input.good()) { + return {}; + } + return std::string(std::istreambuf_iterator(input), + std::istreambuf_iterator()); +} + +std::string quote_arg_posix(const std::string &arg) { + std::string output = "'"; + for (const char value : arg) { + if (value == '\'') { + output += "'\\''"; + } else { + output.push_back(value); + } + } + output += "'"; + return output; +} + +int command_exit_code(const std::string &command) { + const int status = std::system(command.c_str()); + if (status == -1) { + return -1; + } +#if defined(_WIN32) + return status; +#else + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +#endif +} + +} // namespace + +TEST_CASE("personaplex compare math reports exact and partial token behavior") { + const std::string module = quote_arg_posix(compare_script_path().string()); + const std::string program = + "import importlib.util; " + "spec=importlib.util.spec_from_file_location('personaplex_compare', " + + module + + "); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m); " + "assert m.match_fraction([[1,2],[3,4]], [[1,2],[3,4]], 2) == 1.0; " + "assert m.match_fraction([[1,9]], [[1,2]], 2) == 0.5; " + "assert m.match_fraction([[1]], [[1,2]], 2) == 0.0; " + "assert m.common_token_prefix([[1,2]], [[1,3]]) == 1; " + "assert abs(m.correlation([1.0,2.0,3.0], [2.0,4.0,6.0])-1.0) < 1e-12; " + "reasons=[]; m.append_text_parity_reason(reasons, [1,2], [1,3], 0.5); " + "assert reasons == ['text token match is 0.500000, expected 1.0']; " + "reasons=[]; m.append_text_parity_reason(reasons, [1], [1,2], 0.0); " + "assert reasons == ['text frame count mismatch']; " + "reasons=[]; m.append_text_parity_reason(reasons, [1,2], [1,2], 1.0); " + "assert reasons == []"; + CHECK(command_exit_code("python3 -c " + quote_arg_posix(program)) == 0); +} + +TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { + const std::string compare_source = read_file(compare_script_path()); + const std::string wrapper_source = + read_file(repo_root() / "scripts" / "bench_personaplex_compare.sh"); + const std::string cmake_source = + read_file(repo_root() / "tools" / "bench" / "CMakeLists.txt"); + const std::string emel_source = + read_file(repo_root() / "tools" / "bench" / "speech" / + "personaplex_emel_runner.cpp"); + const std::string setup_source = + read_file(repo_root() / "scripts" / "setup_moshi_cpp_reference.sh"); + const std::string converter_source = + read_file(repo_root() / "tools" / "bench" / "moshi_gguf_convert.py"); + const std::string inference_source = + read_file(repo_root() / "tools" / "bench" / "personaplex-inference.json"); + const std::string mimi_wrapper_source = + read_file(repo_root() / "scripts" / "bench_mimi_compare.sh"); + const std::string reference_source = + read_file(repo_root() / "tools" / "bench" / "speech" / + "moshi_reference_driver.cpp"); + + const std::size_t first_public_match = + compare_source.find("public_codebook_match_fraction"); + REQUIRE(first_public_match != std::string::npos); + CHECK(compare_source.find("public_codebook_match_fraction", + first_public_match + 1u) != std::string::npos); + CHECK(compare_source.find("best_log_energy_correlation") != + std::string::npos); + CHECK(compare_source.find("reference_over_emel") != std::string::npos); + CHECK(wrapper_source.find("--threads \"$THREADS\"") != std::string::npos); + CHECK(wrapper_source.find("--audio \"$AUDIO\"") != std::string::npos); + CHECK(wrapper_source.find("--inference-config \"$INFERENCE_CONFIG\"") != + std::string::npos); + CHECK(wrapper_source.find("--emel-lm PATH") != std::string::npos); + CHECK(wrapper_source.find("--emel-lm) EMEL_LM=") != std::string::npos); + CHECK(wrapper_source.find("--emel-mimi PATH") != std::string::npos); + CHECK(wrapper_source.find("--emel-mimi) EMEL_MIMI=") != std::string::npos); + CHECK(wrapper_source.find("--data-format=LEI16@24000") != std::string::npos); + CHECK(wrapper_source.find("mimi-e351c8d8-125-personaplex-emel.gguf") != + std::string::npos); + CHECK(wrapper_source.find("Hey, I'm Gabe. How are you doing?") != + std::string::npos); + CHECK(cmake_source.find("set(GGML_METAL OFF CACHE BOOL \"CPU-only " + "PersonaPlex reference lane\" FORCE)") != + std::string::npos); + CHECK(emel_source.find("#include --audio <24 kHz mono s16 wav> [--n-q N] // decode --mimi --codes --out // [--n-q N] +// personaplex --config --lm --mimi +// --voice --audio <24 kHz mono s16 wav> +// --out --frames --seed [--n-q N] +#include #include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -92,13 +99,56 @@ bool load_wav_24khz_mono(const std::filesystem::path &path, return true; } +void write_u16_le(std::ofstream &out, const uint16_t value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} + +void write_u32_le(std::ofstream &out, const uint32_t value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} + +bool write_wav_24khz_mono(const std::filesystem::path &path, + const std::vector &pcm) { + std::ofstream out(path, std::ios::binary); + if (!out.good() || pcm.size() > (std::numeric_limits::max() / 2u)) { + return false; + } + const uint32_t data_bytes = static_cast(pcm.size() * 2u); + out.write("RIFF", 4); + write_u32_le(out, 36u + data_bytes); + out.write("WAVEfmt ", 8); + write_u32_le(out, 16u); + write_u16_le(out, 1u); + write_u16_le(out, 1u); + write_u32_le(out, 24000u); + write_u32_le(out, 48000u); + write_u16_le(out, 2u); + write_u16_le(out, 16u); + out.write("data", 4); + write_u32_le(out, data_bytes); + for (const float sample : pcm) { + const float clipped = std::max(-1.0f, std::min(1.0f, sample)); + const int16_t s16 = static_cast(std::lrint(clipped * 32767.0f)); + out.write(reinterpret_cast(&s16), sizeof(s16)); + } + return out.good(); +} + struct options { std::string command = {}; + std::filesystem::path config = {}; + std::filesystem::path lm = {}; std::filesystem::path mimi = {}; + std::filesystem::path voice = {}; std::filesystem::path audio = {}; std::filesystem::path codes = {}; std::filesystem::path out = {}; int n_q = 16; + int frames = 0; + int threads = 0; + unsigned int seed = 0u; + float audio_temperature = 0.8f; + float text_temperature = 0.7f; }; bool parse_options(const int argc, char **argv, options &opts) { @@ -109,8 +159,14 @@ bool parse_options(const int argc, char **argv, options &opts) { for (int index = 2; index < argc; ++index) { const std::string_view arg = argv[index]; const bool has_value = index + 1 < argc; - if (arg == "--mimi" && has_value) { + if (arg == "--config" && has_value) { + opts.config = argv[++index]; + } else if (arg == "--lm" && has_value) { + opts.lm = argv[++index]; + } else if (arg == "--mimi" && has_value) { opts.mimi = argv[++index]; + } else if (arg == "--voice" && has_value) { + opts.voice = argv[++index]; } else if (arg == "--audio" && has_value) { opts.audio = argv[++index]; } else if (arg == "--codes" && has_value) { @@ -119,6 +175,17 @@ bool parse_options(const int argc, char **argv, options &opts) { opts.out = argv[++index]; } else if (arg == "--n-q" && has_value) { opts.n_q = std::atoi(argv[++index]); + } else if (arg == "--frames" && has_value) { + opts.frames = std::atoi(argv[++index]); + } else if (arg == "--threads" && has_value) { + opts.threads = std::atoi(argv[++index]); + } else if (arg == "--seed" && has_value) { + opts.seed = + static_cast(std::strtoul(argv[++index], nullptr, 10)); + } else if (arg == "--audio-temperature" && has_value) { + opts.audio_temperature = std::strtof(argv[++index], nullptr); + } else if (arg == "--text-temperature" && has_value) { + opts.text_temperature = std::strtof(argv[++index], nullptr); } else { std::fprintf(stderr, "unknown or incomplete argument: %.*s\n", static_cast(arg.size()), arg.data()); @@ -128,7 +195,12 @@ bool parse_options(const int argc, char **argv, options &opts) { return (opts.command == "encode" && !opts.mimi.empty() && !opts.audio.empty()) || (opts.command == "decode" && !opts.mimi.empty() && - !opts.codes.empty() && !opts.out.empty()); + !opts.codes.empty() && !opts.out.empty()) || + (opts.command == "personaplex" && !opts.config.empty() && + !opts.lm.empty() && !opts.mimi.empty() && !opts.voice.empty() && + !opts.audio.empty() && !opts.out.empty() && opts.frames > 0 && + opts.threads > 0 && opts.seed > 0u && opts.n_q > 0 && + opts.audio_temperature > 0.0f && opts.text_temperature > 0.0f); } bool parse_codes_file(const std::filesystem::path &path, const int n_q, @@ -157,6 +229,128 @@ bool parse_codes_file(const std::filesystem::path &path, const int n_q, return !frames_out.empty(); } +int run_personaplex(const options &opts, ggml_backend_t backend_cpu) { + std::srand(opts.seed); + moshi_context_t *moshi = moshi_alloc(backend_cpu, backend_cpu); + if (moshi == nullptr) { + std::fprintf(stderr, "failed to allocate moshi context\n"); + return 1; + } + moshi_config_t config{}; + if (moshi_get_config(&config, opts.config.string().c_str()) != 0) { + std::fprintf(stderr, "failed to load PersonaPlex config: %s\n", + opts.config.string().c_str()); + return 1; + } + moshi_lm_t *lm = + moshi_lm_from_files(moshi, &config, opts.lm.string().c_str()); + if (lm == nullptr) { + std::fprintf(stderr, "failed to allocate PersonaPlex language model\n"); + return 1; + } + moshi_lm_gen_t *generator = moshi_lm_generator(lm); + mimi_codec_t *codec = mimi_alloc(moshi, opts.mimi.string().c_str(), opts.n_q); + if (generator == nullptr || codec == nullptr || moshi_lm_load(lm) != 0) { + std::fprintf(stderr, "failed to load PersonaPlex reference models\n"); + return 1; + } + mimi_encode_context_t *encoder = mimi_encode_alloc_context(codec); + mimi_decode_context_t *decoder = mimi_decode_alloc_context(codec); + if (encoder == nullptr || decoder == nullptr || + moshi_lm_personaplex_load_voice(moshi, generator, + opts.voice.string().c_str()) != 0) { + std::fprintf(stderr, "failed to initialize PersonaPlex voice/runtime\n"); + return 1; + } + + std::vector pcm = {}; + if (!load_wav_24khz_mono(opts.audio, pcm)) { + std::fprintf(stderr, "failed to load 24 kHz mono s16 wav: %s\n", + opts.audio.string().c_str()); + return 1; + } + const int frame_size = mimi_frame_size(codec); + if (frame_size <= 0) { + std::fprintf(stderr, "invalid Mimi frame size\n"); + return 1; + } + const size_t input_frames = + (pcm.size() + static_cast(frame_size) - 1u) / + static_cast(frame_size); + if (input_frames > static_cast(opts.frames)) { + std::fprintf(stderr, "invalid frame count for input audio\n"); + return 1; + } + + moshi_lm_start(moshi, generator, opts.audio_temperature, + opts.text_temperature); + std::vector input_frame(static_cast(frame_size), 0.0f); + std::vector decoded_frame(static_cast(frame_size), 0.0f); + std::vector output_pcm = {}; + output_pcm.reserve(static_cast(opts.frames) * + static_cast(frame_size)); + std::vector tokens(static_cast(opts.n_q), -1); + int text_token = -1; + const auto started = std::chrono::steady_clock::now(); + for (int frame = 0; frame < opts.frames; ++frame) { + std::fill(input_frame.begin(), input_frame.end(), 0.0f); + if (static_cast(frame) < input_frames) { + const size_t begin = + static_cast(frame) * static_cast(frame_size); + const size_t count = + std::min(static_cast(frame_size), pcm.size() - begin); + std::copy_n(pcm.data() + begin, count, input_frame.data()); + } + mimi_encode_send(encoder, input_frame.data()); + mimi_encode_receive(encoder, tokens.data()); + std::printf("input frame=%d codes=", frame); + for (size_t stream = 0; stream < tokens.size(); ++stream) { + std::printf("%s%d", stream == 0u ? "" : ",", + static_cast(tokens[stream])); + } + std::printf("\n"); + + moshi_lm_send2(generator, tokens); + const int produced = moshi_lm_receive(generator, text_token, tokens); + std::printf("output frame=%d produced=%d text=%d codes=", frame, produced, + produced != 0 ? text_token : -1); + for (size_t stream = 0; stream < tokens.size(); ++stream) { + std::printf("%s%d", stream == 0u ? "" : ",", + produced != 0 ? static_cast(tokens[stream]) : -1); + } + std::printf("\n"); + if (produced != 0) { + mimi_decode_send(decoder, tokens.data()); + mimi_decode_receive(decoder, decoded_frame.data()); + } else { + std::fill(decoded_frame.begin(), decoded_frame.end(), 0.0f); + } + output_pcm.insert(output_pcm.end(), decoded_frame.begin(), + decoded_frame.end()); + } + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - started) + .count(); + if (output_pcm.empty() || !write_wav_24khz_mono(opts.out, output_pcm)) { + std::fprintf(stderr, "failed to write PersonaPlex output wav: %s\n", + opts.out.string().c_str()); + return 1; + } + std::fprintf(stderr, + "personaplex frames=%d output_frames=%zu seed=%u seconds=%.3f " + "frames_per_second=%.6f output=%s\n", + opts.frames, output_pcm.size() / static_cast(frame_size), + opts.seed, elapsed, static_cast(opts.frames) / elapsed, + opts.out.string().c_str()); + unref(decoder); + unref(encoder); + unref(codec); + unref(generator); + unref(lm); + unref(moshi); + return 0; +} + } // namespace int main(int argc, char **argv) { @@ -167,7 +361,11 @@ int main(int argc, char **argv) { " moshi_reference_driver encode --mimi --audio " "[--n-q N]\n" " moshi_reference_driver decode --mimi --codes " - "--out [--n-q N]\n"); + "--out [--n-q N]\n" + " moshi_reference_driver personaplex --config " + "--lm --mimi --voice --audio " + "--out --frames --threads --seed " + "[--n-q N]\n"); return 2; } @@ -176,6 +374,14 @@ int main(int argc, char **argv) { std::fprintf(stderr, "failed to init ggml cpu backend\n"); return 1; } + if (opts.threads > 0) { + ggml_backend_cpu_set_n_threads(backend_cpu, opts.threads); + } + if (opts.command == "personaplex") { + const int status = run_personaplex(opts, backend_cpu); + ggml_backend_free(backend_cpu); + return status; + } moshi_context_t *moshi = moshi_alloc(backend_cpu, backend_cpu); if (moshi == nullptr) { std::fprintf(stderr, "failed to alloc moshi context\n"); diff --git a/tools/bench/speech/personaplex_emel_runner.cpp b/tools/bench/speech/personaplex_emel_runner.cpp new file mode 100644 index 00000000..40979580 --- /dev/null +++ b/tools/bench/speech/personaplex_emel_runner.cpp @@ -0,0 +1,799 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "emel/batch/planner/sm.hpp" +#include "emel/error/error.hpp" +#include "emel/gguf/loader/detail.hpp" +#include "emel/gguf/loader/events.hpp" +#include "emel/gguf/loader/sm.hpp" +#include "emel/kernel/sm.hpp" +#include "emel/memory/streaming/sm.hpp" +#include "emel/model/data.hpp" +#include "emel/model/detail.hpp" +#include "emel/speech/codec/mimi/any.hpp" +#include "emel/speech/generator/any.hpp" +#include "emel/speech/predictor/moshi/any.hpp" +#include "emel/speech/predictor/moshi/executor/any.hpp" +#include "emel/speech/tokenizer/moshi/any.hpp" + +namespace { + +namespace mimi = emel::speech::codec::mimi; +namespace generator = emel::speech::generator; +namespace predictor = emel::speech::predictor::moshi; +namespace runtime = emel::speech::predictor::moshi::executor; +namespace tokenizer = emel::speech::tokenizer::moshi; + +struct runner_config { + const char *mimi_path = nullptr; + const char *lm_path = nullptr; + const char *voice_path = nullptr; + const char *input_path = nullptr; + const char *output_path = nullptr; + size_t target_output_frames = 0; + uint32_t sampling_seed = 0; + float audio_temperature = 0.0f; + float text_temperature = 0.0f; + int32_t audio_top_k = 0; + int32_t text_top_k = 0; + int32_t max_blocks = 0; + int32_t block_tokens = 0; + int32_t prompt_text_token = 0; +}; + +void noop_probe_done(const emel::gguf::loader::events::probe_done &) {} +void noop_probe_error(const emel::gguf::loader::events::probe_error &) {} +void noop_bind_done(const emel::gguf::loader::events::bind_done &) {} +void noop_bind_error(const emel::gguf::loader::events::bind_error &) {} +void noop_parse_done(const emel::gguf::loader::events::parse_done &) {} +void noop_parse_error(const emel::gguf::loader::events::parse_error &) {} + +struct loaded_model { + std::vector file_bytes = {}; + std::vector kv_arena = {}; + std::vector kv_entries = {}; + std::unique_ptr model = {}; +}; + +struct streaming_cache { + std::vector key_cache = {}; + std::vector value_cache = {}; + std::vector layer_offsets = {}; +}; + +struct cache_views { + runtime::detail::temporal_kv_view temporal = {}; + runtime::detail::depformer_kv_view secondary = {}; +}; + +struct personaplex_dependencies { + using generator_mode = generator::duplex; + using voice_condition_event = predictor::event::prefill_voice; + using prompt_begin_event = predictor::event::begin_personaplex_prompt; + using prompt_condition_event = predictor::event::prefill_personaplex_prompt; + using encode_event = mimi::event::encode_frame; + using tokenizer_initialize_event = tokenizer::event::initialize; + using tokenize_event = tokenizer::event::tokenize; + using predict_event = predictor::event::predict; + using graph_event = predictor::event::execute; + using sample_event = predictor::event::sample; + using detokenize_event = tokenizer::event::detokenize; + using capture_tokenizer_state_event = + predictor::event::capture_tokenizer_state; + using restore_tokenizer_state_event = tokenizer::event::restore_cache; + using decode_event = mimi::event::decode_frame; + + emel::memory::streaming::sm &temporal_positions; + emel::memory::streaming::sm &secondary_positions; + emel::batch::planner::sm &planner; + mimi::sm &encoder; + tokenizer::sm &tokenizer; + mimi::sm &decoder; + predictor::sm &predictor; + predictor::sm &graph; + predictor::sm &sampler; + predictor::event::predict::workspace &prediction_workspace; + mimi::event::initialize encoder_initialize; + mimi::event::initialize decoder_initialize; + predictor::event::initialize predictor_initialize; + predictor::event::load_voice conditioning_initialize; + predictor::event::begin_personaplex_prompt prompt_begin = {}; + std::span silence_pcm = {}; + std::span input_codes = {}; + std::span tokenize_input_codes = {}; + std::span model_codes = {}; + std::span predicted_codes = {}; + std::span output_codes = {}; + std::span tokenizer_cache_snapshot = {}; + int32_t frame_samples = 0; + int32_t codebook_count = 0; + emel::batch::planner::event::plan_mode frame_plan_mode = + emel::batch::planner::event::plan_mode::simple; + int32_t frame_plan_steps = 0; + int32_t frame_plan_token_count = 0; + bool frame_plan_output_all = false; +}; + +bool read_binary_file(const std::filesystem::path &path, + std::vector &bytes_out) { + std::ifstream file{path, std::ios::binary}; + if (!file.good()) { + return false; + } + file.seekg(0, std::ios::end); + const std::streamoff size = file.tellg(); + if (size <= 0) { + return false; + } + file.seekg(0, std::ios::beg); + bytes_out.resize(static_cast(size)); + file.read(reinterpret_cast(bytes_out.data()), size); + return file.good(); +} + +bool materialize_tensor_names(emel::model::data &model, + const std::vector &file) { + model.name_bytes_used = 0; + for (uint32_t index = 0; index < model.n_tensors; ++index) { + auto &tensor = model.tensors[index]; + const size_t source_offset = static_cast(tensor.name_offset); + const size_t length = static_cast(tensor.name_length); + if (source_offset + length > file.size() || + static_cast(model.name_bytes_used) + length > + model.name_storage.size()) { + return false; + } + std::copy_n(file.data() + source_offset, length, + model.name_storage.data() + model.name_bytes_used); + tensor.name_offset = model.name_bytes_used; + model.name_bytes_used += static_cast(length); + } + return true; +} + +bool load_model(const std::filesystem::path &path, loaded_model &loaded) { + loaded = {}; + loaded.model = std::make_unique(); + if (!read_binary_file(path, loaded.file_bytes)) { + std::fprintf(stderr, "failed to read model: %s\n", path.string().c_str()); + return false; + } + + emel::gguf::loader::sm loader{}; + emel::gguf::loader::requirements requirements{}; + if (!loader.process_event(emel::gguf::loader::event::probe{ + std::span{loaded.file_bytes}, + requirements, + emel::gguf::loader::event::probe_done_fn::from<&noop_probe_done>(), + emel::gguf::loader::event::probe_error_fn::from<&noop_probe_error>(), + }) || + requirements.tensor_count > loaded.model->tensors.size()) { + std::fprintf(stderr, "failed to probe model: %s\n", path.string().c_str()); + return false; + } + + const uint64_t arena_bytes = + emel::gguf::loader::detail::required_kv_arena_bytes(requirements); + if (arena_bytes == std::numeric_limits::max()) { + return false; + } + loaded.kv_arena.resize(static_cast(arena_bytes)); + loaded.kv_entries.resize(requirements.kv_count); + loaded.model->n_tensors = requirements.tensor_count; + + if (!loader.process_event(emel::gguf::loader::event::bind_storage{ + std::span{loaded.kv_arena}, + std::span{loaded.kv_entries}, + std::span{ + loaded.model->tensors.data(), loaded.model->n_tensors}, + emel::gguf::loader::event::bind_done_fn::from<&noop_bind_done>(), + emel::gguf::loader::event::bind_error_fn::from<&noop_bind_error>(), + }) || + !loader.process_event(emel::gguf::loader::event::parse{ + std::span{loaded.file_bytes}, + emel::gguf::loader::event::parse_done_fn::from<&noop_parse_done>(), + emel::gguf::loader::event::parse_error_fn::from<&noop_parse_error>(), + })) { + std::fprintf(stderr, "failed to parse model: %s\n", path.string().c_str()); + return false; + } + + const emel::model::detail::kv_binding binding{ + .arena = std::span{loaded.kv_arena}, + .entries = + std::span{loaded.kv_entries}, + }; + if (!emel::model::detail::load_hparams_from_gguf(binding, *loaded.model)) { + std::fprintf(stderr, "failed to load hparams: %s\n", path.string().c_str()); + return false; + } + loaded.model->weights_data = loaded.file_bytes.data(); + loaded.model->weights_size = loaded.file_bytes.size(); + return materialize_tensor_names(*loaded.model, loaded.file_bytes); +} + +uint16_t read_u16_le(const uint8_t *data) { + uint16_t value = 0; + std::memcpy(&value, data, sizeof(value)); + return value; +} + +uint32_t read_u32_le(const uint8_t *data) { + uint32_t value = 0; + std::memcpy(&value, data, sizeof(value)); + return value; +} + +bool load_wav_24khz_mono_s16(const std::filesystem::path &path, + std::vector &pcm_out) { + std::vector bytes = {}; + if (!read_binary_file(path, bytes) || bytes.size() < 44 || + std::memcmp(bytes.data(), "RIFF", 4) != 0 || + std::memcmp(bytes.data() + 8, "WAVE", 4) != 0) { + return false; + } + + uint16_t format = 0; + uint16_t channels = 0; + uint32_t sample_rate = 0; + uint16_t bits = 0; + size_t data_offset = 0; + size_t data_size = 0; + size_t offset = 12; + while (offset + 8 <= bytes.size()) { + const uint8_t *chunk = bytes.data() + offset; + const uint32_t chunk_size = read_u32_le(chunk + 4); + const size_t payload = offset + 8; + if (payload + chunk_size > bytes.size()) { + return false; + } + if (std::memcmp(chunk, "fmt ", 4) == 0 && chunk_size >= 16) { + format = read_u16_le(bytes.data() + payload); + channels = read_u16_le(bytes.data() + payload + 2); + sample_rate = read_u32_le(bytes.data() + payload + 4); + bits = read_u16_le(bytes.data() + payload + 14); + } else if (std::memcmp(chunk, "data", 4) == 0) { + data_offset = payload; + data_size = chunk_size; + } + offset = payload + chunk_size + (chunk_size & 1u); + } + if (format != 1 || channels != 1 || sample_rate != 24000 || bits != 16 || + data_offset == 0 || data_size == 0) { + return false; + } + + const size_t samples = data_size / sizeof(int16_t); + pcm_out.resize(samples); + for (size_t index = 0; index < samples; ++index) { + int16_t sample = 0; + std::memcpy(&sample, bytes.data() + data_offset + index * sizeof(sample), + sizeof(sample)); + pcm_out[index] = static_cast(sample) / 32768.0f; + } + return true; +} + +void write_u16_le(std::ofstream &out, const uint16_t value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} + +void write_u32_le(std::ofstream &out, const uint32_t value) { + out.write(reinterpret_cast(&value), sizeof(value)); +} + +bool write_wav_24khz_mono_s16(const std::filesystem::path &path, + const std::span pcm) { + std::ofstream out{path, std::ios::binary}; + if (!out.good()) { + return false; + } + const uint32_t data_bytes = static_cast(pcm.size() * 2u); + out.write("RIFF", 4); + write_u32_le(out, 36u + data_bytes); + out.write("WAVEfmt ", 8); + write_u32_le(out, 16u); + write_u16_le(out, 1u); + write_u16_le(out, 1u); + write_u32_le(out, 24000u); + write_u32_le(out, 24000u * 2u); + write_u16_le(out, 2u); + write_u16_le(out, 16u); + out.write("data", 4); + write_u32_le(out, data_bytes); + for (const float sample : pcm) { + const float clipped = std::max(-1.0f, std::min(1.0f, sample)); + const int16_t s16 = static_cast(std::lrint(clipped * 32767.0f)); + out.write(reinterpret_cast(&s16), sizeof(s16)); + } + return out.good(); +} + +bool prepare_streaming_cache(streaming_cache &cache, const int32_t layer_count, + const int32_t position_capacity, + const int32_t kv_dim) { + if (layer_count <= 0 || position_capacity <= 0 || kv_dim <= 0) { + return false; + } + const size_t per_layer = + static_cast(position_capacity) * static_cast(kv_dim); + cache.layer_offsets.resize(static_cast(layer_count)); + for (int32_t layer = 0; layer < layer_count; ++layer) { + cache.layer_offsets[static_cast(layer)] = + static_cast(layer) * per_layer; + } + cache.key_cache.assign(static_cast(layer_count) * per_layer, 0u); + cache.value_cache.assign(cache.key_cache.size(), 0u); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc != 15) { + std::fprintf(stderr, + "usage: %s " + " " + " " + " " + "\n", + argv[0]); + return 2; + } + + char *target_end = nullptr; + const unsigned long parsed_target = std::strtoul(argv[6], &target_end, 10); + if (target_end == argv[6] || *target_end != '\0' || parsed_target == 0) { + std::fprintf(stderr, "target output frames must be a positive integer\n"); + return 2; + } + char *seed_end = nullptr; + const unsigned long parsed_seed = std::strtoul(argv[7], &seed_end, 10); + char *audio_temperature_end = nullptr; + const float parsed_audio_temperature = + std::strtof(argv[8], &audio_temperature_end); + char *text_temperature_end = nullptr; + const float parsed_text_temperature = + std::strtof(argv[9], &text_temperature_end); + char *audio_top_k_end = nullptr; + const long parsed_audio_top_k = std::strtol(argv[10], &audio_top_k_end, 10); + char *text_top_k_end = nullptr; + const long parsed_text_top_k = std::strtol(argv[11], &text_top_k_end, 10); + char *max_blocks_end = nullptr; + const long parsed_max_blocks = std::strtol(argv[12], &max_blocks_end, 10); + char *block_tokens_end = nullptr; + const long parsed_block_tokens = std::strtol(argv[13], &block_tokens_end, 10); + char *prompt_text_token_end = nullptr; + const long parsed_prompt_text_token = + std::strtol(argv[14], &prompt_text_token_end, 10); + if (seed_end == argv[7] || *seed_end != '\0' || parsed_seed == 0 || + parsed_seed > std::numeric_limits::max() || + audio_temperature_end == argv[8] || *audio_temperature_end != '\0' || + parsed_audio_temperature <= 0.0f || text_temperature_end == argv[9] || + *text_temperature_end != '\0' || parsed_text_temperature <= 0.0f || + audio_top_k_end == argv[10] || *audio_top_k_end != '\0' || + parsed_audio_top_k <= 0 || + parsed_audio_top_k > std::numeric_limits::max() || + text_top_k_end == argv[11] || *text_top_k_end != '\0' || + parsed_text_top_k <= 0 || + parsed_text_top_k > std::numeric_limits::max() || + max_blocks_end == argv[12] || *max_blocks_end != '\0' || + parsed_max_blocks <= 0 || + parsed_max_blocks > std::numeric_limits::max() || + block_tokens_end == argv[13] || *block_tokens_end != '\0' || + parsed_block_tokens <= 0 || + parsed_block_tokens > std::numeric_limits::max() || + prompt_text_token_end == argv[14] || *prompt_text_token_end != '\0' || + parsed_prompt_text_token < std::numeric_limits::min() || + parsed_prompt_text_token > std::numeric_limits::max()) { + std::fprintf(stderr, "invalid sampling configuration\n"); + return 2; + } + const runner_config config{ + .mimi_path = argv[1], + .lm_path = argv[2], + .voice_path = argv[3], + .input_path = argv[4], + .output_path = argv[5], + .target_output_frames = static_cast(parsed_target), + .sampling_seed = static_cast(parsed_seed), + .audio_temperature = parsed_audio_temperature, + .text_temperature = parsed_text_temperature, + .audio_top_k = static_cast(parsed_audio_top_k), + .text_top_k = static_cast(parsed_text_top_k), + .max_blocks = static_cast(parsed_max_blocks), + .block_tokens = static_cast(parsed_block_tokens), + .prompt_text_token = static_cast(parsed_prompt_text_token), + }; + + loaded_model mimi_model = {}; + loaded_model lm_model = {}; + loaded_model voice_model = {}; + if (!load_model(config.mimi_path, mimi_model) || + !load_model(config.lm_path, lm_model) || + !load_model(config.voice_path, voice_model)) { + return 1; + } + + std::vector input_pcm = {}; + if (!load_wav_24khz_mono_s16(config.input_path, input_pcm)) { + std::fprintf(stderr, "input must be 24 kHz mono s16 WAV: %s\n", + config.input_path); + return 1; + } + + const int32_t frame_samples_i32 = static_cast( + std::lround(static_cast(mimi_model.model->mimi.sample_rate) / + static_cast(mimi_model.model->mimi.frame_rate))); + const int32_t public_n_q = lm_model.model->moshi_lm.inference_dep_q; + if (frame_samples_i32 <= 0 || public_n_q <= 0 || + public_n_q != mimi_model.model->mimi.n_q || + mimi_model.model->mimi.card != lm_model.model->moshi_lm.card) { + std::fprintf(stderr, "invalid PersonaPlex/Mimi model contract\n"); + return 1; + } + const size_t frame_samples = static_cast(frame_samples_i32); + const size_t input_frames = + (input_pcm.size() + frame_samples - 1u) / frame_samples; + if (input_frames > config.target_output_frames) { + std::fprintf(stderr, + "input requires %zu frames, exceeding target output %zu\n", + input_frames, config.target_output_frames); + return 1; + } + + std::vector encoder_prepared( + mimi::prepared_arena_floats(*mimi_model.model)); + std::vector encoder_state(mimi::state_arena_floats(*mimi_model.model)); + std::vector encoder_workspace( + mimi::workspace_arena_floats(*mimi_model.model)); + std::vector encoder_frame(mimi::frame_arena_floats(*mimi_model.model)); + std::vector decoder_prepared( + mimi::prepared_arena_floats(*mimi_model.model)); + std::vector decoder_state(mimi::state_arena_floats(*mimi_model.model)); + std::vector decoder_workspace( + mimi::workspace_arena_floats(*mimi_model.model)); + std::vector decoder_frame(mimi::frame_arena_floats(*mimi_model.model)); + streaming_cache temporal_cache = {}; + streaming_cache depformer_cache = {}; + if (!prepare_streaming_cache( + temporal_cache, lm_model.model->moshi_lm.num_layers, + lm_model.model->moshi_lm.context, lm_model.model->moshi_lm.dim) || + !prepare_streaming_cache(depformer_cache, + lm_model.model->moshi_lm.depformer_num_layers, + lm_model.model->moshi_lm.depformer_context, + lm_model.model->moshi_lm.depformer_dim)) { + std::fprintf(stderr, "failed to allocate Moshi streaming caches\n"); + return 1; + } + + std::vector pcm_frame(frame_samples, 0.0f); + std::vector output_pcm(config.target_output_frames * frame_samples, + 0.0f); + std::vector input_codes(static_cast(public_n_q), -1); + const int32_t model_codebook_count = lm_model.model->moshi_lm.n_q + 1; + const auto delay_begin = lm_model.model->moshi_lm.delays.begin(); + const auto delay_end = delay_begin + lm_model.model->moshi_lm.delay_count; + const int32_t maximum_delay = *std::max_element(delay_begin, delay_end); + const int32_t tokenizer_cache_rows = maximum_delay + 3; + std::vector model_codes(static_cast(model_codebook_count), + -1); + std::vector predicted_codes( + static_cast(lm_model.model->moshi_lm.dep_q), -1); + std::vector output_codes(static_cast(public_n_q), -1); + std::vector tokenizer_storage( + static_cast(tokenizer_cache_rows * model_codebook_count), -2); + std::vector tokenizer_cache_snapshot( + static_cast(tokenizer_cache_rows * model_codebook_count), -2); + + emel::memory::streaming::sm temporal_positions{ + emel::memory::streaming::dependencies{ + .capacity = lm_model.model->moshi_lm.context}}; + emel::memory::streaming::sm secondary_positions{ + emel::memory::streaming::dependencies{ + .capacity = lm_model.model->moshi_lm.depformer_context}}; + cache_views cache_view_set{ + .temporal = + { + .key_cache = std::span{temporal_cache.key_cache}, + .value_cache = std::span{temporal_cache.value_cache}, + .layer_cache_offsets = + std::span{temporal_cache.layer_offsets}, + .layer_count = lm_model.model->moshi_lm.num_layers, + .position_capacity = lm_model.model->moshi_lm.context, + .kv_dim = lm_model.model->moshi_lm.dim, + }, + .secondary = + { + .key_cache = std::span{depformer_cache.key_cache}, + .value_cache = std::span{depformer_cache.value_cache}, + .layer_cache_offsets = + std::span{depformer_cache.layer_offsets}, + .layer_count = lm_model.model->moshi_lm.depformer_num_layers, + .position_capacity = lm_model.model->moshi_lm.depformer_context, + .kv_dim = lm_model.model->moshi_lm.depformer_dim, + }, + }; + emel::kernel::sm prediction_kernel{}; + emel::logits::sampler::sm prediction_sampler{}; + runtime::sm prediction_runtime{runtime::dependencies{ + .kv = + runtime::kv_views{ + .temporal = cache_view_set.temporal, + .depformer = cache_view_set.secondary, + .temporal_positions = &temporal_positions, + .depformer_positions = &secondary_positions, + }, + .kernel = prediction_kernel, + .sampler = &prediction_sampler, + .policy = + runtime::policies{ + .rms_norm_epsilon = 1.0e-8f, + .zero_seed_state = 123459876u, + .sampling_modulus = 2147483647u, + .token_zero = -1, + }, + .capacity = + runtime::capacities{ + .hidden_dim = static_cast( + std::max(lm_model.model->moshi_lm.dim, + lm_model.model->moshi_lm.depformer_dim)), + .temporal_context = + static_cast(lm_model.model->moshi_lm.context), + .depformer_context = static_cast( + lm_model.model->moshi_lm.depformer_context), + .sampling_card = static_cast( + std::max(lm_model.model->moshi_lm.text_card, + lm_model.model->moshi_lm.card)), + .sampling_top_k = static_cast( + std::max(config.audio_top_k, config.text_top_k)), + }, + }}; + emel::memory::hybrid::sm prediction_memory{}; + predictor::sm token_predictor{predictor::dependencies{ + .memory = prediction_memory, + .graph = prediction_runtime, + .policy = + predictor::policies{ + .token_zero = -1, + .token_ungenerated = -2, + .prediction_step_size = 1, + .prediction_output_count = 1, + }, + }}; + std::vector prediction_temporal_state( + static_cast(lm_model.model->moshi_lm.dim)); + predictor::event::predict::workspace prediction_workspace{}; + prediction_workspace.temporal_state = + std::span{prediction_temporal_state}; + tokenizer::sm token_delay{tokenizer::dependencies{ + .delays = std::span{delay_begin, delay_end}, + .cache = std::span{tokenizer_storage}, + .codebooks = model_codebook_count, + .generated_audio_codebooks = lm_model.model->moshi_lm.dep_q, + .delayed_audio_codebooks = lm_model.model->moshi_lm.inference_dep_q, + .cache_rows = tokenizer_cache_rows, + .maximum_delay = maximum_delay, + .initial_delay_frames = 0, + .text_initial_token = lm_model.model->moshi_lm.text_card, + .audio_initial_token = lm_model.model->moshi_lm.card, + .token_zero = -1, + .token_ungenerated = -2, + }}; + mimi::sm encoder{}; + mimi::sm decoder{}; + emel::batch::planner::sm frame_planner{}; + + predictor::event::initialize predictor_initialize{*lm_model.model}; + predictor_initialize.max_sequences = 1; + predictor_initialize.max_blocks = config.max_blocks; + predictor_initialize.block_tokens = config.block_tokens; + predictor_initialize.sequence_id = 0; + predictor_initialize.codebook_capacity = model_codebook_count; + predictor_initialize.delay_cache_row_capacity = tokenizer_cache_rows; + predictor_initialize.sampling_enabled = true; + predictor_initialize.sampling_consume_forced_text = true; + predictor_initialize.sampling_audio_temperature = config.audio_temperature; + predictor_initialize.sampling_text_temperature = config.text_temperature; + predictor_initialize.sampling_audio_top_k = config.audio_top_k; + predictor_initialize.sampling_text_top_k = config.text_top_k; + predictor_initialize.sampling_seed = config.sampling_seed; + + personaplex_dependencies dependencies{ + .temporal_positions = temporal_positions, + .secondary_positions = secondary_positions, + .planner = frame_planner, + .encoder = encoder, + .tokenizer = token_delay, + .decoder = decoder, + .predictor = token_predictor, + .graph = token_predictor, + .sampler = token_predictor, + .prediction_workspace = prediction_workspace, + .encoder_initialize = + mimi::event::initialize{*mimi_model.model, + std::span{encoder_prepared}, + std::span{encoder_state}, + std::span{encoder_workspace}, + std::span{encoder_frame}}, + .decoder_initialize = + mimi::event::initialize{*mimi_model.model, + std::span{decoder_prepared}, + std::span{decoder_state}, + std::span{decoder_workspace}, + std::span{decoder_frame}}, + .predictor_initialize = predictor_initialize, + .conditioning_initialize = + predictor::event::load_voice{*voice_model.model}, + .prompt_begin = + predictor::event::begin_personaplex_prompt{ + .text_token_count = config.prompt_text_token >= 0 ? 1 : 0, + .pre_text_silence_frames = + lm_model.model->moshi_lm.inference_pre_text_silence_frames, + .post_text_silence_frames = + lm_model.model->moshi_lm.inference_post_text_silence_frames, + }, + .silence_pcm = std::span{pcm_frame}, + .input_codes = std::span{input_codes}, + .tokenize_input_codes = std::span{input_codes}, + .model_codes = std::span{model_codes}, + .predicted_codes = std::span{predicted_codes}, + .output_codes = std::span{output_codes}, + .tokenizer_cache_snapshot = std::span{tokenizer_cache_snapshot}, + .frame_samples = frame_samples_i32, + .codebook_count = public_n_q, + .frame_plan_mode = emel::batch::planner::event::plan_mode::simple, + .frame_plan_steps = 1, + .frame_plan_token_count = 1, + .frame_plan_output_all = true, + }; + generator::sm session{dependencies}; + emel::error::type session_err = emel::error::cast(generator::error::none); + if (!session.process_event(generator::event::initialize{session_err})) { + std::fprintf(stderr, "PersonaPlex session initialize failed err=%d\n", + static_cast(session_err)); + return 1; + } + + while ( + session.is(stateforward::sml::state)) { + bool complete = false; + int32_t remaining = -1; + session_err = emel::error::cast(generator::error::none); + (void)session.process_event(generator::event::condition{ + config.prompt_text_token, complete, remaining, session_err}); + } + while ( + session.is(stateforward::sml::state)) { + bool complete = false; + int32_t remaining = -1; + session_err = emel::error::cast(generator::error::none); + (void)session.process_event(generator::event::condition{ + config.prompt_text_token, complete, remaining, session_err}); + } + if (!session.is(stateforward::sml::state)) { + std::fprintf(stderr, "PersonaPlex prefill failed err=%d\n", + static_cast(session_err)); + return 1; + } + for (size_t frame_index = 0; frame_index < input_frames; ++frame_index) { + std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); + const size_t begin = frame_index * frame_samples; + const size_t copy_count = std::min(frame_samples, input_pcm.size() - begin); + std::copy_n(input_pcm.data() + begin, copy_count, pcm_frame.data()); + int32_t text_token = -1; + bool produced = false; + int32_t sample_count = 0; + session_err = emel::error::cast(generator::error::none); + generator::event::stream_frame frame{ + std::span{pcm_frame}, + std::span{output_pcm.data() + frame_index * frame_samples, + frame_samples}, + std::span{input_codes}, + std::span{output_codes}, + text_token, + sample_count, + produced, + session_err}; + if (!session.process_event(frame)) { + std::fprintf(stderr, "PersonaPlex stream frame=%zu failed err=%d\n", + frame_index, static_cast(session_err)); + return 1; + } + std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); + for (int32_t index = 0; index < public_n_q; ++index) { + std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", + input_codes[static_cast(index)]); + } + std::fprintf(stderr, "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, + text_token); + for (int32_t index = 0; index < public_n_q; ++index) { + std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", + output_codes[static_cast(index)]); + } + std::fprintf(stderr, "\n"); + } + + std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); + const size_t flush_steps = config.target_output_frames - input_frames; + for (size_t index = 0; index < flush_steps; ++index) { + const size_t frame_index = input_frames + index; + int32_t text_token = -1; + int32_t sample_count = 0; + bool complete = false; + session_err = emel::error::cast(generator::error::none); + generator::event::flush frame{ + std::span{output_pcm.data() + frame_index * frame_samples, + frame_samples}, + std::span{input_codes}, + std::span{output_codes}, + text_token, + sample_count, + complete, + session_err}; + if (!session.process_event(frame)) { + std::fprintf(stderr, "PersonaPlex flush frame=%zu failed err=%d\n", + frame_index, static_cast(session_err)); + return 1; + } + std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + input_codes[static_cast(codebook)]); + } + std::fprintf(stderr, "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, + text_token); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + output_codes[static_cast(codebook)]); + } + std::fprintf(stderr, "\n"); + } + const bool session_finished = + flush_steps == 0 + ? session.is(stateforward::sml::state) + : session.is(stateforward::sml::state); + if (!session_finished) { + std::fprintf(stderr, "PersonaPlex session failed err=%d\n", + static_cast(session_err)); + return 1; + } + + if (output_pcm.empty()) { + std::fprintf(stderr, "no generated audio frames were produced\n"); + return 1; + } + + double energy = 0.0; + float peak = 0.0f; + for (const float sample : output_pcm) { + energy += static_cast(sample) * static_cast(sample); + peak = std::max(peak, std::abs(sample)); + } + const double rms = std::sqrt(energy / static_cast(output_pcm.size())); + if (!write_wav_24khz_mono_s16(config.output_path, + std::span{output_pcm})) { + std::fprintf(stderr, "failed to write output wav: %s\n", + config.output_path); + return 1; + } + + std::printf("input_frames=%zu output_frames=%zu output_seconds=%.3f " + "peak=%.6f rms=%.6f seed=%u output=%s\n", + input_frames, output_pcm.size() / frame_samples, + static_cast(output_pcm.size()) / 24000.0, peak, rms, + config.sampling_seed, config.output_path); + return 0; +} diff --git a/tools/paritychecker/CMakeLists.txt b/tools/paritychecker/CMakeLists.txt index f76450ec..b7164021 100644 --- a/tools/paritychecker/CMakeLists.txt +++ b/tools/paritychecker/CMakeLists.txt @@ -76,11 +76,20 @@ function(emel_patch_reference_metadata_logging reference_source_dir) set(original_print " model.load_stats(ml);\n model.print_info();") set(patched_print " model.load_stats(ml);\n // EMEL parity disables reference metadata-only logging here; decode remains unchanged.") string(FIND "${llama_source}" "${patched_print}" print_patch_pos) + if(print_patch_pos EQUAL -1) + set(patched_print_arrow " model->load_stats(ml);\n // EMEL parity disables reference metadata-only logging here; decode remains unchanged.") + string(FIND "${llama_source}" "${patched_print_arrow}" print_patch_pos) + endif() if(print_patch_pos EQUAL -1) string(FIND "${llama_source}" "${original_print}" print_original_pos) if(print_original_pos EQUAL -1) - message(FATAL_ERROR - "Unable to patch reference model info logging in ${llama_path}") + set(original_print " model->load_stats(ml);\n model->print_info();") + set(patched_print " model->load_stats(ml);\n // EMEL parity disables reference metadata-only logging here; decode remains unchanged.") + string(FIND "${llama_source}" "${original_print}" print_original_pos) + if(print_original_pos EQUAL -1) + message(FATAL_ERROR + "Unable to patch reference model info logging in ${llama_path}") + endif() endif() string(REPLACE "${original_print}" "${patched_print}" llama_source "${llama_source}") file(WRITE "${llama_path}" "${llama_source}")