From e29f18a8e86c047fb4aa790d2d89966e3c604a06 Mon Sep 17 00:00:00 2001 From: Francesco Conti Date: Wed, 10 Jun 2026 11:06:22 +0200 Subject: [PATCH 01/13] backend: Import the datamover transpose engine Verbatim copy of rtl/datamover_engine.sv from pulp-platform/datamover@d58a985 (branch cdurrer/konark), the transpose core of the Ratha HWPE. Co-authored-by: Sergio Mazzola Co-authored-by: Cyrill Durrer --- rtl/datamover_engine.sv | 385 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 rtl/datamover_engine.sv diff --git a/rtl/datamover_engine.sv b/rtl/datamover_engine.sv new file mode 100644 index 00000000..6237efb5 --- /dev/null +++ b/rtl/datamover_engine.sv @@ -0,0 +1,385 @@ +/* + * Copyright (C) 2025-2026 ETH Zurich and University of Bologna + * + * Copyright and related rights are licensed under the Solderpad Hardware + * License, Version 0.51 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law + * or agreed to in writing, software, hardware and materials distributed under + * this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * Authors: Francesco Conti + * Sergio Mazzola + * Cyrill Durrer + */ + +module datamover_engine + import hwpe_stream_package::*; + import hci_package::*; + import datamover_package::*; +#( + parameter int unsigned FIFO_DEPTH = 2, + parameter int unsigned BANDWIDTH_ALIGNED = 512, + parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a bank word + parameter int unsigned ELEM_WIDTH = 8, // element width (in bits) + // Dependent parameters: do not modify! + localparam int unsigned WORD_WIDTH = NUM_ELEM_WORD * ELEM_WIDTH // should correspond to bank width +) ( + // global signals + input logic clk_i, + input logic rst_ni, + input logic test_mode_i, // unused + // local enable & clear + input logic enable_i, // unused + input logic clear_i, + // control registers + input ctrl_engine_t ctrl_i, + // input data stream + handshake + hwpe_stream_intf_stream.sink data_in, + // output data stream + handshake + hwpe_stream_intf_stream.source data_out +); + + // number of elements (in the full bandwidth, not a single bank word) + localparam int unsigned NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; + localparam int unsigned NB_ELEM_LOG2 = $clog2(NB_ELEMENTS); + + // Type def and internal signals + typedef enum logic { WRITE, READ } datamover_engine_fsm_t; + datamover_engine_fsm_t fsm_d, fsm_q; + logic clear_elem_matrix; + logic [NB_ELEM_LOG2-1:0] cnt_q, cnt_d; + logic [17:0] tot_cnt_q, tot_cnt_d; + logic cnt_en; + logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_unrolled; + logic data_in_valid; + logic data_in_ready; + logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_out_unrolled; + logic data_out_valid; + logic data_out_ready; + logic [NB_ELEM_LOG2-1:0] remaining_elems; + logic [17:0] total_accesses_copy_mode, total_accesses; + logic [15:0] y_elem_cnt, expanded_y_elems; + logic [9:0] y_tiles, n_tiles, n_tile_cnt; + logic [NB_ELEM_LOG2:0] leftover_rows, leftover_cols; + logic last_y_tile, last_n_tile; + + logic execution_done; + + // FSM: WRITE -> READ on input handshake at end of write, READ -> WRITE on output handshake at end of read + always_comb + begin + fsm_d = fsm_q; + case (fsm_q) + WRITE: begin + if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_in_valid & data_in_ready)) begin + fsm_d = READ; + end + end + READ: begin + if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_out_valid & data_out_ready)) begin + fsm_d = WRITE; + end + end + default: begin + fsm_d = WRITE; + end + endcase + end + + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + fsm_q <= WRITE; + end else if (clear_i || execution_done) begin + fsm_q <= WRITE; + end else begin + fsm_q <= fsm_d; + end + end + + assign clear_elem_matrix = (fsm_q == READ && fsm_d == WRITE); + + // internal interfaces and unrolling + hwpe_stream_intf_stream #( + .DATA_WIDTH ( BANDWIDTH_ALIGNED ), + .ELEMENT_WIDTH ( ELEM_WIDTH ), + .STRB_WIDTH ( NB_ELEMENTS ) + ) data_in_postfifo ( + .clk ( clk_i ) + ); + hwpe_stream_intf_stream #( + .DATA_WIDTH ( BANDWIDTH_ALIGNED ), + .ELEMENT_WIDTH ( ELEM_WIDTH ), + .STRB_WIDTH ( NB_ELEMENTS ) + ) data_out_prefifo ( + .clk ( clk_i ) + ); + + // decouple in/out with FIFOs + hwpe_stream_fifo #( + .DATA_WIDTH ( BANDWIDTH_ALIGNED ), + .ELEMENT_WIDTH ( ELEM_WIDTH ), + .FIFO_DEPTH ( FIFO_DEPTH ) + ) i_fifo_in ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .flags_o ( ), + .push_i ( data_in ), + .pop_o ( data_in_postfifo ) + ); + assign data_in_unrolled = data_in_postfifo.data; + assign data_in_valid = data_in_postfifo.valid; + assign data_in_postfifo.ready = data_in_ready; + hwpe_stream_fifo #( + .DATA_WIDTH ( BANDWIDTH_ALIGNED ), + .ELEMENT_WIDTH ( ELEM_WIDTH ), + .FIFO_DEPTH ( FIFO_DEPTH ) + ) i_fifo_out ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .flags_o ( ), + .push_i ( data_out_prefifo ), + .pop_o ( data_out ) + ); + + + // Partial tile / leftover elements handling + // Due to the streamer address generation, matrices need to be word-aligned in n-dimension for transposition + localparam logic [NB_ELEMENTS-1:0] STRB_ONE = {{(NB_ELEMENTS-1){1'b0}}, 1'b1}; // Necessary to force the shifting operation to the correct bitwidth (default would be only 32b) + assign remaining_elems = ctrl_i.total_elements & (NB_ELEMENTS - 1); // modulo (NB_ELEMENTS: power of two) - this signal is only used in copy mode + assign total_accesses_copy_mode = (ctrl_i.total_elements >> NB_ELEM_LOG2) + ((remaining_elems != 0) ? 1 : 0); + + // y_tiles represents the number of tiles in c-dimension for unfold/fold modes, and the number of tiles in m-dimension for all other modes + assign y_tiles = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? + (ctrl_i.num_channels + NB_ELEMENTS - 1) >> NB_ELEM_LOG2 : + (ctrl_i.tensor_size_m + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division + assign n_tiles = (ctrl_i.tensor_size_n + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division + assign total_accesses = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? (y_tiles * ctrl_i.tensor_size_m * n_tiles) << NB_ELEM_LOG2 : (y_tiles * n_tiles) << NB_ELEM_LOG2; // NB_ELEMENTS is a power of 2, so multiply by shifting; ToDo: remaining MUL overhead, could be pre-computed in HAL and configured in control register + assign leftover_rows = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? ctrl_i.num_channels & (NB_ELEMENTS - 1) : ctrl_i.tensor_size_m & (NB_ELEMENTS - 1); + assign leftover_cols = ctrl_i.tensor_size_n & (NB_ELEMENTS - 1); + assign expanded_y_elems = y_tiles << NB_ELEM_LOG2; // taking into account partial tiles + assign y_elem_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q % expanded_y_elems); // ToDo: restructure without modulo (pre-compute in HAL?) + assign n_tile_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q / expanded_y_elems); // ToDo: restructure without division (pre-compute in HAL?) + assign last_y_tile = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? + ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.num_channels >> NB_ELEM_LOG2)) : + ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.tensor_size_m >> NB_ELEM_LOG2)); + assign last_n_tile = (n_tile_cnt >= (ctrl_i.tensor_size_n >> NB_ELEM_LOG2)); + + always_comb begin + data_out_prefifo.strb = '1; + if(ctrl_i.total_elements != 0) begin + if(ctrl_i.datamover_mode == DATAMOVER_COPY) begin // Copy mode + data_out_prefifo.strb = ((tot_cnt_q >= total_accesses_copy_mode-1) && (remaining_elems != 0)) ? ((STRB_ONE << remaining_elems) - 1) : '1; + end else if(ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) begin // Transpose mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else if(last_n_tile && leftover_cols != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_CIM_CONVERSION) begin // CIM layout conversion mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_n_tile && leftover_cols != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_UNFOLD) begin // Unfold mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else if(last_n_tile && leftover_cols != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_FOLD) begin // Fold mode (inverse of unfold: leftover_rows <-> leftover_cols roles swapped) + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_n_tile && leftover_cols != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else if(last_y_tile && leftover_rows != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end + end + end + + assign data_out_prefifo.data = data_out_unrolled; + assign data_out_prefifo.valid = data_out_valid; + assign data_out_ready = data_out_prefifo.ready; + + // Write counter + always_ff @(posedge clk_i or negedge rst_ni) + begin + if (~rst_ni) begin + cnt_q <= '0; + tot_cnt_q <= '0; + end + else if(clear_i) begin + cnt_q <= '0; + tot_cnt_q <= '0; + end + else if(execution_done) begin + cnt_q <= '0; + tot_cnt_q <= '0; + end + else if(cnt_en) begin + cnt_q <= cnt_d; + tot_cnt_q <= tot_cnt_d; + end + end + assign cnt_d = (cnt_q < (ctrl_i.transp_len-ctrl_i.transp_stride)) ? cnt_q+ctrl_i.transp_stride : '0; + assign cnt_en = fsm_q == WRITE ? data_in_valid & data_in_ready : data_out_valid & data_out_ready; + assign execution_done = (ctrl_i.datamover_mode == DATAMOVER_COPY) ? (total_accesses_copy_mode != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses_copy_mode - 1) : + (total_accesses != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses - 1); + + // count total number of write accesses + assign tot_cnt_d = (data_out_prefifo.valid & data_out_prefifo.ready) ? tot_cnt_q + 1 : tot_cnt_q; + + // "Smart shifting": this set of combinational blocks shifts data_in_unrolled + // appropriately, depending on the configuration. + // E.g., if you have a classical configuration with + // - NUM_ELEM_WORD = 8 and + // - ELEM_WIDTH = 8 bits, i.e. total is 64 bits per word + // the configurations are: 8b transpose, 16b transpose, 32b transpose. We assume + // that transposes >= 64b can be done efficiently by Snitch processors through SSRs + // and those < 8b are not interesting in our use case. + localparam MAX_SHIFTING = (NUM_ELEM_WORD > 4) ? NUM_ELEM_WORD : 4; + // e.g., in a classical configuration (ELEM_WIDTH = 8), MAX_SHIFTING is + // in bytes, i.e., "4" for 32b transpose (includes shifting by 0 bytes) + logic [MAX_SHIFTING-1:0][NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_shifted; + + for(genvar ii=0; ii> 2) == (ii >> 2)) & data_in_valid & data_in_ready : + ctrl_i.transp_mode == TRANSP_2ELEM ? ((cnt_q >> 1) == (ii >> 1)) & data_in_valid & data_in_ready : + ( cnt_q == ii ) & data_in_valid & data_in_ready; + // select appropriately shifted rows + logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_selected; + assign data_in_selected = ctrl_i.transp_mode == TRANSP_4ELEM? data_in_shifted[ii % 4] : + ctrl_i.transp_mode == TRANSP_2ELEM ? data_in_shifted[ii % 2] : + data_in_shifted[0]; + + for(genvar jj=0; jj Date: Wed, 10 Jun 2026 16:03:38 +0200 Subject: [PATCH 02/13] backend: Adapt the datamover engine into idma_otf_transpose Rework the imported datamover_engine.sv to iDMA conventions: plain valid/ready with byte/strb ports, no hwpe_stream/hci dependencies, transpose only. Runtime element size (int8/fp16/fp32), element-granular edge strobe, ping-pong tile banks with a half-area FullDuplex=0 option, and a standalone DPI-C golden regression. --- rtl/datamover_engine.sv | 385 ------------------------------ src/backend/idma_otf_transpose.sv | 234 ++++++++++++++++++ test/idma_transpose_dpi.c | 39 +++ test/tb_idma_otf_transpose.sv | 167 +++++++++++++ 4 files changed, 440 insertions(+), 385 deletions(-) delete mode 100644 rtl/datamover_engine.sv create mode 100644 src/backend/idma_otf_transpose.sv create mode 100644 test/idma_transpose_dpi.c create mode 100644 test/tb_idma_otf_transpose.sv diff --git a/rtl/datamover_engine.sv b/rtl/datamover_engine.sv deleted file mode 100644 index 6237efb5..00000000 --- a/rtl/datamover_engine.sv +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright (C) 2025-2026 ETH Zurich and University of Bologna - * - * Copyright and related rights are licensed under the Solderpad Hardware - * License, Version 0.51 (the "License"); you may not use this file except in - * compliance with the License. You may obtain a copy of the License at - * http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law - * or agreed to in writing, software, hardware and materials distributed under - * this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Authors: Francesco Conti - * Sergio Mazzola - * Cyrill Durrer - */ - -module datamover_engine - import hwpe_stream_package::*; - import hci_package::*; - import datamover_package::*; -#( - parameter int unsigned FIFO_DEPTH = 2, - parameter int unsigned BANDWIDTH_ALIGNED = 512, - parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a bank word - parameter int unsigned ELEM_WIDTH = 8, // element width (in bits) - // Dependent parameters: do not modify! - localparam int unsigned WORD_WIDTH = NUM_ELEM_WORD * ELEM_WIDTH // should correspond to bank width -) ( - // global signals - input logic clk_i, - input logic rst_ni, - input logic test_mode_i, // unused - // local enable & clear - input logic enable_i, // unused - input logic clear_i, - // control registers - input ctrl_engine_t ctrl_i, - // input data stream + handshake - hwpe_stream_intf_stream.sink data_in, - // output data stream + handshake - hwpe_stream_intf_stream.source data_out -); - - // number of elements (in the full bandwidth, not a single bank word) - localparam int unsigned NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; - localparam int unsigned NB_ELEM_LOG2 = $clog2(NB_ELEMENTS); - - // Type def and internal signals - typedef enum logic { WRITE, READ } datamover_engine_fsm_t; - datamover_engine_fsm_t fsm_d, fsm_q; - logic clear_elem_matrix; - logic [NB_ELEM_LOG2-1:0] cnt_q, cnt_d; - logic [17:0] tot_cnt_q, tot_cnt_d; - logic cnt_en; - logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_unrolled; - logic data_in_valid; - logic data_in_ready; - logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_out_unrolled; - logic data_out_valid; - logic data_out_ready; - logic [NB_ELEM_LOG2-1:0] remaining_elems; - logic [17:0] total_accesses_copy_mode, total_accesses; - logic [15:0] y_elem_cnt, expanded_y_elems; - logic [9:0] y_tiles, n_tiles, n_tile_cnt; - logic [NB_ELEM_LOG2:0] leftover_rows, leftover_cols; - logic last_y_tile, last_n_tile; - - logic execution_done; - - // FSM: WRITE -> READ on input handshake at end of write, READ -> WRITE on output handshake at end of read - always_comb - begin - fsm_d = fsm_q; - case (fsm_q) - WRITE: begin - if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_in_valid & data_in_ready)) begin - fsm_d = READ; - end - end - READ: begin - if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_out_valid & data_out_ready)) begin - fsm_d = WRITE; - end - end - default: begin - fsm_d = WRITE; - end - endcase - end - - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - fsm_q <= WRITE; - end else if (clear_i || execution_done) begin - fsm_q <= WRITE; - end else begin - fsm_q <= fsm_d; - end - end - - assign clear_elem_matrix = (fsm_q == READ && fsm_d == WRITE); - - // internal interfaces and unrolling - hwpe_stream_intf_stream #( - .DATA_WIDTH ( BANDWIDTH_ALIGNED ), - .ELEMENT_WIDTH ( ELEM_WIDTH ), - .STRB_WIDTH ( NB_ELEMENTS ) - ) data_in_postfifo ( - .clk ( clk_i ) - ); - hwpe_stream_intf_stream #( - .DATA_WIDTH ( BANDWIDTH_ALIGNED ), - .ELEMENT_WIDTH ( ELEM_WIDTH ), - .STRB_WIDTH ( NB_ELEMENTS ) - ) data_out_prefifo ( - .clk ( clk_i ) - ); - - // decouple in/out with FIFOs - hwpe_stream_fifo #( - .DATA_WIDTH ( BANDWIDTH_ALIGNED ), - .ELEMENT_WIDTH ( ELEM_WIDTH ), - .FIFO_DEPTH ( FIFO_DEPTH ) - ) i_fifo_in ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .clear_i ( clear_i ), - .flags_o ( ), - .push_i ( data_in ), - .pop_o ( data_in_postfifo ) - ); - assign data_in_unrolled = data_in_postfifo.data; - assign data_in_valid = data_in_postfifo.valid; - assign data_in_postfifo.ready = data_in_ready; - hwpe_stream_fifo #( - .DATA_WIDTH ( BANDWIDTH_ALIGNED ), - .ELEMENT_WIDTH ( ELEM_WIDTH ), - .FIFO_DEPTH ( FIFO_DEPTH ) - ) i_fifo_out ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .clear_i ( clear_i ), - .flags_o ( ), - .push_i ( data_out_prefifo ), - .pop_o ( data_out ) - ); - - - // Partial tile / leftover elements handling - // Due to the streamer address generation, matrices need to be word-aligned in n-dimension for transposition - localparam logic [NB_ELEMENTS-1:0] STRB_ONE = {{(NB_ELEMENTS-1){1'b0}}, 1'b1}; // Necessary to force the shifting operation to the correct bitwidth (default would be only 32b) - assign remaining_elems = ctrl_i.total_elements & (NB_ELEMENTS - 1); // modulo (NB_ELEMENTS: power of two) - this signal is only used in copy mode - assign total_accesses_copy_mode = (ctrl_i.total_elements >> NB_ELEM_LOG2) + ((remaining_elems != 0) ? 1 : 0); - - // y_tiles represents the number of tiles in c-dimension for unfold/fold modes, and the number of tiles in m-dimension for all other modes - assign y_tiles = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? - (ctrl_i.num_channels + NB_ELEMENTS - 1) >> NB_ELEM_LOG2 : - (ctrl_i.tensor_size_m + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division - assign n_tiles = (ctrl_i.tensor_size_n + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division - assign total_accesses = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? (y_tiles * ctrl_i.tensor_size_m * n_tiles) << NB_ELEM_LOG2 : (y_tiles * n_tiles) << NB_ELEM_LOG2; // NB_ELEMENTS is a power of 2, so multiply by shifting; ToDo: remaining MUL overhead, could be pre-computed in HAL and configured in control register - assign leftover_rows = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? ctrl_i.num_channels & (NB_ELEMENTS - 1) : ctrl_i.tensor_size_m & (NB_ELEMENTS - 1); - assign leftover_cols = ctrl_i.tensor_size_n & (NB_ELEMENTS - 1); - assign expanded_y_elems = y_tiles << NB_ELEM_LOG2; // taking into account partial tiles - assign y_elem_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q % expanded_y_elems); // ToDo: restructure without modulo (pre-compute in HAL?) - assign n_tile_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q / expanded_y_elems); // ToDo: restructure without division (pre-compute in HAL?) - assign last_y_tile = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? - ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.num_channels >> NB_ELEM_LOG2)) : - ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.tensor_size_m >> NB_ELEM_LOG2)); - assign last_n_tile = (n_tile_cnt >= (ctrl_i.tensor_size_n >> NB_ELEM_LOG2)); - - always_comb begin - data_out_prefifo.strb = '1; - if(ctrl_i.total_elements != 0) begin - if(ctrl_i.datamover_mode == DATAMOVER_COPY) begin // Copy mode - data_out_prefifo.strb = ((tot_cnt_q >= total_accesses_copy_mode-1) && (remaining_elems != 0)) ? ((STRB_ONE << remaining_elems) - 1) : '1; - end else if(ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) begin // Transpose mode - if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); - end else begin - data_out_prefifo.strb = '0; - end - end else if(last_y_tile && leftover_rows != 0) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); - end else if(last_n_tile && leftover_cols != 0) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin - data_out_prefifo.strb = '1; - end else begin - data_out_prefifo.strb = '0; - end - end else begin - data_out_prefifo.strb = '1; - end - end else if(ctrl_i.datamover_mode == DATAMOVER_CIM_CONVERSION) begin // CIM layout conversion mode - if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); - end else begin - data_out_prefifo.strb = '0; - end - end else if(last_y_tile && leftover_rows != 0) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin - data_out_prefifo.strb = '1; - end else begin - data_out_prefifo.strb = '0; - end - end else if(last_n_tile && leftover_cols != 0) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); - end else begin - data_out_prefifo.strb = '1; - end - end else if(ctrl_i.datamover_mode == DATAMOVER_UNFOLD) begin // Unfold mode - if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); - end else begin - data_out_prefifo.strb = '0; - end - end else if(last_y_tile && leftover_rows != 0) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); - end else if(last_n_tile && leftover_cols != 0) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin - data_out_prefifo.strb = '1; - end else begin - data_out_prefifo.strb = '0; - end - end else begin - data_out_prefifo.strb = '1; - end - end else if(ctrl_i.datamover_mode == DATAMOVER_FOLD) begin // Fold mode (inverse of unfold: leftover_rows <-> leftover_cols roles swapped) - if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); - end else begin - data_out_prefifo.strb = '0; - end - end else if(last_n_tile && leftover_cols != 0) begin - data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); - end else if(last_y_tile && leftover_rows != 0) begin - if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin - data_out_prefifo.strb = '1; - end else begin - data_out_prefifo.strb = '0; - end - end else begin - data_out_prefifo.strb = '1; - end - end - end - end - - assign data_out_prefifo.data = data_out_unrolled; - assign data_out_prefifo.valid = data_out_valid; - assign data_out_ready = data_out_prefifo.ready; - - // Write counter - always_ff @(posedge clk_i or negedge rst_ni) - begin - if (~rst_ni) begin - cnt_q <= '0; - tot_cnt_q <= '0; - end - else if(clear_i) begin - cnt_q <= '0; - tot_cnt_q <= '0; - end - else if(execution_done) begin - cnt_q <= '0; - tot_cnt_q <= '0; - end - else if(cnt_en) begin - cnt_q <= cnt_d; - tot_cnt_q <= tot_cnt_d; - end - end - assign cnt_d = (cnt_q < (ctrl_i.transp_len-ctrl_i.transp_stride)) ? cnt_q+ctrl_i.transp_stride : '0; - assign cnt_en = fsm_q == WRITE ? data_in_valid & data_in_ready : data_out_valid & data_out_ready; - assign execution_done = (ctrl_i.datamover_mode == DATAMOVER_COPY) ? (total_accesses_copy_mode != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses_copy_mode - 1) : - (total_accesses != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses - 1); - - // count total number of write accesses - assign tot_cnt_d = (data_out_prefifo.valid & data_out_prefifo.ready) ? tot_cnt_q + 1 : tot_cnt_q; - - // "Smart shifting": this set of combinational blocks shifts data_in_unrolled - // appropriately, depending on the configuration. - // E.g., if you have a classical configuration with - // - NUM_ELEM_WORD = 8 and - // - ELEM_WIDTH = 8 bits, i.e. total is 64 bits per word - // the configurations are: 8b transpose, 16b transpose, 32b transpose. We assume - // that transposes >= 64b can be done efficiently by Snitch processors through SSRs - // and those < 8b are not interesting in our use case. - localparam MAX_SHIFTING = (NUM_ELEM_WORD > 4) ? NUM_ELEM_WORD : 4; - // e.g., in a classical configuration (ELEM_WIDTH = 8), MAX_SHIFTING is - // in bytes, i.e., "4" for 32b transpose (includes shifting by 0 bytes) - logic [MAX_SHIFTING-1:0][NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_shifted; - - for(genvar ii=0; ii> 2) == (ii >> 2)) & data_in_valid & data_in_ready : - ctrl_i.transp_mode == TRANSP_2ELEM ? ((cnt_q >> 1) == (ii >> 1)) & data_in_valid & data_in_ready : - ( cnt_q == ii ) & data_in_valid & data_in_ready; - // select appropriately shifted rows - logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_selected; - assign data_in_selected = ctrl_i.transp_mode == TRANSP_4ELEM? data_in_shifted[ii % 4] : - ctrl_i.transp_mode == TRANSP_2ELEM ? data_in_shifted[ii % 2] : - data_in_shifted[0]; - - for(genvar jj=0; jj +// +// On-the-fly matrix transpose engine (ping-pong tile banks) for the iDMA +// transport write seam. Adapted from the datamover (Ratha) HWPE: +// pulp-platform/datamover@d58a985, rtl/datamover_engine.sv. +// +// Contract: input padded to full tiles, fed (col-tile, row-tile, row) order; +// out_T[nt*NE+k][rt*NE+r] = in[rt*NE+r][nt*NE+k]. strb_o masks partial edge tiles. +// Throughput: 1 + 1/NE cycles per NE-beat tile (one handoff bubble per tile). + +module idma_otf_transpose #( + /// Byte lanes per beat (= DataWidth/8) + parameter int unsigned StrbWidth = 32'd8, + /// Tensor dimension width in elements (matches idma_pkg::TransposeDimWidth) + parameter int unsigned DimWidth = 32'd12, + /// 1: two tile banks, fill while draining; 0: one bank (half area, half rate) + parameter bit FullDuplex = 1'b1, + localparam int unsigned NumBanks = FullDuplex ? 32'd2 : 32'd1, + localparam int unsigned LaneW = $clog2(StrbWidth) +) ( + input logic clk_i, + input logic rst_ni, + input logic clear_i, + + /// Element size select: 0->1B, 1->2B, 2->4B (E = 1<StrbWidth) degrades to NE=1 + assign eff_mode = (transp_mode_i > LaneW) ? LaneW[1:0] : transp_mode_i; + assign ne_m1 = (1 << (LaneW - eff_mode)) - 1; // NE-1 + assign log2_ne = LaneW - eff_mode; + // Widen the ceil-div add by one bit so it cannot wrap at the dim range. + assign y_tiles = DimWidth'(((DimWidth+1)'(tensor_size_m_i) + ne_m1) >> log2_ne); + assign n_tiles = DimWidth'(((DimWidth+1)'(tensor_size_n_i) + ne_m1) >> log2_ne); + assign leftover_rows = tensor_size_m_i & ne_m1; + assign leftover_cols = tensor_size_n_i & ne_m1; + + // FF tile banks (ping-pong when FullDuplex), E=1 worst case (StrbWidth x StrbWidth B) + logic [StrbWidth-1:0][7:0] tile_q [NumBanks][StrbWidth]; + + // internal output + handshakes + logic [StrbWidth-1:0][7:0] data_int; + logic [StrbWidth-1:0] strb_int; + logic valid_int, ready_int; + logic in_hs, out_hs; + assign in_hs = valid_i & ready_o; + assign out_hs = valid_int & ready_int; + + // full_q[b]: bank b holds a complete tile. Producer sets on fill-complete, + // consumer clears on drain-complete. + logic [NumBanks-1:0] full_q; + logic wr_bank, rd_bank; + logic [LaneW-1:0] wr_cnt, rd_cnt; // intra-tile beat index (write / read) + logic wr_last, rd_last; + + assign wr_last = (wr_cnt == ne_m1[LaneW-1:0]); + assign rd_last = (rd_cnt == ne_m1[LaneW-1:0]); + + assign ready_o = ~full_q[wr_bank]; + assign valid_int = full_q[rd_bank]; + + // tile walkers (col-tile outer, row-tile inner); drain trails fill by up to one tile + logic [DimWidth-1:0] rtw, ntw; // write walker: row-tile, col-tile + logic [DimWidth-1:0] rtr, ntr; // read walker: row-tile, col-tile + + logic last_y_tile_w, last_n_tile_w; // edge flags of the tile being filled + assign last_y_tile_w = (rtw == y_tiles - 1); + assign last_n_tile_w = (ntw == n_tiles - 1); + + logic last_y_tile_r, last_n_tile_r; // edge flags of the tile being drained + assign last_y_tile_r = (rtr == y_tiles - 1); + assign last_n_tile_r = (ntr == n_tiles - 1); + + // per-bank edge flags, captured at fill-complete, consumed by the drain strobe + logic shadow_last_y [NumBanks]; + logic shadow_last_n [NumBanks]; + + // fill-/drain-complete events + logic fill_done, drain_done, exec_done; + assign fill_done = in_hs & wr_last; + assign drain_done = out_hs & rd_last; + // transfer done once the final tile drains + assign exec_done = drain_done & last_y_tile_r & last_n_tile_r; + + // producer (input) side + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + for (int b = 0; b < NumBanks; b++) + for (int r = 0; r < StrbWidth; r++) + tile_q[b][r] <= '0; + wr_cnt <= '0; + wr_bank <= 1'b0; + rtw <= '0; + ntw <= '0; + for (int b = 0; b < NumBanks; b++) begin + shadow_last_y[b] <= 1'b0; + shadow_last_n[b] <= 1'b0; + end + end else if (clear_i || exec_done) begin + wr_cnt <= '0; + wr_bank <= 1'b0; + rtw <= '0; + ntw <= '0; + for (int b = 0; b < NumBanks; b++) begin + shadow_last_y[b] <= 1'b0; + shadow_last_n[b] <= 1'b0; + end + end else begin + if (in_hs) begin + tile_q[wr_bank][wr_cnt] <= data_i; + wr_cnt <= wr_last ? '0 : (wr_cnt + 1'b1); + end + if (fill_done) begin + shadow_last_y[wr_bank] <= last_y_tile_w; + shadow_last_n[wr_bank] <= last_n_tile_w; + wr_bank <= FullDuplex ? ~wr_bank : 1'b0; + if (rtw == y_tiles - 1) begin + rtw <= '0; + ntw <= ntw + 1'b1; + end else begin + rtw <= rtw + 1'b1; + end + end + end + end + + // consumer (output) side + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i || exec_done) begin + rd_cnt <= '0; + rd_bank <= 1'b0; + rtr <= '0; + ntr <= '0; + end else begin + if (out_hs) begin + rd_cnt <= rd_last ? '0 : (rd_cnt + 1'b1); + end + if (drain_done) begin + rd_bank <= FullDuplex ? ~rd_bank : 1'b0; + if (rtr == y_tiles - 1) begin + rtr <= '0; + ntr <= ntr + 1'b1; + end else begin + rtr <= rtr + 1'b1; + end + end + end + end + + // full/empty token + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i || exec_done) begin + full_q <= 2'b00; + end else begin + if (fill_done) full_q[wr_bank] <= 1'b1; + if (drain_done) full_q[rd_bank] <= 1'b0; + end + end + + // transposed readout: byte p (element e=p>>logE, byte b=p&(E-1)) reads + // tile_q[rd_bank][e][rd_cnt*E + b] + always_comb begin + for (int p = 0; p < StrbWidth; p++) begin + automatic int unsigned e = p >> eff_mode; + automatic int unsigned b = p & ((1 << eff_mode) - 1); + automatic int unsigned col = (rd_cnt << eff_mode) | b; + data_int[p] = tile_q[rd_bank][e][col]; + end + end + + // output strobe: element-granular edge masking from the drain-side shadow flags + always_comb begin + logic [StrbWidth-1:0] em; // per-element valid (only low NE bits meaningful) + logic ly, ln; + ly = shadow_last_y[rd_bank]; + ln = shadow_last_n[rd_bank]; + for (int e = 0; e < StrbWidth; e++) begin + logic v; + if ((ly && leftover_rows != 0) && (ln && leftover_cols != 0)) + v = (rd_cnt < leftover_cols) && (e < leftover_rows); + else if (ly && leftover_rows != 0) + v = (e < leftover_rows); + else if (ln && leftover_cols != 0) + v = (rd_cnt < leftover_cols); + else + v = 1'b1; + em[e] = v; + end + for (int p = 0; p < StrbWidth; p++) + strb_int[p] = em[p >> eff_mode]; + end + + // output register; not cleared by exec_done so the final beat is held until accepted + assign ready_int = ~valid_o | ready_i; + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i) begin + valid_o <= 1'b0; + data_o <= '0; + strb_o <= '0; + end else if (ready_int) begin + valid_o <= valid_int; + data_o <= data_int; + strb_o <= strb_int; + end + end + +endmodule : idma_otf_transpose diff --git a/test/idma_transpose_dpi.c b/test/idma_transpose_dpi.c new file mode 100644 index 00000000..0f7faf3f --- /dev/null +++ b/test/idma_transpose_dpi.c @@ -0,0 +1,39 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Authors: +// - Daniel Keller +// +// DPI-C golden model for idma_otf_transpose: an element-granular matrix +// transpose. Element size E in {1,2,4} bytes (int8/fp16/fp32); each E-byte +// element is kept intact while the M x N element grid is transposed to N x M. +// Accessor-based (no open-array marshalling): the testbench loads the row-major +// input byte by byte, calls gm_transpose(m,n,e), then reads back the transposed +// output bytes. Reference: out_elem[c][r] = in_elem[r][c]. + +#include + +#define GM_MAX_BYTES (1 << 24) // 16 MiB + +static uint8_t gm_in[GM_MAX_BYTES]; +static uint8_t gm_out[GM_MAX_BYTES]; + +// Load one input byte at flat byte index. +void gm_load(int idx, int val) { + if (idx >= 0 && idx < GM_MAX_BYTES) gm_in[idx] = (uint8_t)val; +} + +// Transpose an m x n matrix of e-byte elements (row-major) into n x m. +void gm_transpose(int m, int n, int e) { + for (int r = 0; r < m; r++) + for (int c = 0; c < n; c++) + for (int b = 0; b < e; b++) + gm_out[((long)c * m + r) * e + b] = gm_in[((long)r * n + c) * e + b]; +} + +// Read one transposed output byte at flat byte index. +int gm_get(int idx) { + if (idx >= 0 && idx < GM_MAX_BYTES) return (int)gm_out[idx]; + return -1; +} diff --git a/test/tb_idma_otf_transpose.sv b/test/tb_idma_otf_transpose.sv new file mode 100644 index 00000000..23e29a3d --- /dev/null +++ b/test/tb_idma_otf_transpose.sv @@ -0,0 +1,167 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller +// +// Standalone self-checking testbench for idma_otf_transpose (element transpose, +// no iDMA backend / no protocol deps). Verifies a full M x N transpose of +// EB-byte elements (EB in {1,2,4} = int8/fp16/fp32), split into NE-square tiles +// (NE = StrbWidth/EB): input padded to full tiles, fed in (col-tile, row-tile, +// row) order; output collected through the engine's per-byte strb_o edge mask. +// Expected result comes from the DPI-C golden (idma_transpose_dpi.c). Checks +// correctness vs golden, full coverage, and no strobe asserted on padding. +// Optional two-sided backpressure via +BP. Override with -gM= -gN= -gEB=. + +`timescale 1ns/1ps + +module tb_idma_otf_transpose #( + parameter int unsigned StrbWidth = 32'd8, + parameter bit FullDuplex = 1'b1, + parameter int unsigned M = 32'd8, // matrix rows (elements) + parameter int unsigned N = 32'd8, // matrix cols (elements) + parameter int unsigned EB = 32'd1 // element size in bytes (1/2/4) +); + + import "DPI-C" function void gm_load(input int idx, input int val); + import "DPI-C" function void gm_transpose(input int m, input int n, input int e); + import "DPI-C" function int gm_get(input int idx); + + localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; // log2(EB) + localparam int unsigned NE = StrbWidth / EB; // elements/beat = tile side + localparam int unsigned YT = (M + NE - 1) / NE; // row-tiles + localparam int unsigned NT = (N + NE - 1) / NE; // col-tiles + localparam int unsigned LR = M % NE; + localparam int unsigned LC = N % NE; + localparam logic [7:0] PAD = 8'hFF; + + logic clk = 1'b0, rst_n = 1'b0, clear = 1'b0; + always #5 clk = ~clk; + + logic [StrbWidth-1:0][7:0] din_data; + logic din_valid, din_ready; + logic [StrbWidth-1:0][7:0] dout_data; + logic [StrbWidth-1:0] dout_strb; + logic dout_valid, dout_ready; + + idma_otf_transpose #( + .StrbWidth (StrbWidth), + .FullDuplex (FullDuplex) + ) i_dut ( + .clk_i (clk), + .rst_ni (rst_n), + .clear_i (clear), + .transp_mode_i (2'(MODE)), + .tensor_size_m_i (12'(M)), + .tensor_size_n_i (12'(N)), + .data_i (din_data), + .valid_i (din_valid), + .ready_o (din_ready), + .data_o (dout_data), + .strb_o (dout_strb), + .valid_o (dout_valid), + .ready_i (dout_ready) + ); + + logic [7:0] inb [M*N*EB]; // row-major input bytes + bit wrote [N*M]; // per transposed-element coverage + int unsigned errors = 0; + bit backpressure = 1'b0; + + task automatic drive_inputs(); + int unsigned beat = 0; + din_valid = 1'b0; din_data = '0; + @(posedge clk); + for (int nt = 0; nt < NT; nt++) + for (int rt = 0; rt < YT; rt++) + for (int row = 0; row < NE; row++) begin + if (backpressure) begin din_valid = 1'b0; repeat (beat % 3) @(posedge clk); end + for (int c = 0; c < NE; c++) begin + int gr = rt*NE + row; + int gc = nt*NE + c; + for (int b = 0; b < EB; b++) + din_data[c*EB + b] = (gr < M && gc < N) ? inb[(gr*N + gc)*EB + b] : PAD; + end + din_valid = 1'b1; + do @(posedge clk); while (!din_ready); + beat++; + end + din_valid = 1'b0; + endtask + + task automatic capture_outputs(); + int unsigned beat = 0; + dout_ready = 1'b0; + for (int nt = 0; nt < NT; nt++) + for (int rt = 0; rt < YT; rt++) + for (int k = 0; k < NE; k++) begin + if (backpressure) begin dout_ready = 1'b0; repeat (beat % 4) @(posedge clk); end + dout_ready = 1'b1; + do @(posedge clk); while (!dout_valid); + for (int e = 0; e < NE; e++) begin + if (dout_strb[e*EB]) begin // element e valid (element-granular mask) + int tr = nt*NE + k; // transposed row (= original col, 0..N-1) + int tc = rt*NE + e; // transposed col (= original row, 0..M-1) + if (tr >= N || tc >= M) begin + errors++; + if (errors <= 16) $display("STRB-ON-PAD beat(nt%0d rt%0d k%0d) elem %0d -> (%0d,%0d) OOB", nt, rt, k, e, tr, tc); + end else begin + for (int b = 0; b < EB; b++) begin + automatic int gold = gm_get((tr*M + tc)*EB + b); + if (int'(dout_data[e*EB + b]) !== gold) begin + errors++; + if (errors <= 16) $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", tr, tc, b, dout_data[e*EB+b], gold); + end + end + wrote[tr*M + tc] = 1'b1; + end + end + end + beat++; + end + @(posedge clk); + dout_ready = 1'b0; + endtask + + initial begin + for (int i = 0; i < M*N*EB; i++) inb[i] = 8'((i * 7 + 3) & 8'hFF); // varied stimulus + for (int i = 0; i < M*N*EB; i++) gm_load(i, int'(inb[i])); + gm_transpose(M, N, EB); + for (int i = 0; i < N*M; i++) wrote[i] = 1'b0; + + din_valid = 1'b0; dout_ready = 1'b0; + if ($test$plusargs("BP")) backpressure = 1'b1; + $display("[TB] idma_otf_transpose M=%0d N=%0d EB=%0d (tile=%0d elems, %0dx%0d tiles, LR=%0d LC=%0d) BP=%0d", + M, N, EB, NE, YT, NT, LR, LC, backpressure); + + rst_n = 1'b0; clear = 1'b1; + repeat (4) @(posedge clk); + rst_n = 1'b1; + repeat (2) @(posedge clk); + clear = 1'b0; + @(posedge clk); + + fork drive_inputs(); capture_outputs(); join + + for (int tr = 0; tr < N; tr++) + for (int tc = 0; tc < M; tc++) + if (!wrote[tr*M + tc]) begin + errors++; + if (errors <= 16) $display("MISSING transposed elem (%0d,%0d)", tr, tc); + end + + if (errors == 0) $display("[TB] PASS: %0dx%0d EB=%0d transpose matches DPI golden", M, N, EB); + else $display("[TB] FAIL: %0d errors", errors); + + repeat (5) @(posedge clk); + $finish; + end + + initial begin + #5000000; + $display("[TB] FAIL: timeout"); + $finish; + end + +endmodule From 6e590eb9caa7de0271ad719313e4901c5bf34baf Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 10 Jun 2026 16:03:38 +0200 Subject: [PATCH 03/13] idma_pkg: Add the per-transfer compute request model compute_options_t carries {enable, op, params} in the request options; transpose_options_t packs the element mode and tensor shape; compute_enable_t is the compile-time per-op build gate. --- src/idma_pkg.sv | 33 +++++++++++++++++++++++++++++++++ src/include/idma/typedef.svh | 5 +++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/idma_pkg.sv b/src/idma_pkg.sv index c89f4b5f..92c1bcf9 100644 --- a/src/idma_pkg.sv +++ b/src/idma_pkg.sv @@ -81,6 +81,39 @@ package idma_pkg; logic dst_reduce_len; } backend_options_t; + /// On-the-fly compute operation selector + typedef enum logic [3:0] { + COMPUTE_NONE = 4'd0, + COMPUTE_TRANSPOSE = 4'd1 + } compute_op_e; + + /// Transpose tensor dimension width in elements (bound by the inst64 DMCPY argb encoding) + localparam int unsigned TransposeDimWidth = 32'd12; + + /// Transpose option type: E = 1< 1/2/4 B); tensor in elements + typedef struct packed { + logic [1:0] mode; + logic [TransposeDimWidth-1:0] tensor_m; + logic [TransposeDimWidth-1:0] tensor_n; + } transpose_options_t; + + /// Per-op compute parameter union (members must be equal width) + typedef union packed { + transpose_options_t transpose; + } compute_params_t; + + /// Compute option type: per-transfer on-the-fly compute selection + typedef struct packed { + logic enable; + compute_op_e op; + compute_params_t params; + } compute_options_t; + + /// Compile-time per-op compute feature enables + typedef struct packed { + logic transpose; + } compute_enable_t; + /// Supported Protocols /// - `AXI`: Full AXI /// - `AXILITE`: AXI Lite diff --git a/src/include/idma/typedef.svh b/src/include/idma/typedef.svh index 841200a1..642923f0 100644 --- a/src/include/idma/typedef.svh +++ b/src/include/idma/typedef.svh @@ -27,8 +27,9 @@ axi_id_t axi_id; \ idma_pkg::axi_options_t src; \ idma_pkg::axi_options_t dst; \ - idma_pkg::backend_options_t beo; \ - logic last; \ + idma_pkg::backend_options_t beo; \ + idma_pkg::compute_options_t compute; \ + logic last; \ } options_t; `define IDMA_TYPEDEF_ERR_PAYLOAD_T(err_payload_t, axi_addr_t) \ typedef struct packed { \ From d59e16d1f9e7555922c536a86b8689f20c03d839 Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 10 Jun 2026 16:03:38 +0200 Subject: [PATCH 04/13] midend: Add the transpose geometry expander idma_transpose_midend derives the NumDim=4 tiled walk (row / row-tile / col-tile) from the tensor shape and the bus StrbWidth, leaving the generic nd_midend to walk it; the geometry folds to shifts except one stride product. Guards the domain (StrbWidth >= 4, reserved mode, zero dims) and documents the tile-padded access contract; nd_midend asserts strides match the address width. --- src/midend/idma_nd_midend.sv | 7 ++ src/midend/idma_transpose_midend.sv | 123 ++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 src/midend/idma_transpose_midend.sv diff --git a/src/midend/idma_nd_midend.sv b/src/midend/idma_nd_midend.sv index 07267c97..fd3adaee 100644 --- a/src/midend/idma_nd_midend.sv +++ b/src/midend/idma_nd_midend.sv @@ -65,6 +65,13 @@ module idma_nd_midend #( /// How many bits are required to index the counters localparam int unsigned StrideSelWidth = $clog2(NumDim-1) + 'd1; +`ifndef SYNTHESIS + // strides are added with same-width arithmetic; narrower strides would not sign-extend + initial assert ($bits(nd_req_i.d_req[0].src_strides) == $bits(nd_req_i.burst_req.src_addr)) + else $fatal(1, "idma_nd_midend: stride width (%0d) != address width (%0d)", + $bits(nd_req_i.d_req[0].src_strides), $bits(nd_req_i.burst_req.src_addr)); +`endif + // The counter currently active (this is added to the address) logic [StrideSelWidth-1:0] stride_sel_d, stride_sel_q; diff --git a/src/midend/idma_transpose_midend.sv b/src/midend/idma_transpose_midend.sv new file mode 100644 index 00000000..9d79d551 --- /dev/null +++ b/src/midend/idma_transpose_midend.sv @@ -0,0 +1,123 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Author: Daniel Keller + +/// Transpose geometry expander. For a request carrying opt.compute = TRANSPOSE, +/// derives the NumDim=4 tiled ND walk (row / row-tile / col-tile) from the tensor +/// shape (M, N, element mode) and the bus StrbWidth, leaving the generic +/// idma_nd_midend to walk it. Non-transpose requests pass through untouched. +/// The walk reads the source up to the tile-padded bounds (ceil to NE in both +/// dims) and writes the full padded dst extent (writes strb-masked, reads not). +/// Combinational; outputs are quasi-static per request (multicycle-safe in STA). +module idma_transpose_midend #( + /// Number of ND dimensions (must be >= 4 to express the tiled walk) + parameter int unsigned NumDim = 32'd4, + /// Write data-path width in bytes (tile side NE = StrbWidth / element bytes) + parameter int unsigned StrbWidth = 32'd64, + /// Address type + parameter type addr_t = logic, + /// ND request type + parameter type idma_nd_req_t = logic +)( + input idma_nd_req_t nd_req_i, + input logic valid_i, + output logic ready_o, + output idma_nd_req_t nd_req_o, + output logic valid_o, + input logic ready_i +); + + localparam int unsigned Log2Strb = $clog2(StrbWidth); + localparam int unsigned LenW = $bits(nd_req_o.burst_req.length); + localparam int unsigned RepW = $bits(nd_req_o.d_req[0].reps); + localparam int unsigned ModeW = $bits(nd_req_o.burst_req.opt.compute.params.transpose.mode); + localparam int unsigned TensorW = + $bits(nd_req_o.burst_req.opt.compute.params.transpose.tensor_m); + localparam int unsigned AddrW = $bits(addr_t); + // working width: largest term (YT*N)< AddrW) ? ProdW : AddrW; + + assign valid_o = valid_i; + assign ready_o = ready_i; + + logic is_transpose; + assign is_transpose = nd_req_i.burst_req.opt.compute.enable & + (nd_req_i.burst_req.opt.compute.op == idma_pkg::COMPUTE_TRANSPOSE); + + // NE and E are powers of two: all geometry folds to shifts except the YT*N + // stride product. + always_comb begin : proc_expand + logic [ModeW-1:0] mode; + logic [TensorW-1:0] tm, tn; + logic signed [WorkW-1:0] m, n, log2ne, ne, yt, nt, nxe, mpe; + logic signed [WorkW-1:0] strb_c; // NE*E == StrbWidth (mode cancels) + + nd_req_o = nd_req_i; // passthrough + + if (is_transpose) begin + mode = nd_req_i.burst_req.opt.compute.params.transpose.mode; + tm = nd_req_i.burst_req.opt.compute.params.transpose.tensor_m; + tn = nd_req_i.burst_req.opt.compute.params.transpose.tensor_n; + // zero-extend bounded dims into the signed working width + m = $signed({{(WorkW-TensorW){1'b0}}, tm}); // M + n = $signed({{(WorkW-TensorW){1'b0}}, tn}); // N + log2ne = $signed(WorkW'(Log2Strb)) - $signed({{(WorkW-ModeW){1'b0}}, mode}); + ne = $signed(WorkW'(1)) <<< log2ne; // tile side (elements) + yt = (m + ne - 1) >>> log2ne; // ceil(M/NE) + nt = (n + ne - 1) >>> log2ne; // ceil(N/NE) + nxe = n <<< mode; // N*E (E = 1<= 4) else + $fatal(1, "idma_transpose_midend requires NumDim >= 4 (got %0d)", NumDim); + // mode 0..2 needs NE >= 1, i.e. log2(StrbWidth) >= 2 + initial assert (Log2Strb >= 2) else + $fatal(1, "idma_transpose_midend requires StrbWidth >= 4 (got %0d)", StrbWidth); + // reps must hold tile counts (<= 2^TensorW) and ne (<= StrbWidth); length StrbWidth. + initial assert (RepW >= TensorW && RepW > Log2Strb) else + $fatal(1, "idma_transpose_midend: reps field %0d b too narrow (need >= %0d)", + RepW, (TensorW > Log2Strb+1) ? TensorW : Log2Strb+1); + initial assert (LenW > Log2Strb) else + $fatal(1, "idma_transpose_midend: length field %0d b cannot hold StrbWidth", LenW); + // reserved mode 3 (EB=8) and zero-size tensors are out of contract + always_comb begin : check_domain + if (is_transpose) begin + assert (nd_req_i.burst_req.opt.compute.params.transpose.mode != 2'd3) else + $error("idma_transpose_midend: reserved element mode 3 (EB=8)"); + assert (nd_req_i.burst_req.opt.compute.params.transpose.tensor_m != '0 && + nd_req_i.burst_req.opt.compute.params.transpose.tensor_n != '0) else + $error("idma_transpose_midend: zero-size tensor (M or N == 0)"); + end + end +`endif + +endmodule From 4d80deb9aec4a6159577f7c89cb257794ca5974e Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 10 Jun 2026 16:03:38 +0200 Subject: [PATCH 05/13] backend: Dispatch on-the-fly compute at the transport write seam idma_otf_compute latches the per-transfer compute options and runtime-selects one op per transfer; the AXI write manager gains an external strobe mask and a strobe-independent beat-done so edge tiles drain. Compute support is decided at generation time: IDMA_VIDMA_IDS entries (variant[:ops][:fd|hd]) render the seam, the per-op ComputeEnable set and the transpose duplex into the listed variants only, non-compute variants are untouched. The write-side FIFOs grow by a tile to clear the legalizer in-flight bound and compute variants require NO_ERROR_HANDLING. --- idma.mk | 89 ++++++++++++++++- src/backend/idma_axi_write.sv | 10 +- src/backend/idma_otf_compute.sv | 100 ++++++++++++++++++++ src/backend/tpl/idma_backend.sv.tpl | 23 ++++- src/backend/tpl/idma_legalizer.sv.tpl | 14 ++- src/backend/tpl/idma_transport_layer.sv.tpl | 55 ++++++++++- src/db/idma_axi.yml | 7 +- src/db/idma_tilelink.yml | 3 +- util/gen_idma.py | 11 ++- util/mario/backend.py | 10 +- util/mario/legalizer.py | 3 +- util/mario/transport_layer.py | 8 +- util/mario/util.py | 20 ++++ 13 files changed, 328 insertions(+), 25 deletions(-) create mode 100644 src/backend/idma_otf_compute.sv diff --git a/idma.mk b/idma.mk index e76fab2a..6137ded3 100644 --- a/idma.mk +++ b/idma.mk @@ -16,6 +16,8 @@ VCS ?= vcs VERILATOR ?= verilator VLOGAN ?= vlogan VSIM ?= vsim +VLOG ?= vlog +VLIB ?= vlib # Provision a local uv venv when the generator deps are not already available. IDMA_VENV_PY := $(IDMA_ROOT)/.venv/bin/python @@ -51,6 +53,8 @@ IDMA_OCCAMY_IDS := \ rw_axi_rw_init_rw_obi IDMA_ADD_IDS ?= IDMA_BACKEND_IDS ?= $(IDMA_BASE_IDS) $(IDMA_OCCAMY_IDS) $(IDMA_ADD_IDS) +# Backend variants that host the on-the-fly compute dispatcher (single AXI write) +IDMA_VIDMA_IDS ?= rw_axi # generated frontends IDMA_BASE_FE_IDS := reg32_3d reg64_2d reg64_1d @@ -120,17 +124,17 @@ IDMA_RTL_FILES := $(IDMA_RTL_DIR)/idma_transport_layer \ IDMA_VSIM_DIR := $(IDMA_ROOT)/target/sim/vsim define idma_gen - $(PYTHON) $(IDMA_GEN) --entity $1 --tpl $2 --db $3 --ids $4 --fids $5 > $6 + $(PYTHON) $(IDMA_GEN) --entity $1 --tpl $2 --db $3 --ids $4 --fids $5 $(if $7,--compute-ids $7) > $6 endef $(IDMA_RTL_DIR)/idma_transport_layer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,transport,$(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl,$(IDMA_DB_FILES),$*,,$@) + $(call idma_gen,transport,$(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) $(IDMA_RTL_DIR)/idma_legalizer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,legalizer,$(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl,$(IDMA_DB_FILES),$*,,$@) + $(call idma_gen,legalizer,$(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) $(IDMA_RTL_DIR)/idma_backend_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_legalizer_%.sv $(IDMA_RTL_DIR)/idma_transport_layer_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,backend,$(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl,$(IDMA_DB_FILES),$*,,$@) + $(call idma_gen,backend,$(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) $(IDMA_RTL_DIR)/idma_backend_synth_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_backend_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend_synth.sv.tpl $(IDMA_DB_FILES) $(call idma_gen,synth_wrapper,$(IDMA_ROOT)/src/backend/tpl/idma_backend_synth.sv.tpl,$(IDMA_DB_FILES),$*,,$@) @@ -327,7 +331,81 @@ define idma_generate_vsim endef $(IDMA_VSIM_DIR)/compile.tcl: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) $(IDMA_WAVE_ALL) - $(call idma_generate_vsim, $@, -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster,../../..) + $(call idma_generate_vsim, $@, -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster -t split_rtl,../../..) + +.PHONY: idma_sim_tb_idma_rt_midend + +idma_sim_tb_idma_rt_midend: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc \ + tb_idma_rt_midend -do "run -all; quit" + +# Standalone self-checking transpose-engine regression (DPI-C golden, no backend deps). +# Run with the Questa SEPP wrapper, e.g.: +# make idma_sim_tb_idma_otf_transpose VSIM="questa-2023.4 vsim" VLOG="questa-2023.4 vlog" VLIB="questa-2023.4 vlib" +IDMA_OTF_TP_RTL := $(abspath $(IDMA_ROOT)/src/backend/idma_otf_transpose.sv) +IDMA_OTF_TP_TB := $(abspath $(IDMA_ROOT)/test/tb_idma_otf_transpose.sv) +IDMA_OTF_TP_DPI := $(abspath $(IDMA_ROOT)/test/idma_transpose_dpi.c) +IDMA_OTF_TP_DIR := $(abspath $(IDMA_VSIM_DIR))/otf_transpose + +.PHONY: idma_sim_tb_idma_otf_transpose +idma_sim_tb_idma_otf_transpose: + mkdir -p $(IDMA_OTF_TP_DIR) + cd $(IDMA_OTF_TP_DIR); $(VLIB) work + cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv $(IDMA_OTF_TP_DPI) + cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv -svinputport=compat -timescale "1ns/1fs" $(IDMA_OTF_TP_RTL) $(IDMA_OTF_TP_TB) + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=13 -gN=19 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=7 -gN=5 -gEB=2 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=5 -gN=3 -gEB=4 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gM=130 -gN=70 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=8 -gM=13 -gN=19 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=8 -gM=7 -gN=5 -gEB=2 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=64 -gM=130 -gN=70 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" + +# Multi-tile transpose via the ND midend (transposed strides) -> rw_axi backend +# (engine spliced at the write seam) -> axi_sim_mem. Covers aligned and edge +# (M or N not a multiple of NE) geometries for int8/fp16/fp32. Needs the +# split_rtl flow (per-variant routing). Run with the Questa SEPP wrapper: +# make idma_sim_tb_idma_transpose_nd VSIM="questa-2023.4 vsim" +.PHONY: idma_sim_tb_idma_transpose_nd +idma_sim_tb_idma_transpose_nd: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + # ── aligned (regression: M,N multiples of NE) ── + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=16 -gN=16 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=16 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=32 -gN=24 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=2 tb_idma_transpose_nd -do "run -all; quit" + # ── edge: partial output cols only (M%NE!=0, N%NE==0; within-beat wstrb) ── + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + # ── edge: partial output rows only (N%NE!=0; zero-strobe drain beats) ── + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + # ── edge: both partial (int8) ── + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=7 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=10 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + # ── edge: fp16 (EB=2) and fp32 (EB=4) ── + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=5 -gEB=2 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=9 -gN=5 -gEB=4 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=13 -gN=19 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + +# Back-to-back regressions: the ND midend must reload each new transfer's base +# address (it does, for a protocol-compliant producer that drops nd_req_valid on +# accept). tb_idma_nd_midend_b2b checks the midend's burst-address sequence under +# backpressure; tb_idma_transpose_b2b checks two end-to-end transposes to distinct +# destinations. Run with the Questa SEPP wrapper. +.PHONY: idma_sim_tb_idma_nd_midend_b2b +idma_sim_tb_idma_nd_midend_b2b: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc tb_idma_nd_midend_b2b -do "run -all; quit" + +.PHONY: idma_sim_tb_idma_transpose_b2b +idma_sim_tb_idma_transpose_b2b: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=8 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=13 -gN=19 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=5 -gEB=2 tb_idma_transpose_b2b -do "run -all; quit" .PHONY: idma_sim_tb_idma_rt_midend @@ -336,6 +414,7 @@ idma_sim_tb_idma_rt_midend: $(IDMA_VSIM_DIR)/compile.tcl cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc tb_idma_rt_midend -do "run -all; quit" idma_sim_clean: + rm -rf $(IDMA_OTF_TP_DIR) rm -rf $(IDMA_VSIM_DIR)/compile.tcl rm -rf $(IDMA_VSIM_DIR)/work rm -f $(IDMA_VSIM_DIR)/dma_trace_* diff --git a/src/backend/idma_axi_write.sv b/src/backend/idma_axi_write.sv index ee27349d..faa309db 100644 --- a/src/backend/idma_axi_write.sv +++ b/src/backend/idma_axi_write.sv @@ -73,7 +73,11 @@ module idma_axi_write #( /// Valid from buffer input strb_t buffer_out_valid_i, /// Ready to buffer - output strb_t buffer_out_ready_o + output strb_t buffer_out_ready_o, + /// External write-strobe mask (ANDed into wstrb); tie to '1 when unused + input strb_t mask_ext_i, + /// Pulses when a write beat is accepted on the bus (strobe-independent) + output logic w_beat_done_o ); // offsets needed for masks to empty buffer strb_t w_first_mask; @@ -141,6 +145,8 @@ module idma_axi_write #( if (w_dp_req_i.tailer != '0 & last_w) begin mask_out = mask_out & w_last_mask; end + // external mask: some bytes may be masked by an OTF engine + mask_out = mask_out & mask_ext_i; end @@ -170,6 +176,8 @@ module idma_axi_write #( // write happening: both the bus (w_ready) and the buffer (ready_to_write) is high assign write_happening = ready_to_write & write_rsp_i.w_ready; + assign w_beat_done_o = write_happening; + // the main buffer is conditionally to the write mask popped assign buffer_out_ready_o = write_happening ? mask_out : '0; diff --git a/src/backend/idma_otf_compute.sv b/src/backend/idma_otf_compute.sv new file mode 100644 index 00000000..88bfff32 --- /dev/null +++ b/src/backend/idma_otf_compute.sv @@ -0,0 +1,100 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +/// On-the-fly compute dispatcher at the transport write seam: latches the +/// per-transfer compute options and dispatches one op per transfer to its sub-unit. +module idma_otf_compute #( + /// Byte lanes per beat (= DataWidth/8) + parameter int unsigned StrbWidth = 32'd8, + /// Compile-time per-op feature enables (value rendered by the generator) + parameter idma_pkg::compute_enable_t ComputeEnable = '0, + /// Transpose engine duplex (1: two banks full rate, 0: one bank half area) + parameter bit TransposeFullDuplex = 1'b1 +) ( + input logic clk_i, + input logic rst_ni, + + /// Per-transfer compute config; valid only while `cfg_valid_i` + input idma_pkg::compute_options_t compute_i, + input logic cfg_valid_i, + /// A supported compute op is armed for this transfer + output logic active_o, + + /// Input beat stream (from the dataflow buffer) + input logic [StrbWidth-1:0][7:0] data_i, + input logic valid_i, + output logic in_ready_o, + + /// Output beat stream (computed) with per-byte strobe for edge masking + output logic [StrbWidth-1:0][7:0] data_o, + output logic [StrbWidth-1:0] strb_o, + output logic valid_o, + input logic ready_i +); + + // config latch with first-beat bypass + idma_pkg::compute_options_t latched_q, eff_compute; + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) latched_q <= '0; + else if (cfg_valid_i) latched_q <= compute_i; + end + assign eff_compute = cfg_valid_i ? compute_i : latched_q; + + // per-op select + logic sel_transpose; + assign sel_transpose = eff_compute.enable & + (eff_compute.op == idma_pkg::COMPUTE_TRANSPOSE) & ComputeEnable.transpose; + + assign active_o = sel_transpose; + + // transpose sub-unit + logic [StrbWidth-1:0][7:0] tp_data; + logic [StrbWidth-1:0] tp_strb; + logic tp_valid, tp_in_ready; + + if (ComputeEnable.transpose) begin : gen_transpose + idma_otf_transpose #( + .StrbWidth ( StrbWidth ), + .DimWidth ( idma_pkg::TransposeDimWidth ), + .FullDuplex ( TransposeFullDuplex ) + ) i_idma_otf_transpose ( + .clk_i, + .rst_ni, + .clear_i ( ~sel_transpose ), + .transp_mode_i ( eff_compute.params.transpose.mode ), + .tensor_size_m_i ( eff_compute.params.transpose.tensor_m ), + .tensor_size_n_i ( eff_compute.params.transpose.tensor_n ), + .data_i ( data_i ), + .valid_i ( valid_i & sel_transpose ), + .ready_o ( tp_in_ready ), + .data_o ( tp_data ), + .strb_o ( tp_strb ), + .valid_o ( tp_valid ), + .ready_i ( ready_i & sel_transpose ) + ); + end else begin : gen_no_transpose + assign tp_data = '0; assign tp_strb = '0; assign tp_valid = 1'b0; assign tp_in_ready = 1'b0; + end + + // output dispatch + always_comb begin + data_o = '0; + strb_o = '0; + valid_o = 1'b0; + in_ready_o = 1'b0; + unique case (1'b1) + sel_transpose: begin + data_o = tp_data; + strb_o = tp_strb; + valid_o = tp_valid; + in_ready_o = tp_in_ready; + end + default: ; + endcase + end + +endmodule : idma_otf_compute diff --git a/src/backend/tpl/idma_backend.sv.tpl b/src/backend/tpl/idma_backend.sv.tpl index 134c33e6..0459d567 100644 --- a/src/backend/tpl/idma_backend.sv.tpl +++ b/src/backend/tpl/idma_backend.sv.tpl @@ -172,9 +172,23 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, output idma_busy_t busy_o ); + /// Extra write-descriptor slots covering the compute (transpose) tile-fill latency + localparam int unsigned ComputeFifoDepth = ${"StrbWidth" if enable_compute else "32'd0"}; +% if enable_compute: + + /// Per-op compute set baked into this variant (frontends may cross-check) + localparam idma_pkg::compute_enable_t ComputeEnable = + '{${', '.join("%s: 1'b1" % op for op in compute_ops)}}; +`ifndef SYNTHESIS + // no engine flush on abort: compute is incompatible with error handling + initial assert (ErrorCap == idma_pkg::NO_ERROR_HANDLING) else + $fatal(1, "compute requires ErrorCap == NO_ERROR_HANDLING"); +`endif +% endif + /// The localparam MetaFifoDepth holds the maximum number of transfers that can be /// in-flight under any circumstances. - localparam int unsigned MetaFifoDepth = BufferDepth + NumAxInFlight + MemSysDepth; + localparam int unsigned MetaFifoDepth = BufferDepth + NumAxInFlight + MemSysDepth + ComputeFifoDepth; /// Address type typedef logic [AddrWidth-1:0] addr_t; @@ -235,6 +249,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, offset_t shift; axi_pkg::len_t num_beats; logic is_single; + idma_pkg::compute_options_t compute; } w_dp_req_t; /// The datapath write response type provides feedback from the write part of the datapath: @@ -298,6 +313,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, idma_pkg::axi_options_t src_axi_opt; idma_pkg::axi_options_t dst_axi_opt; logic super_last; + idma_pkg::compute_options_t compute; } idma_mut_tf_opt_t; /// The mutable transfer type holds important information that is mutated by the @@ -494,7 +510,8 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, tailer: OffsetWidth'(idma_req_i.length + idma_req_i.dst_addr[OffsetWidth-1:0]), shift: OffsetWidth'(- idma_req_i.dst_addr[OffsetWidth-1:0]), num_beats: len, - is_single: len == '0 + is_single: len == '0, + compute: idma_req_i.opt.compute }; // if the legalizer is bypassed; every burst is the last of the 1D transfer @@ -621,7 +638,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, ); stream_fifo_optimal_wrap #( - .Depth ( NumAxInFlight ), + .Depth ( NumAxInFlight + ComputeFifoDepth ), .type_t ( w_dp_req_t ), .PrintInfo ( PrintFifoInfo ) ) i_w_dp_req ( diff --git a/src/backend/tpl/idma_legalizer.sv.tpl b/src/backend/tpl/idma_legalizer.sv.tpl index 9301602c..9b6090b6 100644 --- a/src/backend/tpl/idma_legalizer.sv.tpl +++ b/src/backend/tpl/idma_legalizer.sv.tpl @@ -484,8 +484,13 @@ w_num_bytes_to_pb = w_page_num_bytes_to_pb; dst_head: req_i.opt.dst_head, read_shift: '0, write_shift: '0, +% if enable_compute: + decouple_rw: req_i.opt.beo.decouple_rw | req_i.opt.compute.enable, + decouple_aw: req_i.opt.beo.decouple_aw | req_i.opt.compute.enable, +% else: decouple_rw: req_i.opt.beo.decouple_rw, decouple_aw: req_i.opt.beo.decouple_aw, +% endif src_max_llen: req_i.opt.beo.src_max_llen, dst_max_llen: req_i.opt.beo.dst_max_llen, src_reduce_len: req_i.opt.beo.src_reduce_len, @@ -493,7 +498,8 @@ w_num_bytes_to_pb = w_page_num_bytes_to_pb; axi_id: req_i.opt.axi_id, src_axi_opt: req_i.opt.src, dst_axi_opt: req_i.opt.dst, - super_last: req_i.opt.last + super_last: req_i.opt.last, + compute: req_i.opt.compute }; // determine shift amount if (CombinedShifter) begin @@ -557,7 +563,8 @@ ${database[used_write_protocols[0]]['legalizer_write_data_path']} tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: 1'b1 + is_single: 1'b1, + compute: opt_tf_q.compute }; % endif end @@ -592,7 +599,8 @@ ${database[protocol]['legalizer_write_data_path']} tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: 1'b1 + is_single: 1'b1, + compute: opt_tf_q.compute }; endcase end diff --git a/src/backend/tpl/idma_transport_layer.sv.tpl b/src/backend/tpl/idma_transport_layer.sv.tpl index 59bcb029..7f27de83 100644 --- a/src/backend/tpl/idma_transport_layer.sv.tpl +++ b/src/backend/tpl/idma_transport_layer.sv.tpl @@ -229,6 +229,10 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, byte_t [2*StrbWidth-1:0] buffer_out_tmp; byte_t [StrbWidth-1:0] buffer_out; byte_t [StrbWidth-1:0] buffer_out_shifted; + // compute write seam (passthrough when compute is off) + byte_t [StrbWidth-1:0] wr_data; + strb_t wr_valid, wr_strb, mask_ext_shifted, dataflow_ready_in; + logic w_beat_done; % if not one_read_port: // Read multiplexed signals @@ -374,16 +378,61 @@ ${rendered_read_ports[read_port]} .ready_o ( buffer_in_ready ), .data_o ( buffer_out ), .valid_o ( buffer_out_valid ), - .ready_i ( buffer_out_ready_shifted ) + .ready_i ( dataflow_ready_in ) ); + //-------------------------------------- + // On-the-fly compute (write seam) + //-------------------------------------- + +% if enable_compute: + logic cmp_active; + logic cmp_in_ready, cmp_out_valid; + byte_t [StrbWidth-1:0] cmp_data_o; + strb_t cmp_strb_o; + + // beats retire on w_beat_done (strobe-independent) + idma_otf_compute #( + .StrbWidth ( StrbWidth ), + .ComputeEnable ( '{${', '.join("%s: 1'b1" % op for op in compute_ops)}} ), + .TransposeFullDuplex ( 1'b${'1' if compute_full_duplex else '0'} ) + ) i_idma_otf_compute ( + .clk_i, + .rst_ni, + .compute_i ( w_dp_req_i.compute ), + .cfg_valid_i ( w_dp_valid_i ), + .active_o ( cmp_active ), + .data_i ( buffer_out ), + .valid_i ( &buffer_out_valid ), + .in_ready_o ( cmp_in_ready ), + .data_o ( cmp_data_o ), + .strb_o ( cmp_strb_o ), + .valid_o ( cmp_out_valid ), + .ready_i ( w_beat_done ) + ); + + // whole-beat valid; edge masking carried on wr_strb + assign wr_data = cmp_active ? cmp_data_o : buffer_out; + assign wr_valid = cmp_active ? {StrbWidth{cmp_out_valid}} : buffer_out_valid; + assign wr_strb = cmp_active ? cmp_strb_o : '1; + // pop the buffer only on a compute input handshake + assign dataflow_ready_in = cmp_active ? {StrbWidth{(&buffer_out_valid) & cmp_in_ready}} + : buffer_out_ready_shifted; +% else: + assign wr_data = buffer_out; + assign wr_valid = buffer_out_valid; + assign wr_strb = '1; + assign dataflow_ready_in = buffer_out_ready_shifted; +% endif + //-------------------------------------- // Write Barrel shifter //-------------------------------------- - assign buffer_out_tmp = {buffer_out, buffer_out} >> (w_dp_req_i.shift*8); + assign buffer_out_tmp = {wr_data, wr_data} >> (w_dp_req_i.shift*8); assign buffer_out_shifted = buffer_out_tmp[$bits(buffer_out_shifted)/8-1:0]; - assign buffer_out_valid_shifted = strb_t'({buffer_out_valid, buffer_out_valid} >> w_dp_req_i.shift); + assign buffer_out_valid_shifted = strb_t'({wr_valid, wr_valid} >> w_dp_req_i.shift); + assign mask_ext_shifted = strb_t'({wr_strb, wr_strb} >> w_dp_req_i.shift); assign buffer_out_ready_shifted = strb_t'({buffer_out_ready, buffer_out_ready} >> - w_dp_req_i.shift); % if not one_write_port: diff --git a/src/db/idma_axi.yml b/src/db/idma_axi.yml index 267ce4b2..1baa8650 100644 --- a/src/db/idma_axi.yml +++ b/src/db/idma_axi.yml @@ -67,7 +67,8 @@ legalizer_write_data_path: | tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: w_req_o.aw_req.axi.aw_chan.len, - is_single: w_req_o.aw_req.axi.aw_chan.len == '0 + is_single: w_req_o.aw_req.axi.aw_chan.len == '0, + compute: opt_tf_q.compute }; read_template: | idma_axi_read #( @@ -128,7 +129,9 @@ write_template: | .write_rsp_i ( ${write_response} ), .buffer_out_i ( buffer_out_shifted ), .buffer_out_valid_i ( buffer_out_valid_shifted ), - .buffer_out_ready_o ( ${buffer_out_ready} ) + .buffer_out_ready_o ( ${buffer_out_ready} ), + .mask_ext_i ( mask_ext_shifted ), + .w_beat_done_o ( w_beat_done ) ); synth_wrapper_ports_write: | output id_t axi_aw_id_o, diff --git a/src/db/idma_tilelink.yml b/src/db/idma_tilelink.yml index 15e97eb0..c737d806 100644 --- a/src/db/idma_tilelink.yml +++ b/src/db/idma_tilelink.yml @@ -85,7 +85,8 @@ legalizer_write_data_path: | tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: w_num_bytes <= StrbWidth + is_single: w_num_bytes <= StrbWidth, + compute: opt_tf_q.compute }; read_template: | idma_tilelink_read #( diff --git a/util/gen_idma.py b/util/gen_idma.py index 2aff908e..537b9adb 100644 --- a/util/gen_idma.py +++ b/util/gen_idma.py @@ -12,7 +12,7 @@ import argparse import sys -from mario.util import prepare_ids, prepare_fids +from mario.util import prepare_ids, prepare_compute_ids, prepare_fids from mario.database import read_database from mario.transport_layer import render_transport_layer from mario.legalizer import render_legalizer @@ -44,6 +44,8 @@ def main(): parser.add_argument('--entity', choices=sorted(GENABLE_ENTITIES), dest='entity', required=True, help='The entity to generate from a given configuration.') parser.add_argument('--ids', dest='ids', nargs='*', help='configuration IDs') + parser.add_argument('--compute-ids', dest='compute_ids', nargs='*', default=[], + help='configuration IDs with on-the-fly compute enabled (IDMA_VIDMA_IDS)') parser.add_argument('--fids', dest='fids', nargs='*', help='frontend IDs') parser.add_argument('--db', dest='db', nargs='*', help='Database files') parser.add_argument('--tpl', dest='tpl', required=True, help='Template file') @@ -51,16 +53,17 @@ def main(): # prepare database and ids protocol_ids = prepare_ids(args.ids) + compute_cfg = prepare_compute_ids(args.compute_ids) frontend_ids = prepare_fids(args.fids) protocol_db = read_database(args.db) # decide what to render if args.entity == 'transport': - print(render_transport_layer(protocol_ids, protocol_db, args.tpl)) + print(render_transport_layer(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'legalizer': - print(render_legalizer(protocol_ids, protocol_db, args.tpl)) + print(render_legalizer(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'backend': - print(render_backend(protocol_ids, protocol_db, args.tpl)) + print(render_backend(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'vsim_wave': print(render_vsim_wave(protocol_ids, protocol_db, args.tpl)) elif args.entity == 'synth_wrapper': diff --git a/util/mario/backend.py b/util/mario/backend.py index 3a6545bd..daa3e934 100644 --- a/util/mario/backend.py +++ b/util/mario/backend.py @@ -12,7 +12,7 @@ from mario.util import eval_key, prot_key -def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_backend(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None) -> str: """Generate backend""" backend_rendered = '' @@ -42,6 +42,12 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: srp = len(used_read_prots) == 1 and not any_mh_r swp = len(used_write_prots) == 1 and not any_mh_w + # on-the-fly compute is hosted at the (single) AXI write seam + enable_compute = prot_id in (compute_cfg or {}) + if enable_compute and not (swp and used_write_prots[0] == 'axi'): + raise ValueError( + f'compute (IDMA_VIDMA_IDS) requires a single AXI write port: {prot_id}') + # create context context = { 'name_uniqueifier': prot_id, @@ -51,6 +57,8 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: 'used_protocols': prot_ids[prot_id]['used'], 'one_read_port': srp, 'one_write_port': swp, + 'enable_compute': enable_compute, + 'compute_ops': compute_cfg[prot_id]['ops'] if enable_compute else [], 'used_non_bursting_write_protocols': prot_key(used_write_prots, 'bursts', 'not_supported', db), 'combined_aw_and_w': diff --git a/util/mario/legalizer.py b/util/mario/legalizer.py index 1724c00e..51757771 100644 --- a/util/mario/legalizer.py +++ b/util/mario/legalizer.py @@ -21,7 +21,7 @@ def prot_force_decouple(used_prots: list, db: dict) -> list: return res -def render_legalizer(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_legalizer(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None) -> str: """Generate legalizer""" legalizer_rendered = '' @@ -71,6 +71,7 @@ def render_legalizer(prot_ids: dict, db: dict, tpl_file: str) -> str: 'used_protocols': prot_ids[prot_id]['used'], 'one_read_port': srp, 'one_write_port': swp, + 'enable_compute': prot_id in (compute_cfg or {}), 'no_read_bursting': not has_read_bursting, 'has_page_read_bursting': diff --git a/util/mario/transport_layer.py b/util/mario/transport_layer.py index 1c6850af..41c5df6f 100644 --- a/util/mario/transport_layer.py +++ b/util/mario/transport_layer.py @@ -253,7 +253,8 @@ def render_write_mgr_inst(prot_id: str, prot_ids: dict, db: dict) -> dict: return res -def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None + ) -> str: """Generate Transport Layer""" transport_rendered = '' @@ -286,6 +287,11 @@ def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str) -> str: 'one_write_port': len(prot_ids[prot_id]['aw']) == 1 and not any_mh['aw'], 'mh_format': mh_format, 'any_mh': any_mh, + 'enable_compute': prot_id in (compute_cfg or {}), + 'compute_ops': + compute_cfg[prot_id]['ops'] if prot_id in (compute_cfg or {}) else [], + 'compute_full_duplex': + compute_cfg[prot_id]['full_duplex'] if prot_id in (compute_cfg or {}) else True, 'rendered_read_ports': render_read_mgr_inst(prot_id, prot_ids, db), 'rendered_write_ports': render_write_mgr_inst(prot_id, prot_ids, db) } diff --git a/util/mario/util.py b/util/mario/util.py index 8cc2d492..bd655d91 100644 --- a/util/mario/util.py +++ b/util/mario/util.py @@ -164,3 +164,23 @@ def prepare_fids(fe_strs: list) -> dict: res[f'reg{reg[0]}_{reg[1]}d'] = reg return res + + +def prepare_compute_ids(compute_id_strs: list) -> dict: + """Parses compute configuration IDs: [:[,...]][:fd|hd]""" + res = {} + for cid_str in (compute_id_strs or []): + parts = cid_str.split(':') + ops = ['transpose'] + full_duplex = True + for part in parts[1:]: + if part in ('fd', 'hd'): + full_duplex = part == 'fd' + else: + ops = part.split(',') + for op in ops: + if op not in ('transpose',): + print(f'[MARIO] {op} is a non-supported compute op in {cid_str}', file=sys.stderr) + sys.exit(1) + res[parts[0]] = {'ops': ops, 'full_duplex': full_duplex} + return res From 20e1e5210dc44ea28abc2564716c4c7235cd9b8e Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 10 Jun 2026 16:03:38 +0200 Subject: [PATCH 06/13] test: Add the transpose regressions Multi-tile aligned and edge transposes through the rw_axi backend, back-to-back geometry-leak checks, an nd_midend burst-address regression, a field-for-field midend unit test and launch_tf transpose options; the engine regression runs in both duplex modes. --- Bender.yml | 62 +++++- jobs/backend_rw_axi/transpose_none.txt | 0 test/idma_test.sv | 12 +- test/midend/tb_idma_nd_midend_b2b.sv | 146 +++++++++++++ test/midend/tb_idma_transpose_midend.sv | 102 +++++++++ test/tb_idma_transpose_b2b.sv | 214 +++++++++++++++++++ test/tb_idma_transpose_nd.sv | 263 ++++++++++++++++++++++++ 7 files changed, 796 insertions(+), 3 deletions(-) create mode 100644 jobs/backend_rw_axi/transpose_none.txt create mode 100644 test/midend/tb_idma_nd_midend_b2b.sv create mode 100644 test/midend/tb_idma_transpose_midend.sv create mode 100644 test/tb_idma_transpose_b2b.sv create mode 100644 test/tb_idma_transpose_nd.sv diff --git a/Bender.yml b/Bender.yml index 6f16b87e..33d662f5 100644 --- a/Bender.yml +++ b/Bender.yml @@ -44,6 +44,8 @@ sources: - src/backend/idma_axis_write.sv - src/backend/idma_channel_coupler.sv - src/backend/idma_dataflow_element.sv + - src/backend/idma_otf_transpose.sv + - src/backend/idma_otf_compute.sv - src/backend/idma_error_handler.sv - src/backend/idma_init_read.sv - src/backend/idma_init_write.sv @@ -54,11 +56,58 @@ sources: - src/backend/idma_tilelink_read.sv - src/backend/idma_tilelink_write.sv - # Generated content - - target: rtl + # Generated content (bundled single-file; default flow) + - target: all(rtl, not(split_rtl)) files: - target/rtl/idma_generated.sv + # Generated content (per-variant files; opt-in via `-t split_rtl`). + # Used for the transpose-engine prototype where the rw_axi backend/transport/ + # legalizer are hand-edited in place before being ported back to templates. + # Mutually exclusive with idma_generated.sv to avoid duplicate module defs. + - target: all(rtl, split_rtl) + files: + - target/rtl/idma_transport_layer_rw_axi.sv + - target/rtl/idma_transport_layer_r_obi_w_axi.sv + - target/rtl/idma_transport_layer_r_axi_w_obi.sv + - target/rtl/idma_transport_layer_rw_axi_rw_axis.sv + - target/rtl/idma_transport_layer_rw_obi.sv + - target/rtl/idma_transport_layer_r_obi_rw_init_w_axi.sv + - target/rtl/idma_transport_layer_r_axi_rw_init_rw_obi.sv + - target/rtl/idma_legalizer_rw_axi.sv + - target/rtl/idma_legalizer_r_obi_w_axi.sv + - target/rtl/idma_legalizer_r_axi_w_obi.sv + - target/rtl/idma_legalizer_rw_axi_rw_axis.sv + - target/rtl/idma_legalizer_rw_obi.sv + - target/rtl/idma_legalizer_r_obi_rw_init_w_axi.sv + - target/rtl/idma_legalizer_r_axi_rw_init_rw_obi.sv + - target/rtl/idma_backend_rw_axi.sv + - target/rtl/idma_backend_r_obi_w_axi.sv + - target/rtl/idma_backend_r_axi_w_obi.sv + - target/rtl/idma_backend_rw_axi_rw_axis.sv + - target/rtl/idma_backend_rw_obi.sv + - target/rtl/idma_backend_r_obi_rw_init_w_axi.sv + - target/rtl/idma_backend_r_axi_rw_init_rw_obi.sv + - target/rtl/idma_backend_synth_rw_axi.sv + - target/rtl/idma_backend_synth_r_obi_w_axi.sv + - target/rtl/idma_backend_synth_r_axi_w_obi.sv + - target/rtl/idma_backend_synth_rw_axi_rw_axis.sv + - target/rtl/idma_backend_synth_rw_obi.sv + - target/rtl/idma_backend_synth_r_obi_rw_init_w_axi.sv + - target/rtl/idma_backend_synth_r_axi_rw_init_rw_obi.sv + - target/rtl/idma_desc64_reg_pkg.sv + - target/rtl/idma_reg32_3d_reg_pkg.sv + - target/rtl/idma_reg64_2d_reg_pkg.sv + - target/rtl/idma_reg64_1d_reg_pkg.sv + - target/rtl/idma_desc64_reg_top.sv + - target/rtl/idma_reg32_3d_reg_top.sv + - target/rtl/idma_reg64_2d_reg_top.sv + - target/rtl/idma_reg64_1d_reg_top.sv + - target/rtl/idma_desc64_top.sv + - target/rtl/idma_reg32_3d_top.sv + - target/rtl/idma_reg64_2d_top.sv + - target/rtl/idma_reg64_1d_top.sv + # Midends - target: rtl files: @@ -67,6 +116,7 @@ sources: - src/midend/idma_mp_split_midend.sv - src/midend/idma_nd_midend.sv - src/midend/idma_rt_midend.sv + - src/midend/idma_transpose_midend.sv # RISC-V opcode package for ooc use of inst64 - target: all(rtl,snitch_cluster) @@ -95,6 +145,7 @@ sources: # Level 2 - src/frontend/desc64/idma_desc64_top.sv + # Synthesis wrappers - target: synth files: @@ -120,7 +171,9 @@ sources: - test/future/idma_tb_per2axi.sv - test/future/TLToAXI4.v - test/midend/tb_idma_nd_midend.sv + - test/midend/tb_idma_nd_midend_b2b.sv - test/midend/tb_idma_rt_midend.sv + - test/midend/tb_idma_transpose_midend.sv # Level 2 - test/future/idma_obi2axi_bridge.sv - test/future/idma_tilelink2axi_bridge.sv @@ -129,6 +182,11 @@ sources: - target: idma_test files: - target/rtl/tb_idma_generated.sv + # Multi-tile transpose path: ND-midend (transposed strides) -> backend + - target: all(idma_test, split_rtl) + files: + - test/tb_idma_transpose_nd.sv + - test/tb_idma_transpose_b2b.sv # Multi-head directed backend testbenches - target: multihead diff --git a/jobs/backend_rw_axi/transpose_none.txt b/jobs/backend_rw_axi/transpose_none.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/idma_test.sv b/test/idma_test.sv index f4115a3e..931a580d 100644 --- a/test/idma_test.sv +++ b/test/idma_test.sv @@ -689,7 +689,11 @@ package idma_test; input logic dst_reduce_len, input id_t id, input idma_pkg::multihead_t src_head = '0, - input idma_pkg::multihead_t dst_head = '0 + input idma_pkg::multihead_t dst_head = '0, + input logic transpose_en = 1'b0, + input logic [1:0] transp_mode = '0, + input logic [11:0] tensor_m = '0, + input logic [11:0] tensor_n = '0 ); idma.req.length <= #TA length; idma.req.src_addr <= #TA src_addr; @@ -705,6 +709,12 @@ package idma_test; idma.req.opt.beo.dst_max_llen <= #TA dst_max_llen; idma.req.opt.beo.src_reduce_len <= #TA src_reduce_len; idma.req.opt.beo.dst_reduce_len <= #TA dst_reduce_len; + idma.req.opt.compute.enable <= #TA transpose_en; + idma.req.opt.compute.op <= #TA transpose_en ? idma_pkg::COMPUTE_TRANSPOSE + : idma_pkg::COMPUTE_NONE; + idma.req.opt.compute.params.transpose.mode <= #TA transp_mode; + idma.req.opt.compute.params.transpose.tensor_m <= #TA tensor_m; + idma.req.opt.compute.params.transpose.tensor_n <= #TA tensor_n; idma.req_valid <= #TA 1; cycle_start(); while (idma.req_ready != 1) begin cycle_end(); cycle_start(); end diff --git a/test/midend/tb_idma_nd_midend_b2b.sv b/test/midend/tb_idma_nd_midend_b2b.sv new file mode 100644 index 00000000..aaba907f --- /dev/null +++ b/test/midend/tb_idma_nd_midend_b2b.sv @@ -0,0 +1,146 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Authors: +// - Daniel Keller +// +// Back-to-back ND regression for idma_nd_midend. Drives the midend directly and +// golden-checks the burst_req address sequence: two back-to-back transfers (no +// gap) plus one after an idle gap must each walk from their own base. Catches a +// stale base-address reuse across transfers. + +`include "idma/typedef.svh" + +module tb_idma_nd_midend_b2b; + + localparam time TCK = 10ns; + localparam int unsigned AddrWidth = 32; + localparam int unsigned NumDim = 4; // 1D burst + 3 strided dims + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [31:0] tf_len_t; + typedef logic [11:0] id_t; + typedef logic [31:0] reps_t; + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + // ── Program (shared by all three transfers; only the base addresses differ) ── + localparam int unsigned R0 = 3, R1 = 2, R2 = 2; + localparam int unsigned NB = R0 * R1 * R2; // bursts per transfer + localparam addr_t SS0 = 'h10, DS0 = 'h100; + localparam addr_t SS1 = 'h40, DS1 = 'h400; + localparam addr_t SS2 = 'h1000, DS2 = 'h4000; + localparam addr_t S1 = 'h0000_1000, D1 = 'h0001_0000; // transfer 1 base + localparam addr_t S2 = 'h0000_2000, D2 = 'h0002_0000; // transfer 2 base (back-to-back) + localparam addr_t S3 = 'h0000_3000, D3 = 'h0003_0000; // transfer 3 base (after idle gap) + + logic clk, rst_n; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + idma_req_t burst_req; logic burst_req_valid, burst_req_ready; + idma_rsp_t burst_rsp; logic burst_rsp_valid, burst_rsp_ready; + logic busy; + + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_dut ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(burst_req), .burst_req_valid_o(burst_req_valid), .burst_req_ready_i(burst_req_ready), + .burst_rsp_i(burst_rsp), .burst_rsp_valid_i(burst_rsp_valid), .burst_rsp_ready_o(burst_rsp_ready), + .busy_o(busy) + ); + + // Backpressure on burst_req_ready is essential: during a stall stride_sel_q + // collapses toward 0, which is what can defeat the base reload. ready always-1 hides it. + logic [2:0] bp_lfsr; + always @(posedge clk or negedge rst_n) + if (!rst_n) bp_lfsr <= 3'b101; + else bp_lfsr <= {bp_lfsr[1:0], bp_lfsr[2] ^ bp_lfsr[1]}; + assign burst_req_ready = bp_lfsr[0]; // stalls ~half the cycles, incl. boundaries + assign burst_rsp = '0; + assign burst_rsp_valid = 1'b0; + assign nd_rsp_ready = 1'b1; + + // ── Capture every issued burst's src/dst address in order ── + addr_t cap_src [$]; + addr_t cap_dst [$]; + always @(posedge clk) if (rst_n && burst_req_valid && burst_req_ready) begin + cap_src.push_back(burst_req.src_addr); + cap_dst.push_back(burst_req.dst_addr); + end + + // build a NumDim=4 ND program with a given base + function automatic idma_nd_req_t mk_req(input addr_t s, input addr_t d); + idma_nd_req_t r = '0; + r.burst_req.length = tf_len_t'('h8); + r.burst_req.src_addr = s; + r.burst_req.dst_addr = d; + r.burst_req.opt.src_protocol = idma_pkg::AXI; + r.burst_req.opt.dst_protocol = idma_pkg::AXI; + r.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + r.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + r.d_req[0].reps = reps_t'(R0); r.d_req[0].src_strides = SS0; r.d_req[0].dst_strides = DS0; + r.d_req[1].reps = reps_t'(R1); r.d_req[1].src_strides = SS1; r.d_req[1].dst_strides = DS1; + r.d_req[2].reps = reps_t'(R2); r.d_req[2].src_strides = SS2; r.d_req[2].dst_strides = DS2; + return r; + endfunction + + initial begin + automatic int unsigned errs = 0; + nd_req = '0; nd_req_valid = 1'b0; + @(posedge rst_n); + repeat (3) @(posedge clk); + + // ── transfer 1 ── + nd_req = mk_req(S1, D1); nd_req_valid = 1'b1; + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + // ── transfer 2 : BACK-TO-BACK (keep valid high, swap payload the cycle after accept) ── + nd_req = mk_req(S2, D2); + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + nd_req_valid = 1'b0; + nd_req = '0; + // ── idle gap ── + repeat (5) @(posedge clk); + // ── transfer 3 : after the gap ── + nd_req = mk_req(S3, D3); nd_req_valid = 1'b1; + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + nd_req_valid = 1'b0; + repeat (3) @(posedge clk); + + // ── checks ── + if (cap_src.size() != 3*NB) + begin errs++; $display("[B2B] burst count %0d != %0d", cap_src.size(), 3*NB); end + else begin + // first burst of each transfer must equal its OWN base (reload happened) + if (cap_src[0] !== S1 || cap_dst[0] !== D1) begin errs++; $display("[B2B] T1[0]=(%0h,%0h) exp (%0h,%0h)", cap_src[0], cap_dst[0], S1, D1); end + if (cap_src[NB] !== S2 || cap_dst[NB] !== D2) begin errs++; $display("[B2B] T2[0]=(%0h,%0h) exp (%0h,%0h) -- back-to-back base NOT reloaded", cap_src[NB], cap_dst[NB], S2, D2); end + if (cap_src[2*NB] !== S3 || cap_dst[2*NB] !== D3) begin errs++; $display("[B2B] T3[0]=(%0h,%0h) exp (%0h,%0h)", cap_src[2*NB], cap_dst[2*NB], S3, D3); end + // full-sequence independence: T2 and T3 must be T1 shifted by their base delta + for (int unsigned i = 0; i < NB; i++) begin + if ((cap_src[NB+i] - cap_src[i]) !== (S2 - S1) || (cap_dst[NB+i] - cap_dst[i]) !== (D2 - D1)) begin + errs++; if (errs <= 8) $display("[B2B] T2[%0d] not T1+delta: src %0h vs %0h (Δexp %0h)", i, cap_src[NB+i], cap_src[i], S2-S1); end + if ((cap_src[2*NB+i] - cap_src[i]) !== (S3 - S1) || (cap_dst[2*NB+i] - cap_dst[i]) !== (D3 - D1)) begin + errs++; if (errs <= 8) $display("[B2B] T3[%0d] not T1+delta: src %0h vs %0h (Δexp %0h)", i, cap_src[2*NB+i], cap_src[i], S3-S1); end + end + end + + if (errs == 0) $display("[B2B] PASS: %0d back-to-back + gapped ND transfers each walked from their own base", 3*NB); + else $fatal(1, "[B2B] FAIL: %0d errors (back-to-back ND base-address reuse)", errs); + $finish(); + end + + initial begin #500_000; $fatal(1, "[B2B] timeout"); end + +endmodule diff --git a/test/midend/tb_idma_transpose_midend.sv b/test/midend/tb_idma_transpose_midend.sv new file mode 100644 index 00000000..4f7440e2 --- /dev/null +++ b/test/midend/tb_idma_transpose_midend.sv @@ -0,0 +1,102 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Author: Daniel Keller + +`include "idma/typedef.svh" + +/// Unit check for idma_transpose_midend: the expanded NumDim=4 ND request must +/// match the golden geometry that tb_idma_transpose_nd hand-builds, and a +/// non-transpose request must pass through unchanged. +module tb_idma_transpose_midend #( + parameter int unsigned DataWidth = 512, + parameter int unsigned AddrWidth = 64, + parameter int unsigned M = 40, + parameter int unsigned N = 24, + parameter int unsigned EB = 1 +); + import idma_pkg::*; + + localparam int unsigned StrbWidth = DataWidth/8; + localparam int unsigned NE = StrbWidth/EB; + localparam int unsigned MODE = (EB==4) ? 2 : (EB==2) ? 1 : 0; + localparam int unsigned YT = (M + NE - 1)/NE; + localparam int unsigned NT = (N + NE - 1)/NE; + localparam int unsigned MP = YT*NE; + localparam int unsigned NumDim = 4; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [31:0] tf_len_t; + typedef logic [2:0] id_t; + typedef logic [31:0] reps_t; + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + idma_nd_req_t nd_in, nd_out; + logic vi, ro, vo, ri; + assign vi = 1'b1; + assign ri = 1'b1; + + idma_transpose_midend #( + .NumDim(NumDim), .StrbWidth(StrbWidth), .addr_t(addr_t), .idma_nd_req_t(idma_nd_req_t) + ) i_dut ( + .nd_req_i(nd_in), .valid_i(vi), .ready_o(ro), + .nd_req_o(nd_out), .valid_o(vo), .ready_i(ri) + ); + + int errs = 0; + task automatic chk(input string name, input logic [63:0] got, input logic [63:0] exp); + if (got !== exp) begin + errs++; + $display("[MID] %s: got %0d exp %0d", name, $signed(got), $signed(exp)); + end + endtask + + initial begin + // --- passthrough case --- + nd_in = '0; + nd_in.burst_req.src_addr = 64'hDEAD; + nd_in.burst_req.dst_addr = 64'hBEEF; + nd_in.d_req[0].reps = 7; + #1; + if (nd_out !== nd_in) begin + errs++; + $display("[MID] passthrough altered a non-transpose request"); + end + + // --- transpose case --- + nd_in = '0; + nd_in.burst_req.src_addr = 64'h1000; + nd_in.burst_req.dst_addr = 64'h2000; + nd_in.burst_req.opt.compute.enable = 1'b1; + nd_in.burst_req.opt.compute.op = COMPUTE_TRANSPOSE; + nd_in.burst_req.opt.compute.params.transpose.mode = 2'(MODE); + nd_in.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); + nd_in.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + #1; + + // golden geometry (same formulas as tb_idma_transpose_nd) + chk("length", nd_out.burst_req.length, NE*EB); + chk("d0.reps", nd_out.d_req[0].reps, NE); + chk("d0.src", nd_out.d_req[0].src_strides, addr_t'(N*EB)); + chk("d0.dst", nd_out.d_req[0].dst_strides, addr_t'(MP*EB)); + chk("d1.reps", nd_out.d_req[1].reps, YT); + chk("d1.src", nd_out.d_req[1].src_strides, addr_t'(N*EB)); + chk("d1.dst", nd_out.d_req[1].dst_strides, addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB))); + chk("d2.reps", nd_out.d_req[2].reps, NT); + chk("d2.src", nd_out.d_req[2].src_strides, addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB))); + chk("d2.dst", nd_out.d_req[2].dst_strides, addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB))); + // addresses + compute must survive untouched + chk("src_addr", nd_out.burst_req.src_addr, 64'h1000); + chk("dst_addr", nd_out.burst_req.dst_addr, 64'h2000); + chk("cmp_en", nd_out.burst_req.opt.compute.enable, 1); + + if (errs == 0) + $display("[MID] PASS: %0dx%0d EB=%0d golden (NE=%0d YT=%0d NT=%0d MP=%0d)", + M, N, EB, NE, YT, NT, MP); + else $fatal(1, "[MID] FAIL: %0d mismatches", errs); + $finish; + end +endmodule diff --git a/test/tb_idma_transpose_b2b.sv b/test/tb_idma_transpose_b2b.sv new file mode 100644 index 00000000..4337dce1 --- /dev/null +++ b/test/tb_idma_transpose_b2b.sv @@ -0,0 +1,214 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Authors: +// - Daniel Keller +// +// End-to-end back-to-back transpose regression: two transposes of one source to +// DIFFERENT dst bases through the ND midend -> rw_axi backend -> axi_sim_mem. A +// stale base across transfers would leave the second dst untouched. Both checked. + +`include "axi/typedef.svh" +`include "idma/typedef.svh" + +module tb_idma_transpose_b2b + import idma_pkg::*; +#( + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, + parameter int unsigned AxiIdWidth = 12, + parameter int unsigned TFLenWidth = 32, + parameter int unsigned M = 6, + parameter int unsigned N = 8, + parameter int unsigned EB = 1 +); + + localparam time TA = 1ns, TT = 9ns, TCK = 10ns; + localparam int unsigned StrbWidth = DataWidth / 8; + localparam int unsigned NE = StrbWidth / EB; + localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; + localparam int unsigned YT = (M + NE - 1) / NE; + localparam int unsigned NT = (N + NE - 1) / NE; + localparam int unsigned MP = YT * NE; + localparam int unsigned NumDim = 4; + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [DataWidth-1:0] data_t; + typedef logic [StrbWidth-1:0] strb_t; + typedef logic [AxiIdWidth-1:0] id_t; + typedef logic [UserWidth-1:0] user_t; + typedef logic [TFLenWidth-1:0] tf_len_t; + typedef logic [31:0] reps_t; + + `AXI_TYPEDEF_AW_CHAN_T(axi_aw_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_W_CHAN_T(axi_w_chan_t, data_t, strb_t, user_t) + `AXI_TYPEDEF_B_CHAN_T(axi_b_chan_t, id_t, user_t) + `AXI_TYPEDEF_AR_CHAN_T(axi_ar_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_R_CHAN_T(axi_r_chan_t, data_t, id_t, user_t) + `AXI_TYPEDEF_REQ_T(axi_req_t, axi_aw_chan_t, axi_w_chan_t, axi_ar_chan_t) + `AXI_TYPEDEF_RESP_T(axi_rsp_t, axi_b_chan_t, axi_r_chan_t) + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + typedef struct packed { axi_ar_chan_t ar_chan; } axi_read_meta_channel_t; + typedef struct packed { axi_read_meta_channel_t axi; } read_meta_channel_t; + typedef struct packed { axi_aw_chan_t aw_chan; } axi_write_meta_channel_t; + typedef struct packed { axi_write_meta_channel_t axi; } write_meta_channel_t; + + logic clk, rst_n; + idma_req_t idma_req; logic req_valid, req_ready; + idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; + idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; + axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; + idma_busy_t busy; logic nd_busy; + + assign idma_eh_req = '0; + assign eh_req_valid = 1'b0; + + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( + .clk_i(clk), .rst_ni(rst_n), + .slv_read_req_i(axi_read_req), .slv_read_resp_o(axi_read_rsp), + .slv_write_req_i(axi_write_req), .slv_write_resp_o(axi_write_rsp), + .mst_req_o(axi_req), .mst_resp_i(axi_rsp) + ); + assign axi_req_mem = axi_req; + assign axi_rsp = axi_rsp_mem; + + axi_sim_mem #( + .AddrWidth(AddrWidth), .DataWidth(DataWidth), .IdWidth(AxiIdWidth), .UserWidth(UserWidth), + .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .WarnUninitialized(1'b0), .ClearErrOnAccess(1'b1), .ApplDelay(TA), .AcqDelay(TT) + ) i_axi_sim_mem ( + .clk_i(clk), .rst_ni(rst_n), .axi_req_i(axi_req_mem), .axi_rsp_o(axi_rsp_mem), + .mon_r_last_o(), .mon_r_beat_count_o(), .mon_r_user_o(), .mon_r_id_o(), + .mon_r_data_o(), .mon_r_addr_o(), .mon_r_valid_o(), + .mon_w_last_o(), .mon_w_beat_count_o(), .mon_w_user_o(), .mon_w_id_o(), + .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() + ); + + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_nd_midend ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), + .busy_o(nd_busy) + ); + + idma_backend_rw_axi #( + .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), + .UserWidth(UserWidth), .TFLenWidth(TFLenWidth), .MaskInvalidData(1'b1), .BufferDepth(3), + .RAWCouplingAvail(1'b1), .HardwareLegalizer(1'b1), .RejectZeroTransfers(1'b1), + .ErrorCap(idma_pkg::NO_ERROR_HANDLING), .PrintFifoInfo(1'b0), .NumAxInFlight(StrbWidth), .MemSysDepth(0), + .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_eh_req_t(idma_eh_req_t), + .idma_busy_t(idma_busy_t), .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .write_meta_channel_t(write_meta_channel_t), .read_meta_channel_t(read_meta_channel_t) + ) i_idma_backend ( + .clk_i(clk), .rst_ni(rst_n), .testmode_i(1'b0), + .idma_req_i(idma_req), .req_valid_i(req_valid), .req_ready_o(req_ready), + .idma_rsp_o(idma_rsp), .rsp_valid_o(rsp_valid), .rsp_ready_i(rsp_ready), + .idma_eh_req_i(idma_eh_req), .eh_req_valid_i(eh_req_valid), .eh_req_ready_o(eh_req_ready), + .axi_read_req_o(axi_read_req), .axi_read_rsp_i(axi_read_rsp), + .axi_write_req_o(axi_write_req), .axi_write_rsp_i(axi_write_rsp), .busy_o(busy) + ); + + stream_watchdog #(.NumCycles(4000)) i_r_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_rsp.r_valid), .ready_i(axi_req.r_ready)); + stream_watchdog #(.NumCycles(4000)) i_w_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_req.w_valid), .ready_i(axi_rsp.w_ready)); + + addr_t sb = 'h0000_1000; + + task automatic wr_mem(input addr_t a, input logic [7:0] d); i_axi_sim_mem.mem[a] = d; endtask + function automatic logic [7:0] rd_mem(input addr_t a); + return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; + endfunction + + // one transpose of the source at sb -> dst base `db`; returns error count + task automatic do_transpose(input addr_t db, output int unsigned errs); + errs = 0; + // pre-fill full padded dst extent with sentinel + for (int unsigned i = 0; i < NT*NE; i++) + for (int unsigned j = 0; j < MP; j++) + for (int unsigned b = 0; b < EB; b++) + wr_mem(db + (i*MP + j)*EB + b, 8'hCC); + nd_req = '0; + nd_req.burst_req.length = tf_len_t'(NE*EB); + nd_req.burst_req.src_addr = sb; + nd_req.burst_req.dst_addr = db; + nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.beo.decouple_rw = 1'b1; + nd_req.burst_req.opt.beo.decouple_aw = 1'b1; + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(MODE); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + nd_req.burst_req.opt.last = 1'b1; + nd_req.d_req[0].reps = reps_t'(NE); nd_req.d_req[0].src_strides = addr_t'(int'(N*EB)); nd_req.d_req[0].dst_strides = addr_t'(int'(MP*EB)); + nd_req.d_req[1].reps = reps_t'(YT); nd_req.d_req[1].src_strides = addr_t'(int'(N*EB)); nd_req.d_req[1].dst_strides = addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB)); + nd_req.d_req[2].reps = reps_t'(NT); nd_req.d_req[2].src_strides = addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB)); nd_req.d_req[2].dst_strides = addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB)); + nd_req_valid = 1'b1; + do @(posedge clk); while (!nd_req_ready); // drop valid the cycle accept is seen (compliant) + nd_req_valid = 1'b0; + nd_req = '0; + while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + repeat (20) @(posedge clk); + // data + padding checks + for (int unsigned c = 0; c < N; c++) + for (int unsigned r = 0; r < M; r++) + for (int unsigned b = 0; b < EB; b++) + if (rd_mem(db + (c*MP + r)*EB + b) !== rd_mem(sb + (r*N + c)*EB + b)) begin + errs++; if (errs <= 8) $display("[B2BT] @db=%0h MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", db, c, r, b, rd_mem(db+(c*MP+r)*EB+b), rd_mem(sb+(r*N+c)*EB+b)); + end + for (int unsigned i = 0; i < NT*NE; i++) + for (int unsigned j = 0; j < MP; j++) + if (i >= N || j >= M) + for (int unsigned b = 0; b < EB; b++) + if (rd_mem(db + (i*MP + j)*EB + b) !== 8'hCC) begin + errs++; if (errs <= 8) $display("[B2BT] @db=%0h PADDING CLOBBERED row=%0d col=%0d", db, i, j); + end + endtask + + initial begin + automatic int unsigned e1, e2; + automatic addr_t db1 = 'h0000_4000; + automatic addr_t db2 = 'h0000_8000; // DIFFERENT base — a stale-addr bug misplaces xfer 2 + nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + @(posedge rst_n); + repeat (5) @(posedge clk); + for (int unsigned r = 0; r < M; r++) + for (int unsigned c = 0; c < N; c++) + for (int unsigned b = 0; b < EB; b++) + wr_mem(sb + (r*N + c)*EB + b, 8'((( (r*N+c)*EB + b )*7 + 3) & 8'hFF)); + + $display("[B2BT] transfer 1 -> db=%0h", db1); + do_transpose(db1, e1); + $display("[B2BT] transfer 2 (back-to-back) -> db=%0h", db2); + do_transpose(db2, e2); + + if (e1 == 0 && e2 == 0) + $display("[B2BT] PASS: two back-to-back %0dx%0d EB=%0d transposes both correct (xfer2 landed at its own dst)", M, N, EB); + else + $fatal(1, "[B2BT] FAIL: xfer1 errs=%0d xfer2 errs=%0d", e1, e2); + repeat (5) @(posedge clk); + $finish(); + end + + initial begin #5_000_000; $fatal(1, "[B2BT] timeout"); end + +endmodule diff --git a/test/tb_idma_transpose_nd.sv b/test/tb_idma_transpose_nd.sv new file mode 100644 index 00000000..1fa3e3ba --- /dev/null +++ b/test/tb_idma_transpose_nd.sv @@ -0,0 +1,263 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// +// Authors: +// - Daniel Keller +// +// Self-checking multi-tile transpose testbench: idma_nd_midend (NumDim=4, +// transposed-stride program) -> idma_backend_rw_axi (EnableTranspose) -> +// axi_sim_mem. The ND midend generates the tiled read order (col-tile, +// row-tile, local-row) and the transposed destination placement the engine +// requires, so a full M x N transpose works end-to-end (not just one tile). +// Reference: out_T[c][r] = in[r][c] over E-byte elements. + +`include "axi/typedef.svh" +`include "idma/typedef.svh" + +module tb_idma_transpose_nd + import idma_pkg::*; +#( + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, + parameter int unsigned AxiIdWidth = 12, + parameter int unsigned TFLenWidth = 32, + parameter int unsigned M = 8, // matrix rows (elements) + parameter int unsigned N = 8, // matrix cols (elements) + parameter int unsigned EB = 1, // element size in bytes (1/2/4) + // 0 = auto: NumAxInFlight = StrbWidth (transpose needs >= NE in-flight; NE = StrbWidth at E=1) + parameter int unsigned NumAxInFlight = 0, + parameter int unsigned BufferDepth = 3 +); + + localparam time TA = 1ns; + localparam time TT = 9ns; + localparam time TCK = 10ns; + + localparam int unsigned StrbWidth = DataWidth / 8; + localparam int unsigned NE = StrbWidth / EB; // tile side (elements) + localparam int unsigned AxIF = (NumAxInFlight == 0) ? StrbWidth : NumAxInFlight; + localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; + localparam int unsigned YT = (M + NE - 1) / NE; // row-tiles + localparam int unsigned NT = (N + NE - 1) / NE; // col-tiles + // padded Aᵀ row pitch: M rounded up to a tile so every row is StrbWidth-aligned + localparam int unsigned MP = YT * NE; + localparam int unsigned NumDim = 4; // 1D + {row, row-tile, col-tile} + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + // ── Types ── + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [DataWidth-1:0] data_t; + typedef logic [StrbWidth-1:0] strb_t; + typedef logic [AxiIdWidth-1:0] id_t; + typedef logic [UserWidth-1:0] user_t; + typedef logic [TFLenWidth-1:0] tf_len_t; + typedef logic [31:0] reps_t; + + `AXI_TYPEDEF_AW_CHAN_T(axi_aw_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_W_CHAN_T(axi_w_chan_t, data_t, strb_t, user_t) + `AXI_TYPEDEF_B_CHAN_T(axi_b_chan_t, id_t, user_t) + `AXI_TYPEDEF_AR_CHAN_T(axi_ar_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_R_CHAN_T(axi_r_chan_t, data_t, id_t, user_t) + `AXI_TYPEDEF_REQ_T(axi_req_t, axi_aw_chan_t, axi_w_chan_t, axi_ar_chan_t) + `AXI_TYPEDEF_RESP_T(axi_rsp_t, axi_b_chan_t, axi_r_chan_t) + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + typedef struct packed { axi_ar_chan_t ar_chan; } axi_read_meta_channel_t; + typedef struct packed { axi_read_meta_channel_t axi; } read_meta_channel_t; + typedef struct packed { axi_aw_chan_t aw_chan; } axi_write_meta_channel_t; + typedef struct packed { axi_write_meta_channel_t axi; } write_meta_channel_t; + + // ── Signals ── + logic clk, rst_n; + idma_req_t idma_req; logic req_valid, req_ready; + idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; + idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; + axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; + idma_busy_t busy; logic nd_busy; + + assign idma_eh_req = '0; + assign eh_req_valid = 1'b0; + + // ── Clock / reset ── + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + // ── AXI sim memory (read+write joined) ── + axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( + .clk_i(clk), .rst_ni(rst_n), + .slv_read_req_i(axi_read_req), .slv_read_resp_o(axi_read_rsp), + .slv_write_req_i(axi_write_req), .slv_write_resp_o(axi_write_rsp), + .mst_req_o(axi_req), .mst_resp_i(axi_rsp) + ); + assign axi_req_mem = axi_req; + assign axi_rsp = axi_rsp_mem; + + axi_sim_mem #( + .AddrWidth(AddrWidth), .DataWidth(DataWidth), .IdWidth(AxiIdWidth), .UserWidth(UserWidth), + .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .WarnUninitialized(1'b0), .ClearErrOnAccess(1'b1), .ApplDelay(TA), .AcqDelay(TT) + ) i_axi_sim_mem ( + .clk_i(clk), .rst_ni(rst_n), .axi_req_i(axi_req_mem), .axi_rsp_o(axi_rsp_mem), + .mon_r_last_o(), .mon_r_beat_count_o(), .mon_r_user_o(), .mon_r_id_o(), + .mon_r_data_o(), .mon_r_addr_o(), .mon_r_valid_o(), + .mon_w_last_o(), .mon_w_beat_count_o(), .mon_w_user_o(), .mon_w_id_o(), + .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() + ); + + // ── ND midend: ND transpose descriptor -> 1D bursts ── + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_nd_midend ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), + .busy_o(nd_busy) + ); + + // ── Backend (rw_axi) with transpose engine ── + idma_backend_rw_axi #( + .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), + .UserWidth(UserWidth), .TFLenWidth(TFLenWidth), .MaskInvalidData(1'b1), .BufferDepth(BufferDepth), + .RAWCouplingAvail(1'b1), .HardwareLegalizer(1'b1), .RejectZeroTransfers(1'b1), + .ErrorCap(idma_pkg::NO_ERROR_HANDLING), .PrintFifoInfo(1'b0), .NumAxInFlight(AxIF), .MemSysDepth(0), + .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_eh_req_t(idma_eh_req_t), + .idma_busy_t(idma_busy_t), .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .write_meta_channel_t(write_meta_channel_t), .read_meta_channel_t(read_meta_channel_t) + ) i_idma_backend ( + .clk_i(clk), .rst_ni(rst_n), .testmode_i(1'b0), + .idma_req_i(idma_req), .req_valid_i(req_valid), .req_ready_o(req_ready), + .idma_rsp_o(idma_rsp), .rsp_valid_o(rsp_valid), .rsp_ready_i(rsp_ready), + .idma_eh_req_i(idma_eh_req), .eh_req_valid_i(eh_req_valid), .eh_req_ready_o(eh_req_ready), + .axi_read_req_o(axi_read_req), .axi_read_rsp_i(axi_read_rsp), + .axi_write_req_o(axi_write_req), .axi_write_rsp_i(axi_write_rsp), .busy_o(busy) + ); + + // watchdogs to surface deadlocks rather than hang forever + stream_watchdog #(.NumCycles(2000)) i_r_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_rsp.r_valid), .ready_i(axi_req.r_ready)); + stream_watchdog #(.NumCycles(2000)) i_w_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_req.w_valid), .ready_i(axi_rsp.w_ready)); + + // ── Stimulus + check via sim-memory backdoor ── + addr_t sb = 'h0000_1000; + addr_t db = 'h0000_4000; + + // every AW (incl. wstrb=0 padding rows) must stay in the padded dst allocation + // [db, db+NT*NE*MP*EB) — else a strict slave would DECERR + always @(posedge clk) if (rst_n && axi_write_req.aw_valid && axi_write_rsp.aw_ready) begin + automatic addr_t aw_end = db + addr_t'(NT*NE*MP*EB); + if (axi_write_req.aw.addr < db || axi_write_req.aw.addr >= aw_end) + $fatal(1, "[TPN] AW 0x%0h outside dst alloc [0x%0h,0x%0h) — would DECERR on a strict slave", + axi_write_req.aw.addr, db, aw_end); + end + + task automatic wr_mem(input addr_t a, input logic [7:0] d); i_axi_sim_mem.mem[a] = d; endtask + function automatic logic [7:0] rd_mem(input addr_t a); + return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; + endfunction + + initial begin + automatic int unsigned errs = 0; + nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + @(posedge rst_n); + repeat (5) @(posedge clk); + + // init source matrix (row-major, M x N elements of EB bytes) + for (int unsigned r = 0; r < M; r++) + for (int unsigned c = 0; c < N; c++) + for (int unsigned b = 0; b < EB; b++) + wr_mem(sb + (r*N + c)*EB + b, 8'((( (r*N+c)*EB + b )*7 + 3) & 8'hFF)); + + // sentinel-fill the full padded Aᵀ extent; padding cols/rows must stay sentinel + // (the engine strobe suppresses them) — checked after the transfer + for (int unsigned i = 0; i < NT*NE; i++) + for (int unsigned j = 0; j < MP; j++) + for (int unsigned b = 0; b < EB; b++) + wr_mem(db + (i*MP + j)*EB + b, 8'hCC); + + // ── transposed-stride ND program (routing-plan §4.2) ── + nd_req = '0; + nd_req.burst_req.length = tf_len_t'(NE*EB); // one tile-row = StrbWidth bytes + nd_req.burst_req.src_addr = sb; + nd_req.burst_req.dst_addr = db; + nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.beo.decouple_rw = 1'b1; + nd_req.burst_req.opt.beo.decouple_aw = 1'b1; + nd_req.burst_req.opt.beo.src_max_llen = '0; + nd_req.burst_req.opt.beo.dst_max_llen = '0; + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(MODE); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + nd_req.burst_req.opt.last = 1'b1; + // ND midend strides are INCREMENTAL deltas (added on dim roll-over), NOT + // absolute pitches. Aᵀ uses padded pitch MP*EB (aligned writes); src keeps + // N*EB (misaligned reads coalesce in the pre-engine buffer). + // d_req[0] = local row within tile (reps NE) + nd_req.d_req[0].reps = reps_t'(NE); + nd_req.d_req[0].src_strides = addr_t'(int'(N*EB)); // next source row + nd_req.d_req[0].dst_strides = addr_t'(int'(MP*EB)); // next Aᵀ row (padded pitch) + // d_req[1] = row-tile (reps YT) + nd_req.d_req[1].reps = reps_t'(YT); + nd_req.d_req[1].src_strides = addr_t'(int'(N*EB)); // rows are consecutive + nd_req.d_req[1].dst_strides = addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB)); // back up cols, next col-block + // d_req[2] = col-tile (reps NT). src rewind uses the padded row extent + // (YT*NE-1, not M-1): the read walks padding rows of the last row-tile. + nd_req.d_req[2].reps = reps_t'(NT); + nd_req.d_req[2].src_strides = addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB)); // back to padded row0, next col-block + nd_req.d_req[2].dst_strides = addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB)); // next Aᵀ col-block + + $display("[TPN] launching %0dx%0d EB=%0d transpose via ND midend (NE=%0d, %0dx%0d tiles)", M, N, EB, NE, YT, NT); + nd_req_valid = 1'b1; + // drop valid on accept; holding it one cycle past makes the midend re-walk the request + do @(posedge clk); while (!nd_req_ready); + nd_req_valid = 1'b0; + nd_req = '0; + + // wait for ND completion + while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + repeat (20) @(posedge clk); + + // check 1 (data): out_T[c][r] == in[r][c], Aᵀ at padded pitch MP + for (int unsigned c = 0; c < N; c++) + for (int unsigned r = 0; r < M; r++) + for (int unsigned b = 0; b < EB; b++) begin + automatic logic [7:0] got = rd_mem(db + (c*MP + r)*EB + b); + automatic logic [7:0] exp = rd_mem(sb + (r*N + c)*EB + b); + if (got !== exp) begin + errs++; + if (errs <= 12) $display("[TPN] MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", c, r, b, got, exp); + end + end + // check 2: padding cols [M,MP) and padding rows [N,NT*NE) must stay sentinel (strobe-suppressed) + for (int unsigned i = 0; i < NT*NE; i++) + for (int unsigned j = 0; j < MP; j++) + if (i >= N || j >= M) + for (int unsigned b = 0; b < EB; b++) begin + automatic logic [7:0] got = rd_mem(db + (i*MP + j)*EB + b); + if (got !== 8'hCC) begin + errs++; + if (errs <= 12) $display("[TPN] PADDING CLOBBERED at row=%0d col=%0d b%0d=%02h (exp CC)", i, j, b, got); + end + end + if (errs == 0) $display("[TPN] PASS: %0dx%0d EB=%0d multi-tile transpose matches golden (padding intact)", M, N, EB); + else $fatal(1, "[TPN] FAIL: %0d mismatches", errs); + repeat (5) @(posedge clk); + $finish(); + end + + initial begin #5_000_000; $fatal(1, "[TPN] timeout"); end + +endmodule From f9853a5079f223cfd1d1439dec9d5d2007a62dcc Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Tue, 16 Jun 2026 15:48:03 +0200 Subject: [PATCH 07/13] backend: Address transpose-engine review feedback Validate IDMA_VIDMA_IDS against the built backend variants, reject empty and duplicate compute IDs, and assert StrbWidth is a power of two >= 2 in the transpose engine. Drop the standalone routing-plan doc (folded into the docs PR) and trim the midend header. --- idma.mk | 6 ++++++ src/backend/idma_otf_transpose.sv | 4 ++++ src/midend/idma_transpose_midend.sv | 12 +++++------- util/mario/util.py | 9 ++++++++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/idma.mk b/idma.mk index 6137ded3..27d29384 100644 --- a/idma.mk +++ b/idma.mk @@ -55,6 +55,12 @@ IDMA_ADD_IDS ?= IDMA_BACKEND_IDS ?= $(IDMA_BASE_IDS) $(IDMA_OCCAMY_IDS) $(IDMA_ADD_IDS) # Backend variants that host the on-the-fly compute dispatcher (single AXI write) IDMA_VIDMA_IDS ?= rw_axi +# Compute variants (strip the optional :op:fd suffix) must be built backends +_idma_vidma_unknown := $(filter-out $(IDMA_BACKEND_IDS),\ + $(foreach c,$(IDMA_VIDMA_IDS),$(firstword $(subst :, ,$(c))))) +ifneq ($(_idma_vidma_unknown),) + $(error iDMA: IDMA_VIDMA_IDS variant(s) not in IDMA_BACKEND_IDS: $(_idma_vidma_unknown)) +endif # generated frontends IDMA_BASE_FE_IDS := reg32_3d reg64_2d reg64_1d diff --git a/src/backend/idma_otf_transpose.sv b/src/backend/idma_otf_transpose.sv index 5c543f99..3e71fe60 100644 --- a/src/backend/idma_otf_transpose.sv +++ b/src/backend/idma_otf_transpose.sv @@ -45,6 +45,10 @@ module idma_otf_transpose #( input logic ready_i ); + // StrbWidth must be a power of two >= 2 so LaneW>=1 and the shift geometry holds + initial assert (StrbWidth >= 2 && (StrbWidth & (StrbWidth-1)) == 0) else + $fatal(1, "idma_otf_transpose: StrbWidth (%0d) must be a power of two >= 2", StrbWidth); + // geometry: NE is a power of two, so only shifts and AND-masks logic [1:0] eff_mode; // element-size mode, saturated at LaneW logic [LaneW:0] ne_m1; // NE-1 diff --git a/src/midend/idma_transpose_midend.sv b/src/midend/idma_transpose_midend.sv index 9d79d551..d9054cbc 100644 --- a/src/midend/idma_transpose_midend.sv +++ b/src/midend/idma_transpose_midend.sv @@ -4,13 +4,11 @@ // // Author: Daniel Keller -/// Transpose geometry expander. For a request carrying opt.compute = TRANSPOSE, -/// derives the NumDim=4 tiled ND walk (row / row-tile / col-tile) from the tensor -/// shape (M, N, element mode) and the bus StrbWidth, leaving the generic -/// idma_nd_midend to walk it. Non-transpose requests pass through untouched. -/// The walk reads the source up to the tile-padded bounds (ceil to NE in both -/// dims) and writes the full padded dst extent (writes strb-masked, reads not). -/// Combinational; outputs are quasi-static per request (multicycle-safe in STA). +/// Transpose geometry expander: turns an opt.compute=TRANSPOSE request into a +/// NumDim=4 tiled ND walk (row / row-tile / col-tile) from (M, N, mode) and +/// StrbWidth, for the generic idma_nd_midend to walk. Non-transpose passes +/// through. Reads to tile-padded bounds; writes the padded dst strb-masked. +/// Combinational, quasi-static per request (multicycle-safe in STA). module idma_transpose_midend #( /// Number of ND dimensions (must be >= 4 to express the tiled walk) parameter int unsigned NumDim = 32'd4, diff --git a/util/mario/util.py b/util/mario/util.py index bd655d91..f19f9d01 100644 --- a/util/mario/util.py +++ b/util/mario/util.py @@ -171,6 +171,13 @@ def prepare_compute_ids(compute_id_strs: list) -> dict: res = {} for cid_str in (compute_id_strs or []): parts = cid_str.split(':') + variant = parts[0] + if variant == '': + print(f'[MARIO] empty compute variant in "{cid_str}"', file=sys.stderr) + sys.exit(1) + if variant in res: + print(f'[MARIO] duplicate compute variant "{variant}"', file=sys.stderr) + sys.exit(1) ops = ['transpose'] full_duplex = True for part in parts[1:]: @@ -182,5 +189,5 @@ def prepare_compute_ids(compute_id_strs: list) -> dict: if op not in ('transpose',): print(f'[MARIO] {op} is a non-supported compute op in {cid_str}', file=sys.stderr) sys.exit(1) - res[parts[0]] = {'ops': ops, 'full_duplex': full_duplex} + res[variant] = {'ops': ops, 'full_duplex': full_duplex} return res From af05bcd7a78f77aaa5249bd8343d89a7bf0e1133 Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Tue, 16 Jun 2026 15:59:39 +0200 Subject: [PATCH 08/13] ci: Fix file-header format for lint-author The transpose files used a '//' empty-comment separator and singular 'Author:'; lint-authors.py requires a blank line after the SPDX line and after the author list (its header-regex is a folded YAML scalar with a trailing newline) plus a plural 'Authors:' bullet list. --- src/backend/idma_otf_transpose.sv | 2 +- src/midend/idma_transpose_midend.sv | 5 +++-- test/idma_transpose_dpi.c | 4 ++-- test/midend/tb_idma_nd_midend_b2b.sv | 4 ++-- test/midend/tb_idma_transpose_midend.sv | 5 +++-- test/tb_idma_otf_transpose.sv | 2 +- test/tb_idma_transpose_b2b.sv | 4 ++-- test/tb_idma_transpose_nd.sv | 4 ++-- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/backend/idma_otf_transpose.sv b/src/backend/idma_otf_transpose.sv index 3e71fe60..5d3c8004 100644 --- a/src/backend/idma_otf_transpose.sv +++ b/src/backend/idma_otf_transpose.sv @@ -4,7 +4,7 @@ // Authors: // - Daniel Keller -// + // On-the-fly matrix transpose engine (ping-pong tile banks) for the iDMA // transport write seam. Adapted from the datamover (Ratha) HWPE: // pulp-platform/datamover@d58a985, rtl/datamover_engine.sv. diff --git a/src/midend/idma_transpose_midend.sv b/src/midend/idma_transpose_midend.sv index d9054cbc..5d984c76 100644 --- a/src/midend/idma_transpose_midend.sv +++ b/src/midend/idma_transpose_midend.sv @@ -1,8 +1,9 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// -// Author: Daniel Keller + +// Authors: +// - Daniel Keller /// Transpose geometry expander: turns an opt.compute=TRANSPOSE request into a /// NumDim=4 tiled ND walk (row / row-tile / col-tile) from (M, N, mode) and diff --git a/test/idma_transpose_dpi.c b/test/idma_transpose_dpi.c index 0f7faf3f..6497a0c5 100644 --- a/test/idma_transpose_dpi.c +++ b/test/idma_transpose_dpi.c @@ -1,10 +1,10 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// + // Authors: // - Daniel Keller -// + // DPI-C golden model for idma_otf_transpose: an element-granular matrix // transpose. Element size E in {1,2,4} bytes (int8/fp16/fp32); each E-byte // element is kept intact while the M x N element grid is transposed to N x M. diff --git a/test/midend/tb_idma_nd_midend_b2b.sv b/test/midend/tb_idma_nd_midend_b2b.sv index aaba907f..e849a1e0 100644 --- a/test/midend/tb_idma_nd_midend_b2b.sv +++ b/test/midend/tb_idma_nd_midend_b2b.sv @@ -1,10 +1,10 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// + // Authors: // - Daniel Keller -// + // Back-to-back ND regression for idma_nd_midend. Drives the midend directly and // golden-checks the burst_req address sequence: two back-to-back transfers (no // gap) plus one after an idle gap must each walk from their own base. Catches a diff --git a/test/midend/tb_idma_transpose_midend.sv b/test/midend/tb_idma_transpose_midend.sv index 4f7440e2..41598289 100644 --- a/test/midend/tb_idma_transpose_midend.sv +++ b/test/midend/tb_idma_transpose_midend.sv @@ -1,8 +1,9 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// -// Author: Daniel Keller + +// Authors: +// - Daniel Keller `include "idma/typedef.svh" diff --git a/test/tb_idma_otf_transpose.sv b/test/tb_idma_otf_transpose.sv index 23e29a3d..fc0950a7 100644 --- a/test/tb_idma_otf_transpose.sv +++ b/test/tb_idma_otf_transpose.sv @@ -4,7 +4,7 @@ // Authors: // - Daniel Keller -// + // Standalone self-checking testbench for idma_otf_transpose (element transpose, // no iDMA backend / no protocol deps). Verifies a full M x N transpose of // EB-byte elements (EB in {1,2,4} = int8/fp16/fp32), split into NE-square tiles diff --git a/test/tb_idma_transpose_b2b.sv b/test/tb_idma_transpose_b2b.sv index 4337dce1..62c10c5a 100644 --- a/test/tb_idma_transpose_b2b.sv +++ b/test/tb_idma_transpose_b2b.sv @@ -1,10 +1,10 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// + // Authors: // - Daniel Keller -// + // End-to-end back-to-back transpose regression: two transposes of one source to // DIFFERENT dst bases through the ND midend -> rw_axi backend -> axi_sim_mem. A // stale base across transfers would leave the second dst untouched. Both checked. diff --git a/test/tb_idma_transpose_nd.sv b/test/tb_idma_transpose_nd.sv index 1fa3e3ba..59d12b2d 100644 --- a/test/tb_idma_transpose_nd.sv +++ b/test/tb_idma_transpose_nd.sv @@ -1,10 +1,10 @@ // Copyright 2026 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 -// + // Authors: // - Daniel Keller -// + // Self-checking multi-tile transpose testbench: idma_nd_midend (NumDim=4, // transposed-stride program) -> idma_backend_rw_axi (EnableTranspose) -> // axi_sim_mem. The ND midend generates the tiled read order (col-tile, From 38f32647b9ef65a8689df504aa2926231bdbe15e Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Tue, 16 Jun 2026 16:02:08 +0200 Subject: [PATCH 09/13] build: Add desc64/reg addrmap packages to the split_rtl file list The PeakRDL addrmap packages are bundled in idma_generated.sv but were missing from the per-variant split_rtl source list, so the split_rtl compile flow could not find idma_*_addrmap_pkg (used by the desc64 reg wrapper). Order them after the reg_top files, matching the bundle. --- Bender.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Bender.yml b/Bender.yml index 33d662f5..0b158979 100644 --- a/Bender.yml +++ b/Bender.yml @@ -103,6 +103,10 @@ sources: - target/rtl/idma_reg32_3d_reg_top.sv - target/rtl/idma_reg64_2d_reg_top.sv - target/rtl/idma_reg64_1d_reg_top.sv + - target/rtl/idma_desc64_addrmap_pkg.sv + - target/rtl/idma_reg32_3d_addrmap_pkg.sv + - target/rtl/idma_reg64_2d_addrmap_pkg.sv + - target/rtl/idma_reg64_1d_addrmap_pkg.sv - target/rtl/idma_desc64_top.sv - target/rtl/idma_reg32_3d_top.sv - target/rtl/idma_reg64_2d_top.sv From 821236bc5f067fd6275ad423ff1a93394496353b Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Tue, 16 Jun 2026 17:38:19 +0200 Subject: [PATCH 10/13] build: Default IDMA_VIDMA_IDS to empty and regen RTL on change Stock backends carry no on-the-fly compute by default; opt in via IDMA_VIDMA_IDS=. A stamp file forces regeneration of the compute- bearing RTL when the value changes (Make tracks timestamps, not variable values). The transpose sim targets opt in to rw_axi via a target-specific variable. --- idma.mk | 22 +++++++++++++++++----- target/rtl/.gitignore | 1 + 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/idma.mk b/idma.mk index 27d29384..db4d6618 100644 --- a/idma.mk +++ b/idma.mk @@ -53,8 +53,8 @@ IDMA_OCCAMY_IDS := \ rw_axi_rw_init_rw_obi IDMA_ADD_IDS ?= IDMA_BACKEND_IDS ?= $(IDMA_BASE_IDS) $(IDMA_OCCAMY_IDS) $(IDMA_ADD_IDS) -# Backend variants that host the on-the-fly compute dispatcher (single AXI write) -IDMA_VIDMA_IDS ?= rw_axi +# Compute-hosting variants (single AXI write); empty default = stock has no compute +IDMA_VIDMA_IDS ?= # Compute variants (strip the optional :op:fd suffix) must be built backends _idma_vidma_unknown := $(filter-out $(IDMA_BACKEND_IDS),\ $(foreach c,$(IDMA_VIDMA_IDS),$(firstword $(subst :, ,$(c))))) @@ -133,13 +133,22 @@ define idma_gen $(PYTHON) $(IDMA_GEN) --entity $1 --tpl $2 --db $3 --ids $4 --fids $5 $(if $7,--compute-ids $7) > $6 endef -$(IDMA_RTL_DIR)/idma_transport_layer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl $(IDMA_DB_FILES) +# Force an RTL regen when IDMA_VIDMA_IDS changes; rewritten only on change +IDMA_VIDMA_STAMP := $(IDMA_RTL_DIR)/.vidma_ids +.PHONY: idma_vidma_stamp_check +idma_vidma_stamp_check: + @mkdir -p $(IDMA_RTL_DIR) + @printf '%s' '$(IDMA_VIDMA_IDS)' | cmp -s - $(IDMA_VIDMA_STAMP) 2>/dev/null || \ + printf '%s' '$(IDMA_VIDMA_IDS)' > $(IDMA_VIDMA_STAMP) +$(IDMA_VIDMA_STAMP): idma_vidma_stamp_check ; + +$(IDMA_RTL_DIR)/idma_transport_layer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) $(call idma_gen,transport,$(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) -$(IDMA_RTL_DIR)/idma_legalizer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl $(IDMA_DB_FILES) +$(IDMA_RTL_DIR)/idma_legalizer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) $(call idma_gen,legalizer,$(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) -$(IDMA_RTL_DIR)/idma_backend_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_legalizer_%.sv $(IDMA_RTL_DIR)/idma_transport_layer_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl $(IDMA_DB_FILES) +$(IDMA_RTL_DIR)/idma_backend_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_legalizer_%.sv $(IDMA_RTL_DIR)/idma_transport_layer_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) $(call idma_gen,backend,$(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) $(IDMA_RTL_DIR)/idma_backend_synth_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_backend_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend_synth.sv.tpl $(IDMA_DB_FILES) @@ -373,6 +382,9 @@ idma_sim_tb_idma_otf_transpose: # (M or N not a multiple of NE) geometries for int8/fp16/fp32. Needs the # split_rtl flow (per-variant routing). Run with the Questa SEPP wrapper: # make idma_sim_tb_idma_transpose_nd VSIM="questa-2023.4 vsim" +# These tests need the write-seam engine: build rw_axi with compute (stamp regens) +idma_sim_tb_idma_transpose_nd idma_sim_tb_idma_transpose_b2b: IDMA_VIDMA_IDS := rw_axi + .PHONY: idma_sim_tb_idma_transpose_nd idma_sim_tb_idma_transpose_nd: $(IDMA_VSIM_DIR)/compile.tcl cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" diff --git a/target/rtl/.gitignore b/target/rtl/.gitignore index 625b14c8..9f111a44 100644 --- a/target/rtl/.gitignore +++ b/target/rtl/.gitignore @@ -3,3 +3,4 @@ include/* !include/.gitkeep *.sv *.hjson +.vidma_ids From 449daf6e99091827435e5567792862e3ee5689da Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Tue, 16 Jun 2026 18:25:59 +0200 Subject: [PATCH 11/13] build: Clean .vidma_ids in idma_rtl_clean and drop orphan job file idma_rtl_clean left the .vidma_ids stamp behind; remove it for hygiene (the stamp only triggers regen, so this was not a correctness bug). Also delete the empty, unreferenced jobs/backend_rw_axi/transpose_none.txt. --- idma.mk | 1 + jobs/backend_rw_axi/transpose_none.txt | 0 2 files changed, 1 insertion(+) delete mode 100644 jobs/backend_rw_axi/transpose_none.txt diff --git a/idma.mk b/idma.mk index db4d6618..accf12e3 100644 --- a/idma.mk +++ b/idma.mk @@ -170,6 +170,7 @@ idma_rtl_clean: rm -f $(IDMA_VSIM_DIR)/wave/*.do rm -f $(IDMA_RTL_DIR)/include/idma/tracer.svh rm -rf $(IDMA_RTL_DIR)/include/idma + rm -f $(IDMA_VIDMA_STAMP) # assemble the required files IDMA_INCLUDE_ALL += $(IDMA_RTL_DIR)/include/idma/tracer.svh diff --git a/jobs/backend_rw_axi/transpose_none.txt b/jobs/backend_rw_axi/transpose_none.txt deleted file mode 100644 index e69de29b..00000000 From fe534915dd6e5c33bb1768657857892abfbf23e6 Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 17 Jun 2026 12:02:43 +0200 Subject: [PATCH 12/13] test: Sweep transpose geometries inside the testbenches The transpose TBs took the M/N/EB geometry as elaboration-time parameters, so the makefile drove coverage with a long list of per-geometry vsim runs. M/N/EB are runtime loop bounds and addressing, not packed dimensions, so the sweep now lives in the TB: each self-checks a geometry list in one elaboration and the make target runs once per bus width, matching the single-vsim convention of the other self-checking testbenches. --- test/idma_transpose_dpi.c | 8 +- test/tb_idma_otf_transpose.sv | 161 ++++++++++++++++------------- test/tb_idma_transpose_b2b.sv | 121 ++++++++++++---------- test/tb_idma_transpose_nd.sv | 186 +++++++++++++++++++--------------- 4 files changed, 261 insertions(+), 215 deletions(-) diff --git a/test/idma_transpose_dpi.c b/test/idma_transpose_dpi.c index 6497a0c5..4082b21d 100644 --- a/test/idma_transpose_dpi.c +++ b/test/idma_transpose_dpi.c @@ -5,12 +5,8 @@ // Authors: // - Daniel Keller -// DPI-C golden model for idma_otf_transpose: an element-granular matrix -// transpose. Element size E in {1,2,4} bytes (int8/fp16/fp32); each E-byte -// element is kept intact while the M x N element grid is transposed to N x M. -// Accessor-based (no open-array marshalling): the testbench loads the row-major -// input byte by byte, calls gm_transpose(m,n,e), then reads back the transposed -// output bytes. Reference: out_elem[c][r] = in_elem[r][c]. +// DPI-C golden model for idma_otf_transpose: element-granular M x N -> N x M +// transpose, element size E in {1,2,4} B. Reference: out_elem[c][r] = in_elem[r][c]. #include diff --git a/test/tb_idma_otf_transpose.sv b/test/tb_idma_otf_transpose.sv index fc0950a7..d57c1f29 100644 --- a/test/tb_idma_otf_transpose.sv +++ b/test/tb_idma_otf_transpose.sv @@ -5,40 +5,36 @@ // Authors: // - Daniel Keller -// Standalone self-checking testbench for idma_otf_transpose (element transpose, -// no iDMA backend / no protocol deps). Verifies a full M x N transpose of -// EB-byte elements (EB in {1,2,4} = int8/fp16/fp32), split into NE-square tiles -// (NE = StrbWidth/EB): input padded to full tiles, fed in (col-tile, row-tile, -// row) order; output collected through the engine's per-byte strb_o edge mask. -// Expected result comes from the DPI-C golden (idma_transpose_dpi.c). Checks -// correctness vs golden, full coverage, and no strobe asserted on padding. -// Optional two-sided backpressure via +BP. Override with -gM= -gN= -gEB=. +// Standalone self-checking testbench for idma_otf_transpose: checks a full +// M x N transpose of EB-byte elements (EB in {1,2,4}) against the DPI-C golden +// (idma_transpose_dpi.c). Sweeps a geometry list internally; M/N/EB are runtime +// DUT inputs, so one elaboration covers all geometries for a given StrbWidth/ +// FullDuplex. Optional backpressure via +BP. `timescale 1ns/1ps module tb_idma_otf_transpose #( parameter int unsigned StrbWidth = 32'd8, - parameter bit FullDuplex = 1'b1, - parameter int unsigned M = 32'd8, // matrix rows (elements) - parameter int unsigned N = 32'd8, // matrix cols (elements) - parameter int unsigned EB = 32'd1 // element size in bytes (1/2/4) + parameter bit FullDuplex = 1'b1 ); import "DPI-C" function void gm_load(input int idx, input int val); import "DPI-C" function void gm_transpose(input int m, input int n, input int e); import "DPI-C" function int gm_get(input int idx); - localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; // log2(EB) - localparam int unsigned NE = StrbWidth / EB; // elements/beat = tile side - localparam int unsigned YT = (M + NE - 1) / NE; // row-tiles - localparam int unsigned NT = (N + NE - 1) / NE; // col-tiles - localparam int unsigned LR = M % NE; - localparam int unsigned LC = N % NE; - localparam logic [7:0] PAD = 8'hFF; + localparam logic [7:0] PAD = 8'hFF; + + // Geometry cases (M, N, EB); EB>StrbWidth cases skip. + localparam int unsigned NCases = 4; + localparam int unsigned Cases [NCases][3] = '{ '{13, 19, 1}, '{7, 5, 2}, '{5, 3, 4}, '{130, 70, 1} }; logic clk = 1'b0, rst_n = 1'b0, clear = 1'b0; always #5 clk = ~clk; + // runtime DUT control (held stable per case) + logic [1:0] mode_q; + logic [11:0] m_q, n_q; + logic [StrbWidth-1:0][7:0] din_data; logic din_valid, din_ready; logic [StrbWidth-1:0][7:0] dout_data; @@ -52,9 +48,9 @@ module tb_idma_otf_transpose #( .clk_i (clk), .rst_ni (rst_n), .clear_i (clear), - .transp_mode_i (2'(MODE)), - .tensor_size_m_i (12'(M)), - .tensor_size_n_i (12'(N)), + .transp_mode_i (mode_q), + .tensor_size_m_i (m_q), + .tensor_size_n_i (n_q), .data_i (din_data), .valid_i (din_valid), .ready_o (din_ready), @@ -64,24 +60,24 @@ module tb_idma_otf_transpose #( .ready_i (dout_ready) ); - logic [7:0] inb [M*N*EB]; // row-major input bytes - bit wrote [N*M]; // per transposed-element coverage - int unsigned errors = 0; - bit backpressure = 1'b0; + logic [7:0] inb []; // row-major input bytes (sized per case) + bit wrote []; // per transposed-element coverage (sized per case) + int unsigned errors = 0; // running total across all cases + bit backpressure = 1'b0; - task automatic drive_inputs(); + task automatic drive_inputs(input int unsigned m, n, eb, ne, yt, nt); int unsigned beat = 0; din_valid = 1'b0; din_data = '0; @(posedge clk); - for (int nt = 0; nt < NT; nt++) - for (int rt = 0; rt < YT; rt++) - for (int row = 0; row < NE; row++) begin + for (int unsigned ct = 0; ct < nt; ct++) + for (int unsigned rt = 0; rt < yt; rt++) + for (int unsigned row = 0; row < ne; row++) begin if (backpressure) begin din_valid = 1'b0; repeat (beat % 3) @(posedge clk); end - for (int c = 0; c < NE; c++) begin - int gr = rt*NE + row; - int gc = nt*NE + c; - for (int b = 0; b < EB; b++) - din_data[c*EB + b] = (gr < M && gc < N) ? inb[(gr*N + gc)*EB + b] : PAD; + for (int unsigned c = 0; c < ne; c++) begin + automatic int unsigned gr = rt*ne + row; + automatic int unsigned gc = ct*ne + c; + for (int unsigned b = 0; b < eb; b++) + din_data[c*eb + b] = (gr < m && gc < n) ? inb[(gr*n + gc)*eb + b] : PAD; end din_valid = 1'b1; do @(posedge clk); while (!din_ready); @@ -90,31 +86,31 @@ module tb_idma_otf_transpose #( din_valid = 1'b0; endtask - task automatic capture_outputs(); + task automatic capture_outputs(input int unsigned m, n, eb, ne, yt, nt); int unsigned beat = 0; dout_ready = 1'b0; - for (int nt = 0; nt < NT; nt++) - for (int rt = 0; rt < YT; rt++) - for (int k = 0; k < NE; k++) begin + for (int unsigned ct = 0; ct < nt; ct++) + for (int unsigned rt = 0; rt < yt; rt++) + for (int unsigned k = 0; k < ne; k++) begin if (backpressure) begin dout_ready = 1'b0; repeat (beat % 4) @(posedge clk); end dout_ready = 1'b1; do @(posedge clk); while (!dout_valid); - for (int e = 0; e < NE; e++) begin - if (dout_strb[e*EB]) begin // element e valid (element-granular mask) - int tr = nt*NE + k; // transposed row (= original col, 0..N-1) - int tc = rt*NE + e; // transposed col (= original row, 0..M-1) - if (tr >= N || tc >= M) begin + for (int unsigned e = 0; e < ne; e++) begin + if (dout_strb[e*eb]) begin // element e valid (element-granular mask) + automatic int unsigned tr = ct*ne + k; // transposed row (= original col, 0..n-1) + automatic int unsigned tc = rt*ne + e; // transposed col (= original row, 0..m-1) + if (tr >= n || tc >= m) begin errors++; - if (errors <= 16) $display("STRB-ON-PAD beat(nt%0d rt%0d k%0d) elem %0d -> (%0d,%0d) OOB", nt, rt, k, e, tr, tc); + if (errors <= 16) $display("STRB-ON-PAD beat(ct%0d rt%0d k%0d) elem %0d -> (%0d,%0d) OOB", ct, rt, k, e, tr, tc); end else begin - for (int b = 0; b < EB; b++) begin - automatic int gold = gm_get((tr*M + tc)*EB + b); - if (int'(dout_data[e*EB + b]) !== gold) begin + for (int unsigned b = 0; b < eb; b++) begin + automatic int gold = gm_get((tr*m + tc)*eb + b); + if (int'(dout_data[e*eb + b]) !== gold) begin errors++; - if (errors <= 16) $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", tr, tc, b, dout_data[e*EB+b], gold); + if (errors <= 16) $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", tr, tc, b, dout_data[e*eb+b], gold); end end - wrote[tr*M + tc] = 1'b1; + wrote[tr*m + tc] = 1'b1; end end end @@ -124,16 +120,43 @@ module tb_idma_otf_transpose #( dout_ready = 1'b0; endtask - initial begin - for (int i = 0; i < M*N*EB; i++) inb[i] = 8'((i * 7 + 3) & 8'hFF); // varied stimulus - for (int i = 0; i < M*N*EB; i++) gm_load(i, int'(inb[i])); - gm_transpose(M, N, EB); - for (int i = 0; i < N*M; i++) wrote[i] = 1'b0; + // Run one m x n transpose of eb-byte elements; returns the mismatch count. + task automatic run_case(input int unsigned m, n, eb, output int unsigned errs); + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned ne = StrbWidth / eb; + automatic int unsigned yt = (m + ne - 1) / ne; + automatic int unsigned nt = (n + ne - 1) / ne; + automatic int unsigned e0 = errors; + + inb = new[m*n*eb]; + wrote = new[n*m]; + for (int unsigned i = 0; i < m*n*eb; i++) begin + inb[i] = 8'((i * 7 + 3) & 8'hFF); + gm_load(i, int'(inb[i])); + end + gm_transpose(m, n, eb); + for (int unsigned i = 0; i < n*m; i++) wrote[i] = 1'b0; + + mode_q = 2'(mode); m_q = 12'(m); n_q = 12'(n); + // clear the engine between cases (resets banks / walkers) + clear = 1'b1; repeat (2) @(posedge clk); clear = 1'b0; @(posedge clk); + + $display("[TB] case M=%0d N=%0d EB=%0d (tile=%0d elems, %0dx%0d tiles) BP=%0d", m, n, eb, ne, yt, nt, backpressure); + fork drive_inputs(m, n, eb, ne, yt, nt); capture_outputs(m, n, eb, ne, yt, nt); join + + for (int unsigned tr = 0; tr < n; tr++) + for (int unsigned tc = 0; tc < m; tc++) + if (!wrote[tr*m + tc]) begin + errors++; + if (errors <= 16) $display("MISSING transposed elem (%0d,%0d)", tr, tc); + end + errs = errors - e0; + endtask - din_valid = 1'b0; dout_ready = 1'b0; + initial begin + automatic int unsigned total = 0, ce; + din_valid = 1'b0; dout_ready = 1'b0; mode_q = '0; m_q = '0; n_q = '0; if ($test$plusargs("BP")) backpressure = 1'b1; - $display("[TB] idma_otf_transpose M=%0d N=%0d EB=%0d (tile=%0d elems, %0dx%0d tiles, LR=%0d LC=%0d) BP=%0d", - M, N, EB, NE, YT, NT, LR, LC, backpressure); rst_n = 1'b0; clear = 1'b1; repeat (4) @(posedge clk); @@ -142,24 +165,22 @@ module tb_idma_otf_transpose #( clear = 1'b0; @(posedge clk); - fork drive_inputs(); capture_outputs(); join - - for (int tr = 0; tr < N; tr++) - for (int tc = 0; tc < M; tc++) - if (!wrote[tr*M + tc]) begin - errors++; - if (errors <= 16) $display("MISSING transposed elem (%0d,%0d)", tr, tc); - end - - if (errors == 0) $display("[TB] PASS: %0dx%0d EB=%0d transpose matches DPI golden", M, N, EB); - else $display("[TB] FAIL: %0d errors", errors); + for (int unsigned k = 0; k < NCases; k++) begin + if (Cases[k][2] > StrbWidth) continue; // element must fit the bus + run_case(Cases[k][0], Cases[k][1], Cases[k][2], ce); + if (ce == 0) $display("[TB] PASS: %0dx%0d EB=%0d transpose matches DPI golden", Cases[k][0], Cases[k][1], Cases[k][2]); + else $display("[TB] FAIL: %0dx%0d EB=%0d (%0d errors)", Cases[k][0], Cases[k][1], Cases[k][2], ce); + total += ce; + end + if (total == 0) $display("[TB] ALL PASS (%0d cases, StrbWidth=%0d, FullDuplex=%0d)", NCases, StrbWidth, FullDuplex); + else $display("[TB] FAIL: %0d total errors", total); repeat (5) @(posedge clk); $finish; end initial begin - #5000000; + #100000000; $display("[TB] FAIL: timeout"); $finish; end diff --git a/test/tb_idma_transpose_b2b.sv b/test/tb_idma_transpose_b2b.sv index 62c10c5a..bdb77f04 100644 --- a/test/tb_idma_transpose_b2b.sv +++ b/test/tb_idma_transpose_b2b.sv @@ -5,9 +5,10 @@ // Authors: // - Daniel Keller -// End-to-end back-to-back transpose regression: two transposes of one source to -// DIFFERENT dst bases through the ND midend -> rw_axi backend -> axi_sim_mem. A -// stale base across transfers would leave the second dst untouched. Both checked. +// End-to-end back-to-back transpose regression: per geometry, two transposes of +// one source to DIFFERENT dst bases through the ND midend -> rw_axi backend -> +// axi_sim_mem. A stale base across transfers would leave the second dst +// untouched. Sweeps a geometry list internally (one run per bus width). `include "axi/typedef.svh" `include "idma/typedef.svh" @@ -15,26 +16,22 @@ module tb_idma_transpose_b2b import idma_pkg::*; #( - parameter int unsigned DataWidth = 32, - parameter int unsigned AddrWidth = 32, - parameter int unsigned UserWidth = 1, + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, parameter int unsigned AxiIdWidth = 12, - parameter int unsigned TFLenWidth = 32, - parameter int unsigned M = 6, - parameter int unsigned N = 8, - parameter int unsigned EB = 1 + parameter int unsigned TFLenWidth = 32 ); localparam time TA = 1ns, TT = 9ns, TCK = 10ns; localparam int unsigned StrbWidth = DataWidth / 8; - localparam int unsigned NE = StrbWidth / EB; - localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; - localparam int unsigned YT = (M + NE - 1) / NE; - localparam int unsigned NT = (N + NE - 1) / NE; - localparam int unsigned MP = YT * NE; localparam int unsigned NumDim = 4; localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + // Geometry cases (M, N, EB); EB>StrbWidth cases skip. + localparam int unsigned NCases = 4; + localparam int unsigned Cases [NCases][3] = '{ '{6, 8, 1}, '{8, 8, 1}, '{13, 19, 1}, '{5, 5, 2} }; + typedef logic [AddrWidth-1:0] addr_t; typedef logic [DataWidth-1:0] data_t; typedef logic [StrbWidth-1:0] strb_t; @@ -135,16 +132,22 @@ module tb_idma_transpose_b2b return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; endfunction - // one transpose of the source at sb -> dst base `db`; returns error count - task automatic do_transpose(input addr_t db, output int unsigned errs); + // one m x n transpose of the source at sb -> dst base `db`; returns error count + task automatic do_transpose(input int unsigned m, input int unsigned n, input int unsigned eb, + input addr_t db, output int unsigned errs); + automatic int unsigned ne = StrbWidth / eb; + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned yt = (m + ne - 1) / ne; + automatic int unsigned nt = (n + ne - 1) / ne; + automatic int unsigned mp = yt * ne; errs = 0; // pre-fill full padded dst extent with sentinel - for (int unsigned i = 0; i < NT*NE; i++) - for (int unsigned j = 0; j < MP; j++) - for (int unsigned b = 0; b < EB; b++) - wr_mem(db + (i*MP + j)*EB + b, 8'hCC); + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(db + (i*mp + j)*eb + b, 8'hCC); nd_req = '0; - nd_req.burst_req.length = tf_len_t'(NE*EB); + nd_req.burst_req.length = tf_len_t'(ne*eb); nd_req.burst_req.src_addr = sb; nd_req.burst_req.dst_addr = db; nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; @@ -153,15 +156,15 @@ module tb_idma_transpose_b2b nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; nd_req.burst_req.opt.beo.decouple_rw = 1'b1; nd_req.burst_req.opt.beo.decouple_aw = 1'b1; - nd_req.burst_req.opt.compute.enable = 1'b1; - nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; - nd_req.burst_req.opt.compute.params.transpose.mode = 2'(MODE); - nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); - nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); nd_req.burst_req.opt.last = 1'b1; - nd_req.d_req[0].reps = reps_t'(NE); nd_req.d_req[0].src_strides = addr_t'(int'(N*EB)); nd_req.d_req[0].dst_strides = addr_t'(int'(MP*EB)); - nd_req.d_req[1].reps = reps_t'(YT); nd_req.d_req[1].src_strides = addr_t'(int'(N*EB)); nd_req.d_req[1].dst_strides = addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB)); - nd_req.d_req[2].reps = reps_t'(NT); nd_req.d_req[2].src_strides = addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB)); nd_req.d_req[2].dst_strides = addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB)); + nd_req.d_req[0].reps = reps_t'(ne); nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); + nd_req.d_req[1].reps = reps_t'(yt); nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); + nd_req.d_req[2].reps = reps_t'(nt); nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); nd_req_valid = 1'b1; do @(posedge clk); while (!nd_req_ready); // drop valid the cycle accept is seen (compliant) nd_req_valid = 1'b0; @@ -169,46 +172,52 @@ module tb_idma_transpose_b2b while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); repeat (20) @(posedge clk); // data + padding checks - for (int unsigned c = 0; c < N; c++) - for (int unsigned r = 0; r < M; r++) - for (int unsigned b = 0; b < EB; b++) - if (rd_mem(db + (c*MP + r)*EB + b) !== rd_mem(sb + (r*N + c)*EB + b)) begin - errs++; if (errs <= 8) $display("[B2BT] @db=%0h MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", db, c, r, b, rd_mem(db+(c*MP+r)*EB+b), rd_mem(sb+(r*N+c)*EB+b)); + for (int unsigned c = 0; c < n; c++) + for (int unsigned r = 0; r < m; r++) + for (int unsigned b = 0; b < eb; b++) + if (rd_mem(db + (c*mp + r)*eb + b) !== rd_mem(sb + (r*n + c)*eb + b)) begin + errs++; if (errs <= 8) $display("[B2BT] @db=%0h MISMATCH out_T[%0d][%0d].b%0d", db, c, r, b); end - for (int unsigned i = 0; i < NT*NE; i++) - for (int unsigned j = 0; j < MP; j++) - if (i >= N || j >= M) - for (int unsigned b = 0; b < EB; b++) - if (rd_mem(db + (i*MP + j)*EB + b) !== 8'hCC) begin + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + if (i >= n || j >= m) + for (int unsigned b = 0; b < eb; b++) + if (rd_mem(db + (i*mp + j)*eb + b) !== 8'hCC) begin errs++; if (errs <= 8) $display("[B2BT] @db=%0h PADDING CLOBBERED row=%0d col=%0d", db, i, j); end endtask initial begin - automatic int unsigned e1, e2; + automatic int unsigned total = 0, e1, e2; automatic addr_t db1 = 'h0000_4000; automatic addr_t db2 = 'h0000_8000; // DIFFERENT base — a stale-addr bug misplaces xfer 2 + automatic int unsigned m, n, eb; nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; @(posedge rst_n); repeat (5) @(posedge clk); - for (int unsigned r = 0; r < M; r++) - for (int unsigned c = 0; c < N; c++) - for (int unsigned b = 0; b < EB; b++) - wr_mem(sb + (r*N + c)*EB + b, 8'((( (r*N+c)*EB + b )*7 + 3) & 8'hFF)); - - $display("[B2BT] transfer 1 -> db=%0h", db1); - do_transpose(db1, e1); - $display("[B2BT] transfer 2 (back-to-back) -> db=%0h", db2); - do_transpose(db2, e2); - - if (e1 == 0 && e2 == 0) - $display("[B2BT] PASS: two back-to-back %0dx%0d EB=%0d transposes both correct (xfer2 landed at its own dst)", M, N, EB); - else - $fatal(1, "[B2BT] FAIL: xfer1 errs=%0d xfer2 errs=%0d", e1, e2); + + for (int unsigned k = 0; k < NCases; k++) begin + m = Cases[k][0]; n = Cases[k][1]; eb = Cases[k][2]; + if (eb > StrbWidth) continue; + // (re)init source for this geometry + for (int unsigned r = 0; r < m; r++) + for (int unsigned c = 0; c < n; c++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); + $display("[B2BT] %0dx%0d EB=%0d: xfer1 -> db=%0h, xfer2 -> db=%0h", m, n, eb, db1, db2); + do_transpose(m, n, eb, db1, e1); + do_transpose(m, n, eb, db2, e2); // back-to-back, distinct base + if (e1 == 0 && e2 == 0) $display("[B2BT] PASS: %0dx%0d EB=%0d both back-to-back transposes correct", m, n, eb); + else $display("[B2BT] FAIL: %0dx%0d EB=%0d xfer1=%0d xfer2=%0d", m, n, eb, e1, e2); + total += e1 + e2; + end + + if (total == 0) $display("[B2BT] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + else $fatal(1, "[B2BT] FAIL: %0d total mismatches", total); repeat (5) @(posedge clk); $finish(); end - initial begin #5_000_000; $fatal(1, "[B2BT] timeout"); end + initial begin #50_000_000; $fatal(1, "[B2BT] timeout"); end endmodule diff --git a/test/tb_idma_transpose_nd.sv b/test/tb_idma_transpose_nd.sv index 59d12b2d..86cf4c44 100644 --- a/test/tb_idma_transpose_nd.sv +++ b/test/tb_idma_transpose_nd.sv @@ -6,11 +6,9 @@ // - Daniel Keller // Self-checking multi-tile transpose testbench: idma_nd_midend (NumDim=4, -// transposed-stride program) -> idma_backend_rw_axi (EnableTranspose) -> -// axi_sim_mem. The ND midend generates the tiled read order (col-tile, -// row-tile, local-row) and the transposed destination placement the engine -// requires, so a full M x N transpose works end-to-end (not just one tile). -// Reference: out_T[c][r] = in[r][c] over E-byte elements. +// transposed-stride program) -> idma_backend_rw_axi -> axi_sim_mem, exercising +// a full M x N transpose end-to-end. Reference: out_T[c][r] = in[r][c]. +// Sweeps a list of geometries internally (one elaboration per bus width). `include "axi/typedef.svh" `include "idma/typedef.svh" @@ -18,17 +16,12 @@ module tb_idma_transpose_nd import idma_pkg::*; #( - parameter int unsigned DataWidth = 32, - parameter int unsigned AddrWidth = 32, - parameter int unsigned UserWidth = 1, - parameter int unsigned AxiIdWidth = 12, - parameter int unsigned TFLenWidth = 32, - parameter int unsigned M = 8, // matrix rows (elements) - parameter int unsigned N = 8, // matrix cols (elements) - parameter int unsigned EB = 1, // element size in bytes (1/2/4) - // 0 = auto: NumAxInFlight = StrbWidth (transpose needs >= NE in-flight; NE = StrbWidth at E=1) - parameter int unsigned NumAxInFlight = 0, - parameter int unsigned BufferDepth = 3 + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, + parameter int unsigned AxiIdWidth = 12, + parameter int unsigned TFLenWidth = 32, + parameter int unsigned BufferDepth = 3 ); localparam time TA = 1ns; @@ -36,16 +29,20 @@ module tb_idma_transpose_nd localparam time TCK = 10ns; localparam int unsigned StrbWidth = DataWidth / 8; - localparam int unsigned NE = StrbWidth / EB; // tile side (elements) - localparam int unsigned AxIF = (NumAxInFlight == 0) ? StrbWidth : NumAxInFlight; - localparam int unsigned MODE = (EB == 4) ? 2 : (EB == 2) ? 1 : 0; - localparam int unsigned YT = (M + NE - 1) / NE; // row-tiles - localparam int unsigned NT = (N + NE - 1) / NE; // col-tiles - // padded Aᵀ row pitch: M rounded up to a tile so every row is StrbWidth-aligned - localparam int unsigned MP = YT * NE; + // transpose buffers a full NE-beat tile before the first write; NE <= StrbWidth + localparam int unsigned AxIF = StrbWidth; localparam int unsigned NumDim = 4; // 1D + {row, row-tile, col-tile} localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + // Geometry cases (M, N, EB) swept in one elaboration: aligned + edge + // (M or N not a multiple of NE) for int8/fp16/fp32. EB>StrbWidth cases skip. + localparam int unsigned NCases = 13; + localparam int unsigned Cases [NCases][3] = '{ + '{ 8, 8, 1}, '{16, 16, 1}, '{16, 8, 1}, '{ 8, 8, 2}, '{ 6, 8, 1}, + '{ 8, 6, 1}, '{ 6, 6, 1}, '{ 5, 7, 1}, '{10, 6, 1}, '{ 5, 5, 2}, + '{32, 24, 1}, '{ 9, 5, 4}, '{13, 19, 1} + }; + // ── Types ── typedef logic [AddrWidth-1:0] addr_t; typedef logic [DataWidth-1:0] data_t; @@ -150,13 +147,14 @@ module tb_idma_transpose_nd addr_t sb = 'h0000_1000; addr_t db = 'h0000_4000; - // every AW (incl. wstrb=0 padding rows) must stay in the padded dst allocation - // [db, db+NT*NE*MP*EB) — else a strict slave would DECERR - always @(posedge clk) if (rst_n && axi_write_req.aw_valid && axi_write_rsp.aw_ready) begin - automatic addr_t aw_end = db + addr_t'(NT*NE*MP*EB); - if (axi_write_req.aw.addr < db || axi_write_req.aw.addr >= aw_end) + // every AW (incl. wstrb=0 padding rows) must stay in the active case's padded + // dst allocation [chk_db, chk_aw_hi) — else a strict slave would DECERR + logic chk_active = 1'b0; + addr_t chk_db, chk_aw_hi; + always @(posedge clk) if (rst_n && chk_active && axi_write_req.aw_valid && axi_write_rsp.aw_ready) begin + if (axi_write_req.aw.addr < chk_db || axi_write_req.aw.addr >= chk_aw_hi) $fatal(1, "[TPN] AW 0x%0h outside dst alloc [0x%0h,0x%0h) — would DECERR on a strict slave", - axi_write_req.aw.addr, db, aw_end); + axi_write_req.aw.addr, chk_db, chk_aw_hi); end task automatic wr_mem(input addr_t a, input logic [7:0] d); i_axi_sim_mem.mem[a] = d; endtask @@ -164,28 +162,36 @@ module tb_idma_transpose_nd return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; endfunction - initial begin - automatic int unsigned errs = 0; - nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; - @(posedge rst_n); - repeat (5) @(posedge clk); + // Run one M x N transpose of EB-byte elements; returns the mismatch count. + task automatic run_case(input int unsigned m, input int unsigned n, input int unsigned eb, + output int unsigned errs); + automatic int unsigned ne = StrbWidth / eb; // tile side (elements) + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned yt = (m + ne - 1) / ne; // row-tiles + automatic int unsigned nt = (n + ne - 1) / ne; // col-tiles + automatic int unsigned mp = yt * ne; // padded Aᵀ row pitch (StrbWidth-aligned) + errs = 0; - // init source matrix (row-major, M x N elements of EB bytes) - for (int unsigned r = 0; r < M; r++) - for (int unsigned c = 0; c < N; c++) - for (int unsigned b = 0; b < EB; b++) - wr_mem(sb + (r*N + c)*EB + b, 8'((( (r*N+c)*EB + b )*7 + 3) & 8'hFF)); + // init source matrix (row-major, m x n elements of eb bytes) + for (int unsigned r = 0; r < m; r++) + for (int unsigned c = 0; c < n; c++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); // sentinel-fill the full padded Aᵀ extent; padding cols/rows must stay sentinel - // (the engine strobe suppresses them) — checked after the transfer - for (int unsigned i = 0; i < NT*NE; i++) - for (int unsigned j = 0; j < MP; j++) - for (int unsigned b = 0; b < EB; b++) - wr_mem(db + (i*MP + j)*EB + b, 8'hCC); + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(db + (i*mp + j)*eb + b, 8'hCC); + + // arm the AW-bounds guard for this case + chk_db = db; + chk_aw_hi = db + addr_t'(nt*ne*mp*eb); + chk_active = 1'b1; // ── transposed-stride ND program (routing-plan §4.2) ── nd_req = '0; - nd_req.burst_req.length = tf_len_t'(NE*EB); // one tile-row = StrbWidth bytes + nd_req.burst_req.length = tf_len_t'(ne*eb); // one tile-row = StrbWidth bytes nd_req.burst_req.src_addr = sb; nd_req.burst_req.dst_addr = db; nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; @@ -196,68 +202,82 @@ module tb_idma_transpose_nd nd_req.burst_req.opt.beo.decouple_aw = 1'b1; nd_req.burst_req.opt.beo.src_max_llen = '0; nd_req.burst_req.opt.beo.dst_max_llen = '0; - nd_req.burst_req.opt.compute.enable = 1'b1; - nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; - nd_req.burst_req.opt.compute.params.transpose.mode = 2'(MODE); - nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); - nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); nd_req.burst_req.opt.last = 1'b1; // ND midend strides are INCREMENTAL deltas (added on dim roll-over), NOT - // absolute pitches. Aᵀ uses padded pitch MP*EB (aligned writes); src keeps - // N*EB (misaligned reads coalesce in the pre-engine buffer). - // d_req[0] = local row within tile (reps NE) - nd_req.d_req[0].reps = reps_t'(NE); - nd_req.d_req[0].src_strides = addr_t'(int'(N*EB)); // next source row - nd_req.d_req[0].dst_strides = addr_t'(int'(MP*EB)); // next Aᵀ row (padded pitch) - // d_req[1] = row-tile (reps YT) - nd_req.d_req[1].reps = reps_t'(YT); - nd_req.d_req[1].src_strides = addr_t'(int'(N*EB)); // rows are consecutive - nd_req.d_req[1].dst_strides = addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB)); // back up cols, next col-block - // d_req[2] = col-tile (reps NT). src rewind uses the padded row extent - // (YT*NE-1, not M-1): the read walks padding rows of the last row-tile. - nd_req.d_req[2].reps = reps_t'(NT); - nd_req.d_req[2].src_strides = addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB)); // back to padded row0, next col-block - nd_req.d_req[2].dst_strides = addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB)); // next Aᵀ col-block - - $display("[TPN] launching %0dx%0d EB=%0d transpose via ND midend (NE=%0d, %0dx%0d tiles)", M, N, EB, NE, YT, NT); + // absolute pitches. Aᵀ uses padded pitch mp*eb (aligned writes); src keeps + // n*eb (misaligned reads coalesce in the pre-engine buffer). + nd_req.d_req[0].reps = reps_t'(ne); + nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); + nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); + nd_req.d_req[1].reps = reps_t'(yt); + nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); + nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); + nd_req.d_req[2].reps = reps_t'(nt); + nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); + nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); + + $display("[TPN] case %0dx%0d EB=%0d (NE=%0d, %0dx%0d tiles)", m, n, eb, ne, yt, nt); nd_req_valid = 1'b1; // drop valid on accept; holding it one cycle past makes the midend re-walk the request do @(posedge clk); while (!nd_req_ready); nd_req_valid = 1'b0; nd_req = '0; - // wait for ND completion + // wait for ND completion + drain while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); repeat (20) @(posedge clk); + chk_active = 1'b0; - // check 1 (data): out_T[c][r] == in[r][c], Aᵀ at padded pitch MP - for (int unsigned c = 0; c < N; c++) - for (int unsigned r = 0; r < M; r++) - for (int unsigned b = 0; b < EB; b++) begin - automatic logic [7:0] got = rd_mem(db + (c*MP + r)*EB + b); - automatic logic [7:0] exp = rd_mem(sb + (r*N + c)*EB + b); + // check 1 (data): out_T[c][r] == in[r][c], Aᵀ at padded pitch mp + for (int unsigned c = 0; c < n; c++) + for (int unsigned r = 0; r < m; r++) + for (int unsigned b = 0; b < eb; b++) begin + automatic logic [7:0] got = rd_mem(db + (c*mp + r)*eb + b); + automatic logic [7:0] exp = rd_mem(sb + (r*n + c)*eb + b); if (got !== exp) begin errs++; if (errs <= 12) $display("[TPN] MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", c, r, b, got, exp); end end - // check 2: padding cols [M,MP) and padding rows [N,NT*NE) must stay sentinel (strobe-suppressed) - for (int unsigned i = 0; i < NT*NE; i++) - for (int unsigned j = 0; j < MP; j++) - if (i >= N || j >= M) - for (int unsigned b = 0; b < EB; b++) begin - automatic logic [7:0] got = rd_mem(db + (i*MP + j)*EB + b); + // check 2: padding cols [m,mp) and padding rows [n,nt*ne) must stay sentinel + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + if (i >= n || j >= m) + for (int unsigned b = 0; b < eb; b++) begin + automatic logic [7:0] got = rd_mem(db + (i*mp + j)*eb + b); if (got !== 8'hCC) begin errs++; if (errs <= 12) $display("[TPN] PADDING CLOBBERED at row=%0d col=%0d b%0d=%02h (exp CC)", i, j, b, got); end end - if (errs == 0) $display("[TPN] PASS: %0dx%0d EB=%0d multi-tile transpose matches golden (padding intact)", M, N, EB); - else $fatal(1, "[TPN] FAIL: %0d mismatches", errs); + endtask + + initial begin + automatic int unsigned total = 0; + automatic int unsigned ce; + nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + @(posedge rst_n); + repeat (5) @(posedge clk); + + for (int unsigned k = 0; k < NCases; k++) begin + if (Cases[k][2] > StrbWidth) continue; // element must fit the bus + run_case(Cases[k][0], Cases[k][1], Cases[k][2], ce); + if (ce == 0) $display("[TPN] PASS: %0dx%0d EB=%0d", Cases[k][0], Cases[k][1], Cases[k][2]); + else $display("[TPN] FAIL: %0dx%0d EB=%0d (%0d mismatches)", Cases[k][0], Cases[k][1], Cases[k][2], ce); + total += ce; + end + + if (total == 0) $display("[TPN] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + else $fatal(1, "[TPN] FAIL: %0d total mismatches", total); repeat (5) @(posedge clk); $finish(); end - initial begin #5_000_000; $fatal(1, "[TPN] timeout"); end + initial begin #100_000_000; $fatal(1, "[TPN] timeout"); end endmodule From 5a47a212b97ed39bb351a14205995704231778d0 Mon Sep 17 00:00:00 2001 From: Daniel Keller Date: Wed, 17 Jun 2026 12:02:44 +0200 Subject: [PATCH 13/13] build: Drop split_rtl prototype flow and address review comments Remove the split_rtl Bender target and -t split_rtl from the compile scripts: it existed only for the hand-edited transpose prototype, and the generated bundle (idma_generated.sv) carries the compute-enabled rw_axi when IDMA_VIDMA_IDS=rw_axi (the transpose sim targets opt in). Also remove a duplicate idma_sim_tb_idma_rt_midend target, drop the internal 'write seam' wording, and trim verbose comments. --- Bender.yml | 62 +-------------------- idma.mk | 62 ++++++--------------- src/backend/idma_axi_write.sv | 2 +- src/backend/idma_otf_compute.sv | 4 +- src/backend/idma_otf_transpose.sv | 2 +- src/backend/tpl/idma_transport_layer.sv.tpl | 3 +- src/idma_pkg.sv | 6 +- src/midend/idma_transpose_midend.sv | 8 +-- util/mario/backend.py | 2 +- 9 files changed, 32 insertions(+), 119 deletions(-) diff --git a/Bender.yml b/Bender.yml index 0b158979..2049f69e 100644 --- a/Bender.yml +++ b/Bender.yml @@ -56,62 +56,10 @@ sources: - src/backend/idma_tilelink_read.sv - src/backend/idma_tilelink_write.sv - # Generated content (bundled single-file; default flow) - - target: all(rtl, not(split_rtl)) + - target: rtl files: - target/rtl/idma_generated.sv - # Generated content (per-variant files; opt-in via `-t split_rtl`). - # Used for the transpose-engine prototype where the rw_axi backend/transport/ - # legalizer are hand-edited in place before being ported back to templates. - # Mutually exclusive with idma_generated.sv to avoid duplicate module defs. - - target: all(rtl, split_rtl) - files: - - target/rtl/idma_transport_layer_rw_axi.sv - - target/rtl/idma_transport_layer_r_obi_w_axi.sv - - target/rtl/idma_transport_layer_r_axi_w_obi.sv - - target/rtl/idma_transport_layer_rw_axi_rw_axis.sv - - target/rtl/idma_transport_layer_rw_obi.sv - - target/rtl/idma_transport_layer_r_obi_rw_init_w_axi.sv - - target/rtl/idma_transport_layer_r_axi_rw_init_rw_obi.sv - - target/rtl/idma_legalizer_rw_axi.sv - - target/rtl/idma_legalizer_r_obi_w_axi.sv - - target/rtl/idma_legalizer_r_axi_w_obi.sv - - target/rtl/idma_legalizer_rw_axi_rw_axis.sv - - target/rtl/idma_legalizer_rw_obi.sv - - target/rtl/idma_legalizer_r_obi_rw_init_w_axi.sv - - target/rtl/idma_legalizer_r_axi_rw_init_rw_obi.sv - - target/rtl/idma_backend_rw_axi.sv - - target/rtl/idma_backend_r_obi_w_axi.sv - - target/rtl/idma_backend_r_axi_w_obi.sv - - target/rtl/idma_backend_rw_axi_rw_axis.sv - - target/rtl/idma_backend_rw_obi.sv - - target/rtl/idma_backend_r_obi_rw_init_w_axi.sv - - target/rtl/idma_backend_r_axi_rw_init_rw_obi.sv - - target/rtl/idma_backend_synth_rw_axi.sv - - target/rtl/idma_backend_synth_r_obi_w_axi.sv - - target/rtl/idma_backend_synth_r_axi_w_obi.sv - - target/rtl/idma_backend_synth_rw_axi_rw_axis.sv - - target/rtl/idma_backend_synth_rw_obi.sv - - target/rtl/idma_backend_synth_r_obi_rw_init_w_axi.sv - - target/rtl/idma_backend_synth_r_axi_rw_init_rw_obi.sv - - target/rtl/idma_desc64_reg_pkg.sv - - target/rtl/idma_reg32_3d_reg_pkg.sv - - target/rtl/idma_reg64_2d_reg_pkg.sv - - target/rtl/idma_reg64_1d_reg_pkg.sv - - target/rtl/idma_desc64_reg_top.sv - - target/rtl/idma_reg32_3d_reg_top.sv - - target/rtl/idma_reg64_2d_reg_top.sv - - target/rtl/idma_reg64_1d_reg_top.sv - - target/rtl/idma_desc64_addrmap_pkg.sv - - target/rtl/idma_reg32_3d_addrmap_pkg.sv - - target/rtl/idma_reg64_2d_addrmap_pkg.sv - - target/rtl/idma_reg64_1d_addrmap_pkg.sv - - target/rtl/idma_desc64_top.sv - - target/rtl/idma_reg32_3d_top.sv - - target/rtl/idma_reg64_2d_top.sv - - target/rtl/idma_reg64_1d_top.sv - # Midends - target: rtl files: @@ -149,7 +97,6 @@ sources: # Level 2 - src/frontend/desc64/idma_desc64_top.sv - # Synthesis wrappers - target: synth files: @@ -186,11 +133,8 @@ sources: - target: idma_test files: - target/rtl/tb_idma_generated.sv - # Multi-tile transpose path: ND-midend (transposed strides) -> backend - - target: all(idma_test, split_rtl) - files: - - test/tb_idma_transpose_nd.sv - - test/tb_idma_transpose_b2b.sv + - test/tb_idma_transpose_nd.sv + - test/tb_idma_transpose_b2b.sv # Multi-head directed backend testbenches - target: multihead diff --git a/idma.mk b/idma.mk index accf12e3..2a9d1523 100644 --- a/idma.mk +++ b/idma.mk @@ -324,7 +324,7 @@ IDMA_PICKLE_ALL += $(IDMA_PICKLE_DIR)/idma_mp_midend_synth.sv $(IDMA_VSIM_DIR)/compile_%.tcl: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) $(IDMA_WAVE_ALL) echo 'set ROOT [file normalize [file dirname [info script]]/../../..]' > $@ set -o pipefail; $(BENDER) script vsim --vlog-arg="$(IDMA_VLOG_ARGS)" \ - -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster -t split_rtl \ + -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster \ --top $* | grep -v "set ROOT" >> $@ echo >> $@ @@ -347,14 +347,7 @@ define idma_generate_vsim endef $(IDMA_VSIM_DIR)/compile.tcl: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) $(IDMA_WAVE_ALL) - $(call idma_generate_vsim, $@, -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster -t split_rtl,../../..) - -.PHONY: idma_sim_tb_idma_rt_midend - -idma_sim_tb_idma_rt_midend: $(IDMA_VSIM_DIR)/compile.tcl - cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc \ - tb_idma_rt_midend -do "run -all; quit" + $(call idma_generate_vsim, $@, -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster,../../..) # Standalone self-checking transpose-engine regression (DPI-C golden, no backend deps). # Run with the Questa SEPP wrapper, e.g.: @@ -370,43 +363,23 @@ idma_sim_tb_idma_otf_transpose: cd $(IDMA_OTF_TP_DIR); $(VLIB) work cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv $(IDMA_OTF_TP_DPI) cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv -svinputport=compat -timescale "1ns/1fs" $(IDMA_OTF_TP_RTL) $(IDMA_OTF_TP_TB) - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=13 -gN=19 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=7 -gN=5 -gEB=2 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gM=5 -gN=3 -gEB=4 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gM=130 -gN=70 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=8 -gM=13 -gN=19 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=8 -gM=7 -gN=5 -gEB=2 tb_idma_otf_transpose +BP -do "run -all; quit" - cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gFullDuplex=0 -gStrbWidth=64 -gM=130 -gN=70 -gEB=1 tb_idma_otf_transpose +BP -do "run -all; quit" - -# Multi-tile transpose via the ND midend (transposed strides) -> rw_axi backend -# (engine spliced at the write seam) -> axi_sim_mem. Covers aligned and edge -# (M or N not a multiple of NE) geometries for int8/fp16/fp32. Needs the -# split_rtl flow (per-variant routing). Run with the Questa SEPP wrapper: -# make idma_sim_tb_idma_transpose_nd VSIM="questa-2023.4 vsim" -# These tests need the write-seam engine: build rw_axi with compute (stamp regens) + # the TB sweeps the geometry list internally; one run per StrbWidth x FullDuplex + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gFullDuplex=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gFullDuplex=0 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gFullDuplex=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gFullDuplex=0 tb_idma_otf_transpose +BP -do "run -all; quit" + +# Multi-tile transpose via the ND midend -> rw_axi backend -> axi_sim_mem. +# Run with the Questa SEPP wrapper: make idma_sim_tb_idma_transpose_nd VSIM="questa-2023.4 vsim" +# These tests need the compute engine: build rw_axi with compute (stamp regens). idma_sim_tb_idma_transpose_nd idma_sim_tb_idma_transpose_b2b: IDMA_VIDMA_IDS := rw_axi .PHONY: idma_sim_tb_idma_transpose_nd idma_sim_tb_idma_transpose_nd: $(IDMA_VSIM_DIR)/compile.tcl cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" - # ── aligned (regression: M,N multiples of NE) ── - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=16 -gN=16 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=16 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=32 -gN=24 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=2 tb_idma_transpose_nd -do "run -all; quit" - # ── edge: partial output cols only (M%NE!=0, N%NE==0; within-beat wstrb) ── - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=8 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - # ── edge: partial output rows only (N%NE!=0; zero-strobe drain beats) ── - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - # ── edge: both partial (int8) ── - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=7 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=10 -gN=6 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" - # ── edge: fp16 (EB=2) and fp32 (EB=4) ── - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=5 -gEB=2 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=9 -gN=5 -gEB=4 tb_idma_transpose_nd -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=13 -gN=19 -gEB=1 tb_idma_transpose_nd -do "run -all; quit" + # the TB sweeps the geometry list internally; one run per bus width + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_transpose_nd -do "run -all; quit" # Back-to-back regressions: the ND midend must reload each new transfer's base # address (it does, for a protocol-compliant producer that drops nd_req_valid on @@ -421,10 +394,9 @@ idma_sim_tb_idma_nd_midend_b2b: $(IDMA_VSIM_DIR)/compile.tcl .PHONY: idma_sim_tb_idma_transpose_b2b idma_sim_tb_idma_transpose_b2b: $(IDMA_VSIM_DIR)/compile.tcl cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=6 -gN=8 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=8 -gN=8 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 -gM=13 -gN=19 -gEB=1 tb_idma_transpose_b2b -do "run -all; quit" - cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 -gM=5 -gN=5 -gEB=2 tb_idma_transpose_b2b -do "run -all; quit" + # the TB sweeps the geometry list internally; one run per bus width + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_transpose_b2b -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_transpose_b2b -do "run -all; quit" .PHONY: idma_sim_tb_idma_rt_midend diff --git a/src/backend/idma_axi_write.sv b/src/backend/idma_axi_write.sv index faa309db..b0b40d0b 100644 --- a/src/backend/idma_axi_write.sv +++ b/src/backend/idma_axi_write.sv @@ -145,7 +145,7 @@ module idma_axi_write #( if (w_dp_req_i.tailer != '0 & last_w) begin mask_out = mask_out & w_last_mask; end - // external mask: some bytes may be masked by an OTF engine + // external mask (OTF compute) mask_out = mask_out & mask_ext_i; end diff --git a/src/backend/idma_otf_compute.sv b/src/backend/idma_otf_compute.sv index 88bfff32..4ad8a5af 100644 --- a/src/backend/idma_otf_compute.sv +++ b/src/backend/idma_otf_compute.sv @@ -5,8 +5,8 @@ // Authors: // - Daniel Keller -/// On-the-fly compute dispatcher at the transport write seam: latches the -/// per-transfer compute options and dispatches one op per transfer to its sub-unit. +/// On-the-fly compute dispatcher: latches the per-transfer compute options +/// and dispatches one op per transfer to its sub-unit. module idma_otf_compute #( /// Byte lanes per beat (= DataWidth/8) parameter int unsigned StrbWidth = 32'd8, diff --git a/src/backend/idma_otf_transpose.sv b/src/backend/idma_otf_transpose.sv index 5d3c8004..efe93155 100644 --- a/src/backend/idma_otf_transpose.sv +++ b/src/backend/idma_otf_transpose.sv @@ -6,7 +6,7 @@ // - Daniel Keller // On-the-fly matrix transpose engine (ping-pong tile banks) for the iDMA -// transport write seam. Adapted from the datamover (Ratha) HWPE: +// transport datapath. Adapted from the datamover (Ratha) HWPE: // pulp-platform/datamover@d58a985, rtl/datamover_engine.sv. // // Contract: input padded to full tiles, fed (col-tile, row-tile, row) order; diff --git a/src/backend/tpl/idma_transport_layer.sv.tpl b/src/backend/tpl/idma_transport_layer.sv.tpl index 7f27de83..6219855e 100644 --- a/src/backend/tpl/idma_transport_layer.sv.tpl +++ b/src/backend/tpl/idma_transport_layer.sv.tpl @@ -229,7 +229,6 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, byte_t [2*StrbWidth-1:0] buffer_out_tmp; byte_t [StrbWidth-1:0] buffer_out; byte_t [StrbWidth-1:0] buffer_out_shifted; - // compute write seam (passthrough when compute is off) byte_t [StrbWidth-1:0] wr_data; strb_t wr_valid, wr_strb, mask_ext_shifted, dataflow_ready_in; logic w_beat_done; @@ -382,7 +381,7 @@ ${rendered_read_ports[read_port]} ); //-------------------------------------- - // On-the-fly compute (write seam) + // On-the-fly compute //-------------------------------------- % if enable_compute: diff --git a/src/idma_pkg.sv b/src/idma_pkg.sv index 92c1bcf9..920740d5 100644 --- a/src/idma_pkg.sv +++ b/src/idma_pkg.sv @@ -81,16 +81,16 @@ package idma_pkg; logic dst_reduce_len; } backend_options_t; - /// On-the-fly compute operation selector + /// Compute operation selector typedef enum logic [3:0] { COMPUTE_NONE = 4'd0, COMPUTE_TRANSPOSE = 4'd1 } compute_op_e; - /// Transpose tensor dimension width in elements (bound by the inst64 DMCPY argb encoding) + /// Transpose tensor dimension width (elements) localparam int unsigned TransposeDimWidth = 32'd12; - /// Transpose option type: E = 1< 1/2/4 B); tensor in elements + /// Transpose options (E = 1< -/// Transpose geometry expander: turns an opt.compute=TRANSPOSE request into a -/// NumDim=4 tiled ND walk (row / row-tile / col-tile) from (M, N, mode) and -/// StrbWidth, for the generic idma_nd_midend to walk. Non-transpose passes -/// through. Reads to tile-padded bounds; writes the padded dst strb-masked. -/// Combinational, quasi-static per request (multicycle-safe in STA). +/// Transpose geometry expander: expands an opt.compute=TRANSPOSE request into a +/// NumDim=4 tiled ND walk for the generic idma_nd_midend. Non-transpose passes +/// through. Combinational, quasi-static per request. module idma_transpose_midend #( /// Number of ND dimensions (must be >= 4 to express the tiled walk) parameter int unsigned NumDim = 32'd4, diff --git a/util/mario/backend.py b/util/mario/backend.py index daa3e934..ff527926 100644 --- a/util/mario/backend.py +++ b/util/mario/backend.py @@ -42,7 +42,7 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = srp = len(used_read_prots) == 1 and not any_mh_r swp = len(used_write_prots) == 1 and not any_mh_w - # on-the-fly compute is hosted at the (single) AXI write seam + # on-the-fly compute requires a single AXI write port enable_compute = prot_id in (compute_cfg or {}) if enable_compute and not (swp and used_write_prots[0] == 'axi'): raise ValueError(