inst64: Peak-bandwidth transpose via the FF engine on the multi-write backend

src/backend/idma_obi_write.sv

@@ -66,7 +66,11 @@ module idma_obi_write #(
     /// Valid from buffer
     input  strb_t buffer_out_valid_i,
     /// Ready to buffer
-    output strb_t buffer_out_ready_o
+    output strb_t buffer_out_ready_o,
+    /// External (compute) byte mask ANDed into the write strobe ('1 when unused)
+    input  strb_t mask_ext_i,
+    /// One write beat was accepted on the bus (strobe-independent retire pulse)
+    output logic  w_beat_done_o
 );
     // corresponds to the strobe: the write aligned data that is currently valid in the buffer
     strb_t mask_out;
@@ -91,7 +95,7 @@ module idma_obi_write #(
 
     assign mask_out = ('1 << w_dp_req_i.offset) &
         ((w_dp_req_i.tailer != '0) ? ('1 >> (StrbWidth - w_dp_req_i.tailer))
-        : '1);
+        : '1) & mask_ext_i;
 
     //--------------------------------------
     // Write control
@@ -106,6 +110,9 @@ module idma_obi_write #(
     // write happening: both the bus (w_ready) and the buffer (ready_to_write) is high
     assign write_happening = ready_to_write & write_rsp_i.gnt;
 
+    // strobe-independent beat-accept pulse for the compute engine retire (pulses even for be='0)
+    assign w_beat_done_o = write_happening;
+
     // the main buffer is conditionally to the write mask popped
     assign buffer_out_ready_o = write_happening ? mask_out : '0;
 

src/backend/tpl/idma_backend.sv.tpl

@@ -873,10 +873,16 @@ w_req.decouple_aw || (w_req.w_dp_req.dst_protocol inside {\
 % endif
     end else begin : gen_r_aw_bypass
 % if combined_aw_and_w:
+    % if enable_compute:
+        // combined aw+w read-meta buffer; deepened for the compute engine tile read-ahead (cf. i_w_dp_req)
+        stream_fifo_optimal_wrap #(
+            .Depth        ( NumAxInFlight + ComputeFifoDepth ),
+    % else:
         // Atleast one write protocol uses combined aw and w -> Need to buffer read meta requests
         // As a write could depend on up to two reads
         stream_fifo_optimal_wrap #(
             .Depth        ( 2                    ),
+    % endif
             .type_t       (\
     % if one_write_port:
  write_meta_channel_t ),

src/backend/tpl/idma_transport_layer.sv.tpl

@@ -231,7 +231,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i,
     byte_t [StrbWidth-1:0] buffer_out_shifted;
     byte_t [StrbWidth-1:0] wr_data;
     strb_t                 wr_valid, wr_strb, mask_ext_shifted, dataflow_ready_in;
-% if one_write_port:
+% if one_write_port or enable_compute:
     logic                  w_beat_done;
 % endif
 
@@ -489,6 +489,22 @@ ${rendered_read_ports[read_port]}
         endcase
     end
 
+% if enable_compute:
+    // route the active write port's beat-done to the compute engine retire
+    always_comb begin : gen_write_beat_done_mux
+        case(w_dp_req_i.dst_protocol)
+% for wp in used_write_protocols:
+    % if wp in ('axi', 'obi') and mh_format['aw'][wp] == '':
+        idma_pkg::${database[wp]['protocol_enum']}: w_beat_done = ${wp}_w_beat_done;
+    % elif wp in ('axi', 'obi'):
+        idma_pkg::${database[wp]['protocol_enum']}: w_beat_done = ${wp}_w_beat_done [w_dp_req_i.dst_head];
+    % endif
+% endfor
+        default: w_beat_done = 1'b0;
+        endcase
+    end
+
+% endif
 % endif
     //--------------------------------------
     // Write Ports

src/db/idma_obi.yml

@@ -117,7 +117,9 @@ write_template: |
         .write_rsp_i        ( ${write_response} ),
         .buffer_out_i       ( buffer_out_shifted ),
         .buffer_out_valid_i ( buffer_out_valid_shifted ),
-        .buffer_out_ready_o ( ${buffer_out_ready} )
+        .buffer_out_ready_o ( ${buffer_out_ready} ),
+        .mask_ext_i         ( mask_ext_shifted ),
+        .w_beat_done_o      ( ${w_beat_done} )
     );
 synth_wrapper_ports_write: |
     output logic                   obi_write_req_a_req_o,

src/frontend/inst64/idma_inst64_top.sv

@@ -22,6 +22,7 @@ module idma_inst64_top #(
     parameter int unsigned NumChannels     = 32'd1,
     parameter bit          TCDMAliasEnable = 1'b0,
     parameter int unsigned DMATracing      = 32'd0,
+    parameter idma_pkg::compute_enable_t ComputeEnable = '0,
     parameter type         axi_ar_chan_t   = logic,
     parameter type         axi_aw_chan_t   = logic,
     parameter type         axi_req_t       = logic,
@@ -70,7 +71,7 @@ module idma_inst64_top #(
     localparam int unsigned TfIdWidth    = 32'd32;
     localparam int unsigned TFLenWidth   = AxiAddrWidth;
     localparam int unsigned RepWidth     = 32'd32;
-    localparam int unsigned NumDim       = 32'd2;
+    localparam int unsigned NumDim       = ComputeEnable.transpose ? 32'd4 : 32'd2;
     localparam int unsigned BufferDepth  = 32'd3;
     localparam int unsigned NumRules     = 32'd5;
 
@@ -84,7 +85,7 @@ module idma_inst64_top #(
     localparam type id_t                 = logic[AxiIdWidth-1:0];
     localparam type tf_len_t             = logic[TFLenWidth-1:0];
     localparam type offset_t             = logic[OffsetWidth-1:0];
-    localparam type strides_t            = logic[RepWidth-1:0];
+    localparam type strides_t            = addr_t;
     localparam type reps_t               = logic[RepWidth-1:0];
     localparam type tf_id_t              = logic[TfIdWidth-1:0];
 
@@ -178,6 +179,7 @@ module idma_inst64_top #(
     logic [1:0] idma_fe_status;
     logic [2:0] idma_fe_sel_chan;
     logic       idma_fe_twod;
+    logic       idma_fe_tp_reject;
 
     // busy signals
     idma_pkg::idma_busy_t [NumChannels-1:0] idma_busy;
@@ -348,7 +350,7 @@ module idma_inst64_top #(
             .idma_req_t    ( idma_req_t    ),
             .idma_rsp_t    ( idma_rsp_t    ),
             .idma_nd_req_t ( idma_nd_req_t ),
-            .RepWidths     ( RepWidth      )
+            .RepWidths     ( {NumDim{RepWidth}} )
         ) i_idma_nd_midend (
             .clk_i,
             .rst_ni,
@@ -367,6 +369,31 @@ module idma_inst64_top #(
             .busy_o            ( idma_nd_busy      [c] )
         );
 
+        // FIFO output, before transpose expansion
+        idma_nd_req_t fifo_nd_req;
+        logic         fifo_nd_valid, fifo_nd_ready;
+
+        // expand a transpose into the engine's tiled walk; padded leading dim MP = ceil(M/NE)*NE
+        if (ComputeEnable.transpose) begin : gen_transpose
+            idma_transpose_midend #(
+                .NumDim        ( NumDim        ),
+                .StrbWidth     ( StrbWidth     ),
+                .addr_t        ( addr_t        ),
+                .idma_nd_req_t ( idma_nd_req_t )
+            ) i_idma_transpose_midend (
+                .nd_req_i ( fifo_nd_req           ),
+                .valid_i  ( fifo_nd_valid         ),
+                .ready_o  ( fifo_nd_ready         ),
+                .nd_req_o ( idma_nd_req       [c] ),
+                .valid_o  ( idma_nd_req_valid [c] ),
+                .ready_i  ( idma_nd_req_ready [c] )
+            );
+        end else begin : gen_no_transpose
+            assign idma_nd_req       [c] = fifo_nd_req;
+            assign idma_nd_req_valid [c] = fifo_nd_valid;
+            assign fifo_nd_ready         = idma_nd_req_ready [c];
+        end
+
         stream_fifo_optimal_wrap #(
             .Depth     ( DMAReqFifoDepth ),
             .type_t    ( idma_nd_req_t   ),
@@ -380,9 +407,9 @@ module idma_inst64_top #(
             .data_i     ( idma_fe_req           ),
             .valid_i    ( idma_fe_req_valid [c] ),
             .ready_o    ( idma_fe_req_ready [c] ),
-            .data_o     ( idma_nd_req       [c] ),
-            .valid_o    ( idma_nd_req_valid [c] ),
-            .ready_i    ( idma_nd_req_ready [c] )
+            .data_o     ( fifo_nd_req           ),
+            .valid_o    ( fifo_nd_valid         ),
+            .ready_i    ( fifo_nd_ready         )
         );
     end
 
@@ -519,10 +546,12 @@ module idma_inst64_top #(
         idma_fe_req_d.burst_req.opt.beo.src_reduce_len = 1'b0;
         idma_fe_req_d.burst_req.opt.beo.dst_reduce_len = 1'b0;
         idma_fe_req_d.burst_req.opt.last               = 1'b0;
+        idma_fe_req_d.burst_req.opt.compute            = '0;
 
         // frontend config
         idma_fe_cfg      = '0;
         idma_fe_status   = '0;
+        idma_fe_tp_reject = 1'b0;
         idma_fe_sel_chan = '0;
 
         // default handshaking
@@ -573,6 +602,30 @@ module idma_inst64_top #(
                         idma_inst64_snitch_pkg::DMCPY : begin
                             idma_fe_cfg      = acc_req_i.data_argb[1:0];
                             idma_fe_sel_chan = acc_req_i.data_argb[4:2];
+                            // transpose: argb = {enable, mode, tensor_m, tensor_n}
+                            if (ComputeEnable.transpose && acc_req_i.data_argb[5]) begin
+                                idma_fe_req_d.burst_req.opt.compute.enable = 1'b1;
+                                idma_fe_req_d.burst_req.opt.compute.op     =
+                                    idma_pkg::COMPUTE_TRANSPOSE;
+                                idma_fe_req_d.burst_req.opt.compute.params.transpose.mode     =
+                                    acc_req_i.data_argb[7:6];
+                                idma_fe_req_d.burst_req.opt.compute.params.transpose.tensor_m =
+                                    acc_req_i.data_argb[19:8];
+                                idma_fe_req_d.burst_req.opt.compute.params.transpose.tensor_n =
+                                    acc_req_i.data_argb[31:20];
+                            end
+                            // reject bad transpose: off, mode 3, zero dim, twod, unaligned
+                            if (acc_req_i.data_argb[5]) begin
+                                idma_fe_tp_reject = !ComputeEnable.transpose
+                                    | (acc_req_i.data_argb[7:6] == 2'd3)
+                                    | (acc_req_i.data_argb[19:8] == '0)
+                                    | (acc_req_i.data_argb[31:20] == '0)
+                                    | acc_req_i.data_argb[1]
+                                    | (|(idma_fe_req_d.burst_req.src_addr
+                                         & addr_t'((32'd1 << acc_req_i.data_argb[7:6]) - 32'd1)))
+                                    | (|(idma_fe_req_d.burst_req.dst_addr
+                                         & addr_t'((32'd1 << acc_req_i.data_argb[7:6]) - 32'd1)));
+                            end
                         end
                         default:;
                     endcase
@@ -588,7 +641,15 @@ module idma_inst64_top #(
                     // 3. wait for twod transfer to be accepted (ready)
                     // 4. send acc response (pvalid)
                     // 5. acknowledge acc request (qready)
-                    if (acc_res_ready) begin
+                    // DMCPY launch; transpose requests reject malformed configs
+                    if (idma_fe_tp_reject) begin
+                        // error response; the transfer is not launched
+                        if (acc_res_ready) begin
+                            acc_res.id      = acc_req_i.id;
+                            acc_res_valid   = 1'b1;
+                            acc_req_ready_o = 1'b1;
+                        end
+                    end else if (acc_res_ready) begin
                         idma_fe_req_valid[idma_fe_sel_chan] = 1'b1;
                         if (idma_fe_req_ready[idma_fe_sel_chan]) begin
                             acc_res.id      = acc_req_i.id;
@@ -750,6 +811,12 @@ module idma_inst64_top #(
         if (!idma_fe_twod) begin
             idma_fe_req.d_req[0].reps = 'd1;
         end
+        // keep higher dims inert for plain requests (the transpose expander overwrites them)
+        for (int d = 1; d <= NumDim-2; d++) begin
+            idma_fe_req.d_req[d].reps        = 'd1;
+            idma_fe_req.d_req[d].src_strides = '0;
+            idma_fe_req.d_req[d].dst_strides = '0;
+        end
     end
 
     //--------------------------------------
@@ -763,6 +830,15 @@ module idma_inst64_top #(
     //--------------------------------------
     // only activate tracer if requested
 `ifndef SYNTHESIS
+    initial assert (idma_pkg::TransposeDimWidth == 32'd12) else
+        $fatal(1, "DMCPY argb transpose packing requires TransposeDimWidth == 12");
+`ifndef VERILATOR
+    // a transpose-enabled frontend needs the engine baked into the backend
+    if (ComputeEnable.transpose) begin : gen_compute_check
+        initial assert (gen_backend[0].i_idma_backend_rw_axi_rw_init_rw_obi.ComputeEnable.transpose)
+            else $fatal(1, "transpose frontend needs the engine backend (IDMA_VIDMA_IDS)");
+    end
+`endif
     if (DMATracing) begin : gen_tracer
         for (genvar c = 0; c < NumChannels; c++) begin : gen_channels
             // derive the name of the trace file from the hart and channel IDs

util/mario/backend.py

@@ -42,11 +42,11 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict =
         srp = len(used_read_prots) == 1 and not any_mh_r
         swp = len(used_write_prots) == 1 and not any_mh_w
 
-        # on-the-fly compute requires a single AXI write port
+        # compute needs a data-carrying write port (AXI or OBI); INIT carries none
         enable_compute = prot_id in (compute_cfg or {})
-        if enable_compute and not (swp and used_write_prots[0] == 'axi'):
+        if enable_compute and not (set(used_write_prots) & {'axi', 'obi'}):
             raise ValueError(
-                f'compute (IDMA_VIDMA_IDS) requires a single AXI write port: {prot_id}')
+                f'compute (IDMA_VIDMA_IDS) needs an AXI or OBI write port: {prot_id}')
 
         # create context
         context = {