Skip to content

inst64: On-the-fly transpose via address generation (no engine) #141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DanielKellerM wants to merge 3 commits into
base: devel
Choose a base branch
from
Open

inst64: On-the-fly transpose via address generation (no engine) #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca225c2
midend: Add address-gen transpose for engine-less backends
DanielKellerM Jun 23, 2026
a33ca68
frontend: Drive on-the-fly transpose through the inst64 frontend
DanielKellerM Jun 23, 2026
a3334c4
test: Add address-gen transpose regressions
DanielKellerM Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Bender.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ sources:
- target/rtl/tb_idma_generated.sv
- test/tb_idma_transpose_nd.sv
- test/tb_idma_transpose_b2b.sv
- test/tb_idma_addrgen_transpose.sv
- test/tb_idma_addrgen_transpose_obi.sv

# Multi-head directed backend testbenches
- target: multihead
Expand Down
15 changes: 15 additions & 0 deletions idma.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,21 @@ idma_sim_tb_idma_transpose_b2b: $(IDMA_VSIM_DIR)/compile.tcl
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_transpose_b2b -do "run -all; quit"
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_transpose_b2b -do "run -all; quit"

# Address-gen transpose: idma_transpose_midend (AddrGen) -> idma_nd_midend ->
# stock rw_axi / rw_obi backend. No compute engine. TB sweeps geometries; one
# run per bus width.
.PHONY: idma_sim_tb_idma_addrgen_transpose
idma_sim_tb_idma_addrgen_transpose: $(IDMA_VSIM_DIR)/compile.tcl
cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit"
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_addrgen_transpose -do "run -all; quit"
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_addrgen_transpose -do "run -all; quit"

.PHONY: idma_sim_tb_idma_addrgen_transpose_obi
idma_sim_tb_idma_addrgen_transpose_obi: $(IDMA_VSIM_DIR)/compile.tcl
cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit"
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_addrgen_transpose_obi -do "run -all; quit"
cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_addrgen_transpose_obi -do "run -all; quit"

.PHONY: idma_sim_tb_idma_transpose_midend
idma_sim_tb_idma_transpose_midend: $(IDMA_VSIM_DIR)/compile.tcl
cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit"
Expand Down
91 changes: 84 additions & 7 deletions src/frontend/inst64/idma_inst64_top.sv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ module idma_inst64_top #(
parameter int unsigned NumChannels = 32'd1,
parameter bit TCDMAliasEnable = 1'b0,
parameter int unsigned DMATracing = 32'd0,
/// Compile-time on-the-fly compute feature enables (e.g. transpose). This
/// multi-write backend cannot host the FF transpose engine, so transpose is
/// always executed by address generation (idma_transpose_midend AddrGen).
parameter idma_pkg::compute_enable_t ComputeEnable = '0,
parameter type axi_ar_chan_t = logic,
parameter type axi_aw_chan_t = logic,
parameter type axi_req_t = logic,
Expand Down Expand Up @@ -70,7 +74,7 @@ module idma_inst64_top #(
localparam int unsigned TfIdWidth = 32'd32;
localparam int unsigned TFLenWidth = AxiAddrWidth;
localparam int unsigned RepWidth = 32'd32;
localparam int unsigned NumDim = 32'd2;
localparam int unsigned NumDim = ComputeEnable.transpose ? 32'd4 : 32'd2;
localparam int unsigned BufferDepth = 32'd3;
localparam int unsigned NumRules = 32'd5;

Expand All @@ -84,7 +88,8 @@ module idma_inst64_top #(
localparam type id_t = logic[AxiIdWidth-1:0];
localparam type tf_len_t = logic[TFLenWidth-1:0];
localparam type offset_t = logic[OffsetWidth-1:0];
localparam type strides_t = logic[RepWidth-1:0];
// strides must match addr_t: signed transpose deltas would not sign-extend if narrower
localparam type strides_t = addr_t;
localparam type reps_t = logic[RepWidth-1:0];
localparam type tf_id_t = logic[TfIdWidth-1:0];

Expand Down Expand Up @@ -178,6 +183,7 @@ module idma_inst64_top #(
logic [1:0] idma_fe_status;
logic [2:0] idma_fe_sel_chan;
logic idma_fe_twod;
logic idma_fe_tp_reject;

// busy signals
idma_pkg::idma_busy_t [NumChannels-1:0] idma_busy;
Expand Down Expand Up @@ -348,7 +354,7 @@ module idma_inst64_top #(
.idma_req_t ( idma_req_t ),
.idma_rsp_t ( idma_rsp_t ),
.idma_nd_req_t ( idma_nd_req_t ),
.RepWidths ( RepWidth )
.RepWidths ( {NumDim{RepWidth}} )
) i_idma_nd_midend (
.clk_i,
.rst_ni,
Expand All @@ -367,6 +373,33 @@ module idma_inst64_top #(
.busy_o ( idma_nd_busy [c] )
);

// FIFO output, before transpose expansion
idma_nd_req_t fifo_nd_req;
logic fifo_nd_valid, fifo_nd_ready;

// expand transpose requests into the address-gen ND walk (this backend
// has no FF engine, so transpose is always address generation)
if (ComputeEnable.transpose) begin : gen_transpose
idma_transpose_midend #(
.NumDim ( NumDim ),
.AddrGenTranspose ( 1'b1 ),
.StrbWidth ( StrbWidth ),
.addr_t ( addr_t ),
.idma_nd_req_t ( idma_nd_req_t )
) i_idma_transpose_midend (
.nd_req_i ( fifo_nd_req ),
.valid_i ( fifo_nd_valid ),
.ready_o ( fifo_nd_ready ),
.nd_req_o ( idma_nd_req [c] ),
.valid_o ( idma_nd_req_valid [c] ),
.ready_i ( idma_nd_req_ready [c] )
);
end else begin : gen_no_transpose
assign idma_nd_req [c] = fifo_nd_req;
assign idma_nd_req_valid [c] = fifo_nd_valid;
assign fifo_nd_ready = idma_nd_req_ready [c];
end

stream_fifo_optimal_wrap #(
.Depth ( DMAReqFifoDepth ),
.type_t ( idma_nd_req_t ),
Expand All @@ -380,9 +413,9 @@ module idma_inst64_top #(
.data_i ( idma_fe_req ),
.valid_i ( idma_fe_req_valid [c] ),
.ready_o ( idma_fe_req_ready [c] ),
.data_o ( idma_nd_req [c] ),
.valid_o ( idma_nd_req_valid [c] ),
.ready_i ( idma_nd_req_ready [c] )
.data_o ( fifo_nd_req ),
.valid_o ( fifo_nd_valid ),
.ready_i ( fifo_nd_ready )
);
end

Expand Down Expand Up @@ -519,10 +552,12 @@ module idma_inst64_top #(
idma_fe_req_d.burst_req.opt.beo.src_reduce_len = 1'b0;
idma_fe_req_d.burst_req.opt.beo.dst_reduce_len = 1'b0;
idma_fe_req_d.burst_req.opt.last = 1'b0;
idma_fe_req_d.burst_req.opt.compute = '0;

// frontend config
idma_fe_cfg = '0;
idma_fe_status = '0;
idma_fe_tp_reject = 1'b0;
idma_fe_sel_chan = '0;

// default handshaking
Expand Down Expand Up @@ -573,6 +608,32 @@ module idma_inst64_top #(
idma_inst64_snitch_pkg::DMCPY : begin
idma_fe_cfg = acc_req_i.data_argb[1:0];
idma_fe_sel_chan = acc_req_i.data_argb[4:2];
// transpose request (register form only): argb spare bits
// carry {enable, mode, tensor_m, tensor_n}
if (ComputeEnable.transpose && acc_req_i.data_argb[5]) begin
idma_fe_req_d.burst_req.opt.compute.enable = 1'b1;
idma_fe_req_d.burst_req.opt.compute.op =
idma_pkg::COMPUTE_TRANSPOSE;
idma_fe_req_d.burst_req.opt.compute.params.transpose.mode =
acc_req_i.data_argb[7:6];
idma_fe_req_d.burst_req.opt.compute.params.transpose.tensor_m =
acc_req_i.data_argb[19:8];
idma_fe_req_d.burst_req.opt.compute.params.transpose.tensor_n =
acc_req_i.data_argb[31:20];
end
// reject malformed transpose: feature off, reserved
// mode, zero dim, twod (cfg[1]), or src/dst not E-aligned
if (acc_req_i.data_argb[5]) begin
idma_fe_tp_reject = !ComputeEnable.transpose
| (acc_req_i.data_argb[7:6] == 2'd3)
| (acc_req_i.data_argb[19:8] == '0)
| (acc_req_i.data_argb[31:20] == '0)
| acc_req_i.data_argb[1]
| (|(idma_fe_req_d.burst_req.src_addr
& addr_t'((32'd1 << acc_req_i.data_argb[7:6]) - 32'd1)))
| (|(idma_fe_req_d.burst_req.dst_addr
& addr_t'((32'd1 << acc_req_i.data_argb[7:6]) - 32'd1)));
end
end
default:;
endcase
Expand All @@ -588,7 +649,15 @@ module idma_inst64_top #(
// 3. wait for twod transfer to be accepted (ready)
// 4. send acc response (pvalid)
// 5. acknowledge acc request (qready)
if (acc_res_ready) begin
// DMCPY launch; transpose requests reject malformed configs
if (idma_fe_tp_reject) begin
// error response; the transfer is not launched
if (acc_res_ready) begin
acc_res.id = acc_req_i.id;
acc_res_valid = 1'b1;
acc_req_ready_o = 1'b1;
end
end else if (acc_res_ready) begin
idma_fe_req_valid[idma_fe_sel_chan] = 1'b1;
if (idma_fe_req_ready[idma_fe_sel_chan]) begin
acc_res.id = acc_req_i.id;
Expand Down Expand Up @@ -750,6 +819,12 @@ module idma_inst64_top #(
if (!idma_fe_twod) begin
idma_fe_req.d_req[0].reps = 'd1;
end
// keep higher dims inert for plain requests (the transpose expander overwrites them)
for (int d = 1; d <= NumDim-2; d++) begin
idma_fe_req.d_req[d].reps = 'd1;
idma_fe_req.d_req[d].src_strides = '0;
idma_fe_req.d_req[d].dst_strides = '0;
end
end

//--------------------------------------
Expand All @@ -763,6 +838,8 @@ module idma_inst64_top #(
//--------------------------------------
// only activate tracer if requested
`ifndef SYNTHESIS
initial assert (idma_pkg::TransposeDimWidth == 32'd12) else
$fatal(1, "DMCPY argb transpose packing requires TransposeDimWidth == 12");
if (DMATracing) begin : gen_tracer
for (genvar c = 0; c < NumChannels; c++) begin : gen_channels
// derive the name of the trace file from the hart and channel IDs
Expand Down
72 changes: 47 additions & 25 deletions src/midend/idma_transpose_midend.sv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
// Authors:
// - Daniel Keller <dankeller@iis.ee.ethz.ch>

/// Transpose geometry expander: expands an opt.compute=TRANSPOSE request into a
/// NumDim=4 tiled ND walk for the generic idma_nd_midend. Non-transpose passes
/// through. Combinational, quasi-static per request.
/// Expand a TRANSPOSE request into an idma_nd_midend walk: engine (tiled) or address-gen.
module idma_transpose_midend #(
/// Number of ND dimensions (must be >= 4 to express the tiled walk)
/// Number of ND dimensions (engine walk needs >= 4; address-gen needs >= 3)
parameter int unsigned NumDim = 32'd4,
/// Address-gen mode: element-granular swapped-stride transpose, no engine
parameter bit AddrGenTranspose = 1'b0,
/// Write data-path width in bytes (tile side NE = StrbWidth / element bytes)
parameter int unsigned StrbWidth = 32'd64,
/// Address type
Expand Down Expand Up @@ -51,6 +51,7 @@ module idma_transpose_midend #(
logic [TensorW-1:0] tm, tn;
logic signed [WorkW-1:0] m, n, log2ne, ne, yt, nt, nxe, mpe;
logic signed [WorkW-1:0] strb_c; // NE*E == StrbWidth (mode cancels)
logic signed [WorkW-1:0] e, me; // address-gen: E (=1<<mode), M*E

nd_req_o = nd_req_i; // passthrough

Expand All @@ -69,33 +70,54 @@ module idma_transpose_midend #(
mpe = yt <<< Log2Strb; // MP*E = YT*NE*E = YT*StrbWidth
strb_c = $signed(WorkW'(StrbWidth)); // NE*E (one tile-row = StrbWidth B)

nd_req_o.burst_req.length = LenW'(StrbWidth);
if (AddrGenTranspose) begin
// address-gen: swapped-stride element walk, no engine (out_T[c][r]=in[r][c])
e = $signed(WorkW'(1)) <<< mode; // E = 1<<mode
me = m <<< mode; // M*E
nd_req_o.burst_req.opt.compute.enable = 1'b0;
nd_req_o.burst_req.length = LenW'(e);
// d_req[0] = column walk (reps N): src +E, dst +M*E
nd_req_o.d_req[0].reps = n[RepW-1:0];
nd_req_o.d_req[0].src_strides = addr_t'(e);
nd_req_o.d_req[0].dst_strides = addr_t'(me);
// d_req[1] = row walk (reps M): src +E, dst +E - (N-1)*M*E (rewind)
nd_req_o.d_req[1].reps = m[RepW-1:0];
nd_req_o.d_req[1].src_strides = addr_t'(e);
nd_req_o.d_req[1].dst_strides = addr_t'(e - (n - 1) * me);
for (int unsigned d = 2; d < NumDim-1; d++) begin
nd_req_o.d_req[d].reps = RepW'(1);
nd_req_o.d_req[d].src_strides = '0;
nd_req_o.d_req[d].dst_strides = '0;
end
end else begin
nd_req_o.burst_req.length = LenW'(StrbWidth);

// d_req[0] = local row within tile (reps NE)
nd_req_o.d_req[0].reps = ne[RepW-1:0];
nd_req_o.d_req[0].src_strides = addr_t'(nxe);
nd_req_o.d_req[0].dst_strides = addr_t'(mpe);
// d_req[1] = row-tile (reps YT). (NE-1)*MPE = (MPE<<log2ne) - MPE.
nd_req_o.d_req[1].reps = yt[RepW-1:0];
nd_req_o.d_req[1].src_strides = addr_t'(nxe);
nd_req_o.d_req[1].dst_strides = addr_t'(strb_c - (mpe <<< log2ne) + mpe);
// d_req[2] = col-tile (reps NT). (YT*NE-1)*NXE = ((YT*N)<<Log2Strb) - NXE;
// the dst rewind MPE-(YT-1)*StrbWidth collapses to StrbWidth.
nd_req_o.d_req[2].reps = nt[RepW-1:0];
nd_req_o.d_req[2].src_strides = addr_t'(strb_c - ((yt * n) <<< Log2Strb) + nxe);
nd_req_o.d_req[2].dst_strides = addr_t'(strb_c);
// the walk is exactly 4-D: neutralize any higher dims
for (int unsigned d = 3; d < NumDim-1; d++) begin
nd_req_o.d_req[d].reps = RepW'(1);
nd_req_o.d_req[d].src_strides = '0;
nd_req_o.d_req[d].dst_strides = '0;
// d_req[0] = local row within tile (reps NE)
nd_req_o.d_req[0].reps = ne[RepW-1:0];
nd_req_o.d_req[0].src_strides = addr_t'(nxe);
nd_req_o.d_req[0].dst_strides = addr_t'(mpe);
// d_req[1] = row-tile (reps YT). (NE-1)*MPE = (MPE<<log2ne) - MPE.
nd_req_o.d_req[1].reps = yt[RepW-1:0];
nd_req_o.d_req[1].src_strides = addr_t'(nxe);
nd_req_o.d_req[1].dst_strides = addr_t'(strb_c - (mpe <<< log2ne) + mpe);
// d_req[2] = col-tile (reps NT). (YT*NE-1)*NXE = ((YT*N)<<Log2Strb) - NXE;
// the dst rewind MPE-(YT-1)*StrbWidth collapses to StrbWidth.
nd_req_o.d_req[2].reps = nt[RepW-1:0];
nd_req_o.d_req[2].src_strides = addr_t'(strb_c - ((yt * n) <<< Log2Strb) + nxe);
nd_req_o.d_req[2].dst_strides = addr_t'(strb_c);
// the walk is exactly 4-D: neutralize any higher dims
for (int unsigned d = 3; d < NumDim-1; d++) begin
nd_req_o.d_req[d].reps = RepW'(1);
nd_req_o.d_req[d].src_strides = '0;
nd_req_o.d_req[d].dst_strides = '0;
end
end
end
end

`ifndef SYNTHESIS
initial assert (NumDim >= 4) else
$fatal(1, "idma_transpose_midend requires NumDim >= 4 (got %0d)", NumDim);
initial assert (NumDim >= (AddrGenTranspose ? 32'd3 : 32'd4)) else
$fatal(1, "idma_transpose_midend: NumDim too small (got %0d)", NumDim);
// mode 0..2 needs NE >= 1, i.e. log2(StrbWidth) >= 2
initial assert (Log2Strb >= 2) else
$fatal(1, "idma_transpose_midend requires StrbWidth >= 4 (got %0d)", StrbWidth);
Expand Down
Loading
Loading