diff --git a/Bender.yml b/Bender.yml index 6f16b87e..2049f69e 100644 --- a/Bender.yml +++ b/Bender.yml @@ -44,6 +44,8 @@ sources: - src/backend/idma_axis_write.sv - src/backend/idma_channel_coupler.sv - src/backend/idma_dataflow_element.sv + - src/backend/idma_otf_transpose.sv + - src/backend/idma_otf_compute.sv - src/backend/idma_error_handler.sv - src/backend/idma_init_read.sv - src/backend/idma_init_write.sv @@ -54,7 +56,6 @@ sources: - src/backend/idma_tilelink_read.sv - src/backend/idma_tilelink_write.sv - # Generated content - target: rtl files: - target/rtl/idma_generated.sv @@ -67,6 +68,7 @@ sources: - src/midend/idma_mp_split_midend.sv - src/midend/idma_nd_midend.sv - src/midend/idma_rt_midend.sv + - src/midend/idma_transpose_midend.sv # RISC-V opcode package for ooc use of inst64 - target: all(rtl,snitch_cluster) @@ -120,7 +122,9 @@ sources: - test/future/idma_tb_per2axi.sv - test/future/TLToAXI4.v - test/midend/tb_idma_nd_midend.sv + - test/midend/tb_idma_nd_midend_b2b.sv - test/midend/tb_idma_rt_midend.sv + - test/midend/tb_idma_transpose_midend.sv # Level 2 - test/future/idma_obi2axi_bridge.sv - test/future/idma_tilelink2axi_bridge.sv @@ -129,6 +133,8 @@ sources: - target: idma_test files: - target/rtl/tb_idma_generated.sv + - test/tb_idma_transpose_nd.sv + - test/tb_idma_transpose_b2b.sv # Multi-head directed backend testbenches - target: multihead diff --git a/idma.mk b/idma.mk index e76fab2a..2a9d1523 100644 --- a/idma.mk +++ b/idma.mk @@ -16,6 +16,8 @@ VCS ?= vcs VERILATOR ?= verilator VLOGAN ?= vlogan VSIM ?= vsim +VLOG ?= vlog +VLIB ?= vlib # Provision a local uv venv when the generator deps are not already available. IDMA_VENV_PY := $(IDMA_ROOT)/.venv/bin/python @@ -51,6 +53,14 @@ IDMA_OCCAMY_IDS := \ rw_axi_rw_init_rw_obi IDMA_ADD_IDS ?= IDMA_BACKEND_IDS ?= $(IDMA_BASE_IDS) $(IDMA_OCCAMY_IDS) $(IDMA_ADD_IDS) +# Compute-hosting variants (single AXI write); empty default = stock has no compute +IDMA_VIDMA_IDS ?= +# Compute variants (strip the optional :op:fd suffix) must be built backends +_idma_vidma_unknown := $(filter-out $(IDMA_BACKEND_IDS),\ + $(foreach c,$(IDMA_VIDMA_IDS),$(firstword $(subst :, ,$(c))))) +ifneq ($(_idma_vidma_unknown),) + $(error iDMA: IDMA_VIDMA_IDS variant(s) not in IDMA_BACKEND_IDS: $(_idma_vidma_unknown)) +endif # generated frontends IDMA_BASE_FE_IDS := reg32_3d reg64_2d reg64_1d @@ -120,17 +130,26 @@ IDMA_RTL_FILES := $(IDMA_RTL_DIR)/idma_transport_layer \ IDMA_VSIM_DIR := $(IDMA_ROOT)/target/sim/vsim define idma_gen - $(PYTHON) $(IDMA_GEN) --entity $1 --tpl $2 --db $3 --ids $4 --fids $5 > $6 + $(PYTHON) $(IDMA_GEN) --entity $1 --tpl $2 --db $3 --ids $4 --fids $5 $(if $7,--compute-ids $7) > $6 endef -$(IDMA_RTL_DIR)/idma_transport_layer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,transport,$(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl,$(IDMA_DB_FILES),$*,,$@) +# Force an RTL regen when IDMA_VIDMA_IDS changes; rewritten only on change +IDMA_VIDMA_STAMP := $(IDMA_RTL_DIR)/.vidma_ids +.PHONY: idma_vidma_stamp_check +idma_vidma_stamp_check: + @mkdir -p $(IDMA_RTL_DIR) + @printf '%s' '$(IDMA_VIDMA_IDS)' | cmp -s - $(IDMA_VIDMA_STAMP) 2>/dev/null || \ + printf '%s' '$(IDMA_VIDMA_IDS)' > $(IDMA_VIDMA_STAMP) +$(IDMA_VIDMA_STAMP): idma_vidma_stamp_check ; + +$(IDMA_RTL_DIR)/idma_transport_layer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) + $(call idma_gen,transport,$(IDMA_ROOT)/src/backend/tpl/idma_transport_layer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) -$(IDMA_RTL_DIR)/idma_legalizer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,legalizer,$(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl,$(IDMA_DB_FILES),$*,,$@) +$(IDMA_RTL_DIR)/idma_legalizer_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) + $(call idma_gen,legalizer,$(IDMA_ROOT)/src/backend/tpl/idma_legalizer.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) -$(IDMA_RTL_DIR)/idma_backend_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_legalizer_%.sv $(IDMA_RTL_DIR)/idma_transport_layer_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl $(IDMA_DB_FILES) - $(call idma_gen,backend,$(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl,$(IDMA_DB_FILES),$*,,$@) +$(IDMA_RTL_DIR)/idma_backend_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_legalizer_%.sv $(IDMA_RTL_DIR)/idma_transport_layer_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl $(IDMA_DB_FILES) $(IDMA_VIDMA_STAMP) + $(call idma_gen,backend,$(IDMA_ROOT)/src/backend/tpl/idma_backend.sv.tpl,$(IDMA_DB_FILES),$*,,$@,$(IDMA_VIDMA_IDS)) $(IDMA_RTL_DIR)/idma_backend_synth_%.sv: $(IDMA_GEN) $(IDMA_GEN_SRC) $(IDMA_RTL_DIR)/idma_backend_%.sv $(IDMA_ROOT)/src/backend/tpl/idma_backend_synth.sv.tpl $(IDMA_DB_FILES) $(call idma_gen,synth_wrapper,$(IDMA_ROOT)/src/backend/tpl/idma_backend_synth.sv.tpl,$(IDMA_DB_FILES),$*,,$@) @@ -151,6 +170,7 @@ idma_rtl_clean: rm -f $(IDMA_VSIM_DIR)/wave/*.do rm -f $(IDMA_RTL_DIR)/include/idma/tracer.svh rm -rf $(IDMA_RTL_DIR)/include/idma + rm -f $(IDMA_VIDMA_STAMP) # assemble the required files IDMA_INCLUDE_ALL += $(IDMA_RTL_DIR)/include/idma/tracer.svh @@ -304,7 +324,7 @@ IDMA_PICKLE_ALL += $(IDMA_PICKLE_DIR)/idma_mp_midend_synth.sv $(IDMA_VSIM_DIR)/compile_%.tcl: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) $(IDMA_WAVE_ALL) echo 'set ROOT [file normalize [file dirname [info script]]/../../..]' > $@ set -o pipefail; $(BENDER) script vsim --vlog-arg="$(IDMA_VLOG_ARGS)" \ - -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster -t split_rtl \ + -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster \ --top $* | grep -v "set ROOT" >> $@ echo >> $@ @@ -329,6 +349,55 @@ endef $(IDMA_VSIM_DIR)/compile.tcl: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) $(IDMA_WAVE_ALL) $(call idma_generate_vsim, $@, -t sim -t test -t idma_test -t synth -t rtl -t asic -t snitch_cluster,../../..) +# Standalone self-checking transpose-engine regression (DPI-C golden, no backend deps). +# Run with the Questa SEPP wrapper, e.g.: +# make idma_sim_tb_idma_otf_transpose VSIM="questa-2023.4 vsim" VLOG="questa-2023.4 vlog" VLIB="questa-2023.4 vlib" +IDMA_OTF_TP_RTL := $(abspath $(IDMA_ROOT)/src/backend/idma_otf_transpose.sv) +IDMA_OTF_TP_TB := $(abspath $(IDMA_ROOT)/test/tb_idma_otf_transpose.sv) +IDMA_OTF_TP_DPI := $(abspath $(IDMA_ROOT)/test/idma_transpose_dpi.c) +IDMA_OTF_TP_DIR := $(abspath $(IDMA_VSIM_DIR))/otf_transpose + +.PHONY: idma_sim_tb_idma_otf_transpose +idma_sim_tb_idma_otf_transpose: + mkdir -p $(IDMA_OTF_TP_DIR) + cd $(IDMA_OTF_TP_DIR); $(VLIB) work + cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv $(IDMA_OTF_TP_DPI) + cd $(IDMA_OTF_TP_DIR); $(VLOG) -sv -svinputport=compat -timescale "1ns/1fs" $(IDMA_OTF_TP_RTL) $(IDMA_OTF_TP_TB) + # the TB sweeps the geometry list internally; one run per StrbWidth x FullDuplex + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gFullDuplex=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=8 -gFullDuplex=0 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gFullDuplex=1 tb_idma_otf_transpose +BP -do "run -all; quit" + cd $(IDMA_OTF_TP_DIR); $(VSIM) -c -t 1ps -gStrbWidth=64 -gFullDuplex=0 tb_idma_otf_transpose +BP -do "run -all; quit" + +# Multi-tile transpose via the ND midend -> rw_axi backend -> axi_sim_mem. +# Run with the Questa SEPP wrapper: make idma_sim_tb_idma_transpose_nd VSIM="questa-2023.4 vsim" +# These tests need the compute engine: build rw_axi with compute (stamp regens). +idma_sim_tb_idma_transpose_nd idma_sim_tb_idma_transpose_b2b: IDMA_VIDMA_IDS := rw_axi + +.PHONY: idma_sim_tb_idma_transpose_nd +idma_sim_tb_idma_transpose_nd: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + # the TB sweeps the geometry list internally; one run per bus width + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_transpose_nd -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_transpose_nd -do "run -all; quit" + +# Back-to-back regressions: the ND midend must reload each new transfer's base +# address (it does, for a protocol-compliant producer that drops nd_req_valid on +# accept). tb_idma_nd_midend_b2b checks the midend's burst-address sequence under +# backpressure; tb_idma_transpose_b2b checks two end-to-end transposes to distinct +# destinations. Run with the Questa SEPP wrapper. +.PHONY: idma_sim_tb_idma_nd_midend_b2b +idma_sim_tb_idma_nd_midend_b2b: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc tb_idma_nd_midend_b2b -do "run -all; quit" + +.PHONY: idma_sim_tb_idma_transpose_b2b +idma_sim_tb_idma_transpose_b2b: $(IDMA_VSIM_DIR)/compile.tcl + cd $(IDMA_VSIM_DIR); $(VSIM) -c -do "source compile.tcl; quit" + # the TB sweeps the geometry list internally; one run per bus width + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=32 tb_idma_transpose_b2b -do "run -all; quit" + cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc -gDataWidth=64 tb_idma_transpose_b2b -do "run -all; quit" + .PHONY: idma_sim_tb_idma_rt_midend idma_sim_tb_idma_rt_midend: $(IDMA_VSIM_DIR)/compile.tcl @@ -336,6 +405,7 @@ idma_sim_tb_idma_rt_midend: $(IDMA_VSIM_DIR)/compile.tcl cd $(IDMA_VSIM_DIR); $(VSIM) -c -t 1ps -voptargs=+acc tb_idma_rt_midend -do "run -all; quit" idma_sim_clean: + rm -rf $(IDMA_OTF_TP_DIR) rm -rf $(IDMA_VSIM_DIR)/compile.tcl rm -rf $(IDMA_VSIM_DIR)/work rm -f $(IDMA_VSIM_DIR)/dma_trace_* diff --git a/src/backend/idma_axi_write.sv b/src/backend/idma_axi_write.sv index ee27349d..b0b40d0b 100644 --- a/src/backend/idma_axi_write.sv +++ b/src/backend/idma_axi_write.sv @@ -73,7 +73,11 @@ module idma_axi_write #( /// Valid from buffer input strb_t buffer_out_valid_i, /// Ready to buffer - output strb_t buffer_out_ready_o + output strb_t buffer_out_ready_o, + /// External write-strobe mask (ANDed into wstrb); tie to '1 when unused + input strb_t mask_ext_i, + /// Pulses when a write beat is accepted on the bus (strobe-independent) + output logic w_beat_done_o ); // offsets needed for masks to empty buffer strb_t w_first_mask; @@ -141,6 +145,8 @@ module idma_axi_write #( if (w_dp_req_i.tailer != '0 & last_w) begin mask_out = mask_out & w_last_mask; end + // external mask (OTF compute) + mask_out = mask_out & mask_ext_i; end @@ -170,6 +176,8 @@ module idma_axi_write #( // write happening: both the bus (w_ready) and the buffer (ready_to_write) is high assign write_happening = ready_to_write & write_rsp_i.w_ready; + assign w_beat_done_o = write_happening; + // the main buffer is conditionally to the write mask popped assign buffer_out_ready_o = write_happening ? mask_out : '0; diff --git a/src/backend/idma_otf_compute.sv b/src/backend/idma_otf_compute.sv new file mode 100644 index 00000000..4ad8a5af --- /dev/null +++ b/src/backend/idma_otf_compute.sv @@ -0,0 +1,100 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +/// On-the-fly compute dispatcher: latches the per-transfer compute options +/// and dispatches one op per transfer to its sub-unit. +module idma_otf_compute #( + /// Byte lanes per beat (= DataWidth/8) + parameter int unsigned StrbWidth = 32'd8, + /// Compile-time per-op feature enables (value rendered by the generator) + parameter idma_pkg::compute_enable_t ComputeEnable = '0, + /// Transpose engine duplex (1: two banks full rate, 0: one bank half area) + parameter bit TransposeFullDuplex = 1'b1 +) ( + input logic clk_i, + input logic rst_ni, + + /// Per-transfer compute config; valid only while `cfg_valid_i` + input idma_pkg::compute_options_t compute_i, + input logic cfg_valid_i, + /// A supported compute op is armed for this transfer + output logic active_o, + + /// Input beat stream (from the dataflow buffer) + input logic [StrbWidth-1:0][7:0] data_i, + input logic valid_i, + output logic in_ready_o, + + /// Output beat stream (computed) with per-byte strobe for edge masking + output logic [StrbWidth-1:0][7:0] data_o, + output logic [StrbWidth-1:0] strb_o, + output logic valid_o, + input logic ready_i +); + + // config latch with first-beat bypass + idma_pkg::compute_options_t latched_q, eff_compute; + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) latched_q <= '0; + else if (cfg_valid_i) latched_q <= compute_i; + end + assign eff_compute = cfg_valid_i ? compute_i : latched_q; + + // per-op select + logic sel_transpose; + assign sel_transpose = eff_compute.enable & + (eff_compute.op == idma_pkg::COMPUTE_TRANSPOSE) & ComputeEnable.transpose; + + assign active_o = sel_transpose; + + // transpose sub-unit + logic [StrbWidth-1:0][7:0] tp_data; + logic [StrbWidth-1:0] tp_strb; + logic tp_valid, tp_in_ready; + + if (ComputeEnable.transpose) begin : gen_transpose + idma_otf_transpose #( + .StrbWidth ( StrbWidth ), + .DimWidth ( idma_pkg::TransposeDimWidth ), + .FullDuplex ( TransposeFullDuplex ) + ) i_idma_otf_transpose ( + .clk_i, + .rst_ni, + .clear_i ( ~sel_transpose ), + .transp_mode_i ( eff_compute.params.transpose.mode ), + .tensor_size_m_i ( eff_compute.params.transpose.tensor_m ), + .tensor_size_n_i ( eff_compute.params.transpose.tensor_n ), + .data_i ( data_i ), + .valid_i ( valid_i & sel_transpose ), + .ready_o ( tp_in_ready ), + .data_o ( tp_data ), + .strb_o ( tp_strb ), + .valid_o ( tp_valid ), + .ready_i ( ready_i & sel_transpose ) + ); + end else begin : gen_no_transpose + assign tp_data = '0; assign tp_strb = '0; assign tp_valid = 1'b0; assign tp_in_ready = 1'b0; + end + + // output dispatch + always_comb begin + data_o = '0; + strb_o = '0; + valid_o = 1'b0; + in_ready_o = 1'b0; + unique case (1'b1) + sel_transpose: begin + data_o = tp_data; + strb_o = tp_strb; + valid_o = tp_valid; + in_ready_o = tp_in_ready; + end + default: ; + endcase + end + +endmodule : idma_otf_compute diff --git a/src/backend/idma_otf_transpose.sv b/src/backend/idma_otf_transpose.sv new file mode 100644 index 00000000..efe93155 --- /dev/null +++ b/src/backend/idma_otf_transpose.sv @@ -0,0 +1,238 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// On-the-fly matrix transpose engine (ping-pong tile banks) for the iDMA +// transport datapath. Adapted from the datamover (Ratha) HWPE: +// pulp-platform/datamover@d58a985, rtl/datamover_engine.sv. +// +// Contract: input padded to full tiles, fed (col-tile, row-tile, row) order; +// out_T[nt*NE+k][rt*NE+r] = in[rt*NE+r][nt*NE+k]. strb_o masks partial edge tiles. +// Throughput: 1 + 1/NE cycles per NE-beat tile (one handoff bubble per tile). + +module idma_otf_transpose #( + /// Byte lanes per beat (= DataWidth/8) + parameter int unsigned StrbWidth = 32'd8, + /// Tensor dimension width in elements (matches idma_pkg::TransposeDimWidth) + parameter int unsigned DimWidth = 32'd12, + /// 1: two tile banks, fill while draining; 0: one bank (half area, half rate) + parameter bit FullDuplex = 1'b1, + localparam int unsigned NumBanks = FullDuplex ? 32'd2 : 32'd1, + localparam int unsigned LaneW = $clog2(StrbWidth) +) ( + input logic clk_i, + input logic rst_ni, + input logic clear_i, + + /// Element size select: 0->1B, 1->2B, 2->4B (E = 1<= 2 so LaneW>=1 and the shift geometry holds + initial assert (StrbWidth >= 2 && (StrbWidth & (StrbWidth-1)) == 0) else + $fatal(1, "idma_otf_transpose: StrbWidth (%0d) must be a power of two >= 2", StrbWidth); + + // geometry: NE is a power of two, so only shifts and AND-masks + logic [1:0] eff_mode; // element-size mode, saturated at LaneW + logic [LaneW:0] ne_m1; // NE-1 + logic [3:0] log2_ne; // log2(NE) = LaneW - eff_mode + logic [DimWidth-1:0] y_tiles, n_tiles; // row-tiles, col-tiles + logic [LaneW:0] leftover_rows, leftover_cols; // M%NE, N%NE (run-global) + + // saturate at LaneW: out-of-contract mode (E>StrbWidth) degrades to NE=1 + assign eff_mode = (transp_mode_i > LaneW) ? LaneW[1:0] : transp_mode_i; + assign ne_m1 = (1 << (LaneW - eff_mode)) - 1; // NE-1 + assign log2_ne = LaneW - eff_mode; + // Widen the ceil-div add by one bit so it cannot wrap at the dim range. + assign y_tiles = DimWidth'(((DimWidth+1)'(tensor_size_m_i) + ne_m1) >> log2_ne); + assign n_tiles = DimWidth'(((DimWidth+1)'(tensor_size_n_i) + ne_m1) >> log2_ne); + assign leftover_rows = tensor_size_m_i & ne_m1; + assign leftover_cols = tensor_size_n_i & ne_m1; + + // FF tile banks (ping-pong when FullDuplex), E=1 worst case (StrbWidth x StrbWidth B) + logic [StrbWidth-1:0][7:0] tile_q [NumBanks][StrbWidth]; + + // internal output + handshakes + logic [StrbWidth-1:0][7:0] data_int; + logic [StrbWidth-1:0] strb_int; + logic valid_int, ready_int; + logic in_hs, out_hs; + assign in_hs = valid_i & ready_o; + assign out_hs = valid_int & ready_int; + + // full_q[b]: bank b holds a complete tile. Producer sets on fill-complete, + // consumer clears on drain-complete. + logic [NumBanks-1:0] full_q; + logic wr_bank, rd_bank; + logic [LaneW-1:0] wr_cnt, rd_cnt; // intra-tile beat index (write / read) + logic wr_last, rd_last; + + assign wr_last = (wr_cnt == ne_m1[LaneW-1:0]); + assign rd_last = (rd_cnt == ne_m1[LaneW-1:0]); + + assign ready_o = ~full_q[wr_bank]; + assign valid_int = full_q[rd_bank]; + + // tile walkers (col-tile outer, row-tile inner); drain trails fill by up to one tile + logic [DimWidth-1:0] rtw, ntw; // write walker: row-tile, col-tile + logic [DimWidth-1:0] rtr, ntr; // read walker: row-tile, col-tile + + logic last_y_tile_w, last_n_tile_w; // edge flags of the tile being filled + assign last_y_tile_w = (rtw == y_tiles - 1); + assign last_n_tile_w = (ntw == n_tiles - 1); + + logic last_y_tile_r, last_n_tile_r; // edge flags of the tile being drained + assign last_y_tile_r = (rtr == y_tiles - 1); + assign last_n_tile_r = (ntr == n_tiles - 1); + + // per-bank edge flags, captured at fill-complete, consumed by the drain strobe + logic shadow_last_y [NumBanks]; + logic shadow_last_n [NumBanks]; + + // fill-/drain-complete events + logic fill_done, drain_done, exec_done; + assign fill_done = in_hs & wr_last; + assign drain_done = out_hs & rd_last; + // transfer done once the final tile drains + assign exec_done = drain_done & last_y_tile_r & last_n_tile_r; + + // producer (input) side + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + for (int b = 0; b < NumBanks; b++) + for (int r = 0; r < StrbWidth; r++) + tile_q[b][r] <= '0; + wr_cnt <= '0; + wr_bank <= 1'b0; + rtw <= '0; + ntw <= '0; + for (int b = 0; b < NumBanks; b++) begin + shadow_last_y[b] <= 1'b0; + shadow_last_n[b] <= 1'b0; + end + end else if (clear_i || exec_done) begin + wr_cnt <= '0; + wr_bank <= 1'b0; + rtw <= '0; + ntw <= '0; + for (int b = 0; b < NumBanks; b++) begin + shadow_last_y[b] <= 1'b0; + shadow_last_n[b] <= 1'b0; + end + end else begin + if (in_hs) begin + tile_q[wr_bank][wr_cnt] <= data_i; + wr_cnt <= wr_last ? '0 : (wr_cnt + 1'b1); + end + if (fill_done) begin + shadow_last_y[wr_bank] <= last_y_tile_w; + shadow_last_n[wr_bank] <= last_n_tile_w; + wr_bank <= FullDuplex ? ~wr_bank : 1'b0; + if (rtw == y_tiles - 1) begin + rtw <= '0; + ntw <= ntw + 1'b1; + end else begin + rtw <= rtw + 1'b1; + end + end + end + end + + // consumer (output) side + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i || exec_done) begin + rd_cnt <= '0; + rd_bank <= 1'b0; + rtr <= '0; + ntr <= '0; + end else begin + if (out_hs) begin + rd_cnt <= rd_last ? '0 : (rd_cnt + 1'b1); + end + if (drain_done) begin + rd_bank <= FullDuplex ? ~rd_bank : 1'b0; + if (rtr == y_tiles - 1) begin + rtr <= '0; + ntr <= ntr + 1'b1; + end else begin + rtr <= rtr + 1'b1; + end + end + end + end + + // full/empty token + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i || exec_done) begin + full_q <= 2'b00; + end else begin + if (fill_done) full_q[wr_bank] <= 1'b1; + if (drain_done) full_q[rd_bank] <= 1'b0; + end + end + + // transposed readout: byte p (element e=p>>logE, byte b=p&(E-1)) reads + // tile_q[rd_bank][e][rd_cnt*E + b] + always_comb begin + for (int p = 0; p < StrbWidth; p++) begin + automatic int unsigned e = p >> eff_mode; + automatic int unsigned b = p & ((1 << eff_mode) - 1); + automatic int unsigned col = (rd_cnt << eff_mode) | b; + data_int[p] = tile_q[rd_bank][e][col]; + end + end + + // output strobe: element-granular edge masking from the drain-side shadow flags + always_comb begin + logic [StrbWidth-1:0] em; // per-element valid (only low NE bits meaningful) + logic ly, ln; + ly = shadow_last_y[rd_bank]; + ln = shadow_last_n[rd_bank]; + for (int e = 0; e < StrbWidth; e++) begin + logic v; + if ((ly && leftover_rows != 0) && (ln && leftover_cols != 0)) + v = (rd_cnt < leftover_cols) && (e < leftover_rows); + else if (ly && leftover_rows != 0) + v = (e < leftover_rows); + else if (ln && leftover_cols != 0) + v = (rd_cnt < leftover_cols); + else + v = 1'b1; + em[e] = v; + end + for (int p = 0; p < StrbWidth; p++) + strb_int[p] = em[p >> eff_mode]; + end + + // output register; not cleared by exec_done so the final beat is held until accepted + assign ready_int = ~valid_o | ready_i; + always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni || clear_i) begin + valid_o <= 1'b0; + data_o <= '0; + strb_o <= '0; + end else if (ready_int) begin + valid_o <= valid_int; + data_o <= data_int; + strb_o <= strb_int; + end + end + +endmodule : idma_otf_transpose diff --git a/src/backend/tpl/idma_backend.sv.tpl b/src/backend/tpl/idma_backend.sv.tpl index 134c33e6..0459d567 100644 --- a/src/backend/tpl/idma_backend.sv.tpl +++ b/src/backend/tpl/idma_backend.sv.tpl @@ -172,9 +172,23 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, output idma_busy_t busy_o ); + /// Extra write-descriptor slots covering the compute (transpose) tile-fill latency + localparam int unsigned ComputeFifoDepth = ${"StrbWidth" if enable_compute else "32'd0"}; +% if enable_compute: + + /// Per-op compute set baked into this variant (frontends may cross-check) + localparam idma_pkg::compute_enable_t ComputeEnable = + '{${', '.join("%s: 1'b1" % op for op in compute_ops)}}; +`ifndef SYNTHESIS + // no engine flush on abort: compute is incompatible with error handling + initial assert (ErrorCap == idma_pkg::NO_ERROR_HANDLING) else + $fatal(1, "compute requires ErrorCap == NO_ERROR_HANDLING"); +`endif +% endif + /// The localparam MetaFifoDepth holds the maximum number of transfers that can be /// in-flight under any circumstances. - localparam int unsigned MetaFifoDepth = BufferDepth + NumAxInFlight + MemSysDepth; + localparam int unsigned MetaFifoDepth = BufferDepth + NumAxInFlight + MemSysDepth + ComputeFifoDepth; /// Address type typedef logic [AddrWidth-1:0] addr_t; @@ -235,6 +249,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, offset_t shift; axi_pkg::len_t num_beats; logic is_single; + idma_pkg::compute_options_t compute; } w_dp_req_t; /// The datapath write response type provides feedback from the write part of the datapath: @@ -298,6 +313,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, idma_pkg::axi_options_t src_axi_opt; idma_pkg::axi_options_t dst_axi_opt; logic super_last; + idma_pkg::compute_options_t compute; } idma_mut_tf_opt_t; /// The mutable transfer type holds important information that is mutated by the @@ -494,7 +510,8 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, tailer: OffsetWidth'(idma_req_i.length + idma_req_i.dst_addr[OffsetWidth-1:0]), shift: OffsetWidth'(- idma_req_i.dst_addr[OffsetWidth-1:0]), num_beats: len, - is_single: len == '0 + is_single: len == '0, + compute: idma_req_i.opt.compute }; // if the legalizer is bypassed; every burst is the last of the 1D transfer @@ -621,7 +638,7 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, ); stream_fifo_optimal_wrap #( - .Depth ( NumAxInFlight ), + .Depth ( NumAxInFlight + ComputeFifoDepth ), .type_t ( w_dp_req_t ), .PrintInfo ( PrintFifoInfo ) ) i_w_dp_req ( diff --git a/src/backend/tpl/idma_legalizer.sv.tpl b/src/backend/tpl/idma_legalizer.sv.tpl index 9301602c..9b6090b6 100644 --- a/src/backend/tpl/idma_legalizer.sv.tpl +++ b/src/backend/tpl/idma_legalizer.sv.tpl @@ -484,8 +484,13 @@ w_num_bytes_to_pb = w_page_num_bytes_to_pb; dst_head: req_i.opt.dst_head, read_shift: '0, write_shift: '0, +% if enable_compute: + decouple_rw: req_i.opt.beo.decouple_rw | req_i.opt.compute.enable, + decouple_aw: req_i.opt.beo.decouple_aw | req_i.opt.compute.enable, +% else: decouple_rw: req_i.opt.beo.decouple_rw, decouple_aw: req_i.opt.beo.decouple_aw, +% endif src_max_llen: req_i.opt.beo.src_max_llen, dst_max_llen: req_i.opt.beo.dst_max_llen, src_reduce_len: req_i.opt.beo.src_reduce_len, @@ -493,7 +498,8 @@ w_num_bytes_to_pb = w_page_num_bytes_to_pb; axi_id: req_i.opt.axi_id, src_axi_opt: req_i.opt.src, dst_axi_opt: req_i.opt.dst, - super_last: req_i.opt.last + super_last: req_i.opt.last, + compute: req_i.opt.compute }; // determine shift amount if (CombinedShifter) begin @@ -557,7 +563,8 @@ ${database[used_write_protocols[0]]['legalizer_write_data_path']} tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: 1'b1 + is_single: 1'b1, + compute: opt_tf_q.compute }; % endif end @@ -592,7 +599,8 @@ ${database[protocol]['legalizer_write_data_path']} tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: 1'b1 + is_single: 1'b1, + compute: opt_tf_q.compute }; endcase end diff --git a/src/backend/tpl/idma_transport_layer.sv.tpl b/src/backend/tpl/idma_transport_layer.sv.tpl index 59bcb029..6219855e 100644 --- a/src/backend/tpl/idma_transport_layer.sv.tpl +++ b/src/backend/tpl/idma_transport_layer.sv.tpl @@ -229,6 +229,9 @@ _rsp_t ${mh_format['aw'][protocol]}${protocol}_write_rsp_i, byte_t [2*StrbWidth-1:0] buffer_out_tmp; byte_t [StrbWidth-1:0] buffer_out; byte_t [StrbWidth-1:0] buffer_out_shifted; + byte_t [StrbWidth-1:0] wr_data; + strb_t wr_valid, wr_strb, mask_ext_shifted, dataflow_ready_in; + logic w_beat_done; % if not one_read_port: // Read multiplexed signals @@ -374,16 +377,61 @@ ${rendered_read_ports[read_port]} .ready_o ( buffer_in_ready ), .data_o ( buffer_out ), .valid_o ( buffer_out_valid ), - .ready_i ( buffer_out_ready_shifted ) + .ready_i ( dataflow_ready_in ) ); + //-------------------------------------- + // On-the-fly compute + //-------------------------------------- + +% if enable_compute: + logic cmp_active; + logic cmp_in_ready, cmp_out_valid; + byte_t [StrbWidth-1:0] cmp_data_o; + strb_t cmp_strb_o; + + // beats retire on w_beat_done (strobe-independent) + idma_otf_compute #( + .StrbWidth ( StrbWidth ), + .ComputeEnable ( '{${', '.join("%s: 1'b1" % op for op in compute_ops)}} ), + .TransposeFullDuplex ( 1'b${'1' if compute_full_duplex else '0'} ) + ) i_idma_otf_compute ( + .clk_i, + .rst_ni, + .compute_i ( w_dp_req_i.compute ), + .cfg_valid_i ( w_dp_valid_i ), + .active_o ( cmp_active ), + .data_i ( buffer_out ), + .valid_i ( &buffer_out_valid ), + .in_ready_o ( cmp_in_ready ), + .data_o ( cmp_data_o ), + .strb_o ( cmp_strb_o ), + .valid_o ( cmp_out_valid ), + .ready_i ( w_beat_done ) + ); + + // whole-beat valid; edge masking carried on wr_strb + assign wr_data = cmp_active ? cmp_data_o : buffer_out; + assign wr_valid = cmp_active ? {StrbWidth{cmp_out_valid}} : buffer_out_valid; + assign wr_strb = cmp_active ? cmp_strb_o : '1; + // pop the buffer only on a compute input handshake + assign dataflow_ready_in = cmp_active ? {StrbWidth{(&buffer_out_valid) & cmp_in_ready}} + : buffer_out_ready_shifted; +% else: + assign wr_data = buffer_out; + assign wr_valid = buffer_out_valid; + assign wr_strb = '1; + assign dataflow_ready_in = buffer_out_ready_shifted; +% endif + //-------------------------------------- // Write Barrel shifter //-------------------------------------- - assign buffer_out_tmp = {buffer_out, buffer_out} >> (w_dp_req_i.shift*8); + assign buffer_out_tmp = {wr_data, wr_data} >> (w_dp_req_i.shift*8); assign buffer_out_shifted = buffer_out_tmp[$bits(buffer_out_shifted)/8-1:0]; - assign buffer_out_valid_shifted = strb_t'({buffer_out_valid, buffer_out_valid} >> w_dp_req_i.shift); + assign buffer_out_valid_shifted = strb_t'({wr_valid, wr_valid} >> w_dp_req_i.shift); + assign mask_ext_shifted = strb_t'({wr_strb, wr_strb} >> w_dp_req_i.shift); assign buffer_out_ready_shifted = strb_t'({buffer_out_ready, buffer_out_ready} >> - w_dp_req_i.shift); % if not one_write_port: diff --git a/src/db/idma_axi.yml b/src/db/idma_axi.yml index 267ce4b2..1baa8650 100644 --- a/src/db/idma_axi.yml +++ b/src/db/idma_axi.yml @@ -67,7 +67,8 @@ legalizer_write_data_path: | tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: w_req_o.aw_req.axi.aw_chan.len, - is_single: w_req_o.aw_req.axi.aw_chan.len == '0 + is_single: w_req_o.aw_req.axi.aw_chan.len == '0, + compute: opt_tf_q.compute }; read_template: | idma_axi_read #( @@ -128,7 +129,9 @@ write_template: | .write_rsp_i ( ${write_response} ), .buffer_out_i ( buffer_out_shifted ), .buffer_out_valid_i ( buffer_out_valid_shifted ), - .buffer_out_ready_o ( ${buffer_out_ready} ) + .buffer_out_ready_o ( ${buffer_out_ready} ), + .mask_ext_i ( mask_ext_shifted ), + .w_beat_done_o ( w_beat_done ) ); synth_wrapper_ports_write: | output id_t axi_aw_id_o, diff --git a/src/db/idma_tilelink.yml b/src/db/idma_tilelink.yml index 15e97eb0..c737d806 100644 --- a/src/db/idma_tilelink.yml +++ b/src/db/idma_tilelink.yml @@ -85,7 +85,8 @@ legalizer_write_data_path: | tailer: OffsetWidth'(w_num_bytes + w_addr_offset), shift: opt_tf_q.write_shift, num_beats: 'd0, - is_single: w_num_bytes <= StrbWidth + is_single: w_num_bytes <= StrbWidth, + compute: opt_tf_q.compute }; read_template: | idma_tilelink_read #( diff --git a/src/idma_pkg.sv b/src/idma_pkg.sv index c89f4b5f..920740d5 100644 --- a/src/idma_pkg.sv +++ b/src/idma_pkg.sv @@ -81,6 +81,39 @@ package idma_pkg; logic dst_reduce_len; } backend_options_t; + /// Compute operation selector + typedef enum logic [3:0] { + COMPUTE_NONE = 4'd0, + COMPUTE_TRANSPOSE = 4'd1 + } compute_op_e; + + /// Transpose tensor dimension width (elements) + localparam int unsigned TransposeDimWidth = 32'd12; + + /// Transpose options (E = 1< + +/// Transpose geometry expander: expands an opt.compute=TRANSPOSE request into a +/// NumDim=4 tiled ND walk for the generic idma_nd_midend. Non-transpose passes +/// through. Combinational, quasi-static per request. +module idma_transpose_midend #( + /// Number of ND dimensions (must be >= 4 to express the tiled walk) + parameter int unsigned NumDim = 32'd4, + /// Write data-path width in bytes (tile side NE = StrbWidth / element bytes) + parameter int unsigned StrbWidth = 32'd64, + /// Address type + parameter type addr_t = logic, + /// ND request type + parameter type idma_nd_req_t = logic +)( + input idma_nd_req_t nd_req_i, + input logic valid_i, + output logic ready_o, + output idma_nd_req_t nd_req_o, + output logic valid_o, + input logic ready_i +); + + localparam int unsigned Log2Strb = $clog2(StrbWidth); + localparam int unsigned LenW = $bits(nd_req_o.burst_req.length); + localparam int unsigned RepW = $bits(nd_req_o.d_req[0].reps); + localparam int unsigned ModeW = $bits(nd_req_o.burst_req.opt.compute.params.transpose.mode); + localparam int unsigned TensorW = + $bits(nd_req_o.burst_req.opt.compute.params.transpose.tensor_m); + localparam int unsigned AddrW = $bits(addr_t); + // working width: largest term (YT*N)< AddrW) ? ProdW : AddrW; + + assign valid_o = valid_i; + assign ready_o = ready_i; + + logic is_transpose; + assign is_transpose = nd_req_i.burst_req.opt.compute.enable & + (nd_req_i.burst_req.opt.compute.op == idma_pkg::COMPUTE_TRANSPOSE); + + // NE and E are powers of two: all geometry folds to shifts except the YT*N + // stride product. + always_comb begin : proc_expand + logic [ModeW-1:0] mode; + logic [TensorW-1:0] tm, tn; + logic signed [WorkW-1:0] m, n, log2ne, ne, yt, nt, nxe, mpe; + logic signed [WorkW-1:0] strb_c; // NE*E == StrbWidth (mode cancels) + + nd_req_o = nd_req_i; // passthrough + + if (is_transpose) begin + mode = nd_req_i.burst_req.opt.compute.params.transpose.mode; + tm = nd_req_i.burst_req.opt.compute.params.transpose.tensor_m; + tn = nd_req_i.burst_req.opt.compute.params.transpose.tensor_n; + // zero-extend bounded dims into the signed working width + m = $signed({{(WorkW-TensorW){1'b0}}, tm}); // M + n = $signed({{(WorkW-TensorW){1'b0}}, tn}); // N + log2ne = $signed(WorkW'(Log2Strb)) - $signed({{(WorkW-ModeW){1'b0}}, mode}); + ne = $signed(WorkW'(1)) <<< log2ne; // tile side (elements) + yt = (m + ne - 1) >>> log2ne; // ceil(M/NE) + nt = (n + ne - 1) >>> log2ne; // ceil(N/NE) + nxe = n <<< mode; // N*E (E = 1<= 4) else + $fatal(1, "idma_transpose_midend requires NumDim >= 4 (got %0d)", NumDim); + // mode 0..2 needs NE >= 1, i.e. log2(StrbWidth) >= 2 + initial assert (Log2Strb >= 2) else + $fatal(1, "idma_transpose_midend requires StrbWidth >= 4 (got %0d)", StrbWidth); + // reps must hold tile counts (<= 2^TensorW) and ne (<= StrbWidth); length StrbWidth. + initial assert (RepW >= TensorW && RepW > Log2Strb) else + $fatal(1, "idma_transpose_midend: reps field %0d b too narrow (need >= %0d)", + RepW, (TensorW > Log2Strb+1) ? TensorW : Log2Strb+1); + initial assert (LenW > Log2Strb) else + $fatal(1, "idma_transpose_midend: length field %0d b cannot hold StrbWidth", LenW); + // reserved mode 3 (EB=8) and zero-size tensors are out of contract + always_comb begin : check_domain + if (is_transpose) begin + assert (nd_req_i.burst_req.opt.compute.params.transpose.mode != 2'd3) else + $error("idma_transpose_midend: reserved element mode 3 (EB=8)"); + assert (nd_req_i.burst_req.opt.compute.params.transpose.tensor_m != '0 && + nd_req_i.burst_req.opt.compute.params.transpose.tensor_n != '0) else + $error("idma_transpose_midend: zero-size tensor (M or N == 0)"); + end + end +`endif + +endmodule diff --git a/target/rtl/.gitignore b/target/rtl/.gitignore index 625b14c8..9f111a44 100644 --- a/target/rtl/.gitignore +++ b/target/rtl/.gitignore @@ -3,3 +3,4 @@ include/* !include/.gitkeep *.sv *.hjson +.vidma_ids diff --git a/test/idma_test.sv b/test/idma_test.sv index f4115a3e..931a580d 100644 --- a/test/idma_test.sv +++ b/test/idma_test.sv @@ -689,7 +689,11 @@ package idma_test; input logic dst_reduce_len, input id_t id, input idma_pkg::multihead_t src_head = '0, - input idma_pkg::multihead_t dst_head = '0 + input idma_pkg::multihead_t dst_head = '0, + input logic transpose_en = 1'b0, + input logic [1:0] transp_mode = '0, + input logic [11:0] tensor_m = '0, + input logic [11:0] tensor_n = '0 ); idma.req.length <= #TA length; idma.req.src_addr <= #TA src_addr; @@ -705,6 +709,12 @@ package idma_test; idma.req.opt.beo.dst_max_llen <= #TA dst_max_llen; idma.req.opt.beo.src_reduce_len <= #TA src_reduce_len; idma.req.opt.beo.dst_reduce_len <= #TA dst_reduce_len; + idma.req.opt.compute.enable <= #TA transpose_en; + idma.req.opt.compute.op <= #TA transpose_en ? idma_pkg::COMPUTE_TRANSPOSE + : idma_pkg::COMPUTE_NONE; + idma.req.opt.compute.params.transpose.mode <= #TA transp_mode; + idma.req.opt.compute.params.transpose.tensor_m <= #TA tensor_m; + idma.req.opt.compute.params.transpose.tensor_n <= #TA tensor_n; idma.req_valid <= #TA 1; cycle_start(); while (idma.req_ready != 1) begin cycle_end(); cycle_start(); end diff --git a/test/idma_transpose_dpi.c b/test/idma_transpose_dpi.c new file mode 100644 index 00000000..4082b21d --- /dev/null +++ b/test/idma_transpose_dpi.c @@ -0,0 +1,35 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// DPI-C golden model for idma_otf_transpose: element-granular M x N -> N x M +// transpose, element size E in {1,2,4} B. Reference: out_elem[c][r] = in_elem[r][c]. + +#include + +#define GM_MAX_BYTES (1 << 24) // 16 MiB + +static uint8_t gm_in[GM_MAX_BYTES]; +static uint8_t gm_out[GM_MAX_BYTES]; + +// Load one input byte at flat byte index. +void gm_load(int idx, int val) { + if (idx >= 0 && idx < GM_MAX_BYTES) gm_in[idx] = (uint8_t)val; +} + +// Transpose an m x n matrix of e-byte elements (row-major) into n x m. +void gm_transpose(int m, int n, int e) { + for (int r = 0; r < m; r++) + for (int c = 0; c < n; c++) + for (int b = 0; b < e; b++) + gm_out[((long)c * m + r) * e + b] = gm_in[((long)r * n + c) * e + b]; +} + +// Read one transposed output byte at flat byte index. +int gm_get(int idx) { + if (idx >= 0 && idx < GM_MAX_BYTES) return (int)gm_out[idx]; + return -1; +} diff --git a/test/midend/tb_idma_nd_midend_b2b.sv b/test/midend/tb_idma_nd_midend_b2b.sv new file mode 100644 index 00000000..e849a1e0 --- /dev/null +++ b/test/midend/tb_idma_nd_midend_b2b.sv @@ -0,0 +1,146 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// Back-to-back ND regression for idma_nd_midend. Drives the midend directly and +// golden-checks the burst_req address sequence: two back-to-back transfers (no +// gap) plus one after an idle gap must each walk from their own base. Catches a +// stale base-address reuse across transfers. + +`include "idma/typedef.svh" + +module tb_idma_nd_midend_b2b; + + localparam time TCK = 10ns; + localparam int unsigned AddrWidth = 32; + localparam int unsigned NumDim = 4; // 1D burst + 3 strided dims + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [31:0] tf_len_t; + typedef logic [11:0] id_t; + typedef logic [31:0] reps_t; + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + // ── Program (shared by all three transfers; only the base addresses differ) ── + localparam int unsigned R0 = 3, R1 = 2, R2 = 2; + localparam int unsigned NB = R0 * R1 * R2; // bursts per transfer + localparam addr_t SS0 = 'h10, DS0 = 'h100; + localparam addr_t SS1 = 'h40, DS1 = 'h400; + localparam addr_t SS2 = 'h1000, DS2 = 'h4000; + localparam addr_t S1 = 'h0000_1000, D1 = 'h0001_0000; // transfer 1 base + localparam addr_t S2 = 'h0000_2000, D2 = 'h0002_0000; // transfer 2 base (back-to-back) + localparam addr_t S3 = 'h0000_3000, D3 = 'h0003_0000; // transfer 3 base (after idle gap) + + logic clk, rst_n; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + idma_req_t burst_req; logic burst_req_valid, burst_req_ready; + idma_rsp_t burst_rsp; logic burst_rsp_valid, burst_rsp_ready; + logic busy; + + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_dut ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(burst_req), .burst_req_valid_o(burst_req_valid), .burst_req_ready_i(burst_req_ready), + .burst_rsp_i(burst_rsp), .burst_rsp_valid_i(burst_rsp_valid), .burst_rsp_ready_o(burst_rsp_ready), + .busy_o(busy) + ); + + // Backpressure on burst_req_ready is essential: during a stall stride_sel_q + // collapses toward 0, which is what can defeat the base reload. ready always-1 hides it. + logic [2:0] bp_lfsr; + always @(posedge clk or negedge rst_n) + if (!rst_n) bp_lfsr <= 3'b101; + else bp_lfsr <= {bp_lfsr[1:0], bp_lfsr[2] ^ bp_lfsr[1]}; + assign burst_req_ready = bp_lfsr[0]; // stalls ~half the cycles, incl. boundaries + assign burst_rsp = '0; + assign burst_rsp_valid = 1'b0; + assign nd_rsp_ready = 1'b1; + + // ── Capture every issued burst's src/dst address in order ── + addr_t cap_src [$]; + addr_t cap_dst [$]; + always @(posedge clk) if (rst_n && burst_req_valid && burst_req_ready) begin + cap_src.push_back(burst_req.src_addr); + cap_dst.push_back(burst_req.dst_addr); + end + + // build a NumDim=4 ND program with a given base + function automatic idma_nd_req_t mk_req(input addr_t s, input addr_t d); + idma_nd_req_t r = '0; + r.burst_req.length = tf_len_t'('h8); + r.burst_req.src_addr = s; + r.burst_req.dst_addr = d; + r.burst_req.opt.src_protocol = idma_pkg::AXI; + r.burst_req.opt.dst_protocol = idma_pkg::AXI; + r.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + r.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + r.d_req[0].reps = reps_t'(R0); r.d_req[0].src_strides = SS0; r.d_req[0].dst_strides = DS0; + r.d_req[1].reps = reps_t'(R1); r.d_req[1].src_strides = SS1; r.d_req[1].dst_strides = DS1; + r.d_req[2].reps = reps_t'(R2); r.d_req[2].src_strides = SS2; r.d_req[2].dst_strides = DS2; + return r; + endfunction + + initial begin + automatic int unsigned errs = 0; + nd_req = '0; nd_req_valid = 1'b0; + @(posedge rst_n); + repeat (3) @(posedge clk); + + // ── transfer 1 ── + nd_req = mk_req(S1, D1); nd_req_valid = 1'b1; + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + // ── transfer 2 : BACK-TO-BACK (keep valid high, swap payload the cycle after accept) ── + nd_req = mk_req(S2, D2); + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + nd_req_valid = 1'b0; + nd_req = '0; + // ── idle gap ── + repeat (5) @(posedge clk); + // ── transfer 3 : after the gap ── + nd_req = mk_req(S3, D3); nd_req_valid = 1'b1; + @(posedge clk); + while (!nd_req_ready) @(posedge clk); + nd_req_valid = 1'b0; + repeat (3) @(posedge clk); + + // ── checks ── + if (cap_src.size() != 3*NB) + begin errs++; $display("[B2B] burst count %0d != %0d", cap_src.size(), 3*NB); end + else begin + // first burst of each transfer must equal its OWN base (reload happened) + if (cap_src[0] !== S1 || cap_dst[0] !== D1) begin errs++; $display("[B2B] T1[0]=(%0h,%0h) exp (%0h,%0h)", cap_src[0], cap_dst[0], S1, D1); end + if (cap_src[NB] !== S2 || cap_dst[NB] !== D2) begin errs++; $display("[B2B] T2[0]=(%0h,%0h) exp (%0h,%0h) -- back-to-back base NOT reloaded", cap_src[NB], cap_dst[NB], S2, D2); end + if (cap_src[2*NB] !== S3 || cap_dst[2*NB] !== D3) begin errs++; $display("[B2B] T3[0]=(%0h,%0h) exp (%0h,%0h)", cap_src[2*NB], cap_dst[2*NB], S3, D3); end + // full-sequence independence: T2 and T3 must be T1 shifted by their base delta + for (int unsigned i = 0; i < NB; i++) begin + if ((cap_src[NB+i] - cap_src[i]) !== (S2 - S1) || (cap_dst[NB+i] - cap_dst[i]) !== (D2 - D1)) begin + errs++; if (errs <= 8) $display("[B2B] T2[%0d] not T1+delta: src %0h vs %0h (Δexp %0h)", i, cap_src[NB+i], cap_src[i], S2-S1); end + if ((cap_src[2*NB+i] - cap_src[i]) !== (S3 - S1) || (cap_dst[2*NB+i] - cap_dst[i]) !== (D3 - D1)) begin + errs++; if (errs <= 8) $display("[B2B] T3[%0d] not T1+delta: src %0h vs %0h (Δexp %0h)", i, cap_src[2*NB+i], cap_src[i], S3-S1); end + end + end + + if (errs == 0) $display("[B2B] PASS: %0d back-to-back + gapped ND transfers each walked from their own base", 3*NB); + else $fatal(1, "[B2B] FAIL: %0d errors (back-to-back ND base-address reuse)", errs); + $finish(); + end + + initial begin #500_000; $fatal(1, "[B2B] timeout"); end + +endmodule diff --git a/test/midend/tb_idma_transpose_midend.sv b/test/midend/tb_idma_transpose_midend.sv new file mode 100644 index 00000000..41598289 --- /dev/null +++ b/test/midend/tb_idma_transpose_midend.sv @@ -0,0 +1,103 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +`include "idma/typedef.svh" + +/// Unit check for idma_transpose_midend: the expanded NumDim=4 ND request must +/// match the golden geometry that tb_idma_transpose_nd hand-builds, and a +/// non-transpose request must pass through unchanged. +module tb_idma_transpose_midend #( + parameter int unsigned DataWidth = 512, + parameter int unsigned AddrWidth = 64, + parameter int unsigned M = 40, + parameter int unsigned N = 24, + parameter int unsigned EB = 1 +); + import idma_pkg::*; + + localparam int unsigned StrbWidth = DataWidth/8; + localparam int unsigned NE = StrbWidth/EB; + localparam int unsigned MODE = (EB==4) ? 2 : (EB==2) ? 1 : 0; + localparam int unsigned YT = (M + NE - 1)/NE; + localparam int unsigned NT = (N + NE - 1)/NE; + localparam int unsigned MP = YT*NE; + localparam int unsigned NumDim = 4; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [31:0] tf_len_t; + typedef logic [2:0] id_t; + typedef logic [31:0] reps_t; + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + idma_nd_req_t nd_in, nd_out; + logic vi, ro, vo, ri; + assign vi = 1'b1; + assign ri = 1'b1; + + idma_transpose_midend #( + .NumDim(NumDim), .StrbWidth(StrbWidth), .addr_t(addr_t), .idma_nd_req_t(idma_nd_req_t) + ) i_dut ( + .nd_req_i(nd_in), .valid_i(vi), .ready_o(ro), + .nd_req_o(nd_out), .valid_o(vo), .ready_i(ri) + ); + + int errs = 0; + task automatic chk(input string name, input logic [63:0] got, input logic [63:0] exp); + if (got !== exp) begin + errs++; + $display("[MID] %s: got %0d exp %0d", name, $signed(got), $signed(exp)); + end + endtask + + initial begin + // --- passthrough case --- + nd_in = '0; + nd_in.burst_req.src_addr = 64'hDEAD; + nd_in.burst_req.dst_addr = 64'hBEEF; + nd_in.d_req[0].reps = 7; + #1; + if (nd_out !== nd_in) begin + errs++; + $display("[MID] passthrough altered a non-transpose request"); + end + + // --- transpose case --- + nd_in = '0; + nd_in.burst_req.src_addr = 64'h1000; + nd_in.burst_req.dst_addr = 64'h2000; + nd_in.burst_req.opt.compute.enable = 1'b1; + nd_in.burst_req.opt.compute.op = COMPUTE_TRANSPOSE; + nd_in.burst_req.opt.compute.params.transpose.mode = 2'(MODE); + nd_in.burst_req.opt.compute.params.transpose.tensor_m = 12'(M); + nd_in.burst_req.opt.compute.params.transpose.tensor_n = 12'(N); + #1; + + // golden geometry (same formulas as tb_idma_transpose_nd) + chk("length", nd_out.burst_req.length, NE*EB); + chk("d0.reps", nd_out.d_req[0].reps, NE); + chk("d0.src", nd_out.d_req[0].src_strides, addr_t'(N*EB)); + chk("d0.dst", nd_out.d_req[0].dst_strides, addr_t'(MP*EB)); + chk("d1.reps", nd_out.d_req[1].reps, YT); + chk("d1.src", nd_out.d_req[1].src_strides, addr_t'(N*EB)); + chk("d1.dst", nd_out.d_req[1].dst_strides, addr_t'(int'(NE*EB) - int'((NE-1)*MP*EB))); + chk("d2.reps", nd_out.d_req[2].reps, NT); + chk("d2.src", nd_out.d_req[2].src_strides, addr_t'(int'(NE*EB) - int'((YT*NE-1)*N*EB))); + chk("d2.dst", nd_out.d_req[2].dst_strides, addr_t'(int'(MP*EB) - int'((YT-1)*NE*EB))); + // addresses + compute must survive untouched + chk("src_addr", nd_out.burst_req.src_addr, 64'h1000); + chk("dst_addr", nd_out.burst_req.dst_addr, 64'h2000); + chk("cmp_en", nd_out.burst_req.opt.compute.enable, 1); + + if (errs == 0) + $display("[MID] PASS: %0dx%0d EB=%0d golden (NE=%0d YT=%0d NT=%0d MP=%0d)", + M, N, EB, NE, YT, NT, MP); + else $fatal(1, "[MID] FAIL: %0d mismatches", errs); + $finish; + end +endmodule diff --git a/test/tb_idma_otf_transpose.sv b/test/tb_idma_otf_transpose.sv new file mode 100644 index 00000000..d57c1f29 --- /dev/null +++ b/test/tb_idma_otf_transpose.sv @@ -0,0 +1,188 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// Standalone self-checking testbench for idma_otf_transpose: checks a full +// M x N transpose of EB-byte elements (EB in {1,2,4}) against the DPI-C golden +// (idma_transpose_dpi.c). Sweeps a geometry list internally; M/N/EB are runtime +// DUT inputs, so one elaboration covers all geometries for a given StrbWidth/ +// FullDuplex. Optional backpressure via +BP. + +`timescale 1ns/1ps + +module tb_idma_otf_transpose #( + parameter int unsigned StrbWidth = 32'd8, + parameter bit FullDuplex = 1'b1 +); + + import "DPI-C" function void gm_load(input int idx, input int val); + import "DPI-C" function void gm_transpose(input int m, input int n, input int e); + import "DPI-C" function int gm_get(input int idx); + + localparam logic [7:0] PAD = 8'hFF; + + // Geometry cases (M, N, EB); EB>StrbWidth cases skip. + localparam int unsigned NCases = 4; + localparam int unsigned Cases [NCases][3] = '{ '{13, 19, 1}, '{7, 5, 2}, '{5, 3, 4}, '{130, 70, 1} }; + + logic clk = 1'b0, rst_n = 1'b0, clear = 1'b0; + always #5 clk = ~clk; + + // runtime DUT control (held stable per case) + logic [1:0] mode_q; + logic [11:0] m_q, n_q; + + logic [StrbWidth-1:0][7:0] din_data; + logic din_valid, din_ready; + logic [StrbWidth-1:0][7:0] dout_data; + logic [StrbWidth-1:0] dout_strb; + logic dout_valid, dout_ready; + + idma_otf_transpose #( + .StrbWidth (StrbWidth), + .FullDuplex (FullDuplex) + ) i_dut ( + .clk_i (clk), + .rst_ni (rst_n), + .clear_i (clear), + .transp_mode_i (mode_q), + .tensor_size_m_i (m_q), + .tensor_size_n_i (n_q), + .data_i (din_data), + .valid_i (din_valid), + .ready_o (din_ready), + .data_o (dout_data), + .strb_o (dout_strb), + .valid_o (dout_valid), + .ready_i (dout_ready) + ); + + logic [7:0] inb []; // row-major input bytes (sized per case) + bit wrote []; // per transposed-element coverage (sized per case) + int unsigned errors = 0; // running total across all cases + bit backpressure = 1'b0; + + task automatic drive_inputs(input int unsigned m, n, eb, ne, yt, nt); + int unsigned beat = 0; + din_valid = 1'b0; din_data = '0; + @(posedge clk); + for (int unsigned ct = 0; ct < nt; ct++) + for (int unsigned rt = 0; rt < yt; rt++) + for (int unsigned row = 0; row < ne; row++) begin + if (backpressure) begin din_valid = 1'b0; repeat (beat % 3) @(posedge clk); end + for (int unsigned c = 0; c < ne; c++) begin + automatic int unsigned gr = rt*ne + row; + automatic int unsigned gc = ct*ne + c; + for (int unsigned b = 0; b < eb; b++) + din_data[c*eb + b] = (gr < m && gc < n) ? inb[(gr*n + gc)*eb + b] : PAD; + end + din_valid = 1'b1; + do @(posedge clk); while (!din_ready); + beat++; + end + din_valid = 1'b0; + endtask + + task automatic capture_outputs(input int unsigned m, n, eb, ne, yt, nt); + int unsigned beat = 0; + dout_ready = 1'b0; + for (int unsigned ct = 0; ct < nt; ct++) + for (int unsigned rt = 0; rt < yt; rt++) + for (int unsigned k = 0; k < ne; k++) begin + if (backpressure) begin dout_ready = 1'b0; repeat (beat % 4) @(posedge clk); end + dout_ready = 1'b1; + do @(posedge clk); while (!dout_valid); + for (int unsigned e = 0; e < ne; e++) begin + if (dout_strb[e*eb]) begin // element e valid (element-granular mask) + automatic int unsigned tr = ct*ne + k; // transposed row (= original col, 0..n-1) + automatic int unsigned tc = rt*ne + e; // transposed col (= original row, 0..m-1) + if (tr >= n || tc >= m) begin + errors++; + if (errors <= 16) $display("STRB-ON-PAD beat(ct%0d rt%0d k%0d) elem %0d -> (%0d,%0d) OOB", ct, rt, k, e, tr, tc); + end else begin + for (int unsigned b = 0; b < eb; b++) begin + automatic int gold = gm_get((tr*m + tc)*eb + b); + if (int'(dout_data[e*eb + b]) !== gold) begin + errors++; + if (errors <= 16) $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", tr, tc, b, dout_data[e*eb+b], gold); + end + end + wrote[tr*m + tc] = 1'b1; + end + end + end + beat++; + end + @(posedge clk); + dout_ready = 1'b0; + endtask + + // Run one m x n transpose of eb-byte elements; returns the mismatch count. + task automatic run_case(input int unsigned m, n, eb, output int unsigned errs); + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned ne = StrbWidth / eb; + automatic int unsigned yt = (m + ne - 1) / ne; + automatic int unsigned nt = (n + ne - 1) / ne; + automatic int unsigned e0 = errors; + + inb = new[m*n*eb]; + wrote = new[n*m]; + for (int unsigned i = 0; i < m*n*eb; i++) begin + inb[i] = 8'((i * 7 + 3) & 8'hFF); + gm_load(i, int'(inb[i])); + end + gm_transpose(m, n, eb); + for (int unsigned i = 0; i < n*m; i++) wrote[i] = 1'b0; + + mode_q = 2'(mode); m_q = 12'(m); n_q = 12'(n); + // clear the engine between cases (resets banks / walkers) + clear = 1'b1; repeat (2) @(posedge clk); clear = 1'b0; @(posedge clk); + + $display("[TB] case M=%0d N=%0d EB=%0d (tile=%0d elems, %0dx%0d tiles) BP=%0d", m, n, eb, ne, yt, nt, backpressure); + fork drive_inputs(m, n, eb, ne, yt, nt); capture_outputs(m, n, eb, ne, yt, nt); join + + for (int unsigned tr = 0; tr < n; tr++) + for (int unsigned tc = 0; tc < m; tc++) + if (!wrote[tr*m + tc]) begin + errors++; + if (errors <= 16) $display("MISSING transposed elem (%0d,%0d)", tr, tc); + end + errs = errors - e0; + endtask + + initial begin + automatic int unsigned total = 0, ce; + din_valid = 1'b0; dout_ready = 1'b0; mode_q = '0; m_q = '0; n_q = '0; + if ($test$plusargs("BP")) backpressure = 1'b1; + + rst_n = 1'b0; clear = 1'b1; + repeat (4) @(posedge clk); + rst_n = 1'b1; + repeat (2) @(posedge clk); + clear = 1'b0; + @(posedge clk); + + for (int unsigned k = 0; k < NCases; k++) begin + if (Cases[k][2] > StrbWidth) continue; // element must fit the bus + run_case(Cases[k][0], Cases[k][1], Cases[k][2], ce); + if (ce == 0) $display("[TB] PASS: %0dx%0d EB=%0d transpose matches DPI golden", Cases[k][0], Cases[k][1], Cases[k][2]); + else $display("[TB] FAIL: %0dx%0d EB=%0d (%0d errors)", Cases[k][0], Cases[k][1], Cases[k][2], ce); + total += ce; + end + + if (total == 0) $display("[TB] ALL PASS (%0d cases, StrbWidth=%0d, FullDuplex=%0d)", NCases, StrbWidth, FullDuplex); + else $display("[TB] FAIL: %0d total errors", total); + repeat (5) @(posedge clk); + $finish; + end + + initial begin + #100000000; + $display("[TB] FAIL: timeout"); + $finish; + end + +endmodule diff --git a/test/tb_idma_transpose_b2b.sv b/test/tb_idma_transpose_b2b.sv new file mode 100644 index 00000000..bdb77f04 --- /dev/null +++ b/test/tb_idma_transpose_b2b.sv @@ -0,0 +1,223 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// End-to-end back-to-back transpose regression: per geometry, two transposes of +// one source to DIFFERENT dst bases through the ND midend -> rw_axi backend -> +// axi_sim_mem. A stale base across transfers would leave the second dst +// untouched. Sweeps a geometry list internally (one run per bus width). + +`include "axi/typedef.svh" +`include "idma/typedef.svh" + +module tb_idma_transpose_b2b + import idma_pkg::*; +#( + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, + parameter int unsigned AxiIdWidth = 12, + parameter int unsigned TFLenWidth = 32 +); + + localparam time TA = 1ns, TT = 9ns, TCK = 10ns; + localparam int unsigned StrbWidth = DataWidth / 8; + localparam int unsigned NumDim = 4; + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + // Geometry cases (M, N, EB); EB>StrbWidth cases skip. + localparam int unsigned NCases = 4; + localparam int unsigned Cases [NCases][3] = '{ '{6, 8, 1}, '{8, 8, 1}, '{13, 19, 1}, '{5, 5, 2} }; + + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [DataWidth-1:0] data_t; + typedef logic [StrbWidth-1:0] strb_t; + typedef logic [AxiIdWidth-1:0] id_t; + typedef logic [UserWidth-1:0] user_t; + typedef logic [TFLenWidth-1:0] tf_len_t; + typedef logic [31:0] reps_t; + + `AXI_TYPEDEF_AW_CHAN_T(axi_aw_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_W_CHAN_T(axi_w_chan_t, data_t, strb_t, user_t) + `AXI_TYPEDEF_B_CHAN_T(axi_b_chan_t, id_t, user_t) + `AXI_TYPEDEF_AR_CHAN_T(axi_ar_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_R_CHAN_T(axi_r_chan_t, data_t, id_t, user_t) + `AXI_TYPEDEF_REQ_T(axi_req_t, axi_aw_chan_t, axi_w_chan_t, axi_ar_chan_t) + `AXI_TYPEDEF_RESP_T(axi_rsp_t, axi_b_chan_t, axi_r_chan_t) + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + typedef struct packed { axi_ar_chan_t ar_chan; } axi_read_meta_channel_t; + typedef struct packed { axi_read_meta_channel_t axi; } read_meta_channel_t; + typedef struct packed { axi_aw_chan_t aw_chan; } axi_write_meta_channel_t; + typedef struct packed { axi_write_meta_channel_t axi; } write_meta_channel_t; + + logic clk, rst_n; + idma_req_t idma_req; logic req_valid, req_ready; + idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; + idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; + axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; + idma_busy_t busy; logic nd_busy; + + assign idma_eh_req = '0; + assign eh_req_valid = 1'b0; + + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( + .clk_i(clk), .rst_ni(rst_n), + .slv_read_req_i(axi_read_req), .slv_read_resp_o(axi_read_rsp), + .slv_write_req_i(axi_write_req), .slv_write_resp_o(axi_write_rsp), + .mst_req_o(axi_req), .mst_resp_i(axi_rsp) + ); + assign axi_req_mem = axi_req; + assign axi_rsp = axi_rsp_mem; + + axi_sim_mem #( + .AddrWidth(AddrWidth), .DataWidth(DataWidth), .IdWidth(AxiIdWidth), .UserWidth(UserWidth), + .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .WarnUninitialized(1'b0), .ClearErrOnAccess(1'b1), .ApplDelay(TA), .AcqDelay(TT) + ) i_axi_sim_mem ( + .clk_i(clk), .rst_ni(rst_n), .axi_req_i(axi_req_mem), .axi_rsp_o(axi_rsp_mem), + .mon_r_last_o(), .mon_r_beat_count_o(), .mon_r_user_o(), .mon_r_id_o(), + .mon_r_data_o(), .mon_r_addr_o(), .mon_r_valid_o(), + .mon_w_last_o(), .mon_w_beat_count_o(), .mon_w_user_o(), .mon_w_id_o(), + .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() + ); + + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_nd_midend ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), + .busy_o(nd_busy) + ); + + idma_backend_rw_axi #( + .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), + .UserWidth(UserWidth), .TFLenWidth(TFLenWidth), .MaskInvalidData(1'b1), .BufferDepth(3), + .RAWCouplingAvail(1'b1), .HardwareLegalizer(1'b1), .RejectZeroTransfers(1'b1), + .ErrorCap(idma_pkg::NO_ERROR_HANDLING), .PrintFifoInfo(1'b0), .NumAxInFlight(StrbWidth), .MemSysDepth(0), + .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_eh_req_t(idma_eh_req_t), + .idma_busy_t(idma_busy_t), .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .write_meta_channel_t(write_meta_channel_t), .read_meta_channel_t(read_meta_channel_t) + ) i_idma_backend ( + .clk_i(clk), .rst_ni(rst_n), .testmode_i(1'b0), + .idma_req_i(idma_req), .req_valid_i(req_valid), .req_ready_o(req_ready), + .idma_rsp_o(idma_rsp), .rsp_valid_o(rsp_valid), .rsp_ready_i(rsp_ready), + .idma_eh_req_i(idma_eh_req), .eh_req_valid_i(eh_req_valid), .eh_req_ready_o(eh_req_ready), + .axi_read_req_o(axi_read_req), .axi_read_rsp_i(axi_read_rsp), + .axi_write_req_o(axi_write_req), .axi_write_rsp_i(axi_write_rsp), .busy_o(busy) + ); + + stream_watchdog #(.NumCycles(4000)) i_r_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_rsp.r_valid), .ready_i(axi_req.r_ready)); + stream_watchdog #(.NumCycles(4000)) i_w_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_req.w_valid), .ready_i(axi_rsp.w_ready)); + + addr_t sb = 'h0000_1000; + + task automatic wr_mem(input addr_t a, input logic [7:0] d); i_axi_sim_mem.mem[a] = d; endtask + function automatic logic [7:0] rd_mem(input addr_t a); + return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; + endfunction + + // one m x n transpose of the source at sb -> dst base `db`; returns error count + task automatic do_transpose(input int unsigned m, input int unsigned n, input int unsigned eb, + input addr_t db, output int unsigned errs); + automatic int unsigned ne = StrbWidth / eb; + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned yt = (m + ne - 1) / ne; + automatic int unsigned nt = (n + ne - 1) / ne; + automatic int unsigned mp = yt * ne; + errs = 0; + // pre-fill full padded dst extent with sentinel + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(db + (i*mp + j)*eb + b, 8'hCC); + nd_req = '0; + nd_req.burst_req.length = tf_len_t'(ne*eb); + nd_req.burst_req.src_addr = sb; + nd_req.burst_req.dst_addr = db; + nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.beo.decouple_rw = 1'b1; + nd_req.burst_req.opt.beo.decouple_aw = 1'b1; + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); + nd_req.burst_req.opt.last = 1'b1; + nd_req.d_req[0].reps = reps_t'(ne); nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); + nd_req.d_req[1].reps = reps_t'(yt); nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); + nd_req.d_req[2].reps = reps_t'(nt); nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); + nd_req_valid = 1'b1; + do @(posedge clk); while (!nd_req_ready); // drop valid the cycle accept is seen (compliant) + nd_req_valid = 1'b0; + nd_req = '0; + while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + repeat (20) @(posedge clk); + // data + padding checks + for (int unsigned c = 0; c < n; c++) + for (int unsigned r = 0; r < m; r++) + for (int unsigned b = 0; b < eb; b++) + if (rd_mem(db + (c*mp + r)*eb + b) !== rd_mem(sb + (r*n + c)*eb + b)) begin + errs++; if (errs <= 8) $display("[B2BT] @db=%0h MISMATCH out_T[%0d][%0d].b%0d", db, c, r, b); + end + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + if (i >= n || j >= m) + for (int unsigned b = 0; b < eb; b++) + if (rd_mem(db + (i*mp + j)*eb + b) !== 8'hCC) begin + errs++; if (errs <= 8) $display("[B2BT] @db=%0h PADDING CLOBBERED row=%0d col=%0d", db, i, j); + end + endtask + + initial begin + automatic int unsigned total = 0, e1, e2; + automatic addr_t db1 = 'h0000_4000; + automatic addr_t db2 = 'h0000_8000; // DIFFERENT base — a stale-addr bug misplaces xfer 2 + automatic int unsigned m, n, eb; + nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + @(posedge rst_n); + repeat (5) @(posedge clk); + + for (int unsigned k = 0; k < NCases; k++) begin + m = Cases[k][0]; n = Cases[k][1]; eb = Cases[k][2]; + if (eb > StrbWidth) continue; + // (re)init source for this geometry + for (int unsigned r = 0; r < m; r++) + for (int unsigned c = 0; c < n; c++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); + $display("[B2BT] %0dx%0d EB=%0d: xfer1 -> db=%0h, xfer2 -> db=%0h", m, n, eb, db1, db2); + do_transpose(m, n, eb, db1, e1); + do_transpose(m, n, eb, db2, e2); // back-to-back, distinct base + if (e1 == 0 && e2 == 0) $display("[B2BT] PASS: %0dx%0d EB=%0d both back-to-back transposes correct", m, n, eb); + else $display("[B2BT] FAIL: %0dx%0d EB=%0d xfer1=%0d xfer2=%0d", m, n, eb, e1, e2); + total += e1 + e2; + end + + if (total == 0) $display("[B2BT] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + else $fatal(1, "[B2BT] FAIL: %0d total mismatches", total); + repeat (5) @(posedge clk); + $finish(); + end + + initial begin #50_000_000; $fatal(1, "[B2BT] timeout"); end + +endmodule diff --git a/test/tb_idma_transpose_nd.sv b/test/tb_idma_transpose_nd.sv new file mode 100644 index 00000000..86cf4c44 --- /dev/null +++ b/test/tb_idma_transpose_nd.sv @@ -0,0 +1,283 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Daniel Keller + +// Self-checking multi-tile transpose testbench: idma_nd_midend (NumDim=4, +// transposed-stride program) -> idma_backend_rw_axi -> axi_sim_mem, exercising +// a full M x N transpose end-to-end. Reference: out_T[c][r] = in[r][c]. +// Sweeps a list of geometries internally (one elaboration per bus width). + +`include "axi/typedef.svh" +`include "idma/typedef.svh" + +module tb_idma_transpose_nd + import idma_pkg::*; +#( + parameter int unsigned DataWidth = 32, + parameter int unsigned AddrWidth = 32, + parameter int unsigned UserWidth = 1, + parameter int unsigned AxiIdWidth = 12, + parameter int unsigned TFLenWidth = 32, + parameter int unsigned BufferDepth = 3 +); + + localparam time TA = 1ns; + localparam time TT = 9ns; + localparam time TCK = 10ns; + + localparam int unsigned StrbWidth = DataWidth / 8; + // transpose buffers a full NE-beat tile before the first write; NE <= StrbWidth + localparam int unsigned AxIF = StrbWidth; + localparam int unsigned NumDim = 4; // 1D + {row, row-tile, col-tile} + localparam logic [NumDim-1:0][31:0] RepWidths = '{default: 32'd16}; + + // Geometry cases (M, N, EB) swept in one elaboration: aligned + edge + // (M or N not a multiple of NE) for int8/fp16/fp32. EB>StrbWidth cases skip. + localparam int unsigned NCases = 13; + localparam int unsigned Cases [NCases][3] = '{ + '{ 8, 8, 1}, '{16, 16, 1}, '{16, 8, 1}, '{ 8, 8, 2}, '{ 6, 8, 1}, + '{ 8, 6, 1}, '{ 6, 6, 1}, '{ 5, 7, 1}, '{10, 6, 1}, '{ 5, 5, 2}, + '{32, 24, 1}, '{ 9, 5, 4}, '{13, 19, 1} + }; + + // ── Types ── + typedef logic [AddrWidth-1:0] addr_t; + typedef logic [DataWidth-1:0] data_t; + typedef logic [StrbWidth-1:0] strb_t; + typedef logic [AxiIdWidth-1:0] id_t; + typedef logic [UserWidth-1:0] user_t; + typedef logic [TFLenWidth-1:0] tf_len_t; + typedef logic [31:0] reps_t; + + `AXI_TYPEDEF_AW_CHAN_T(axi_aw_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_W_CHAN_T(axi_w_chan_t, data_t, strb_t, user_t) + `AXI_TYPEDEF_B_CHAN_T(axi_b_chan_t, id_t, user_t) + `AXI_TYPEDEF_AR_CHAN_T(axi_ar_chan_t, addr_t, id_t, user_t) + `AXI_TYPEDEF_R_CHAN_T(axi_r_chan_t, data_t, id_t, user_t) + `AXI_TYPEDEF_REQ_T(axi_req_t, axi_aw_chan_t, axi_w_chan_t, axi_ar_chan_t) + `AXI_TYPEDEF_RESP_T(axi_rsp_t, axi_b_chan_t, axi_r_chan_t) + + `IDMA_TYPEDEF_FULL_REQ_T(idma_req_t, id_t, addr_t, tf_len_t) + `IDMA_TYPEDEF_FULL_RSP_T(idma_rsp_t, addr_t) + `IDMA_TYPEDEF_FULL_ND_REQ_T(idma_nd_req_t, idma_req_t, reps_t, addr_t) + + typedef struct packed { axi_ar_chan_t ar_chan; } axi_read_meta_channel_t; + typedef struct packed { axi_read_meta_channel_t axi; } read_meta_channel_t; + typedef struct packed { axi_aw_chan_t aw_chan; } axi_write_meta_channel_t; + typedef struct packed { axi_write_meta_channel_t axi; } write_meta_channel_t; + + // ── Signals ── + logic clk, rst_n; + idma_req_t idma_req; logic req_valid, req_ready; + idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; + idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; + idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; + axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; + axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; + idma_busy_t busy; logic nd_busy; + + assign idma_eh_req = '0; + assign eh_req_valid = 1'b0; + + // ── Clock / reset ── + clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + + // ── AXI sim memory (read+write joined) ── + axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( + .clk_i(clk), .rst_ni(rst_n), + .slv_read_req_i(axi_read_req), .slv_read_resp_o(axi_read_rsp), + .slv_write_req_i(axi_write_req), .slv_write_resp_o(axi_write_rsp), + .mst_req_o(axi_req), .mst_resp_i(axi_rsp) + ); + assign axi_req_mem = axi_req; + assign axi_rsp = axi_rsp_mem; + + axi_sim_mem #( + .AddrWidth(AddrWidth), .DataWidth(DataWidth), .IdWidth(AxiIdWidth), .UserWidth(UserWidth), + .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .WarnUninitialized(1'b0), .ClearErrOnAccess(1'b1), .ApplDelay(TA), .AcqDelay(TT) + ) i_axi_sim_mem ( + .clk_i(clk), .rst_ni(rst_n), .axi_req_i(axi_req_mem), .axi_rsp_o(axi_rsp_mem), + .mon_r_last_o(), .mon_r_beat_count_o(), .mon_r_user_o(), .mon_r_id_o(), + .mon_r_data_o(), .mon_r_addr_o(), .mon_r_valid_o(), + .mon_w_last_o(), .mon_w_beat_count_o(), .mon_w_user_o(), .mon_w_id_o(), + .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() + ); + + // ── ND midend: ND transpose descriptor -> 1D bursts ── + idma_nd_midend #( + .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), + .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) + ) i_nd_midend ( + .clk_i(clk), .rst_ni(rst_n), + .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), + .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), + .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), + .busy_o(nd_busy) + ); + + // ── Backend (rw_axi) with transpose engine ── + idma_backend_rw_axi #( + .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), + .UserWidth(UserWidth), .TFLenWidth(TFLenWidth), .MaskInvalidData(1'b1), .BufferDepth(BufferDepth), + .RAWCouplingAvail(1'b1), .HardwareLegalizer(1'b1), .RejectZeroTransfers(1'b1), + .ErrorCap(idma_pkg::NO_ERROR_HANDLING), .PrintFifoInfo(1'b0), .NumAxInFlight(AxIF), .MemSysDepth(0), + .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_eh_req_t(idma_eh_req_t), + .idma_busy_t(idma_busy_t), .axi_req_t(axi_req_t), .axi_rsp_t(axi_rsp_t), + .write_meta_channel_t(write_meta_channel_t), .read_meta_channel_t(read_meta_channel_t) + ) i_idma_backend ( + .clk_i(clk), .rst_ni(rst_n), .testmode_i(1'b0), + .idma_req_i(idma_req), .req_valid_i(req_valid), .req_ready_o(req_ready), + .idma_rsp_o(idma_rsp), .rsp_valid_o(rsp_valid), .rsp_ready_i(rsp_ready), + .idma_eh_req_i(idma_eh_req), .eh_req_valid_i(eh_req_valid), .eh_req_ready_o(eh_req_ready), + .axi_read_req_o(axi_read_req), .axi_read_rsp_i(axi_read_rsp), + .axi_write_req_o(axi_write_req), .axi_write_rsp_i(axi_write_rsp), .busy_o(busy) + ); + + // watchdogs to surface deadlocks rather than hang forever + stream_watchdog #(.NumCycles(2000)) i_r_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_rsp.r_valid), .ready_i(axi_req.r_ready)); + stream_watchdog #(.NumCycles(2000)) i_w_wd (.clk_i(clk), .rst_ni(rst_n), .valid_i(axi_req.w_valid), .ready_i(axi_rsp.w_ready)); + + // ── Stimulus + check via sim-memory backdoor ── + addr_t sb = 'h0000_1000; + addr_t db = 'h0000_4000; + + // every AW (incl. wstrb=0 padding rows) must stay in the active case's padded + // dst allocation [chk_db, chk_aw_hi) — else a strict slave would DECERR + logic chk_active = 1'b0; + addr_t chk_db, chk_aw_hi; + always @(posedge clk) if (rst_n && chk_active && axi_write_req.aw_valid && axi_write_rsp.aw_ready) begin + if (axi_write_req.aw.addr < chk_db || axi_write_req.aw.addr >= chk_aw_hi) + $fatal(1, "[TPN] AW 0x%0h outside dst alloc [0x%0h,0x%0h) — would DECERR on a strict slave", + axi_write_req.aw.addr, chk_db, chk_aw_hi); + end + + task automatic wr_mem(input addr_t a, input logic [7:0] d); i_axi_sim_mem.mem[a] = d; endtask + function automatic logic [7:0] rd_mem(input addr_t a); + return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; + endfunction + + // Run one M x N transpose of EB-byte elements; returns the mismatch count. + task automatic run_case(input int unsigned m, input int unsigned n, input int unsigned eb, + output int unsigned errs); + automatic int unsigned ne = StrbWidth / eb; // tile side (elements) + automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; + automatic int unsigned yt = (m + ne - 1) / ne; // row-tiles + automatic int unsigned nt = (n + ne - 1) / ne; // col-tiles + automatic int unsigned mp = yt * ne; // padded Aᵀ row pitch (StrbWidth-aligned) + errs = 0; + + // init source matrix (row-major, m x n elements of eb bytes) + for (int unsigned r = 0; r < m; r++) + for (int unsigned c = 0; c < n; c++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); + + // sentinel-fill the full padded Aᵀ extent; padding cols/rows must stay sentinel + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + for (int unsigned b = 0; b < eb; b++) + wr_mem(db + (i*mp + j)*eb + b, 8'hCC); + + // arm the AW-bounds guard for this case + chk_db = db; + chk_aw_hi = db + addr_t'(nt*ne*mp*eb); + chk_active = 1'b1; + + // ── transposed-stride ND program (routing-plan §4.2) ── + nd_req = '0; + nd_req.burst_req.length = tf_len_t'(ne*eb); // one tile-row = StrbWidth bytes + nd_req.burst_req.src_addr = sb; + nd_req.burst_req.dst_addr = db; + nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; + nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + nd_req.burst_req.opt.beo.decouple_rw = 1'b1; + nd_req.burst_req.opt.beo.decouple_aw = 1'b1; + nd_req.burst_req.opt.beo.src_max_llen = '0; + nd_req.burst_req.opt.beo.dst_max_llen = '0; + nd_req.burst_req.opt.compute.enable = 1'b1; + nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); + nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); + nd_req.burst_req.opt.last = 1'b1; + // ND midend strides are INCREMENTAL deltas (added on dim roll-over), NOT + // absolute pitches. Aᵀ uses padded pitch mp*eb (aligned writes); src keeps + // n*eb (misaligned reads coalesce in the pre-engine buffer). + nd_req.d_req[0].reps = reps_t'(ne); + nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); + nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); + nd_req.d_req[1].reps = reps_t'(yt); + nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); + nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); + nd_req.d_req[2].reps = reps_t'(nt); + nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); + nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); + + $display("[TPN] case %0dx%0d EB=%0d (NE=%0d, %0dx%0d tiles)", m, n, eb, ne, yt, nt); + nd_req_valid = 1'b1; + // drop valid on accept; holding it one cycle past makes the midend re-walk the request + do @(posedge clk); while (!nd_req_ready); + nd_req_valid = 1'b0; + nd_req = '0; + + // wait for ND completion + drain + while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + repeat (20) @(posedge clk); + chk_active = 1'b0; + + // check 1 (data): out_T[c][r] == in[r][c], Aᵀ at padded pitch mp + for (int unsigned c = 0; c < n; c++) + for (int unsigned r = 0; r < m; r++) + for (int unsigned b = 0; b < eb; b++) begin + automatic logic [7:0] got = rd_mem(db + (c*mp + r)*eb + b); + automatic logic [7:0] exp = rd_mem(sb + (r*n + c)*eb + b); + if (got !== exp) begin + errs++; + if (errs <= 12) $display("[TPN] MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", c, r, b, got, exp); + end + end + // check 2: padding cols [m,mp) and padding rows [n,nt*ne) must stay sentinel + for (int unsigned i = 0; i < nt*ne; i++) + for (int unsigned j = 0; j < mp; j++) + if (i >= n || j >= m) + for (int unsigned b = 0; b < eb; b++) begin + automatic logic [7:0] got = rd_mem(db + (i*mp + j)*eb + b); + if (got !== 8'hCC) begin + errs++; + if (errs <= 12) $display("[TPN] PADDING CLOBBERED at row=%0d col=%0d b%0d=%02h (exp CC)", i, j, b, got); + end + end + endtask + + initial begin + automatic int unsigned total = 0; + automatic int unsigned ce; + nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + @(posedge rst_n); + repeat (5) @(posedge clk); + + for (int unsigned k = 0; k < NCases; k++) begin + if (Cases[k][2] > StrbWidth) continue; // element must fit the bus + run_case(Cases[k][0], Cases[k][1], Cases[k][2], ce); + if (ce == 0) $display("[TPN] PASS: %0dx%0d EB=%0d", Cases[k][0], Cases[k][1], Cases[k][2]); + else $display("[TPN] FAIL: %0dx%0d EB=%0d (%0d mismatches)", Cases[k][0], Cases[k][1], Cases[k][2], ce); + total += ce; + end + + if (total == 0) $display("[TPN] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + else $fatal(1, "[TPN] FAIL: %0d total mismatches", total); + repeat (5) @(posedge clk); + $finish(); + end + + initial begin #100_000_000; $fatal(1, "[TPN] timeout"); end + +endmodule diff --git a/util/gen_idma.py b/util/gen_idma.py index 2aff908e..537b9adb 100644 --- a/util/gen_idma.py +++ b/util/gen_idma.py @@ -12,7 +12,7 @@ import argparse import sys -from mario.util import prepare_ids, prepare_fids +from mario.util import prepare_ids, prepare_compute_ids, prepare_fids from mario.database import read_database from mario.transport_layer import render_transport_layer from mario.legalizer import render_legalizer @@ -44,6 +44,8 @@ def main(): parser.add_argument('--entity', choices=sorted(GENABLE_ENTITIES), dest='entity', required=True, help='The entity to generate from a given configuration.') parser.add_argument('--ids', dest='ids', nargs='*', help='configuration IDs') + parser.add_argument('--compute-ids', dest='compute_ids', nargs='*', default=[], + help='configuration IDs with on-the-fly compute enabled (IDMA_VIDMA_IDS)') parser.add_argument('--fids', dest='fids', nargs='*', help='frontend IDs') parser.add_argument('--db', dest='db', nargs='*', help='Database files') parser.add_argument('--tpl', dest='tpl', required=True, help='Template file') @@ -51,16 +53,17 @@ def main(): # prepare database and ids protocol_ids = prepare_ids(args.ids) + compute_cfg = prepare_compute_ids(args.compute_ids) frontend_ids = prepare_fids(args.fids) protocol_db = read_database(args.db) # decide what to render if args.entity == 'transport': - print(render_transport_layer(protocol_ids, protocol_db, args.tpl)) + print(render_transport_layer(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'legalizer': - print(render_legalizer(protocol_ids, protocol_db, args.tpl)) + print(render_legalizer(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'backend': - print(render_backend(protocol_ids, protocol_db, args.tpl)) + print(render_backend(protocol_ids, protocol_db, args.tpl, compute_cfg)) elif args.entity == 'vsim_wave': print(render_vsim_wave(protocol_ids, protocol_db, args.tpl)) elif args.entity == 'synth_wrapper': diff --git a/util/mario/backend.py b/util/mario/backend.py index 3a6545bd..ff527926 100644 --- a/util/mario/backend.py +++ b/util/mario/backend.py @@ -12,7 +12,7 @@ from mario.util import eval_key, prot_key -def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_backend(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None) -> str: """Generate backend""" backend_rendered = '' @@ -42,6 +42,12 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: srp = len(used_read_prots) == 1 and not any_mh_r swp = len(used_write_prots) == 1 and not any_mh_w + # on-the-fly compute requires a single AXI write port + enable_compute = prot_id in (compute_cfg or {}) + if enable_compute and not (swp and used_write_prots[0] == 'axi'): + raise ValueError( + f'compute (IDMA_VIDMA_IDS) requires a single AXI write port: {prot_id}') + # create context context = { 'name_uniqueifier': prot_id, @@ -51,6 +57,8 @@ def render_backend(prot_ids: dict, db: dict, tpl_file: str) -> str: 'used_protocols': prot_ids[prot_id]['used'], 'one_read_port': srp, 'one_write_port': swp, + 'enable_compute': enable_compute, + 'compute_ops': compute_cfg[prot_id]['ops'] if enable_compute else [], 'used_non_bursting_write_protocols': prot_key(used_write_prots, 'bursts', 'not_supported', db), 'combined_aw_and_w': diff --git a/util/mario/legalizer.py b/util/mario/legalizer.py index 1724c00e..51757771 100644 --- a/util/mario/legalizer.py +++ b/util/mario/legalizer.py @@ -21,7 +21,7 @@ def prot_force_decouple(used_prots: list, db: dict) -> list: return res -def render_legalizer(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_legalizer(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None) -> str: """Generate legalizer""" legalizer_rendered = '' @@ -71,6 +71,7 @@ def render_legalizer(prot_ids: dict, db: dict, tpl_file: str) -> str: 'used_protocols': prot_ids[prot_id]['used'], 'one_read_port': srp, 'one_write_port': swp, + 'enable_compute': prot_id in (compute_cfg or {}), 'no_read_bursting': not has_read_bursting, 'has_page_read_bursting': diff --git a/util/mario/transport_layer.py b/util/mario/transport_layer.py index 1c6850af..41c5df6f 100644 --- a/util/mario/transport_layer.py +++ b/util/mario/transport_layer.py @@ -253,7 +253,8 @@ def render_write_mgr_inst(prot_id: str, prot_ids: dict, db: dict) -> dict: return res -def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str) -> str: +def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str, compute_cfg: dict = None + ) -> str: """Generate Transport Layer""" transport_rendered = '' @@ -286,6 +287,11 @@ def render_transport_layer(prot_ids: dict, db: dict, tpl_file: str) -> str: 'one_write_port': len(prot_ids[prot_id]['aw']) == 1 and not any_mh['aw'], 'mh_format': mh_format, 'any_mh': any_mh, + 'enable_compute': prot_id in (compute_cfg or {}), + 'compute_ops': + compute_cfg[prot_id]['ops'] if prot_id in (compute_cfg or {}) else [], + 'compute_full_duplex': + compute_cfg[prot_id]['full_duplex'] if prot_id in (compute_cfg or {}) else True, 'rendered_read_ports': render_read_mgr_inst(prot_id, prot_ids, db), 'rendered_write_ports': render_write_mgr_inst(prot_id, prot_ids, db) } diff --git a/util/mario/util.py b/util/mario/util.py index 8cc2d492..f19f9d01 100644 --- a/util/mario/util.py +++ b/util/mario/util.py @@ -164,3 +164,30 @@ def prepare_fids(fe_strs: list) -> dict: res[f'reg{reg[0]}_{reg[1]}d'] = reg return res + + +def prepare_compute_ids(compute_id_strs: list) -> dict: + """Parses compute configuration IDs: [:[,...]][:fd|hd]""" + res = {} + for cid_str in (compute_id_strs or []): + parts = cid_str.split(':') + variant = parts[0] + if variant == '': + print(f'[MARIO] empty compute variant in "{cid_str}"', file=sys.stderr) + sys.exit(1) + if variant in res: + print(f'[MARIO] duplicate compute variant "{variant}"', file=sys.stderr) + sys.exit(1) + ops = ['transpose'] + full_duplex = True + for part in parts[1:]: + if part in ('fd', 'hd'): + full_duplex = part == 'fd' + else: + ops = part.split(',') + for op in ops: + if op not in ('transpose',): + print(f'[MARIO] {op} is a non-supported compute op in {cid_str}', file=sys.stderr) + sys.exit(1) + res[variant] = {'ops': ops, 'full_duplex': full_duplex} + return res