From 0d0c6d0376bcf9ce8366de615efb98d7f5605d00 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Tue, 14 Jul 2026 20:40:43 +0800 Subject: [PATCH] Optimize matmul DMA with chained 256-beat bursts --- .../Bus_interconnects/axi2sram_sp_external.v | 32 ++++++-- rtl/ip/matmul/matmul_axi_slave.v | 77 +++++++++++++++---- 2 files changed, 88 insertions(+), 21 deletions(-) diff --git a/rtl/ip/Bus_interconnects/axi2sram_sp_external.v b/rtl/ip/Bus_interconnects/axi2sram_sp_external.v index fa6bedf..32bac78 100644 --- a/rtl/ip/Bus_interconnects/axi2sram_sp_external.v +++ b/rtl/ip/Bus_interconnects/axi2sram_sp_external.v @@ -106,7 +106,7 @@ module axi2sram_sp_external #( {{(AXI_ADDR_WIDTH-1){1'b0}}, 1'b1} << LOG_NR_BYTES; reg [AXI_ADDR_WIDTH-1:0] req_addr_d, req_addr_q; - reg [7:0] cnt_d, cnt_q; + reg [8:0] cnt_d, cnt_q; reg [AXI_ADDR_WIDTH-1:0] wrap_mask_d, wrap_mask_q; reg req_bank_d, req_bank_q; reg [AXI_DATA_WIDTH-1:0] read_data_q; @@ -240,7 +240,7 @@ module axi2sram_sp_external #( s_rvalid = 1'b1; s_rdata = read_data_q; s_rid = ax_req_q_id; - s_rlast = (cnt_q == ax_req_q_len + 1); + s_rlast = (cnt_q == ({1'b0, ax_req_q_len} + 9'd1)); if (FAST_READ_BURST && !s_rlast) begin req_o = 1'b1; addr_o = next_addr; @@ -249,8 +249,30 @@ module axi2sram_sp_external #( if (s_rready) begin // we sent the last byte -> go back to idle if (s_rlast) begin - state_d = IDLE; - // we already got everything + if (s_arvalid) begin + // Accept the next burst without an intervening + // IDLE cycle and start its SRAM address phase. + s_arready = 1'b1; + ax_req_d_id = s_arid; + ax_req_d_addr = s_araddr; + ax_req_d_len = s_arlen; + ax_req_d_size = s_arsize; + ax_req_d_burst = s_arburst; + wrap_mask_d = ar_wrap_mask; + req_bank_d = s_araddr[22]; + req_o = 1'b1; + addr_o = s_araddr; + bank_o = s_araddr[22]; + req_addr_d = s_araddr; + // ARVALID was presented during the preceding + // beat, so the async-SRAM address has a full + // cycle to settle before this capture edge. + read_capture = 1'b1; + cnt_d = 1; + state_d = READ; + end else begin + state_d = IDLE; + end end else begin if (FAST_READ_BURST) begin @@ -323,7 +345,7 @@ module axi2sram_sp_external #( ax_req_q_len <= 8'h0; ax_req_q_size <= 3'h0; req_addr_q <= 'h0; - cnt_q <= 8'h0; + cnt_q <= 9'h0; wrap_mask_q <= 'h0; req_bank_q <= 1'b0; read_data_q <= 'h0; diff --git a/rtl/ip/matmul/matmul_axi_slave.v b/rtl/ip/matmul/matmul_axi_slave.v index d291f2e..5a4f595 100644 --- a/rtl/ip/matmul/matmul_axi_slave.v +++ b/rtl/ip/matmul/matmul_axi_slave.v @@ -481,10 +481,19 @@ module matmul_axi_slave ( localparam DMA_RD_ADDR = 2'd1; localparam DMA_RD_DATA = 2'd2; reg [1:0] dma_rd_state; - reg [4:0] dma_rd_count; + reg [7:0] dma_rd_count; reg dma_rd_bank; + wire [4:0] dma_rd_group_word_idx = dma_rd_count[4:0]; wire [3:0] dma_rd_word_idx = dma_rd_count[3:0]; + wire dma_rd_fill_bank = dma_rd_bank ^ dma_rd_count[5]; wire dma_rd_commit = (dma_rd_state == DMA_RD_DATA) && m_axi_rvalid && m_axi_rready; + wire dma_bank0_free = !bank_loaded[0] && !bank_compute_pending[0] && !req_0 && + !(state == C_RUN && cur_bank == 1'b0); + wire dma_bank1_free = !bank_loaded[1] && !bank_compute_pending[1] && !req_1 && + !(state == C_RUN && cur_bank == 1'b1); + wire dma_load_bank_free = dma_load_bank ? dma_bank1_free : dma_bank0_free; + wire dma_rd_fill_bank_free = dma_rd_fill_bank ? dma_bank1_free : dma_bank0_free; + wire dma_rd_next_bank_free = dma_rd_fill_bank ? dma_bank0_free : dma_bank1_free; localparam DMA_WR_IDLE = 2'd0; localparam DMA_WR_ADDR = 2'd1; @@ -519,13 +528,13 @@ module matmul_axi_slave ( dma_done_clear_pulse <= 1'b0; if (dma_rd_commit) begin - if (dma_rd_bank == 1'b0) begin - if (dma_rd_count < 5'd16) + if (dma_rd_fill_bank == 1'b0) begin + if (dma_rd_group_word_idx < 5'd16) b0_a[dma_rd_word_idx] <= m_axi_rdata; else b0_b[dma_rd_word_idx] <= m_axi_rdata; end else begin - if (dma_rd_count < 5'd16) + if (dma_rd_group_word_idx < 5'd16) b1_a[dma_rd_word_idx] <= m_axi_rdata; else b1_b[dma_rd_word_idx] <= m_axi_rdata; @@ -960,7 +969,7 @@ module matmul_axi_slave ( dma_load_bank <= 1'b0; dma_store_bank <= 1'b0; dma_rd_state <= DMA_RD_IDLE; - dma_rd_count <= 5'd0; + dma_rd_count <= 8'd0; dma_rd_bank <= 1'b0; dma_wr_state <= DMA_WR_IDLE; dma_wr_count <= 6'd0; @@ -1058,14 +1067,12 @@ module matmul_axi_slave ( DMA_RD_IDLE: begin m_axi_arvalid <= 1'b0; m_axi_rready <= 1'b0; - if (dma_busy && load_group < dma_groups && !bank_loaded[dma_load_bank] && - !bank_compute_pending[dma_load_bank] && - !((dma_load_bank == 1'b0) ? req_0 : req_1) && - !(state == C_RUN && cur_bank == dma_load_bank)) begin + if (dma_busy && load_group < dma_groups && dma_load_bank_free) begin dma_rd_bank <= dma_load_bank; - dma_rd_count <= 5'd0; + dma_rd_count <= 8'd0; m_axi_araddr <= dma_next_rd_addr; - m_axi_arlen <= 8'd31; + m_axi_arlen <= ((dma_src_stride == 32'd128) && + ((dma_groups - load_group) >= 32'd8)) ? 8'd255 : 8'd31; m_axi_arsize <= 3'd2; m_axi_arburst <= 2'b01; m_axi_arvalid <= 1'b1; @@ -1080,16 +1087,54 @@ module matmul_axi_slave ( end end DMA_RD_DATA: begin + if (!m_axi_rready && dma_rd_fill_bank_free) + m_axi_rready <= 1'b1; + if (m_axi_rvalid && m_axi_rready) begin - if (m_axi_rlast) begin - m_axi_rready <= 1'b0; - bank_loaded[dma_rd_bank] <= 1'b1; + // Present the next request one beat early so the SRAM + // slave can accept it together with the current RLAST. + if ((dma_rd_count == 8'd254) && dma_busy && + ((load_group + 32'd1) < dma_groups) && dma_rd_next_bank_free) begin + m_axi_araddr <= dma_next_rd_addr + dma_src_stride; + m_axi_arlen <= ((dma_src_stride == 32'd128) && + ((dma_groups - (load_group + 32'd1)) >= 32'd8)) ? + 8'd255 : 8'd31; + m_axi_arvalid <= 1'b1; + end + + if (dma_rd_group_word_idx == 5'd31) begin + bank_loaded[dma_rd_fill_bank] <= 1'b1; load_group <= load_group + 32'd1; dma_next_rd_addr <= dma_next_rd_addr + dma_src_stride; dma_load_bank <= ~dma_load_bank; - dma_rd_state <= DMA_RD_IDLE; + end + + if (m_axi_rlast) begin + m_axi_rready <= 1'b0; + if (dma_busy && ((load_group + 32'd1) < dma_groups) && + dma_rd_next_bank_free) begin + dma_rd_bank <= ~dma_rd_fill_bank; + dma_rd_count <= 8'd0; + if (m_axi_arvalid && m_axi_arready) begin + m_axi_arvalid <= 1'b0; + m_axi_rready <= 1'b1; + dma_rd_state <= DMA_RD_DATA; + end else begin + m_axi_araddr <= dma_next_rd_addr + dma_src_stride; + m_axi_arlen <= ((dma_src_stride == 32'd128) && + ((dma_groups - (load_group + 32'd1)) >= 32'd8)) ? + 8'd255 : 8'd31; + m_axi_arvalid <= 1'b1; + dma_rd_state <= DMA_RD_ADDR; + end + end else begin + m_axi_arvalid <= 1'b0; + dma_rd_state <= DMA_RD_IDLE; + end end else begin - dma_rd_count <= dma_rd_count + 5'd1; + dma_rd_count <= dma_rd_count + 8'd1; + if ((dma_rd_group_word_idx == 5'd31) && !dma_rd_next_bank_free) + m_axi_rready <= 1'b0; end end end