From 27cdfb713ac8af2d79e122f6b7647d0baaed8a1a Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 4 Jul 2026 19:59:49 +0800 Subject: [PATCH] Optimize CDMA matmul bridge scheduling --- rtl/ip/cdma/cdma_matmul_bridge.sv | 38 ++++++++++++++---- rtl/ip/cdma/snix_axil_cdma_mux.sv | 41 ++++++++++--------- rtl/ip/matmul/matmul_axi_slave.v | 67 +++++++++++++++++++++++++------ 3 files changed, 106 insertions(+), 40 deletions(-) diff --git a/rtl/ip/cdma/cdma_matmul_bridge.sv b/rtl/ip/cdma/cdma_matmul_bridge.sv index 61e7e89..88e8177 100644 --- a/rtl/ip/cdma/cdma_matmul_bridge.sv +++ b/rtl/ip/cdma/cdma_matmul_bridge.sv @@ -103,6 +103,9 @@ module cdma_matmul_bridge #( logic [31:0] seq_store_idx; logic seq_compute_bank; logic seq_skip_writeback; + logic seq_defer_load_start; + logic seq_deferred_start_valid; + logic seq_deferred_start_bank; wire [7:0] wr_off = s_axil_awaddr[7:0]; wire [7:0] rd_off = s_axil_araddr[7:0]; @@ -223,6 +226,9 @@ module cdma_matmul_bridge #( seq_store_idx <= 32'd0; seq_compute_bank <= 1'b0; seq_skip_writeback <= 1'b0; + seq_defer_load_start <= 1'b0; + seq_deferred_start_valid <= 1'b0; + seq_deferred_start_bank <= 1'b0; cdma_req_valid <= 1'b0; cdma_req_mode <= MODE_MEM2EP; cdma_req_src <= '0; @@ -254,6 +260,9 @@ module cdma_matmul_bridge #( seq_store_idx <= 32'd0; seq_compute_bank <= 1'b0; seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1]; + seq_defer_load_start <= 1'b0; + seq_deferred_start_valid <= 1'b0; + seq_deferred_start_bank <= 1'b0; mat_crc_clear <= 1'b1; mat_done_clear <= 1'b1; seq_state <= SQ_CLEAR; @@ -285,13 +294,20 @@ module cdma_matmul_bridge #( SQ_LOAD_WAIT: begin if (cdma_req_done) begin - mat_start_valid <= 1'b1; - mat_start_bank <= cdma_req_ep_bank[0]; - seq_load_idx <= seq_load_idx + 32'd1; - if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin - seq_state <= SQ_LOAD_START; + seq_load_idx <= seq_load_idx + 32'd1; + if (seq_defer_load_start) begin + seq_defer_load_start <= 1'b0; + seq_deferred_start_valid <= 1'b1; + seq_deferred_start_bank <= cdma_req_ep_bank[0]; + seq_state <= SQ_STORE_START; end else begin - seq_state <= SQ_WAIT_DONE; + mat_start_valid <= 1'b1; + mat_start_bank <= cdma_req_ep_bank[0]; + if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin + seq_state <= SQ_LOAD_START; + end else begin + seq_state <= SQ_WAIT_DONE; + end end end end @@ -309,6 +325,9 @@ module cdma_matmul_bridge #( end else begin seq_state <= SQ_WAIT_DONE; end + end else if (seq_load_idx < seq_groups) begin + seq_defer_load_start <= 1'b1; + seq_state <= SQ_LOAD_START; end else begin seq_state <= SQ_STORE_START; end @@ -338,10 +357,15 @@ module cdma_matmul_bridge #( if (cdma_req_done) begin seq_store_idx <= seq_store_idx + 32'd1; seq_compute_bank <= ~seq_compute_bank; + if (seq_deferred_start_valid) begin + mat_start_valid <= 1'b1; + mat_start_bank <= seq_deferred_start_bank; + seq_deferred_start_valid <= 1'b0; + end if ((seq_store_idx + 32'd1) >= seq_groups) begin seq_state <= SQ_DONE; end else if (seq_load_idx < seq_groups) begin - seq_state <= SQ_LOAD_START; + seq_state <= SQ_WAIT_DONE; end else begin seq_state <= SQ_WAIT_DONE; end diff --git a/rtl/ip/cdma/snix_axil_cdma_mux.sv b/rtl/ip/cdma/snix_axil_cdma_mux.sv index b1cfc96..2af0e14 100644 --- a/rtl/ip/cdma/snix_axil_cdma_mux.sv +++ b/rtl/ip/cdma/snix_axil_cdma_mux.sv @@ -401,7 +401,6 @@ module snix_axil_cdma_mux #( logic ep_rready; logic ext_active; - logic ext_start_pulse; logic [1:0] ext_mode_r; logic [ADDR_WIDTH-1:0] ext_src_r; logic [ADDR_WIDTH-1:0] ext_dst_r; @@ -410,22 +409,23 @@ module snix_axil_cdma_mux #( logic [3:0] ext_ep_bank_r; logic [7:0] ext_ep_word_off_r; - wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_start_pulse; - wire select_ext_ep = ext_active || ext_start_pulse; + wire ext_accept = ext_req_valid && ext_req_ready; + wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_accept; + wire select_ext_ep = ext_active || ext_accept; // Dynamic routing to the engine based on current active channel - assign engine_src = select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]; - assign engine_dst = select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]; - assign engine_bytes = select_ext_ep ? ext_bytes_r : ch_len[cur_ch]; + assign engine_src = ext_accept ? ext_req_src : (select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]); + assign engine_dst = ext_accept ? ext_req_dst : (select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]); + assign engine_bytes = ext_accept ? ext_req_bytes : (select_ext_ep ? ext_bytes_r : ch_len[cur_ch]); assign engine_len = ch_ctrl[cur_ch][13:6]; assign engine_size = ch_ctrl[cur_ch][5:3]; - assign engine_mode = select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]; - assign engine_ep_bank = select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]; - assign engine_ep_word_off = select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]; - assign engine_ep_id = select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]; + assign engine_mode = ext_accept ? ext_req_mode : (select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]); + assign engine_ep_bank = ext_accept ? ext_req_ep_bank : (select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]); + assign engine_ep_word_off = ext_accept ? ext_req_ep_word_off : (select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]); + assign engine_ep_id = ext_accept ? ext_req_ep_id : (select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]); assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done; assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM); - assign ep_start = ext_start_pulse || (engine_start && (engine_mode != MODE_MEM2MEM)); + assign ep_start = ext_accept || (engine_start && (engine_mode != MODE_MEM2MEM)); assign core_awready = (!use_ep_engine) && mm2mm_awready; assign core_wready = (!use_ep_engine) && mm2mm_wready; @@ -562,12 +562,11 @@ module snix_axil_cdma_mux #( // ========================================== // Generic external endpoint requester // ========================================== - assign ext_req_ready = !ext_active && (state == IDLE); + assign ext_req_ready = !ext_active && (state == IDLE) && (ep_state == EP_IDLE); always_ff @(posedge clk) begin if (!rst_n) begin ext_active <= 1'b0; - ext_start_pulse <= 1'b0; ext_req_done <= 1'b0; ext_mode_r <= MODE_MEM2EP; ext_src_r <= '0; @@ -577,12 +576,10 @@ module snix_axil_cdma_mux #( ext_ep_bank_r <= '0; ext_ep_word_off_r <= '0; end else begin - ext_start_pulse <= 1'b0; ext_req_done <= 1'b0; - if (ext_req_valid && ext_req_ready) begin + if (ext_accept) begin ext_active <= 1'b1; - ext_start_pulse <= 1'b1; ext_mode_r <= ext_req_mode; ext_src_r <= ext_req_src; ext_dst_r <= ext_req_dst; @@ -640,8 +637,10 @@ module snix_axil_cdma_mux #( if (mm2mm_rvalid && ep_rready) begin ep_word_addr_r <= ep_word_addr_r + 8'd1; ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0; - if (mm2mm_rlast) - ep_state <= EP_DONE; + if (mm2mm_rlast) begin + ep_done <= 1'b1; + ep_state <= EP_IDLE; + end end end @@ -660,8 +659,10 @@ module snix_axil_cdma_mux #( end EP_BRESP: begin - if (mm2mm_bvalid && ep_bready) - ep_state <= EP_DONE; + if (mm2mm_bvalid && ep_bready) begin + ep_done <= 1'b1; + ep_state <= EP_IDLE; + end end EP_DONE: begin diff --git a/rtl/ip/matmul/matmul_axi_slave.v b/rtl/ip/matmul/matmul_axi_slave.v index 9c4fbea..39112bc 100644 --- a/rtl/ip/matmul/matmul_axi_slave.v +++ b/rtl/ip/matmul/matmul_axi_slave.v @@ -582,14 +582,39 @@ module matmul_axi_slave ( // Pipeline delay shift registers for accumulation (4 stages = 3 multiplier pipe + 1) reg [3:0] acc_idx_d1, acc_idx_d2, acc_idx_d3, acc_idx_d4; reg acc_bank_d1, acc_bank_d2, acc_bank_d3, acc_bank_d4; + reg acc_valid_d1, acc_valid_d2, acc_valid_d3, acc_valid_d4; reg [4:0] pipe_cnt; wire [65:0] calc_c_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) + ({2'b0, mult_p[2]} + {2'b0, mult_p[3]}); - wire calc_crc_valid = (state == C_RUN) && (pipe_cnt >= 5'd4) && (pipe_cnt < 5'd20); + wire calc_crc_valid = acc_valid_d4; wire [31:0] crc_calc_0 = crc32_word_next(crc_reg, calc_c_result[31:0]); wire [31:0] crc_calc_1 = crc32_word_next(crc_calc_0, calc_c_result[63:32]); wire [31:0] crc_calc_2 = crc32_word_next(crc_calc_1, {30'b0, calc_c_result[65:64]}); + task start_calc_bank; + input bank; + begin + cur_bank <= bank; + busy <= 1'b1; + state <= C_RUN; + pipe_cnt <= 5'd1; + acc_idx_d1 <= 4'd0; + acc_bank_d1 <= bank; + acc_valid_d1 <= 1'b1; + if (bank == 1'b0) begin + m_in_a[0] <= b0_a[4'd0]; m_in_b[0] <= b0_b[4'd0]; + m_in_a[1] <= b0_a[4'd1]; m_in_b[1] <= b0_b[4'd4]; + m_in_a[2] <= b0_a[4'd2]; m_in_b[2] <= b0_b[4'd8]; + m_in_a[3] <= b0_a[4'd3]; m_in_b[3] <= b0_b[4'd12]; + end else begin + m_in_a[0] <= b1_a[4'd0]; m_in_b[0] <= b1_b[4'd0]; + m_in_a[1] <= b1_a[4'd1]; m_in_b[1] <= b1_b[4'd4]; + m_in_a[2] <= b1_a[4'd2]; m_in_b[2] <= b1_b[4'd8]; + m_in_a[3] <= b1_a[4'd3]; m_in_b[3] <= b1_b[4'd12]; + end + end + endtask + // CRC tracks completed C elements directly, independent of result writeback. always @(posedge clk) begin if (!resetn) begin @@ -616,6 +641,10 @@ module matmul_axi_slave ( acc_bank_d2 <= 1'b0; acc_bank_d3 <= 1'b0; acc_bank_d4 <= 1'b0; + acc_valid_d1 <= 1'b0; + acc_valid_d2 <= 1'b0; + acc_valid_d3 <= 1'b0; + acc_valid_d4 <= 1'b0; end else begin calc_done_pulse <= 1'b0; @@ -623,24 +652,24 @@ module matmul_axi_slave ( acc_idx_d4 <= acc_idx_d3; acc_idx_d3 <= acc_idx_d2; acc_idx_d2 <= acc_idx_d1; - acc_idx_d1 <= pipe_cnt[3:0]; + acc_idx_d1 <= 4'd0; acc_bank_d4 <= acc_bank_d3; acc_bank_d3 <= acc_bank_d2; acc_bank_d2 <= acc_bank_d1; - acc_bank_d1 <= cur_bank; + acc_bank_d1 <= 1'b0; + acc_valid_d4 <= acc_valid_d3; + acc_valid_d3 <= acc_valid_d2; + acc_valid_d2 <= acc_valid_d1; + acc_valid_d1 <= 1'b0; case (state) C_IDLE: begin - pipe_cnt <= 5'd0; if (req_0) begin - cur_bank <= 1'b0; - busy <= 1'b1; - state <= C_RUN; + start_calc_bank(1'b0); end else if (req_1) begin - cur_bank <= 1'b1; - busy <= 1'b1; - state <= C_RUN; + start_calc_bank(1'b1); end else begin + pipe_cnt <= 5'd0; busy <= 1'b0; end end @@ -648,6 +677,9 @@ module matmul_axi_slave ( C_RUN: begin // Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds) if (pipe_cnt < 5'd16) begin + acc_idx_d1 <= pipe_cnt[3:0]; + acc_bank_d1 <= cur_bank; + acc_valid_d1 <= 1'b1; if (cur_bank == 1'b0) begin m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}]; m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}]; @@ -661,16 +693,25 @@ module matmul_axi_slave ( end end - // Accumulate results from 4 cycles ago (pipe_cnt 4-19) - if (pipe_cnt >= 5'd4 && pipe_cnt < 5'd20) begin + // Accumulate results from 4 cycles ago. + if (acc_valid_d4) begin if (acc_bank_d4 == 1'b0) b0_c[acc_idx_d4] <= calc_c_result; else b1_c[acc_idx_d4] <= calc_c_result; end - if (pipe_cnt == 5'd20) begin + if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin calc_done_pulse <= 1'b1; + end + + if (pipe_cnt < 5'd16) begin + pipe_cnt <= pipe_cnt + 5'd1; + end else if (cur_bank == 1'b0 && req_1) begin + start_calc_bank(1'b1); + end else if (cur_bank == 1'b1 && req_0) begin + start_calc_bank(1'b0); + end else if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin busy <= 1'b0; state <= C_IDLE; end else begin