From d3a08b1f9dae1ca7277d167823234da80320fc7b Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 4 Jul 2026 20:42:31 +0800 Subject: [PATCH] Split CDMA matmul endpoint engines --- rtl/ip/cdma/cdma_matmul_bridge.sv | 369 +++++++++++++++--------------- rtl/ip/cdma/snix_axil_cdma_mux.sv | 274 ++++++++++++++-------- rtl/soc_top.v | 80 ++++--- 3 files changed, 408 insertions(+), 315 deletions(-) diff --git a/rtl/ip/cdma/cdma_matmul_bridge.sv b/rtl/ip/cdma/cdma_matmul_bridge.sv index 00ede7f..bca9dd5 100644 --- a/rtl/ip/cdma/cdma_matmul_bridge.sv +++ b/rtl/ip/cdma/cdma_matmul_bridge.sv @@ -41,16 +41,23 @@ module cdma_matmul_bridge #( output logic s_axil_rvalid, input logic s_axil_rready, - output logic cdma_req_valid, - input logic cdma_req_ready, - output logic [1:0] cdma_req_mode, - output logic [ADDR_WIDTH-1:0] cdma_req_src, - output logic [ADDR_WIDTH-1:0] cdma_req_dst, - output logic [31:0] cdma_req_bytes, - output logic [3:0] cdma_req_ep_id, - output logic [3:0] cdma_req_ep_bank, - output logic [7:0] cdma_req_ep_word_off, - input logic cdma_req_done, + output logic cdma_rd_req_valid, + input logic cdma_rd_req_ready, + output logic [ADDR_WIDTH-1:0] cdma_rd_req_src, + output logic [31:0] cdma_rd_req_bytes, + output logic [3:0] cdma_rd_req_ep_id, + output logic [3:0] cdma_rd_req_ep_bank, + output logic [7:0] cdma_rd_req_ep_word_off, + input logic cdma_rd_req_done, + + output logic cdma_wr_req_valid, + input logic cdma_wr_req_ready, + output logic [ADDR_WIDTH-1:0] cdma_wr_req_dst, + output logic [31:0] cdma_wr_req_bytes, + output logic [3:0] cdma_wr_req_ep_id, + output logic [3:0] cdma_wr_req_ep_bank, + output logic [7:0] cdma_wr_req_ep_word_off, + input logic cdma_wr_req_done, output logic mat_start_valid, output logic mat_start_bank, @@ -61,9 +68,6 @@ module cdma_matmul_bridge #( output logic seq_done_irq ); - localparam logic [1:0] MODE_MEM2EP = 2'd1; - localparam logic [1:0] MODE_EP2MEM = 2'd2; - localparam logic [7:0] REG_CTRL = 8'h00; localparam logic [7:0] REG_STATUS = 8'h04; localparam logic [7:0] REG_SRC_BASE = 8'h08; @@ -76,21 +80,6 @@ module cdma_matmul_bridge #( localparam logic [31:0] MAT_AB_BYTES = 32'd128; localparam logic [31:0] MAT_C_BYTES = 32'd192; - typedef enum logic [3:0] { - SQ_IDLE, - SQ_CLEAR, - SQ_LOAD_START, - SQ_LOAD_REQ, - SQ_LOAD_WAIT, - SQ_WAIT_DONE, - SQ_STORE_START, - SQ_STORE_REQ, - SQ_STORE_WAIT, - SQ_DONE - } seq_state_t; - - seq_state_t seq_state; - logic [31:0] seq_src_base; logic [31:0] seq_dst_base; logic [31:0] seq_groups; @@ -99,21 +88,51 @@ module cdma_matmul_bridge #( logic [31:0] seq_dst_stride; logic seq_busy; logic seq_done_flag; - logic [31:0] seq_load_idx; - logic [31:0] seq_store_idx; - logic seq_compute_bank; logic seq_skip_writeback; - logic seq_defer_load_start; - logic seq_deferred_start_valid; - logic seq_deferred_start_bank; + + logic [31:0] load_next_idx; + logic [31:0] groups_retired; + logic [1:0] bank_load_active; + logic [1:0] bank_ab_valid; + logic [1:0] bank_compute_active; + logic [1:0] bank_store_pending; + logic [1:0] bank_store_active; + logic [31:0] bank_group_idx [2]; + + logic rd_busy; + logic wr_busy; + logic rd_active_bank; + logic wr_active_bank; + logic [1:0] compute_q; + logic [1:0] compute_q_count; + logic mat_done_seen; wire [7:0] wr_off = s_axil_awaddr[7:0]; wire [7:0] rd_off = s_axil_araddr[7:0]; + wire do_write; wire start_req = do_write && (wr_off == REG_CTRL) && s_axil_wstrb[0] && s_axil_wdata[0]; wire done_clear_req = do_write && (wr_off == REG_STATUS) && s_axil_wstrb[0] && s_axil_wdata[1]; - wire [31:0] src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride); - wire [31:0] dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride); - wire loaded_bank = seq_load_idx[0]; + wire load_target_bank = load_next_idx[0]; + wire load_target_free = !bank_load_active[load_target_bank] && + !bank_ab_valid[load_target_bank] && + !bank_compute_active[load_target_bank] && + !bank_store_pending[load_target_bank] && + !bank_store_active[load_target_bank]; + wire bank0_start_ready = bank_ab_valid[0] && !bank_compute_active[0] && + !bank_store_pending[0] && !bank_store_active[0]; + wire bank1_start_ready = bank_ab_valid[1] && !bank_compute_active[1] && + !bank_store_pending[1] && !bank_store_active[1]; + wire start_sel_bank = bank1_start_ready && + (!bank0_start_ready || (bank_group_idx[1] < bank_group_idx[0])); + wire start_ready_any = bank0_start_ready || bank1_start_ready; + wire store_sel_bank = bank_store_pending[1] && !bank_store_active[1] && + !(bank_store_pending[0] && !bank_store_active[0]); + wire store_ready_any = (bank_store_pending[0] && !bank_store_active[0]) || + (bank_store_pending[1] && !bank_store_active[1]); + wire mat_done_event = mat_done && !mat_done_seen; + wire retire_from_store = seq_busy && cdma_wr_req_done; + wire retire_from_compute = seq_busy && seq_skip_writeback && mat_done_event && (compute_q_count != 2'd0); + wire [31:0] retire_inc = {31'd0, retire_from_store} + {31'd0, retire_from_compute}; function automatic logic [31:0] apply_wstrb( input logic [31:0] old_val, @@ -128,35 +147,6 @@ module cdma_matmul_bridge #( end endfunction - task issue_load_req; - input [31:0] load_idx; - begin - cdma_req_mode <= MODE_MEM2EP; - cdma_req_src <= seq_src_base + (load_idx * seq_src_stride); - cdma_req_dst <= '0; - cdma_req_bytes <= MAT_AB_BYTES; - cdma_req_ep_id <= 4'd0; - cdma_req_ep_bank <= {3'b000, load_idx[0]}; - cdma_req_ep_word_off <= 8'd0; - cdma_req_valid <= 1'b1; - seq_state <= SQ_LOAD_REQ; - end - endtask - - task issue_store_req; - begin - cdma_req_mode <= MODE_EP2MEM; - cdma_req_src <= '0; - cdma_req_dst <= dst_addr_for_store; - cdma_req_bytes <= MAT_C_BYTES; - cdma_req_ep_id <= 4'd0; - cdma_req_ep_bank <= {3'b000, seq_compute_bank}; - cdma_req_ep_word_off <= 8'd0; - cdma_req_valid <= 1'b1; - seq_state <= SQ_STORE_REQ; - end - endtask - assign s_axil_bresp = 2'b00; assign s_axil_rresp = 2'b00; assign seq_done_irq = seq_done_flag; @@ -176,7 +166,7 @@ module cdma_matmul_bridge #( end end - wire do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid; + assign do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid; always_ff @(posedge clk) begin if (!rst_n) begin @@ -209,7 +199,7 @@ module cdma_matmul_bridge #( end else if (do_read) begin s_axil_rvalid <= 1'b1; case (rd_off) - REG_CTRL: s_axil_rdata <= {31'd0, start_req}; + REG_CTRL: s_axil_rdata <= {30'd0, seq_skip_writeback, 1'b0}; REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy}; REG_SRC_BASE: s_axil_rdata <= seq_src_base; REG_DST_BASE: s_axil_rdata <= seq_dst_base; @@ -248,151 +238,150 @@ module cdma_matmul_bridge #( always_ff @(posedge clk) begin if (!rst_n) begin - seq_state <= SQ_IDLE; - seq_busy <= 1'b0; - seq_done_flag <= 1'b0; - seq_load_idx <= 32'd0; - seq_store_idx <= 32'd0; - seq_compute_bank <= 1'b0; - seq_skip_writeback <= 1'b0; - seq_defer_load_start <= 1'b0; - seq_deferred_start_valid <= 1'b0; - seq_deferred_start_bank <= 1'b0; - cdma_req_valid <= 1'b0; - cdma_req_mode <= MODE_MEM2EP; - cdma_req_src <= '0; - cdma_req_dst <= '0; - cdma_req_bytes <= '0; - cdma_req_ep_id <= 4'd0; - cdma_req_ep_bank <= 4'd0; - cdma_req_ep_word_off <= 8'd0; - mat_start_valid <= 1'b0; - mat_start_bank <= 1'b0; - mat_done_clear <= 1'b0; - mat_crc_clear <= 1'b0; + seq_busy <= 1'b0; + seq_done_flag <= 1'b0; + seq_skip_writeback <= 1'b0; + load_next_idx <= 32'd0; + groups_retired <= 32'd0; + bank_load_active <= 2'b00; + bank_ab_valid <= 2'b00; + bank_compute_active <= 2'b00; + bank_store_pending <= 2'b00; + bank_store_active <= 2'b00; + bank_group_idx[0] <= 32'd0; + bank_group_idx[1] <= 32'd0; + rd_busy <= 1'b0; + wr_busy <= 1'b0; + rd_active_bank <= 1'b0; + wr_active_bank <= 1'b0; + compute_q <= 2'b00; + compute_q_count <= 2'd0; + mat_done_seen <= 1'b0; + cdma_rd_req_valid <= 1'b0; + cdma_rd_req_src <= '0; + cdma_rd_req_bytes <= '0; + cdma_rd_req_ep_id <= 4'd0; + cdma_rd_req_ep_bank <= 4'd0; + cdma_rd_req_ep_word_off <= 8'd0; + cdma_wr_req_valid <= 1'b0; + cdma_wr_req_dst <= '0; + cdma_wr_req_bytes <= '0; + cdma_wr_req_ep_id <= 4'd0; + cdma_wr_req_ep_bank <= 4'd0; + cdma_wr_req_ep_word_off <= 8'd0; + mat_start_valid <= 1'b0; + mat_start_bank <= 1'b0; + mat_done_clear <= 1'b0; + mat_crc_clear <= 1'b0; end else begin mat_start_valid <= 1'b0; mat_done_clear <= 1'b0; mat_crc_clear <= 1'b0; + mat_done_seen <= mat_done; + if (done_clear_req) seq_done_flag <= 1'b0; - case (seq_state) - SQ_IDLE: begin - seq_busy <= 1'b0; - cdma_req_valid <= 1'b0; - if (start_req && (seq_groups != 32'd0)) begin - seq_busy <= 1'b1; - seq_done_flag <= 1'b0; - seq_load_idx <= 32'd0; - seq_store_idx <= 32'd0; - seq_compute_bank <= 1'b0; - seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1]; - seq_defer_load_start <= 1'b0; - seq_deferred_start_valid <= 1'b0; - seq_deferred_start_bank <= 1'b0; - mat_crc_clear <= 1'b1; - mat_done_clear <= 1'b1; - seq_state <= SQ_CLEAR; + if (cdma_rd_req_valid && cdma_rd_req_ready) + cdma_rd_req_valid <= 1'b0; + + if (cdma_wr_req_valid && cdma_wr_req_ready) + cdma_wr_req_valid <= 1'b0; + + if (start_req && !seq_busy && (seq_groups != 32'd0)) begin + seq_busy <= 1'b1; + seq_done_flag <= 1'b0; + seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1]; + load_next_idx <= 32'd0; + groups_retired <= 32'd0; + bank_load_active <= 2'b00; + bank_ab_valid <= 2'b00; + bank_compute_active <= 2'b00; + bank_store_pending <= 2'b00; + bank_store_active <= 2'b00; + bank_group_idx[0] <= 32'd0; + bank_group_idx[1] <= 32'd0; + rd_busy <= 1'b0; + wr_busy <= 1'b0; + compute_q <= 2'b00; + compute_q_count <= 2'd0; + mat_done_seen <= 1'b1; + mat_crc_clear <= 1'b1; + mat_done_clear <= 1'b1; + cdma_rd_req_valid <= 1'b0; + cdma_wr_req_valid <= 1'b0; + end else if (seq_busy) begin + if (retire_inc != 32'd0) begin + groups_retired <= groups_retired + retire_inc; + if ((groups_retired + retire_inc) >= seq_groups) begin + seq_busy <= 1'b0; + seq_done_flag <= 1'b1; end end - SQ_CLEAR: begin - issue_load_req(32'd0); + if (cdma_rd_req_done) begin + rd_busy <= 1'b0; + bank_load_active[rd_active_bank] <= 1'b0; + bank_ab_valid[rd_active_bank] <= 1'b1; end - SQ_LOAD_START: begin - issue_load_req(seq_load_idx); + if (cdma_wr_req_done) begin + wr_busy <= 1'b0; + bank_store_active[wr_active_bank] <= 1'b0; end - SQ_LOAD_REQ: begin - if (cdma_req_valid && cdma_req_ready) begin - cdma_req_valid <= 1'b0; - seq_state <= SQ_LOAD_WAIT; + if (mat_done_event && (compute_q_count != 2'd0)) begin + mat_done_clear <= 1'b1; + mat_done_seen <= 1'b1; + bank_compute_active[compute_q[0]] <= 1'b0; + if (!seq_skip_writeback) begin + bank_store_pending[compute_q[0]] <= 1'b1; end + if (compute_q_count == 2'd2) + compute_q[0] <= compute_q[1]; + compute_q_count <= compute_q_count - 2'd1; end - SQ_LOAD_WAIT: begin - if (cdma_req_done) begin - seq_load_idx <= seq_load_idx + 32'd1; - if (seq_defer_load_start) begin - seq_defer_load_start <= 1'b0; - seq_deferred_start_valid <= 1'b1; - seq_deferred_start_bank <= cdma_req_ep_bank[0]; - issue_store_req(); - end else begin - mat_start_valid <= 1'b1; - mat_start_bank <= cdma_req_ep_bank[0]; - if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin - issue_load_req(seq_load_idx + 32'd1); - end else begin - seq_state <= SQ_WAIT_DONE; - end - end - end + if (!rd_busy && !cdma_rd_req_valid && (load_next_idx < seq_groups) && load_target_free) begin + cdma_rd_req_src <= seq_src_base + (load_next_idx * seq_src_stride); + cdma_rd_req_bytes <= MAT_AB_BYTES; + cdma_rd_req_ep_id <= 4'd0; + cdma_rd_req_ep_bank <= {3'b000, load_target_bank}; + cdma_rd_req_ep_word_off <= 8'd0; + cdma_rd_req_valid <= 1'b1; + rd_busy <= 1'b1; + rd_active_bank <= load_target_bank; + bank_load_active[load_target_bank] <= 1'b1; + bank_group_idx[load_target_bank] <= load_next_idx; + load_next_idx <= load_next_idx + 32'd1; end - SQ_WAIT_DONE: begin - if (mat_done) begin - mat_done_clear <= 1'b1; - if (seq_skip_writeback) begin - seq_store_idx <= seq_store_idx + 32'd1; - seq_compute_bank <= ~seq_compute_bank; - if ((seq_store_idx + 32'd1) >= seq_groups) begin - seq_state <= SQ_DONE; - end else if (seq_load_idx < seq_groups) begin - issue_load_req(seq_load_idx); - end else begin - seq_state <= SQ_WAIT_DONE; - end - end else if (seq_load_idx < seq_groups) begin - seq_defer_load_start <= 1'b1; - issue_load_req(seq_load_idx); - end else begin - issue_store_req(); - end - end + if (!seq_skip_writeback && !wr_busy && !cdma_wr_req_valid && store_ready_any) begin + cdma_wr_req_dst <= seq_dst_base + (bank_group_idx[store_sel_bank] * seq_dst_stride); + cdma_wr_req_bytes <= MAT_C_BYTES; + cdma_wr_req_ep_id <= 4'd0; + cdma_wr_req_ep_bank <= {3'b000, store_sel_bank}; + cdma_wr_req_ep_word_off <= 8'd0; + cdma_wr_req_valid <= 1'b1; + wr_busy <= 1'b1; + wr_active_bank <= store_sel_bank; + bank_store_pending[store_sel_bank] <= 1'b0; + bank_store_active[store_sel_bank] <= 1'b1; end - SQ_STORE_START: begin - issue_store_req(); + if ((compute_q_count < 2'd2) && start_ready_any && !(mat_done_event && (compute_q_count != 2'd0))) begin + mat_start_valid <= 1'b1; + mat_start_bank <= start_sel_bank; + bank_ab_valid[start_sel_bank] <= 1'b0; + bank_compute_active[start_sel_bank] <= 1'b1; + if (compute_q_count == 2'd0) + compute_q[0] <= start_sel_bank; + else + compute_q[1] <= start_sel_bank; + compute_q_count <= compute_q_count + 2'd1; end - - SQ_STORE_REQ: begin - if (cdma_req_valid && cdma_req_ready) begin - cdma_req_valid <= 1'b0; - seq_state <= SQ_STORE_WAIT; - end - end - - SQ_STORE_WAIT: begin - if (cdma_req_done) begin - seq_store_idx <= seq_store_idx + 32'd1; - seq_compute_bank <= ~seq_compute_bank; - if (seq_deferred_start_valid) begin - mat_start_valid <= 1'b1; - mat_start_bank <= seq_deferred_start_bank; - seq_deferred_start_valid <= 1'b0; - end - if ((seq_store_idx + 32'd1) >= seq_groups) begin - seq_state <= SQ_DONE; - end else if (seq_load_idx < seq_groups) begin - seq_state <= SQ_WAIT_DONE; - end else begin - seq_state <= SQ_WAIT_DONE; - end - end - end - - SQ_DONE: begin - seq_busy <= 1'b0; - seq_done_flag <= 1'b1; - seq_state <= SQ_IDLE; - end - - default: seq_state <= SQ_IDLE; - endcase + end end end diff --git a/rtl/ip/cdma/snix_axil_cdma_mux.sv b/rtl/ip/cdma/snix_axil_cdma_mux.sv index f3604c8..35586e4 100644 --- a/rtl/ip/cdma/snix_axil_cdma_mux.sv +++ b/rtl/ip/cdma/snix_axil_cdma_mux.sv @@ -114,16 +114,23 @@ module snix_axil_cdma_mux #( // ========================================== // Generic external endpoint requester // ========================================== - input logic ext_req_valid, - output logic ext_req_ready, - input logic [1:0] ext_req_mode, - input logic [ADDR_WIDTH-1:0] ext_req_src, - input logic [ADDR_WIDTH-1:0] ext_req_dst, - input logic [31:0] ext_req_bytes, - input logic [3:0] ext_req_ep_id, - input logic [3:0] ext_req_ep_bank, - input logic [7:0] ext_req_ep_word_off, - output logic ext_req_done, + input logic ext_rd_req_valid, + output logic ext_rd_req_ready, + input logic [ADDR_WIDTH-1:0] ext_rd_req_src, + input logic [31:0] ext_rd_req_bytes, + input logic [3:0] ext_rd_req_ep_id, + input logic [3:0] ext_rd_req_ep_bank, + input logic [7:0] ext_rd_req_ep_word_off, + output logic ext_rd_req_done, + + input logic ext_wr_req_valid, + output logic ext_wr_req_ready, + input logic [ADDR_WIDTH-1:0] ext_wr_req_dst, + input logic [31:0] ext_wr_req_bytes, + input logic [3:0] ext_wr_req_ep_id, + input logic [3:0] ext_wr_req_ep_bank, + input logic [7:0] ext_wr_req_ep_word_off, + output logic ext_wr_req_done, // Global Interrupt (OR'd from all channels) output logic dma_finish @@ -406,71 +413,82 @@ module snix_axil_cdma_mux #( logic [31:0] ep_bytes_left; logic [7:0] ep_word_addr_r; - logic ext_active; - logic [1:0] ext_mode_r; - logic [ADDR_WIDTH-1:0] ext_src_r; - logic [ADDR_WIDTH-1:0] ext_dst_r; - logic [31:0] ext_bytes_r; - logic [3:0] ext_ep_id_r; - logic [3:0] ext_ep_bank_r; - logic [7:0] ext_ep_word_off_r; + typedef enum logic [1:0] {EXT_RD_IDLE, EXT_RD_AR, EXT_RD_READ} ext_rd_state_t; + typedef enum logic [1:0] {EXT_WR_IDLE, EXT_WR_AW, EXT_WR_WRITE, EXT_WR_BRESP} ext_wr_state_t; - wire ext_accept = ext_req_valid && ext_req_ready; - wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_accept; - wire select_ext_ep = ext_active || ext_accept; + ext_rd_state_t ext_rd_state; + ext_wr_state_t ext_wr_state; + logic [ADDR_WIDTH-1:0] ext_rd_addr_r; + logic [31:0] ext_rd_bytes_left; + logic [7:0] ext_rd_word_addr_r; + logic [3:0] ext_rd_ep_id_r; + logic [3:0] ext_rd_ep_bank_r; + + logic [ADDR_WIDTH-1:0] ext_wr_addr_r; + logic [31:0] ext_wr_bytes_left; + logic [7:0] ext_wr_word_addr_r; + logic [3:0] ext_wr_ep_id_r; + logic [3:0] ext_wr_ep_bank_r; + + wire ext_rd_active = (ext_rd_state != EXT_RD_IDLE); + wire ext_wr_active = (ext_wr_state != EXT_WR_IDLE); + wire ext_rd_accept = ext_rd_req_valid && ext_rd_req_ready; + wire ext_wr_accept = ext_wr_req_valid && ext_wr_req_ready; + wire ext_any_busy_or_req = ext_rd_active || ext_wr_active || ext_rd_req_valid || ext_wr_req_valid; + wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)); // Dynamic routing to the engine based on current active channel - assign engine_src = ext_accept ? ext_req_src : (select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]); - assign engine_dst = ext_accept ? ext_req_dst : (select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]); - assign engine_bytes = ext_accept ? ext_req_bytes : (select_ext_ep ? ext_bytes_r : ch_len[cur_ch]); + assign engine_src = ch_src_addr[cur_ch]; + assign engine_dst = ch_dst_addr[cur_ch]; + assign engine_bytes = ch_len[cur_ch]; assign engine_len = ch_ctrl[cur_ch][13:6]; assign engine_size = ch_ctrl[cur_ch][5:3]; - assign engine_mode = ext_accept ? ext_req_mode : (select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]); - assign engine_ep_bank = ext_accept ? ext_req_ep_bank : (select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]); - assign engine_ep_word_off = ext_accept ? ext_req_ep_word_off : (select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]); - assign engine_ep_id = ext_accept ? ext_req_ep_id : (select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]); + assign engine_mode = ch_mode[cur_ch][1:0]; + assign engine_ep_bank = ch_ep_cfg[cur_ch][3:0]; + assign engine_ep_word_off = ch_ep_cfg[cur_ch][15:8]; + assign engine_ep_id = ch_ep_cfg[cur_ch][19:16]; assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done; assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM); - assign ep_start = ext_accept || (engine_start && (engine_mode != MODE_MEM2MEM)); + assign ep_start = engine_start && (engine_mode != MODE_MEM2MEM); - assign core_awready = (!use_ep_engine) && mm2mm_awready; - assign core_wready = (!use_ep_engine) && mm2mm_wready; - assign core_arready = (!use_ep_engine) && mm2mm_arready; - assign ep_awready = use_ep_engine && mm2mm_awready; - assign ep_wready = use_ep_engine && mm2mm_wready; - assign ep_arready = use_ep_engine && mm2mm_arready; + assign core_awready = (!ext_wr_active && !use_ep_engine) && mm2mm_awready; + assign core_wready = (!ext_wr_active && !use_ep_engine) && mm2mm_wready; + assign core_arready = (!ext_rd_active && !use_ep_engine) && mm2mm_arready; + assign ep_awready = (!ext_wr_active && use_ep_engine) && mm2mm_awready; + assign ep_wready = (!ext_wr_active && use_ep_engine) && mm2mm_wready; + assign ep_arready = (!ext_rd_active && use_ep_engine) && mm2mm_arready; - assign mm2mm_awid = use_ep_engine ? ep_awid : core_awid; - assign mm2mm_awaddr = use_ep_engine ? ep_awaddr : core_awaddr; - assign mm2mm_awlen = use_ep_engine ? ep_awlen : core_awlen; - assign mm2mm_awsize = use_ep_engine ? ep_awsize : core_awsize; - assign mm2mm_awburst = use_ep_engine ? ep_awburst : core_awburst; - assign mm2mm_awlock = use_ep_engine ? ep_awlock : core_awlock; - assign mm2mm_awcache = use_ep_engine ? ep_awcache : core_awcache; - assign mm2mm_awprot = use_ep_engine ? ep_awprot : core_awprot; - assign mm2mm_awqos = use_ep_engine ? ep_awqos : core_awqos; - assign mm2mm_awuser = use_ep_engine ? ep_awuser : core_awuser; - assign mm2mm_awvalid = use_ep_engine ? ep_awvalid : core_awvalid; + assign mm2mm_awid = ext_wr_active ? '0 : (use_ep_engine ? ep_awid : core_awid); + assign mm2mm_awaddr = ext_wr_active ? ext_wr_addr_r : (use_ep_engine ? ep_awaddr : core_awaddr); + assign mm2mm_awlen = ext_wr_active ? (((ext_wr_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_wr_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_awlen : core_awlen); + assign mm2mm_awsize = ext_wr_active ? 3'd2 : (use_ep_engine ? ep_awsize : core_awsize); + assign mm2mm_awburst = ext_wr_active ? 2'b01 : (use_ep_engine ? ep_awburst : core_awburst); + assign mm2mm_awlock = ext_wr_active ? 1'b0 : (use_ep_engine ? ep_awlock : core_awlock); + assign mm2mm_awcache = ext_wr_active ? 4'b0 : (use_ep_engine ? ep_awcache : core_awcache); + assign mm2mm_awprot = ext_wr_active ? 3'b0 : (use_ep_engine ? ep_awprot : core_awprot); + assign mm2mm_awqos = ext_wr_active ? 4'b0 : (use_ep_engine ? ep_awqos : core_awqos); + assign mm2mm_awuser = ext_wr_active ? '0 : (use_ep_engine ? ep_awuser : core_awuser); + assign mm2mm_awvalid = ext_wr_active ? (ext_wr_state == EXT_WR_AW) : (use_ep_engine ? ep_awvalid : core_awvalid); - assign mm2mm_wdata = use_ep_engine ? ep_wdata : core_wdata; - assign mm2mm_wstrb = use_ep_engine ? ep_wstrb : core_wstrb; - assign mm2mm_wlast = use_ep_engine ? ep_wlast : core_wlast; - assign mm2mm_wuser = use_ep_engine ? ep_wuser : core_wuser; - assign mm2mm_wvalid = use_ep_engine ? ep_wvalid : core_wvalid; - assign mm2mm_bready = use_ep_engine ? ep_bready : core_bready; + assign mm2mm_wdata = ext_wr_active ? ep_rd_data : (use_ep_engine ? ep_wdata : core_wdata); + assign mm2mm_wstrb = ext_wr_active ? {{(DATA_WIDTH/8){1'b1}}} : (use_ep_engine ? ep_wstrb : core_wstrb); + assign mm2mm_wlast = ext_wr_active ? ((ext_wr_bytes_left <= 32'd4) && mm2mm_wvalid) : (use_ep_engine ? ep_wlast : core_wlast); + assign mm2mm_wuser = ext_wr_active ? '0 : (use_ep_engine ? ep_wuser : core_wuser); + assign mm2mm_wvalid = ext_wr_active ? ((ext_wr_state == EXT_WR_WRITE) && ep_rd_ready) : (use_ep_engine ? ep_wvalid : core_wvalid); + assign mm2mm_bready = ext_wr_active ? (ext_wr_state == EXT_WR_BRESP) : (use_ep_engine ? ep_bready : core_bready); - assign mm2mm_arid = use_ep_engine ? ep_arid : core_arid; - assign mm2mm_araddr = use_ep_engine ? ep_araddr : core_araddr; - assign mm2mm_arlen = use_ep_engine ? ep_arlen : core_arlen; - assign mm2mm_arsize = use_ep_engine ? ep_arsize : core_arsize; - assign mm2mm_arburst = use_ep_engine ? ep_arburst : core_arburst; - assign mm2mm_arlock = use_ep_engine ? ep_arlock : core_arlock; - assign mm2mm_arcache = use_ep_engine ? ep_arcache : core_arcache; - assign mm2mm_arprot = use_ep_engine ? ep_arprot : core_arprot; - assign mm2mm_arqos = use_ep_engine ? ep_arqos : core_arqos; - assign mm2mm_aruser = use_ep_engine ? ep_aruser : core_aruser; - assign mm2mm_arvalid = use_ep_engine ? ep_arvalid : core_arvalid; - assign mm2mm_rready = use_ep_engine ? ep_rready : core_rready; + assign mm2mm_arid = ext_rd_active ? '0 : (use_ep_engine ? ep_arid : core_arid); + assign mm2mm_araddr = ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr); + assign mm2mm_arlen = ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_arlen : core_arlen); + assign mm2mm_arsize = ext_rd_active ? 3'd2 : (use_ep_engine ? ep_arsize : core_arsize); + assign mm2mm_arburst = ext_rd_active ? 2'b01 : (use_ep_engine ? ep_arburst : core_arburst); + assign mm2mm_arlock = ext_rd_active ? 1'b0 : (use_ep_engine ? ep_arlock : core_arlock); + assign mm2mm_arcache = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arcache : core_arcache); + assign mm2mm_arprot = ext_rd_active ? 3'b0 : (use_ep_engine ? ep_arprot : core_arprot); + assign mm2mm_arqos = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arqos : core_arqos); + assign mm2mm_aruser = ext_rd_active ? '0 : (use_ep_engine ? ep_aruser : core_aruser); + assign mm2mm_arvalid = ext_rd_active ? (ext_rd_state == EXT_RD_AR) : (use_ep_engine ? ep_arvalid : core_arvalid); + assign mm2mm_rready = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && ep_wr_ready) : (use_ep_engine ? ep_rready : core_rready); always_ff @(posedge clk) begin if (!rst_n) begin @@ -485,7 +503,7 @@ module snix_axil_cdma_mux #( case (state) IDLE: begin - if (ext_req_valid || ext_active) begin + if (ext_any_busy_or_req) begin state <= IDLE; end else for (int i = 0; i < PORTS; i++) begin logic [CH_BITS:0] check_ch_ext; @@ -572,36 +590,100 @@ module snix_axil_cdma_mux #( wire ep_rd_done_fire = (ep_state == EP_READ) && mm2mm_rvalid && ep_rready && mm2mm_rlast; wire ep_wr_done_fire = (ep_state == EP_BRESP) && mm2mm_bvalid && ep_bready; wire ep_done_fast = ep_rd_done_fire || ep_wr_done_fire; + wire ext_rd_done_fire = (ext_rd_state == EXT_RD_READ) && mm2mm_rvalid && mm2mm_rready && mm2mm_rlast; + wire ext_wr_done_fire = (ext_wr_state == EXT_WR_BRESP) && mm2mm_bvalid && mm2mm_bready; + wire ext_wr_w_fire = (ext_wr_state == EXT_WR_WRITE) && mm2mm_wvalid && mm2mm_wready; // ========================================== // Generic external endpoint requester // ========================================== - assign ext_req_ready = !ext_active && (state == IDLE) && (ep_state == EP_IDLE); - assign ext_req_done = ext_active && ep_done_fast; + assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_rd_active; + assign ext_wr_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active; + assign ext_rd_req_done = ext_rd_done_fire; + assign ext_wr_req_done = ext_wr_done_fire; always_ff @(posedge clk) begin if (!rst_n) begin - ext_active <= 1'b0; - ext_mode_r <= MODE_MEM2EP; - ext_src_r <= '0; - ext_dst_r <= '0; - ext_bytes_r <= '0; - ext_ep_id_r <= '0; - ext_ep_bank_r <= '0; - ext_ep_word_off_r <= '0; + ext_rd_state <= EXT_RD_IDLE; + ext_rd_addr_r <= '0; + ext_rd_bytes_left <= '0; + ext_rd_word_addr_r <= '0; + ext_rd_ep_id_r <= '0; + ext_rd_ep_bank_r <= '0; end else begin - if (ext_accept) begin - ext_active <= 1'b1; - ext_mode_r <= ext_req_mode; - ext_src_r <= ext_req_src; - ext_dst_r <= ext_req_dst; - ext_bytes_r <= ext_req_bytes; - ext_ep_id_r <= ext_req_ep_id; - ext_ep_bank_r <= ext_req_ep_bank; - ext_ep_word_off_r <= ext_req_ep_word_off; - end else if (ext_active && ep_done_fast) begin - ext_active <= 1'b0; - end + case (ext_rd_state) + EXT_RD_IDLE: begin + if (ext_rd_accept) begin + ext_rd_addr_r <= ext_rd_req_src; + ext_rd_bytes_left <= ext_rd_req_bytes; + ext_rd_word_addr_r <= ext_rd_req_ep_word_off; + ext_rd_ep_id_r <= ext_rd_req_ep_id; + ext_rd_ep_bank_r <= ext_rd_req_ep_bank; + ext_rd_state <= EXT_RD_AR; + end + end + + EXT_RD_AR: begin + if (mm2mm_arvalid && mm2mm_arready) + ext_rd_state <= EXT_RD_READ; + end + + EXT_RD_READ: begin + if (mm2mm_rvalid && mm2mm_rready) begin + ext_rd_word_addr_r <= ext_rd_word_addr_r + 8'd1; + ext_rd_bytes_left <= (ext_rd_bytes_left > 32'd4) ? (ext_rd_bytes_left - 32'd4) : 32'd0; + if (mm2mm_rlast) + ext_rd_state <= EXT_RD_IDLE; + end + end + + default: ext_rd_state <= EXT_RD_IDLE; + endcase + end + end + + always_ff @(posedge clk) begin + if (!rst_n) begin + ext_wr_state <= EXT_WR_IDLE; + ext_wr_addr_r <= '0; + ext_wr_bytes_left <= '0; + ext_wr_word_addr_r <= '0; + ext_wr_ep_id_r <= '0; + ext_wr_ep_bank_r <= '0; + end else begin + case (ext_wr_state) + EXT_WR_IDLE: begin + if (ext_wr_accept) begin + ext_wr_addr_r <= ext_wr_req_dst; + ext_wr_bytes_left <= ext_wr_req_bytes; + ext_wr_word_addr_r <= ext_wr_req_ep_word_off; + ext_wr_ep_id_r <= ext_wr_req_ep_id; + ext_wr_ep_bank_r <= ext_wr_req_ep_bank; + ext_wr_state <= EXT_WR_AW; + end + end + + EXT_WR_AW: begin + if (mm2mm_awvalid && mm2mm_awready) + ext_wr_state <= EXT_WR_WRITE; + end + + EXT_WR_WRITE: begin + if (ext_wr_w_fire) begin + ext_wr_word_addr_r <= ext_wr_word_addr_r + 8'd1; + ext_wr_bytes_left <= (ext_wr_bytes_left > 32'd4) ? (ext_wr_bytes_left - 32'd4) : 32'd0; + if (mm2mm_wlast) + ext_wr_state <= EXT_WR_BRESP; + end + end + + EXT_WR_BRESP: begin + if (mm2mm_bvalid && mm2mm_bready) + ext_wr_state <= EXT_WR_IDLE; + end + + default: ext_wr_state <= EXT_WR_IDLE; + endcase end end @@ -688,10 +770,11 @@ module snix_axil_cdma_mux #( assign ep_arvalid = (ep_state == EP_AR); assign ep_rready = (ep_state == EP_READ) && ep_wr_ready; - assign ep_wr_valid = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep; - assign ep_wr_id = engine_ep_id; - assign ep_wr_bank = engine_ep_bank; - assign ep_wr_word_addr = ep_word_addr_r; + assign ep_wr_valid = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && mm2mm_rvalid) : + ((ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep); + assign ep_wr_id = ext_rd_active ? ext_rd_ep_id_r : engine_ep_id; + assign ep_wr_bank = ext_rd_active ? ext_rd_ep_bank_r : engine_ep_bank; + assign ep_wr_word_addr = ext_rd_active ? ext_rd_word_addr_r : ep_word_addr_r; assign ep_wr_data = mm2mm_rdata[31:0]; assign ep_awid = '0; @@ -712,9 +795,10 @@ module snix_axil_cdma_mux #( assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready; assign ep_bready = (ep_state == EP_BRESP); - assign ep_rd_valid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready; - assign ep_rd_id = engine_ep_id; - assign ep_rd_bank = engine_ep_bank; - assign ep_rd_word_addr = ep_word_addr_r; + assign ep_rd_valid = ext_wr_active ? ((ext_wr_state == EXT_WR_WRITE) && mm2mm_wready) : + ((ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready); + assign ep_rd_id = ext_wr_active ? ext_wr_ep_id_r : engine_ep_id; + assign ep_rd_bank = ext_wr_active ? ext_wr_ep_bank_r : engine_ep_bank; + assign ep_rd_word_addr = ext_wr_active ? ext_wr_word_addr_r : ep_word_addr_r; endmodule diff --git a/rtl/soc_top.v b/rtl/soc_top.v index 48688a1..6867608 100644 --- a/rtl/soc_top.v +++ b/rtl/soc_top.v @@ -1619,16 +1619,22 @@ wire [1:0] cdma_bridge_rresp; wire cdma_bridge_rvalid; wire cdma_bridge_rready; -wire bridge_cdma_req_valid; -wire bridge_cdma_req_ready; -wire [1:0] bridge_cdma_req_mode; -wire [31:0] bridge_cdma_req_src; -wire [31:0] bridge_cdma_req_dst; -wire [31:0] bridge_cdma_req_bytes; -wire [3:0] bridge_cdma_req_ep_id; -wire [3:0] bridge_cdma_req_ep_bank; -wire [7:0] bridge_cdma_req_ep_word_off; -wire bridge_cdma_req_done; +wire bridge_cdma_rd_req_valid; +wire bridge_cdma_rd_req_ready; +wire [31:0] bridge_cdma_rd_req_src; +wire [31:0] bridge_cdma_rd_req_bytes; +wire [3:0] bridge_cdma_rd_req_ep_id; +wire [3:0] bridge_cdma_rd_req_ep_bank; +wire [7:0] bridge_cdma_rd_req_ep_word_off; +wire bridge_cdma_rd_req_done; +wire bridge_cdma_wr_req_valid; +wire bridge_cdma_wr_req_ready; +wire [31:0] bridge_cdma_wr_req_dst; +wire [31:0] bridge_cdma_wr_req_bytes; +wire [3:0] bridge_cdma_wr_req_ep_id; +wire [3:0] bridge_cdma_wr_req_ep_bank; +wire [7:0] bridge_cdma_wr_req_ep_word_off; +wire bridge_cdma_wr_req_done; wire cdma_axil_wr_bridge = (cdma_s_awaddr[11:8] == 4'h2); wire cdma_axil_rd_bridge = (cdma_s_araddr[11:8] == 4'h2); @@ -1781,16 +1787,23 @@ cdma_matmul_bridge #( .s_axil_rvalid (cdma_bridge_rvalid), .s_axil_rready (cdma_bridge_rready), - .cdma_req_valid (bridge_cdma_req_valid), - .cdma_req_ready (bridge_cdma_req_ready), - .cdma_req_mode (bridge_cdma_req_mode), - .cdma_req_src (bridge_cdma_req_src), - .cdma_req_dst (bridge_cdma_req_dst), - .cdma_req_bytes (bridge_cdma_req_bytes), - .cdma_req_ep_id (bridge_cdma_req_ep_id), - .cdma_req_ep_bank (bridge_cdma_req_ep_bank), - .cdma_req_ep_word_off(bridge_cdma_req_ep_word_off), - .cdma_req_done (bridge_cdma_req_done), + .cdma_rd_req_valid (bridge_cdma_rd_req_valid), + .cdma_rd_req_ready (bridge_cdma_rd_req_ready), + .cdma_rd_req_src (bridge_cdma_rd_req_src), + .cdma_rd_req_bytes (bridge_cdma_rd_req_bytes), + .cdma_rd_req_ep_id (bridge_cdma_rd_req_ep_id), + .cdma_rd_req_ep_bank (bridge_cdma_rd_req_ep_bank), + .cdma_rd_req_ep_word_off(bridge_cdma_rd_req_ep_word_off), + .cdma_rd_req_done (bridge_cdma_rd_req_done), + + .cdma_wr_req_valid (bridge_cdma_wr_req_valid), + .cdma_wr_req_ready (bridge_cdma_wr_req_ready), + .cdma_wr_req_dst (bridge_cdma_wr_req_dst), + .cdma_wr_req_bytes (bridge_cdma_wr_req_bytes), + .cdma_wr_req_ep_id (bridge_cdma_wr_req_ep_id), + .cdma_wr_req_ep_bank (bridge_cdma_wr_req_ep_bank), + .cdma_wr_req_ep_word_off(bridge_cdma_wr_req_ep_word_off), + .cdma_wr_req_done (bridge_cdma_wr_req_done), .mat_start_valid (matmul_direct_start_valid), .mat_start_bank (matmul_direct_start_bank), @@ -1891,16 +1904,23 @@ snix_axil_cdma_mux #( .ep_rd_data (matmul_ep_rd_data), .ep_rd_ready (matmul_ep_rd_ready), - .ext_req_valid (bridge_cdma_req_valid), - .ext_req_ready (bridge_cdma_req_ready), - .ext_req_mode (bridge_cdma_req_mode), - .ext_req_src (bridge_cdma_req_src), - .ext_req_dst (bridge_cdma_req_dst), - .ext_req_bytes (bridge_cdma_req_bytes), - .ext_req_ep_id (bridge_cdma_req_ep_id), - .ext_req_ep_bank (bridge_cdma_req_ep_bank), - .ext_req_ep_word_off(bridge_cdma_req_ep_word_off), - .ext_req_done (bridge_cdma_req_done), + .ext_rd_req_valid (bridge_cdma_rd_req_valid), + .ext_rd_req_ready (bridge_cdma_rd_req_ready), + .ext_rd_req_src (bridge_cdma_rd_req_src), + .ext_rd_req_bytes (bridge_cdma_rd_req_bytes), + .ext_rd_req_ep_id (bridge_cdma_rd_req_ep_id), + .ext_rd_req_ep_bank (bridge_cdma_rd_req_ep_bank), + .ext_rd_req_ep_word_off(bridge_cdma_rd_req_ep_word_off), + .ext_rd_req_done (bridge_cdma_rd_req_done), + + .ext_wr_req_valid (bridge_cdma_wr_req_valid), + .ext_wr_req_ready (bridge_cdma_wr_req_ready), + .ext_wr_req_dst (bridge_cdma_wr_req_dst), + .ext_wr_req_bytes (bridge_cdma_wr_req_bytes), + .ext_wr_req_ep_id (bridge_cdma_wr_req_ep_id), + .ext_wr_req_ep_bank (bridge_cdma_wr_req_ep_bank), + .ext_wr_req_ep_word_off(bridge_cdma_wr_req_ep_word_off), + .ext_wr_req_done (bridge_cdma_wr_req_done), // 全局中断输出 .dma_finish (cdma_dma_finish)