fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split CDMA matmul endpoint engines

Browse Source
This commit is contained in:
FallenSigh
2026-07-04 20:42:31 +08:00
parent 2725f6eb57
commit d3a08b1f9d
3 changed files with 408 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

369
rtl/ip/cdma/cdma_matmul_bridge.sv
View File

@@ -41,16 +41,23 @@ module cdma_matmul_bridge #(
output logic s_axil_rvalid, output logic s_axil_rvalid,
input logic s_axil_rready, input logic s_axil_rready,
output logic cdma_req_valid, output logic cdma_rd_req_valid,
input logic cdma_req_ready, input logic cdma_rd_req_ready,
output logic [1:0] cdma_req_mode, output logic [ADDR_WIDTH-1:0] cdma_rd_req_src,
output logic [ADDR_WIDTH-1:0] cdma_req_src, output logic [31:0] cdma_rd_req_bytes,
output logic [ADDR_WIDTH-1:0] cdma_req_dst, output logic [3:0] cdma_rd_req_ep_id,
output logic [31:0] cdma_req_bytes, output logic [3:0] cdma_rd_req_ep_bank,
output logic [3:0] cdma_req_ep_id, output logic [7:0] cdma_rd_req_ep_word_off,
output logic [3:0] cdma_req_ep_bank, input logic cdma_rd_req_done,
output logic [7:0] cdma_req_ep_word_off,
input logic cdma_req_done, output logic cdma_wr_req_valid,
input logic cdma_wr_req_ready,
output logic [ADDR_WIDTH-1:0] cdma_wr_req_dst,
output logic [31:0] cdma_wr_req_bytes,
output logic [3:0] cdma_wr_req_ep_id,
output logic [3:0] cdma_wr_req_ep_bank,
output logic [7:0] cdma_wr_req_ep_word_off,
input logic cdma_wr_req_done,
output logic mat_start_valid, output logic mat_start_valid,
output logic mat_start_bank, output logic mat_start_bank,
@@ -61,9 +68,6 @@ module cdma_matmul_bridge #(
output logic seq_done_irq output logic seq_done_irq
); );
localparam logic [1:0] MODE_MEM2EP = 2'd1;
localparam logic [1:0] MODE_EP2MEM = 2'd2;
localparam logic [7:0] REG_CTRL = 8'h00; localparam logic [7:0] REG_CTRL = 8'h00;
localparam logic [7:0] REG_STATUS = 8'h04; localparam logic [7:0] REG_STATUS = 8'h04;
localparam logic [7:0] REG_SRC_BASE = 8'h08; localparam logic [7:0] REG_SRC_BASE = 8'h08;
@@ -76,21 +80,6 @@ module cdma_matmul_bridge #(
localparam logic [31:0] MAT_AB_BYTES = 32'd128; localparam logic [31:0] MAT_AB_BYTES = 32'd128;
localparam logic [31:0] MAT_C_BYTES = 32'd192; localparam logic [31:0] MAT_C_BYTES = 32'd192;
typedef enum logic [3:0] {
SQ_IDLE,
SQ_CLEAR,
SQ_LOAD_START,
SQ_LOAD_REQ,
SQ_LOAD_WAIT,
SQ_WAIT_DONE,
SQ_STORE_START,
SQ_STORE_REQ,
SQ_STORE_WAIT,
SQ_DONE
} seq_state_t;
seq_state_t seq_state;
logic [31:0] seq_src_base; logic [31:0] seq_src_base;
logic [31:0] seq_dst_base; logic [31:0] seq_dst_base;
logic [31:0] seq_groups; logic [31:0] seq_groups;
@@ -99,21 +88,51 @@ module cdma_matmul_bridge #(
logic [31:0] seq_dst_stride; logic [31:0] seq_dst_stride;
logic seq_busy; logic seq_busy;
logic seq_done_flag; logic seq_done_flag;
logic [31:0] seq_load_idx;
logic [31:0] seq_store_idx;
logic seq_compute_bank;
logic seq_skip_writeback; logic seq_skip_writeback;
logic seq_defer_load_start;
logic seq_deferred_start_valid; logic [31:0] load_next_idx;
logic seq_deferred_start_bank; logic [31:0] groups_retired;
logic [1:0] bank_load_active;
logic [1:0] bank_ab_valid;
logic [1:0] bank_compute_active;
logic [1:0] bank_store_pending;
logic [1:0] bank_store_active;
logic [31:0] bank_group_idx [2];
logic rd_busy;
logic wr_busy;
logic rd_active_bank;
logic wr_active_bank;
logic [1:0] compute_q;
logic [1:0] compute_q_count;
logic mat_done_seen;
wire [7:0] wr_off = s_axil_awaddr[7:0]; wire [7:0] wr_off = s_axil_awaddr[7:0];
wire [7:0] rd_off = s_axil_araddr[7:0]; wire [7:0] rd_off = s_axil_araddr[7:0];
wire do_write;
wire start_req = do_write && (wr_off == REG_CTRL) && s_axil_wstrb[0] && s_axil_wdata[0]; wire start_req = do_write && (wr_off == REG_CTRL) && s_axil_wstrb[0] && s_axil_wdata[0];
wire done_clear_req = do_write && (wr_off == REG_STATUS) && s_axil_wstrb[0] && s_axil_wdata[1]; wire done_clear_req = do_write && (wr_off == REG_STATUS) && s_axil_wstrb[0] && s_axil_wdata[1];
wire [31:0] src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride); wire load_target_bank = load_next_idx[0];
wire [31:0] dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride); wire load_target_free = !bank_load_active[load_target_bank] &&
wire loaded_bank = seq_load_idx[0]; !bank_ab_valid[load_target_bank] &&
!bank_compute_active[load_target_bank] &&
!bank_store_pending[load_target_bank] &&
!bank_store_active[load_target_bank];
wire bank0_start_ready = bank_ab_valid[0] && !bank_compute_active[0] &&
!bank_store_pending[0] && !bank_store_active[0];
wire bank1_start_ready = bank_ab_valid[1] && !bank_compute_active[1] &&
!bank_store_pending[1] && !bank_store_active[1];
wire start_sel_bank = bank1_start_ready &&
(!bank0_start_ready || (bank_group_idx[1] < bank_group_idx[0]));
wire start_ready_any = bank0_start_ready || bank1_start_ready;
wire store_sel_bank = bank_store_pending[1] && !bank_store_active[1] &&
!(bank_store_pending[0] && !bank_store_active[0]);
wire store_ready_any = (bank_store_pending[0] && !bank_store_active[0]) ||
(bank_store_pending[1] && !bank_store_active[1]);
wire mat_done_event = mat_done && !mat_done_seen;
wire retire_from_store = seq_busy && cdma_wr_req_done;
wire retire_from_compute = seq_busy && seq_skip_writeback && mat_done_event && (compute_q_count != 2'd0);
wire [31:0] retire_inc = {31'd0, retire_from_store} + {31'd0, retire_from_compute};
function automatic logic [31:0] apply_wstrb( function automatic logic [31:0] apply_wstrb(
input logic [31:0] old_val, input logic [31:0] old_val,
@@ -128,35 +147,6 @@ module cdma_matmul_bridge #(
end end
endfunction endfunction
task issue_load_req;
input [31:0] load_idx;
begin
cdma_req_mode <= MODE_MEM2EP;
cdma_req_src <= seq_src_base + (load_idx * seq_src_stride);
cdma_req_dst <= '0;
cdma_req_bytes <= MAT_AB_BYTES;
cdma_req_ep_id <= 4'd0;
cdma_req_ep_bank <= {3'b000, load_idx[0]};
cdma_req_ep_word_off <= 8'd0;
cdma_req_valid <= 1'b1;
seq_state <= SQ_LOAD_REQ;
end
endtask
task issue_store_req;
begin
cdma_req_mode <= MODE_EP2MEM;
cdma_req_src <= '0;
cdma_req_dst <= dst_addr_for_store;
cdma_req_bytes <= MAT_C_BYTES;
cdma_req_ep_id <= 4'd0;
cdma_req_ep_bank <= {3'b000, seq_compute_bank};
cdma_req_ep_word_off <= 8'd0;
cdma_req_valid <= 1'b1;
seq_state <= SQ_STORE_REQ;
end
endtask
assign s_axil_bresp = 2'b00; assign s_axil_bresp = 2'b00;
assign s_axil_rresp = 2'b00; assign s_axil_rresp = 2'b00;
assign seq_done_irq = seq_done_flag; assign seq_done_irq = seq_done_flag;
@@ -176,7 +166,7 @@ module cdma_matmul_bridge #(
end end
end end
wire do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid; assign do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid;
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
@@ -209,7 +199,7 @@ module cdma_matmul_bridge #(
end else if (do_read) begin end else if (do_read) begin
s_axil_rvalid <= 1'b1; s_axil_rvalid <= 1'b1;
case (rd_off) case (rd_off)
REG_CTRL: s_axil_rdata <= {31'd0, start_req}; REG_CTRL: s_axil_rdata <= {30'd0, seq_skip_writeback, 1'b0};
REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy}; REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
REG_SRC_BASE: s_axil_rdata <= seq_src_base; REG_SRC_BASE: s_axil_rdata <= seq_src_base;
REG_DST_BASE: s_axil_rdata <= seq_dst_base; REG_DST_BASE: s_axil_rdata <= seq_dst_base;
@@ -248,151 +238,150 @@ module cdma_matmul_bridge #(
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
seq_state <= SQ_IDLE; seq_busy <= 1'b0;
seq_busy <= 1'b0; seq_done_flag <= 1'b0;
seq_done_flag <= 1'b0; seq_skip_writeback <= 1'b0;
seq_load_idx <= 32'd0; load_next_idx <= 32'd0;
seq_store_idx <= 32'd0; groups_retired <= 32'd0;
seq_compute_bank <= 1'b0; bank_load_active <= 2'b00;
seq_skip_writeback <= 1'b0; bank_ab_valid <= 2'b00;
seq_defer_load_start <= 1'b0; bank_compute_active <= 2'b00;
seq_deferred_start_valid <= 1'b0; bank_store_pending <= 2'b00;
seq_deferred_start_bank <= 1'b0; bank_store_active <= 2'b00;
cdma_req_valid <= 1'b0; bank_group_idx[0] <= 32'd0;
cdma_req_mode <= MODE_MEM2EP; bank_group_idx[1] <= 32'd0;
cdma_req_src <= '0; rd_busy <= 1'b0;
cdma_req_dst <= '0; wr_busy <= 1'b0;
cdma_req_bytes <= '0; rd_active_bank <= 1'b0;
cdma_req_ep_id <= 4'd0; wr_active_bank <= 1'b0;
cdma_req_ep_bank <= 4'd0; compute_q <= 2'b00;
cdma_req_ep_word_off <= 8'd0; compute_q_count <= 2'd0;
mat_start_valid <= 1'b0; mat_done_seen <= 1'b0;
mat_start_bank <= 1'b0; cdma_rd_req_valid <= 1'b0;
mat_done_clear <= 1'b0; cdma_rd_req_src <= '0;
mat_crc_clear <= 1'b0; cdma_rd_req_bytes <= '0;
cdma_rd_req_ep_id <= 4'd0;
cdma_rd_req_ep_bank <= 4'd0;
cdma_rd_req_ep_word_off <= 8'd0;
cdma_wr_req_valid <= 1'b0;
cdma_wr_req_dst <= '0;
cdma_wr_req_bytes <= '0;
cdma_wr_req_ep_id <= 4'd0;
cdma_wr_req_ep_bank <= 4'd0;
cdma_wr_req_ep_word_off <= 8'd0;
mat_start_valid <= 1'b0;
mat_start_bank <= 1'b0;
mat_done_clear <= 1'b0;
mat_crc_clear <= 1'b0;
end else begin end else begin
mat_start_valid <= 1'b0; mat_start_valid <= 1'b0;
mat_done_clear <= 1'b0; mat_done_clear <= 1'b0;
mat_crc_clear <= 1'b0; mat_crc_clear <= 1'b0;
mat_done_seen <= mat_done;
if (done_clear_req) if (done_clear_req)
seq_done_flag <= 1'b0; seq_done_flag <= 1'b0;
case (seq_state) if (cdma_rd_req_valid && cdma_rd_req_ready)
SQ_IDLE: begin cdma_rd_req_valid <= 1'b0;
seq_busy <= 1'b0;
cdma_req_valid <= 1'b0; if (cdma_wr_req_valid && cdma_wr_req_ready)
if (start_req && (seq_groups != 32'd0)) begin cdma_wr_req_valid <= 1'b0;
seq_busy <= 1'b1;
seq_done_flag <= 1'b0; if (start_req && !seq_busy && (seq_groups != 32'd0)) begin
seq_load_idx <= 32'd0; seq_busy <= 1'b1;
seq_store_idx <= 32'd0; seq_done_flag <= 1'b0;
seq_compute_bank <= 1'b0; seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1];
seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1]; load_next_idx <= 32'd0;
seq_defer_load_start <= 1'b0; groups_retired <= 32'd0;
seq_deferred_start_valid <= 1'b0; bank_load_active <= 2'b00;
seq_deferred_start_bank <= 1'b0; bank_ab_valid <= 2'b00;
mat_crc_clear <= 1'b1; bank_compute_active <= 2'b00;
mat_done_clear <= 1'b1; bank_store_pending <= 2'b00;
seq_state <= SQ_CLEAR; bank_store_active <= 2'b00;
bank_group_idx[0] <= 32'd0;
bank_group_idx[1] <= 32'd0;
rd_busy <= 1'b0;
wr_busy <= 1'b0;
compute_q <= 2'b00;
compute_q_count <= 2'd0;
mat_done_seen <= 1'b1;
mat_crc_clear <= 1'b1;
mat_done_clear <= 1'b1;
cdma_rd_req_valid <= 1'b0;
cdma_wr_req_valid <= 1'b0;
end else if (seq_busy) begin
if (retire_inc != 32'd0) begin
groups_retired <= groups_retired + retire_inc;
if ((groups_retired + retire_inc) >= seq_groups) begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
end end
end end
SQ_CLEAR: begin if (cdma_rd_req_done) begin
issue_load_req(32'd0); rd_busy <= 1'b0;
bank_load_active[rd_active_bank] <= 1'b0;
bank_ab_valid[rd_active_bank] <= 1'b1;
end end
SQ_LOAD_START: begin if (cdma_wr_req_done) begin
issue_load_req(seq_load_idx); wr_busy <= 1'b0;
bank_store_active[wr_active_bank] <= 1'b0;
end end
SQ_LOAD_REQ: begin if (mat_done_event && (compute_q_count != 2'd0)) begin
if (cdma_req_valid && cdma_req_ready) begin mat_done_clear <= 1'b1;
cdma_req_valid <= 1'b0; mat_done_seen <= 1'b1;
seq_state <= SQ_LOAD_WAIT; bank_compute_active[compute_q[0]] <= 1'b0;
if (!seq_skip_writeback) begin
bank_store_pending[compute_q[0]] <= 1'b1;
end end
if (compute_q_count == 2'd2)
compute_q[0] <= compute_q[1];
compute_q_count <= compute_q_count - 2'd1;
end end
SQ_LOAD_WAIT: begin if (!rd_busy && !cdma_rd_req_valid && (load_next_idx < seq_groups) && load_target_free) begin
if (cdma_req_done) begin cdma_rd_req_src <= seq_src_base + (load_next_idx * seq_src_stride);
seq_load_idx <= seq_load_idx + 32'd1; cdma_rd_req_bytes <= MAT_AB_BYTES;
if (seq_defer_load_start) begin cdma_rd_req_ep_id <= 4'd0;
seq_defer_load_start <= 1'b0; cdma_rd_req_ep_bank <= {3'b000, load_target_bank};
seq_deferred_start_valid <= 1'b1; cdma_rd_req_ep_word_off <= 8'd0;
seq_deferred_start_bank <= cdma_req_ep_bank[0]; cdma_rd_req_valid <= 1'b1;
issue_store_req(); rd_busy <= 1'b1;
end else begin rd_active_bank <= load_target_bank;
mat_start_valid <= 1'b1; bank_load_active[load_target_bank] <= 1'b1;
mat_start_bank <= cdma_req_ep_bank[0]; bank_group_idx[load_target_bank] <= load_next_idx;
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin load_next_idx <= load_next_idx + 32'd1;
issue_load_req(seq_load_idx + 32'd1);
end else begin
seq_state <= SQ_WAIT_DONE;
end
end
end
end end
SQ_WAIT_DONE: begin if (!seq_skip_writeback && !wr_busy && !cdma_wr_req_valid && store_ready_any) begin
if (mat_done) begin cdma_wr_req_dst <= seq_dst_base + (bank_group_idx[store_sel_bank] * seq_dst_stride);
mat_done_clear <= 1'b1; cdma_wr_req_bytes <= MAT_C_BYTES;
if (seq_skip_writeback) begin cdma_wr_req_ep_id <= 4'd0;
seq_store_idx <= seq_store_idx + 32'd1; cdma_wr_req_ep_bank <= {3'b000, store_sel_bank};
seq_compute_bank <= ~seq_compute_bank; cdma_wr_req_ep_word_off <= 8'd0;
if ((seq_store_idx + 32'd1) >= seq_groups) begin cdma_wr_req_valid <= 1'b1;
seq_state <= SQ_DONE; wr_busy <= 1'b1;
end else if (seq_load_idx < seq_groups) begin wr_active_bank <= store_sel_bank;
issue_load_req(seq_load_idx); bank_store_pending[store_sel_bank] <= 1'b0;
end else begin bank_store_active[store_sel_bank] <= 1'b1;
seq_state <= SQ_WAIT_DONE;
end
end else if (seq_load_idx < seq_groups) begin
seq_defer_load_start <= 1'b1;
issue_load_req(seq_load_idx);
end else begin
issue_store_req();
end
end
end end
SQ_STORE_START: begin if ((compute_q_count < 2'd2) && start_ready_any && !(mat_done_event && (compute_q_count != 2'd0))) begin
issue_store_req(); mat_start_valid <= 1'b1;
mat_start_bank <= start_sel_bank;
bank_ab_valid[start_sel_bank] <= 1'b0;
bank_compute_active[start_sel_bank] <= 1'b1;
if (compute_q_count == 2'd0)
compute_q[0] <= start_sel_bank;
else
compute_q[1] <= start_sel_bank;
compute_q_count <= compute_q_count + 2'd1;
end end
end
SQ_STORE_REQ: begin
if (cdma_req_valid && cdma_req_ready) begin
cdma_req_valid <= 1'b0;
seq_state <= SQ_STORE_WAIT;
end
end
SQ_STORE_WAIT: begin
if (cdma_req_done) begin
seq_store_idx <= seq_store_idx + 32'd1;
seq_compute_bank <= ~seq_compute_bank;
if (seq_deferred_start_valid) begin
mat_start_valid <= 1'b1;
mat_start_bank <= seq_deferred_start_bank;
seq_deferred_start_valid <= 1'b0;
end
if ((seq_store_idx + 32'd1) >= seq_groups) begin
seq_state <= SQ_DONE;
end else if (seq_load_idx < seq_groups) begin
seq_state <= SQ_WAIT_DONE;
end else begin
seq_state <= SQ_WAIT_DONE;
end
end
end
SQ_DONE: begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
seq_state <= SQ_IDLE;
end
default: seq_state <= SQ_IDLE;
endcase
end end
end end

274
rtl/ip/cdma/snix_axil_cdma_mux.sv
View File

@@ -114,16 +114,23 @@ module snix_axil_cdma_mux #(
// ========================================== // ==========================================
// Generic external endpoint requester // Generic external endpoint requester
// ========================================== // ==========================================
input logic ext_req_valid, input logic ext_rd_req_valid,
output logic ext_req_ready, output logic ext_rd_req_ready,
input logic [1:0] ext_req_mode, input logic [ADDR_WIDTH-1:0] ext_rd_req_src,
input logic [ADDR_WIDTH-1:0] ext_req_src, input logic [31:0] ext_rd_req_bytes,
input logic [ADDR_WIDTH-1:0] ext_req_dst, input logic [3:0] ext_rd_req_ep_id,
input logic [31:0] ext_req_bytes, input logic [3:0] ext_rd_req_ep_bank,
input logic [3:0] ext_req_ep_id, input logic [7:0] ext_rd_req_ep_word_off,
input logic [3:0] ext_req_ep_bank, output logic ext_rd_req_done,
input logic [7:0] ext_req_ep_word_off,
output logic ext_req_done, input logic ext_wr_req_valid,
output logic ext_wr_req_ready,
input logic [ADDR_WIDTH-1:0] ext_wr_req_dst,
input logic [31:0] ext_wr_req_bytes,
input logic [3:0] ext_wr_req_ep_id,
input logic [3:0] ext_wr_req_ep_bank,
input logic [7:0] ext_wr_req_ep_word_off,
output logic ext_wr_req_done,
// Global Interrupt (OR'd from all channels) // Global Interrupt (OR'd from all channels)
output logic dma_finish output logic dma_finish
@@ -406,71 +413,82 @@ module snix_axil_cdma_mux #(
logic [31:0] ep_bytes_left; logic [31:0] ep_bytes_left;
logic [7:0] ep_word_addr_r; logic [7:0] ep_word_addr_r;
logic ext_active; typedef enum logic [1:0] {EXT_RD_IDLE, EXT_RD_AR, EXT_RD_READ} ext_rd_state_t;
logic [1:0] ext_mode_r; typedef enum logic [1:0] {EXT_WR_IDLE, EXT_WR_AW, EXT_WR_WRITE, EXT_WR_BRESP} ext_wr_state_t;
logic [ADDR_WIDTH-1:0] ext_src_r;
logic [ADDR_WIDTH-1:0] ext_dst_r;
logic [31:0] ext_bytes_r;
logic [3:0] ext_ep_id_r;
logic [3:0] ext_ep_bank_r;
logic [7:0] ext_ep_word_off_r;
wire ext_accept = ext_req_valid && ext_req_ready; ext_rd_state_t ext_rd_state;
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_accept; ext_wr_state_t ext_wr_state;
wire select_ext_ep = ext_active || ext_accept; logic [ADDR_WIDTH-1:0] ext_rd_addr_r;
logic [31:0] ext_rd_bytes_left;
logic [7:0] ext_rd_word_addr_r;
logic [3:0] ext_rd_ep_id_r;
logic [3:0] ext_rd_ep_bank_r;
logic [ADDR_WIDTH-1:0] ext_wr_addr_r;
logic [31:0] ext_wr_bytes_left;
logic [7:0] ext_wr_word_addr_r;
logic [3:0] ext_wr_ep_id_r;
logic [3:0] ext_wr_ep_bank_r;
wire ext_rd_active = (ext_rd_state != EXT_RD_IDLE);
wire ext_wr_active = (ext_wr_state != EXT_WR_IDLE);
wire ext_rd_accept = ext_rd_req_valid && ext_rd_req_ready;
wire ext_wr_accept = ext_wr_req_valid && ext_wr_req_ready;
wire ext_any_busy_or_req = ext_rd_active || ext_wr_active || ext_rd_req_valid || ext_wr_req_valid;
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM));
// Dynamic routing to the engine based on current active channel // Dynamic routing to the engine based on current active channel
assign engine_src = ext_accept ? ext_req_src : (select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]); assign engine_src = ch_src_addr[cur_ch];
assign engine_dst = ext_accept ? ext_req_dst : (select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]); assign engine_dst = ch_dst_addr[cur_ch];
assign engine_bytes = ext_accept ? ext_req_bytes : (select_ext_ep ? ext_bytes_r : ch_len[cur_ch]); assign engine_bytes = ch_len[cur_ch];
assign engine_len = ch_ctrl[cur_ch][13:6]; assign engine_len = ch_ctrl[cur_ch][13:6];
assign engine_size = ch_ctrl[cur_ch][5:3]; assign engine_size = ch_ctrl[cur_ch][5:3];
assign engine_mode = ext_accept ? ext_req_mode : (select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]); assign engine_mode = ch_mode[cur_ch][1:0];
assign engine_ep_bank = ext_accept ? ext_req_ep_bank : (select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]); assign engine_ep_bank = ch_ep_cfg[cur_ch][3:0];
assign engine_ep_word_off = ext_accept ? ext_req_ep_word_off : (select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]); assign engine_ep_word_off = ch_ep_cfg[cur_ch][15:8];
assign engine_ep_id = ext_accept ? ext_req_ep_id : (select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]); assign engine_ep_id = ch_ep_cfg[cur_ch][19:16];
assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done; assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM); assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
assign ep_start = ext_accept || (engine_start && (engine_mode != MODE_MEM2MEM)); assign ep_start = engine_start && (engine_mode != MODE_MEM2MEM);
assign core_awready = (!use_ep_engine) && mm2mm_awready; assign core_awready = (!ext_wr_active && !use_ep_engine) && mm2mm_awready;
assign core_wready = (!use_ep_engine) && mm2mm_wready; assign core_wready = (!ext_wr_active && !use_ep_engine) && mm2mm_wready;
assign core_arready = (!use_ep_engine) && mm2mm_arready; assign core_arready = (!ext_rd_active && !use_ep_engine) && mm2mm_arready;
assign ep_awready = use_ep_engine && mm2mm_awready; assign ep_awready = (!ext_wr_active && use_ep_engine) && mm2mm_awready;
assign ep_wready = use_ep_engine && mm2mm_wready; assign ep_wready = (!ext_wr_active && use_ep_engine) && mm2mm_wready;
assign ep_arready = use_ep_engine && mm2mm_arready; assign ep_arready = (!ext_rd_active && use_ep_engine) && mm2mm_arready;
assign mm2mm_awid = use_ep_engine ? ep_awid : core_awid; assign mm2mm_awid = ext_wr_active ? '0 : (use_ep_engine ? ep_awid : core_awid);
assign mm2mm_awaddr = use_ep_engine ? ep_awaddr : core_awaddr; assign mm2mm_awaddr = ext_wr_active ? ext_wr_addr_r : (use_ep_engine ? ep_awaddr : core_awaddr);
assign mm2mm_awlen = use_ep_engine ? ep_awlen : core_awlen; assign mm2mm_awlen = ext_wr_active ? (((ext_wr_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_wr_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_awlen : core_awlen);
assign mm2mm_awsize = use_ep_engine ? ep_awsize : core_awsize; assign mm2mm_awsize = ext_wr_active ? 3'd2 : (use_ep_engine ? ep_awsize : core_awsize);
assign mm2mm_awburst = use_ep_engine ? ep_awburst : core_awburst; assign mm2mm_awburst = ext_wr_active ? 2'b01 : (use_ep_engine ? ep_awburst : core_awburst);
assign mm2mm_awlock = use_ep_engine ? ep_awlock : core_awlock; assign mm2mm_awlock = ext_wr_active ? 1'b0 : (use_ep_engine ? ep_awlock : core_awlock);
assign mm2mm_awcache = use_ep_engine ? ep_awcache : core_awcache; assign mm2mm_awcache = ext_wr_active ? 4'b0 : (use_ep_engine ? ep_awcache : core_awcache);
assign mm2mm_awprot = use_ep_engine ? ep_awprot : core_awprot; assign mm2mm_awprot = ext_wr_active ? 3'b0 : (use_ep_engine ? ep_awprot : core_awprot);
assign mm2mm_awqos = use_ep_engine ? ep_awqos : core_awqos; assign mm2mm_awqos = ext_wr_active ? 4'b0 : (use_ep_engine ? ep_awqos : core_awqos);
assign mm2mm_awuser = use_ep_engine ? ep_awuser : core_awuser; assign mm2mm_awuser = ext_wr_active ? '0 : (use_ep_engine ? ep_awuser : core_awuser);
assign mm2mm_awvalid = use_ep_engine ? ep_awvalid : core_awvalid; assign mm2mm_awvalid = ext_wr_active ? (ext_wr_state == EXT_WR_AW) : (use_ep_engine ? ep_awvalid : core_awvalid);
assign mm2mm_wdata = use_ep_engine ? ep_wdata : core_wdata; assign mm2mm_wdata = ext_wr_active ? ep_rd_data : (use_ep_engine ? ep_wdata : core_wdata);
assign mm2mm_wstrb = use_ep_engine ? ep_wstrb : core_wstrb; assign mm2mm_wstrb = ext_wr_active ? {{(DATA_WIDTH/8){1'b1}}} : (use_ep_engine ? ep_wstrb : core_wstrb);
assign mm2mm_wlast = use_ep_engine ? ep_wlast : core_wlast; assign mm2mm_wlast = ext_wr_active ? ((ext_wr_bytes_left <= 32'd4) && mm2mm_wvalid) : (use_ep_engine ? ep_wlast : core_wlast);
assign mm2mm_wuser = use_ep_engine ? ep_wuser : core_wuser; assign mm2mm_wuser = ext_wr_active ? '0 : (use_ep_engine ? ep_wuser : core_wuser);
assign mm2mm_wvalid = use_ep_engine ? ep_wvalid : core_wvalid; assign mm2mm_wvalid = ext_wr_active ? ((ext_wr_state == EXT_WR_WRITE) && ep_rd_ready) : (use_ep_engine ? ep_wvalid : core_wvalid);
assign mm2mm_bready = use_ep_engine ? ep_bready : core_bready; assign mm2mm_bready = ext_wr_active ? (ext_wr_state == EXT_WR_BRESP) : (use_ep_engine ? ep_bready : core_bready);
assign mm2mm_arid = use_ep_engine ? ep_arid : core_arid; assign mm2mm_arid = ext_rd_active ? '0 : (use_ep_engine ? ep_arid : core_arid);
assign mm2mm_araddr = use_ep_engine ? ep_araddr : core_araddr; assign mm2mm_araddr = ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr);
assign mm2mm_arlen = use_ep_engine ? ep_arlen : core_arlen; assign mm2mm_arlen = ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_arlen : core_arlen);
assign mm2mm_arsize = use_ep_engine ? ep_arsize : core_arsize; assign mm2mm_arsize = ext_rd_active ? 3'd2 : (use_ep_engine ? ep_arsize : core_arsize);
assign mm2mm_arburst = use_ep_engine ? ep_arburst : core_arburst; assign mm2mm_arburst = ext_rd_active ? 2'b01 : (use_ep_engine ? ep_arburst : core_arburst);
assign mm2mm_arlock = use_ep_engine ? ep_arlock : core_arlock; assign mm2mm_arlock = ext_rd_active ? 1'b0 : (use_ep_engine ? ep_arlock : core_arlock);
assign mm2mm_arcache = use_ep_engine ? ep_arcache : core_arcache; assign mm2mm_arcache = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arcache : core_arcache);
assign mm2mm_arprot = use_ep_engine ? ep_arprot : core_arprot; assign mm2mm_arprot = ext_rd_active ? 3'b0 : (use_ep_engine ? ep_arprot : core_arprot);
assign mm2mm_arqos = use_ep_engine ? ep_arqos : core_arqos; assign mm2mm_arqos = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arqos : core_arqos);
assign mm2mm_aruser = use_ep_engine ? ep_aruser : core_aruser; assign mm2mm_aruser = ext_rd_active ? '0 : (use_ep_engine ? ep_aruser : core_aruser);
assign mm2mm_arvalid = use_ep_engine ? ep_arvalid : core_arvalid; assign mm2mm_arvalid = ext_rd_active ? (ext_rd_state == EXT_RD_AR) : (use_ep_engine ? ep_arvalid : core_arvalid);
assign mm2mm_rready = use_ep_engine ? ep_rready : core_rready; assign mm2mm_rready = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && ep_wr_ready) : (use_ep_engine ? ep_rready : core_rready);
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
@@ -485,7 +503,7 @@ module snix_axil_cdma_mux #(
case (state) case (state)
IDLE: begin IDLE: begin
if (ext_req_valid || ext_active) begin if (ext_any_busy_or_req) begin
state <= IDLE; state <= IDLE;
end else for (int i = 0; i < PORTS; i++) begin end else for (int i = 0; i < PORTS; i++) begin
logic [CH_BITS:0] check_ch_ext; logic [CH_BITS:0] check_ch_ext;
@@ -572,36 +590,100 @@ module snix_axil_cdma_mux #(
wire ep_rd_done_fire = (ep_state == EP_READ) && mm2mm_rvalid && ep_rready && mm2mm_rlast; wire ep_rd_done_fire = (ep_state == EP_READ) && mm2mm_rvalid && ep_rready && mm2mm_rlast;
wire ep_wr_done_fire = (ep_state == EP_BRESP) && mm2mm_bvalid && ep_bready; wire ep_wr_done_fire = (ep_state == EP_BRESP) && mm2mm_bvalid && ep_bready;
wire ep_done_fast = ep_rd_done_fire || ep_wr_done_fire; wire ep_done_fast = ep_rd_done_fire || ep_wr_done_fire;
wire ext_rd_done_fire = (ext_rd_state == EXT_RD_READ) && mm2mm_rvalid && mm2mm_rready && mm2mm_rlast;
wire ext_wr_done_fire = (ext_wr_state == EXT_WR_BRESP) && mm2mm_bvalid && mm2mm_bready;
wire ext_wr_w_fire = (ext_wr_state == EXT_WR_WRITE) && mm2mm_wvalid && mm2mm_wready;
// ========================================== // ==========================================
// Generic external endpoint requester // Generic external endpoint requester
// ========================================== // ==========================================
assign ext_req_ready = !ext_active && (state == IDLE) && (ep_state == EP_IDLE); assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_rd_active;
assign ext_req_done = ext_active && ep_done_fast; assign ext_wr_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active;
assign ext_rd_req_done = ext_rd_done_fire;
assign ext_wr_req_done = ext_wr_done_fire;
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
ext_active <= 1'b0; ext_rd_state <= EXT_RD_IDLE;
ext_mode_r <= MODE_MEM2EP; ext_rd_addr_r <= '0;
ext_src_r <= '0; ext_rd_bytes_left <= '0;
ext_dst_r <= '0; ext_rd_word_addr_r <= '0;
ext_bytes_r <= '0; ext_rd_ep_id_r <= '0;
ext_ep_id_r <= '0; ext_rd_ep_bank_r <= '0;
ext_ep_bank_r <= '0;
ext_ep_word_off_r <= '0;
end else begin end else begin
if (ext_accept) begin case (ext_rd_state)
ext_active <= 1'b1; EXT_RD_IDLE: begin
ext_mode_r <= ext_req_mode; if (ext_rd_accept) begin
ext_src_r <= ext_req_src; ext_rd_addr_r <= ext_rd_req_src;
ext_dst_r <= ext_req_dst; ext_rd_bytes_left <= ext_rd_req_bytes;
ext_bytes_r <= ext_req_bytes; ext_rd_word_addr_r <= ext_rd_req_ep_word_off;
ext_ep_id_r <= ext_req_ep_id; ext_rd_ep_id_r <= ext_rd_req_ep_id;
ext_ep_bank_r <= ext_req_ep_bank; ext_rd_ep_bank_r <= ext_rd_req_ep_bank;
ext_ep_word_off_r <= ext_req_ep_word_off; ext_rd_state <= EXT_RD_AR;
end else if (ext_active && ep_done_fast) begin end
ext_active <= 1'b0; end
end
EXT_RD_AR: begin
if (mm2mm_arvalid && mm2mm_arready)
ext_rd_state <= EXT_RD_READ;
end
EXT_RD_READ: begin
if (mm2mm_rvalid && mm2mm_rready) begin
ext_rd_word_addr_r <= ext_rd_word_addr_r + 8'd1;
ext_rd_bytes_left <= (ext_rd_bytes_left > 32'd4) ? (ext_rd_bytes_left - 32'd4) : 32'd0;
if (mm2mm_rlast)
ext_rd_state <= EXT_RD_IDLE;
end
end
default: ext_rd_state <= EXT_RD_IDLE;
endcase
end
end
always_ff @(posedge clk) begin
if (!rst_n) begin
ext_wr_state <= EXT_WR_IDLE;
ext_wr_addr_r <= '0;
ext_wr_bytes_left <= '0;
ext_wr_word_addr_r <= '0;
ext_wr_ep_id_r <= '0;
ext_wr_ep_bank_r <= '0;
end else begin
case (ext_wr_state)
EXT_WR_IDLE: begin
if (ext_wr_accept) begin
ext_wr_addr_r <= ext_wr_req_dst;
ext_wr_bytes_left <= ext_wr_req_bytes;
ext_wr_word_addr_r <= ext_wr_req_ep_word_off;
ext_wr_ep_id_r <= ext_wr_req_ep_id;
ext_wr_ep_bank_r <= ext_wr_req_ep_bank;
ext_wr_state <= EXT_WR_AW;
end
end
EXT_WR_AW: begin
if (mm2mm_awvalid && mm2mm_awready)
ext_wr_state <= EXT_WR_WRITE;
end
EXT_WR_WRITE: begin
if (ext_wr_w_fire) begin
ext_wr_word_addr_r <= ext_wr_word_addr_r + 8'd1;
ext_wr_bytes_left <= (ext_wr_bytes_left > 32'd4) ? (ext_wr_bytes_left - 32'd4) : 32'd0;
if (mm2mm_wlast)
ext_wr_state <= EXT_WR_BRESP;
end
end
EXT_WR_BRESP: begin
if (mm2mm_bvalid && mm2mm_bready)
ext_wr_state <= EXT_WR_IDLE;
end
default: ext_wr_state <= EXT_WR_IDLE;
endcase
end end
end end
@@ -688,10 +770,11 @@ module snix_axil_cdma_mux #(
assign ep_arvalid = (ep_state == EP_AR); assign ep_arvalid = (ep_state == EP_AR);
assign ep_rready = (ep_state == EP_READ) && ep_wr_ready; assign ep_rready = (ep_state == EP_READ) && ep_wr_ready;
assign ep_wr_valid = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep; assign ep_wr_valid = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && mm2mm_rvalid) :
assign ep_wr_id = engine_ep_id; ((ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep);
assign ep_wr_bank = engine_ep_bank; assign ep_wr_id = ext_rd_active ? ext_rd_ep_id_r : engine_ep_id;
assign ep_wr_word_addr = ep_word_addr_r; assign ep_wr_bank = ext_rd_active ? ext_rd_ep_bank_r : engine_ep_bank;
assign ep_wr_word_addr = ext_rd_active ? ext_rd_word_addr_r : ep_word_addr_r;
assign ep_wr_data = mm2mm_rdata[31:0]; assign ep_wr_data = mm2mm_rdata[31:0];
assign ep_awid = '0; assign ep_awid = '0;
@@ -712,9 +795,10 @@ module snix_axil_cdma_mux #(
assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready; assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready;
assign ep_bready = (ep_state == EP_BRESP); assign ep_bready = (ep_state == EP_BRESP);
assign ep_rd_valid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready; assign ep_rd_valid = ext_wr_active ? ((ext_wr_state == EXT_WR_WRITE) && mm2mm_wready) :
assign ep_rd_id = engine_ep_id; ((ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready);
assign ep_rd_bank = engine_ep_bank; assign ep_rd_id = ext_wr_active ? ext_wr_ep_id_r : engine_ep_id;
assign ep_rd_word_addr = ep_word_addr_r; assign ep_rd_bank = ext_wr_active ? ext_wr_ep_bank_r : engine_ep_bank;
assign ep_rd_word_addr = ext_wr_active ? ext_wr_word_addr_r : ep_word_addr_r;
endmodule endmodule

80
rtl/soc_top.v
View File

@@ -1619,16 +1619,22 @@ wire [1:0] cdma_bridge_rresp;
wire cdma_bridge_rvalid; wire cdma_bridge_rvalid;
wire cdma_bridge_rready; wire cdma_bridge_rready;
wire bridge_cdma_req_valid; wire bridge_cdma_rd_req_valid;
wire bridge_cdma_req_ready; wire bridge_cdma_rd_req_ready;
wire [1:0] bridge_cdma_req_mode; wire [31:0] bridge_cdma_rd_req_src;
wire [31:0] bridge_cdma_req_src; wire [31:0] bridge_cdma_rd_req_bytes;
wire [31:0] bridge_cdma_req_dst; wire [3:0] bridge_cdma_rd_req_ep_id;
wire [31:0] bridge_cdma_req_bytes; wire [3:0] bridge_cdma_rd_req_ep_bank;
wire [3:0] bridge_cdma_req_ep_id; wire [7:0] bridge_cdma_rd_req_ep_word_off;
wire [3:0] bridge_cdma_req_ep_bank; wire bridge_cdma_rd_req_done;
wire [7:0] bridge_cdma_req_ep_word_off; wire bridge_cdma_wr_req_valid;
wire bridge_cdma_req_done; wire bridge_cdma_wr_req_ready;
wire [31:0] bridge_cdma_wr_req_dst;
wire [31:0] bridge_cdma_wr_req_bytes;
wire [3:0] bridge_cdma_wr_req_ep_id;
wire [3:0] bridge_cdma_wr_req_ep_bank;
wire [7:0] bridge_cdma_wr_req_ep_word_off;
wire bridge_cdma_wr_req_done;
wire cdma_axil_wr_bridge = (cdma_s_awaddr[11:8] == 4'h2); wire cdma_axil_wr_bridge = (cdma_s_awaddr[11:8] == 4'h2);
wire cdma_axil_rd_bridge = (cdma_s_araddr[11:8] == 4'h2); wire cdma_axil_rd_bridge = (cdma_s_araddr[11:8] == 4'h2);
@@ -1781,16 +1787,23 @@ cdma_matmul_bridge #(
.s_axil_rvalid (cdma_bridge_rvalid), .s_axil_rvalid (cdma_bridge_rvalid),
.s_axil_rready (cdma_bridge_rready), .s_axil_rready (cdma_bridge_rready),
.cdma_req_valid (bridge_cdma_req_valid), .cdma_rd_req_valid (bridge_cdma_rd_req_valid),
.cdma_req_ready (bridge_cdma_req_ready), .cdma_rd_req_ready (bridge_cdma_rd_req_ready),
.cdma_req_mode (bridge_cdma_req_mode), .cdma_rd_req_src (bridge_cdma_rd_req_src),
.cdma_req_src (bridge_cdma_req_src), .cdma_rd_req_bytes (bridge_cdma_rd_req_bytes),
.cdma_req_dst (bridge_cdma_req_dst), .cdma_rd_req_ep_id (bridge_cdma_rd_req_ep_id),
.cdma_req_bytes (bridge_cdma_req_bytes), .cdma_rd_req_ep_bank (bridge_cdma_rd_req_ep_bank),
.cdma_req_ep_id (bridge_cdma_req_ep_id), .cdma_rd_req_ep_word_off(bridge_cdma_rd_req_ep_word_off),
.cdma_req_ep_bank (bridge_cdma_req_ep_bank), .cdma_rd_req_done (bridge_cdma_rd_req_done),
.cdma_req_ep_word_off(bridge_cdma_req_ep_word_off),
.cdma_req_done (bridge_cdma_req_done), .cdma_wr_req_valid (bridge_cdma_wr_req_valid),
.cdma_wr_req_ready (bridge_cdma_wr_req_ready),
.cdma_wr_req_dst (bridge_cdma_wr_req_dst),
.cdma_wr_req_bytes (bridge_cdma_wr_req_bytes),
.cdma_wr_req_ep_id (bridge_cdma_wr_req_ep_id),
.cdma_wr_req_ep_bank (bridge_cdma_wr_req_ep_bank),
.cdma_wr_req_ep_word_off(bridge_cdma_wr_req_ep_word_off),
.cdma_wr_req_done (bridge_cdma_wr_req_done),
.mat_start_valid (matmul_direct_start_valid), .mat_start_valid (matmul_direct_start_valid),
.mat_start_bank (matmul_direct_start_bank), .mat_start_bank (matmul_direct_start_bank),
@@ -1891,16 +1904,23 @@ snix_axil_cdma_mux #(
.ep_rd_data (matmul_ep_rd_data), .ep_rd_data (matmul_ep_rd_data),
.ep_rd_ready (matmul_ep_rd_ready), .ep_rd_ready (matmul_ep_rd_ready),
.ext_req_valid (bridge_cdma_req_valid), .ext_rd_req_valid (bridge_cdma_rd_req_valid),
.ext_req_ready (bridge_cdma_req_ready), .ext_rd_req_ready (bridge_cdma_rd_req_ready),
.ext_req_mode (bridge_cdma_req_mode), .ext_rd_req_src (bridge_cdma_rd_req_src),
.ext_req_src (bridge_cdma_req_src), .ext_rd_req_bytes (bridge_cdma_rd_req_bytes),
.ext_req_dst (bridge_cdma_req_dst), .ext_rd_req_ep_id (bridge_cdma_rd_req_ep_id),
.ext_req_bytes (bridge_cdma_req_bytes), .ext_rd_req_ep_bank (bridge_cdma_rd_req_ep_bank),
.ext_req_ep_id (bridge_cdma_req_ep_id), .ext_rd_req_ep_word_off(bridge_cdma_rd_req_ep_word_off),
.ext_req_ep_bank (bridge_cdma_req_ep_bank), .ext_rd_req_done (bridge_cdma_rd_req_done),
.ext_req_ep_word_off(bridge_cdma_req_ep_word_off),
.ext_req_done (bridge_cdma_req_done), .ext_wr_req_valid (bridge_cdma_wr_req_valid),
.ext_wr_req_ready (bridge_cdma_wr_req_ready),
.ext_wr_req_dst (bridge_cdma_wr_req_dst),
.ext_wr_req_bytes (bridge_cdma_wr_req_bytes),
.ext_wr_req_ep_id (bridge_cdma_wr_req_ep_id),
.ext_wr_req_ep_bank (bridge_cdma_wr_req_ep_bank),
.ext_wr_req_ep_word_off(bridge_cdma_wr_req_ep_word_off),
.ext_wr_req_done (bridge_cdma_wr_req_done),
// 全局中断输出 // 全局中断输出
.dma_finish (cdma_dma_finish) .dma_finish (cdma_dma_finish)