fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize CDMA matmul bridge scheduling

Browse Source
This commit is contained in:
FallenSigh
2026-07-04 19:59:49 +08:00
parent c37f493bcb
commit 27cdfb713a
3 changed files with 106 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
rtl/ip/cdma/cdma_matmul_bridge.sv
View File

@@ -103,6 +103,9 @@ module cdma_matmul_bridge #(
logic [31:0] seq_store_idx; logic [31:0] seq_store_idx;
logic seq_compute_bank; logic seq_compute_bank;
logic seq_skip_writeback; logic seq_skip_writeback;
logic seq_defer_load_start;
logic seq_deferred_start_valid;
logic seq_deferred_start_bank;
wire [7:0] wr_off = s_axil_awaddr[7:0]; wire [7:0] wr_off = s_axil_awaddr[7:0];
wire [7:0] rd_off = s_axil_araddr[7:0]; wire [7:0] rd_off = s_axil_araddr[7:0];
@@ -223,6 +226,9 @@ module cdma_matmul_bridge #(
seq_store_idx <= 32'd0; seq_store_idx <= 32'd0;
seq_compute_bank <= 1'b0; seq_compute_bank <= 1'b0;
seq_skip_writeback <= 1'b0; seq_skip_writeback <= 1'b0;
seq_defer_load_start <= 1'b0;
seq_deferred_start_valid <= 1'b0;
seq_deferred_start_bank <= 1'b0;
cdma_req_valid <= 1'b0; cdma_req_valid <= 1'b0;
cdma_req_mode <= MODE_MEM2EP; cdma_req_mode <= MODE_MEM2EP;
cdma_req_src <= '0; cdma_req_src <= '0;
@@ -254,6 +260,9 @@ module cdma_matmul_bridge #(
seq_store_idx <= 32'd0; seq_store_idx <= 32'd0;
seq_compute_bank <= 1'b0; seq_compute_bank <= 1'b0;
seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1]; seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1];
seq_defer_load_start <= 1'b0;
seq_deferred_start_valid <= 1'b0;
seq_deferred_start_bank <= 1'b0;
mat_crc_clear <= 1'b1; mat_crc_clear <= 1'b1;
mat_done_clear <= 1'b1; mat_done_clear <= 1'b1;
seq_state <= SQ_CLEAR; seq_state <= SQ_CLEAR;
@@ -285,13 +294,20 @@ module cdma_matmul_bridge #(
SQ_LOAD_WAIT: begin SQ_LOAD_WAIT: begin
if (cdma_req_done) begin if (cdma_req_done) begin
mat_start_valid <= 1'b1; seq_load_idx <= seq_load_idx + 32'd1;
mat_start_bank <= cdma_req_ep_bank[0]; if (seq_defer_load_start) begin
seq_load_idx <= seq_load_idx + 32'd1; seq_defer_load_start <= 1'b0;
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin seq_deferred_start_valid <= 1'b1;
seq_state <= SQ_LOAD_START; seq_deferred_start_bank <= cdma_req_ep_bank[0];
seq_state <= SQ_STORE_START;
end else begin end else begin
seq_state <= SQ_WAIT_DONE; mat_start_valid <= 1'b1;
mat_start_bank <= cdma_req_ep_bank[0];
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin
seq_state <= SQ_LOAD_START;
end else begin
seq_state <= SQ_WAIT_DONE;
end
end end
end end
end end
@@ -309,6 +325,9 @@ module cdma_matmul_bridge #(
end else begin end else begin
seq_state <= SQ_WAIT_DONE; seq_state <= SQ_WAIT_DONE;
end end
end else if (seq_load_idx < seq_groups) begin
seq_defer_load_start <= 1'b1;
seq_state <= SQ_LOAD_START;
end else begin end else begin
seq_state <= SQ_STORE_START; seq_state <= SQ_STORE_START;
end end
@@ -338,10 +357,15 @@ module cdma_matmul_bridge #(
if (cdma_req_done) begin if (cdma_req_done) begin
seq_store_idx <= seq_store_idx + 32'd1; seq_store_idx <= seq_store_idx + 32'd1;
seq_compute_bank <= ~seq_compute_bank; seq_compute_bank <= ~seq_compute_bank;
if (seq_deferred_start_valid) begin
mat_start_valid <= 1'b1;
mat_start_bank <= seq_deferred_start_bank;
seq_deferred_start_valid <= 1'b0;
end
if ((seq_store_idx + 32'd1) >= seq_groups) begin if ((seq_store_idx + 32'd1) >= seq_groups) begin
seq_state <= SQ_DONE; seq_state <= SQ_DONE;
end else if (seq_load_idx < seq_groups) begin end else if (seq_load_idx < seq_groups) begin
seq_state <= SQ_LOAD_START; seq_state <= SQ_WAIT_DONE;
end else begin end else begin
seq_state <= SQ_WAIT_DONE; seq_state <= SQ_WAIT_DONE;
end end

41
rtl/ip/cdma/snix_axil_cdma_mux.sv
View File

@@ -401,7 +401,6 @@ module snix_axil_cdma_mux #(
logic ep_rready; logic ep_rready;
logic ext_active; logic ext_active;
logic ext_start_pulse;
logic [1:0] ext_mode_r; logic [1:0] ext_mode_r;
logic [ADDR_WIDTH-1:0] ext_src_r; logic [ADDR_WIDTH-1:0] ext_src_r;
logic [ADDR_WIDTH-1:0] ext_dst_r; logic [ADDR_WIDTH-1:0] ext_dst_r;
@@ -410,22 +409,23 @@ module snix_axil_cdma_mux #(
logic [3:0] ext_ep_bank_r; logic [3:0] ext_ep_bank_r;
logic [7:0] ext_ep_word_off_r; logic [7:0] ext_ep_word_off_r;
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_start_pulse; wire ext_accept = ext_req_valid && ext_req_ready;
wire select_ext_ep = ext_active || ext_start_pulse; wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || ext_active || ext_accept;
wire select_ext_ep = ext_active || ext_accept;
// Dynamic routing to the engine based on current active channel // Dynamic routing to the engine based on current active channel
assign engine_src = select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]; assign engine_src = ext_accept ? ext_req_src : (select_ext_ep ? ext_src_r : ch_src_addr[cur_ch]);
assign engine_dst = select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]; assign engine_dst = ext_accept ? ext_req_dst : (select_ext_ep ? ext_dst_r : ch_dst_addr[cur_ch]);
assign engine_bytes = select_ext_ep ? ext_bytes_r : ch_len[cur_ch]; assign engine_bytes = ext_accept ? ext_req_bytes : (select_ext_ep ? ext_bytes_r : ch_len[cur_ch]);
assign engine_len = ch_ctrl[cur_ch][13:6]; assign engine_len = ch_ctrl[cur_ch][13:6];
assign engine_size = ch_ctrl[cur_ch][5:3]; assign engine_size = ch_ctrl[cur_ch][5:3];
assign engine_mode = select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]; assign engine_mode = ext_accept ? ext_req_mode : (select_ext_ep ? ext_mode_r : ch_mode[cur_ch][1:0]);
assign engine_ep_bank = select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]; assign engine_ep_bank = ext_accept ? ext_req_ep_bank : (select_ext_ep ? ext_ep_bank_r : ch_ep_cfg[cur_ch][3:0]);
assign engine_ep_word_off = select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]; assign engine_ep_word_off = ext_accept ? ext_req_ep_word_off : (select_ext_ep ? ext_ep_word_off_r : ch_ep_cfg[cur_ch][15:8]);
assign engine_ep_id = select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]; assign engine_ep_id = ext_accept ? ext_req_ep_id : (select_ext_ep ? ext_ep_id_r : ch_ep_cfg[cur_ch][19:16]);
assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done; assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM); assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
assign ep_start = ext_start_pulse || (engine_start && (engine_mode != MODE_MEM2MEM)); assign ep_start = ext_accept || (engine_start && (engine_mode != MODE_MEM2MEM));
assign core_awready = (!use_ep_engine) && mm2mm_awready; assign core_awready = (!use_ep_engine) && mm2mm_awready;
assign core_wready = (!use_ep_engine) && mm2mm_wready; assign core_wready = (!use_ep_engine) && mm2mm_wready;
@@ -562,12 +562,11 @@ module snix_axil_cdma_mux #(
// ========================================== // ==========================================
// Generic external endpoint requester // Generic external endpoint requester
// ========================================== // ==========================================
assign ext_req_ready = !ext_active && (state == IDLE); assign ext_req_ready = !ext_active && (state == IDLE) && (ep_state == EP_IDLE);
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
ext_active <= 1'b0; ext_active <= 1'b0;
ext_start_pulse <= 1'b0;
ext_req_done <= 1'b0; ext_req_done <= 1'b0;
ext_mode_r <= MODE_MEM2EP; ext_mode_r <= MODE_MEM2EP;
ext_src_r <= '0; ext_src_r <= '0;
@@ -577,12 +576,10 @@ module snix_axil_cdma_mux #(
ext_ep_bank_r <= '0; ext_ep_bank_r <= '0;
ext_ep_word_off_r <= '0; ext_ep_word_off_r <= '0;
end else begin end else begin
ext_start_pulse <= 1'b0;
ext_req_done <= 1'b0; ext_req_done <= 1'b0;
if (ext_req_valid && ext_req_ready) begin if (ext_accept) begin
ext_active <= 1'b1; ext_active <= 1'b1;
ext_start_pulse <= 1'b1;
ext_mode_r <= ext_req_mode; ext_mode_r <= ext_req_mode;
ext_src_r <= ext_req_src; ext_src_r <= ext_req_src;
ext_dst_r <= ext_req_dst; ext_dst_r <= ext_req_dst;
@@ -640,8 +637,10 @@ module snix_axil_cdma_mux #(
if (mm2mm_rvalid && ep_rready) begin if (mm2mm_rvalid && ep_rready) begin
ep_word_addr_r <= ep_word_addr_r + 8'd1; ep_word_addr_r <= ep_word_addr_r + 8'd1;
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0; ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
if (mm2mm_rlast) if (mm2mm_rlast) begin
ep_state <= EP_DONE; ep_done <= 1'b1;
ep_state <= EP_IDLE;
end
end end
end end
@@ -660,8 +659,10 @@ module snix_axil_cdma_mux #(
end end
EP_BRESP: begin EP_BRESP: begin
if (mm2mm_bvalid && ep_bready) if (mm2mm_bvalid && ep_bready) begin
ep_state <= EP_DONE; ep_done <= 1'b1;
ep_state <= EP_IDLE;
end
end end
EP_DONE: begin EP_DONE: begin

67
rtl/ip/matmul/matmul_axi_slave.v
View File

@@ -582,14 +582,39 @@ module matmul_axi_slave (
// Pipeline delay shift registers for accumulation (4 stages = 3 multiplier pipe + 1) // Pipeline delay shift registers for accumulation (4 stages = 3 multiplier pipe + 1)
reg [3:0] acc_idx_d1, acc_idx_d2, acc_idx_d3, acc_idx_d4; reg [3:0] acc_idx_d1, acc_idx_d2, acc_idx_d3, acc_idx_d4;
reg acc_bank_d1, acc_bank_d2, acc_bank_d3, acc_bank_d4; reg acc_bank_d1, acc_bank_d2, acc_bank_d3, acc_bank_d4;
reg acc_valid_d1, acc_valid_d2, acc_valid_d3, acc_valid_d4;
reg [4:0] pipe_cnt; reg [4:0] pipe_cnt;
wire [65:0] calc_c_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) + wire [65:0] calc_c_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]}); ({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
wire calc_crc_valid = (state == C_RUN) && (pipe_cnt >= 5'd4) && (pipe_cnt < 5'd20); wire calc_crc_valid = acc_valid_d4;
wire [31:0] crc_calc_0 = crc32_word_next(crc_reg, calc_c_result[31:0]); wire [31:0] crc_calc_0 = crc32_word_next(crc_reg, calc_c_result[31:0]);
wire [31:0] crc_calc_1 = crc32_word_next(crc_calc_0, calc_c_result[63:32]); wire [31:0] crc_calc_1 = crc32_word_next(crc_calc_0, calc_c_result[63:32]);
wire [31:0] crc_calc_2 = crc32_word_next(crc_calc_1, {30'b0, calc_c_result[65:64]}); wire [31:0] crc_calc_2 = crc32_word_next(crc_calc_1, {30'b0, calc_c_result[65:64]});
task start_calc_bank;
input bank;
begin
cur_bank <= bank;
busy <= 1'b1;
state <= C_RUN;
pipe_cnt <= 5'd1;
acc_idx_d1 <= 4'd0;
acc_bank_d1 <= bank;
acc_valid_d1 <= 1'b1;
if (bank == 1'b0) begin
m_in_a[0] <= b0_a[4'd0]; m_in_b[0] <= b0_b[4'd0];
m_in_a[1] <= b0_a[4'd1]; m_in_b[1] <= b0_b[4'd4];
m_in_a[2] <= b0_a[4'd2]; m_in_b[2] <= b0_b[4'd8];
m_in_a[3] <= b0_a[4'd3]; m_in_b[3] <= b0_b[4'd12];
end else begin
m_in_a[0] <= b1_a[4'd0]; m_in_b[0] <= b1_b[4'd0];
m_in_a[1] <= b1_a[4'd1]; m_in_b[1] <= b1_b[4'd4];
m_in_a[2] <= b1_a[4'd2]; m_in_b[2] <= b1_b[4'd8];
m_in_a[3] <= b1_a[4'd3]; m_in_b[3] <= b1_b[4'd12];
end
end
endtask
// CRC tracks completed C elements directly, independent of result writeback. // CRC tracks completed C elements directly, independent of result writeback.
always @(posedge clk) begin always @(posedge clk) begin
if (!resetn) begin if (!resetn) begin
@@ -616,6 +641,10 @@ module matmul_axi_slave (
acc_bank_d2 <= 1'b0; acc_bank_d2 <= 1'b0;
acc_bank_d3 <= 1'b0; acc_bank_d3 <= 1'b0;
acc_bank_d4 <= 1'b0; acc_bank_d4 <= 1'b0;
acc_valid_d1 <= 1'b0;
acc_valid_d2 <= 1'b0;
acc_valid_d3 <= 1'b0;
acc_valid_d4 <= 1'b0;
end else begin end else begin
calc_done_pulse <= 1'b0; calc_done_pulse <= 1'b0;
@@ -623,24 +652,24 @@ module matmul_axi_slave (
acc_idx_d4 <= acc_idx_d3; acc_idx_d4 <= acc_idx_d3;
acc_idx_d3 <= acc_idx_d2; acc_idx_d3 <= acc_idx_d2;
acc_idx_d2 <= acc_idx_d1; acc_idx_d2 <= acc_idx_d1;
acc_idx_d1 <= pipe_cnt[3:0]; acc_idx_d1 <= 4'd0;
acc_bank_d4 <= acc_bank_d3; acc_bank_d4 <= acc_bank_d3;
acc_bank_d3 <= acc_bank_d2; acc_bank_d3 <= acc_bank_d2;
acc_bank_d2 <= acc_bank_d1; acc_bank_d2 <= acc_bank_d1;
acc_bank_d1 <= cur_bank; acc_bank_d1 <= 1'b0;
acc_valid_d4 <= acc_valid_d3;
acc_valid_d3 <= acc_valid_d2;
acc_valid_d2 <= acc_valid_d1;
acc_valid_d1 <= 1'b0;
case (state) case (state)
C_IDLE: begin C_IDLE: begin
pipe_cnt <= 5'd0;
if (req_0) begin if (req_0) begin
cur_bank <= 1'b0; start_calc_bank(1'b0);
busy <= 1'b1;
state <= C_RUN;
end else if (req_1) begin end else if (req_1) begin
cur_bank <= 1'b1; start_calc_bank(1'b1);
busy <= 1'b1;
state <= C_RUN;
end else begin end else begin
pipe_cnt <= 5'd0;
busy <= 1'b0; busy <= 1'b0;
end end
end end
@@ -648,6 +677,9 @@ module matmul_axi_slave (
C_RUN: begin C_RUN: begin
// Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds) // Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds)
if (pipe_cnt < 5'd16) begin if (pipe_cnt < 5'd16) begin
acc_idx_d1 <= pipe_cnt[3:0];
acc_bank_d1 <= cur_bank;
acc_valid_d1 <= 1'b1;
if (cur_bank == 1'b0) begin if (cur_bank == 1'b0) begin
m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}]; m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}];
m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}]; m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}];
@@ -661,16 +693,25 @@ module matmul_axi_slave (
end end
end end
// Accumulate results from 4 cycles ago (pipe_cnt 4-19) // Accumulate results from 4 cycles ago.
if (pipe_cnt >= 5'd4 && pipe_cnt < 5'd20) begin if (acc_valid_d4) begin
if (acc_bank_d4 == 1'b0) if (acc_bank_d4 == 1'b0)
b0_c[acc_idx_d4] <= calc_c_result; b0_c[acc_idx_d4] <= calc_c_result;
else else
b1_c[acc_idx_d4] <= calc_c_result; b1_c[acc_idx_d4] <= calc_c_result;
end end
if (pipe_cnt == 5'd20) begin if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
calc_done_pulse <= 1'b1; calc_done_pulse <= 1'b1;
end
if (pipe_cnt < 5'd16) begin
pipe_cnt <= pipe_cnt + 5'd1;
end else if (cur_bank == 1'b0 && req_1) begin
start_calc_bank(1'b1);
end else if (cur_bank == 1'b1 && req_0) begin
start_calc_bank(1'b0);
end else if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
busy <= 1'b0; busy <= 1'b0;
state <= C_IDLE; state <= C_IDLE;
end else begin end else begin