fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize matmul CRC pipeline and expose DMA cycles

Browse Source
This commit is contained in:
FallenSigh
2026-07-14 18:21:45 +08:00
parent 7c60da78cb
commit f3af271788
2 changed files with 116 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
rtl/ip/matmul/matmul_axi_slave.v
View File

@@ -148,6 +148,7 @@ module matmul_axi_slave (
localparam ADDR_DMA_GROUPS = 12'h028;
localparam ADDR_DMA_SRC_STRIDE = 12'h02C;
localparam ADDR_DMA_DST_STRIDE = 12'h030;
localparam ADDR_DMA_CYCLES = 12'h034;
localparam ADDR_B0_A = 12'h100;
localparam ADDR_B0_B = 12'h140;
@@ -179,8 +180,11 @@ module matmul_axi_slave (
reg [31:0] dma_groups;
reg [31:0] dma_src_stride;
reg [31:0] dma_dst_stride;
// Full sys_clk cycles spent in dma_busy; held after DMA completion.
reg [31:0] dma_cycle_count;
reg dma_busy;
reg dma_done;
reg dma_compute_complete;
reg [31:0] load_group;
reg [31:0] store_group;
reg [31:0] dma_next_rd_addr;
@@ -257,10 +261,9 @@ module matmul_axi_slave (
//
// AXI-stream: single-cycle crc32_update_word (4 LUT lookup depth, ok for
// AXI timing—the bus runs at or below internal compute rate).
// DMA-calc: 2-stage pipelined word engine (2 bytes/stage) + 6-cycle
// serial FSM for 66-bit C-elements, with 16-deep FIFO to absorb the
// compute pipeline burst. Critical path: 2 table lookups + 1 XOR per
// stage vs 12 table lookups in the original single-cycle design.
// DMA-calc: two 6-byte CRC steps per 66-bit C-element. A result FIFO
// absorbs the 16-cycle compute burst so the recursive CRC state never
// backpressures the multiplier under the normal DMA load cadence.
// ==========================================
reg [31:0] crc32_table [0:255];
reg [31:0] crc_reg;
@@ -287,23 +290,38 @@ module matmul_axi_slave (
end
endfunction
// ---- DMA-calc 3-stage shift register pipeline -----------------------
// Stage 0: capture acc_result, compute word0 CRC.
// Stage 1: shift, compute word1 CRC.
// Final: crc_reg updated when stage 1 valid (word2 CRC).
// Latency 3 cycles; compute FSM inserts issue gaps so result spacing
// is ≥3 cycles, ensuring crc_reg is committed before the next capture.
reg crc_c66_s0_valid, crc_c66_s1_valid;
reg [31:0] crc_c66_s0_crc, crc_c66_s1_crc;
reg [31:0] crc_c66_s0_word1, crc_c66_s1_word2;
reg [31:0] crc_c66_s0_word2;
reg crc_c66_s0_last, crc_c66_s1_last;
function [31:0] crc32_update_6bytes;
input [31:0] crc_in;
input [47:0] data_in;
reg [31:0] c;
integer byte_idx;
begin
c = crc_in;
for (byte_idx = 0; byte_idx < 6; byte_idx = byte_idx + 1)
c = crc32_table[(c[7:0] ^ data_in[byte_idx*8 +: 8])] ^
{8'h00, c[31:8]};
crc32_update_6bytes = c;
end
endfunction
wire [31:0] crc_c66_s0_next = crc32_update_word(crc_reg, acc_result[31:0]);
wire [31:0] crc_c66_s1_next = crc32_update_word(crc_c66_s0_crc, crc_c66_s0_word1);
wire [31:0] crc_c66_final_next = crc32_update_word(crc_c66_s1_crc, crc_c66_s1_word2);
// ---- DMA result FIFO and two-cycle recursive CRC consumer -----------
localparam CRC_FIFO_DEPTH = 32;
reg [65:0] crc_fifo [0:CRC_FIFO_DEPTH-1];
reg [4:0] crc_fifo_wr_ptr;
reg [4:0] crc_fifo_rd_ptr;
reg [5:0] crc_fifo_count;
reg crc_c66_busy;
reg [65:0] crc_c66_data;
reg [31:0] crc_c66_half;
wire crc_c66_busy = crc_c66_s0_valid || crc_c66_s1_valid;
wire crc_fifo_empty = (crc_fifo_count == 6'd0);
wire crc_fifo_push = dma_busy && acc_valid_d4;
wire crc_fifo_pop = !crc_c66_busy && !crc_fifo_empty;
wire [31:0] crc_c66_first_next =
crc32_update_6bytes(crc_reg, crc_fifo[crc_fifo_rd_ptr][47:0]);
wire [47:0] crc_c66_second_data = {30'b0, crc_c66_data[65:48]};
wire [31:0] crc_c66_final_next =
crc32_update_6bytes(crc_c66_half, crc_c66_second_data);
initial begin
crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096;
@@ -672,7 +690,8 @@ module matmul_axi_slave (
else if (current_r_offset == ADDR_CRC)
s_axi_rdata = crc_reg ^ 32'hFFFFFFFF;
else if (current_r_offset == ADDR_STATUS)
s_axi_rdata = {26'b0, crc_c66_busy, dma_done, dma_busy, error, done, busy};
s_axi_rdata = {26'b0, (crc_c66_busy || !crc_fifo_empty),
dma_done, dma_busy, error, done, busy};
else if (current_r_offset == ADDR_DMA_SRC_BASE)
s_axi_rdata = dma_src_base;
else if (current_r_offset == ADDR_DMA_DST_BASE)
@@ -683,6 +702,8 @@ module matmul_axi_slave (
s_axi_rdata = dma_src_stride;
else if (current_r_offset == ADDR_DMA_DST_STRIDE)
s_axi_rdata = dma_dst_stride;
else if (current_r_offset == ADDR_DMA_CYCLES)
s_axi_rdata = dma_cycle_count;
else if (current_r_offset >= ADDR_B0_A && current_r_offset < ADDR_B0_B)
s_axi_rdata = b0_a[(current_r_offset - ADDR_B0_A) >> 2];
else if (current_r_offset >= ADDR_B0_B && current_r_offset < ADDR_B0_C)
@@ -731,66 +752,51 @@ module matmul_axi_slave (
reg [4:0] pipe_cnt;
wire [65:0] acc_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
wire crc_calc_valid = dma_busy && acc_valid_d4;
// DMA CRC issue spacing: compute FSM stalls via this counter to ensure
// ≥3 cycles between consecutive acc_result captures, matching the 3-stage
// CRC pipeline latency so crc_reg is committed before the next element.
reg [1:0] dma_crc_issue_gap;
wire dma_crc_issue_ready = !dma_busy || ((dma_crc_issue_gap == 2'd0) && !crc_c66_busy);
// Leave eight entries for results already in the multiplier pipeline.
// Under the normal DMA cadence the FIFO stays well below this threshold.
wire dma_crc_issue_ready = !dma_busy || (crc_fifo_count < 6'd24);
// CRC register update: DMA calc result (pipelined, 3-cycle latency)
// takes priority over AXI-stream (single-cycle). The stream path is
// only permitted when the DMA pipeline is idle (!crc_c66_busy).
// DMA CRC takes priority over the software-driven AXI read stream.
always @(posedge clk) begin
if (!resetn) begin
crc_reg <= 32'hFFFFFFFF;
crc_c66_s0_valid <= 1'b0;
crc_c66_s0_crc <= 32'b0;
crc_c66_s0_word1 <= 32'b0;
crc_c66_s0_word2 <= 32'b0;
crc_c66_s0_last <= 1'b0;
crc_c66_s1_valid <= 1'b0;
crc_c66_s1_crc <= 32'b0;
crc_c66_s1_word2 <= 32'b0;
crc_c66_s1_last <= 1'b0;
crc_reg <= 32'hFFFFFFFF;
crc_fifo_wr_ptr <= 5'd0;
crc_fifo_rd_ptr <= 5'd0;
crc_fifo_count <= 6'd0;
crc_c66_busy <= 1'b0;
crc_c66_data <= 66'd0;
crc_c66_half <= 32'd0;
end else if (crc_clear_pulse) begin
crc_reg <= 32'hFFFFFFFF;
crc_c66_s0_valid <= 1'b0;
crc_c66_s0_crc <= 32'b0;
crc_c66_s0_word1 <= 32'b0;
crc_c66_s0_word2 <= 32'b0;
crc_c66_s0_last <= 1'b0;
crc_c66_s1_valid <= 1'b0;
crc_c66_s1_crc <= 32'b0;
crc_c66_s1_word2 <= 32'b0;
crc_c66_s1_last <= 1'b0;
crc_reg <= 32'hFFFFFFFF;
crc_fifo_wr_ptr <= 5'd0;
crc_fifo_rd_ptr <= 5'd0;
crc_fifo_count <= 6'd0;
crc_c66_busy <= 1'b0;
crc_c66_data <= 66'd0;
crc_c66_half <= 32'd0;
end else begin
// Stage 1 → crc_reg finalize (word2 CRC)
if (crc_c66_s1_valid)
crc_reg <= crc_c66_final_next;
else if (crc_stream_valid && !crc_c66_busy)
crc_reg <= crc_next;
// Stage 0 → Stage 1 shift
crc_c66_s1_valid <= crc_c66_s0_valid;
if (crc_c66_s0_valid) begin
crc_c66_s1_crc <= crc_c66_s1_next;
crc_c66_s1_word2 <= crc_c66_s0_word2;
crc_c66_s1_last <= crc_c66_s0_last;
end else begin
crc_c66_s1_last <= 1'b0;
if (crc_fifo_push) begin
crc_fifo[crc_fifo_wr_ptr] <= acc_result;
crc_fifo_wr_ptr <= crc_fifo_wr_ptr + 5'd1;
end
// Capture → Stage 0
crc_c66_s0_valid <= crc_calc_valid;
if (crc_calc_valid) begin
crc_c66_s0_crc <= crc_c66_s0_next;
crc_c66_s0_word1 <= acc_result[63:32];
crc_c66_s0_word2 <= {30'b0, acc_result[65:64]};
crc_c66_s0_last <= (acc_idx_d4 == 4'd15);
end else begin
crc_c66_s0_last <= 1'b0;
case ({crc_fifo_push, crc_fifo_pop})
2'b10: crc_fifo_count <= crc_fifo_count + 6'd1;
2'b01: crc_fifo_count <= crc_fifo_count - 6'd1;
default: crc_fifo_count <= crc_fifo_count;
endcase
if (crc_c66_busy) begin
crc_reg <= crc_c66_final_next;
crc_c66_busy <= 1'b0;
end else if (crc_fifo_pop) begin
crc_c66_data <= crc_fifo[crc_fifo_rd_ptr];
crc_c66_half <= crc_c66_first_next;
crc_c66_busy <= 1'b1;
crc_fifo_rd_ptr <= crc_fifo_rd_ptr + 5'd1;
end else if (crc_stream_valid) begin
crc_reg <= crc_next;
end
end
end
@@ -802,7 +808,6 @@ module matmul_axi_slave (
busy <= 1'b1;
state <= C_RUN;
pipe_cnt <= 5'd1;
dma_crc_issue_gap <= dma_busy ? 2'd2 : 2'd0;
acc_idx_d1 <= 4'd0;
acc_bank_d1 <= bank;
acc_valid_d1 <= 1'b1;
@@ -827,7 +832,6 @@ module matmul_axi_slave (
cur_bank <= 1'b0;
calc_done_pulse <= 1'b0;
pipe_cnt <= 5'd0;
dma_crc_issue_gap <= 2'd0;
acc_idx_d1 <= 4'd0;
acc_idx_d2 <= 4'd0;
acc_idx_d3 <= 4'd0;
@@ -842,8 +846,6 @@ module matmul_axi_slave (
acc_valid_d4 <= 1'b0;
end else begin
calc_done_pulse <= 1'b0;
if (dma_crc_issue_gap != 2'd0)
dma_crc_issue_gap <= dma_crc_issue_gap - 2'd1;
// Pipeline shift (4 cycles total)
acc_idx_d4 <= acc_idx_d3;
@@ -872,9 +874,8 @@ module matmul_axi_slave (
end
C_RUN: begin
// Issue 4 new operands per round. DMA mode inserts gap
// so the 3-stage CRC pipeline can commit before the next
// C-element capture.
// Issue one C element per cycle unless the CRC FIFO is
// approaching capacity.
if (pipe_cnt < 5'd16) begin
if (dma_crc_issue_ready) begin
acc_idx_d1 <= pipe_cnt[3:0];
@@ -891,8 +892,6 @@ module matmul_axi_slave (
m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}];
end
if (dma_busy)
dma_crc_issue_gap <= 2'd2;
pipe_cnt <= pipe_cnt + 5'd1;
end
end
@@ -946,6 +945,8 @@ module matmul_axi_slave (
if (!resetn) begin
dma_busy <= 1'b0;
dma_done <= 1'b0;
dma_compute_complete <= 1'b0;
dma_cycle_count <= 32'd0;
load_group <= 32'd0;
store_group <= 32'd0;
dma_next_rd_addr <= 32'd0;
@@ -1001,6 +1002,8 @@ module matmul_axi_slave (
if (dma_start_pulse && !dma_busy && dma_groups != 32'd0) begin
dma_busy <= 1'b1;
dma_done <= 1'b0;
dma_compute_complete <= 1'b0;
dma_cycle_count <= 32'd0;
load_group <= 32'd0;
store_group <= 32'd0;
dma_next_rd_addr <= dma_src_base;
@@ -1015,6 +1018,8 @@ module matmul_axi_slave (
dma_store_bank <= 1'b0;
dma_rd_state <= DMA_RD_IDLE;
dma_wr_state <= DMA_WR_IDLE;
end else if (dma_busy) begin
dma_cycle_count <= dma_cycle_count + 32'd1;
end
if (dma_busy) begin
@@ -1039,10 +1044,14 @@ module matmul_axi_slave (
if (calc_done_pulse && dma_busy) begin
store_group <= store_group + 32'd1;
completed_groups <= completed_groups + 32'd1;
if ((completed_groups + 32'd1) >= dma_groups) begin
dma_busy <= 1'b0;
dma_done <= 1'b1;
end
if ((completed_groups + 32'd1) >= dma_groups)
dma_compute_complete <= 1'b1;
end
if (dma_compute_complete && crc_fifo_empty && !crc_c66_busy) begin
dma_busy <= 1'b0;
dma_done <= 1'b1;
dma_compute_complete <= 1'b0;
end
case (dma_rd_state)