fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize matmul CRC pipeline and expose DMA cycles

Browse Source
This commit is contained in:
FallenSigh
2026-07-14 18:21:45 +08:00
parent 7c60da78cb
commit f3af271788
2 changed files with 116 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
rtl/ip/matmul/matmul_axi_slave.v
View File

@@ -148,6 +148,7 @@ module matmul_axi_slave (
localparam ADDR_DMA_GROUPS = 12'h028; localparam ADDR_DMA_GROUPS = 12'h028;
localparam ADDR_DMA_SRC_STRIDE = 12'h02C; localparam ADDR_DMA_SRC_STRIDE = 12'h02C;
localparam ADDR_DMA_DST_STRIDE = 12'h030; localparam ADDR_DMA_DST_STRIDE = 12'h030;
localparam ADDR_DMA_CYCLES = 12'h034;
localparam ADDR_B0_A = 12'h100; localparam ADDR_B0_A = 12'h100;
localparam ADDR_B0_B = 12'h140; localparam ADDR_B0_B = 12'h140;
@@ -179,8 +180,11 @@ module matmul_axi_slave (
reg [31:0] dma_groups; reg [31:0] dma_groups;
reg [31:0] dma_src_stride; reg [31:0] dma_src_stride;
reg [31:0] dma_dst_stride; reg [31:0] dma_dst_stride;
// Full sys_clk cycles spent in dma_busy; held after DMA completion.
reg [31:0] dma_cycle_count;
reg dma_busy; reg dma_busy;
reg dma_done; reg dma_done;
reg dma_compute_complete;
reg [31:0] load_group; reg [31:0] load_group;
reg [31:0] store_group; reg [31:0] store_group;
reg [31:0] dma_next_rd_addr; reg [31:0] dma_next_rd_addr;
@@ -257,10 +261,9 @@ module matmul_axi_slave (
// //
// AXI-stream: single-cycle crc32_update_word (4 LUT lookup depth, ok for // AXI-stream: single-cycle crc32_update_word (4 LUT lookup depth, ok for
// AXI timing—the bus runs at or below internal compute rate). // AXI timing—the bus runs at or below internal compute rate).
// DMA-calc: 2-stage pipelined word engine (2 bytes/stage) + 6-cycle // DMA-calc: two 6-byte CRC steps per 66-bit C-element. A result FIFO
// serial FSM for 66-bit C-elements, with 16-deep FIFO to absorb the // absorbs the 16-cycle compute burst so the recursive CRC state never
// compute pipeline burst. Critical path: 2 table lookups + 1 XOR per // backpressures the multiplier under the normal DMA load cadence.
// stage vs 12 table lookups in the original single-cycle design.
// ========================================== // ==========================================
reg [31:0] crc32_table [0:255]; reg [31:0] crc32_table [0:255];
reg [31:0] crc_reg; reg [31:0] crc_reg;
@@ -287,23 +290,38 @@ module matmul_axi_slave (
end end
endfunction endfunction
// ---- DMA-calc 3-stage shift register pipeline ----------------------- function [31:0] crc32_update_6bytes;
// Stage 0: capture acc_result, compute word0 CRC. input [31:0] crc_in;
// Stage 1: shift, compute word1 CRC. input [47:0] data_in;
// Final: crc_reg updated when stage 1 valid (word2 CRC). reg [31:0] c;
// Latency 3 cycles; compute FSM inserts issue gaps so result spacing integer byte_idx;
// is ≥3 cycles, ensuring crc_reg is committed before the next capture. begin
reg crc_c66_s0_valid, crc_c66_s1_valid; c = crc_in;
reg [31:0] crc_c66_s0_crc, crc_c66_s1_crc; for (byte_idx = 0; byte_idx < 6; byte_idx = byte_idx + 1)
reg [31:0] crc_c66_s0_word1, crc_c66_s1_word2; c = crc32_table[(c[7:0] ^ data_in[byte_idx*8 +: 8])] ^
reg [31:0] crc_c66_s0_word2; {8'h00, c[31:8]};
reg crc_c66_s0_last, crc_c66_s1_last; crc32_update_6bytes = c;
end
endfunction
wire [31:0] crc_c66_s0_next = crc32_update_word(crc_reg, acc_result[31:0]); // ---- DMA result FIFO and two-cycle recursive CRC consumer -----------
wire [31:0] crc_c66_s1_next = crc32_update_word(crc_c66_s0_crc, crc_c66_s0_word1); localparam CRC_FIFO_DEPTH = 32;
wire [31:0] crc_c66_final_next = crc32_update_word(crc_c66_s1_crc, crc_c66_s1_word2); reg [65:0] crc_fifo [0:CRC_FIFO_DEPTH-1];
reg [4:0] crc_fifo_wr_ptr;
reg [4:0] crc_fifo_rd_ptr;
reg [5:0] crc_fifo_count;
reg crc_c66_busy;
reg [65:0] crc_c66_data;
reg [31:0] crc_c66_half;
wire crc_c66_busy = crc_c66_s0_valid || crc_c66_s1_valid; wire crc_fifo_empty = (crc_fifo_count == 6'd0);
wire crc_fifo_push = dma_busy && acc_valid_d4;
wire crc_fifo_pop = !crc_c66_busy && !crc_fifo_empty;
wire [31:0] crc_c66_first_next =
crc32_update_6bytes(crc_reg, crc_fifo[crc_fifo_rd_ptr][47:0]);
wire [47:0] crc_c66_second_data = {30'b0, crc_c66_data[65:48]};
wire [31:0] crc_c66_final_next =
crc32_update_6bytes(crc_c66_half, crc_c66_second_data);
initial begin initial begin
crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096; crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096;
@@ -672,7 +690,8 @@ module matmul_axi_slave (
else if (current_r_offset == ADDR_CRC) else if (current_r_offset == ADDR_CRC)
s_axi_rdata = crc_reg ^ 32'hFFFFFFFF; s_axi_rdata = crc_reg ^ 32'hFFFFFFFF;
else if (current_r_offset == ADDR_STATUS) else if (current_r_offset == ADDR_STATUS)
s_axi_rdata = {26'b0, crc_c66_busy, dma_done, dma_busy, error, done, busy}; s_axi_rdata = {26'b0, (crc_c66_busy || !crc_fifo_empty),
dma_done, dma_busy, error, done, busy};
else if (current_r_offset == ADDR_DMA_SRC_BASE) else if (current_r_offset == ADDR_DMA_SRC_BASE)
s_axi_rdata = dma_src_base; s_axi_rdata = dma_src_base;
else if (current_r_offset == ADDR_DMA_DST_BASE) else if (current_r_offset == ADDR_DMA_DST_BASE)
@@ -683,6 +702,8 @@ module matmul_axi_slave (
s_axi_rdata = dma_src_stride; s_axi_rdata = dma_src_stride;
else if (current_r_offset == ADDR_DMA_DST_STRIDE) else if (current_r_offset == ADDR_DMA_DST_STRIDE)
s_axi_rdata = dma_dst_stride; s_axi_rdata = dma_dst_stride;
else if (current_r_offset == ADDR_DMA_CYCLES)
s_axi_rdata = dma_cycle_count;
else if (current_r_offset >= ADDR_B0_A && current_r_offset < ADDR_B0_B) else if (current_r_offset >= ADDR_B0_A && current_r_offset < ADDR_B0_B)
s_axi_rdata = b0_a[(current_r_offset - ADDR_B0_A) >> 2]; s_axi_rdata = b0_a[(current_r_offset - ADDR_B0_A) >> 2];
else if (current_r_offset >= ADDR_B0_B && current_r_offset < ADDR_B0_C) else if (current_r_offset >= ADDR_B0_B && current_r_offset < ADDR_B0_C)
@@ -731,66 +752,51 @@ module matmul_axi_slave (
reg [4:0] pipe_cnt; reg [4:0] pipe_cnt;
wire [65:0] acc_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) + wire [65:0] acc_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]}); ({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
wire crc_calc_valid = dma_busy && acc_valid_d4;
// DMA CRC issue spacing: compute FSM stalls via this counter to ensure // Leave eight entries for results already in the multiplier pipeline.
// ≥3 cycles between consecutive acc_result captures, matching the 3-stage // Under the normal DMA cadence the FIFO stays well below this threshold.
// CRC pipeline latency so crc_reg is committed before the next element. wire dma_crc_issue_ready = !dma_busy || (crc_fifo_count < 6'd24);
reg [1:0] dma_crc_issue_gap;
wire dma_crc_issue_ready = !dma_busy || ((dma_crc_issue_gap == 2'd0) && !crc_c66_busy);
// CRC register update: DMA calc result (pipelined, 3-cycle latency) // DMA CRC takes priority over the software-driven AXI read stream.
// takes priority over AXI-stream (single-cycle). The stream path is
// only permitted when the DMA pipeline is idle (!crc_c66_busy).
always @(posedge clk) begin always @(posedge clk) begin
if (!resetn) begin if (!resetn) begin
crc_reg <= 32'hFFFFFFFF; crc_reg <= 32'hFFFFFFFF;
crc_c66_s0_valid <= 1'b0; crc_fifo_wr_ptr <= 5'd0;
crc_c66_s0_crc <= 32'b0; crc_fifo_rd_ptr <= 5'd0;
crc_c66_s0_word1 <= 32'b0; crc_fifo_count <= 6'd0;
crc_c66_s0_word2 <= 32'b0; crc_c66_busy <= 1'b0;
crc_c66_s0_last <= 1'b0; crc_c66_data <= 66'd0;
crc_c66_s1_valid <= 1'b0; crc_c66_half <= 32'd0;
crc_c66_s1_crc <= 32'b0;
crc_c66_s1_word2 <= 32'b0;
crc_c66_s1_last <= 1'b0;
end else if (crc_clear_pulse) begin end else if (crc_clear_pulse) begin
crc_reg <= 32'hFFFFFFFF; crc_reg <= 32'hFFFFFFFF;
crc_c66_s0_valid <= 1'b0; crc_fifo_wr_ptr <= 5'd0;
crc_c66_s0_crc <= 32'b0; crc_fifo_rd_ptr <= 5'd0;
crc_c66_s0_word1 <= 32'b0; crc_fifo_count <= 6'd0;
crc_c66_s0_word2 <= 32'b0; crc_c66_busy <= 1'b0;
crc_c66_s0_last <= 1'b0; crc_c66_data <= 66'd0;
crc_c66_s1_valid <= 1'b0; crc_c66_half <= 32'd0;
crc_c66_s1_crc <= 32'b0;
crc_c66_s1_word2 <= 32'b0;
crc_c66_s1_last <= 1'b0;
end else begin end else begin
// Stage 1 → crc_reg finalize (word2 CRC) if (crc_fifo_push) begin
if (crc_c66_s1_valid) crc_fifo[crc_fifo_wr_ptr] <= acc_result;
crc_reg <= crc_c66_final_next; crc_fifo_wr_ptr <= crc_fifo_wr_ptr + 5'd1;
else if (crc_stream_valid && !crc_c66_busy)
crc_reg <= crc_next;
// Stage 0 → Stage 1 shift
crc_c66_s1_valid <= crc_c66_s0_valid;
if (crc_c66_s0_valid) begin
crc_c66_s1_crc <= crc_c66_s1_next;
crc_c66_s1_word2 <= crc_c66_s0_word2;
crc_c66_s1_last <= crc_c66_s0_last;
end else begin
crc_c66_s1_last <= 1'b0;
end end
// Capture → Stage 0 case ({crc_fifo_push, crc_fifo_pop})
crc_c66_s0_valid <= crc_calc_valid; 2'b10: crc_fifo_count <= crc_fifo_count + 6'd1;
if (crc_calc_valid) begin 2'b01: crc_fifo_count <= crc_fifo_count - 6'd1;
crc_c66_s0_crc <= crc_c66_s0_next; default: crc_fifo_count <= crc_fifo_count;
crc_c66_s0_word1 <= acc_result[63:32]; endcase
crc_c66_s0_word2 <= {30'b0, acc_result[65:64]};
crc_c66_s0_last <= (acc_idx_d4 == 4'd15); if (crc_c66_busy) begin
end else begin crc_reg <= crc_c66_final_next;
crc_c66_s0_last <= 1'b0; crc_c66_busy <= 1'b0;
end else if (crc_fifo_pop) begin
crc_c66_data <= crc_fifo[crc_fifo_rd_ptr];
crc_c66_half <= crc_c66_first_next;
crc_c66_busy <= 1'b1;
crc_fifo_rd_ptr <= crc_fifo_rd_ptr + 5'd1;
end else if (crc_stream_valid) begin
crc_reg <= crc_next;
end end
end end
end end
@@ -802,7 +808,6 @@ module matmul_axi_slave (
busy <= 1'b1; busy <= 1'b1;
state <= C_RUN; state <= C_RUN;
pipe_cnt <= 5'd1; pipe_cnt <= 5'd1;
dma_crc_issue_gap <= dma_busy ? 2'd2 : 2'd0;
acc_idx_d1 <= 4'd0; acc_idx_d1 <= 4'd0;
acc_bank_d1 <= bank; acc_bank_d1 <= bank;
acc_valid_d1 <= 1'b1; acc_valid_d1 <= 1'b1;
@@ -827,7 +832,6 @@ module matmul_axi_slave (
cur_bank <= 1'b0; cur_bank <= 1'b0;
calc_done_pulse <= 1'b0; calc_done_pulse <= 1'b0;
pipe_cnt <= 5'd0; pipe_cnt <= 5'd0;
dma_crc_issue_gap <= 2'd0;
acc_idx_d1 <= 4'd0; acc_idx_d1 <= 4'd0;
acc_idx_d2 <= 4'd0; acc_idx_d2 <= 4'd0;
acc_idx_d3 <= 4'd0; acc_idx_d3 <= 4'd0;
@@ -842,8 +846,6 @@ module matmul_axi_slave (
acc_valid_d4 <= 1'b0; acc_valid_d4 <= 1'b0;
end else begin end else begin
calc_done_pulse <= 1'b0; calc_done_pulse <= 1'b0;
if (dma_crc_issue_gap != 2'd0)
dma_crc_issue_gap <= dma_crc_issue_gap - 2'd1;
// Pipeline shift (4 cycles total) // Pipeline shift (4 cycles total)
acc_idx_d4 <= acc_idx_d3; acc_idx_d4 <= acc_idx_d3;
@@ -872,9 +874,8 @@ module matmul_axi_slave (
end end
C_RUN: begin C_RUN: begin
// Issue 4 new operands per round. DMA mode inserts gap // Issue one C element per cycle unless the CRC FIFO is
// so the 3-stage CRC pipeline can commit before the next // approaching capacity.
// C-element capture.
if (pipe_cnt < 5'd16) begin if (pipe_cnt < 5'd16) begin
if (dma_crc_issue_ready) begin if (dma_crc_issue_ready) begin
acc_idx_d1 <= pipe_cnt[3:0]; acc_idx_d1 <= pipe_cnt[3:0];
@@ -891,8 +892,6 @@ module matmul_axi_slave (
m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}]; m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}]; m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}];
end end
if (dma_busy)
dma_crc_issue_gap <= 2'd2;
pipe_cnt <= pipe_cnt + 5'd1; pipe_cnt <= pipe_cnt + 5'd1;
end end
end end
@@ -946,6 +945,8 @@ module matmul_axi_slave (
if (!resetn) begin if (!resetn) begin
dma_busy <= 1'b0; dma_busy <= 1'b0;
dma_done <= 1'b0; dma_done <= 1'b0;
dma_compute_complete <= 1'b0;
dma_cycle_count <= 32'd0;
load_group <= 32'd0; load_group <= 32'd0;
store_group <= 32'd0; store_group <= 32'd0;
dma_next_rd_addr <= 32'd0; dma_next_rd_addr <= 32'd0;
@@ -1001,6 +1002,8 @@ module matmul_axi_slave (
if (dma_start_pulse && !dma_busy && dma_groups != 32'd0) begin if (dma_start_pulse && !dma_busy && dma_groups != 32'd0) begin
dma_busy <= 1'b1; dma_busy <= 1'b1;
dma_done <= 1'b0; dma_done <= 1'b0;
dma_compute_complete <= 1'b0;
dma_cycle_count <= 32'd0;
load_group <= 32'd0; load_group <= 32'd0;
store_group <= 32'd0; store_group <= 32'd0;
dma_next_rd_addr <= dma_src_base; dma_next_rd_addr <= dma_src_base;
@@ -1015,6 +1018,8 @@ module matmul_axi_slave (
dma_store_bank <= 1'b0; dma_store_bank <= 1'b0;
dma_rd_state <= DMA_RD_IDLE; dma_rd_state <= DMA_RD_IDLE;
dma_wr_state <= DMA_WR_IDLE; dma_wr_state <= DMA_WR_IDLE;
end else if (dma_busy) begin
dma_cycle_count <= dma_cycle_count + 32'd1;
end end
if (dma_busy) begin if (dma_busy) begin
@@ -1039,10 +1044,14 @@ module matmul_axi_slave (
if (calc_done_pulse && dma_busy) begin if (calc_done_pulse && dma_busy) begin
store_group <= store_group + 32'd1; store_group <= store_group + 32'd1;
completed_groups <= completed_groups + 32'd1; completed_groups <= completed_groups + 32'd1;
if ((completed_groups + 32'd1) >= dma_groups) begin if ((completed_groups + 32'd1) >= dma_groups)
dma_compute_complete <= 1'b1;
end
if (dma_compute_complete && crc_fifo_empty && !crc_c66_busy) begin
dma_busy <= 1'b0; dma_busy <= 1'b0;
dma_done <= 1'b1; dma_done <= 1'b1;
end dma_compute_complete <= 1'b0;
end end
case (dma_rd_state) case (dma_rd_state)

37
sdk/software/examples/asm/user-sample.c
View File

@@ -33,6 +33,23 @@ static void uart_put_matmul_start_fifo(void)
UART_THR = 'T'; UART_THR = 'T';
} }
static void uart_put_crc_prefix_fifo(void)
{
UART_THR = 'M';
UART_THR = 'A';
UART_THR = 'T';
UART_THR = 'M';
UART_THR = 'U';
UART_THR = 'L';
UART_THR = '_';
UART_THR = 'C';
UART_THR = 'R';
UART_THR = 'C';
UART_THR = '3';
UART_THR = '2';
UART_THR = '=';
}
static inline void uart_wait_tx_empty(void) static inline void uart_wait_tx_empty(void)
{ {
while ((UART_LSR & LSR_THRE) == 0) { while ((UART_LSR & LSR_THRE) == 0) {
@@ -49,24 +66,9 @@ static inline unsigned char hex_digit(unsigned int val, int shift)
static void uart_put_result_fifo(unsigned int crc) static void uart_put_result_fifo(unsigned int crc)
{ {
uart_wait_tx_empty(); uart_wait_tx_empty();
UART_THR = 'M';
UART_THR = 'A';
UART_THR = 'T';
UART_THR = 'M';
UART_THR = 'U';
UART_THR = 'L';
UART_THR = '_';
UART_THR = 'C';
UART_THR = 'R';
UART_THR = 'C';
UART_THR = '3';
UART_THR = '2';
UART_THR = '=';
UART_THR = hex_digit(crc, 28); UART_THR = hex_digit(crc, 28);
UART_THR = hex_digit(crc, 24); UART_THR = hex_digit(crc, 24);
UART_THR = hex_digit(crc, 20); UART_THR = hex_digit(crc, 20);
uart_wait_tx_empty();
UART_THR = hex_digit(crc, 16); UART_THR = hex_digit(crc, 16);
UART_THR = hex_digit(crc, 12); UART_THR = hex_digit(crc, 12);
UART_THR = hex_digit(crc, 8); UART_THR = hex_digit(crc, 8);
@@ -107,6 +109,7 @@ unsigned long UART_BASE = 0xbf000000;
#define MATMUL_DMA_SRC_BASE (MATMUL_BASE_UNCACHED + 0x020) #define MATMUL_DMA_SRC_BASE (MATMUL_BASE_UNCACHED + 0x020)
#define MATMUL_DMA_GROUPS (MATMUL_BASE_UNCACHED + 0x028) #define MATMUL_DMA_GROUPS (MATMUL_BASE_UNCACHED + 0x028)
#define MATMUL_DMA_SRC_STRIDE (MATMUL_BASE_UNCACHED + 0x02C) #define MATMUL_DMA_SRC_STRIDE (MATMUL_BASE_UNCACHED + 0x02C)
#define MATMUL_DMA_CYCLES (MATMUL_BASE_UNCACHED + 0x034)
#define CTRL_START_B0 0x1 #define CTRL_START_B0 0x1
#define CTRL_START_B1 0x2 #define CTRL_START_B1 0x2
@@ -121,6 +124,8 @@ unsigned long UART_BASE = 0xbf000000;
#define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT) #define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT)
#define MATMUL_EP_ID 0 #define MATMUL_EP_ID 0
volatile unsigned int matmul_dma_cycles;
int main(void) int main(void)
{ {
if (MATMUL_GROUP_NUM <= 0) { if (MATMUL_GROUP_NUM <= 0) {
@@ -141,7 +146,9 @@ int main(void)
*(volatile unsigned int *)MATMUL_DMA_GROUPS = MATMUL_GROUP_NUM; *(volatile unsigned int *)MATMUL_DMA_GROUPS = MATMUL_GROUP_NUM;
*(volatile unsigned int *)MATMUL_DMA_SRC_STRIDE = 128; *(volatile unsigned int *)MATMUL_DMA_SRC_STRIDE = 128;
*mat_ctrl = CTRL_DMA_START; *mat_ctrl = CTRL_DMA_START;
uart_put_crc_prefix_fifo();
while (!(*mat_status & STATUS_DMA_DONE)) {} while (!(*mat_status & STATUS_DMA_DONE)) {}
matmul_dma_cycles = *(volatile unsigned int *)MATMUL_DMA_CYCLES;
*mat_status = STATUS_DMA_DONE; *mat_status = STATUS_DMA_DONE;
crc = *mat_crc; crc = *mat_crc;