From f3af27178879441a84e66cadd1b906ad7a1fba46 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Tue, 14 Jul 2026 18:21:45 +0800 Subject: [PATCH] Optimize matmul CRC pipeline and expose DMA cycles --- rtl/ip/matmul/matmul_axi_slave.v | 179 +++++++++++++----------- sdk/software/examples/asm/user-sample.c | 37 +++-- 2 files changed, 116 insertions(+), 100 deletions(-) diff --git a/rtl/ip/matmul/matmul_axi_slave.v b/rtl/ip/matmul/matmul_axi_slave.v index 7c400a6..d291f2e 100644 --- a/rtl/ip/matmul/matmul_axi_slave.v +++ b/rtl/ip/matmul/matmul_axi_slave.v @@ -148,6 +148,7 @@ module matmul_axi_slave ( localparam ADDR_DMA_GROUPS = 12'h028; localparam ADDR_DMA_SRC_STRIDE = 12'h02C; localparam ADDR_DMA_DST_STRIDE = 12'h030; + localparam ADDR_DMA_CYCLES = 12'h034; localparam ADDR_B0_A = 12'h100; localparam ADDR_B0_B = 12'h140; @@ -179,8 +180,11 @@ module matmul_axi_slave ( reg [31:0] dma_groups; reg [31:0] dma_src_stride; reg [31:0] dma_dst_stride; + // Full sys_clk cycles spent in dma_busy; held after DMA completion. + reg [31:0] dma_cycle_count; reg dma_busy; reg dma_done; + reg dma_compute_complete; reg [31:0] load_group; reg [31:0] store_group; reg [31:0] dma_next_rd_addr; @@ -257,10 +261,9 @@ module matmul_axi_slave ( // // AXI-stream: single-cycle crc32_update_word (4 LUT lookup depth, ok for // AXI timing—the bus runs at or below internal compute rate). - // DMA-calc: 2-stage pipelined word engine (2 bytes/stage) + 6-cycle - // serial FSM for 66-bit C-elements, with 16-deep FIFO to absorb the - // compute pipeline burst. Critical path: 2 table lookups + 1 XOR per - // stage vs 12 table lookups in the original single-cycle design. + // DMA-calc: two 6-byte CRC steps per 66-bit C-element. A result FIFO + // absorbs the 16-cycle compute burst so the recursive CRC state never + // backpressures the multiplier under the normal DMA load cadence. // ========================================== reg [31:0] crc32_table [0:255]; reg [31:0] crc_reg; @@ -287,23 +290,38 @@ module matmul_axi_slave ( end endfunction - // ---- DMA-calc 3-stage shift register pipeline ----------------------- - // Stage 0: capture acc_result, compute word0 CRC. - // Stage 1: shift, compute word1 CRC. - // Final: crc_reg updated when stage 1 valid (word2 CRC). - // Latency 3 cycles; compute FSM inserts issue gaps so result spacing - // is ≥3 cycles, ensuring crc_reg is committed before the next capture. - reg crc_c66_s0_valid, crc_c66_s1_valid; - reg [31:0] crc_c66_s0_crc, crc_c66_s1_crc; - reg [31:0] crc_c66_s0_word1, crc_c66_s1_word2; - reg [31:0] crc_c66_s0_word2; - reg crc_c66_s0_last, crc_c66_s1_last; + function [31:0] crc32_update_6bytes; + input [31:0] crc_in; + input [47:0] data_in; + reg [31:0] c; + integer byte_idx; + begin + c = crc_in; + for (byte_idx = 0; byte_idx < 6; byte_idx = byte_idx + 1) + c = crc32_table[(c[7:0] ^ data_in[byte_idx*8 +: 8])] ^ + {8'h00, c[31:8]}; + crc32_update_6bytes = c; + end + endfunction - wire [31:0] crc_c66_s0_next = crc32_update_word(crc_reg, acc_result[31:0]); - wire [31:0] crc_c66_s1_next = crc32_update_word(crc_c66_s0_crc, crc_c66_s0_word1); - wire [31:0] crc_c66_final_next = crc32_update_word(crc_c66_s1_crc, crc_c66_s1_word2); + // ---- DMA result FIFO and two-cycle recursive CRC consumer ----------- + localparam CRC_FIFO_DEPTH = 32; + reg [65:0] crc_fifo [0:CRC_FIFO_DEPTH-1]; + reg [4:0] crc_fifo_wr_ptr; + reg [4:0] crc_fifo_rd_ptr; + reg [5:0] crc_fifo_count; + reg crc_c66_busy; + reg [65:0] crc_c66_data; + reg [31:0] crc_c66_half; - wire crc_c66_busy = crc_c66_s0_valid || crc_c66_s1_valid; + wire crc_fifo_empty = (crc_fifo_count == 6'd0); + wire crc_fifo_push = dma_busy && acc_valid_d4; + wire crc_fifo_pop = !crc_c66_busy && !crc_fifo_empty; + wire [31:0] crc_c66_first_next = + crc32_update_6bytes(crc_reg, crc_fifo[crc_fifo_rd_ptr][47:0]); + wire [47:0] crc_c66_second_data = {30'b0, crc_c66_data[65:48]}; + wire [31:0] crc_c66_final_next = + crc32_update_6bytes(crc_c66_half, crc_c66_second_data); initial begin crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096; @@ -672,7 +690,8 @@ module matmul_axi_slave ( else if (current_r_offset == ADDR_CRC) s_axi_rdata = crc_reg ^ 32'hFFFFFFFF; else if (current_r_offset == ADDR_STATUS) - s_axi_rdata = {26'b0, crc_c66_busy, dma_done, dma_busy, error, done, busy}; + s_axi_rdata = {26'b0, (crc_c66_busy || !crc_fifo_empty), + dma_done, dma_busy, error, done, busy}; else if (current_r_offset == ADDR_DMA_SRC_BASE) s_axi_rdata = dma_src_base; else if (current_r_offset == ADDR_DMA_DST_BASE) @@ -683,6 +702,8 @@ module matmul_axi_slave ( s_axi_rdata = dma_src_stride; else if (current_r_offset == ADDR_DMA_DST_STRIDE) s_axi_rdata = dma_dst_stride; + else if (current_r_offset == ADDR_DMA_CYCLES) + s_axi_rdata = dma_cycle_count; else if (current_r_offset >= ADDR_B0_A && current_r_offset < ADDR_B0_B) s_axi_rdata = b0_a[(current_r_offset - ADDR_B0_A) >> 2]; else if (current_r_offset >= ADDR_B0_B && current_r_offset < ADDR_B0_C) @@ -731,66 +752,51 @@ module matmul_axi_slave ( reg [4:0] pipe_cnt; wire [65:0] acc_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) + ({2'b0, mult_p[2]} + {2'b0, mult_p[3]}); - wire crc_calc_valid = dma_busy && acc_valid_d4; - // DMA CRC issue spacing: compute FSM stalls via this counter to ensure - // ≥3 cycles between consecutive acc_result captures, matching the 3-stage - // CRC pipeline latency so crc_reg is committed before the next element. - reg [1:0] dma_crc_issue_gap; - wire dma_crc_issue_ready = !dma_busy || ((dma_crc_issue_gap == 2'd0) && !crc_c66_busy); + // Leave eight entries for results already in the multiplier pipeline. + // Under the normal DMA cadence the FIFO stays well below this threshold. + wire dma_crc_issue_ready = !dma_busy || (crc_fifo_count < 6'd24); - // CRC register update: DMA calc result (pipelined, 3-cycle latency) - // takes priority over AXI-stream (single-cycle). The stream path is - // only permitted when the DMA pipeline is idle (!crc_c66_busy). + // DMA CRC takes priority over the software-driven AXI read stream. always @(posedge clk) begin if (!resetn) begin - crc_reg <= 32'hFFFFFFFF; - crc_c66_s0_valid <= 1'b0; - crc_c66_s0_crc <= 32'b0; - crc_c66_s0_word1 <= 32'b0; - crc_c66_s0_word2 <= 32'b0; - crc_c66_s0_last <= 1'b0; - crc_c66_s1_valid <= 1'b0; - crc_c66_s1_crc <= 32'b0; - crc_c66_s1_word2 <= 32'b0; - crc_c66_s1_last <= 1'b0; + crc_reg <= 32'hFFFFFFFF; + crc_fifo_wr_ptr <= 5'd0; + crc_fifo_rd_ptr <= 5'd0; + crc_fifo_count <= 6'd0; + crc_c66_busy <= 1'b0; + crc_c66_data <= 66'd0; + crc_c66_half <= 32'd0; end else if (crc_clear_pulse) begin - crc_reg <= 32'hFFFFFFFF; - crc_c66_s0_valid <= 1'b0; - crc_c66_s0_crc <= 32'b0; - crc_c66_s0_word1 <= 32'b0; - crc_c66_s0_word2 <= 32'b0; - crc_c66_s0_last <= 1'b0; - crc_c66_s1_valid <= 1'b0; - crc_c66_s1_crc <= 32'b0; - crc_c66_s1_word2 <= 32'b0; - crc_c66_s1_last <= 1'b0; + crc_reg <= 32'hFFFFFFFF; + crc_fifo_wr_ptr <= 5'd0; + crc_fifo_rd_ptr <= 5'd0; + crc_fifo_count <= 6'd0; + crc_c66_busy <= 1'b0; + crc_c66_data <= 66'd0; + crc_c66_half <= 32'd0; end else begin - // Stage 1 → crc_reg finalize (word2 CRC) - if (crc_c66_s1_valid) - crc_reg <= crc_c66_final_next; - else if (crc_stream_valid && !crc_c66_busy) - crc_reg <= crc_next; - - // Stage 0 → Stage 1 shift - crc_c66_s1_valid <= crc_c66_s0_valid; - if (crc_c66_s0_valid) begin - crc_c66_s1_crc <= crc_c66_s1_next; - crc_c66_s1_word2 <= crc_c66_s0_word2; - crc_c66_s1_last <= crc_c66_s0_last; - end else begin - crc_c66_s1_last <= 1'b0; + if (crc_fifo_push) begin + crc_fifo[crc_fifo_wr_ptr] <= acc_result; + crc_fifo_wr_ptr <= crc_fifo_wr_ptr + 5'd1; end - // Capture → Stage 0 - crc_c66_s0_valid <= crc_calc_valid; - if (crc_calc_valid) begin - crc_c66_s0_crc <= crc_c66_s0_next; - crc_c66_s0_word1 <= acc_result[63:32]; - crc_c66_s0_word2 <= {30'b0, acc_result[65:64]}; - crc_c66_s0_last <= (acc_idx_d4 == 4'd15); - end else begin - crc_c66_s0_last <= 1'b0; + case ({crc_fifo_push, crc_fifo_pop}) + 2'b10: crc_fifo_count <= crc_fifo_count + 6'd1; + 2'b01: crc_fifo_count <= crc_fifo_count - 6'd1; + default: crc_fifo_count <= crc_fifo_count; + endcase + + if (crc_c66_busy) begin + crc_reg <= crc_c66_final_next; + crc_c66_busy <= 1'b0; + end else if (crc_fifo_pop) begin + crc_c66_data <= crc_fifo[crc_fifo_rd_ptr]; + crc_c66_half <= crc_c66_first_next; + crc_c66_busy <= 1'b1; + crc_fifo_rd_ptr <= crc_fifo_rd_ptr + 5'd1; + end else if (crc_stream_valid) begin + crc_reg <= crc_next; end end end @@ -802,7 +808,6 @@ module matmul_axi_slave ( busy <= 1'b1; state <= C_RUN; pipe_cnt <= 5'd1; - dma_crc_issue_gap <= dma_busy ? 2'd2 : 2'd0; acc_idx_d1 <= 4'd0; acc_bank_d1 <= bank; acc_valid_d1 <= 1'b1; @@ -827,7 +832,6 @@ module matmul_axi_slave ( cur_bank <= 1'b0; calc_done_pulse <= 1'b0; pipe_cnt <= 5'd0; - dma_crc_issue_gap <= 2'd0; acc_idx_d1 <= 4'd0; acc_idx_d2 <= 4'd0; acc_idx_d3 <= 4'd0; @@ -842,8 +846,6 @@ module matmul_axi_slave ( acc_valid_d4 <= 1'b0; end else begin calc_done_pulse <= 1'b0; - if (dma_crc_issue_gap != 2'd0) - dma_crc_issue_gap <= dma_crc_issue_gap - 2'd1; // Pipeline shift (4 cycles total) acc_idx_d4 <= acc_idx_d3; @@ -872,9 +874,8 @@ module matmul_axi_slave ( end C_RUN: begin - // Issue 4 new operands per round. DMA mode inserts gap - // so the 3-stage CRC pipeline can commit before the next - // C-element capture. + // Issue one C element per cycle unless the CRC FIFO is + // approaching capacity. if (pipe_cnt < 5'd16) begin if (dma_crc_issue_ready) begin acc_idx_d1 <= pipe_cnt[3:0]; @@ -891,8 +892,6 @@ module matmul_axi_slave ( m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}]; m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}]; end - if (dma_busy) - dma_crc_issue_gap <= 2'd2; pipe_cnt <= pipe_cnt + 5'd1; end end @@ -946,6 +945,8 @@ module matmul_axi_slave ( if (!resetn) begin dma_busy <= 1'b0; dma_done <= 1'b0; + dma_compute_complete <= 1'b0; + dma_cycle_count <= 32'd0; load_group <= 32'd0; store_group <= 32'd0; dma_next_rd_addr <= 32'd0; @@ -1001,6 +1002,8 @@ module matmul_axi_slave ( if (dma_start_pulse && !dma_busy && dma_groups != 32'd0) begin dma_busy <= 1'b1; dma_done <= 1'b0; + dma_compute_complete <= 1'b0; + dma_cycle_count <= 32'd0; load_group <= 32'd0; store_group <= 32'd0; dma_next_rd_addr <= dma_src_base; @@ -1015,6 +1018,8 @@ module matmul_axi_slave ( dma_store_bank <= 1'b0; dma_rd_state <= DMA_RD_IDLE; dma_wr_state <= DMA_WR_IDLE; + end else if (dma_busy) begin + dma_cycle_count <= dma_cycle_count + 32'd1; end if (dma_busy) begin @@ -1039,10 +1044,14 @@ module matmul_axi_slave ( if (calc_done_pulse && dma_busy) begin store_group <= store_group + 32'd1; completed_groups <= completed_groups + 32'd1; - if ((completed_groups + 32'd1) >= dma_groups) begin - dma_busy <= 1'b0; - dma_done <= 1'b1; - end + if ((completed_groups + 32'd1) >= dma_groups) + dma_compute_complete <= 1'b1; + end + + if (dma_compute_complete && crc_fifo_empty && !crc_c66_busy) begin + dma_busy <= 1'b0; + dma_done <= 1'b1; + dma_compute_complete <= 1'b0; end case (dma_rd_state) diff --git a/sdk/software/examples/asm/user-sample.c b/sdk/software/examples/asm/user-sample.c index bfe91ab..1307f90 100644 --- a/sdk/software/examples/asm/user-sample.c +++ b/sdk/software/examples/asm/user-sample.c @@ -33,6 +33,23 @@ static void uart_put_matmul_start_fifo(void) UART_THR = 'T'; } +static void uart_put_crc_prefix_fifo(void) +{ + UART_THR = 'M'; + UART_THR = 'A'; + UART_THR = 'T'; + UART_THR = 'M'; + UART_THR = 'U'; + UART_THR = 'L'; + UART_THR = '_'; + UART_THR = 'C'; + UART_THR = 'R'; + UART_THR = 'C'; + UART_THR = '3'; + UART_THR = '2'; + UART_THR = '='; +} + static inline void uart_wait_tx_empty(void) { while ((UART_LSR & LSR_THRE) == 0) { @@ -49,24 +66,9 @@ static inline unsigned char hex_digit(unsigned int val, int shift) static void uart_put_result_fifo(unsigned int crc) { uart_wait_tx_empty(); - UART_THR = 'M'; - UART_THR = 'A'; - UART_THR = 'T'; - UART_THR = 'M'; - UART_THR = 'U'; - UART_THR = 'L'; - UART_THR = '_'; - UART_THR = 'C'; - UART_THR = 'R'; - UART_THR = 'C'; - UART_THR = '3'; - UART_THR = '2'; - UART_THR = '='; UART_THR = hex_digit(crc, 28); UART_THR = hex_digit(crc, 24); UART_THR = hex_digit(crc, 20); - - uart_wait_tx_empty(); UART_THR = hex_digit(crc, 16); UART_THR = hex_digit(crc, 12); UART_THR = hex_digit(crc, 8); @@ -107,6 +109,7 @@ unsigned long UART_BASE = 0xbf000000; #define MATMUL_DMA_SRC_BASE (MATMUL_BASE_UNCACHED + 0x020) #define MATMUL_DMA_GROUPS (MATMUL_BASE_UNCACHED + 0x028) #define MATMUL_DMA_SRC_STRIDE (MATMUL_BASE_UNCACHED + 0x02C) +#define MATMUL_DMA_CYCLES (MATMUL_BASE_UNCACHED + 0x034) #define CTRL_START_B0 0x1 #define CTRL_START_B1 0x2 @@ -121,6 +124,8 @@ unsigned long UART_BASE = 0xbf000000; #define DMA_CTRL_VAL (CTRL_BURST_SIZE(2) | CTRL_BURST_LEN(63) | CTRL_START_BIT) #define MATMUL_EP_ID 0 +volatile unsigned int matmul_dma_cycles; + int main(void) { if (MATMUL_GROUP_NUM <= 0) { @@ -141,7 +146,9 @@ int main(void) *(volatile unsigned int *)MATMUL_DMA_GROUPS = MATMUL_GROUP_NUM; *(volatile unsigned int *)MATMUL_DMA_SRC_STRIDE = 128; *mat_ctrl = CTRL_DMA_START; + uart_put_crc_prefix_fifo(); while (!(*mat_status & STATUS_DMA_DONE)) {} + matmul_dma_cycles = *(volatile unsigned int *)MATMUL_DMA_CYCLES; *mat_status = STATUS_DMA_DONE; crc = *mat_crc;