fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize matmul CRC-only path

Browse Source
This commit is contained in:
FallenSigh
2026-07-04 16:13:26 +08:00
parent 094157ff05
commit 1540087071
4 changed files with 139 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
rtl/ip/matmul/matmul_axi_slave.v
View File

@@ -217,15 +217,54 @@ module matmul_axi_slave (
reg dma_req0_pulse;
reg dma_req1_pulse;
function [3:0] c_elem_from_word;
input [5:0] word_idx;
begin
case (word_idx)
6'd0, 6'd1, 6'd2: c_elem_from_word = 4'd0;
6'd3, 6'd4, 6'd5: c_elem_from_word = 4'd1;
6'd6, 6'd7, 6'd8: c_elem_from_word = 4'd2;
6'd9, 6'd10, 6'd11: c_elem_from_word = 4'd3;
6'd12, 6'd13, 6'd14: c_elem_from_word = 4'd4;
6'd15, 6'd16, 6'd17: c_elem_from_word = 4'd5;
6'd18, 6'd19, 6'd20: c_elem_from_word = 4'd6;
6'd21, 6'd22, 6'd23: c_elem_from_word = 4'd7;
6'd24, 6'd25, 6'd26: c_elem_from_word = 4'd8;
6'd27, 6'd28, 6'd29: c_elem_from_word = 4'd9;
6'd30, 6'd31, 6'd32: c_elem_from_word = 4'd10;
6'd33, 6'd34, 6'd35: c_elem_from_word = 4'd11;
6'd36, 6'd37, 6'd38: c_elem_from_word = 4'd12;
6'd39, 6'd40, 6'd41: c_elem_from_word = 4'd13;
6'd42, 6'd43, 6'd44: c_elem_from_word = 4'd14;
default: c_elem_from_word = 4'd15;
endcase
end
endfunction
function [1:0] c_part_from_word;
input [5:0] word_idx;
begin
case (word_idx)
6'd0, 6'd3, 6'd6, 6'd9,
6'd12, 6'd15, 6'd18, 6'd21,
6'd24, 6'd27, 6'd30, 6'd33,
6'd36, 6'd39, 6'd42, 6'd45: c_part_from_word = 2'd0;
6'd1, 6'd4, 6'd7, 6'd10,
6'd13, 6'd16, 6'd19, 6'd22,
6'd25, 6'd28, 6'd31, 6'd34,
6'd37, 6'd40, 6'd43, 6'd46: c_part_from_word = 2'd1;
default: c_part_from_word = 2'd2;
endcase
end
endfunction
assign ep_wr_ready = 1'b1;
assign ep_rd_ready = 1'b1;
wire [3:0] ep_wr_lo_idx = ep_wr_word_addr[3:0];
wire [3:0] ep_wr_hi_idx = ep_wr_word_addr[3:0];
wire [7:0] ep_rd_elem_raw = ep_rd_word_addr / 8'd3;
wire [7:0] ep_rd_part_raw = ep_rd_word_addr % 8'd3;
wire [3:0] ep_rd_elem = ep_rd_elem_raw[3:0];
wire [1:0] ep_rd_part = ep_rd_part_raw[1:0];
wire [3:0] ep_rd_elem = c_elem_from_word(ep_rd_word_addr[5:0]);
wire [1:0] ep_rd_part = c_part_from_word(ep_rd_word_addr[5:0]);
wire [65:0] ep_rd_c0 = b0_c[ep_rd_elem];
wire [65:0] ep_rd_c1 = b1_c[ep_rd_elem];
@@ -263,25 +302,40 @@ module matmul_axi_slave (
reg [31:0] crc32_table [0:255];
reg [31:0] crc_reg;
wire crc_dma_valid;
wire [31:0] crc_dma_data;
wire [31:0] crc_stream_data = crc_dma_valid ? crc_dma_data :
(ep_rd_valid ? ep_rd_data : s_axi_rdata);
wire [31:0] crc_stream_data = ep_rd_valid ? ep_rd_data : s_axi_rdata;
wire crc_axi_valid = (r_state == R_DATA) && s_axi_rvalid && s_axi_rready &&
((r_offset >= ADDR_B0_C && r_offset < (ADDR_B0_C + 12'h0C0)) ||
(r_offset >= ADDR_B1_C && r_offset < (ADDR_B1_C + 12'h0C0)));
wire crc_ep_valid = ep_rd_valid && ep_rd_ready && (ep_rd_word_addr < 8'd48);
wire crc_stream_valid = crc_axi_valid || crc_ep_valid || crc_dma_valid;
wire crc_stream_valid = crc_axi_valid || crc_ep_valid;
wire [31:0] crc_t0, crc_c0, crc_t1, crc_c1, crc_t2, crc_c2, crc_t3, crc_next;
assign crc_t0 = crc32_table[(crc_reg[7:0] ^ crc_stream_data[7:0])];
assign crc_c0 = crc_t0 ^ {8'h00, crc_reg[31:8]};
assign crc_t1 = crc32_table[(crc_c0[7:0] ^ crc_stream_data[15:8])];
assign crc_c1 = crc_t1 ^ {8'h00, crc_c0[31:8]};
assign crc_t2 = crc32_table[(crc_c1[7:0] ^ crc_stream_data[23:16])];
assign crc_c2 = crc_t2 ^ {8'h00, crc_c1[31:8]};
assign crc_t3 = crc32_table[(crc_c2[7:0] ^ crc_stream_data[31:24])];
assign crc_next = crc_t3 ^ {8'h00, crc_c2[31:8]};
function [31:0] crc32_update_word;
input [31:0] crc_in;
input [31:0] data_in;
reg [31:0] c0;
reg [31:0] c1;
reg [31:0] c2;
begin
c0 = crc32_table[(crc_in[7:0] ^ data_in[7:0])] ^ {8'h00, crc_in[31:8]};
c1 = crc32_table[(c0[7:0] ^ data_in[15:8])] ^ {8'h00, c0[31:8]};
c2 = crc32_table[(c1[7:0] ^ data_in[23:16])] ^ {8'h00, c1[31:8]};
crc32_update_word = crc32_table[(c2[7:0] ^ data_in[31:24])] ^ {8'h00, c2[31:8]};
end
endfunction
function [31:0] crc32_update_c66;
input [31:0] crc_in;
input [65:0] c_in;
reg [31:0] crc_mid0;
reg [31:0] crc_mid1;
begin
crc_mid0 = crc32_update_word(crc_in, c_in[31:0]);
crc_mid1 = crc32_update_word(crc_mid0, c_in[63:32]);
crc32_update_c66 = crc32_update_word(crc_mid1, {30'b0, c_in[65:64]});
end
endfunction
wire [31:0] crc_next = crc32_update_word(crc_reg, crc_stream_data);
initial begin
crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096;
@@ -454,30 +508,6 @@ module matmul_axi_slave (
reg [1:0] dma_wr_state;
reg [5:0] dma_wr_count;
wire [7:0] dma_wr_elem_raw = dma_wr_count / 6'd3;
wire [7:0] dma_wr_part_raw = dma_wr_count % 6'd3;
wire [3:0] dma_wr_elem = dma_wr_elem_raw[3:0];
wire [1:0] dma_wr_part = dma_wr_part_raw[1:0];
wire [65:0] dma_wr_c0 = b0_c[dma_wr_elem];
wire [65:0] dma_wr_c1 = b1_c[dma_wr_elem];
wire [65:0] dma_wr_c = dma_store_bank ? dma_wr_c1 : dma_wr_c0;
wire [31:0] dma_wr_word =
(dma_wr_part == 2'd0) ? dma_wr_c[31:0] :
(dma_wr_part == 2'd1) ? dma_wr_c[63:32] :
{30'b0, dma_wr_c[65:64]};
wire [5:0] dma_wr_next_count = dma_wr_count + 6'd1;
wire [7:0] dma_wr_next_elem_raw = dma_wr_next_count / 6'd3;
wire [7:0] dma_wr_next_part_raw = dma_wr_next_count % 6'd3;
wire [3:0] dma_wr_next_elem = dma_wr_next_elem_raw[3:0];
wire [1:0] dma_wr_next_part = dma_wr_next_part_raw[1:0];
wire [65:0] dma_wr_next_c0 = b0_c[dma_wr_next_elem];
wire [65:0] dma_wr_next_c1 = b1_c[dma_wr_next_elem];
wire [65:0] dma_wr_next_c = dma_store_bank ? dma_wr_next_c1 : dma_wr_next_c0;
wire [31:0] dma_wr_word_next =
(dma_wr_next_part == 2'd0) ? dma_wr_next_c[31:0] :
(dma_wr_next_part == 2'd1) ? dma_wr_next_c[63:32] :
{30'b0, dma_wr_next_c[65:64]};
// 【修改点 1】单独引出一个写脉冲信号给 CRC 清零用
wire crc_clear_pulse = (w_state == W_DATA) && s_axi_wvalid && s_axi_wready && (w_offset == ADDR_CRC);
wire done_clear_pulse = (w_state == W_DATA) && s_axi_wvalid && s_axi_wready &&
@@ -630,13 +660,13 @@ module matmul_axi_slave (
assign s_axi_rresp = 2'b00;
wire [5:0] c0_offset = (r_offset - ADDR_B0_C) >> 2;
wire [3:0] c0_elem = c0_offset / 3;
wire [1:0] c0_part = c0_offset % 3;
wire [3:0] c0_elem = c_elem_from_word(c0_offset);
wire [1:0] c0_part = c_part_from_word(c0_offset);
wire [65:0] target_c0 = b0_c[c0_elem];
wire [5:0] c1_offset = (r_offset - ADDR_B1_C) >> 2;
wire [3:0] c1_elem = c1_offset / 3;
wire [1:0] c1_part = c1_offset % 3;
wire [3:0] c1_elem = c_elem_from_word(c1_offset);
wire [1:0] c1_part = c_part_from_word(c1_offset);
wire [65:0] target_c1 = b1_c[c1_elem];
wire crc_read_en = (r_offset >= ADDR_B0_C && r_offset < (ADDR_B0_C + 12'h0C0)) ||
@@ -727,17 +757,6 @@ module matmul_axi_slave (
end
end
// CRC register: update on each C data read beat, clear on write to 0x010
always @(posedge clk) begin
if (!resetn) begin
crc_reg <= 32'hFFFFFFFF;
end else if (crc_clear_pulse) begin
crc_reg <= 32'hFFFFFFFF;
end else if (crc_stream_valid) begin
crc_reg <= crc_next;
end
end
// ==========================================
// 计算引擎 (Compute FSM)
// ==========================================
@@ -763,6 +782,24 @@ module matmul_axi_slave (
reg acc_bank_d1, acc_bank_d2, acc_bank_d3, acc_bank_d4;
reg acc_valid_d1, acc_valid_d2, acc_valid_d3, acc_valid_d4;
reg [4:0] pipe_cnt;
wire [65:0] acc_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
wire crc_calc_valid = dma_busy && acc_valid_d4;
wire [31:0] crc_calc_next = crc32_update_c66(crc_reg, acc_result);
// CRC register: software/EP reads keep the legacy 32-bit path; autonomous DMA
// folds each 66-bit C element directly as low/mid/high words.
always @(posedge clk) begin
if (!resetn) begin
crc_reg <= 32'hFFFFFFFF;
end else if (crc_clear_pulse) begin
crc_reg <= 32'hFFFFFFFF;
end else if (crc_calc_valid) begin
crc_reg <= crc_calc_next;
end else if (crc_stream_valid) begin
crc_reg <= crc_next;
end
end
task start_calc_bank;
input bank;
@@ -858,11 +895,9 @@ module matmul_axi_slave (
// Accumulate results from 4 cycles ago.
if (acc_valid_d4) begin
if (acc_bank_d4 == 1'b0)
b0_c[acc_idx_d4] <= ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
b0_c[acc_idx_d4] <= acc_result;
else
b1_c[acc_idx_d4] <= ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
b1_c[acc_idx_d4] <= acc_result;
end
if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
@@ -887,11 +922,8 @@ module matmul_axi_slave (
end
// ==========================================
// Autonomous AXI master loader/storer
// Autonomous AXI master loader with inline result CRC.
// ==========================================
assign crc_dma_valid = (dma_wr_state == DMA_WR_DATA) && m_axi_wvalid && m_axi_wready;
assign crc_dma_data = m_axi_wdata;
always @(posedge clk) begin
if (!resetn) begin
dma_busy <= 1'b0;
@@ -971,13 +1003,13 @@ module matmul_axi_slave (
if (state == C_RUN)
bank_compute_pending[cur_bank] <= 1'b0;
if (bank_loaded[0] && !req_0 && !bank_store_pending[0] && !bank_storing[0] &&
if (bank_loaded[0] && !req_0 &&
!bank_compute_pending[0] && !(state == C_RUN && cur_bank == 1'b0)) begin
dma_req0_pulse <= 1'b1;
bank_loaded[0] <= 1'b0;
bank_compute_pending[0] <= 1'b1;
dma_compute_bank <= 1'b0;
end else if (bank_loaded[1] && !req_1 && !bank_store_pending[1] && !bank_storing[1] &&
end else if (bank_loaded[1] && !req_1 &&
!bank_compute_pending[1] && !(state == C_RUN && cur_bank == 1'b1)) begin
dma_req1_pulse <= 1'b1;
bank_loaded[1] <= 1'b0;
@@ -987,7 +1019,12 @@ module matmul_axi_slave (
end
if (calc_done_pulse && dma_busy) begin
bank_store_pending[cur_bank] <= 1'b1;
store_group <= store_group + 32'd1;
completed_groups <= completed_groups + 32'd1;
if ((completed_groups + 32'd1) >= dma_groups) begin
dma_busy <= 1'b0;
dma_done <= 1'b1;
end
end
case (dma_rd_state)
@@ -1037,56 +1074,32 @@ module matmul_axi_slave (
m_axi_awvalid <= 1'b0;
m_axi_wvalid <= 1'b0;
m_axi_bready <= 1'b0;
if (dma_busy && bank_store_pending[dma_store_bank]) begin
dma_wr_count <= 6'd0;
bank_store_pending[dma_store_bank] <= 1'b0;
bank_storing[dma_store_bank] <= 1'b1;
m_axi_awaddr <= dma_next_wr_addr;
m_axi_awlen <= 8'd47;
m_axi_awsize <= 3'd2;
m_axi_awburst <= 2'b01;
m_axi_awvalid <= 1'b1;
dma_wr_state <= DMA_WR_ADDR;
end
end
DMA_WR_ADDR: begin
if (m_axi_awvalid && m_axi_awready) begin
m_axi_awvalid <= 1'b0;
m_axi_wvalid <= 1'b1;
m_axi_wdata <= dma_wr_word;
m_axi_wlast <= 1'b0;
dma_wr_state <= DMA_WR_DATA;
end
m_axi_awvalid <= 1'b0;
dma_wr_state <= DMA_WR_DATA;
end
DMA_WR_DATA: begin
if (m_axi_wvalid && m_axi_wready) begin
if (dma_wr_count == 6'd47) begin
m_axi_wvalid <= 1'b0;
m_axi_wlast <= 1'b0;
m_axi_bready <= 1'b1;
dma_wr_state <= DMA_WR_RESP;
end else begin
dma_wr_count <= dma_wr_count + 6'd1;
m_axi_wdata <= dma_wr_word_next;
m_axi_wlast <= (dma_wr_count == 6'd46);
end
end
end
DMA_WR_RESP: begin
if (m_axi_bvalid && m_axi_bready) begin
m_axi_bready <= 1'b0;
m_axi_wvalid <= 1'b0;
m_axi_wlast <= 1'b0;
if (dma_wr_count == 6'd47) begin
bank_storing[dma_store_bank] <= 1'b0;
dma_store_bank <= ~dma_store_bank;
store_group <= store_group + 32'd1;
dma_next_wr_addr <= dma_next_wr_addr + dma_dst_stride;
completed_groups <= completed_groups + 32'd1;
if ((completed_groups + 32'd1) >= dma_groups) begin
dma_busy <= 1'b0;
dma_done <= 1'b1;
end
dma_wr_state <= DMA_WR_IDLE;
end else begin
dma_wr_count <= dma_wr_count + 6'd1;
end
end
DMA_WR_RESP: begin
m_axi_bready <= 1'b0;
dma_wr_state <= DMA_WR_IDLE;
end
default: dma_wr_state <= DMA_WR_IDLE;
endcase
end