diff --git a/rtl/ip/PLL_2019_2/clk_pll.xci b/rtl/ip/PLL_2019_2/clk_pll.xci index 92ef6f9..3971b3b 100644 --- a/rtl/ip/PLL_2019_2/clk_pll.xci +++ b/rtl/ip/PLL_2019_2/clk_pll.xci @@ -35,6 +35,12 @@ 100000000 0 0.000 + + + + 100000000 + 0 + 0.000 1 LEVEL_HIGH @@ -136,7 +142,7 @@ 100.000 0.000 1 - 0 + 1 0000 0000 100.000 @@ -249,7 +255,7 @@ 20.000 0.000 FALSE - 20.000 + 20.0 10.0 30.000 0.500 @@ -259,11 +265,7 @@ 0.500 0.000 FALSE -<<<<<<< Updated upstream - 1 -======= 13 ->>>>>>> Stashed changes 0.500 0.000 FALSE @@ -293,18 +295,12 @@ FALSE 64.000 2.000 - 2 + 3 Output Output Phase Duty Cycle Pk-to-Pk Phase Clock Freq (MHz) (degrees) (%) Jitter (ps) Error (ps) -<<<<<<< Updated upstream - _cpu_clk__32.97872______0.000______50.0______130.093____123.600 - _sys_clk__50.00000______0.000______50.0______122.035____123.600 - no_CLK_OUT3_output -======= _cpu_clk__33.33333______0.000______50.0______215.691____164.985 _sys_clk__50.00000______0.000______50.0______192.113____164.985 mlkem_clk__76.92308______0.000______50.0______172.249____164.985 ->>>>>>> Stashed changes no_CLK_OUT4_output no_CLK_OUT5_output no_CLK_OUT6_output @@ -424,32 +420,32 @@ 100.0 0.010 BUFG - 130.093 + 243.156 false - 123.600 + 144.334 50.000 33.000 0.000 1 true BUFG - 122.035 + 222.587 false - 123.600 + 144.334 50.000 50.000 0.000 1 true BUFG - 110.420 + 186.330 false - 123.600 + 144.334 50.000 100.000 0.000 1 - false + true BUFG 0.0 false @@ -538,7 +534,7 @@ 20 0.000 false - 20.000 + 20.0 10.0 30 0.500 @@ -548,11 +544,7 @@ 0.500 0.000 false -<<<<<<< Updated upstream - 1 -======= 13 ->>>>>>> Stashed changes 0.500 0.000 false @@ -580,13 +572,8 @@ 0.010 0.010 false -<<<<<<< Updated upstream - 2 - false -======= 3 true ->>>>>>> Stashed changes false WAVEFORM false @@ -726,7 +713,6 @@ - @@ -735,7 +721,6 @@ - diff --git a/rtl/ip/cdma/cdma_matmul_bridge.sv b/rtl/ip/cdma/cdma_matmul_bridge.sv index 65d9399..d1e36dc 100644 --- a/rtl/ip/cdma/cdma_matmul_bridge.sv +++ b/rtl/ip/cdma/cdma_matmul_bridge.sv @@ -64,6 +64,7 @@ module cdma_matmul_bridge #( output logic mat_done_clear, output logic mat_crc_clear, input logic mat_done, + input logic mat_crc_idle, output logic seq_done_irq ); @@ -102,10 +103,10 @@ module cdma_matmul_bridge #( logic [31:0] load_next_dst_addr; logic [31:0] bank_dst_addr [2]; - logic rd_busy; logic wr_busy; - logic rd_active_bank; logic wr_active_bank; + logic [1:0] rd_outstanding_count; + logic [1:0] rd_bank_fifo; logic [1:0] compute_q; logic [1:0] compute_q_count; logic mat_done_seen; @@ -136,6 +137,7 @@ module cdma_matmul_bridge #( wire retire_from_store = seq_busy && cdma_wr_req_done; wire retire_from_compute = seq_busy && seq_skip_writeback && mat_done_event && (compute_q_count != 2'd0); wire [31:0] retire_inc = {31'd0, retire_from_store} + {31'd0, retire_from_compute}; + wire rd_req_fire = cdma_rd_req_valid && cdma_rd_req_ready; function automatic logic [31:0] apply_wstrb( input logic [31:0] old_val, @@ -257,10 +259,10 @@ module cdma_matmul_bridge #( load_next_dst_addr <= 32'd0; bank_dst_addr[0] <= 32'd0; bank_dst_addr[1] <= 32'd0; - rd_busy <= 1'b0; wr_busy <= 1'b0; - rd_active_bank <= 1'b0; wr_active_bank <= 1'b0; + rd_outstanding_count <= 2'd0; + rd_bank_fifo <= 2'b00; compute_q <= 2'b00; compute_q_count <= 2'd0; mat_done_seen <= 1'b0; @@ -290,7 +292,7 @@ module cdma_matmul_bridge #( if (done_clear_req) seq_done_flag <= 1'b0; - if (cdma_rd_req_valid && cdma_rd_req_ready) + if (rd_req_fire) cdma_rd_req_valid <= 1'b0; if (cdma_wr_req_valid && cdma_wr_req_ready) @@ -313,8 +315,9 @@ module cdma_matmul_bridge #( load_next_dst_addr <= seq_dst_base; bank_dst_addr[0] <= 32'd0; bank_dst_addr[1] <= 32'd0; - rd_busy <= 1'b0; wr_busy <= 1'b0; + rd_outstanding_count <= 2'd0; + rd_bank_fifo <= 2'b00; compute_q <= 2'b00; compute_q_count <= 2'd0; mat_done_seen <= 1'b1; @@ -325,18 +328,50 @@ module cdma_matmul_bridge #( end else if (seq_busy) begin if (retire_inc != 32'd0) begin groups_retired <= groups_retired + retire_inc; - if ((groups_retired + retire_inc) >= seq_groups) begin - seq_busy <= 1'b0; - seq_done_flag <= 1'b1; - end end - if (cdma_rd_req_done) begin - rd_busy <= 1'b0; - bank_load_active[rd_active_bank] <= 1'b0; - bank_ab_valid[rd_active_bank] <= 1'b1; + // The final compute completion can precede the recursive CRC + // consumer. Do not expose sequence completion until it drains. + if ((groups_retired >= seq_groups) && mat_crc_idle) begin + seq_busy <= 1'b0; + seq_done_flag <= 1'b1; end + if (cdma_rd_req_done && (rd_outstanding_count != 2'd0)) begin + bank_load_active[rd_bank_fifo[0]] <= 1'b0; + bank_ab_valid[rd_bank_fifo[0]] <= 1'b1; + end + + + // Track the active read and one prefetched read. Completion + // pulses are in request order, so a two-entry bank FIFO is + // sufficient to associate each RLAST with its destination. + case ({rd_req_fire, cdma_rd_req_done}) + 2'b10: begin + if (rd_outstanding_count == 2'd0) + rd_bank_fifo[0] <= cdma_rd_req_ep_bank[0]; + else + rd_bank_fifo[1] <= cdma_rd_req_ep_bank[0]; + rd_outstanding_count <= rd_outstanding_count + 2'd1; + end + 2'b01: begin + if (rd_outstanding_count == 2'd2) + rd_bank_fifo[0] <= rd_bank_fifo[1]; + if (rd_outstanding_count != 2'd0) + rd_outstanding_count <= rd_outstanding_count - 2'd1; + end + 2'b11: begin + if (rd_outstanding_count == 2'd1) begin + rd_bank_fifo[0] <= cdma_rd_req_ep_bank[0]; + end else begin + rd_bank_fifo[0] <= rd_bank_fifo[1]; + rd_bank_fifo[1] <= cdma_rd_req_ep_bank[0]; + end + end + default: begin + end + endcase + if (cdma_wr_req_done) begin wr_busy <= 1'b0; bank_store_active[wr_active_bank] <= 1'b0; @@ -354,15 +389,14 @@ module cdma_matmul_bridge #( compute_q_count <= compute_q_count - 2'd1; end - if (!rd_busy && !cdma_rd_req_valid && (load_next_idx < seq_groups) && load_target_free) begin + if ((rd_outstanding_count < 2'd2) && !cdma_rd_req_valid && + (load_next_idx < seq_groups) && load_target_free) begin cdma_rd_req_src <= load_next_src_addr; cdma_rd_req_bytes <= MAT_AB_BYTES; cdma_rd_req_ep_id <= 4'd0; cdma_rd_req_ep_bank <= {3'b000, load_target_bank}; cdma_rd_req_ep_word_off <= 8'd0; cdma_rd_req_valid <= 1'b1; - rd_busy <= 1'b1; - rd_active_bank <= load_target_bank; bank_load_active[load_target_bank] <= 1'b1; bank_group_idx[load_target_bank] <= load_next_idx; bank_dst_addr[load_target_bank] <= load_next_dst_addr; diff --git a/rtl/ip/cdma/snix_axil_cdma_mux.sv b/rtl/ip/cdma/snix_axil_cdma_mux.sv index 35586e4..cf16694 100644 --- a/rtl/ip/cdma/snix_axil_cdma_mux.sv +++ b/rtl/ip/cdma/snix_axil_cdma_mux.sv @@ -423,6 +423,13 @@ module snix_axil_cdma_mux #( logic [7:0] ext_rd_word_addr_r; logic [3:0] ext_rd_ep_id_r; logic [3:0] ext_rd_ep_bank_r; + logic ext_rd_queue_valid; + logic ext_rd_queue_ar_sent; + logic [ADDR_WIDTH-1:0] ext_rd_queue_addr; + logic [31:0] ext_rd_queue_bytes; + logic [7:0] ext_rd_queue_word_addr; + logic [3:0] ext_rd_queue_ep_id; + logic [3:0] ext_rd_queue_ep_bank; logic [ADDR_WIDTH-1:0] ext_wr_addr_r; logic [31:0] ext_wr_bytes_left; @@ -434,7 +441,10 @@ module snix_axil_cdma_mux #( wire ext_wr_active = (ext_wr_state != EXT_WR_IDLE); wire ext_rd_accept = ext_rd_req_valid && ext_rd_req_ready; wire ext_wr_accept = ext_wr_req_valid && ext_wr_req_ready; - wire ext_any_busy_or_req = ext_rd_active || ext_wr_active || ext_rd_req_valid || ext_wr_req_valid; + wire ext_rd_prefetch_ar = (ext_rd_state == EXT_RD_READ) && + ext_rd_queue_valid && !ext_rd_queue_ar_sent; + wire ext_any_busy_or_req = ext_rd_active || ext_rd_queue_valid || ext_wr_active || + ext_rd_req_valid || ext_wr_req_valid; wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)); // Dynamic routing to the engine based on current active channel @@ -478,8 +488,12 @@ module snix_axil_cdma_mux #( assign mm2mm_bready = ext_wr_active ? (ext_wr_state == EXT_WR_BRESP) : (use_ep_engine ? ep_bready : core_bready); assign mm2mm_arid = ext_rd_active ? '0 : (use_ep_engine ? ep_arid : core_arid); - assign mm2mm_araddr = ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr); - assign mm2mm_arlen = ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_arlen : core_arlen); + assign mm2mm_araddr = ext_rd_prefetch_ar ? ext_rd_queue_addr : + (ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr)); + assign mm2mm_arlen = ext_rd_prefetch_ar ? + (((ext_rd_queue_bytes[31:2] == 30'd0) ? 8'd1 : ext_rd_queue_bytes[9:2]) - 8'd1) : + (ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) : + (use_ep_engine ? ep_arlen : core_arlen)); assign mm2mm_arsize = ext_rd_active ? 3'd2 : (use_ep_engine ? ep_arsize : core_arsize); assign mm2mm_arburst = ext_rd_active ? 2'b01 : (use_ep_engine ? ep_arburst : core_arburst); assign mm2mm_arlock = ext_rd_active ? 1'b0 : (use_ep_engine ? ep_arlock : core_arlock); @@ -487,7 +501,8 @@ module snix_axil_cdma_mux #( assign mm2mm_arprot = ext_rd_active ? 3'b0 : (use_ep_engine ? ep_arprot : core_arprot); assign mm2mm_arqos = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arqos : core_arqos); assign mm2mm_aruser = ext_rd_active ? '0 : (use_ep_engine ? ep_aruser : core_aruser); - assign mm2mm_arvalid = ext_rd_active ? (ext_rd_state == EXT_RD_AR) : (use_ep_engine ? ep_arvalid : core_arvalid); + assign mm2mm_arvalid = ext_rd_active ? ((ext_rd_state == EXT_RD_AR) || ext_rd_prefetch_ar) : + (use_ep_engine ? ep_arvalid : core_arvalid); assign mm2mm_rready = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && ep_wr_ready) : (use_ep_engine ? ep_rready : core_rready); always_ff @(posedge clk) begin @@ -597,7 +612,10 @@ module snix_axil_cdma_mux #( // ========================================== // Generic external endpoint requester // ========================================== - assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_rd_active; + assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active && + ((ext_rd_state == EXT_RD_IDLE) || + ((ext_rd_state == EXT_RD_READ) && !ext_rd_queue_valid && + !ext_rd_done_fire)); assign ext_wr_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active; assign ext_rd_req_done = ext_rd_done_fire; assign ext_wr_req_done = ext_wr_done_fire; @@ -610,7 +628,17 @@ module snix_axil_cdma_mux #( ext_rd_word_addr_r <= '0; ext_rd_ep_id_r <= '0; ext_rd_ep_bank_r <= '0; + ext_rd_queue_valid <= 1'b0; + ext_rd_queue_ar_sent <= 1'b0; + ext_rd_queue_addr <= '0; + ext_rd_queue_bytes <= '0; + ext_rd_queue_word_addr <= '0; + ext_rd_queue_ep_id <= '0; + ext_rd_queue_ep_bank <= '0; end else begin + if (ext_rd_prefetch_ar && mm2mm_arvalid && mm2mm_arready) + ext_rd_queue_ar_sent <= 1'b1; + case (ext_rd_state) EXT_RD_IDLE: begin if (ext_rd_accept) begin @@ -629,11 +657,37 @@ module snix_axil_cdma_mux #( end EXT_RD_READ: begin + if (ext_rd_accept) begin + ext_rd_queue_valid <= 1'b1; + ext_rd_queue_ar_sent <= 1'b0; + ext_rd_queue_addr <= ext_rd_req_src; + ext_rd_queue_bytes <= ext_rd_req_bytes; + ext_rd_queue_word_addr <= ext_rd_req_ep_word_off; + ext_rd_queue_ep_id <= ext_rd_req_ep_id; + ext_rd_queue_ep_bank <= ext_rd_req_ep_bank; + end + if (mm2mm_rvalid && mm2mm_rready) begin ext_rd_word_addr_r <= ext_rd_word_addr_r + 8'd1; ext_rd_bytes_left <= (ext_rd_bytes_left > 32'd4) ? (ext_rd_bytes_left - 32'd4) : 32'd0; - if (mm2mm_rlast) - ext_rd_state <= EXT_RD_IDLE; + if (mm2mm_rlast) begin + if (ext_rd_queue_valid) begin + ext_rd_addr_r <= ext_rd_queue_addr; + ext_rd_bytes_left <= ext_rd_queue_bytes; + ext_rd_word_addr_r <= ext_rd_queue_word_addr; + ext_rd_ep_id_r <= ext_rd_queue_ep_id; + ext_rd_ep_bank_r <= ext_rd_queue_ep_bank; + ext_rd_queue_valid <= 1'b0; + ext_rd_queue_ar_sent <= 1'b0; + if (ext_rd_queue_ar_sent || + (ext_rd_prefetch_ar && mm2mm_arvalid && mm2mm_arready)) + ext_rd_state <= EXT_RD_READ; + else + ext_rd_state <= EXT_RD_AR; + end else begin + ext_rd_state <= EXT_RD_IDLE; + end + end end end diff --git a/rtl/ip/matmul/matmul_axi_slave.v b/rtl/ip/matmul/matmul_axi_slave.v index 14eb17f..2da4523 100644 --- a/rtl/ip/matmul/matmul_axi_slave.v +++ b/rtl/ip/matmul/matmul_axi_slave.v @@ -102,6 +102,7 @@ module matmul_axi_slave ( input mat_done_clear, input mat_crc_clear, output mat_done, + output mat_crc_idle, // Interrupt output (active high level, pulsed by calc_done) output intr @@ -184,22 +185,44 @@ module matmul_axi_slave ( reg [31:0] crc32_table [0:255]; reg [31:0] crc_reg; - function [31:0] crc32_word_next; + function [31:0] crc32_update_word; input [31:0] crc_in; input [31:0] data_in; - reg [31:0] t0, c0, t1, c1, t2, c2, t3; + reg [31:0] c0; + reg [31:0] c1; + reg [31:0] c2; begin - t0 = crc32_table[(crc_in[7:0] ^ data_in[7:0])]; - c0 = t0 ^ {8'h00, crc_in[31:8]}; - t1 = crc32_table[(c0[7:0] ^ data_in[15:8])]; - c1 = t1 ^ {8'h00, c0[31:8]}; - t2 = crc32_table[(c1[7:0] ^ data_in[23:16])]; - c2 = t2 ^ {8'h00, c1[31:8]}; - t3 = crc32_table[(c2[7:0] ^ data_in[31:24])]; - crc32_word_next = t3 ^ {8'h00, c2[31:8]}; + c0 = crc32_table[(crc_in[7:0] ^ data_in[7:0])] ^ {8'h00, crc_in[31:8]}; + c1 = crc32_table[(c0[7:0] ^ data_in[15:8])] ^ {8'h00, c0[31:8]}; + c2 = crc32_table[(c1[7:0] ^ data_in[23:16])] ^ {8'h00, c1[31:8]}; + crc32_update_word = crc32_table[(c2[7:0] ^ data_in[31:24])] ^ + {8'h00, c2[31:8]}; end endfunction + function [31:0] crc32_update_6bytes; + input [31:0] crc_in; + input [47:0] data_in; + reg [31:0] c; + integer byte_idx; + begin + c = crc_in; + for (byte_idx = 0; byte_idx < 6; byte_idx = byte_idx + 1) + c = crc32_table[(c[7:0] ^ data_in[byte_idx*8 +: 8])] ^ + {8'h00, c[31:8]}; + crc32_update_6bytes = c; + end + endfunction + + localparam CRC_FIFO_DEPTH = 32; + reg [65:0] crc_fifo [0:CRC_FIFO_DEPTH-1]; + reg [4:0] crc_fifo_wr_ptr; + reg [4:0] crc_fifo_rd_ptr; + reg [5:0] crc_fifo_count; + reg crc_c66_busy; + reg [65:0] crc_c66_data; + reg [31:0] crc_c66_half; + initial begin crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096; crc32_table[2 ] = 32'hEE0E612C; crc32_table[3 ] = 32'h990951BA; @@ -587,10 +610,21 @@ module matmul_axi_slave ( reg [4:0] pipe_cnt; wire [65:0] calc_c_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) + ({2'b0, mult_p[2]} + {2'b0, mult_p[3]}); - wire calc_crc_valid = acc_valid_d4; - wire [31:0] crc_calc_0 = crc32_word_next(crc_reg, calc_c_result[31:0]); - wire [31:0] crc_calc_1 = crc32_word_next(crc_calc_0, calc_c_result[63:32]); - wire [31:0] crc_calc_2 = crc32_word_next(crc_calc_1, {30'b0, calc_c_result[65:64]}); + + wire crc_fifo_empty = (crc_fifo_count == 6'd0); + wire crc_fifo_push = acc_valid_d4; + wire crc_fifo_pop = !crc_c66_busy && !crc_fifo_empty; + wire [31:0] crc_c66_first_next = + crc32_update_6bytes(crc_reg, crc_fifo[crc_fifo_rd_ptr][47:0]); + wire [47:0] crc_c66_second_data = {30'b0, crc_c66_data[65:48]}; + wire [31:0] crc_c66_final_next = + crc32_update_6bytes(crc_c66_half, crc_c66_second_data); + + // Reserve space for results already in the four-stage multiplier pipeline. + wire crc_issue_ready = (crc_fifo_count < 6'd24); + assign mat_crc_idle = crc_fifo_empty && !crc_c66_busy && + !acc_valid_d1 && !acc_valid_d2 && + !acc_valid_d3 && !acc_valid_d4; task start_calc_bank; input bank; @@ -616,14 +650,46 @@ module matmul_axi_slave ( end endtask - // CRC tracks completed C elements directly, independent of result writeback. + // Buffer completed C elements so the recursive CRC path is split across + // two cycles without reducing the multiplier's burst throughput. always @(posedge clk) begin if (!resetn) begin - crc_reg <= 32'hFFFFFFFF; + crc_reg <= 32'hFFFFFFFF; + crc_fifo_wr_ptr <= 5'd0; + crc_fifo_rd_ptr <= 5'd0; + crc_fifo_count <= 6'd0; + crc_c66_busy <= 1'b0; + crc_c66_data <= 66'd0; + crc_c66_half <= 32'd0; end else if (crc_clear_pulse) begin - crc_reg <= 32'hFFFFFFFF; - end else if (calc_crc_valid) begin - crc_reg <= crc_calc_2; + crc_reg <= 32'hFFFFFFFF; + crc_fifo_wr_ptr <= 5'd0; + crc_fifo_rd_ptr <= 5'd0; + crc_fifo_count <= 6'd0; + crc_c66_busy <= 1'b0; + crc_c66_data <= 66'd0; + crc_c66_half <= 32'd0; + end else begin + if (crc_fifo_push) begin + crc_fifo[crc_fifo_wr_ptr] <= calc_c_result; + crc_fifo_wr_ptr <= crc_fifo_wr_ptr + 5'd1; + end + + case ({crc_fifo_push, crc_fifo_pop}) + 2'b10: crc_fifo_count <= crc_fifo_count + 6'd1; + 2'b01: crc_fifo_count <= crc_fifo_count - 6'd1; + default: crc_fifo_count <= crc_fifo_count; + endcase + + if (crc_c66_busy) begin + crc_reg <= crc_c66_final_next; + crc_c66_busy <= 1'b0; + end else if (crc_fifo_pop) begin + crc_c66_data <= crc_fifo[crc_fifo_rd_ptr]; + crc_c66_half <= crc_c66_first_next; + crc_c66_busy <= 1'b1; + crc_fifo_rd_ptr <= crc_fifo_rd_ptr + 5'd1; + end end end @@ -676,21 +742,25 @@ module matmul_axi_slave ( end C_RUN: begin - // Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds) + // Issue one C element per cycle while the CRC result FIFO + // has room for this result and the in-flight pipeline. if (pipe_cnt < 5'd16) begin - acc_idx_d1 <= pipe_cnt[3:0]; - acc_bank_d1 <= cur_bank; - acc_valid_d1 <= 1'b1; - if (cur_bank == 1'b0) begin - m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}]; - m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}]; - m_in_a[2] <= b0_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b0_b[{2'd2, pipe_cnt[1:0]}]; - m_in_a[3] <= b0_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b0_b[{2'd3, pipe_cnt[1:0]}]; - end else begin - m_in_a[0] <= b1_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b1_b[{2'd0, pipe_cnt[1:0]}]; - m_in_a[1] <= b1_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b1_b[{2'd1, pipe_cnt[1:0]}]; - m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}]; - m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}]; + if (crc_issue_ready) begin + acc_idx_d1 <= pipe_cnt[3:0]; + acc_bank_d1 <= cur_bank; + acc_valid_d1 <= 1'b1; + if (cur_bank == 1'b0) begin + m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}]; + m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}]; + m_in_a[2] <= b0_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b0_b[{2'd2, pipe_cnt[1:0]}]; + m_in_a[3] <= b0_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b0_b[{2'd3, pipe_cnt[1:0]}]; + end else begin + m_in_a[0] <= b1_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b1_b[{2'd0, pipe_cnt[1:0]}]; + m_in_a[1] <= b1_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b1_b[{2'd1, pipe_cnt[1:0]}]; + m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}]; + m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}]; + end + pipe_cnt <= pipe_cnt + 5'd1; end end @@ -706,16 +776,14 @@ module matmul_axi_slave ( calc_done_pulse <= 1'b1; end - if (pipe_cnt < 5'd16) begin - pipe_cnt <= pipe_cnt + 5'd1; - end else if (cur_bank == 1'b0 && req_1) begin + if (pipe_cnt >= 5'd16 && cur_bank == 1'b0 && req_1) begin start_calc_bank(1'b1); - end else if (cur_bank == 1'b1 && req_0) begin + end else if (pipe_cnt >= 5'd16 && cur_bank == 1'b1 && req_0) begin start_calc_bank(1'b0); - end else if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin + end else if (pipe_cnt >= 5'd16 && acc_valid_d4 && acc_idx_d4 == 4'd15) begin busy <= 1'b0; state <= C_IDLE; - end else begin + end else if (pipe_cnt >= 5'd16) begin pipe_cnt <= pipe_cnt + 5'd1; end end diff --git a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v index 49c7fa8..e6ed66a 100644 --- a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v +++ b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v @@ -30,15 +30,16 @@ `include "defines.vh" -/* verilator lint_off UNUSED */ +/* verilator lint_off UNUSEDPARAM */ +(* use_dsp = "no" *) module sample_ntt_sync #(parameter K = 4) ( -/* verilator lint_on UNUSED */ +/* verilator lint_on UNUSEDPARAM */ input clk, input rst_n, input [255:0] rho_i, - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ input [2:0] k_i, - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ input [1:0] i_idx, input [1:0] j_idx, input valid_i, @@ -72,12 +73,12 @@ module sample_ntt_sync #(parameter K = 4) ( // ================================================================ // Registered inputs (captured on valid_i) // ================================================================ - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ reg [255:0] rho_r; reg [2:0] k_r; reg [1:0] i_r; reg [1:0] j_r; - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ // ================================================================ // Coefficient counter (0..256). 9 bits to avoid overflow when @@ -163,9 +164,9 @@ module sample_ntt_sync #(parameter K = 4) ( // ================================================================ wire kc_valid_i; wire [1599:0] kc_state_i; - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ wire kc_ready_o; - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ wire [1599:0] kc_state_o; wire kc_valid_o; @@ -209,6 +210,10 @@ module sample_ntt_sync #(parameter K = 4) ( // d2 rejected, no longer need coefficients, or d2 was just accepted out. wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) && (!d2_acc_r || !need_more || (valid_o_r && ready_i)); + // `need_more` observes the registered counter before the final d2 + // handshake increments it. Detect that boundary explicitly so a full + // rate block does not launch an unnecessary final re-permutation. + wire completes_output = valid_o_r && ready_i && (coeff_cnt_r == 9'd255); // ================================================================ // FSM: state_next (combinational) @@ -234,7 +239,7 @@ module sample_ntt_sync #(parameter K = 4) ( // next group in this block, re-permute (block exhausted), or // finish. if (grp_done) begin - if (!need_more) + if (!need_more || completes_output) state_next = ST_DONE; else if (grp_ptr_r < GRP_MAX) state_next = ST_SQUEEZE; // next group, no re-permute diff --git a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v index d5ed5cc..dc9462b 100644 --- a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v +++ b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v @@ -43,15 +43,16 @@ `include "defines.vh" -/* verilator lint_off UNUSED */ +/* verilator lint_off UNUSEDPARAM */ +(* use_dsp = "no" *) module sample_ntt_sync_shared #(parameter K = 4) ( -/* verilator lint_on UNUSED */ +/* verilator lint_on UNUSEDPARAM */ input clk, input rst_n, input [255:0] rho_i, - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ input [2:0] k_i, - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ input [1:0] i_idx, input [1:0] j_idx, input valid_i, @@ -67,9 +68,9 @@ module sample_ntt_sync_shared #(parameter K = 4) ( output kc_ready_i, output [1599:0] kc_state_i, output kc_valid_i, - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ input kc_ready_o - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ ); // ================================================================ @@ -95,12 +96,12 @@ module sample_ntt_sync_shared #(parameter K = 4) ( // ================================================================ // Registered inputs (captured on valid_i) // ================================================================ - /* verilator lint_off UNUSED */ + /* verilator lint_off UNUSEDSIGNAL */ reg [255:0] rho_r; reg [2:0] k_r; reg [1:0] i_r; reg [1:0] j_r; - /* verilator lint_on UNUSED */ + /* verilator lint_on UNUSEDSIGNAL */ // ================================================================ // Coefficient counter (0..256). 9 bits to avoid overflow when @@ -223,6 +224,10 @@ module sample_ntt_sync_shared #(parameter K = 4) ( // d2 rejected, no longer need coefficients, or d2 was just accepted out. wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) && (!d2_acc_r || !need_more || (valid_o_r && ready_i)); + // `need_more` observes the registered counter before the final d2 + // handshake increments it. Detect that boundary explicitly so a full + // rate block does not launch an unnecessary final re-permutation. + wire completes_output = valid_o_r && ready_i && (coeff_cnt_r == 9'd255); // ================================================================ // FSM: state_next (combinational) @@ -248,7 +253,7 @@ module sample_ntt_sync_shared #(parameter K = 4) ( // next group in this block, re-permute (block exhausted), or // finish. if (grp_done) begin - if (!need_more) + if (!need_more || completes_output) state_next = ST_DONE; else if (grp_ptr_r < GRP_MAX) state_next = ST_SQUEEZE; // next group, no re-permute diff --git a/rtl/soc_top.v b/rtl/soc_top.v index 1bb6d80..fbe4d95 100644 --- a/rtl/soc_top.v +++ b/rtl/soc_top.v @@ -635,9 +635,10 @@ wire [31:0] matmul_ep_rd_data; wire matmul_ep_rd_ready; wire matmul_direct_start_valid; wire matmul_direct_start_bank; -wire matmul_direct_done_clear; -wire matmul_direct_crc_clear; -wire matmul_direct_done; +wire matmul_direct_done_clear; +wire matmul_direct_crc_clear; +wire matmul_direct_done; +wire matmul_direct_crc_idle; wire mlkem_intr; AxiCrossbar_2x8 u_AxiCrossbar_2x8 ( @@ -1577,8 +1578,9 @@ matmul_axi_slave u_matmul_axi_slave ( .mat_start_valid(matmul_direct_start_valid), .mat_start_bank (matmul_direct_start_bank), .mat_done_clear (matmul_direct_done_clear), - .mat_crc_clear (matmul_direct_crc_clear), - .mat_done (matmul_direct_done), + .mat_crc_clear (matmul_direct_crc_clear), + .mat_done (matmul_direct_done), + .mat_crc_idle (matmul_direct_crc_idle), .intr (matmul_intr) ); @@ -1828,8 +1830,9 @@ cdma_matmul_bridge #( .mat_start_valid (matmul_direct_start_valid), .mat_start_bank (matmul_direct_start_bank), .mat_done_clear (matmul_direct_done_clear), - .mat_crc_clear (matmul_direct_crc_clear), - .mat_done (matmul_direct_done), + .mat_crc_clear (matmul_direct_crc_clear), + .mat_done (matmul_direct_done), + .mat_crc_idle (matmul_direct_crc_idle), .seq_done_irq (cdma_matmul_seq_irq) );