fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix samples ntt bugs & set PLL & optimize matmul

Browse Source
This commit is contained in:
FallenSigh
2026-07-15 10:36:38 +08:00
parent fc91511950
commit c4d36515cc
7 changed files with 274 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
rtl/ip/PLL_2019_2/clk_pll.xci
View File

@@ -35,6 +35,12 @@
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT2.FREQ_HZ">100000000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT2.INSERT_VIP">0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT2.PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.ASSOCIATED_BUSIF"/>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.ASSOCIATED_RESET"/>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.CLK_DOMAIN"/>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.FREQ_HZ">100000000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.INSERT_VIP">0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.CLOCK_CLK_OUT3.PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.INTR.PortWidth">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.INTR.SENSITIVITY">LEVEL_HIGH</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="BUSIFPARAM_VALUE.REF_CLK.ASSOCIATED_BUSIF"/>
@@ -136,7 +142,7 @@
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT3_REQUESTED_OUT_FREQ">100.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT3_REQUESTED_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT3_SEQUENCE_NUMBER">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT3_USED">0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT3_USED">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT4_1">0000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT4_2">0000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_CLKOUT4_ACTUAL_FREQ">100.000</spirit:configurableElementValue>
@@ -249,7 +255,7 @@
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKFBOUT_MULT_F">20.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKFBOUT_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKFBOUT_USE_FINE_PS">FALSE</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKIN1_PERIOD">20.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKIN1_PERIOD">20.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKIN2_PERIOD">10.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT0_DIVIDE_F">30.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT0_DUTY_CYCLE">0.500</spirit:configurableElementValue>
@@ -259,11 +265,7 @@
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT1_DUTY_CYCLE">0.500</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT1_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT1_USE_FINE_PS">FALSE</spirit:configurableElementValue>
<<<<<<< Updated upstream
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT2_DIVIDE">1</spirit:configurableElementValue>
=======
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT2_DIVIDE">13</spirit:configurableElementValue>
>>>>>>> Stashed changes
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT2_DUTY_CYCLE">0.500</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT2_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_CLKOUT2_USE_FINE_PS">FALSE</spirit:configurableElementValue>
@@ -293,18 +295,12 @@
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_MMCM_STARTUP_WAIT">FALSE</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_M_MAX">64.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_M_MIN">2.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_NUM_OUT_CLKS">2</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_NUM_OUT_CLKS">3</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW0A"> Output Output Phase Duty Cycle Pk-to-Pk Phase</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW0B"> Clock Freq (MHz) (degrees) (%) Jitter (ps) Error (ps)</spirit:configurableElementValue>
<<<<<<< Updated upstream
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW1">_cpu_clk__32.97872______0.000______50.0______130.093____123.600</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW2">_sys_clk__50.00000______0.000______50.0______122.035____123.600</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW3">no_CLK_OUT3_output</spirit:configurableElementValue>
=======
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW1">_cpu_clk__33.33333______0.000______50.0______215.691____164.985</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW2">_sys_clk__50.00000______0.000______50.0______192.113____164.985</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW3">mlkem_clk__76.92308______0.000______50.0______172.249____164.985</spirit:configurableElementValue>
>>>>>>> Stashed changes
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW4">no_CLK_OUT4_output</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW5">no_CLK_OUT5_output</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="MODELPARAM_VALUE.C_OUTCLK_SUM_ROW6">no_CLK_OUT6_output</spirit:configurableElementValue>
@@ -424,32 +420,32 @@
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKIN2_JITTER_PS">100.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKIN2_UI_JITTER">0.010</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_DRIVES">BUFG</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_JITTER">130.093</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_JITTER">243.156</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_MATCHED_ROUTING">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_PHASE_ERROR">123.600</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_PHASE_ERROR">144.334</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_REQUESTED_DUTY_CYCLE">50.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_REQUESTED_OUT_FREQ">33.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_REQUESTED_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_SEQUENCE_NUMBER">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT1_USED">true</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_DRIVES">BUFG</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_JITTER">122.035</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_JITTER">222.587</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_MATCHED_ROUTING">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_PHASE_ERROR">123.600</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_PHASE_ERROR">144.334</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_REQUESTED_DUTY_CYCLE">50.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_REQUESTED_OUT_FREQ">50.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_REQUESTED_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_SEQUENCE_NUMBER">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT2_USED">true</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_DRIVES">BUFG</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_JITTER">110.420</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_JITTER">186.330</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_MATCHED_ROUTING">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_PHASE_ERROR">123.600</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_PHASE_ERROR">144.334</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_REQUESTED_DUTY_CYCLE">50.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_REQUESTED_OUT_FREQ">100.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_REQUESTED_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_SEQUENCE_NUMBER">1</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_USED">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT3_USED">true</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT4_DRIVES">BUFG</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT4_JITTER">0.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.CLKOUT4_MATCHED_ROUTING">false</spirit:configurableElementValue>
@@ -538,7 +534,7 @@
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKFBOUT_MULT_F">20</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKFBOUT_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKFBOUT_USE_FINE_PS">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKIN1_PERIOD">20.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKIN1_PERIOD">20.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKIN2_PERIOD">10.0</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT0_DIVIDE_F">30</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT0_DUTY_CYCLE">0.500</spirit:configurableElementValue>
@@ -548,11 +544,7 @@
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT1_DUTY_CYCLE">0.500</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT1_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT1_USE_FINE_PS">false</spirit:configurableElementValue>
<<<<<<< Updated upstream
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT2_DIVIDE">1</spirit:configurableElementValue>
=======
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT2_DIVIDE">13</spirit:configurableElementValue>
>>>>>>> Stashed changes
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT2_DUTY_CYCLE">0.500</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT2_PHASE">0.000</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_CLKOUT2_USE_FINE_PS">false</spirit:configurableElementValue>
@@ -580,13 +572,8 @@
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_REF_JITTER1">0.010</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_REF_JITTER2">0.010</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.MMCM_STARTUP_WAIT">false</spirit:configurableElementValue>
<<<<<<< Updated upstream
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.NUM_OUT_CLKS">2</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.OVERRIDE_MMCM">false</spirit:configurableElementValue>
=======
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.NUM_OUT_CLKS">3</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.OVERRIDE_MMCM">true</spirit:configurableElementValue>
>>>>>>> Stashed changes
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.OVERRIDE_PLL">false</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.PHASESHIFT_MODE">WAVEFORM</spirit:configurableElementValue>
<spirit:configurableElementValue spirit:referenceId="PARAM_VALUE.PHASE_DUTY_CONFIG">false</spirit:configurableElementValue>
@@ -726,7 +713,6 @@
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT3_DRIVES" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT3_JITTER" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT3_PHASE_ERROR" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT3_REQUESTED_OUT_FREQ" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT3_USED" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT4_DRIVES" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLKOUT5_DRIVES" xilinx:valueSource="user"/>
@@ -735,7 +721,6 @@
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLK_OUT1_PORT" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLK_OUT2_PORT" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.CLK_OUT3_PORT" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.ENABLE_CLOCK_MONITOR" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.FEEDBACK_SOURCE" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.MMCM_BANDWIDTH" xilinx:valueSource="user"/>
<xilinx:configElementInfo xilinx:referenceId="PARAM_VALUE.MMCM_CLKFBOUT_MULT_F" xilinx:valueSource="user"/>

68
rtl/ip/cdma/cdma_matmul_bridge.sv
View File

@@ -64,6 +64,7 @@ module cdma_matmul_bridge #(
output logic mat_done_clear,
output logic mat_crc_clear,
input logic mat_done,
input logic mat_crc_idle,
output logic seq_done_irq
);
@@ -102,10 +103,10 @@ module cdma_matmul_bridge #(
logic [31:0] load_next_dst_addr;
logic [31:0] bank_dst_addr [2];
logic rd_busy;
logic wr_busy;
logic rd_active_bank;
logic wr_active_bank;
logic [1:0] rd_outstanding_count;
logic [1:0] rd_bank_fifo;
logic [1:0] compute_q;
logic [1:0] compute_q_count;
logic mat_done_seen;
@@ -136,6 +137,7 @@ module cdma_matmul_bridge #(
wire retire_from_store = seq_busy && cdma_wr_req_done;
wire retire_from_compute = seq_busy && seq_skip_writeback && mat_done_event && (compute_q_count != 2'd0);
wire [31:0] retire_inc = {31'd0, retire_from_store} + {31'd0, retire_from_compute};
wire rd_req_fire = cdma_rd_req_valid && cdma_rd_req_ready;
function automatic logic [31:0] apply_wstrb(
input logic [31:0] old_val,
@@ -257,10 +259,10 @@ module cdma_matmul_bridge #(
load_next_dst_addr <= 32'd0;
bank_dst_addr[0] <= 32'd0;
bank_dst_addr[1] <= 32'd0;
rd_busy <= 1'b0;
wr_busy <= 1'b0;
rd_active_bank <= 1'b0;
wr_active_bank <= 1'b0;
rd_outstanding_count <= 2'd0;
rd_bank_fifo <= 2'b00;
compute_q <= 2'b00;
compute_q_count <= 2'd0;
mat_done_seen <= 1'b0;
@@ -290,7 +292,7 @@ module cdma_matmul_bridge #(
if (done_clear_req)
seq_done_flag <= 1'b0;
if (cdma_rd_req_valid && cdma_rd_req_ready)
if (rd_req_fire)
cdma_rd_req_valid <= 1'b0;
if (cdma_wr_req_valid && cdma_wr_req_ready)
@@ -313,8 +315,9 @@ module cdma_matmul_bridge #(
load_next_dst_addr <= seq_dst_base;
bank_dst_addr[0] <= 32'd0;
bank_dst_addr[1] <= 32'd0;
rd_busy <= 1'b0;
wr_busy <= 1'b0;
rd_outstanding_count <= 2'd0;
rd_bank_fifo <= 2'b00;
compute_q <= 2'b00;
compute_q_count <= 2'd0;
mat_done_seen <= 1'b1;
@@ -325,18 +328,50 @@ module cdma_matmul_bridge #(
end else if (seq_busy) begin
if (retire_inc != 32'd0) begin
groups_retired <= groups_retired + retire_inc;
if ((groups_retired + retire_inc) >= seq_groups) begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
end
end
if (cdma_rd_req_done) begin
rd_busy <= 1'b0;
bank_load_active[rd_active_bank] <= 1'b0;
bank_ab_valid[rd_active_bank] <= 1'b1;
// The final compute completion can precede the recursive CRC
// consumer. Do not expose sequence completion until it drains.
if ((groups_retired >= seq_groups) && mat_crc_idle) begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
end
if (cdma_rd_req_done && (rd_outstanding_count != 2'd0)) begin
bank_load_active[rd_bank_fifo[0]] <= 1'b0;
bank_ab_valid[rd_bank_fifo[0]] <= 1'b1;
end
// Track the active read and one prefetched read. Completion
// pulses are in request order, so a two-entry bank FIFO is
// sufficient to associate each RLAST with its destination.
case ({rd_req_fire, cdma_rd_req_done})
2'b10: begin
if (rd_outstanding_count == 2'd0)
rd_bank_fifo[0] <= cdma_rd_req_ep_bank[0];
else
rd_bank_fifo[1] <= cdma_rd_req_ep_bank[0];
rd_outstanding_count <= rd_outstanding_count + 2'd1;
end
2'b01: begin
if (rd_outstanding_count == 2'd2)
rd_bank_fifo[0] <= rd_bank_fifo[1];
if (rd_outstanding_count != 2'd0)
rd_outstanding_count <= rd_outstanding_count - 2'd1;
end
2'b11: begin
if (rd_outstanding_count == 2'd1) begin
rd_bank_fifo[0] <= cdma_rd_req_ep_bank[0];
end else begin
rd_bank_fifo[0] <= rd_bank_fifo[1];
rd_bank_fifo[1] <= cdma_rd_req_ep_bank[0];
end
end
default: begin
end
endcase
if (cdma_wr_req_done) begin
wr_busy <= 1'b0;
bank_store_active[wr_active_bank] <= 1'b0;
@@ -354,15 +389,14 @@ module cdma_matmul_bridge #(
compute_q_count <= compute_q_count - 2'd1;
end
if (!rd_busy && !cdma_rd_req_valid && (load_next_idx < seq_groups) && load_target_free) begin
if ((rd_outstanding_count < 2'd2) && !cdma_rd_req_valid &&
(load_next_idx < seq_groups) && load_target_free) begin
cdma_rd_req_src <= load_next_src_addr;
cdma_rd_req_bytes <= MAT_AB_BYTES;
cdma_rd_req_ep_id <= 4'd0;
cdma_rd_req_ep_bank <= {3'b000, load_target_bank};
cdma_rd_req_ep_word_off <= 8'd0;
cdma_rd_req_valid <= 1'b1;
rd_busy <= 1'b1;
rd_active_bank <= load_target_bank;
bank_load_active[load_target_bank] <= 1'b1;
bank_group_idx[load_target_bank] <= load_next_idx;
bank_dst_addr[load_target_bank] <= load_next_dst_addr;

68
rtl/ip/cdma/snix_axil_cdma_mux.sv
View File

@@ -423,6 +423,13 @@ module snix_axil_cdma_mux #(
logic [7:0] ext_rd_word_addr_r;
logic [3:0] ext_rd_ep_id_r;
logic [3:0] ext_rd_ep_bank_r;
logic ext_rd_queue_valid;
logic ext_rd_queue_ar_sent;
logic [ADDR_WIDTH-1:0] ext_rd_queue_addr;
logic [31:0] ext_rd_queue_bytes;
logic [7:0] ext_rd_queue_word_addr;
logic [3:0] ext_rd_queue_ep_id;
logic [3:0] ext_rd_queue_ep_bank;
logic [ADDR_WIDTH-1:0] ext_wr_addr_r;
logic [31:0] ext_wr_bytes_left;
@@ -434,7 +441,10 @@ module snix_axil_cdma_mux #(
wire ext_wr_active = (ext_wr_state != EXT_WR_IDLE);
wire ext_rd_accept = ext_rd_req_valid && ext_rd_req_ready;
wire ext_wr_accept = ext_wr_req_valid && ext_wr_req_ready;
wire ext_any_busy_or_req = ext_rd_active || ext_wr_active || ext_rd_req_valid || ext_wr_req_valid;
wire ext_rd_prefetch_ar = (ext_rd_state == EXT_RD_READ) &&
ext_rd_queue_valid && !ext_rd_queue_ar_sent;
wire ext_any_busy_or_req = ext_rd_active || ext_rd_queue_valid || ext_wr_active ||
ext_rd_req_valid || ext_wr_req_valid;
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM));
// Dynamic routing to the engine based on current active channel
@@ -478,8 +488,12 @@ module snix_axil_cdma_mux #(
assign mm2mm_bready = ext_wr_active ? (ext_wr_state == EXT_WR_BRESP) : (use_ep_engine ? ep_bready : core_bready);
assign mm2mm_arid = ext_rd_active ? '0 : (use_ep_engine ? ep_arid : core_arid);
assign mm2mm_araddr = ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr);
assign mm2mm_arlen = ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) : (use_ep_engine ? ep_arlen : core_arlen);
assign mm2mm_araddr = ext_rd_prefetch_ar ? ext_rd_queue_addr :
(ext_rd_active ? ext_rd_addr_r : (use_ep_engine ? ep_araddr : core_araddr));
assign mm2mm_arlen = ext_rd_prefetch_ar ?
(((ext_rd_queue_bytes[31:2] == 30'd0) ? 8'd1 : ext_rd_queue_bytes[9:2]) - 8'd1) :
(ext_rd_active ? (((ext_rd_bytes_left[31:2] == 30'd0) ? 8'd1 : ext_rd_bytes_left[9:2]) - 8'd1) :
(use_ep_engine ? ep_arlen : core_arlen));
assign mm2mm_arsize = ext_rd_active ? 3'd2 : (use_ep_engine ? ep_arsize : core_arsize);
assign mm2mm_arburst = ext_rd_active ? 2'b01 : (use_ep_engine ? ep_arburst : core_arburst);
assign mm2mm_arlock = ext_rd_active ? 1'b0 : (use_ep_engine ? ep_arlock : core_arlock);
@@ -487,7 +501,8 @@ module snix_axil_cdma_mux #(
assign mm2mm_arprot = ext_rd_active ? 3'b0 : (use_ep_engine ? ep_arprot : core_arprot);
assign mm2mm_arqos = ext_rd_active ? 4'b0 : (use_ep_engine ? ep_arqos : core_arqos);
assign mm2mm_aruser = ext_rd_active ? '0 : (use_ep_engine ? ep_aruser : core_aruser);
assign mm2mm_arvalid = ext_rd_active ? (ext_rd_state == EXT_RD_AR) : (use_ep_engine ? ep_arvalid : core_arvalid);
assign mm2mm_arvalid = ext_rd_active ? ((ext_rd_state == EXT_RD_AR) || ext_rd_prefetch_ar) :
(use_ep_engine ? ep_arvalid : core_arvalid);
assign mm2mm_rready = ext_rd_active ? ((ext_rd_state == EXT_RD_READ) && ep_wr_ready) : (use_ep_engine ? ep_rready : core_rready);
always_ff @(posedge clk) begin
@@ -597,7 +612,10 @@ module snix_axil_cdma_mux #(
// ==========================================
// Generic external endpoint requester
// ==========================================
assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_rd_active;
assign ext_rd_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active &&
((ext_rd_state == EXT_RD_IDLE) ||
((ext_rd_state == EXT_RD_READ) && !ext_rd_queue_valid &&
!ext_rd_done_fire));
assign ext_wr_req_ready = (state == IDLE) && (ep_state == EP_IDLE) && !ext_wr_active;
assign ext_rd_req_done = ext_rd_done_fire;
assign ext_wr_req_done = ext_wr_done_fire;
@@ -610,7 +628,17 @@ module snix_axil_cdma_mux #(
ext_rd_word_addr_r <= '0;
ext_rd_ep_id_r <= '0;
ext_rd_ep_bank_r <= '0;
ext_rd_queue_valid <= 1'b0;
ext_rd_queue_ar_sent <= 1'b0;
ext_rd_queue_addr <= '0;
ext_rd_queue_bytes <= '0;
ext_rd_queue_word_addr <= '0;
ext_rd_queue_ep_id <= '0;
ext_rd_queue_ep_bank <= '0;
end else begin
if (ext_rd_prefetch_ar && mm2mm_arvalid && mm2mm_arready)
ext_rd_queue_ar_sent <= 1'b1;
case (ext_rd_state)
EXT_RD_IDLE: begin
if (ext_rd_accept) begin
@@ -629,11 +657,37 @@ module snix_axil_cdma_mux #(
end
EXT_RD_READ: begin
if (ext_rd_accept) begin
ext_rd_queue_valid <= 1'b1;
ext_rd_queue_ar_sent <= 1'b0;
ext_rd_queue_addr <= ext_rd_req_src;
ext_rd_queue_bytes <= ext_rd_req_bytes;
ext_rd_queue_word_addr <= ext_rd_req_ep_word_off;
ext_rd_queue_ep_id <= ext_rd_req_ep_id;
ext_rd_queue_ep_bank <= ext_rd_req_ep_bank;
end
if (mm2mm_rvalid && mm2mm_rready) begin
ext_rd_word_addr_r <= ext_rd_word_addr_r + 8'd1;
ext_rd_bytes_left <= (ext_rd_bytes_left > 32'd4) ? (ext_rd_bytes_left - 32'd4) : 32'd0;
if (mm2mm_rlast)
ext_rd_state <= EXT_RD_IDLE;
if (mm2mm_rlast) begin
if (ext_rd_queue_valid) begin
ext_rd_addr_r <= ext_rd_queue_addr;
ext_rd_bytes_left <= ext_rd_queue_bytes;
ext_rd_word_addr_r <= ext_rd_queue_word_addr;
ext_rd_ep_id_r <= ext_rd_queue_ep_id;
ext_rd_ep_bank_r <= ext_rd_queue_ep_bank;
ext_rd_queue_valid <= 1'b0;
ext_rd_queue_ar_sent <= 1'b0;
if (ext_rd_queue_ar_sent ||
(ext_rd_prefetch_ar && mm2mm_arvalid && mm2mm_arready))
ext_rd_state <= EXT_RD_READ;
else
ext_rd_state <= EXT_RD_AR;
end else begin
ext_rd_state <= EXT_RD_IDLE;
end
end
end
end

146
rtl/ip/matmul/matmul_axi_slave.v
View File

@@ -102,6 +102,7 @@ module matmul_axi_slave (
input mat_done_clear,
input mat_crc_clear,
output mat_done,
output mat_crc_idle,
// Interrupt output (active high level, pulsed by calc_done)
output intr
@@ -184,22 +185,44 @@ module matmul_axi_slave (
reg [31:0] crc32_table [0:255];
reg [31:0] crc_reg;
function [31:0] crc32_word_next;
function [31:0] crc32_update_word;
input [31:0] crc_in;
input [31:0] data_in;
reg [31:0] t0, c0, t1, c1, t2, c2, t3;
reg [31:0] c0;
reg [31:0] c1;
reg [31:0] c2;
begin
t0 = crc32_table[(crc_in[7:0] ^ data_in[7:0])];
c0 = t0 ^ {8'h00, crc_in[31:8]};
t1 = crc32_table[(c0[7:0] ^ data_in[15:8])];
c1 = t1 ^ {8'h00, c0[31:8]};
t2 = crc32_table[(c1[7:0] ^ data_in[23:16])];
c2 = t2 ^ {8'h00, c1[31:8]};
t3 = crc32_table[(c2[7:0] ^ data_in[31:24])];
crc32_word_next = t3 ^ {8'h00, c2[31:8]};
c0 = crc32_table[(crc_in[7:0] ^ data_in[7:0])] ^ {8'h00, crc_in[31:8]};
c1 = crc32_table[(c0[7:0] ^ data_in[15:8])] ^ {8'h00, c0[31:8]};
c2 = crc32_table[(c1[7:0] ^ data_in[23:16])] ^ {8'h00, c1[31:8]};
crc32_update_word = crc32_table[(c2[7:0] ^ data_in[31:24])] ^
{8'h00, c2[31:8]};
end
endfunction
function [31:0] crc32_update_6bytes;
input [31:0] crc_in;
input [47:0] data_in;
reg [31:0] c;
integer byte_idx;
begin
c = crc_in;
for (byte_idx = 0; byte_idx < 6; byte_idx = byte_idx + 1)
c = crc32_table[(c[7:0] ^ data_in[byte_idx*8 +: 8])] ^
{8'h00, c[31:8]};
crc32_update_6bytes = c;
end
endfunction
localparam CRC_FIFO_DEPTH = 32;
reg [65:0] crc_fifo [0:CRC_FIFO_DEPTH-1];
reg [4:0] crc_fifo_wr_ptr;
reg [4:0] crc_fifo_rd_ptr;
reg [5:0] crc_fifo_count;
reg crc_c66_busy;
reg [65:0] crc_c66_data;
reg [31:0] crc_c66_half;
initial begin
crc32_table[0 ] = 32'h00000000; crc32_table[1 ] = 32'h77073096;
crc32_table[2 ] = 32'hEE0E612C; crc32_table[3 ] = 32'h990951BA;
@@ -587,10 +610,21 @@ module matmul_axi_slave (
reg [4:0] pipe_cnt;
wire [65:0] calc_c_result = ({2'b0, mult_p[0]} + {2'b0, mult_p[1]}) +
({2'b0, mult_p[2]} + {2'b0, mult_p[3]});
wire calc_crc_valid = acc_valid_d4;
wire [31:0] crc_calc_0 = crc32_word_next(crc_reg, calc_c_result[31:0]);
wire [31:0] crc_calc_1 = crc32_word_next(crc_calc_0, calc_c_result[63:32]);
wire [31:0] crc_calc_2 = crc32_word_next(crc_calc_1, {30'b0, calc_c_result[65:64]});
wire crc_fifo_empty = (crc_fifo_count == 6'd0);
wire crc_fifo_push = acc_valid_d4;
wire crc_fifo_pop = !crc_c66_busy && !crc_fifo_empty;
wire [31:0] crc_c66_first_next =
crc32_update_6bytes(crc_reg, crc_fifo[crc_fifo_rd_ptr][47:0]);
wire [47:0] crc_c66_second_data = {30'b0, crc_c66_data[65:48]};
wire [31:0] crc_c66_final_next =
crc32_update_6bytes(crc_c66_half, crc_c66_second_data);
// Reserve space for results already in the four-stage multiplier pipeline.
wire crc_issue_ready = (crc_fifo_count < 6'd24);
assign mat_crc_idle = crc_fifo_empty && !crc_c66_busy &&
!acc_valid_d1 && !acc_valid_d2 &&
!acc_valid_d3 && !acc_valid_d4;
task start_calc_bank;
input bank;
@@ -616,14 +650,46 @@ module matmul_axi_slave (
end
endtask
// CRC tracks completed C elements directly, independent of result writeback.
// Buffer completed C elements so the recursive CRC path is split across
// two cycles without reducing the multiplier's burst throughput.
always @(posedge clk) begin
if (!resetn) begin
crc_reg <= 32'hFFFFFFFF;
crc_reg <= 32'hFFFFFFFF;
crc_fifo_wr_ptr <= 5'd0;
crc_fifo_rd_ptr <= 5'd0;
crc_fifo_count <= 6'd0;
crc_c66_busy <= 1'b0;
crc_c66_data <= 66'd0;
crc_c66_half <= 32'd0;
end else if (crc_clear_pulse) begin
crc_reg <= 32'hFFFFFFFF;
end else if (calc_crc_valid) begin
crc_reg <= crc_calc_2;
crc_reg <= 32'hFFFFFFFF;
crc_fifo_wr_ptr <= 5'd0;
crc_fifo_rd_ptr <= 5'd0;
crc_fifo_count <= 6'd0;
crc_c66_busy <= 1'b0;
crc_c66_data <= 66'd0;
crc_c66_half <= 32'd0;
end else begin
if (crc_fifo_push) begin
crc_fifo[crc_fifo_wr_ptr] <= calc_c_result;
crc_fifo_wr_ptr <= crc_fifo_wr_ptr + 5'd1;
end
case ({crc_fifo_push, crc_fifo_pop})
2'b10: crc_fifo_count <= crc_fifo_count + 6'd1;
2'b01: crc_fifo_count <= crc_fifo_count - 6'd1;
default: crc_fifo_count <= crc_fifo_count;
endcase
if (crc_c66_busy) begin
crc_reg <= crc_c66_final_next;
crc_c66_busy <= 1'b0;
end else if (crc_fifo_pop) begin
crc_c66_data <= crc_fifo[crc_fifo_rd_ptr];
crc_c66_half <= crc_c66_first_next;
crc_c66_busy <= 1'b1;
crc_fifo_rd_ptr <= crc_fifo_rd_ptr + 5'd1;
end
end
end
@@ -676,21 +742,25 @@ module matmul_axi_slave (
end
C_RUN: begin
// Issue 4 new operands every cycle (pipe_cnt 0-15, 16 rounds)
// Issue one C element per cycle while the CRC result FIFO
// has room for this result and the in-flight pipeline.
if (pipe_cnt < 5'd16) begin
acc_idx_d1 <= pipe_cnt[3:0];
acc_bank_d1 <= cur_bank;
acc_valid_d1 <= 1'b1;
if (cur_bank == 1'b0) begin
m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}];
m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}];
m_in_a[2] <= b0_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b0_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b0_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b0_b[{2'd3, pipe_cnt[1:0]}];
end else begin
m_in_a[0] <= b1_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b1_b[{2'd0, pipe_cnt[1:0]}];
m_in_a[1] <= b1_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b1_b[{2'd1, pipe_cnt[1:0]}];
m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}];
if (crc_issue_ready) begin
acc_idx_d1 <= pipe_cnt[3:0];
acc_bank_d1 <= cur_bank;
acc_valid_d1 <= 1'b1;
if (cur_bank == 1'b0) begin
m_in_a[0] <= b0_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b0_b[{2'd0, pipe_cnt[1:0]}];
m_in_a[1] <= b0_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b0_b[{2'd1, pipe_cnt[1:0]}];
m_in_a[2] <= b0_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b0_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b0_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b0_b[{2'd3, pipe_cnt[1:0]}];
end else begin
m_in_a[0] <= b1_a[{pipe_cnt[3:2], 2'd0}]; m_in_b[0] <= b1_b[{2'd0, pipe_cnt[1:0]}];
m_in_a[1] <= b1_a[{pipe_cnt[3:2], 2'd1}]; m_in_b[1] <= b1_b[{2'd1, pipe_cnt[1:0]}];
m_in_a[2] <= b1_a[{pipe_cnt[3:2], 2'd2}]; m_in_b[2] <= b1_b[{2'd2, pipe_cnt[1:0]}];
m_in_a[3] <= b1_a[{pipe_cnt[3:2], 2'd3}]; m_in_b[3] <= b1_b[{2'd3, pipe_cnt[1:0]}];
end
pipe_cnt <= pipe_cnt + 5'd1;
end
end
@@ -706,16 +776,14 @@ module matmul_axi_slave (
calc_done_pulse <= 1'b1;
end
if (pipe_cnt < 5'd16) begin
pipe_cnt <= pipe_cnt + 5'd1;
end else if (cur_bank == 1'b0 && req_1) begin
if (pipe_cnt >= 5'd16 && cur_bank == 1'b0 && req_1) begin
start_calc_bank(1'b1);
end else if (cur_bank == 1'b1 && req_0) begin
end else if (pipe_cnt >= 5'd16 && cur_bank == 1'b1 && req_0) begin
start_calc_bank(1'b0);
end else if (acc_valid_d4 && acc_idx_d4 == 4'd15) begin
end else if (pipe_cnt >= 5'd16 && acc_valid_d4 && acc_idx_d4 == 4'd15) begin
busy <= 1'b0;
state <= C_IDLE;
end else begin
end else if (pipe_cnt >= 5'd16) begin
pipe_cnt <= pipe_cnt + 5'd1;
end
end

23
rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v
View File

@@ -30,15 +30,16 @@
`include "defines.vh"
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDPARAM */
(* use_dsp = "no" *)
module sample_ntt_sync #(parameter K = 4) (
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDPARAM */
input clk,
input rst_n,
input [255:0] rho_i,
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
input [2:0] k_i,
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
input [1:0] i_idx,
input [1:0] j_idx,
input valid_i,
@@ -72,12 +73,12 @@ module sample_ntt_sync #(parameter K = 4) (
// ================================================================
// Registered inputs (captured on valid_i)
// ================================================================
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
reg [255:0] rho_r;
reg [2:0] k_r;
reg [1:0] i_r;
reg [1:0] j_r;
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
// ================================================================
// Coefficient counter (0..256). 9 bits to avoid overflow when
@@ -163,9 +164,9 @@ module sample_ntt_sync #(parameter K = 4) (
// ================================================================
wire kc_valid_i;
wire [1599:0] kc_state_i;
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
wire kc_ready_o;
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_valid_o;
@@ -209,6 +210,10 @@ module sample_ntt_sync #(parameter K = 4) (
// d2 rejected, no longer need coefficients, or d2 was just accepted out.
wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) &&
(!d2_acc_r || !need_more || (valid_o_r && ready_i));
// `need_more` observes the registered counter before the final d2
// handshake increments it. Detect that boundary explicitly so a full
// rate block does not launch an unnecessary final re-permutation.
wire completes_output = valid_o_r && ready_i && (coeff_cnt_r == 9'd255);
// ================================================================
// FSM: state_next (combinational)
@@ -234,7 +239,7 @@ module sample_ntt_sync #(parameter K = 4) (
// next group in this block, re-permute (block exhausted), or
// finish.
if (grp_done) begin
if (!need_more)
if (!need_more || completes_output)
state_next = ST_DONE;
else if (grp_ptr_r < GRP_MAX)
state_next = ST_SQUEEZE; // next group, no re-permute

23
rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v
View File

@@ -43,15 +43,16 @@
`include "defines.vh"
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDPARAM */
(* use_dsp = "no" *)
module sample_ntt_sync_shared #(parameter K = 4) (
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDPARAM */
input clk,
input rst_n,
input [255:0] rho_i,
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
input [2:0] k_i,
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
input [1:0] i_idx,
input [1:0] j_idx,
input valid_i,
@@ -67,9 +68,9 @@ module sample_ntt_sync_shared #(parameter K = 4) (
output kc_ready_i,
output [1599:0] kc_state_i,
output kc_valid_i,
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
input kc_ready_o
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
);
// ================================================================
@@ -95,12 +96,12 @@ module sample_ntt_sync_shared #(parameter K = 4) (
// ================================================================
// Registered inputs (captured on valid_i)
// ================================================================
/* verilator lint_off UNUSED */
/* verilator lint_off UNUSEDSIGNAL */
reg [255:0] rho_r;
reg [2:0] k_r;
reg [1:0] i_r;
reg [1:0] j_r;
/* verilator lint_on UNUSED */
/* verilator lint_on UNUSEDSIGNAL */
// ================================================================
// Coefficient counter (0..256). 9 bits to avoid overflow when
@@ -223,6 +224,10 @@ module sample_ntt_sync_shared #(parameter K = 4) (
// d2 rejected, no longer need coefficients, or d2 was just accepted out.
wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) &&
(!d2_acc_r || !need_more || (valid_o_r && ready_i));
// `need_more` observes the registered counter before the final d2
// handshake increments it. Detect that boundary explicitly so a full
// rate block does not launch an unnecessary final re-permutation.
wire completes_output = valid_o_r && ready_i && (coeff_cnt_r == 9'd255);
// ================================================================
// FSM: state_next (combinational)
@@ -248,7 +253,7 @@ module sample_ntt_sync_shared #(parameter K = 4) (
// next group in this block, re-permute (block exhausted), or
// finish.
if (grp_done) begin
if (!need_more)
if (!need_more || completes_output)
state_next = ST_DONE;
else if (grp_ptr_r < GRP_MAX)
state_next = ST_SQUEEZE; // next group, no re-permute