From 75c350c1e4c5b42722caf5fc622d8606bf504a74 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sun, 28 Jun 2026 16:37:07 +0800 Subject: [PATCH] refactor(kg): registered read-ahead for ST_M accumulate (bank_se/bank_t) Third per-consumer step of stage 2b, the read-modify-write one. m_acc_src is now a registered read (m_acc_rd) whose address leads by pm_vo (m_oidx+1 when consuming) so the value lands exactly on the pm_vo cycle. Same-index RMW on bank_t is read-old: the running-t_hat source was written by the previous term and is long settled; read addr (m_oidx+1) leads write addr (m_oidx). 11/11 KAT PASS incl. K=3/4 j>0 paths, byte-exact. --- sync_rtl/top/mlkem_top.v | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/sync_rtl/top/mlkem_top.v b/sync_rtl/top/mlkem_top.v index fe81a4f..f17a1ed 100644 --- a/sync_rtl/top/mlkem_top.v +++ b/sync_rtl/top/mlkem_top.v @@ -506,10 +506,20 @@ module mlkem_top #( // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]. // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i. // t_hat[i] lives in bank_t at relative slot m_i. - wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_oidx; // K+m_i - wire [13:0] m_tacc_full = m_i*256 + m_oidx; - wire [11:0] m_acc_src = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]] - : bank_t [m_tacc_full[PT_AW-1:0]]; + // Registered read-ahead: present the index that the NEXT pm_vo will consume + // m_acc_radr = pm_vo ? m_oidx+1 : m_oidx + // and register the selected bank into m_acc_rd, consumed one cycle later when + // pm_vo is high. The same-index read-modify-write on bank_t is read-old: + // the running t_hat source was written by the previous (j-1) term, long + // settled; the value consumed this cycle was registered before this cycle's + // write to m_oidx. Cadence is CALC(vo=0)/C0(vo=1)/C1(vo=1). + wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx; + wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i + wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; + wire [11:0] m_acc_rd_next = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]] + : bank_t [m_tacc_full[PT_AW-1:0]]; + reg [11:0] m_acc_rd; + wire [11:0] m_acc_src = m_acc_rd; // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0]; @@ -566,6 +576,7 @@ module mlkem_top #( pm_valid <= 1'b0; pm_a_rd <= 12'd0; pm_b_rd <= 12'd0; + m_acc_rd <= 12'd0; e_poly <= 3'd0; e_pair <= 8'd0; e_byte <= 2'd0; @@ -752,6 +763,11 @@ module mlkem_top #( // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ---- if (st == ST_M) begin + // registered accumulator read (sd_bram timing): captures the + // index the next pm_vo will consume. Free-running within ST_M; + // primed during LOAD (m_oidx held at 0). + m_acc_rd <= m_acc_rd_next; + // LOAD: present read-ahead addr to bank_a/bank_se; the pair // registered last cycle (pm_a_rd/pm_b_rd) is consumed by // poly_mul this cycle (pm_valid). poly_mul holds ready high