From e30a65bd72c63fe17ce82f5454da3a141bef80f4 Mon Sep 17 00:00:00 2001
From: FallenSigh <fallensigh@gmail.com>
Date: Sat, 11 Jul 2026 07:42:35 +0800
Subject: [PATCH] perf(mlkem): pipeline CBD->bank write to cut 100 MHz critical
 path

At 100 MHz the worst path (WNS -1.58 ns) was CBD sample: squeeze_buf[buf_ptr]
extraction -> CBD add/sub -> mod-Q -> bank_t/bank_se write, all combinational
in one cycle and routing-dominated (squeeze_buf and the banks place far apart).
Register the CBD write (data/addr/enable) so the bank write commits one cycle
later; the registered write takes priority in the bank_se/bank_t mux so the
1-cycle spillover into the next state uses the correct addr/data. No consumer
reads these coeffs until the following stage (>1 cycle later), so no RAW hazard.

Verified byte-exact vs NIST KAT: Encaps 9/9, Decaps 9/9, KeyGen 11/11,
hello_world e2e -- all PASS.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rtl/ip/mlkem/top/mlkem_top.v | 44 +++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v
index 306df7a..04362d8 100644
--- a/rtl/ip/mlkem/top/mlkem_top.v
+++ b/rtl/ip/mlkem/top/mlkem_top.v
@@ -312,15 +312,15 @@ module mlkem_top #(
     // D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT].
     wire dec_v_we  = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo;
     wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx;
-    assign bt_we = ((st == ST_M) && pm_vo) || e2_we ||
+    assign bt_we = ((st == ST_M) && pm_vo) || cbdw_bt_en ||
                    u_psum_we || u_intt_we || u_v_we || u_w_we;
-    assign bt_wa = e2_we ? (c_widx & ((1<<PT_AW)-1)) :
+    assign bt_wa = cbdw_bt_en ? ({6'd0, cbdw_widx} & ((1<<PT_AW)-1)) :
                    u_psum_we ? ((UPSUM*256 + u_oidx) & ((1<<PT_AW)-1)) :
                    u_intt_we ? (u_intt_wr[PT_AW-1:0]) :
                    u_v_we    ? (u_vadd_wr[PT_AW-1:0]) :
                    u_w_we    ? (u_wsub_wr[PT_AW-1:0]) :
                            ((m_i*256 + m_oidx) & ((1<<PT_AW)-1));
-    assign bt_wd = e2_we ? cbd_modq :
+    assign bt_wd = cbdw_bt_en ? cbdw_data :
                    u_psum_we ? u_accq :
                    u_intt_we ? ntt_coeff :
                    u_v_we    ? u_vq :
@@ -381,16 +381,16 @@ module mlkem_top #(
     // bank_se rel slot dec_poly (0..K-1), at coeff index dec_widx.
     wire dec_u_we  = (st == ST_DEC_DECOMP) && !dec_in_c2 && cd_vo;
     wire [13:0] dec_u_wr = {2'b0, dec_poly}*256 + dec_widx;
-    assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
-                    ((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
-                     (c_poly < {1'b0, k_r, 1'b0})) ||
+    assign bse_we = cbdw_se_en ||
                     (((st == ST_N) || (st == ST_ENC_N) || (st == ST_DEC_NTT)) && ntt_vo) ||
                     u_add_we || dec_u_we;
-    assign bse_wa = u_add_we ? u_add_uwr[PSE_AW-1:0] :
+    assign bse_wa = cbdw_se_en ? (({1'b0, cbdw_poly}*256 + cbdw_widx) & ((1<<PSE_AW)-1)) :
+                    u_add_we ? u_add_uwr[PSE_AW-1:0] :
                     dec_u_we  ? dec_u_wr[PSE_AW-1:0] :
                     (st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ((n_slot*256 + n_widx) & ((1<<PSE_AW)-1))
                                  : ((c_poly*256 + c_widx) & ((1<<PSE_AW)-1));
-    assign bse_wd = u_add_we ? u_uq :
+    assign bse_wd = cbdw_se_en ? cbdw_data :
+                    u_add_we ? u_uq :
                     dec_u_we  ? cd_out :
                     (st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ntt_coeff : cbd_modq;
     always @(posedge clk) begin
@@ -797,6 +797,34 @@ module mlkem_top #(
     // signed (two's complement) -> [0,Q): add Q when negative
     wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff;
 
+    // Pipeline the CBD -> bank write by one cycle. The squeeze_buf[buf_ptr]
+    // extraction -> CBD add/sub -> mod-Q -> BRAM-write route is long and
+    // routing-dominated (the mlkem_clk critical path at 100 MHz). Register the
+    // write data/addr/enable here; the bank write commits one cycle later. No
+    // consumer reads these coeffs until the following stage, which begins more
+    // than one cycle after the last CBD coeff, so there is no read-after-write
+    // hazard. The registered write takes priority in the bank_se/bank_t mux so
+    // the 1-cycle spillover into the next state uses the correct addr/data.
+    reg        cbdw_se_en, cbdw_bt_en;
+    reg [4:0]  cbdw_poly;
+    reg [7:0]  cbdw_widx;
+    reg [11:0] cbdw_data;
+    always @(posedge clk or negedge rst_n) begin
+        if (!rst_n) begin
+            cbdw_se_en <= 1'b0; cbdw_bt_en <= 1'b0;
+            cbdw_poly <= 5'd0; cbdw_widx <= 8'd0; cbdw_data <= 12'd0;
+        end else begin
+            cbdw_se_en <= ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
+                          ((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
+                           (c_poly < {1'b0, k_r, 1'b0}));
+            cbdw_bt_en <= (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
+                          (c_poly == {1'b0, k_r, 1'b0});
+            cbdw_poly  <= c_poly;
+            cbdw_widx  <= c_widx;
+            cbdw_data  <= cbd_modq;
+        end
+    end
+
     // ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ----
     // N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot).
     reg  [4:0]  n_slot;        // 0..2K (process s_hat[0..K-1] then e_hat[0..K-1])