build(vivado): add sd_bram to create_project.tcl, fix run instructions

mlkem_top instantiates sd_bram (ek/dk byte buffers) but create_project.tcl
was missing it -- elaboration would fail. Add sync_rtl/storage/sd_bram.v to
match xsim_run.tcl exactly. Also clarify the run flow: runtime=all means
launch_simulation runs to the TB $finish; a manual 'run all' afterwards trips
the 120ms watchdog and prints a spurious timeout.

Verified: vivado -mode batch builds the project (0 errors) and the simulation
prints 'K=2 CASE 0 PASS: ek==pk, dk==sk'.

.claude/plans/phase2_polymem_bram.md

@@ -0,0 +1,47 @@
+# Phase 2: polymem -> BRAM banks
+
+## Goal
+Replace the single multi-port async-read `reg [11:0] polymem [0:28*256-1]` with
+registered-read `sd_bram` banks that infer real BRAM (ASIC: compiled SRAM).
+
+## Bank split (all sd_bram, W=12, 1R+1W registered)
+- bank_a : A_hat[i][j], slots 0..K^2-1   -> D=4096 (KMAX^2*256), A=12
+- bank_se: s_hat[i] then e_hat[i]         -> D=2048 (2*KMAX*256), A=11
+            relative slot = abs_slot - slot_s_rt   (s: 0..K-1, e: K..2K-1)
+- bank_t : t_hat[i], slots slot_t_rt..    -> D=1024 (KMAX*256), A=10
+            relative slot = abs_slot - slot_t_rt
+
+## Port budget (verified disjoint)
+- poly_mul LOAD vs ACCUMULATE never overlap; ST_N/ST_M/ST_E are disjoint
+  top-states. => each bank needs only 1R+1W. sd_bram fits.
+- bank_t ST_M-acc reads t_hat[idx] (j>0 source) and writes t_hat[idx] result
+  same cycle: 1R+1W, read-old/write-new. Prefetch keeps read AHEAD of write
+  (monotonic idx), so no same-pass RAW hazard.
+
+## Read sites (all need 1-cycle read-ahead vs async today)
+- ntt_in       (ST_N load)      <- bank_se[n_slot]
+- e_c0,e_c1    (ST_E)           <- bank_se (dk) or bank_t (ek); SERIALIZE the
+                                   two reads across the 3-byte/pair window
+- pm_a_in      (ST_M load)      <- bank_a[m_aslot]
+- pm_b_in      (ST_M load)      <- bank_se[m_j]
+- m_acc_src    (ST_M acc)       <- bank_se[K+m_i] (j==0) or bank_t[m_i] (j>0)
+- dbg_coeff_o  (not on KAT path) <- route by slot range (compile-only)
+
+## Write sites
+- ST_A: bank_a[a_slot]            <= snt_coeff
+- ST_C: bank_se[c_slot-slot_s]    <= cbd_modq
+- ST_N: bank_se[n_slot]           <= ntt_coeff   (write-back, same slot)
+- ST_M-acc: bank_t[m_i]           <= m_accq
+
+## Read-ahead strategy
+Cores hold ready_o high through entire LOAD (no mid-stream backpressure) =>
+fixed 1-cycle skew. Pattern: advance a read-address pointer 1 cycle ahead of
+the consumer index; delay the consumer's valid by 1 cycle ("prime the pipe").
+For ST_M-acc (irregular pm_vo cadence 0,1,1): 2-entry prefetch skid buffer,
+read pointer runs monotonically ahead of write pointer.
+
+## Checkpoints (KAT-gated: 11 cases, byte-exact, 0 file-not-found)
+- 2a: split polymem -> 4 ASYNC-read banks (pure refactor, zero timing change).
+      Validates bank sizing + base-relative addressing + debug mux. COMMIT.
+- 2b: convert banks to registered sd_bram + add read-ahead pipelines to every
+      consumer FSM (ST_N, ST_E, ST_M load, ST_M acc). COMMIT.

create_project.tcl

@@ -2,7 +2,7 @@
 # 顶层 mlkem_top 及其全部叶子算子与参数化 KAT testbench。
 #
 # 与已验证的 XSIM 流程（sync_rtl/top/TB/xsim_run.tcl）保持一致：
-#   - 仅加载 mlkem_top 实际依赖的 14 个源文件
+#   - 加载 mlkem_top 实际依赖的全部 RTL 源文件（含 sd_bram 存储）
 #   - 顶层仿真模块 = tb_mlkem_kg_katK_xsim
 #   - 运行时安全等级由 generic KP（2/3/4）选择，用例号由 +CASE 选择
 #
@@ -30,7 +30,7 @@ create_project -force ${PROJECT_NAME} ${PROJECT_DIR}/vivado_prj
 set_property target_simulator XSim [current_project]
 
 # ===================================================================
-# RTL 源文件 —— 与 xsim_run.tcl 完全一致的 14 个文件
+# RTL 源文件 —— 与 xsim_run.tcl 完全一致
 # ===================================================================
 
 # ── SHA3 / Keccak ──
@@ -53,6 +53,9 @@ read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/basecase_mul.v
 read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/poly_mul_zeta_rom.v
 read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/poly_mul_sync.v
 
+# ── 存储（ek/dk 字节缓冲，registered-read BRAM）──
+read_verilog -sv ${PROJECT_DIR}/sync_rtl/storage/sd_bram.v
+
 # ── 顶层 KeyGen 集成 ──
 read_verilog -sv ${PROJECT_DIR}/sync_rtl/top/mlkem_top.v
 
@@ -107,7 +110,13 @@ set_property -name {xsim.simulate.xsim.more_options} \
 puts "========================================"
 puts " Project created: ${PROJECT_DIR}/vivado_prj/${PROJECT_NAME}.xpr"
 puts " 仿真配置: ML-KEM K=${SIM_KP}, KAT CASE=${SIM_CASE}"
-puts " 运行仿真:  launch_simulation; run all"
+puts ""
+puts " 运行仿真（runtime=all，会自动跑到 TB 的 \$finish）："
+puts "   - GUI:    打开工程后点 Run Simulation，或 Tcl Console: launch_simulation"
+puts "   - batch:  vivado -mode batch -source create_project.tcl 后，"
+puts "             另开: open_project vivado_prj/mlkem.xpr; launch_simulation"
+puts "   （勿在 launch_simulation 后再手动 run all —— 仿真已到 \$finish，"
+puts "     再 run 会触发 TB 的 120ms 看门狗误报 timeout）"
 puts " 期望输出:  K=${SIM_KP} CASE ${SIM_CASE} PASS: ek==pk, dk==sk"
 puts ""
 puts " 切换配置（如 K=4）：编辑脚本顶部 SIM_KP/SIM_CASE 重跑本脚本，"

run_tb.sh

@@ -1,10 +1,18 @@
 #!/bin/bash
 # run_tb.sh - Run Vivado XSIM testbench for a module
 #
-# Usage: ./run_tb.sh <module>
-#        ./run_tb.sh mod_add
+# Usage: ./run_tb.sh <module> [K] [CASE]
+#        ./run_tb.sh mod_add              # run the module's full xsim_run.tcl
+#        ./run_tb.sh top                  # ML-KEM KeyGen: all K, all cases
+#        ./run_tb.sh top 2                # only K=2 (ML-KEM-512), all its cases
+#        ./run_tb.sh top 2 3              # only K=2, only CASE=3
+#        ./run_tb.sh top 4 0              # only K=4 (ML-KEM-1024), CASE=0
 #        ./run_tb.sh --list
 #
+# For the 'top' module, K (2/3/4) and CASE select a single KAT run so you can
+# iterate quickly instead of elaborating 3 snapshots and running all 11 cases.
+# Other modules ignore the extra args and run their xsim_run.tcl verbatim.
+#
 # Prerequisites:
 #   Vivado 2019.2 at /opt/Xilinx/Vivado/2019.2/
 
@@ -31,6 +39,8 @@ if [ -z "$1" ]; then
 fi
 
 MODULE="$1"
+SEL_K="$2"          # optional: 2/3/4 (top module only)
+SEL_CASE="$3"       # optional: KAT case index (top module only)
 TB_DIR="$SCRIPT_DIR/sync_rtl/$MODULE/TB"
 TCL_FILE="$TB_DIR/xsim_run.tcl"
 
@@ -80,4 +90,48 @@ execute_tcl() {
     rm -f "$tmp_script"
 }
 
+# Fast single-K / single-case path for the 'top' (ML-KEM KeyGen) module.
+# Compiles the same RTL as the tcl, but elaborates ONLY the requested K and
+# runs ONLY the requested CASE(s). Avoids the 3-snapshot + 11-run full sweep.
+run_top_selected() {
+    local tcl_file="$1" k="$2" csel="$3"
+    set +e   # we do our own error handling / result parsing here
+
+    rm -rf xsim.dir .Xil
+
+    # Compile every xvlog line from the tcl, verbatim.
+    while read -r cmd; do
+        eval "$cmd" || { echo "COMPILE FAILED: $cmd"; return 1; }
+    done < <(grep -E '^xvlog ' "$tcl_file")
+
+    # Cases per K: K=2 has 0..4, K=3/4 have 0..2.
+    local cases
+    if [ "$k" = "2" ]; then cases="0 1 2 3 4"; else cases="0 1 2"; fi
+    if [ -n "$csel" ]; then cases="$csel"; fi
+
+    echo "  xelab tb_mlkem_kg_katK_xsim -generic_top KP=$k -s mlkem_kg_k$k --timescale 1ns/1ps"
+    xelab tb_mlkem_kg_katK_xsim -generic_top KP=$k -s mlkem_kg_k$k --timescale 1ns/1ps \
+        || { echo "ELAB FAILED for K=$k"; return 1; }
+
+    local fail=0
+    for c in $cases; do
+        local log="/tmp/run_tb_top_k${k}_c${c}.log"
+        echo "  xsim mlkem_kg_k$k -R -testplusarg CASE=$c"
+        echo "========================================" | tee "$log"
+        xsim "mlkem_kg_k$k" -R -testplusarg "CASE=$c" 2>&1 | tee -a "$log"
+        echo "========================================" | tee -a "$log"
+        local pf nf
+        pf=$(grep -oE 'PASS|FAIL' "$log" | tail -1)
+        nf=$(grep -c 'cannot be opened' "$log")
+        echo "    K=$k CASE=$c -> ${pf:-NORESULT}  (file-not-found=$nf, log: $log)"
+        { [ "$pf" = "PASS" ] && [ "$nf" -eq 0 ]; } || fail=1
+    done
+    return $fail
+}
+
+if [ "$MODULE" = "top" ] && [ -n "$SEL_K" ]; then
+    run_top_selected "$TCL_FILE" "$SEL_K" "$SEL_CASE"
+    exit $?
+fi
+
 execute_tcl "$TCL_FILE"

sync_rtl/top/mlkem_top.v

@@ -82,11 +82,40 @@ module mlkem_top #(
     localparam NUM_SLOTS = KMAX*KMAX + 3*KMAX;
     localparam SAW       = 5;             // slot-address width (>=clog2(28))
 
-    reg [11:0] polymem [0:NUM_SLOTS*256-1];
+    // ================================================================
+    // Coefficient storage as 3 banks (Phase 2). Was one async-read reg
+    // array polymem[0:28*256-1]; split by polynomial so each becomes a
+    // registered-read sd_bram (infers BRAM / ASIC SRAM) in the final step.
+    //   bank_a : A_hat[i][j]  slots 0..K^2-1   -> D=KMAX*KMAX*256=4096
+    //   bank_se: s_hat||e_hat slots slot_s..   -> D=2*KMAX*256=2048 (rel 0..2K-1)
+    //   bank_t : t_hat[i]     slots slot_t..   -> D=KMAX*256=1024  (rel 0..K-1)
+    // Addresses are base-relative: bank index = abs_slot - base_slot.
+    // Stage 2b (in progress): per-consumer registered read-ahead. Banks are
+    // still async reg arrays here; converted consumers read via their own
+    // 1-cycle pipeline reg (== sd_bram timing). Once every consumer of a
+    // bank is registered, the array is replaced by an sd_bram with its read
+    // port muxed across the (phase-disjoint) consumers.
+    // ================================================================
+    localparam PA_AW = 12;                // bank_a addr width (4096)
+    localparam PSE_AW = 11;               // bank_se addr width (2048)
+    localparam PT_AW = 10;                // bank_t addr width (1024)
+    reg [11:0] bank_a  [0:(1<<PA_AW)-1];
+    reg [11:0] bank_se [0:(1<<PSE_AW)-1];
+    reg [11:0] bank_t  [0:(1<<PT_AW)-1];
 
     // Debug readback (registered for timing)
     reg [11:0] dbg_coeff_r;
-    always @(posedge clk) dbg_coeff_r <= polymem[dbg_slot_i*256 + dbg_idx_i];
+    // Route debug read by absolute slot range (not on the KAT correctness
+    // path — TB reads ek/dk BRAMs only). A: 0..K^2-1; s/e: slot_s..slot_t-1;
+    // t: slot_t..  Index within bank is (abs_slot-base)*256 + idx.
+    wire [13:0] dbg_a_addr  = dbg_slot_i*256 + dbg_idx_i;
+    wire [13:0] dbg_se_addr = (dbg_slot_i - slot_s_rt)*256 + dbg_idx_i;
+    wire [13:0] dbg_t_addr  = (dbg_slot_i - slot_t_rt)*256 + dbg_idx_i;
+    always @(posedge clk) begin
+        if (dbg_slot_i >= slot_t_rt)      dbg_coeff_r <= bank_t [dbg_t_addr[PT_AW-1:0]];
+        else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]];
+        else                              dbg_coeff_r <= bank_a [dbg_a_addr[PA_AW-1:0]];
+    end
     assign dbg_coeff_o = dbg_coeff_r;
 
     // ek and dk_pke byte memories sized for KMAX.
@@ -377,17 +406,22 @@ module mlkem_top #(
     // ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ----
     // N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot).
     reg  [4:0]  n_slot;        // 0..2K (process s_hat[0..K-1] then e_hat[0..K-1])
-    reg  [8:0]  n_ridx;        // load read index 0..256
+    reg  [8:0]  n_ridx;        // load read-AHEAD pointer 0..256 (leads consume by 1)
     reg  [7:0]  n_widx;        // output write index 0..255
-    reg         n_valid;       // feeding coeffs to ntt_core
+    reg         n_valid;       // feeding coeffs to ntt_core (delayed 1 cyc vs n_ridx)
+    reg         n_loading;     // 1 while presenting load addresses to bank_se
     reg         n_pending;     // waiting for ntt_core IDLE to start next slot
+    reg  [11:0] n_rd_data;     // registered bank_se read (== sd_bram timing)
     wire [SAW-1:0] n_slot_addr = slot_s_rt + n_slot;   // s_hat then e_hat contiguous
 
     wire        ntt_ready;
     wire [11:0] ntt_coeff;
     wire        ntt_vo;
     wire        ntt_done;
-    wire [11:0] ntt_in = polymem[n_slot_addr*256 + n_ridx[7:0]];
+    // bank_se read addr for the NTT load (relative slot = n_slot); registered
+    // into n_rd_data, which feeds ntt_core 1 cycle later.
+    wire [13:0] ntt_rd_full = n_slot*256 + n_ridx[7:0];
+    wire [11:0] ntt_in = n_rd_data;
 
     ntt_core u_ntt (
         .clk(clk), .rst_n(rst_n),
@@ -423,9 +457,12 @@ module mlkem_top #(
     wire        e_is_dk = (e_poly >= {1'b0, k_r});
     wire [4:0]  e_pidx  = e_is_dk ? (e_poly - {1'b0, k_r}) : e_poly;   // index within target
     wire [SAW-1:0] e_slot = e_is_dk ? (slot_s_rt + e_pidx) : (slot_t_rt + e_pidx);
-    // two coeffs of the current pair
-    wire [11:0] e_c0 = polymem[e_slot*256 + {e_pair, 1'b0}];
-    wire [11:0] e_c1 = polymem[e_slot*256 + {e_pair, 1'b1}];
+    // two coeffs of the current pair: ek half reads t_hat (bank_t), dk half
+    // reads s_hat (bank_se). Relative index within the bank = e_pidx.
+    wire [13:0] e_rd0_full = e_pidx*256 + {e_pair, 1'b0};
+    wire [13:0] e_rd1_full = e_pidx*256 + {e_pair, 1'b1};
+    wire [11:0] e_c0 = e_is_dk ? bank_se[e_rd0_full[PSE_AW-1:0]] : bank_t[e_rd0_full[PT_AW-1:0]];
+    wire [11:0] e_c1 = e_is_dk ? bank_se[e_rd1_full[PSE_AW-1:0]] : bank_t[e_rd1_full[PT_AW-1:0]];
     // 3 packed bytes
     wire [7:0]  e_b0 = e_c0[7:0];
     wire [7:0]  e_b1 = {e_c1[3:0], e_c0[11:8]};
@@ -445,8 +482,15 @@ module mlkem_top #(
     wire         pm_ready;
     wire [11:0]  pm_coeff;
     wire         pm_vo;
-    wire [11:0]  pm_a_in = polymem[m_aslot*256 + m_ld[7:0]];
-    wire [11:0]  pm_b_in = polymem[m_sslot*256 + m_ld[7:0]];
+    // pm_a: A_hat[i][j] in bank_a (abs slot m_aslot). pm_b: s_hat[j] in
+    // bank_se (relative slot = m_sslot - slot_s_rt = m_j). m_ld is a read-ahead
+    // pointer; both bank reads are registered into pm_a_rd/pm_b_rd and fed to
+    // poly_mul one cycle later (pm_valid delayed 1 cyc), like ST_N.
+    wire [13:0] pm_a_full = m_aslot*256 + m_ld[7:0];
+    wire [13:0] pm_b_full = m_j*256 + m_ld[7:0];
+    reg  [11:0]  pm_a_rd, pm_b_rd;       // registered bank reads (sd_bram timing)
+    wire [11:0]  pm_a_in = pm_a_rd;
+    wire [11:0]  pm_b_in = pm_b_rd;
 
     poly_mul_sync u_pmul (
         .clk(clk), .rst_n(rst_n),
@@ -459,9 +503,23 @@ module mlkem_top #(
         .ready_i(1'b1)
     );
 
-    // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]
-    wire [11:0]  m_acc_src = (m_j == 2'd0) ? polymem[m_eslot*256 + m_oidx]
-                                           : polymem[m_tslot*256 + m_oidx];
+    // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i].
+    // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i.
+    // t_hat[i] lives in bank_t at relative slot m_i.
+    // Registered read-ahead: present the index that the NEXT pm_vo will consume
+    //   m_acc_radr = pm_vo ? m_oidx+1 : m_oidx
+    // and register the selected bank into m_acc_rd, consumed one cycle later when
+    // pm_vo is high. The same-index read-modify-write on bank_t is read-old:
+    // the running t_hat source was written by the previous (j-1) term, long
+    // settled; the value consumed this cycle was registered before this cycle's
+    // write to m_oidx. Cadence is CALC(vo=0)/C0(vo=1)/C1(vo=1).
+    wire [7:0]  m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx;
+    wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr;  // K+m_i
+    wire [13:0] m_tacc_full = m_i*256 + m_acc_radr;
+    wire [11:0] m_acc_rd_next = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]]
+                                              : bank_t [m_tacc_full[PT_AW-1:0]];
+    reg  [11:0] m_acc_rd;
+    wire [11:0]  m_acc_src = m_acc_rd;
     // (a + b) mod Q  (both < Q, sum < 2Q): one conditional subtract
     wire [12:0]  m_sum  = {1'b0, m_acc_src} + {1'b0, pm_coeff};
     wire [11:0]  m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0];
@@ -506,6 +564,8 @@ module mlkem_top #(
             n_ridx     <= 9'd0;
             n_widx     <= 8'd0;
             n_valid    <= 1'b0;
+            n_loading  <= 1'b0;
+            n_rd_data  <= 12'd0;
             n_pending  <= 1'b0;
             m_i        <= 2'd0;
             m_j        <= 2'd0;
@@ -514,6 +574,9 @@ module mlkem_top #(
             m_loading  <= 1'b0;
             m_pending  <= 1'b0;
             pm_valid   <= 1'b0;
+            pm_a_rd    <= 12'd0;
+            pm_b_rd    <= 12'd0;
+            m_acc_rd   <= 12'd0;
             e_poly     <= 3'd0;
             e_pair     <= 8'd0;
             e_byte     <= 2'd0;
@@ -578,7 +641,7 @@ module mlkem_top #(
 
                 // store each output coefficient only while busy (ignore stale last coeff from prior poly)
                 if (a_busy && snt_vo && snt_ack) begin
-                    polymem[a_slot*256 + a_widx] <= snt_coeff;
+                    bank_a[(a_slot*256 + a_widx) & ((1<<PA_AW)-1)] <= snt_coeff;
                     if (snt_last) begin
                         // finished this poly; advance (i,j) in row-major order
                         a_pair <= a_pair + 5'd1;
@@ -615,7 +678,8 @@ module mlkem_top #(
                 end
 
                 if (c_busy && cbd_vo && cbd_ack) begin
-                    polymem[c_slot*256 + c_widx] <= cbd_modq;
+                    // bank_se relative slot = c_slot - slot_s_rt = c_poly
+                    bank_se[(c_poly*256 + c_widx) & ((1<<PSE_AW)-1)] <= cbd_modq;
                     if (cbd_last) begin
                         c_poly <= c_poly + 3'd1;
                         c_widx <= 8'd0;
@@ -627,30 +691,40 @@ module mlkem_top #(
                 end
             end
 
-            // Arm N stage when C finishes: start NTT on slot S0
+            // Arm N stage when C finishes: prime load of slot S0. n_ridx is a
+            // read-ahead pointer; bank_se read is registered into n_rd_data and
+            // fed to ntt_core one cycle later, so valid starts low (priming).
             if (st == ST_C && st_next == ST_N) begin
                 n_slot    <= 3'd0;
                 n_ridx    <= 9'd0;
                 n_widx    <= 8'd0;
-                n_valid   <= 1'b1;       // begin loading first poly
+                n_valid   <= 1'b0;
+                n_loading <= 1'b1;       // begin presenting load addresses
                 n_pending <= 1'b0;
             end
 
             // ---- ST_N: forward NTT each of S0,S1,E0,E1 in place ----
             if (st == ST_N) begin
-                // LOAD phase: stream 256 coeffs into ntt_core
-                if (n_valid && ntt_ready) begin
-                    if (n_ridx == 9'd255) begin
-                        n_valid <= 1'b0;       // last coeff presented this cycle
-                        n_ridx  <= 9'd0;
+                // LOAD phase: present read-ahead addr to bank_se; the value
+                // registered last cycle (n_rd_data) is consumed by ntt_core
+                // this cycle (n_valid). Cores hold ready high through LOAD, so
+                // a fixed 1-cycle skew suffices (no backpressure gating).
+                if (n_loading) begin
+                    if (n_ridx == 9'd256) begin
+                        // 256th coeff (bank_se[255]) consumed this cycle; stop
+                        n_loading <= 1'b0;
+                        n_valid   <= 1'b0;
                     end else begin
-                        n_ridx <= n_ridx + 9'd1;
+                        n_rd_data <= bank_se[ntt_rd_full[PSE_AW-1:0]];
+                        n_ridx    <= n_ridx + 9'd1;
+                        n_valid   <= 1'b1;     // data presented last cycle is valid
                     end
                 end
 
                 // OUTPUT phase: collect 256 results, write back to same slot
                 if (ntt_vo) begin
-                    polymem[n_slot_addr*256 + n_widx] <= ntt_coeff;
+                    // bank_se relative slot = n_slot_addr - slot_s_rt = n_slot
+                    bank_se[(n_slot*256 + n_widx) & ((1<<PSE_AW)-1)] <= ntt_coeff;
                     n_widx <= n_widx + 8'd1;   // wraps 255->0 after last
                 end
 
@@ -665,15 +739,18 @@ module mlkem_top #(
                     end
                 end
 
-                // Kick next slot's load once core is back IDLE
+                // Kick next slot's load once core is back IDLE (re-prime)
                 if (n_pending && ntt_ready && !ntt_done) begin
-                    n_valid   <= 1'b1;
                     n_ridx    <= 9'd0;
+                    n_valid   <= 1'b0;
+                    n_loading <= 1'b1;
                     n_pending <= 1'b0;
                 end
             end
 
-            // Arm M stage when N finishes: start first (i=0,j=0) poly_mul load
+            // Arm M stage when N finishes: prime first (i=0,j=0) poly_mul load.
+            // m_ld read-ahead pointer; pm_a_rd/pm_b_rd registered, pm_valid
+            // asserted one cycle after an address is presented (like ST_N).
             if (st == ST_N && st_next == ST_M) begin
                 m_i       <= 2'd0;
                 m_j       <= 2'd0;
@@ -681,26 +758,38 @@ module mlkem_top #(
                 m_oidx    <= 8'd0;
                 m_loading <= 1'b1;
                 m_pending <= 1'b0;
-                pm_valid  <= 1'b1;
+                pm_valid  <= 1'b0;
             end
 
             // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ----
             if (st == ST_M) begin
-                // LOAD: stream 256 (A,shat) pairs into poly_mul
-                if (m_loading && pm_valid && pm_ready) begin
-                    if (m_ld == 9'd255) begin
-                        pm_valid  <= 1'b0;     // last pair presented
+                // registered accumulator read (sd_bram timing): captures the
+                // index the next pm_vo will consume. Free-running within ST_M;
+                // primed during LOAD (m_oidx held at 0).
+                m_acc_rd <= m_acc_rd_next;
+
+                // LOAD: present read-ahead addr to bank_a/bank_se; the pair
+                // registered last cycle (pm_a_rd/pm_b_rd) is consumed by
+                // poly_mul this cycle (pm_valid). poly_mul holds ready high
+                // through LOAD, so a fixed 1-cycle skew suffices.
+                if (m_loading) begin
+                    if (m_ld == 9'd256) begin
+                        pm_valid  <= 1'b0;     // 256th pair consumed this cycle
                         m_loading <= 1'b0;
                         m_ld      <= 9'd0;
                         m_oidx    <= 8'd0;
                     end else begin
-                        m_ld <= m_ld + 9'd1;
+                        pm_a_rd <= bank_a [pm_a_full[PA_AW-1:0]];
+                        pm_b_rd <= bank_se[pm_b_full[PSE_AW-1:0]];
+                        m_ld    <= m_ld + 9'd1;
+                        pm_valid <= 1'b1;      // pair presented last cycle is valid
                     end
                 end
 
                 // ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat
                 if (pm_vo) begin
-                    polymem[m_tslot*256 + m_oidx] <= m_accq;
+                    // bank_t relative slot = m_tslot - slot_t_rt = m_i
+                    bank_t[(m_i*256 + m_oidx) & ((1<<PT_AW)-1)] <= m_accq;
                     if (m_oidx == 8'd255) begin
                         // finished this (i,j) term; advance
                         if (m_j + 2'd1 < k_r) begin
@@ -716,12 +805,12 @@ module mlkem_top #(
                     end
                 end
 
-                // Start next (i,j) poly_mul load once core is IDLE again
+                // Start next (i,j) poly_mul load once core is IDLE again (re-prime)
                 if (m_pending && pm_ready && !pm_vo) begin
-                    pm_valid  <= 1'b1;
                     m_loading <= 1'b1;
                     m_ld      <= 9'd0;
                     m_oidx    <= 8'd0;
+                    pm_valid  <= 1'b0;
                     m_pending <= 1'b0;
                 end
             end