refactor(kg): registered single-port read for ST_E byteEncode (bank_se/bank_t)

Final per-consumer step of stage 2b. ST_E read two coeffs of a pair from one
bank combinationally; now serialized through a single registered read port
(e_rd_data_r) over a 4-cycle micro-phase e_ph: ph0 fetch c0, ph1 write b0 +
fetch c1, ph2 write b1, ph3 write b2. Matches sd_bram timing; +512 cyc on K=2
(one extra fetch per pair-stream). All polymem read consumers (ST_N, ST_M
load, ST_M acc, ST_E) are now registered-read. 11/11 KAT PASS, byte-exact.

sync_rtl/top/mlkem_top.v

@@ -450,28 +450,33 @@ module mlkem_top #(
     // bytes || rho; dk_pke = s_hat[0..K-1] bytes. Walk coeff pairs per poly.
     reg  [4:0]  e_poly;        // 0..2K-1: [0,K) = t_hat -> ek; [K,2K) = s_hat -> dk_pke
     reg  [7:0]  e_pair;        // 0..127 coeff-pair within poly
-    reg  [1:0]  e_byte;        // 0..2 byte within the current pair (BRAM: 1 write/cycle)
     reg  [9:0]  e_rho;         // 0..31 rho byte copy index (ek tail)
     reg         e_done;        // serialization complete
     // source poly slot: t_hat[e_poly] for ek half, s_hat[e_poly-K] for dk half
     wire        e_is_dk = (e_poly >= {1'b0, k_r});
     wire [4:0]  e_pidx  = e_is_dk ? (e_poly - {1'b0, k_r}) : e_poly;   // index within target
-    wire [SAW-1:0] e_slot = e_is_dk ? (slot_s_rt + e_pidx) : (slot_t_rt + e_pidx);
+    // Registered single-port read (sd_bram timing). One bank read per cycle:
-    // two coeffs of the current pair: ek half reads t_hat (bank_t), dk half
+    // ek half reads bank_t (t_hat), dk half reads bank_se (s_hat). The two
-    // reads s_hat (bank_se). Relative index within the bank = e_pidx.
+    // coeffs of a pair are serialized across a 4-cycle micro-phase e_ph:
-    wire [13:0] e_rd0_full = e_pidx*256 + {e_pair, 1'b0};
+    //   0 = fetch c0
-    wire [13:0] e_rd1_full = e_pidx*256 + {e_pair, 1'b1};
+    //   1 = c0 ready: write b0, save c0[11:8], fetch c1
-    wire [11:0] e_c0 = e_is_dk ? bank_se[e_rd0_full[PSE_AW-1:0]] : bank_t[e_rd0_full[PT_AW-1:0]];
+    //   2 = c1 ready: write b1, save c1[11:4]
-    wire [11:0] e_c1 = e_is_dk ? bank_se[e_rd1_full[PSE_AW-1:0]] : bank_t[e_rd1_full[PT_AW-1:0]];
+    //   3 = write b2, advance pair
-    // 3 packed bytes
+    reg  [1:0]  e_ph;          // per-pair micro-phase 0..3
-    wire [7:0]  e_b0 = e_c0[7:0];
+    reg  [3:0]  e_c0_hi;       // saved c0[11:8] for b1
-    wire [7:0]  e_b1 = {e_c1[3:0], e_c0[11:8]};
+    reg  [7:0]  e_c1_hi;       // saved c1[11:4] for b2
-    wire [7:0]  e_b2 = e_c1[11:4];
+    wire        e_rd_half = (e_ph == 2'd1);          // ph0 -> c0, ph1 -> c1
-    // byte base offset within target memory: poly index *384 (= 128 pairs *3)
+    wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half};
-    wire [11:0] e_base = e_pidx * 12'd384;
+    wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]]
+                                     : bank_t [e_rd_full[PT_AW-1:0]];
+    reg  [11:0] e_rd_data_r;   // registered read: value for addr presented last cyc
+    // byteEncode write byte offset within the target memory: pair*3 + byte index.
+    wire [11:0] e_base = e_pidx * 12'd384;           // poly index *384 (=128 pairs*3)
     wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair};  // pair*3
-    // BRAM write is 1 byte/cycle: select packed byte by e_byte sub-counter
+    wire [1:0]  e_wb   = e_ph - 2'd1;                // ph1->byte0, ph2->byte1, ph3->byte2
-    wire [7:0]  e_byte_d = (e_byte == 2'd0) ? e_b0 : (e_byte == 2'd1) ? e_b1 : e_b2;
+    wire [7:0]  e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0]             // b0 = c0[7:0]
+                        : (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi}  // b1 = {c1[3:0],c0[11:8]}
+                        :                  e_c1_hi;                      // b2 = c1[11:4]
 
     wire [SAW-1:0]  m_aslot = m_i*k_r + m_j;     // A_hat[i][j] slot = i*k + j
     wire [SAW-1:0]  m_sslot = slot_s_rt + m_j;     // s_hat[j]
@@ -579,7 +584,10 @@ module mlkem_top #(
             m_acc_rd   <= 12'd0;
             e_poly     <= 3'd0;
             e_pair     <= 8'd0;
-            e_byte     <= 2'd0;
+            e_ph       <= 2'd0;
+            e_rd_data_r<= 12'd0;
+            e_c0_hi    <= 4'd0;
+            e_c1_hi    <= 8'd0;
             e_rho      <= 10'd0;
             e_done     <= 1'b0;
             ek_we      <= 1'b0;
@@ -819,27 +827,39 @@ module mlkem_top #(
             if (st == ST_M && st_next == ST_E) begin
                 e_poly <= 3'd0;
                 e_pair <= 8'd0;
-                e_byte <= 2'd0;
+                e_ph   <= 2'd0;     // start at fetch-c0 of first pair
                 e_rho  <= 10'd0;
                 e_done <= 1'b0;
             end
 
-            // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke, 1 byte/cycle ----
+            // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ----
+            // Single registered bank read per cycle; 4-cycle micro-phase per
+            // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1,
+            // ph3 write b2). e_rd_data_r holds the value read last cycle.
             if (st == ST_E && !e_done) begin
+                // registered read (sd_bram timing): capture addr presented now
+                e_rd_data_r <= e_rd_async;
+
                 if (e_poly < {1'b0, k_r, 1'b0}) begin
-                    // write current byte (e_byte_d) to target BRAM
+                    // ph0: fetch only (prime). ph1..3: write one packed byte.
+                    if (e_ph != 2'd0) begin
                         if (!e_is_dk) begin
                             ek_we <= 1'b1;
-                        ek_wa <= e_boff[10:0] + {9'd0, e_byte};
+                            ek_wa <= e_boff[10:0] + {9'd0, e_wb};
-                        ek_wd <= e_byte_d;
+                            ek_wd <= e_wbyte;
                         end else begin
                             dkp_we <= 1'b1;
-                        dkp_wa <= e_boff[10:0] + {9'd0, e_byte};
+                            dkp_wa <= e_boff[10:0] + {9'd0, e_wb};
-                        dkp_wd <= e_byte_d;
+                            dkp_wd <= e_wbyte;
                         end
-                    // advance sub-counters
+                    end
-                    if (e_byte == 2'd2) begin
+                    // save coeff high nibbles as they become available
-                        e_byte <= 2'd0;
+                    if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8];  // c0[11:8]
+                    if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4];  // c1[11:4]
+
+                    // advance micro-phase / pair / poly
+                    if (e_ph == 2'd3) begin
+                        e_ph <= 2'd0;
                         if (e_pair == 8'd127) begin
                             e_pair <= 8'd0;
                             e_poly <= e_poly + 5'd1;   // next poly or -> rho phase
@@ -847,7 +867,7 @@ module mlkem_top #(
                             e_pair <= e_pair + 8'd1;
                         end
                     end else begin
-                        e_byte <= e_byte + 2'd1;
+                        e_ph <= e_ph + 2'd1;
                     end
                 end else begin
                     // rho copy: ek[384*K + r] = rho byte r  (r = 0..31), 1 byte/cycle