perf(mlkem): move poly_mul coeff storage to even/odd banked BRAM

Replace poly_mul_sync's two 256x12 LUT register arrays (mem_A/mem_B, ~8k LUT)
with four sd_bram banks (A_even/A_odd/B_even/B_odd, 128x12). The base-case
reads mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a
single read port. This removes the multi-port LUT mux and the long
BRAM->register-file route that was the mlkem_clk critical path.

sd_bram reads are synchronous (1-cycle); the existing 2-cycle issue cadence
absorbs it (address on the gap cycle, data on the issue cycle), with a
one-cycle priming gap at LOAD->RUN. Interface and behavior unchanged.

Verified byte-exact vs NIST KAT: poly_mul standalone TB 11/11, Encaps 9/9,
Decaps 9/9 (incl. reject + DECOMP), KeyGen, hello_world e2e — all PASS.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

rtl/ip/mlkem/poly_mul/poly_mul_sync.v

@@ -1,4 +1,4 @@
-// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier
+// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (BRAM storage)
 //
 // Computes pointwise (Karatsuba-like base-case) multiplication of two
 // 256-coefficient NTT-domain polynomials.
@@ -6,18 +6,18 @@
 // Operation flow:
 //   IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
 //
-// RUN issues one base-case multiply every two cycles, matching the two-cycle
+// Coefficient storage moved from 256x12 LUT register arrays to four sd_bram
-// c0/c1 output bandwidth after the basecase pipeline fills.
+// banks (A_even/A_odd/B_even/B_odd, each 128x12): the base-case reads
+// mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single
+// read port. This removes the large multi-port LUT mux (~8k LUT) and the long
+// BRAM->register-file routing that was the mlkem_clk critical path.
 //
-// The LOAD phase accepts both A and B coefficients simultaneously
+// RUN issues one base-case every two cycles. sd_bram reads are synchronous
-// (one pair per cycle) on coeff_a_in/coeff_b_in.
+// (1-cycle latency); the 2-cycle cadence absorbs it (address presented on the
+// gap cycle, data consumed on the issue cycle). A one-cycle priming gap is
+// inserted at LOAD->RUN so the first base-case's operands are ready.
 //
-// The COMPUTE phase outputs the 256 result coefficients one per cycle
+// Interface (unchanged):
-// via valid/ready handshake on coeff_out/valid_o/ready_i.
-//
-// Memory: 256×12-bit register arrays for A and B coefficients.
-//
-// Interface:
 //   clk, rst_n     - Clock, active-low reset
 //   coeff_a_in[11:0]- Polynomial A coefficient input
 //   coeff_b_in[11:0]- Polynomial B coefficient input
@@ -38,18 +38,12 @@ module poly_mul_sync (
     input         ready_i
 );
 
-    // State definitions
     localparam S_IDLE = 3'd0;
     localparam S_LOAD = 3'd1;
     localparam S_RUN  = 3'd2;
 
     reg [2:0] state, next_state;
 
-    // Coefficient storage (register arrays)
-    reg [11:0] mem_A [0:255];
-    reg [11:0] mem_B [0:255];
-
-    // Counters
     reg [7:0] load_cnt;       // 0..256 for loading 256 pairs
     reg [7:0] issue_cnt;      // 0..128, issued base-cases
     reg [7:0] out_cnt;        // 0..255, consumed output coefficients
@@ -57,53 +51,61 @@ module poly_mul_sync (
     reg       out_phase;      // 0=c0, 1=c1
     reg       out_valid_r;
 
-    // Registered basecase_mul inputs/results
     reg [11:0] bc_a0_reg, bc_a1_reg;
     reg [11:0] bc_b0_reg, bc_b1_reg;
     reg [11:0] bc_zeta_reg;
     reg        bc_valid_reg;
     reg [11:0] c0_reg, c1_reg;
 
-    // Combinational read signals for the next base-case request.
+    // ---- Coefficient storage: even/odd banked sd_bram ----
-    wire [6:0] issue_k   = issue_cnt[6:0];
+    // A_even[k]=mem_A[2k], A_odd[k]=mem_A[2k+1]; same for B. Read index during
-    wire [7:0] addr_even = {issue_k, 1'b0};   // issue_k * 2
+    // RUN is issue_k; write index during LOAD is load_cnt>>1 routed by load_cnt[0].
-    wire [7:0] addr_odd  = {issue_k, 1'b1};   // issue_k * 2 + 1
+    wire [6:0] issue_k   = issue_cnt[6:0];       // 0..127 read index
-    wire [11:0] mem_a0 = mem_A[addr_even];
-    wire [11:0] mem_a1 = mem_A[addr_odd];
-    wire [11:0] mem_b0 = mem_B[addr_even];
-    wire [11:0] mem_b1 = mem_B[addr_odd];
 
-    // Zeta ROM
+    wire load_fire = ((state == S_IDLE) || (state == S_LOAD)) && valid_i && ready_o;
-    wire [11:0] zeta;
+    wire [6:0] wr_local = load_cnt[7:1];          // load_cnt/2
-    poly_mul_zeta_rom u_zeta (
+    wire       wr_odd   = load_cnt[0];
-        .addr (issue_k),
+
-        .zeta (zeta)
+    wire we_even = load_fire && !wr_odd;
+    wire we_odd  = load_fire &&  wr_odd;
+
+    wire [11:0] mem_a0, mem_a1, mem_b0, mem_b1;   // sd_bram registered read outputs
+    sd_bram #(.W(12), .D(128), .A(7)) u_a_even (
+        .clk(clk), .rd_addr(issue_k), .rd_data(mem_a0),
+        .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_a_in)
+    );
+    sd_bram #(.W(12), .D(128), .A(7)) u_a_odd (
+        .clk(clk), .rd_addr(issue_k), .rd_data(mem_a1),
+        .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_a_in)
+    );
+    sd_bram #(.W(12), .D(128), .A(7)) u_b_even (
+        .clk(clk), .rd_addr(issue_k), .rd_data(mem_b0),
+        .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_b_in)
+    );
+    sd_bram #(.W(12), .D(128), .A(7)) u_b_odd (
+        .clk(clk), .rd_addr(issue_k), .rd_data(mem_b1),
+        .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_b_in)
     );
 
-    // Pipelined basecase multiply. Requests are issued every two cycles; inputs
+    wire [11:0] zeta;
-    // are registered locally so issue_cnt does not directly drive DSP input muxes.
+    poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta));
+
     wire [11:0] bc_c0, bc_c1;
     wire        bc_vo;
     basecase_mul_pipe u_bc (
-        .clk (clk),
+        .clk(clk), .rst_n(rst_n),
-        .rst_n(rst_n),
         .valid_i(bc_valid_reg),
-        .a0  (bc_a0_reg),
+        .a0(bc_a0_reg), .a1(bc_a1_reg),
-        .a1  (bc_a1_reg),
+        .b0(bc_b0_reg), .b1(bc_b1_reg),
-        .b0  (bc_b0_reg),
-        .b1  (bc_b1_reg),
         .zeta(bc_zeta_reg),
-        .c0  (bc_c0),
+        .c0(bc_c0), .c1(bc_c1),
-        .c1  (bc_c1),
         .valid_o(bc_vo)
     );
 
-    // Output interface
     assign ready_o = (state == S_IDLE) || (state == S_LOAD);
     assign valid_o = out_valid_r;
     assign coeff_out = out_phase ? c1_reg : c0_reg;
 
-    // State transition logic (combinational)
     always @* begin
         next_state = state;
         case (state)
@@ -116,8 +118,6 @@ module poly_mul_sync (
         endcase
     end
 
-    // Sequential logic
-    integer i;
     always @(posedge clk or negedge rst_n) begin
         if (!rst_n) begin
             state    <= S_IDLE;
@@ -127,27 +127,16 @@ module poly_mul_sync (
             issue_gap <= 1'b0;
             out_phase <= 1'b0;
             out_valid_r <= 1'b0;
-            bc_a0_reg <= 12'd0;
+            bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0;
-            bc_a1_reg <= 12'd0;
+            bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0;
-            bc_b0_reg <= 12'd0;
+            bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0;
-            bc_b1_reg <= 12'd0;
+            c0_reg   <= 12'd0; c1_reg <= 12'd0;
-            bc_zeta_reg <= 12'd0;
-            bc_valid_reg <= 1'b0;
-            c0_reg   <= 12'd0;
-            c1_reg   <= 12'd0;
-            for (i = 0; i < 256; i = i + 1) begin
-                mem_A[i] = 12'd0;
-                mem_B[i] = 12'd0;
-            end
         end else begin
             state <= next_state;
             bc_valid_reg <= 1'b0;
 
-            // ---- LOAD phase ----
+            // ---- LOAD phase (writes routed to even/odd banks combinationally) ----
-            // First coefficient captured on IDLE → LOAD transition
             if (state == S_IDLE && valid_i && ready_o) begin
-                mem_A[0] <= coeff_a_in;
-                mem_B[0] <= coeff_b_in;
                 load_cnt <= 8'd1;
                 issue_cnt <= 8'd0;
                 out_cnt <= 8'd0;
@@ -156,25 +145,21 @@ module poly_mul_sync (
                 out_valid_r <= 1'b0;
             end
 
-            // Subsequent coefficients in LOAD state
             if (state == S_LOAD && valid_i && ready_o) begin
-                mem_A[load_cnt] <= coeff_a_in;
-                mem_B[load_cnt] <= coeff_b_in;
                 load_cnt <= load_cnt + 8'd1;
                 if (load_cnt >= 8'd255) begin
                     issue_cnt <= 8'd0;
                     out_cnt <= 8'd0;
-                    issue_gap <= 1'b0;
+                    issue_gap <= 1'b1;   // priming gap: present issue_k=0 addr first
                     out_phase <= 1'b0;
                     out_valid_r <= 1'b0;
                 end
             end
 
             // ---- COMPUTE phase ----
-            // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
+            // Issue one base-case every two cycles. On the gap cycle the sd_bram
-            // regs, then launch requests every other cycle. The two-cycle issue
+            // address (issue_k) settles; on the issue cycle its registered output
-            // interval matches the c0/c1 output bandwidth, so no deep result FIFO
+            // (mem_a0/a1/b0/b1) is captured into the base-case input registers.
-            // is needed for this always-ready system.
             if (state == S_RUN) begin
                 if (issue_cnt < 8'd128) begin
                     if (!issue_gap) begin
@@ -212,7 +197,6 @@ module poly_mul_sync (
                 end
             end
 
-            // ---- final output consumed: reset counters before accepting a new op ----
             if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
                 load_cnt <= 8'd0;
                 issue_cnt <= 8'd0;