From be2e54c8abca7d6d71a3109d992b7be3b7ce3a11 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 03:48:50 +0800 Subject: [PATCH] revert(mlkem): drop poly_mul ping-pong (B2) to reclaim ~7k LUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B2 (ping-pong poly_mul + decoupled MAC load) doubled the 256x12 coeff register file (mem_A/B -> 4 arrays) costing ~7k LUT for only ~2.8% Encaps latency (K=2) — the worst area/latency ratio of the optimization set. Revert restores single-buffer poly_mul and serial MAC. B1 (compress streaming) and R1 (dual-butterfly NTT) are kept. Verified byte-exact vs NIST KAT: Encaps 9/9, Decaps 9/9, KeyGen, hello end-to-end, poly_mul standalone TB 11/11 — all PASS. ENC_U K=2 4177->4689, ENC_V 2089->2345 (B2's hidden load reappears). --- rtl/ip/mlkem/poly_mul/poly_mul_sync.v | 266 ++++++++++++++------------ rtl/ip/mlkem/top/mlkem_top.v | 69 ++----- 2 files changed, 162 insertions(+), 173 deletions(-) diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v index 46e9ac4..61f8f99 100644 --- a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -1,32 +1,30 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (B2 ping-pong) +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. // -// Double-buffered (ping-pong) load so the caller can stream the NEXT operation's -// 256 A+B pairs into the idle buffer WHILE the current operation's results are -// still draining out. This hides the 256-cycle load behind the ~256-cycle -// compute/output of the previous multiply: back-to-back multiplies cost ~256 -// cycles each instead of 512 (load THEN run). +// Operation flow: +// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // -// Contract (backward compatible with the single-shot testbench): -// - ready_o is high whenever a buffer is free to accept a load. A fresh op is -// accepted (load_active started) as soon as the first pair arrives. -// - Each op streams 256 pairs on coeff_a_in/coeff_b_in (one per accepted cyc). -// - When a buffer is full it is handed to the compute engine, which issues one -// base-case every two cycles and emits 256 result coeffs on coeff_out via -// valid_o/ready_i (c0 then c1 per pair). -// - A caller that only ever loads-then-drains (never loads during RUN) sees -// the exact same behaviour as before, just with an extra buffer. +// RUN issues one base-case multiply every two cycles, matching the two-cycle +// c0/c1 output bandwidth after the basecase pipeline fills. // -// Interface (unchanged ports): -// clk, rst_n - Clock, active-low reset -// coeff_a_in[11:0]- Polynomial A coefficient (index = pairs accepted this load) -// coeff_b_in[11:0]- Polynomial B coefficient +// The LOAD phase accepts both A and B coefficients simultaneously +// (one pair per cycle) on coeff_a_in/coeff_b_in. +// +// The COMPUTE phase outputs the 256 result coefficients one per cycle +// via valid/ready handshake on coeff_out/valid_o/ready_i. +// +// Memory: 256×12-bit register arrays for A and B coefficients. +// +// Interface: +// clk, rst_n - Clock, active-low reset +// coeff_a_in[11:0]- Polynomial A coefficient input +// coeff_b_in[11:0]- Polynomial B coefficient input // valid_i - Input valid -// ready_o - Ready to accept a load pair (a buffer is free) +// ready_o - Ready to accept input (high in IDLE/LOAD) // coeff_out[11:0] - Result coefficient output -// valid_o - Output valid +// valid_o - Output valid during RUN output phase // ready_i - Output consumer ready module poly_mul_sync ( @@ -40,133 +38,150 @@ module poly_mul_sync ( input ready_i ); - // Two coefficient buffers (ping-pong). - reg [11:0] mem_A0 [0:255]; - reg [11:0] mem_B0 [0:255]; - reg [11:0] mem_A1 [0:255]; - reg [11:0] mem_B1 [0:255]; + // State definitions + localparam S_IDLE = 3'd0; + localparam S_LOAD = 3'd1; + localparam S_RUN = 3'd2; - // ---- Load side ---- - reg ld_active; // currently accepting a load into ld_ptr - reg ld_ptr; // FIFO write buffer (toggles each completed load) - reg [8:0] load_cnt; // 0..256 pairs written this load - reg buf_full [0:1]; // buffer holds a complete (unconsumed) operand set + reg [2:0] state, next_state; - // ---- Compute side ---- - reg run_active; // compute engine busy on rn_ptr - reg rn_ptr; // FIFO read buffer (toggles each completed run) - reg [7:0] issue_cnt; // 0..128 base-cases issued - reg [7:0] out_cnt; // 0..255 output coeffs consumed - reg issue_gap; // 2-cycle issue spacing - reg out_phase; // 0=c0, 1=c1 - reg out_valid_r; + // Coefficient storage (register arrays) + reg [11:0] mem_A [0:255]; + reg [11:0] mem_B [0:255]; - // Registered basecase inputs/results + // Counters + reg [7:0] load_cnt; // 0..256 for loading 256 pairs + reg [7:0] issue_cnt; // 0..128, issued base-cases + reg [7:0] out_cnt; // 0..255, consumed output coefficients + reg issue_gap; // spaces base-case requests to match c0/c1 output + reg out_phase; // 0=c0, 1=c1 + reg out_valid_r; + + // Registered basecase_mul inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Compute-side memory read (from the buffer being consumed). + // Combinational read signals for the next base-case request. wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; - wire [7:0] addr_odd = {issue_k, 1'b1}; - wire [11:0] mem_a0 = rn_ptr ? mem_A1[addr_even] : mem_A0[addr_even]; - wire [11:0] mem_a1 = rn_ptr ? mem_A1[addr_odd] : mem_A0[addr_odd]; - wire [11:0] mem_b0 = rn_ptr ? mem_B1[addr_even] : mem_B0[addr_even]; - wire [11:0] mem_b1 = rn_ptr ? mem_B1[addr_odd] : mem_B0[addr_odd]; + wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 + wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 + wire [11:0] mem_a0 = mem_A[addr_even]; + wire [11:0] mem_a1 = mem_A[addr_odd]; + wire [11:0] mem_b0 = mem_B[addr_even]; + wire [11:0] mem_b1 = mem_B[addr_odd]; + // Zeta ROM wire [11:0] zeta; - poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); + poly_mul_zeta_rom u_zeta ( + .addr (issue_k), + .zeta (zeta) + ); + // Pipelined basecase multiply. Requests are issued every two cycles; inputs + // are registered locally so issue_cnt does not directly drive DSP input muxes. wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk(clk), .rst_n(rst_n), + .clk (clk), + .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0(bc_a0_reg), .a1(bc_a1_reg), - .b0(bc_b0_reg), .b1(bc_b1_reg), + .a0 (bc_a0_reg), + .a1 (bc_a1_reg), + .b0 (bc_b0_reg), + .b1 (bc_b1_reg), .zeta(bc_zeta_reg), - .c0(bc_c0), .c1(bc_c1), + .c0 (bc_c0), + .c1 (bc_c1), .valid_o(bc_vo) ); - // Accept a load pair whenever the FIFO write buffer (ld_ptr) is free. Once a - // load is in progress it always finishes (ready stays high for its 256). - assign ready_o = ld_active ? 1'b1 : !buf_full[ld_ptr]; - - assign valid_o = out_valid_r; + // Output interface + assign ready_o = (state == S_IDLE) || (state == S_LOAD); + assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - wire ld_fire = valid_i && ready_o; + // State transition logic (combinational) + always @* begin + next_state = state; + case (state) + S_IDLE: if (valid_i && ready_o) next_state = S_LOAD; + S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o) + next_state = S_RUN; + S_RUN: if (valid_o && ready_i && out_cnt == 8'd255) + next_state = S_IDLE; + default: next_state = S_IDLE; + endcase + end + // Sequential logic integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - ld_active <= 1'b0; ld_ptr <= 1'b0; load_cnt <= 9'd0; - buf_full[0] <= 1'b0; buf_full[1] <= 1'b0; - run_active <= 1'b0; rn_ptr <= 1'b0; - issue_cnt <= 8'd0; out_cnt <= 8'd0; issue_gap <= 1'b0; - out_phase <= 1'b0; out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; c1_reg <= 12'd0; + state <= S_IDLE; + load_cnt <= 8'd0; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; + bc_a0_reg <= 12'd0; + bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; + bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; + bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; + c1_reg <= 12'd0; for (i = 0; i < 256; i = i + 1) begin - mem_A0[i] = 12'd0; mem_B0[i] = 12'd0; - mem_A1[i] = 12'd0; mem_B1[i] = 12'd0; + mem_A[i] = 12'd0; + mem_B[i] = 12'd0; end end else begin + state <= next_state; bc_valid_reg <= 1'b0; - // ================= LOAD SIDE (FIFO write into ld_ptr) ================= - if (!ld_active) begin - if (ld_fire) begin - if (!ld_ptr) begin - mem_A0[0] <= coeff_a_in; mem_B0[0] <= coeff_b_in; - end else begin - mem_A1[0] <= coeff_a_in; mem_B1[0] <= coeff_b_in; - end - load_cnt <= 9'd1; - ld_active <= 1'b1; - end - end else begin - if (ld_fire) begin - if (!ld_ptr) begin - mem_A0[load_cnt[7:0]] <= coeff_a_in; - mem_B0[load_cnt[7:0]] <= coeff_b_in; - end else begin - mem_A1[load_cnt[7:0]] <= coeff_a_in; - mem_B1[load_cnt[7:0]] <= coeff_b_in; - end - if (load_cnt == 9'd255) begin - buf_full[ld_ptr] <= 1'b1; // buffer complete - ld_ptr <= ~ld_ptr; // FIFO advance - ld_active <= 1'b0; - load_cnt <= 9'd0; - end else begin - load_cnt <= load_cnt + 9'd1; - end + // ---- LOAD phase ---- + // First coefficient captured on IDLE → LOAD transition + if (state == S_IDLE && valid_i && ready_o) begin + mem_A[0] <= coeff_a_in; + mem_B[0] <= coeff_b_in; + load_cnt <= 8'd1; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; + end + + // Subsequent coefficients in LOAD state + if (state == S_LOAD && valid_i && ready_o) begin + mem_A[load_cnt] <= coeff_a_in; + mem_B[load_cnt] <= coeff_b_in; + load_cnt <= load_cnt + 8'd1; + if (load_cnt >= 8'd255) begin + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; end end - // ================= COMPUTE SIDE (FIFO read from rn_ptr) ============= - if (!run_active) begin - // Start compute on the FIFO head buffer once it is full (and not - // still mid-load, which cannot happen for rn_ptr==ld_ptr while - // ld_active, since buf_full is set only at load completion). - if (buf_full[rn_ptr]) begin - run_active <= 1'b1; - issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - end - end else begin - // Issue one base-case every two cycles. + // ---- COMPUTE phase ---- + // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input + // regs, then launch requests every other cycle. The two-cycle issue + // interval matches the c0/c1 output bandwidth, so no deep result FIFO + // is needed for this always-ready system. + if (state == S_RUN) begin if (issue_cnt < 8'd128) begin if (!issue_gap) begin - bc_a0_reg <= mem_a0; bc_a1_reg <= mem_a1; - bc_b0_reg <= mem_b0; bc_b1_reg <= mem_b1; + bc_a0_reg <= mem_a0; + bc_a1_reg <= mem_a1; + bc_b0_reg <= mem_b0; + bc_b1_reg <= mem_b1; bc_zeta_reg <= zeta; bc_valid_reg <= 1'b1; issue_cnt <= issue_cnt + 8'd1; @@ -179,27 +194,32 @@ module poly_mul_sync ( if (out_valid_r && ready_i) begin out_cnt <= out_cnt + 8'd1; if (out_cnt == 8'd255) begin - out_valid_r <= 1'b0; out_phase <= 1'b0; + out_valid_r <= 1'b0; + out_phase <= 1'b0; end else if (!out_phase) begin out_phase <= 1'b1; end else begin - out_valid_r <= 1'b0; out_phase <= 1'b0; + out_valid_r <= 1'b0; + out_phase <= 1'b0; end end if (bc_vo) begin - c0_reg <= bc_c0; c1_reg <= bc_c1; - out_valid_r <= 1'b1; out_phase <= 1'b0; + c0_reg <= bc_c0; + c1_reg <= bc_c1; + out_valid_r <= 1'b1; + out_phase <= 1'b0; end + end - // Final output consumed: free this buffer, advance FIFO read ptr. - if (out_valid_r && ready_i && out_cnt == 8'd255) begin - buf_full[rn_ptr] <= 1'b0; - rn_ptr <= ~rn_ptr; - run_active <= 1'b0; - issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - end + // ---- final output consumed: reset counters before accepting a new op ---- + if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin + load_cnt <= 8'd0; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; end end end diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 3ee72ac..306df7a 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -896,12 +896,10 @@ module mlkem_top #( reg [2:0] u_row; // output row i 0..K-1 reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD // MAC bookkeeping (mirrors ST_M) - reg [2:0] u_j; // ACCUMULATE term col j 0..K-1 (psum order) - reg [2:0] u_ldj; // LOAD term col j 0..K-1 (runs ahead of u_j) + reg [2:0] u_j; // term col j 0..K-1 reg [8:0] u_ld; // load read-ahead 0..256 reg [7:0] u_oidx; // accumulate index 0..255 reg u_loading; // streaming pairs into poly_mul - reg u_ldwait; // load-side waiting for a free poly_mul buffer reg u_pending; // wait poly_mul IDLE before next j reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select) // INTT bookkeeping (mirrors ST_N) @@ -914,15 +912,10 @@ module mlkem_top #( reg [8:0] u_aidx; // ADD read-ahead 0..256 reg [7:0] u_awidx; // ADD write index 0..255 reg u_avalid; // ADD write enable pipeline - // MAC addresses (B2 decoupled): the LOAD side runs ahead on u_ldj (term being - // streamed into poly_mul's idle buffer); the ACCUMULATE side stays on u_j - // (term whose products are being summed into psum). A^T -> A_hat[u_ldj][u_row] - // slot = u_ldj*k+u_row; y_hat[u_ldj] rel u_ldj. Load reads bank_a/bank_se; - // accumulate reads bank_t (psum) -> no read-port conflict, so the next term's - // load overlaps the current term's accumulate. - wire [SAW-1:0] u_aslot = u_ldj*k_r + u_row; + // MAC addresses: A^T -> A_hat[u_j][u_row] slot = u_j*k+u_row; y_hat[u_j] rel u_j + wire [SAW-1:0] u_aslot = u_j*k_r + u_row; wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a - wire [13:0] u_pm_b_full = u_ldj*256 + u_ld[7:0]; // bank_se y_hat[u_ldj] + wire [13:0] u_pm_b_full = u_j*256 + u_ld[7:0]; // bank_se y_hat[u_j] wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx; wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 @@ -1250,8 +1243,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b0; @@ -1600,8 +1591,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -1610,53 +1599,30 @@ module mlkem_top #( end // ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ---- - // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. - // B2: decoupled load/accumulate. poly_mul is double-buffered, so the - // LOAD of term u_ldj can run AHEAD, overlapping the ACCUMULATE of an - // earlier term u_j (psum is order-dependent, so accumulate stays - // strictly serial; only the 256-cyc load is hidden). Load reads - // bank_a/bank_se; accumulate reads bank_t (UPSUM) — disjoint ports. - // u_ldj = term being streamed into poly_mul (0..K-1, then "done"); - // u_j = term whose products are being accumulated (via u_j0q sel). + // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. Mirrors ST_M + // load/accumulate cadence (read-ahead by 1, j-select via u_j0q). + // U: A^T[u_row][j] (bank_a u_j*K+u_row). V: t_hat[j] (bank_a u_j*K, u_row=0). if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin u_j0q <= (u_j == 3'd0); - // ---- LOAD side: stream term u_ldj into poly_mul's free buffer ---- - // Within a term poly_mul holds ready high for all 256 pairs, so the - // fixed 1-cyc addr->valid skew is kept. Between terms we wait for a - // free buffer (pm_ready) before priming the next (u_ldwait). if (u_loading) begin if (u_ld == 9'd256) begin pm_valid <= 1'b0; u_loading <= 1'b0; u_ld <= 9'd0; - if (u_ldj + 3'd1 < k_r) begin - u_ldj <= u_ldj + 3'd1; // more terms to load - u_ldwait <= 1'b1; // wait for a free buffer - end + u_oidx <= 8'd0; end else begin u_ld <= u_ld + 9'd1; pm_valid <= 1'b1; end - end else if (u_ldwait) begin - // Next term's load: start as soon as poly_mul has a free buffer. - // pm_ready high => buf[ld_ptr] free; begin presenting addr now, - // assert pm_valid next cycle (same skew as term start). - if (pm_ready) begin - u_loading <= 1'b1; - u_ld <= 9'd0; - u_ldwait <= 1'b0; - pm_valid <= 1'b0; - end end - // ---- ACCUMULATE side: consume poly_mul outputs in load order ---- if (pm_vo) begin // bank_t psum write is combinational (u_psum_we); advance idx if (u_oidx == 8'd255) begin - u_oidx <= 8'd0; if (u_j + 3'd1 < k_r) begin - u_j <= u_j + 3'd1; // next term's products + u_j <= u_j + 3'd1; + u_pending <= 1'b1; // next term, same row end else begin // row's MAC done -> INTT sub-phase u_sub <= 2'd1; @@ -1669,6 +1635,15 @@ module mlkem_top #( u_oidx <= u_oidx + 8'd1; end end + + // re-prime next term's poly_mul load + if (u_pending && pm_ready && !pm_vo) begin + u_loading <= 1'b1; + u_ld <= 9'd0; + u_oidx <= 8'd0; + pm_valid <= 1'b0; + u_pending <= 1'b0; + end end // ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ---- @@ -1714,8 +1689,6 @@ module mlkem_top #( u_row <= u_row + 3'd1; u_sub <= 2'd0; // next row MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2271,8 +2244,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2286,11 +2257,9 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; - u_ldwait <= 1'b0; u_pending <= 1'b0; pm_valid <= 1'b0; end