fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(mlkem): ping-pong poly_mul + decoupled MAC load/accumulate (B2)

Browse Source 
Double-buffer poly_mul_sync so the next MAC term's 256-cycle load overlaps
the current term's accumulate. Decouple the top-level MAC load index (u_ldj)
from the accumulate index (u_j) in the shared ENC_U/ENC_V/DEC_W engine; load
reads bank_a/bank_se while accumulate reads bank_t (disjoint ports, no
conflict). Accumulate stays strictly serial (psum order-dependent); only the
load is hidden.

Verified byte-exact vs NIST KAT in ~/Dev/mlkem: Encaps 9/9, Decaps 9/9
(incl. implicit-reject + DECOMP), KeyGen all, hello_world end-to-end.

ENC_U 5585->5073, ENC_V 2793->2537 (K=2). Larger relative win at K=4
(K^2 terms). poly_mul contract unchanged (standalone TB 11/11).
This commit is contained in:
FallenSigh
2026-07-11 02:37:28 +08:00
parent ce40a6df42
commit 975903000e
2 changed files with 173 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
rtl/ip/mlkem/poly_mul/poly_mul_sync.v
View File

@@ -1,30 +1,32 @@
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier // poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (B2 ping-pong)
// //
// Computes pointwise (Karatsuba-like base-case) multiplication of two // Computes pointwise (Karatsuba-like base-case) multiplication of two
// 256-coefficient NTT-domain polynomials. // 256-coefficient NTT-domain polynomials.
// //
// Operation flow: // Double-buffered (ping-pong) load so the caller can stream the NEXT operation's
// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // 256 A+B pairs into the idle buffer WHILE the current operation's results are
// still draining out. This hides the 256-cycle load behind the ~256-cycle
// compute/output of the previous multiply: back-to-back multiplies cost ~256
// cycles each instead of 512 (load THEN run).
// //
// RUN issues one base-case multiply every two cycles, matching the two-cycle // Contract (backward compatible with the single-shot testbench):
// c0/c1 output bandwidth after the basecase pipeline fills. // - ready_o is high whenever a buffer is free to accept a load. A fresh op is
// accepted (load_active started) as soon as the first pair arrives.
// - Each op streams 256 pairs on coeff_a_in/coeff_b_in (one per accepted cyc).
// - When a buffer is full it is handed to the compute engine, which issues one
// base-case every two cycles and emits 256 result coeffs on coeff_out via
// valid_o/ready_i (c0 then c1 per pair).
// - A caller that only ever loads-then-drains (never loads during RUN) sees
// the exact same behaviour as before, just with an extra buffer.
// //
// The LOAD phase accepts both A and B coefficients simultaneously // Interface (unchanged ports):
// (one pair per cycle) on coeff_a_in/coeff_b_in. // clk, rst_n - Clock, active-low reset
// // coeff_a_in[11:0]- Polynomial A coefficient (index = pairs accepted this load)
// The COMPUTE phase outputs the 256 result coefficients one per cycle // coeff_b_in[11:0]- Polynomial B coefficient
// via valid/ready handshake on coeff_out/valid_o/ready_i.
//
// Memory: 256×12-bit register arrays for A and B coefficients.
//
// Interface:
// clk, rst_n - Clock, active-low reset
// coeff_a_in[11:0]- Polynomial A coefficient input
// coeff_b_in[11:0]- Polynomial B coefficient input
// valid_i - Input valid // valid_i - Input valid
// ready_o - Ready to accept input (high in IDLE/LOAD) // ready_o - Ready to accept a load pair (a buffer is free)
// coeff_out[11:0] - Result coefficient output // coeff_out[11:0] - Result coefficient output
// valid_o - Output valid during RUN output phase // valid_o - Output valid
// ready_i - Output consumer ready // ready_i - Output consumer ready
module poly_mul_sync ( module poly_mul_sync (
@@ -38,150 +40,133 @@ module poly_mul_sync (
input ready_i input ready_i
); );
// State definitions // Two coefficient buffers (ping-pong).
localparam S_IDLE = 3'd0; reg [11:0] mem_A0 [0:255];
localparam S_LOAD = 3'd1; reg [11:0] mem_B0 [0:255];
localparam S_RUN = 3'd2; reg [11:0] mem_A1 [0:255];
reg [11:0] mem_B1 [0:255];
reg [2:0] state, next_state; // ---- Load side ----
reg ld_active; // currently accepting a load into ld_ptr
reg ld_ptr; // FIFO write buffer (toggles each completed load)
reg [8:0] load_cnt; // 0..256 pairs written this load
reg buf_full [0:1]; // buffer holds a complete (unconsumed) operand set
// Coefficient storage (register arrays) // ---- Compute side ----
reg [11:0] mem_A [0:255]; reg run_active; // compute engine busy on rn_ptr
reg [11:0] mem_B [0:255]; reg rn_ptr; // FIFO read buffer (toggles each completed run)
reg [7:0] issue_cnt; // 0..128 base-cases issued
reg [7:0] out_cnt; // 0..255 output coeffs consumed
reg issue_gap; // 2-cycle issue spacing
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Counters // Registered basecase inputs/results
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [7:0] issue_cnt; // 0..128, issued base-cases
reg [7:0] out_cnt; // 0..255, consumed output coefficients
reg issue_gap; // spaces base-case requests to match c0/c1 output
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Registered basecase_mul inputs/results
reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_a0_reg, bc_a1_reg;
reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_b0_reg, bc_b1_reg;
reg [11:0] bc_zeta_reg; reg [11:0] bc_zeta_reg;
reg bc_valid_reg; reg bc_valid_reg;
reg [11:0] c0_reg, c1_reg; reg [11:0] c0_reg, c1_reg;
// Combinational read signals for the next base-case request. // Compute-side memory read (from the buffer being consumed).
wire [6:0] issue_k = issue_cnt[6:0]; wire [6:0] issue_k = issue_cnt[6:0];
wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 wire [7:0] addr_even = {issue_k, 1'b0};
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 wire [7:0] addr_odd = {issue_k, 1'b1};
wire [11:0] mem_a0 = mem_A[addr_even]; wire [11:0] mem_a0 = rn_ptr ? mem_A1[addr_even] : mem_A0[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd]; wire [11:0] mem_a1 = rn_ptr ? mem_A1[addr_odd] : mem_A0[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even]; wire [11:0] mem_b0 = rn_ptr ? mem_B1[addr_even] : mem_B0[addr_even];
wire [11:0] mem_b1 = mem_B[addr_odd]; wire [11:0] mem_b1 = rn_ptr ? mem_B1[addr_odd] : mem_B0[addr_odd];
// Zeta ROM
wire [11:0] zeta; wire [11:0] zeta;
poly_mul_zeta_rom u_zeta ( poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta));
.addr (issue_k),
.zeta (zeta)
);
// Pipelined basecase multiply. Requests are issued every two cycles; inputs
// are registered locally so issue_cnt does not directly drive DSP input muxes.
wire [11:0] bc_c0, bc_c1; wire [11:0] bc_c0, bc_c1;
wire bc_vo; wire bc_vo;
basecase_mul_pipe u_bc ( basecase_mul_pipe u_bc (
.clk (clk), .clk(clk), .rst_n(rst_n),
.rst_n(rst_n),
.valid_i(bc_valid_reg), .valid_i(bc_valid_reg),
.a0 (bc_a0_reg), .a0(bc_a0_reg), .a1(bc_a1_reg),
.a1 (bc_a1_reg), .b0(bc_b0_reg), .b1(bc_b1_reg),
.b0 (bc_b0_reg),
.b1 (bc_b1_reg),
.zeta(bc_zeta_reg), .zeta(bc_zeta_reg),
.c0 (bc_c0), .c0(bc_c0), .c1(bc_c1),
.c1 (bc_c1),
.valid_o(bc_vo) .valid_o(bc_vo)
); );
// Output interface // Accept a load pair whenever the FIFO write buffer (ld_ptr) is free. Once a
assign ready_o = (state == S_IDLE) || (state == S_LOAD); // load is in progress it always finishes (ready stays high for its 256).
assign valid_o = out_valid_r; assign ready_o = ld_active ? 1'b1 : !buf_full[ld_ptr];
assign valid_o = out_valid_r;
assign coeff_out = out_phase ? c1_reg : c0_reg; assign coeff_out = out_phase ? c1_reg : c0_reg;
// State transition logic (combinational) wire ld_fire = valid_i && ready_o;
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o)
next_state = S_RUN;
S_RUN: if (valid_o && ready_i && out_cnt == 8'd255)
next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
// Sequential logic
integer i; integer i;
always @(posedge clk or negedge rst_n) begin always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin if (!rst_n) begin
state <= S_IDLE; ld_active <= 1'b0; ld_ptr <= 1'b0; load_cnt <= 9'd0;
load_cnt <= 8'd0; buf_full[0] <= 1'b0; buf_full[1] <= 1'b0;
issue_cnt <= 8'd0; run_active <= 1'b0; rn_ptr <= 1'b0;
out_cnt <= 8'd0; issue_cnt <= 8'd0; out_cnt <= 8'd0; issue_gap <= 1'b0;
issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0;
out_phase <= 1'b0; bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0;
out_valid_r <= 1'b0; bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0;
bc_a0_reg <= 12'd0; bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0;
bc_a1_reg <= 12'd0; c0_reg <= 12'd0; c1_reg <= 12'd0;
bc_b0_reg <= 12'd0;
bc_b1_reg <= 12'd0;
bc_zeta_reg <= 12'd0;
bc_valid_reg <= 1'b0;
c0_reg <= 12'd0;
c1_reg <= 12'd0;
for (i = 0; i < 256; i = i + 1) begin for (i = 0; i < 256; i = i + 1) begin
mem_A[i] = 12'd0; mem_A0[i] = 12'd0; mem_B0[i] = 12'd0;
mem_B[i] = 12'd0; mem_A1[i] = 12'd0; mem_B1[i] = 12'd0;
end end
end else begin end else begin
state <= next_state;
bc_valid_reg <= 1'b0; bc_valid_reg <= 1'b0;
// ---- LOAD phase ---- // ================= LOAD SIDE (FIFO write into ld_ptr) =================
// First coefficient captured on IDLE → LOAD transition if (!ld_active) begin
if (state == S_IDLE && valid_i && ready_o) begin if (ld_fire) begin
mem_A[0] <= coeff_a_in; if (!ld_ptr) begin
mem_B[0] <= coeff_b_in; mem_A0[0] <= coeff_a_in; mem_B0[0] <= coeff_b_in;
load_cnt <= 8'd1; end else begin
issue_cnt <= 8'd0; mem_A1[0] <= coeff_a_in; mem_B1[0] <= coeff_b_in;
out_cnt <= 8'd0; end
issue_gap <= 1'b0; load_cnt <= 9'd1;
out_phase <= 1'b0; ld_active <= 1'b1;
out_valid_r <= 1'b0; end
end end else begin
if (ld_fire) begin
// Subsequent coefficients in LOAD state if (!ld_ptr) begin
if (state == S_LOAD && valid_i && ready_o) begin mem_A0[load_cnt[7:0]] <= coeff_a_in;
mem_A[load_cnt] <= coeff_a_in; mem_B0[load_cnt[7:0]] <= coeff_b_in;
mem_B[load_cnt] <= coeff_b_in; end else begin
load_cnt <= load_cnt + 8'd1; mem_A1[load_cnt[7:0]] <= coeff_a_in;
if (load_cnt >= 8'd255) begin mem_B1[load_cnt[7:0]] <= coeff_b_in;
issue_cnt <= 8'd0; end
out_cnt <= 8'd0; if (load_cnt == 9'd255) begin
issue_gap <= 1'b0; buf_full[ld_ptr] <= 1'b1; // buffer complete
out_phase <= 1'b0; ld_ptr <= ~ld_ptr; // FIFO advance
out_valid_r <= 1'b0; ld_active <= 1'b0;
load_cnt <= 9'd0;
end else begin
load_cnt <= load_cnt + 9'd1;
end
end end
end end
// ---- COMPUTE phase ---- // ================= COMPUTE SIDE (FIFO read from rn_ptr) =============
// RUN: cut issue_cnt -> memory mux -> basecase DSP with local input if (!run_active) begin
// regs, then launch requests every other cycle. The two-cycle issue // Start compute on the FIFO head buffer once it is full (and not
// interval matches the c0/c1 output bandwidth, so no deep result FIFO // still mid-load, which cannot happen for rn_ptr==ld_ptr while
// is needed for this always-ready system. // ld_active, since buf_full is set only at load completion).
if (state == S_RUN) begin if (buf_full[rn_ptr]) begin
run_active <= 1'b1;
issue_cnt <= 8'd0; out_cnt <= 8'd0;
issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0;
end
end else begin
// Issue one base-case every two cycles.
if (issue_cnt < 8'd128) begin if (issue_cnt < 8'd128) begin
if (!issue_gap) begin if (!issue_gap) begin
bc_a0_reg <= mem_a0; bc_a0_reg <= mem_a0; bc_a1_reg <= mem_a1;
bc_a1_reg <= mem_a1; bc_b0_reg <= mem_b0; bc_b1_reg <= mem_b1;
bc_b0_reg <= mem_b0;
bc_b1_reg <= mem_b1;
bc_zeta_reg <= zeta; bc_zeta_reg <= zeta;
bc_valid_reg <= 1'b1; bc_valid_reg <= 1'b1;
issue_cnt <= issue_cnt + 8'd1; issue_cnt <= issue_cnt + 8'd1;
@@ -194,32 +179,27 @@ module poly_mul_sync (
if (out_valid_r && ready_i) begin if (out_valid_r && ready_i) begin
out_cnt <= out_cnt + 8'd1; out_cnt <= out_cnt + 8'd1;
if (out_cnt == 8'd255) begin if (out_cnt == 8'd255) begin
out_valid_r <= 1'b0; out_valid_r <= 1'b0; out_phase <= 1'b0;
out_phase <= 1'b0;
end else if (!out_phase) begin end else if (!out_phase) begin
out_phase <= 1'b1; out_phase <= 1'b1;
end else begin end else begin
out_valid_r <= 1'b0; out_valid_r <= 1'b0; out_phase <= 1'b0;
out_phase <= 1'b0;
end end
end end
if (bc_vo) begin if (bc_vo) begin
c0_reg <= bc_c0; c0_reg <= bc_c0; c1_reg <= bc_c1;
c1_reg <= bc_c1; out_valid_r <= 1'b1; out_phase <= 1'b0;
out_valid_r <= 1'b1;
out_phase <= 1'b0;
end end
end
// ---- final output consumed: reset counters before accepting a new op ---- // Final output consumed: free this buffer, advance FIFO read ptr.
if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin if (out_valid_r && ready_i && out_cnt == 8'd255) begin
load_cnt <= 8'd0; buf_full[rn_ptr] <= 1'b0;
issue_cnt <= 8'd0; rn_ptr <= ~rn_ptr;
out_cnt <= 8'd0; run_active <= 1'b0;
issue_gap <= 1'b0; issue_cnt <= 8'd0; out_cnt <= 8'd0;
out_phase <= 1'b0; issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0;
out_valid_r <= 1'b0; end
end end
end end
end end

69
rtl/ip/mlkem/top/mlkem_top.v
View File

@@ -896,10 +896,12 @@ module mlkem_top #(
reg [2:0] u_row; // output row i 0..K-1 reg [2:0] u_row; // output row i 0..K-1
reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD
// MAC bookkeeping (mirrors ST_M) // MAC bookkeeping (mirrors ST_M)
reg [2:0] u_j; // term col j 0..K-1 reg [2:0] u_j; // ACCUMULATE term col j 0..K-1 (psum order)
reg [2:0] u_ldj; // LOAD term col j 0..K-1 (runs ahead of u_j)
reg [8:0] u_ld; // load read-ahead 0..256 reg [8:0] u_ld; // load read-ahead 0..256
reg [7:0] u_oidx; // accumulate index 0..255 reg [7:0] u_oidx; // accumulate index 0..255
reg u_loading; // streaming pairs into poly_mul reg u_loading; // streaming pairs into poly_mul
reg u_ldwait; // load-side waiting for a free poly_mul buffer
reg u_pending; // wait poly_mul IDLE before next j reg u_pending; // wait poly_mul IDLE before next j
reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select) reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select)
// INTT bookkeeping (mirrors ST_N) // INTT bookkeeping (mirrors ST_N)
@@ -912,10 +914,15 @@ module mlkem_top #(
reg [8:0] u_aidx; // ADD read-ahead 0..256 reg [8:0] u_aidx; // ADD read-ahead 0..256
reg [7:0] u_awidx; // ADD write index 0..255 reg [7:0] u_awidx; // ADD write index 0..255
reg u_avalid; // ADD write enable pipeline reg u_avalid; // ADD write enable pipeline
// MAC addresses: A^T -> A_hat[u_j][u_row] slot = u_j*k+u_row; y_hat[u_j] rel u_j // MAC addresses (B2 decoupled): the LOAD side runs ahead on u_ldj (term being
wire [SAW-1:0] u_aslot = u_j*k_r + u_row; // streamed into poly_mul's idle buffer); the ACCUMULATE side stays on u_j
// (term whose products are being summed into psum). A^T -> A_hat[u_ldj][u_row]
// slot = u_ldj*k+u_row; y_hat[u_ldj] rel u_ldj. Load reads bank_a/bank_se;
// accumulate reads bank_t (psum) -> no read-port conflict, so the next term's
// load overlaps the current term's accumulate.
wire [SAW-1:0] u_aslot = u_ldj*k_r + u_row;
wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a
wire [13:0] u_pm_b_full = u_j*256 + u_ld[7:0]; // bank_se y_hat[u_j] wire [13:0] u_pm_b_full = u_ldj*256 + u_ld[7:0]; // bank_se y_hat[u_ldj]
wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx; wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx;
wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC)
wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0
@@ -1243,6 +1250,8 @@ module mlkem_top #(
u_row <= 3'd0; u_row <= 3'd0;
u_sub <= 2'd0; u_sub <= 2'd0;
u_j <= 3'd0; u_j <= 3'd0;
u_ldj <= 3'd0;
u_ldwait <= 1'b0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; u_oidx <= 8'd0;
u_loading <= 1'b0; u_loading <= 1'b0;
@@ -1591,6 +1600,8 @@ module mlkem_top #(
u_row <= 3'd0; u_row <= 3'd0;
u_sub <= 2'd0; // MAC u_sub <= 2'd0; // MAC
u_j <= 3'd0; u_j <= 3'd0;
u_ldj <= 3'd0;
u_ldwait <= 1'b0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; u_oidx <= 8'd0;
u_loading <= 1'b1; u_loading <= 1'b1;
@@ -1599,30 +1610,53 @@ module mlkem_top #(
end end
// ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ---- // ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ----
// psum (init 0 at j==0) accumulates into bank_t[UPSUM]. Mirrors ST_M // psum (init 0 at j==0) accumulates into bank_t[UPSUM].
// load/accumulate cadence (read-ahead by 1, j-select via u_j0q). // B2: decoupled load/accumulate. poly_mul is double-buffered, so the
// U: A^T[u_row][j] (bank_a u_j*K+u_row). V: t_hat[j] (bank_a u_j*K, u_row=0). // LOAD of term u_ldj can run AHEAD, overlapping the ACCUMULATE of an
// earlier term u_j (psum is order-dependent, so accumulate stays
// strictly serial; only the 256-cyc load is hidden). Load reads
// bank_a/bank_se; accumulate reads bank_t (UPSUM) — disjoint ports.
// u_ldj = term being streamed into poly_mul (0..K-1, then "done");
// u_j = term whose products are being accumulated (via u_j0q sel).
if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin
u_j0q <= (u_j == 3'd0); u_j0q <= (u_j == 3'd0);
// ---- LOAD side: stream term u_ldj into poly_mul's free buffer ----
// Within a term poly_mul holds ready high for all 256 pairs, so the
// fixed 1-cyc addr->valid skew is kept. Between terms we wait for a
// free buffer (pm_ready) before priming the next (u_ldwait).
if (u_loading) begin if (u_loading) begin
if (u_ld == 9'd256) begin if (u_ld == 9'd256) begin
pm_valid <= 1'b0; pm_valid <= 1'b0;
u_loading <= 1'b0; u_loading <= 1'b0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; if (u_ldj + 3'd1 < k_r) begin
u_ldj <= u_ldj + 3'd1; // more terms to load
u_ldwait <= 1'b1; // wait for a free buffer
end
end else begin end else begin
u_ld <= u_ld + 9'd1; u_ld <= u_ld + 9'd1;
pm_valid <= 1'b1; pm_valid <= 1'b1;
end end
end else if (u_ldwait) begin
// Next term's load: start as soon as poly_mul has a free buffer.
// pm_ready high => buf[ld_ptr] free; begin presenting addr now,
// assert pm_valid next cycle (same skew as term start).
if (pm_ready) begin
u_loading <= 1'b1;
u_ld <= 9'd0;
u_ldwait <= 1'b0;
pm_valid <= 1'b0;
end
end end
// ---- ACCUMULATE side: consume poly_mul outputs in load order ----
if (pm_vo) begin if (pm_vo) begin
// bank_t psum write is combinational (u_psum_we); advance idx // bank_t psum write is combinational (u_psum_we); advance idx
if (u_oidx == 8'd255) begin if (u_oidx == 8'd255) begin
u_oidx <= 8'd0;
if (u_j + 3'd1 < k_r) begin if (u_j + 3'd1 < k_r) begin
u_j <= u_j + 3'd1; u_j <= u_j + 3'd1; // next term's products
u_pending <= 1'b1; // next term, same row
end else begin end else begin
// row's MAC done -> INTT sub-phase // row's MAC done -> INTT sub-phase
u_sub <= 2'd1; u_sub <= 2'd1;
@@ -1635,15 +1669,6 @@ module mlkem_top #(
u_oidx <= u_oidx + 8'd1; u_oidx <= u_oidx + 8'd1;
end end
end end
// re-prime next term's poly_mul load
if (u_pending && pm_ready && !pm_vo) begin
u_loading <= 1'b1;
u_ld <= 9'd0;
u_oidx <= 8'd0;
pm_valid <= 1'b0;
u_pending <= 1'b0;
end
end end
// ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ---- // ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ----
@@ -1689,6 +1714,8 @@ module mlkem_top #(
u_row <= u_row + 3'd1; u_row <= u_row + 3'd1;
u_sub <= 2'd0; // next row MAC u_sub <= 2'd0; // next row MAC
u_j <= 3'd0; u_j <= 3'd0;
u_ldj <= 3'd0;
u_ldwait <= 1'b0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; u_oidx <= 8'd0;
u_loading <= 1'b1; u_loading <= 1'b1;
@@ -2244,6 +2271,8 @@ module mlkem_top #(
u_row <= 3'd0; u_row <= 3'd0;
u_sub <= 2'd0; // MAC u_sub <= 2'd0; // MAC
u_j <= 3'd0; u_j <= 3'd0;
u_ldj <= 3'd0;
u_ldwait <= 1'b0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; u_oidx <= 8'd0;
u_loading <= 1'b1; u_loading <= 1'b1;
@@ -2257,9 +2286,11 @@ module mlkem_top #(
u_row <= 3'd0; u_row <= 3'd0;
u_sub <= 2'd0; // MAC u_sub <= 2'd0; // MAC
u_j <= 3'd0; u_j <= 3'd0;
u_ldj <= 3'd0;
u_ld <= 9'd0; u_ld <= 9'd0;
u_oidx <= 8'd0; u_oidx <= 8'd0;
u_loading <= 1'b1; u_loading <= 1'b1;
u_ldwait <= 1'b0;
u_pending <= 1'b0; u_pending <= 1'b0;
pm_valid <= 1'b0; pm_valid <= 1'b0;
end end