From ce40a6df42791b48ff4a123b26b7a48f9ba18325 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 02:00:03 +0800 Subject: [PATCH 1/8] =?UTF-8?q?perf(mlkem):=20stream=20compress=20feed=20(?= =?UTF-8?q?B1)=20=E2=80=94=20~27%=20fewer=20Encaps=20cycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decouple the ENC_C1/C2 compress micro-sequence into a streaming producer/collector instead of feeding one coeff and draining the whole 6-stage comp_decomp pipeline per coeff. - comp_decomp_sync: ready_o = 1'b1 (fully pipelined, consumer never backpressures since ready_i is tied high), so feeders can issue a new coeff every cycle. - mlkem_top ENC_C1/C2: producer feeds one coeff every 2 cycles (<=5.5 bit/cyc, under the single-port ct drain of 8 bit/cyc so the bit buffer stays bounded); a per-cycle collector appends compressed bits and drains one ct byte/cycle. Verified byte-exact against NIST KAT (Vivado 2019.2 XSIM) for KeyGen, Encaps (9/9), Decaps (9/9 incl. reject path + DECOMP), hello_world e2e. K=2 Encaps: ENC_C1 6273->1035, ENC_C2 2945->522, total 27851->20190. Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v | 10 +- rtl/ip/mlkem/top/mlkem_top.v | 164 ++++++++++++-------- 2 files changed, 103 insertions(+), 71 deletions(-) diff --git a/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v index 2690bf1..9861dd1 100644 --- a/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v +++ b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v @@ -75,9 +75,13 @@ module comp_decomp_sync ( wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5; - // Accept a new command only when the pipeline and output slot are both free. - // Current users issue one coefficient at a time and wait for valid_o. - assign ready_o = !pipe_busy && (!valid_r || ready_i); + // Fully pipelined: accept one coefficient per cycle. Every stage advances + // unconditionally (valid_sN <= valid_s(N-1)), and the sole consumer ties + // ready_i high, so the output register never needs to backpressure the + // pipeline. Streaming feeders (Encaps C1/C2, Decaps DECOMP) can therefore + // issue a new coeff every cycle instead of draining the 6-stage pipe each + // time. pipe_busy retained only for the legacy drain-based users' info. + assign ready_o = 1'b1; assign coeff_out = data_r; assign valid_o = valid_r; diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 293188d..306df7a 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -1041,13 +1041,18 @@ module mlkem_top #( .ready_i(1'b1) ); // bit-packer / ct walk bookkeeping - reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v - reg [7:0] cp_idx; // coeff 0..255 within poly - reg [2:0] cp_ph; // micro-phase - reg [24:0] cp_buf; // bit accumulator (LSB-first) - reg [5:0] cp_nbits; // valid bits in cp_buf + reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v (feed side) + reg [7:0] cp_idx; // coeff 0..255 within poly (feed side, drives bank addr) + reg [2:0] cp_ph; // feed micro-phase toggle (0=addr settle, 1=feed) + reg [63:0] cp_buf; // bit accumulator (LSB-first); widened for streaming + reg [6:0] cp_nbits; // valid bits in cp_buf reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2) reg cp_done; // serialization complete (this region) + reg cp_fed_done; // all coeffs for this region have been fed in + reg [10:0] cp_fcnt; // coeffs fed into comp_decomp this region + reg [10:0] cp_ccnt; // compressed coeffs collected this region + // total coeffs this region: c1 = K*256 (u rows), c2 = 256 (single v) + wire [10:0] cp_ntot = (st == ST_ENC_C2) ? 11'd256 : ({8'd0, k_r} << 8); // coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly); // c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0). wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly] @@ -1260,10 +1265,13 @@ module mlkem_top #( cp_poly <= 3'd0; cp_idx <= 8'd0; cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; cp_wa <= 12'd0; cp_done <= 1'b0; + cp_fed_done<= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; ct_we <= 1'b0; ct_wa <= 11'd0; ct_wd <= 8'd0; @@ -1695,69 +1703,86 @@ module mlkem_top #( // Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)). // Walk K polys * 256 coeffs; reset bit-packer + ct write pointer. if (st == ST_ENC_U && st_next == ST_ENC_C1) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_wa <= 12'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_wa <= 12'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; end // ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ---- // C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes). // C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes). - // Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram plus - // variable-latency comp_decomp), then a drain sub-phase emitting whole bytes: - // ph0: present coeff addr (cp_se_full / cp_bt_full by state). - // ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid. - // ph2: drop cd_valid (1-cyc pulse); comp_decomp captures. - // ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first. - // ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance. - // Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at - // each poly boundary (no carry across polys / regions). + // + // Streaming (B1): the comp_decomp pipeline is fully pipelined and its + // sole output consumer never backpressures (ready_i=1'b1), so instead + // of feeding one coeff and draining the whole pipe per coeff (~12 cyc), + // we run a decoupled producer + collector: + // PRODUCER (2-cycle cadence): ph0 lets the bank read address settle + // (1-cyc sd_bram latency), ph1 latches the coeff and pulses + // cd_valid, then advances the address. One coeff enters the pipe + // every 2 cycles = <=5.5 bits/cyc (du<=11), strictly under the + // single-port ct drain rate (8 bits/cyc), so cp_buf stays bounded. + // COLLECTOR (every cycle): when cd_vo, append the compressed cp_d + // bits LSB-first; independently emit one ct byte/cycle whenever + // >=8 bits are buffered. Coeffs come out in-order, one per pipe + // output. Each poly is 256*d bits (multiple of 8), so the buffer + // empties to 0 at region end (cp_done when all collected+drained). if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin - case (cp_ph) - 3'd0: cp_ph <= 3'd1; // addr presented; wait read - 3'd1: begin - cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) - cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse) - cp_ph <= 3'd2; - end - 3'd2: begin - cd_valid <= 1'b0; // comp_decomp captured this cyc - cp_ph <= 3'd3; - end - 3'd3: begin - if (cd_vo) begin - // Append compressed low cp_d bits LSB-first at bit cp_nbits. - cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits); - cp_nbits <= cp_nbits + {1'b0, cp_d}; - cp_ph <= 3'd4; - end - end - default: begin // 3'd4: drain whole bytes - if (cp_nbits >= 6'd8) begin - ct_we <= 1'b1; - ct_wa <= cp_wa; - ct_wd <= cp_buf[7:0]; - cp_wa <= cp_wa + 12'd1; - cp_buf <= cp_buf >> 8; - cp_nbits <= cp_nbits - 6'd8; + // ---- PRODUCER: feed one coeff every 2 cycles ---- + cd_valid <= 1'b0; + if (!cp_fed_done) begin + if (cp_ph == 3'd0) begin + cp_ph <= 3'd1; // address settling + end else begin + cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) + cd_valid <= 1'b1; // 1-cyc feed pulse + cp_fcnt <= cp_fcnt + 11'd1; + if (cp_fcnt + 11'd1 >= cp_ntot) cp_fed_done <= 1'b1; + if (cp_idx == 8'd255) begin + cp_idx <= 8'd0; + cp_poly <= cp_poly + 3'd1; // next u row (c1); c2 unused end else begin - // coeff fully packed; advance coeff / poly - if (cp_idx == 8'd255) begin - cp_idx <= 8'd0; - if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1; - else cp_done <= 1'b1; // region complete - end else begin - cp_idx <= cp_idx + 8'd1; - end - cp_ph <= 3'd0; + cp_idx <= cp_idx + 8'd1; end + cp_ph <= 3'd0; end - endcase + end + + // ---- COLLECTOR: append compressed bits + drain ct bytes ---- + if (cd_vo) begin + // append cp_d bits at cp_nbits, then drain one byte if >=8. + if ((cp_nbits + {1'b0, cp_d}) >= 7'd8) begin + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) & 64'hFF; + cp_wa <= cp_wa + 12'd1; + cp_buf <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) >> 8; + cp_nbits <= (cp_nbits + {1'b0, cp_d}) - 7'd8; + end else begin + cp_buf <= cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits); + cp_nbits <= cp_nbits + {1'b0, cp_d}; + end + cp_ccnt <= cp_ccnt + 11'd1; // one compressed coeff collected + end else if (cp_nbits >= 7'd8) begin + // no new coeff this cycle: keep draining buffered whole bytes. + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= cp_buf[7:0]; + cp_wa <= cp_wa + 12'd1; + cp_buf <= cp_buf >> 8; + cp_nbits <= cp_nbits - 7'd8; + end + + // region complete once all coeffs collected and buffer drained. + if (cp_ccnt >= cp_ntot && cp_nbits < 7'd8) + cp_done <= 1'b1; end // Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)). @@ -1765,13 +1790,16 @@ module mlkem_top #( // from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits // are 0 here (c1 ended on a poly/byte boundary) but reset for safety. if (st == ST_ENC_V && st_next == ST_ENC_C2) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; // cp_wa intentionally preserved (= c1_bytes_rt from C1). end From 975903000e8c1af5045cc85a51748d90f1a2b593 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 02:37:28 +0800 Subject: [PATCH 2/8] perf(mlkem): ping-pong poly_mul + decoupled MAC load/accumulate (B2) Double-buffer poly_mul_sync so the next MAC term's 256-cycle load overlaps the current term's accumulate. Decouple the top-level MAC load index (u_ldj) from the accumulate index (u_j) in the shared ENC_U/ENC_V/DEC_W engine; load reads bank_a/bank_se while accumulate reads bank_t (disjoint ports, no conflict). Accumulate stays strictly serial (psum order-dependent); only the load is hidden. Verified byte-exact vs NIST KAT in ~/Dev/mlkem: Encaps 9/9, Decaps 9/9 (incl. implicit-reject + DECOMP), KeyGen all, hello_world end-to-end. ENC_U 5585->5073, ENC_V 2793->2537 (K=2). Larger relative win at K=4 (K^2 terms). poly_mul contract unchanged (standalone TB 11/11). --- rtl/ip/mlkem/poly_mul/poly_mul_sync.v | 266 ++++++++++++-------------- rtl/ip/mlkem/top/mlkem_top.v | 69 +++++-- 2 files changed, 173 insertions(+), 162 deletions(-) diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v index 61f8f99..46e9ac4 100644 --- a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -1,30 +1,32 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (B2 ping-pong) // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. // -// Operation flow: -// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE +// Double-buffered (ping-pong) load so the caller can stream the NEXT operation's +// 256 A+B pairs into the idle buffer WHILE the current operation's results are +// still draining out. This hides the 256-cycle load behind the ~256-cycle +// compute/output of the previous multiply: back-to-back multiplies cost ~256 +// cycles each instead of 512 (load THEN run). // -// RUN issues one base-case multiply every two cycles, matching the two-cycle -// c0/c1 output bandwidth after the basecase pipeline fills. +// Contract (backward compatible with the single-shot testbench): +// - ready_o is high whenever a buffer is free to accept a load. A fresh op is +// accepted (load_active started) as soon as the first pair arrives. +// - Each op streams 256 pairs on coeff_a_in/coeff_b_in (one per accepted cyc). +// - When a buffer is full it is handed to the compute engine, which issues one +// base-case every two cycles and emits 256 result coeffs on coeff_out via +// valid_o/ready_i (c0 then c1 per pair). +// - A caller that only ever loads-then-drains (never loads during RUN) sees +// the exact same behaviour as before, just with an extra buffer. // -// The LOAD phase accepts both A and B coefficients simultaneously -// (one pair per cycle) on coeff_a_in/coeff_b_in. -// -// The COMPUTE phase outputs the 256 result coefficients one per cycle -// via valid/ready handshake on coeff_out/valid_o/ready_i. -// -// Memory: 256×12-bit register arrays for A and B coefficients. -// -// Interface: -// clk, rst_n - Clock, active-low reset -// coeff_a_in[11:0]- Polynomial A coefficient input -// coeff_b_in[11:0]- Polynomial B coefficient input +// Interface (unchanged ports): +// clk, rst_n - Clock, active-low reset +// coeff_a_in[11:0]- Polynomial A coefficient (index = pairs accepted this load) +// coeff_b_in[11:0]- Polynomial B coefficient // valid_i - Input valid -// ready_o - Ready to accept input (high in IDLE/LOAD) +// ready_o - Ready to accept a load pair (a buffer is free) // coeff_out[11:0] - Result coefficient output -// valid_o - Output valid during RUN output phase +// valid_o - Output valid // ready_i - Output consumer ready module poly_mul_sync ( @@ -38,150 +40,133 @@ module poly_mul_sync ( input ready_i ); - // State definitions - localparam S_IDLE = 3'd0; - localparam S_LOAD = 3'd1; - localparam S_RUN = 3'd2; + // Two coefficient buffers (ping-pong). + reg [11:0] mem_A0 [0:255]; + reg [11:0] mem_B0 [0:255]; + reg [11:0] mem_A1 [0:255]; + reg [11:0] mem_B1 [0:255]; - reg [2:0] state, next_state; + // ---- Load side ---- + reg ld_active; // currently accepting a load into ld_ptr + reg ld_ptr; // FIFO write buffer (toggles each completed load) + reg [8:0] load_cnt; // 0..256 pairs written this load + reg buf_full [0:1]; // buffer holds a complete (unconsumed) operand set - // Coefficient storage (register arrays) - reg [11:0] mem_A [0:255]; - reg [11:0] mem_B [0:255]; + // ---- Compute side ---- + reg run_active; // compute engine busy on rn_ptr + reg rn_ptr; // FIFO read buffer (toggles each completed run) + reg [7:0] issue_cnt; // 0..128 base-cases issued + reg [7:0] out_cnt; // 0..255 output coeffs consumed + reg issue_gap; // 2-cycle issue spacing + reg out_phase; // 0=c0, 1=c1 + reg out_valid_r; - // Counters - reg [7:0] load_cnt; // 0..256 for loading 256 pairs - reg [7:0] issue_cnt; // 0..128, issued base-cases - reg [7:0] out_cnt; // 0..255, consumed output coefficients - reg issue_gap; // spaces base-case requests to match c0/c1 output - reg out_phase; // 0=c0, 1=c1 - reg out_valid_r; - - // Registered basecase_mul inputs/results + // Registered basecase inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Combinational read signals for the next base-case request. + // Compute-side memory read (from the buffer being consumed). wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 - wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 - wire [11:0] mem_a0 = mem_A[addr_even]; - wire [11:0] mem_a1 = mem_A[addr_odd]; - wire [11:0] mem_b0 = mem_B[addr_even]; - wire [11:0] mem_b1 = mem_B[addr_odd]; + wire [7:0] addr_even = {issue_k, 1'b0}; + wire [7:0] addr_odd = {issue_k, 1'b1}; + wire [11:0] mem_a0 = rn_ptr ? mem_A1[addr_even] : mem_A0[addr_even]; + wire [11:0] mem_a1 = rn_ptr ? mem_A1[addr_odd] : mem_A0[addr_odd]; + wire [11:0] mem_b0 = rn_ptr ? mem_B1[addr_even] : mem_B0[addr_even]; + wire [11:0] mem_b1 = rn_ptr ? mem_B1[addr_odd] : mem_B0[addr_odd]; - // Zeta ROM wire [11:0] zeta; - poly_mul_zeta_rom u_zeta ( - .addr (issue_k), - .zeta (zeta) - ); + poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); - // Pipelined basecase multiply. Requests are issued every two cycles; inputs - // are registered locally so issue_cnt does not directly drive DSP input muxes. wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk (clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0 (bc_a0_reg), - .a1 (bc_a1_reg), - .b0 (bc_b0_reg), - .b1 (bc_b1_reg), + .a0(bc_a0_reg), .a1(bc_a1_reg), + .b0(bc_b0_reg), .b1(bc_b1_reg), .zeta(bc_zeta_reg), - .c0 (bc_c0), - .c1 (bc_c1), + .c0(bc_c0), .c1(bc_c1), .valid_o(bc_vo) ); - // Output interface - assign ready_o = (state == S_IDLE) || (state == S_LOAD); - assign valid_o = out_valid_r; + // Accept a load pair whenever the FIFO write buffer (ld_ptr) is free. Once a + // load is in progress it always finishes (ready stays high for its 256). + assign ready_o = ld_active ? 1'b1 : !buf_full[ld_ptr]; + + assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - // State transition logic (combinational) - always @* begin - next_state = state; - case (state) - S_IDLE: if (valid_i && ready_o) next_state = S_LOAD; - S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o) - next_state = S_RUN; - S_RUN: if (valid_o && ready_i && out_cnt == 8'd255) - next_state = S_IDLE; - default: next_state = S_IDLE; - endcase - end + wire ld_fire = valid_i && ready_o; - // Sequential logic integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - state <= S_IDLE; - load_cnt <= 8'd0; - issue_cnt <= 8'd0; - out_cnt <= 8'd0; - issue_gap <= 1'b0; - out_phase <= 1'b0; - out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; - bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; - bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; - bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; - c1_reg <= 12'd0; + ld_active <= 1'b0; ld_ptr <= 1'b0; load_cnt <= 9'd0; + buf_full[0] <= 1'b0; buf_full[1] <= 1'b0; + run_active <= 1'b0; rn_ptr <= 1'b0; + issue_cnt <= 8'd0; out_cnt <= 8'd0; issue_gap <= 1'b0; + out_phase <= 1'b0; out_valid_r <= 1'b0; + bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; c1_reg <= 12'd0; for (i = 0; i < 256; i = i + 1) begin - mem_A[i] = 12'd0; - mem_B[i] = 12'd0; + mem_A0[i] = 12'd0; mem_B0[i] = 12'd0; + mem_A1[i] = 12'd0; mem_B1[i] = 12'd0; end end else begin - state <= next_state; bc_valid_reg <= 1'b0; - // ---- LOAD phase ---- - // First coefficient captured on IDLE → LOAD transition - if (state == S_IDLE && valid_i && ready_o) begin - mem_A[0] <= coeff_a_in; - mem_B[0] <= coeff_b_in; - load_cnt <= 8'd1; - issue_cnt <= 8'd0; - out_cnt <= 8'd0; - issue_gap <= 1'b0; - out_phase <= 1'b0; - out_valid_r <= 1'b0; - end - - // Subsequent coefficients in LOAD state - if (state == S_LOAD && valid_i && ready_o) begin - mem_A[load_cnt] <= coeff_a_in; - mem_B[load_cnt] <= coeff_b_in; - load_cnt <= load_cnt + 8'd1; - if (load_cnt >= 8'd255) begin - issue_cnt <= 8'd0; - out_cnt <= 8'd0; - issue_gap <= 1'b0; - out_phase <= 1'b0; - out_valid_r <= 1'b0; + // ================= LOAD SIDE (FIFO write into ld_ptr) ================= + if (!ld_active) begin + if (ld_fire) begin + if (!ld_ptr) begin + mem_A0[0] <= coeff_a_in; mem_B0[0] <= coeff_b_in; + end else begin + mem_A1[0] <= coeff_a_in; mem_B1[0] <= coeff_b_in; + end + load_cnt <= 9'd1; + ld_active <= 1'b1; + end + end else begin + if (ld_fire) begin + if (!ld_ptr) begin + mem_A0[load_cnt[7:0]] <= coeff_a_in; + mem_B0[load_cnt[7:0]] <= coeff_b_in; + end else begin + mem_A1[load_cnt[7:0]] <= coeff_a_in; + mem_B1[load_cnt[7:0]] <= coeff_b_in; + end + if (load_cnt == 9'd255) begin + buf_full[ld_ptr] <= 1'b1; // buffer complete + ld_ptr <= ~ld_ptr; // FIFO advance + ld_active <= 1'b0; + load_cnt <= 9'd0; + end else begin + load_cnt <= load_cnt + 9'd1; + end end end - // ---- COMPUTE phase ---- - // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input - // regs, then launch requests every other cycle. The two-cycle issue - // interval matches the c0/c1 output bandwidth, so no deep result FIFO - // is needed for this always-ready system. - if (state == S_RUN) begin + // ================= COMPUTE SIDE (FIFO read from rn_ptr) ============= + if (!run_active) begin + // Start compute on the FIFO head buffer once it is full (and not + // still mid-load, which cannot happen for rn_ptr==ld_ptr while + // ld_active, since buf_full is set only at load completion). + if (buf_full[rn_ptr]) begin + run_active <= 1'b1; + issue_cnt <= 8'd0; out_cnt <= 8'd0; + issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; + end + end else begin + // Issue one base-case every two cycles. if (issue_cnt < 8'd128) begin if (!issue_gap) begin - bc_a0_reg <= mem_a0; - bc_a1_reg <= mem_a1; - bc_b0_reg <= mem_b0; - bc_b1_reg <= mem_b1; + bc_a0_reg <= mem_a0; bc_a1_reg <= mem_a1; + bc_b0_reg <= mem_b0; bc_b1_reg <= mem_b1; bc_zeta_reg <= zeta; bc_valid_reg <= 1'b1; issue_cnt <= issue_cnt + 8'd1; @@ -194,32 +179,27 @@ module poly_mul_sync ( if (out_valid_r && ready_i) begin out_cnt <= out_cnt + 8'd1; if (out_cnt == 8'd255) begin - out_valid_r <= 1'b0; - out_phase <= 1'b0; + out_valid_r <= 1'b0; out_phase <= 1'b0; end else if (!out_phase) begin out_phase <= 1'b1; end else begin - out_valid_r <= 1'b0; - out_phase <= 1'b0; + out_valid_r <= 1'b0; out_phase <= 1'b0; end end if (bc_vo) begin - c0_reg <= bc_c0; - c1_reg <= bc_c1; - out_valid_r <= 1'b1; - out_phase <= 1'b0; + c0_reg <= bc_c0; c1_reg <= bc_c1; + out_valid_r <= 1'b1; out_phase <= 1'b0; end - end - // ---- final output consumed: reset counters before accepting a new op ---- - if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin - load_cnt <= 8'd0; - issue_cnt <= 8'd0; - out_cnt <= 8'd0; - issue_gap <= 1'b0; - out_phase <= 1'b0; - out_valid_r <= 1'b0; + // Final output consumed: free this buffer, advance FIFO read ptr. + if (out_valid_r && ready_i && out_cnt == 8'd255) begin + buf_full[rn_ptr] <= 1'b0; + rn_ptr <= ~rn_ptr; + run_active <= 1'b0; + issue_cnt <= 8'd0; out_cnt <= 8'd0; + issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; + end end end end diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 306df7a..3ee72ac 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -896,10 +896,12 @@ module mlkem_top #( reg [2:0] u_row; // output row i 0..K-1 reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD // MAC bookkeeping (mirrors ST_M) - reg [2:0] u_j; // term col j 0..K-1 + reg [2:0] u_j; // ACCUMULATE term col j 0..K-1 (psum order) + reg [2:0] u_ldj; // LOAD term col j 0..K-1 (runs ahead of u_j) reg [8:0] u_ld; // load read-ahead 0..256 reg [7:0] u_oidx; // accumulate index 0..255 reg u_loading; // streaming pairs into poly_mul + reg u_ldwait; // load-side waiting for a free poly_mul buffer reg u_pending; // wait poly_mul IDLE before next j reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select) // INTT bookkeeping (mirrors ST_N) @@ -912,10 +914,15 @@ module mlkem_top #( reg [8:0] u_aidx; // ADD read-ahead 0..256 reg [7:0] u_awidx; // ADD write index 0..255 reg u_avalid; // ADD write enable pipeline - // MAC addresses: A^T -> A_hat[u_j][u_row] slot = u_j*k+u_row; y_hat[u_j] rel u_j - wire [SAW-1:0] u_aslot = u_j*k_r + u_row; + // MAC addresses (B2 decoupled): the LOAD side runs ahead on u_ldj (term being + // streamed into poly_mul's idle buffer); the ACCUMULATE side stays on u_j + // (term whose products are being summed into psum). A^T -> A_hat[u_ldj][u_row] + // slot = u_ldj*k+u_row; y_hat[u_ldj] rel u_ldj. Load reads bank_a/bank_se; + // accumulate reads bank_t (psum) -> no read-port conflict, so the next term's + // load overlaps the current term's accumulate. + wire [SAW-1:0] u_aslot = u_ldj*k_r + u_row; wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a - wire [13:0] u_pm_b_full = u_j*256 + u_ld[7:0]; // bank_se y_hat[u_j] + wire [13:0] u_pm_b_full = u_ldj*256 + u_ld[7:0]; // bank_se y_hat[u_ldj] wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx; wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 @@ -1243,6 +1250,8 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; u_j <= 3'd0; + u_ldj <= 3'd0; + u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b0; @@ -1591,6 +1600,8 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; + u_ldj <= 3'd0; + u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -1599,30 +1610,53 @@ module mlkem_top #( end // ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ---- - // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. Mirrors ST_M - // load/accumulate cadence (read-ahead by 1, j-select via u_j0q). - // U: A^T[u_row][j] (bank_a u_j*K+u_row). V: t_hat[j] (bank_a u_j*K, u_row=0). + // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. + // B2: decoupled load/accumulate. poly_mul is double-buffered, so the + // LOAD of term u_ldj can run AHEAD, overlapping the ACCUMULATE of an + // earlier term u_j (psum is order-dependent, so accumulate stays + // strictly serial; only the 256-cyc load is hidden). Load reads + // bank_a/bank_se; accumulate reads bank_t (UPSUM) — disjoint ports. + // u_ldj = term being streamed into poly_mul (0..K-1, then "done"); + // u_j = term whose products are being accumulated (via u_j0q sel). if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin u_j0q <= (u_j == 3'd0); + // ---- LOAD side: stream term u_ldj into poly_mul's free buffer ---- + // Within a term poly_mul holds ready high for all 256 pairs, so the + // fixed 1-cyc addr->valid skew is kept. Between terms we wait for a + // free buffer (pm_ready) before priming the next (u_ldwait). if (u_loading) begin if (u_ld == 9'd256) begin pm_valid <= 1'b0; u_loading <= 1'b0; u_ld <= 9'd0; - u_oidx <= 8'd0; + if (u_ldj + 3'd1 < k_r) begin + u_ldj <= u_ldj + 3'd1; // more terms to load + u_ldwait <= 1'b1; // wait for a free buffer + end end else begin u_ld <= u_ld + 9'd1; pm_valid <= 1'b1; end + end else if (u_ldwait) begin + // Next term's load: start as soon as poly_mul has a free buffer. + // pm_ready high => buf[ld_ptr] free; begin presenting addr now, + // assert pm_valid next cycle (same skew as term start). + if (pm_ready) begin + u_loading <= 1'b1; + u_ld <= 9'd0; + u_ldwait <= 1'b0; + pm_valid <= 1'b0; + end end + // ---- ACCUMULATE side: consume poly_mul outputs in load order ---- if (pm_vo) begin // bank_t psum write is combinational (u_psum_we); advance idx if (u_oidx == 8'd255) begin + u_oidx <= 8'd0; if (u_j + 3'd1 < k_r) begin - u_j <= u_j + 3'd1; - u_pending <= 1'b1; // next term, same row + u_j <= u_j + 3'd1; // next term's products end else begin // row's MAC done -> INTT sub-phase u_sub <= 2'd1; @@ -1635,15 +1669,6 @@ module mlkem_top #( u_oidx <= u_oidx + 8'd1; end end - - // re-prime next term's poly_mul load - if (u_pending && pm_ready && !pm_vo) begin - u_loading <= 1'b1; - u_ld <= 9'd0; - u_oidx <= 8'd0; - pm_valid <= 1'b0; - u_pending <= 1'b0; - end end // ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ---- @@ -1689,6 +1714,8 @@ module mlkem_top #( u_row <= u_row + 3'd1; u_sub <= 2'd0; // next row MAC u_j <= 3'd0; + u_ldj <= 3'd0; + u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2244,6 +2271,8 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; + u_ldj <= 3'd0; + u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2257,9 +2286,11 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; + u_ldj <= 3'd0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; + u_ldwait <= 1'b0; u_pending <= 1'b0; pm_valid <= 1'b0; end From 454c3e559d7843e7ee755e38784226dc5d0dca74 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 03:05:35 +0800 Subject: [PATCH 3/8] =?UTF-8?q?perf(mlkem):=20dual-butterfly=20NTT=20core?= =?UTF-8?q?=20(R1)=20=E2=80=94=20~12%=20fewer=20Encaps=20cycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue 2 butterflies/cycle from the same NTT group (positions p,p+1). ML-KEM's 7 layers have layer_len in {128..2} (always even, never 1), so each pair shares one zeta and touches 4 distinct coefficient indices — no intra-group RAW conflict, single zeta ROM. Compute per transform ~966->~490 cycles. Verified: standalone ntt TB 19/19 (fwd+inv), full KAT Encaps 9/9 + Decaps 9/9 + KeyGen + hello_world all PASS (all K). --- rtl/ip/mlkem/ntt/ntt_core.v | 137 ++++++++++++++++++++---------------- 1 file changed, 75 insertions(+), 62 deletions(-) diff --git a/rtl/ip/mlkem/ntt/ntt_core.v b/rtl/ip/mlkem/ntt/ntt_core.v index 676583a..74402f4 100644 --- a/rtl/ip/mlkem/ntt/ntt_core.v +++ b/rtl/ip/mlkem/ntt/ntt_core.v @@ -1,9 +1,13 @@ -// ntt_core.v - NTT core with individual coefficient registers +// ntt_core.v - NTT core with individual coefficient registers (2 butterflies) // -// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path. -// The arithmetic hot path is split into: -// address/operand/zeta register -> pipelined Barrett butterfly -> writeback -// Each NTT layer is issued continuously into the butterfly pipeline, then the +// Uses 256 individual 12-bit registers and TWO deeply pipelined butterfly paths +// running in lock-step (R1: dual-butterfly parallelism). Each issue cycle emits +// two butterflies from the SAME group at positions (p, p+1); since ML-KEM's NTT +// has 7 layers with layer_len in {128..2} (always even, never 1), the pair always +// shares one zeta and touches four distinct coefficient indices, so a single +// zeta ROM suffices and there is no intra-group read/write conflict. +// address/operand/zeta register -> 2x pipelined Barrett butterfly -> writeback +// Each NTT layer issues 2 butterflies/cycle (64 cycles/layer vs 128), then the // core drains pending writebacks before starting the next dependent layer. // In inverse mode, final x3303 output scaling also uses a pipelined Barrett // multiplier so the output path does not reintroduce a combinational reducer. @@ -45,13 +49,16 @@ module ntt_core ( reg bf_done; reg mode_r; reg layer_issue_done; - reg [4:0] bf_inflight; + reg [5:0] bf_inflight; // pending issue cycles (pairs) in flight - reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7; - reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7; + // Two writeback address delay lines (lane0: j, lane1: j+1), each +len. + reg [7:0] a0_d0,a0_d1,a0_d2,a0_d3,a0_d4,a0_d5,a0_d6,a0_d7; + reg [7:0] b0_d0,b0_d1,b0_d2,b0_d3,b0_d4,b0_d5,b0_d6,b0_d7; + reg [7:0] a1_d0,a1_d1,a1_d2,a1_d3,a1_d4,a1_d5,a1_d6,a1_d7; + reg [7:0] b1_d0,b1_d1,b1_d2,b1_d3,b1_d4,b1_d5,b1_d6,b1_d7; reg wb_valid_r; - reg [DW-1:0] wb_a_data_r, wb_b_data_r; - reg [7:0] wb_wa_r, wb_wb_r; + reg [DW-1:0] wb_a0_r, wb_b0_r, wb_a1_r, wb_b1_r; + reg [7:0] wb_a0a_r, wb_b0a_r, wb_a1a_r, wb_b1a_r; reg [DW-1:0] coeff_out_r; reg valid_o_r; @@ -63,38 +70,42 @@ module ntt_core ( wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len}; wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len}; - wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end); + // pair (j, j+1) is the last in this group when j+2 reaches group_end + wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end); wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256); wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1); wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done; - wire [DW-1:0] issue_a = cr[j]; - wire [DW-1:0] issue_b = cr[j + layer_len]; - wire [DW-1:0] bf_a_out, bf_b_out; - wire bf_valid; - butterfly_unit_pipe u_bf ( - .clk(clk), - .rst_n(rst_n), + // lane0 = position j, lane1 = position j+1 (same group => same zeta) + wire [DW-1:0] issue_a0 = cr[j]; + wire [DW-1:0] issue_b0 = cr[j + layer_len]; + wire [DW-1:0] issue_a1 = cr[j + 8'd1]; + wire [DW-1:0] issue_b1 = cr[j + 8'd1 + layer_len]; + + wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out; + wire bf0_valid, bf1_valid; + butterfly_unit_pipe u_bf0 ( + .clk(clk), .rst_n(rst_n), .valid_i(issue_fire), - .a(issue_a), - .b(issue_b), - .zeta(zeta), - .mode(mode_r), - .a_out(bf_a_out), - .b_out(bf_b_out), - .valid_o(bf_valid) + .a(issue_a0), .b(issue_b0), + .zeta(zeta), .mode(mode_r), + .a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid) + ); + butterfly_unit_pipe u_bf1 ( + .clk(clk), .rst_n(rst_n), + .valid_i(issue_fire), + .a(issue_a1), .b(issue_b1), + .zeta(zeta), .mode(mode_r), + .a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid) ); wire [DW-1:0] scale_product; wire scale_valid_o; barrett_mul_pipe u_scl ( - .clk(clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(scale_valid_i), - .a(scale_a_i), - .b(12'd3303), - .product(scale_product), - .valid_o(scale_valid_o) + .a(scale_a_i), .b(12'd3303), + .product(scale_product), .valid_o(scale_valid_o) ); assign ready_o = (state == S_IDLE) || (state == S_LOAD); @@ -108,7 +119,7 @@ module ntt_core ( S_IDLE: if (valid_i) next_state = S_LOAD; S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN; S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN; - S_CMP_DRAIN: if (bf_inflight == 5'd0) + S_CMP_DRAIN: if (bf_inflight == 6'd0) next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN; S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; @@ -117,7 +128,6 @@ module ntt_core ( default: next_state = S_IDLE; endcase end - always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; @@ -133,16 +143,14 @@ module ntt_core ( bf_done <= 1'b0; mode_r <= 1'b0; layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; - wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0; - wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0; - wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0; - wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0; + bf_inflight <= 6'd0; + a0_d0<=8'd0;a0_d1<=8'd0;a0_d2<=8'd0;a0_d3<=8'd0;a0_d4<=8'd0;a0_d5<=8'd0;a0_d6<=8'd0;a0_d7<=8'd0; + b0_d0<=8'd0;b0_d1<=8'd0;b0_d2<=8'd0;b0_d3<=8'd0;b0_d4<=8'd0;b0_d5<=8'd0;b0_d6<=8'd0;b0_d7<=8'd0; + a1_d0<=8'd0;a1_d1<=8'd0;a1_d2<=8'd0;a1_d3<=8'd0;a1_d4<=8'd0;a1_d5<=8'd0;a1_d6<=8'd0;a1_d7<=8'd0; + b1_d0<=8'd0;b1_d1<=8'd0;b1_d2<=8'd0;b1_d3<=8'd0;b1_d4<=8'd0;b1_d5<=8'd0;b1_d6<=8'd0;b1_d7<=8'd0; wb_valid_r <= 1'b0; - wb_a_data_r <= 12'd0; - wb_b_data_r <= 12'd0; - wb_wa_r <= 8'd0; - wb_wb_r <= 8'd0; + wb_a0_r <= 12'd0; wb_b0_r <= 12'd0; wb_a1_r <= 12'd0; wb_b1_r <= 12'd0; + wb_a0a_r <= 8'd0; wb_b0a_r <= 8'd0; wb_a1a_r <= 8'd0; wb_b1a_r <= 8'd0; coeff_out_r <= 12'd0; valid_o_r <= 1'b0; scale_valid_i <= 1'b0; @@ -152,15 +160,17 @@ module ntt_core ( state <= next_state; scale_valid_i <= 1'b0; - wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2; - wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6; - wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2; - wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6; + // Writeback address delay lines (8-deep = butterfly latency). + a0_d0<=j; a0_d1<=a0_d0;a0_d2<=a0_d1;a0_d3<=a0_d2;a0_d4<=a0_d3;a0_d5<=a0_d4;a0_d6<=a0_d5;a0_d7<=a0_d6; + b0_d0<=j+layer_len; b0_d1<=b0_d0;b0_d2<=b0_d1;b0_d3<=b0_d2;b0_d4<=b0_d3;b0_d5<=b0_d4;b0_d6<=b0_d5;b0_d7<=b0_d6; + a1_d0<=j+8'd1; a1_d1<=a1_d0;a1_d2<=a1_d1;a1_d3<=a1_d2;a1_d4<=a1_d3;a1_d5<=a1_d4;a1_d6<=a1_d5;a1_d7<=a1_d6; + b1_d0<=j+8'd1+layer_len; b1_d1<=b1_d0;b1_d2<=b1_d1;b1_d3<=b1_d2;b1_d4<=b1_d3;b1_d5<=b1_d4;b1_d6<=b1_d5;b1_d7<=b1_d6; + // In-flight pair accounting (mirrors single-butterfly version). if (issue_fire && !wb_valid_r) - bf_inflight <= bf_inflight + 5'd1; + bf_inflight <= bf_inflight + 6'd1; else if (!issue_fire && wb_valid_r) - bf_inflight <= bf_inflight - 5'd1; + bf_inflight <= bf_inflight - 6'd1; if (state != S_OUTPUT && state != S_OUT_SCALE) valid_o_r <= 1'b0; @@ -176,7 +186,7 @@ module ntt_core ( layer <= 3'd0; bf_done <= 1'b0; layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; + bf_inflight <= 6'd0; mode_r <= mode; if (!mode) begin layer_len <= 8'd128; @@ -192,6 +202,7 @@ module ntt_core ( load_cnt <= load_cnt + 8'd1; end + // ---- issue advance: step j by 2 (two butterflies/cycle) ---- if (state == S_CMP_RUN && !layer_issue_done) begin if (issue_group_last) begin if (!mode_r) zeta_idx <= zeta_idx + 7'd1; @@ -204,24 +215,28 @@ module ntt_core ( j <= next_group_start[7:0]; end end else begin - j <= j + 8'd1; + j <= j + 8'd2; end end + // ---- writeback: commit previous cycle's pair results ---- if (wb_valid_r) begin - cr[wb_wa_r] <= wb_a_data_r; - cr[wb_wb_r] <= wb_b_data_r; + cr[wb_a0a_r] <= wb_a0_r; + cr[wb_b0a_r] <= wb_b0_r; + cr[wb_a1a_r] <= wb_a1_r; + cr[wb_b1a_r] <= wb_b1_r; end - wb_valid_r <= bf_valid; - if (bf_valid) begin - wb_a_data_r <= bf_a_out; - wb_b_data_r <= bf_b_out; - wb_wa_r <= wa_d7; - wb_wb_r <= wb_d7; + wb_valid_r <= bf0_valid; // both lanes share latency/valid + if (bf0_valid) begin + wb_a0_r <= bf0_a_out; wb_b0_r <= bf0_b_out; + wb_a1_r <= bf1_a_out; wb_b1_r <= bf1_b_out; + wb_a0a_r <= a0_d7; wb_b0a_r <= b0_d7; + wb_a1a_r <= a1_d7; wb_b1a_r <= b1_d7; end - if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin + // ---- layer advance after drain ---- + if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin if (layer + 3'd1 >= LAYERS) begin bf_done <= 1'b1; end else begin @@ -260,7 +275,6 @@ module ntt_core ( scale_a_i <= cr[scale_issue_cnt[7:0]]; scale_issue_cnt <= scale_issue_cnt + 9'd1; end - valid_o_r <= 1'b0; if (scale_valid_o) begin coeff_out_r <= scale_product; @@ -275,11 +289,10 @@ module ntt_core ( scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; + bf_inflight <= 6'd0; wb_valid_r <= 1'b0; valid_o_r <= 1'b0; end end end - endmodule From be2e54c8abca7d6d71a3109d992b7be3b7ce3a11 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 03:48:50 +0800 Subject: [PATCH 4/8] revert(mlkem): drop poly_mul ping-pong (B2) to reclaim ~7k LUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B2 (ping-pong poly_mul + decoupled MAC load) doubled the 256x12 coeff register file (mem_A/B -> 4 arrays) costing ~7k LUT for only ~2.8% Encaps latency (K=2) — the worst area/latency ratio of the optimization set. Revert restores single-buffer poly_mul and serial MAC. B1 (compress streaming) and R1 (dual-butterfly NTT) are kept. Verified byte-exact vs NIST KAT: Encaps 9/9, Decaps 9/9, KeyGen, hello end-to-end, poly_mul standalone TB 11/11 — all PASS. ENC_U K=2 4177->4689, ENC_V 2089->2345 (B2's hidden load reappears). --- rtl/ip/mlkem/poly_mul/poly_mul_sync.v | 266 ++++++++++++++------------ rtl/ip/mlkem/top/mlkem_top.v | 69 ++----- 2 files changed, 162 insertions(+), 173 deletions(-) diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v index 46e9ac4..61f8f99 100644 --- a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -1,32 +1,30 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (B2 ping-pong) +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. // -// Double-buffered (ping-pong) load so the caller can stream the NEXT operation's -// 256 A+B pairs into the idle buffer WHILE the current operation's results are -// still draining out. This hides the 256-cycle load behind the ~256-cycle -// compute/output of the previous multiply: back-to-back multiplies cost ~256 -// cycles each instead of 512 (load THEN run). +// Operation flow: +// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // -// Contract (backward compatible with the single-shot testbench): -// - ready_o is high whenever a buffer is free to accept a load. A fresh op is -// accepted (load_active started) as soon as the first pair arrives. -// - Each op streams 256 pairs on coeff_a_in/coeff_b_in (one per accepted cyc). -// - When a buffer is full it is handed to the compute engine, which issues one -// base-case every two cycles and emits 256 result coeffs on coeff_out via -// valid_o/ready_i (c0 then c1 per pair). -// - A caller that only ever loads-then-drains (never loads during RUN) sees -// the exact same behaviour as before, just with an extra buffer. +// RUN issues one base-case multiply every two cycles, matching the two-cycle +// c0/c1 output bandwidth after the basecase pipeline fills. // -// Interface (unchanged ports): -// clk, rst_n - Clock, active-low reset -// coeff_a_in[11:0]- Polynomial A coefficient (index = pairs accepted this load) -// coeff_b_in[11:0]- Polynomial B coefficient +// The LOAD phase accepts both A and B coefficients simultaneously +// (one pair per cycle) on coeff_a_in/coeff_b_in. +// +// The COMPUTE phase outputs the 256 result coefficients one per cycle +// via valid/ready handshake on coeff_out/valid_o/ready_i. +// +// Memory: 256×12-bit register arrays for A and B coefficients. +// +// Interface: +// clk, rst_n - Clock, active-low reset +// coeff_a_in[11:0]- Polynomial A coefficient input +// coeff_b_in[11:0]- Polynomial B coefficient input // valid_i - Input valid -// ready_o - Ready to accept a load pair (a buffer is free) +// ready_o - Ready to accept input (high in IDLE/LOAD) // coeff_out[11:0] - Result coefficient output -// valid_o - Output valid +// valid_o - Output valid during RUN output phase // ready_i - Output consumer ready module poly_mul_sync ( @@ -40,133 +38,150 @@ module poly_mul_sync ( input ready_i ); - // Two coefficient buffers (ping-pong). - reg [11:0] mem_A0 [0:255]; - reg [11:0] mem_B0 [0:255]; - reg [11:0] mem_A1 [0:255]; - reg [11:0] mem_B1 [0:255]; + // State definitions + localparam S_IDLE = 3'd0; + localparam S_LOAD = 3'd1; + localparam S_RUN = 3'd2; - // ---- Load side ---- - reg ld_active; // currently accepting a load into ld_ptr - reg ld_ptr; // FIFO write buffer (toggles each completed load) - reg [8:0] load_cnt; // 0..256 pairs written this load - reg buf_full [0:1]; // buffer holds a complete (unconsumed) operand set + reg [2:0] state, next_state; - // ---- Compute side ---- - reg run_active; // compute engine busy on rn_ptr - reg rn_ptr; // FIFO read buffer (toggles each completed run) - reg [7:0] issue_cnt; // 0..128 base-cases issued - reg [7:0] out_cnt; // 0..255 output coeffs consumed - reg issue_gap; // 2-cycle issue spacing - reg out_phase; // 0=c0, 1=c1 - reg out_valid_r; + // Coefficient storage (register arrays) + reg [11:0] mem_A [0:255]; + reg [11:0] mem_B [0:255]; - // Registered basecase inputs/results + // Counters + reg [7:0] load_cnt; // 0..256 for loading 256 pairs + reg [7:0] issue_cnt; // 0..128, issued base-cases + reg [7:0] out_cnt; // 0..255, consumed output coefficients + reg issue_gap; // spaces base-case requests to match c0/c1 output + reg out_phase; // 0=c0, 1=c1 + reg out_valid_r; + + // Registered basecase_mul inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Compute-side memory read (from the buffer being consumed). + // Combinational read signals for the next base-case request. wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; - wire [7:0] addr_odd = {issue_k, 1'b1}; - wire [11:0] mem_a0 = rn_ptr ? mem_A1[addr_even] : mem_A0[addr_even]; - wire [11:0] mem_a1 = rn_ptr ? mem_A1[addr_odd] : mem_A0[addr_odd]; - wire [11:0] mem_b0 = rn_ptr ? mem_B1[addr_even] : mem_B0[addr_even]; - wire [11:0] mem_b1 = rn_ptr ? mem_B1[addr_odd] : mem_B0[addr_odd]; + wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 + wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 + wire [11:0] mem_a0 = mem_A[addr_even]; + wire [11:0] mem_a1 = mem_A[addr_odd]; + wire [11:0] mem_b0 = mem_B[addr_even]; + wire [11:0] mem_b1 = mem_B[addr_odd]; + // Zeta ROM wire [11:0] zeta; - poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); + poly_mul_zeta_rom u_zeta ( + .addr (issue_k), + .zeta (zeta) + ); + // Pipelined basecase multiply. Requests are issued every two cycles; inputs + // are registered locally so issue_cnt does not directly drive DSP input muxes. wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk(clk), .rst_n(rst_n), + .clk (clk), + .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0(bc_a0_reg), .a1(bc_a1_reg), - .b0(bc_b0_reg), .b1(bc_b1_reg), + .a0 (bc_a0_reg), + .a1 (bc_a1_reg), + .b0 (bc_b0_reg), + .b1 (bc_b1_reg), .zeta(bc_zeta_reg), - .c0(bc_c0), .c1(bc_c1), + .c0 (bc_c0), + .c1 (bc_c1), .valid_o(bc_vo) ); - // Accept a load pair whenever the FIFO write buffer (ld_ptr) is free. Once a - // load is in progress it always finishes (ready stays high for its 256). - assign ready_o = ld_active ? 1'b1 : !buf_full[ld_ptr]; - - assign valid_o = out_valid_r; + // Output interface + assign ready_o = (state == S_IDLE) || (state == S_LOAD); + assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - wire ld_fire = valid_i && ready_o; + // State transition logic (combinational) + always @* begin + next_state = state; + case (state) + S_IDLE: if (valid_i && ready_o) next_state = S_LOAD; + S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o) + next_state = S_RUN; + S_RUN: if (valid_o && ready_i && out_cnt == 8'd255) + next_state = S_IDLE; + default: next_state = S_IDLE; + endcase + end + // Sequential logic integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - ld_active <= 1'b0; ld_ptr <= 1'b0; load_cnt <= 9'd0; - buf_full[0] <= 1'b0; buf_full[1] <= 1'b0; - run_active <= 1'b0; rn_ptr <= 1'b0; - issue_cnt <= 8'd0; out_cnt <= 8'd0; issue_gap <= 1'b0; - out_phase <= 1'b0; out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; c1_reg <= 12'd0; + state <= S_IDLE; + load_cnt <= 8'd0; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; + bc_a0_reg <= 12'd0; + bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; + bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; + bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; + c1_reg <= 12'd0; for (i = 0; i < 256; i = i + 1) begin - mem_A0[i] = 12'd0; mem_B0[i] = 12'd0; - mem_A1[i] = 12'd0; mem_B1[i] = 12'd0; + mem_A[i] = 12'd0; + mem_B[i] = 12'd0; end end else begin + state <= next_state; bc_valid_reg <= 1'b0; - // ================= LOAD SIDE (FIFO write into ld_ptr) ================= - if (!ld_active) begin - if (ld_fire) begin - if (!ld_ptr) begin - mem_A0[0] <= coeff_a_in; mem_B0[0] <= coeff_b_in; - end else begin - mem_A1[0] <= coeff_a_in; mem_B1[0] <= coeff_b_in; - end - load_cnt <= 9'd1; - ld_active <= 1'b1; - end - end else begin - if (ld_fire) begin - if (!ld_ptr) begin - mem_A0[load_cnt[7:0]] <= coeff_a_in; - mem_B0[load_cnt[7:0]] <= coeff_b_in; - end else begin - mem_A1[load_cnt[7:0]] <= coeff_a_in; - mem_B1[load_cnt[7:0]] <= coeff_b_in; - end - if (load_cnt == 9'd255) begin - buf_full[ld_ptr] <= 1'b1; // buffer complete - ld_ptr <= ~ld_ptr; // FIFO advance - ld_active <= 1'b0; - load_cnt <= 9'd0; - end else begin - load_cnt <= load_cnt + 9'd1; - end + // ---- LOAD phase ---- + // First coefficient captured on IDLE → LOAD transition + if (state == S_IDLE && valid_i && ready_o) begin + mem_A[0] <= coeff_a_in; + mem_B[0] <= coeff_b_in; + load_cnt <= 8'd1; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; + end + + // Subsequent coefficients in LOAD state + if (state == S_LOAD && valid_i && ready_o) begin + mem_A[load_cnt] <= coeff_a_in; + mem_B[load_cnt] <= coeff_b_in; + load_cnt <= load_cnt + 8'd1; + if (load_cnt >= 8'd255) begin + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; end end - // ================= COMPUTE SIDE (FIFO read from rn_ptr) ============= - if (!run_active) begin - // Start compute on the FIFO head buffer once it is full (and not - // still mid-load, which cannot happen for rn_ptr==ld_ptr while - // ld_active, since buf_full is set only at load completion). - if (buf_full[rn_ptr]) begin - run_active <= 1'b1; - issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - end - end else begin - // Issue one base-case every two cycles. + // ---- COMPUTE phase ---- + // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input + // regs, then launch requests every other cycle. The two-cycle issue + // interval matches the c0/c1 output bandwidth, so no deep result FIFO + // is needed for this always-ready system. + if (state == S_RUN) begin if (issue_cnt < 8'd128) begin if (!issue_gap) begin - bc_a0_reg <= mem_a0; bc_a1_reg <= mem_a1; - bc_b0_reg <= mem_b0; bc_b1_reg <= mem_b1; + bc_a0_reg <= mem_a0; + bc_a1_reg <= mem_a1; + bc_b0_reg <= mem_b0; + bc_b1_reg <= mem_b1; bc_zeta_reg <= zeta; bc_valid_reg <= 1'b1; issue_cnt <= issue_cnt + 8'd1; @@ -179,27 +194,32 @@ module poly_mul_sync ( if (out_valid_r && ready_i) begin out_cnt <= out_cnt + 8'd1; if (out_cnt == 8'd255) begin - out_valid_r <= 1'b0; out_phase <= 1'b0; + out_valid_r <= 1'b0; + out_phase <= 1'b0; end else if (!out_phase) begin out_phase <= 1'b1; end else begin - out_valid_r <= 1'b0; out_phase <= 1'b0; + out_valid_r <= 1'b0; + out_phase <= 1'b0; end end if (bc_vo) begin - c0_reg <= bc_c0; c1_reg <= bc_c1; - out_valid_r <= 1'b1; out_phase <= 1'b0; + c0_reg <= bc_c0; + c1_reg <= bc_c1; + out_valid_r <= 1'b1; + out_phase <= 1'b0; end + end - // Final output consumed: free this buffer, advance FIFO read ptr. - if (out_valid_r && ready_i && out_cnt == 8'd255) begin - buf_full[rn_ptr] <= 1'b0; - rn_ptr <= ~rn_ptr; - run_active <= 1'b0; - issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - end + // ---- final output consumed: reset counters before accepting a new op ---- + if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin + load_cnt <= 8'd0; + issue_cnt <= 8'd0; + out_cnt <= 8'd0; + issue_gap <= 1'b0; + out_phase <= 1'b0; + out_valid_r <= 1'b0; end end end diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 3ee72ac..306df7a 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -896,12 +896,10 @@ module mlkem_top #( reg [2:0] u_row; // output row i 0..K-1 reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD // MAC bookkeeping (mirrors ST_M) - reg [2:0] u_j; // ACCUMULATE term col j 0..K-1 (psum order) - reg [2:0] u_ldj; // LOAD term col j 0..K-1 (runs ahead of u_j) + reg [2:0] u_j; // term col j 0..K-1 reg [8:0] u_ld; // load read-ahead 0..256 reg [7:0] u_oidx; // accumulate index 0..255 reg u_loading; // streaming pairs into poly_mul - reg u_ldwait; // load-side waiting for a free poly_mul buffer reg u_pending; // wait poly_mul IDLE before next j reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select) // INTT bookkeeping (mirrors ST_N) @@ -914,15 +912,10 @@ module mlkem_top #( reg [8:0] u_aidx; // ADD read-ahead 0..256 reg [7:0] u_awidx; // ADD write index 0..255 reg u_avalid; // ADD write enable pipeline - // MAC addresses (B2 decoupled): the LOAD side runs ahead on u_ldj (term being - // streamed into poly_mul's idle buffer); the ACCUMULATE side stays on u_j - // (term whose products are being summed into psum). A^T -> A_hat[u_ldj][u_row] - // slot = u_ldj*k+u_row; y_hat[u_ldj] rel u_ldj. Load reads bank_a/bank_se; - // accumulate reads bank_t (psum) -> no read-port conflict, so the next term's - // load overlaps the current term's accumulate. - wire [SAW-1:0] u_aslot = u_ldj*k_r + u_row; + // MAC addresses: A^T -> A_hat[u_j][u_row] slot = u_j*k+u_row; y_hat[u_j] rel u_j + wire [SAW-1:0] u_aslot = u_j*k_r + u_row; wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a - wire [13:0] u_pm_b_full = u_ldj*256 + u_ld[7:0]; // bank_se y_hat[u_ldj] + wire [13:0] u_pm_b_full = u_j*256 + u_ld[7:0]; // bank_se y_hat[u_j] wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx; wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 @@ -1250,8 +1243,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b0; @@ -1600,8 +1591,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -1610,53 +1599,30 @@ module mlkem_top #( end // ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ---- - // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. - // B2: decoupled load/accumulate. poly_mul is double-buffered, so the - // LOAD of term u_ldj can run AHEAD, overlapping the ACCUMULATE of an - // earlier term u_j (psum is order-dependent, so accumulate stays - // strictly serial; only the 256-cyc load is hidden). Load reads - // bank_a/bank_se; accumulate reads bank_t (UPSUM) — disjoint ports. - // u_ldj = term being streamed into poly_mul (0..K-1, then "done"); - // u_j = term whose products are being accumulated (via u_j0q sel). + // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. Mirrors ST_M + // load/accumulate cadence (read-ahead by 1, j-select via u_j0q). + // U: A^T[u_row][j] (bank_a u_j*K+u_row). V: t_hat[j] (bank_a u_j*K, u_row=0). if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin u_j0q <= (u_j == 3'd0); - // ---- LOAD side: stream term u_ldj into poly_mul's free buffer ---- - // Within a term poly_mul holds ready high for all 256 pairs, so the - // fixed 1-cyc addr->valid skew is kept. Between terms we wait for a - // free buffer (pm_ready) before priming the next (u_ldwait). if (u_loading) begin if (u_ld == 9'd256) begin pm_valid <= 1'b0; u_loading <= 1'b0; u_ld <= 9'd0; - if (u_ldj + 3'd1 < k_r) begin - u_ldj <= u_ldj + 3'd1; // more terms to load - u_ldwait <= 1'b1; // wait for a free buffer - end + u_oidx <= 8'd0; end else begin u_ld <= u_ld + 9'd1; pm_valid <= 1'b1; end - end else if (u_ldwait) begin - // Next term's load: start as soon as poly_mul has a free buffer. - // pm_ready high => buf[ld_ptr] free; begin presenting addr now, - // assert pm_valid next cycle (same skew as term start). - if (pm_ready) begin - u_loading <= 1'b1; - u_ld <= 9'd0; - u_ldwait <= 1'b0; - pm_valid <= 1'b0; - end end - // ---- ACCUMULATE side: consume poly_mul outputs in load order ---- if (pm_vo) begin // bank_t psum write is combinational (u_psum_we); advance idx if (u_oidx == 8'd255) begin - u_oidx <= 8'd0; if (u_j + 3'd1 < k_r) begin - u_j <= u_j + 3'd1; // next term's products + u_j <= u_j + 3'd1; + u_pending <= 1'b1; // next term, same row end else begin // row's MAC done -> INTT sub-phase u_sub <= 2'd1; @@ -1669,6 +1635,15 @@ module mlkem_top #( u_oidx <= u_oidx + 8'd1; end end + + // re-prime next term's poly_mul load + if (u_pending && pm_ready && !pm_vo) begin + u_loading <= 1'b1; + u_ld <= 9'd0; + u_oidx <= 8'd0; + pm_valid <= 1'b0; + u_pending <= 1'b0; + end end // ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ---- @@ -1714,8 +1689,6 @@ module mlkem_top #( u_row <= u_row + 3'd1; u_sub <= 2'd0; // next row MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2271,8 +2244,6 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; - u_ldwait <= 1'b0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; @@ -2286,11 +2257,9 @@ module mlkem_top #( u_row <= 3'd0; u_sub <= 2'd0; // MAC u_j <= 3'd0; - u_ldj <= 3'd0; u_ld <= 9'd0; u_oidx <= 8'd0; u_loading <= 1'b1; - u_ldwait <= 1'b0; u_pending <= 1'b0; pm_valid <= 1'b0; end From 78ccc345781031f0d1531eee8f4a643fb74592b5 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 05:16:58 +0800 Subject: [PATCH 5/8] perf(mlkem): move NTT coeff storage to 4-way banked BRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ntt_core's 256x12 LUT register file (multi-port async read, ~18.5k LUT) with coeff_bram_4way: 4 sd_bram banks, conflict-free banking function bank(a)={a[0], ^a[7:1]}, local(a)=a[7:1]. The dual-butterfly's four accesses {j, j+1, j+len, j+1+len} map to four distinct banks for every layer_len in {2..128}, so each bank needs only 1 read + 1 write/cycle (single buffer, no ping-pong). Trades ~13k LUT for +4 BRAM (device BRAM at ~6%). Pipeline gains one stage for the synchronous BRAM read; writeback address delay deepened to 9. coeff_out is driven combinationally from the BRAM read during OUTPUT; unused read ports mirror port 0 outside compute so the read crossbar arbitration is not polluted by stale addresses. Verified byte-exact vs NIST KAT: NTT standalone TB 19/19 (fwd+inv), Encaps 9/9, Decaps 9/9 (incl. reject + DECOMP), KeyGen, hello_world e2e — all PASS. Cycle counts unchanged (ENC_N +4 for BRAM latency). Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/ntt/coeff_bram_4way.v | 113 +++++++++++ rtl/ip/mlkem/ntt/ntt_core.v | 290 ++++++++++++++--------------- 2 files changed, 257 insertions(+), 146 deletions(-) create mode 100644 rtl/ip/mlkem/ntt/coeff_bram_4way.v diff --git a/rtl/ip/mlkem/ntt/coeff_bram_4way.v b/rtl/ip/mlkem/ntt/coeff_bram_4way.v new file mode 100644 index 0000000..c68e718 --- /dev/null +++ b/rtl/ip/mlkem/ntt/coeff_bram_4way.v @@ -0,0 +1,113 @@ +// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients +// +// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM +// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree. +// +// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits) +// local(a) = a[7:1] (7-bit index into the 128-deep bank) +// +// This function makes a dual-butterfly's four simultaneous coefficient accesses +// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT +// layer_len in {2,4,...,128}: j is even and len is even, so +// j -> {0, P } +// j+1 -> {1, P } +// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity) +// j+1+len -> {1, P'} +// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four +// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank +// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong. +// +// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this +// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle. +// +// NOTE: the caller must guarantee the four read addresses map to four distinct +// banks (true for the NTT butterfly pattern) and the four write addresses too. +// Sequential access (load/output) uses a single port and is always safe. + +module coeff_bram_4way ( + input clk, rst_n, + // Four read addresses (global 8-bit); data returns 1 cycle later. + input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3, + output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3, + // Four write ports. + input wr_en_0, wr_en_1, wr_en_2, wr_en_3, + input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3, + input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3 +); + + // Bank id and local index for an address. + function [1:0] bank_id; + input [7:0] a; + bank_id = {a[0], ^a[7:1]}; + endfunction + + wire [1:0] rb0 = bank_id(rd_addr_0); + wire [1:0] rb1 = bank_id(rd_addr_1); + wire [1:0] rb2 = bank_id(rd_addr_2); + wire [1:0] rb3 = bank_id(rd_addr_3); + + wire [1:0] wb0 = bank_id(wr_addr_0); + wire [1:0] wb1 = bank_id(wr_addr_1); + wire [1:0] wb2 = bank_id(wr_addr_2); + wire [1:0] wb3 = bank_id(wr_addr_3); + + // Per-bank read-address crossbar: bank b reads the local index of whichever + // read port targets it (exactly one, by construction). + wire [6:0] bank_rd_local [0:3]; + wire [6:0] bank_wr_local [0:3]; + wire bank_wr_en [0:3]; + wire [11:0] bank_wr_data [0:3]; + wire [11:0] bank_rd_data [0:3]; + + genvar b; + generate + for (b = 0; b < 4; b = b + 1) begin : gen_xbar + // read address route (priority: 3>2>1>0; ties impossible for NTT) + assign bank_rd_local[b] = + (rb3 == b[1:0]) ? rd_addr_3[7:1] : + (rb2 == b[1:0]) ? rd_addr_2[7:1] : + (rb1 == b[1:0]) ? rd_addr_1[7:1] : + rd_addr_0[7:1]; + // write route + wire m0 = wr_en_0 && (wb0 == b[1:0]); + wire m1 = wr_en_1 && (wb1 == b[1:0]); + wire m2 = wr_en_2 && (wb2 == b[1:0]); + wire m3 = wr_en_3 && (wb3 == b[1:0]); + assign bank_wr_en[b] = m0 | m1 | m2 | m3; + assign bank_wr_local[b] = + m3 ? wr_addr_3[7:1] : + m2 ? wr_addr_2[7:1] : + m1 ? wr_addr_1[7:1] : wr_addr_0[7:1]; + assign bank_wr_data[b] = + m3 ? wr_data_3 : + m2 ? wr_data_2 : + m1 ? wr_data_1 : wr_data_0; + + sd_bram #(.W(12), .D(128), .A(7)) u_bank ( + .clk(clk), + .rd_addr(bank_rd_local[b]), + .rd_data(bank_rd_data[b]), + .wr_en(bank_wr_en[b]), + .wr_addr(bank_wr_local[b]), + .wr_data(bank_wr_data[b]) + ); + end + endgenerate + + // Register the read bank-ids to align with sd_bram's 1-cycle read latency, + // then route each bank's output back to the requesting read port. + reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0; + end else begin + rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3; + end + end + + assign rd_data_0 = bank_rd_data[rb0_r]; + assign rd_data_1 = bank_rd_data[rb1_r]; + assign rd_data_2 = bank_rd_data[rb2_r]; + assign rd_data_3 = bank_rd_data[rb3_r]; + +endmodule diff --git a/rtl/ip/mlkem/ntt/ntt_core.v b/rtl/ip/mlkem/ntt/ntt_core.v index 74402f4..681c1b8 100644 --- a/rtl/ip/mlkem/ntt/ntt_core.v +++ b/rtl/ip/mlkem/ntt/ntt_core.v @@ -1,16 +1,18 @@ -// ntt_core.v - NTT core with individual coefficient registers (2 butterflies) +// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage // -// Uses 256 individual 12-bit registers and TWO deeply pipelined butterfly paths -// running in lock-step (R1: dual-butterfly parallelism). Each issue cycle emits -// two butterflies from the SAME group at positions (p, p+1); since ML-KEM's NTT -// has 7 layers with layer_len in {128..2} (always even, never 1), the pair always -// shares one zeta and touches four distinct coefficient indices, so a single -// zeta ROM suffices and there is no intra-group read/write conflict. -// address/operand/zeta register -> 2x pipelined Barrett butterfly -> writeback -// Each NTT layer issues 2 butterflies/cycle (64 cycles/layer vs 128), then the -// core drains pending writebacks before starting the next dependent layer. -// In inverse mode, final x3303 output scaling also uses a pipelined Barrett -// multiplier so the output path does not reintroduce a combinational reducer. +// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way +// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port +// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle. +// +// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline +// gains one stage: +// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid) +// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer. +// Read and writeback hit disjoint banks (butterfly index sets in a layer are +// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W). +// +// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read +// latency (ready_i is tied high by all NTT consumers, so no output backpressure). module ntt_core ( input clk, rst_n, @@ -25,9 +27,6 @@ module ntt_core ( ); localparam N = 256, LAYERS = 7, DW = 12; - reg [DW-1:0] cr [0:N-1]; - integer ci; - localparam S_IDLE = 4'd0; localparam S_LOAD = 4'd1; localparam S_CMP_RUN = 4'd2; @@ -40,65 +39,78 @@ module ntt_core ( reg [3:0] state, next_state; reg [7:0] load_cnt; - reg [7:0] out_cnt; - reg [8:0] scale_issue_cnt; - reg [8:0] scale_emit_cnt; + reg [8:0] out_rptr; // output read address pointer (0..256) + reg [8:0] out_cnt; // output coeffs emitted (0..256) + reg [8:0] scale_rptr; // scale read address pointer + reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett + reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett reg [7:0] j, start, layer_len; reg [6:0] zeta_idx; reg [2:0] layer; - reg bf_done; reg mode_r; reg layer_issue_done; - reg [5:0] bf_inflight; // pending issue cycles (pairs) in flight + reg [5:0] bf_inflight; - // Two writeback address delay lines (lane0: j, lane1: j+1), each +len. - reg [7:0] a0_d0,a0_d1,a0_d2,a0_d3,a0_d4,a0_d5,a0_d6,a0_d7; - reg [7:0] b0_d0,b0_d1,b0_d2,b0_d3,b0_d4,b0_d5,b0_d6,b0_d7; - reg [7:0] a1_d0,a1_d1,a1_d2,a1_d3,a1_d4,a1_d5,a1_d6,a1_d7; - reg [7:0] b1_d0,b1_d1,b1_d2,b1_d3,b1_d4,b1_d5,b1_d6,b1_d7; - reg wb_valid_r; - reg [DW-1:0] wb_a0_r, wb_b0_r, wb_a1_r, wb_b1_r; - reg [7:0] wb_a0a_r, wb_b0a_r, wb_a1a_r, wb_b1a_r; + reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed + // 9-deep writeback address delay lines (BRAM read + 8-stage butterfly). + reg [7:0] a0d[0:8]; reg [7:0] b0d[0:8]; + reg [7:0] a1d[0:8]; reg [7:0] b1d[0:8]; + integer k; reg [DW-1:0] coeff_out_r; reg valid_o_r; reg scale_valid_i; reg [DW-1:0] scale_a_i; + reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present) wire [DW-1:0] zeta; zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len}; wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len}; - // pair (j, j+1) is the last in this group when j+2 reaches group_end wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end); wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256); wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1); wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done; - // lane0 = position j, lane1 = position j+1 (same group => same zeta) - wire [DW-1:0] issue_a0 = cr[j]; - wire [DW-1:0] issue_b0 = cr[j + layer_len]; - wire [DW-1:0] issue_a1 = cr[j + 8'd1]; - wire [DW-1:0] issue_b1 = cr[j + 8'd1 + layer_len]; + // ---- coeff_bram_4way port muxing ---- + // Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr). + wire [7:0] cmp_a0 = j; + wire [7:0] cmp_b0 = j + layer_len; + wire [7:0] cmp_a1 = j + 8'd1; + wire [7:0] cmp_b1 = j + 8'd1 + layer_len; + wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] : + (state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0; + // Ports 1..3 carry the other three butterfly operands ONLY during compute; + // otherwise they must mirror port 0 so the read crossbar's per-bank priority + // arbitration doesn't route a stale compute address into port 0's bank. + wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN); + wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0; + wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0; + wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0; + wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3; + + // Butterflies fed from BRAM read data one cycle after the address (rd_fire_r). wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out; wire bf0_valid, bf1_valid; butterfly_unit_pipe u_bf0 ( .clk(clk), .rst_n(rst_n), - .valid_i(issue_fire), - .a(issue_a0), .b(issue_b0), - .zeta(zeta), .mode(mode_r), + .valid_i(rd_fire_r), + .a(rd_data_0), .b(rd_data_1), + .zeta(zeta_fed), .mode(mode_r), .a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid) ); butterfly_unit_pipe u_bf1 ( .clk(clk), .rst_n(rst_n), - .valid_i(issue_fire), - .a(issue_a1), .b(issue_b1), - .zeta(zeta), .mode(mode_r), + .valid_i(rd_fire_r), + .a(rd_data_2), .b(rd_data_3), + .zeta(zeta_fed), .mode(mode_r), .a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid) ); + // zeta must align with the BRAM read data (1 cycle after presentation). + // Scale multiplier (inverse NTT final x3303). wire [DW-1:0] scale_product; wire scale_valid_o; barrett_mul_pipe u_scl ( @@ -108,8 +120,41 @@ module ntt_core ( .product(scale_product), .valid_o(scale_valid_o) ); + // ---- write ports ---- + wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i); + wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt; + + wire wr_en_0 = loading_we | bf0_valid; + wire wr_en_1 = bf0_valid & ~loading_we; + wire wr_en_2 = bf0_valid & ~loading_we; + wire wr_en_3 = bf0_valid & ~loading_we; + wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[8]; + wire [7:0] wr_addr_1 = b0d[8]; + wire [7:0] wr_addr_2 = a1d[8]; + wire [7:0] wr_addr_3 = b1d[8]; + wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out; + wire [11:0] wr_data_1 = bf0_b_out; + wire [11:0] wr_data_2 = bf1_a_out; + wire [11:0] wr_data_3 = bf1_b_out; + + coeff_bram_4way u_cr ( + .clk(clk), .rst_n(rst_n), + .rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1), + .rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3), + .rd_data_0(rd_data_0), .rd_data_1(rd_data_1), + .rd_data_2(rd_data_2), .rd_data_3(rd_data_3), + .wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3), + .wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1), + .wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3), + .wr_data_0(wr_data_0), .wr_data_1(wr_data_1), + .wr_data_2(wr_data_2), .wr_data_3(wr_data_3) + ); + assign ready_o = (state == S_IDLE) || (state == S_LOAD); - assign coeff_out = coeff_out_r; + // During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle + // latency, matching the old combinational-read + 1 output reg). Scale path + // uses the registered barrett result. + assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r; assign valid_o = valid_o_r; assign done_o = (state == S_DONE); @@ -122,157 +167,113 @@ module ntt_core ( S_CMP_DRAIN: if (bf_inflight == 6'd0) next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN; S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; - S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; + S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE; S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE; S_DONE: next_state = S_IDLE; default: next_state = S_IDLE; endcase end + always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer_len <= 8'd0; - zeta_idx <= 7'd0; - layer <= 3'd0; - bf_done <= 1'b0; - mode_r <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; - a0_d0<=8'd0;a0_d1<=8'd0;a0_d2<=8'd0;a0_d3<=8'd0;a0_d4<=8'd0;a0_d5<=8'd0;a0_d6<=8'd0;a0_d7<=8'd0; - b0_d0<=8'd0;b0_d1<=8'd0;b0_d2<=8'd0;b0_d3<=8'd0;b0_d4<=8'd0;b0_d5<=8'd0;b0_d6<=8'd0;b0_d7<=8'd0; - a1_d0<=8'd0;a1_d1<=8'd0;a1_d2<=8'd0;a1_d3<=8'd0;a1_d4<=8'd0;a1_d5<=8'd0;a1_d6<=8'd0;a1_d7<=8'd0; - b1_d0<=8'd0;b1_d1<=8'd0;b1_d2<=8'd0;b1_d3<=8'd0;b1_d4<=8'd0;b1_d5<=8'd0;b1_d6<=8'd0;b1_d7<=8'd0; - wb_valid_r <= 1'b0; - wb_a0_r <= 12'd0; wb_b0_r <= 12'd0; wb_a1_r <= 12'd0; wb_b1_r <= 12'd0; - wb_a0a_r <= 8'd0; wb_b0a_r <= 8'd0; wb_a1a_r <= 8'd0; wb_b1a_r <= 8'd0; - coeff_out_r <= 12'd0; - valid_o_r <= 1'b0; - scale_valid_i <= 1'b0; - scale_a_i <= 12'd0; - for (ci = 0; ci < N; ci = ci + 1) cr[ci] = 12'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer_len <= 8'd0; + zeta_idx <= 7'd0; zeta_fed <= 12'd0; + layer <= 3'd0; mode_r <= 1'b0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; + rd_fire_r <= 1'b0; + for (k = 0; k <= 8; k = k + 1) begin + a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0; + end + coeff_out_r <= 12'd0; valid_o_r <= 1'b0; + scale_valid_i <= 1'b0; scale_a_i <= 12'd0; end else begin state <= next_state; scale_valid_i <= 1'b0; - // Writeback address delay lines (8-deep = butterfly latency). - a0_d0<=j; a0_d1<=a0_d0;a0_d2<=a0_d1;a0_d3<=a0_d2;a0_d4<=a0_d3;a0_d5<=a0_d4;a0_d6<=a0_d5;a0_d7<=a0_d6; - b0_d0<=j+layer_len; b0_d1<=b0_d0;b0_d2<=b0_d1;b0_d3<=b0_d2;b0_d4<=b0_d3;b0_d5<=b0_d4;b0_d6<=b0_d5;b0_d7<=b0_d6; - a1_d0<=j+8'd1; a1_d1<=a1_d0;a1_d2<=a1_d1;a1_d3<=a1_d2;a1_d4<=a1_d3;a1_d5<=a1_d4;a1_d6<=a1_d5;a1_d7<=a1_d6; - b1_d0<=j+8'd1+layer_len; b1_d1<=b1_d0;b1_d2<=b1_d1;b1_d3<=b1_d2;b1_d4<=b1_d3;b1_d5<=b1_d4;b1_d6<=b1_d5;b1_d7<=b1_d6; + // read->feed pipeline register (BRAM latency) + zeta alignment + rd_fire_r <= issue_fire; + zeta_fed <= zeta; - // In-flight pair accounting (mirrors single-butterfly version). - if (issue_fire && !wb_valid_r) - bf_inflight <= bf_inflight + 6'd1; - else if (!issue_fire && wb_valid_r) - bf_inflight <= bf_inflight - 6'd1; + // writeback address delay lines: sample j-derived addrs at read cycle + a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1; + for (k = 1; k <= 8; k = k + 1) begin + a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1]; + a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1]; + end + + // in-flight pair accounting: +1 on issue, -1 on writeback + if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1; + else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1; if (state != S_OUTPUT && state != S_OUT_SCALE) valid_o_r <= 1'b0; if (state == S_IDLE && valid_i) begin - cr[0] <= coeff_in; load_cnt <= 8'd1; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer <= 3'd0; - bf_done <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer <= 3'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; mode_r <= mode; - if (!mode) begin - layer_len <= 8'd128; - zeta_idx <= 7'd1; - end else begin - layer_len <= 8'd2; - zeta_idx <= 7'd127; - end + if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end + else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end end - if (state == S_LOAD && valid_i) begin - cr[load_cnt] <= coeff_in; - load_cnt <= load_cnt + 8'd1; - end + if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1; - // ---- issue advance: step j by 2 (two butterflies/cycle) ---- + // ---- issue advance (read pointer j, step 2) ---- if (state == S_CMP_RUN && !layer_issue_done) begin if (issue_group_last) begin if (!mode_r) zeta_idx <= zeta_idx + 7'd1; else zeta_idx <= zeta_idx - 7'd1; - - if (issue_layer_last) begin - layer_issue_done <= 1'b1; - end else begin - start <= next_group_start[7:0]; - j <= next_group_start[7:0]; - end + if (issue_layer_last) layer_issue_done <= 1'b1; + else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end end else begin j <= j + 8'd2; end end - // ---- writeback: commit previous cycle's pair results ---- - if (wb_valid_r) begin - cr[wb_a0a_r] <= wb_a0_r; - cr[wb_b0a_r] <= wb_b0_r; - cr[wb_a1a_r] <= wb_a1_r; - cr[wb_b1a_r] <= wb_b1_r; - end - - wb_valid_r <= bf0_valid; // both lanes share latency/valid - if (bf0_valid) begin - wb_a0_r <= bf0_a_out; wb_b0_r <= bf0_b_out; - wb_a1_r <= bf1_a_out; wb_b1_r <= bf1_b_out; - wb_a0a_r <= a0_d7; wb_b0a_r <= b0_d7; - wb_a1a_r <= a1_d7; wb_b1a_r <= b1_d7; - end - // ---- layer advance after drain ---- if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin - if (layer + 3'd1 >= LAYERS) begin - bf_done <= 1'b1; - end else begin + if (layer + 3'd1 < LAYERS) begin layer <= layer + 3'd1; layer_len <= next_layer_len; - start <= 8'd0; - j <= 8'd0; + start <= 8'd0; j <= 8'd0; layer_issue_done <= 1'b0; end end + // ---- OUT_PREP: reset output pointers ---- if (state == S_OUT_PREP) begin - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - if (!mode_r) begin - coeff_out_r <= cr[0]; - valid_o_r <= 1'b1; - end + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + // (scale first addr 0 is presented via out_rptr in this PREP cycle) end - if (state == S_OUTPUT && valid_o_r && ready_i) begin - if (out_cnt < 8'd255) begin - out_cnt <= out_cnt + 8'd1; - coeff_out_r <= cr[out_cnt + 8'd1]; - valid_o_r <= 1'b1; + // ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0; + // valid_o is a registered "presented a valid addr last cycle". Present + // addresses out_rptr = 0..255 (256), giving 256 valid outputs. ---- + if (state == S_OUTPUT) begin + if (out_rptr < 9'd256) begin + out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr + valid_o_r <= 1'b1; // data valid next cycle end else begin - out_cnt <= 8'd0; valid_o_r <= 1'b0; end + if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted end + // ---- OUT_SCALE: read coeff, x3303 via barrett, emit ---- if (state == S_OUT_SCALE) begin + if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1; + // feed barrett with the coeff that arrived this cycle if (scale_issue_cnt < 9'd256) begin scale_valid_i <= 1'b1; - scale_a_i <= cr[scale_issue_cnt[7:0]]; + scale_a_i <= rd_data_0; scale_issue_cnt <= scale_issue_cnt + 9'd1; end valid_o_r <= 1'b0; @@ -285,12 +286,9 @@ module ntt_core ( if (state == S_DONE) begin load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; - wb_valid_r <= 1'b0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; valid_o_r <= 1'b0; end end From b2c8d6eaf311afab7018ce8e3544ec82cd5a072c Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 05:41:27 +0800 Subject: [PATCH 6/8] perf(mlkem): move poly_mul coeff storage to even/odd banked BRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace poly_mul_sync's two 256x12 LUT register arrays (mem_A/mem_B, ~8k LUT) with four sd_bram banks (A_even/A_odd/B_even/B_odd, 128x12). The base-case reads mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single read port. This removes the multi-port LUT mux and the long BRAM->register-file route that was the mlkem_clk critical path. sd_bram reads are synchronous (1-cycle); the existing 2-cycle issue cadence absorbs it (address on the gap cycle, data on the issue cycle), with a one-cycle priming gap at LOAD->RUN. Interface and behavior unchanged. Verified byte-exact vs NIST KAT: poly_mul standalone TB 11/11, Encaps 9/9, Decaps 9/9 (incl. reject + DECOMP), KeyGen, hello_world e2e — all PASS. Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/poly_mul/poly_mul_sync.v | 124 +++++++++++--------------- 1 file changed, 54 insertions(+), 70 deletions(-) diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v index 61f8f99..bf7f278 100644 --- a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -1,4 +1,4 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (BRAM storage) // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. @@ -6,18 +6,18 @@ // Operation flow: // IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // -// RUN issues one base-case multiply every two cycles, matching the two-cycle -// c0/c1 output bandwidth after the basecase pipeline fills. +// Coefficient storage moved from 256x12 LUT register arrays to four sd_bram +// banks (A_even/A_odd/B_even/B_odd, each 128x12): the base-case reads +// mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single +// read port. This removes the large multi-port LUT mux (~8k LUT) and the long +// BRAM->register-file routing that was the mlkem_clk critical path. // -// The LOAD phase accepts both A and B coefficients simultaneously -// (one pair per cycle) on coeff_a_in/coeff_b_in. +// RUN issues one base-case every two cycles. sd_bram reads are synchronous +// (1-cycle latency); the 2-cycle cadence absorbs it (address presented on the +// gap cycle, data consumed on the issue cycle). A one-cycle priming gap is +// inserted at LOAD->RUN so the first base-case's operands are ready. // -// The COMPUTE phase outputs the 256 result coefficients one per cycle -// via valid/ready handshake on coeff_out/valid_o/ready_i. -// -// Memory: 256×12-bit register arrays for A and B coefficients. -// -// Interface: +// Interface (unchanged): // clk, rst_n - Clock, active-low reset // coeff_a_in[11:0]- Polynomial A coefficient input // coeff_b_in[11:0]- Polynomial B coefficient input @@ -38,18 +38,12 @@ module poly_mul_sync ( input ready_i ); - // State definitions localparam S_IDLE = 3'd0; localparam S_LOAD = 3'd1; localparam S_RUN = 3'd2; reg [2:0] state, next_state; - // Coefficient storage (register arrays) - reg [11:0] mem_A [0:255]; - reg [11:0] mem_B [0:255]; - - // Counters reg [7:0] load_cnt; // 0..256 for loading 256 pairs reg [7:0] issue_cnt; // 0..128, issued base-cases reg [7:0] out_cnt; // 0..255, consumed output coefficients @@ -57,53 +51,61 @@ module poly_mul_sync ( reg out_phase; // 0=c0, 1=c1 reg out_valid_r; - // Registered basecase_mul inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Combinational read signals for the next base-case request. - wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 - wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 - wire [11:0] mem_a0 = mem_A[addr_even]; - wire [11:0] mem_a1 = mem_A[addr_odd]; - wire [11:0] mem_b0 = mem_B[addr_even]; - wire [11:0] mem_b1 = mem_B[addr_odd]; + // ---- Coefficient storage: even/odd banked sd_bram ---- + // A_even[k]=mem_A[2k], A_odd[k]=mem_A[2k+1]; same for B. Read index during + // RUN is issue_k; write index during LOAD is load_cnt>>1 routed by load_cnt[0]. + wire [6:0] issue_k = issue_cnt[6:0]; // 0..127 read index - // Zeta ROM - wire [11:0] zeta; - poly_mul_zeta_rom u_zeta ( - .addr (issue_k), - .zeta (zeta) + wire load_fire = ((state == S_IDLE) || (state == S_LOAD)) && valid_i && ready_o; + wire [6:0] wr_local = load_cnt[7:1]; // load_cnt/2 + wire wr_odd = load_cnt[0]; + + wire we_even = load_fire && !wr_odd; + wire we_odd = load_fire && wr_odd; + + wire [11:0] mem_a0, mem_a1, mem_b0, mem_b1; // sd_bram registered read outputs + sd_bram #(.W(12), .D(128), .A(7)) u_a_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_a_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_b_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_b_in) ); - // Pipelined basecase multiply. Requests are issued every two cycles; inputs - // are registered locally so issue_cnt does not directly drive DSP input muxes. + wire [11:0] zeta; + poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); + wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk (clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0 (bc_a0_reg), - .a1 (bc_a1_reg), - .b0 (bc_b0_reg), - .b1 (bc_b1_reg), + .a0(bc_a0_reg), .a1(bc_a1_reg), + .b0(bc_b0_reg), .b1(bc_b1_reg), .zeta(bc_zeta_reg), - .c0 (bc_c0), - .c1 (bc_c1), + .c0(bc_c0), .c1(bc_c1), .valid_o(bc_vo) ); - // Output interface assign ready_o = (state == S_IDLE) || (state == S_LOAD); assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - // State transition logic (combinational) always @* begin next_state = state; case (state) @@ -116,8 +118,6 @@ module poly_mul_sync ( endcase end - // Sequential logic - integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; @@ -127,27 +127,16 @@ module poly_mul_sync ( issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; - bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; - bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; - bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; - c1_reg <= 12'd0; - for (i = 0; i < 256; i = i + 1) begin - mem_A[i] = 12'd0; - mem_B[i] = 12'd0; - end + bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; c1_reg <= 12'd0; end else begin state <= next_state; bc_valid_reg <= 1'b0; - // ---- LOAD phase ---- - // First coefficient captured on IDLE → LOAD transition + // ---- LOAD phase (writes routed to even/odd banks combinationally) ---- if (state == S_IDLE && valid_i && ready_o) begin - mem_A[0] <= coeff_a_in; - mem_B[0] <= coeff_b_in; load_cnt <= 8'd1; issue_cnt <= 8'd0; out_cnt <= 8'd0; @@ -156,25 +145,21 @@ module poly_mul_sync ( out_valid_r <= 1'b0; end - // Subsequent coefficients in LOAD state if (state == S_LOAD && valid_i && ready_o) begin - mem_A[load_cnt] <= coeff_a_in; - mem_B[load_cnt] <= coeff_b_in; load_cnt <= load_cnt + 8'd1; if (load_cnt >= 8'd255) begin issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; + issue_gap <= 1'b1; // priming gap: present issue_k=0 addr first out_phase <= 1'b0; out_valid_r <= 1'b0; end end // ---- COMPUTE phase ---- - // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input - // regs, then launch requests every other cycle. The two-cycle issue - // interval matches the c0/c1 output bandwidth, so no deep result FIFO - // is needed for this always-ready system. + // Issue one base-case every two cycles. On the gap cycle the sd_bram + // address (issue_k) settles; on the issue cycle its registered output + // (mem_a0/a1/b0/b1) is captured into the base-case input registers. if (state == S_RUN) begin if (issue_cnt < 8'd128) begin if (!issue_gap) begin @@ -212,7 +197,6 @@ module poly_mul_sync ( end end - // ---- final output consumed: reset counters before accepting a new op ---- if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin load_cnt <= 8'd0; issue_cnt <= 8'd0; From 4538ec496ff1d2385fb85e35f8e7e6ae96430320 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 07:05:12 +0800 Subject: [PATCH 7/8] perf(mlkem): register butterfly inputs to cut the 100 MHz NTT critical path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At mlkem_clk=100 MHz the worst path (WNS -2.14 ns) was coeff_bram read -> butterfly diff (b-a) -> barrett first multiply stage, all in one cycle. The butterfly took its operands combinationally from the BRAM read (unlike poly_mul's basecase, which already registers them). Add an input register stage (a/b/zeta/mode/valid) in butterfly_unit_pipe so the BRAM read and the first multiply land in separate cycles; feed the a/b/mode alignment delay lines from the registered inputs. Butterfly latency 8->9; ntt_core writeback address delay lines deepened 9->10 to match. Verified byte-exact vs NIST KAT: NTT standalone TB 19/19 (fwd+inv), Encaps 9/9, Decaps 9/9, hello_world e2e — all PASS. Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/ntt/butterfly_unit_pipe.v | 29 +++++++++++++++++++------- rtl/ip/mlkem/ntt/ntt_core.v | 16 +++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/rtl/ip/mlkem/ntt/butterfly_unit_pipe.v b/rtl/ip/mlkem/ntt/butterfly_unit_pipe.v index c672953..c365a24 100644 --- a/rtl/ip/mlkem/ntt/butterfly_unit_pipe.v +++ b/rtl/ip/mlkem/ntt/butterfly_unit_pipe.v @@ -18,21 +18,32 @@ module butterfly_unit_pipe ( localparam [11:0] Q12 = 12'd3329; localparam [12:0] Q13 = 13'd3329; - wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12); - wire [11:0] mul_data = mode ? diff : b; + // Plan A: register the inputs. Operands arrive combinationally from the + // coeff_bram read; without this stage the path coeff_bram-read -> diff -> + // first multiply stage sits in one cycle (the mlkem_clk critical path at + // 100 MHz). Registering here splits it into (BRAM read -> reg) and + // (reg -> diff -> multiply). Adds one pipeline cycle (absorbed by the NTT + // writeback delay line). + reg [11:0] a_r, b_r, zeta_r; + reg mode_ri, vld_r; + + wire [11:0] diff = (b_r >= a_r) ? (b_r - a_r) : (b_r - a_r + Q12); + wire [11:0] mul_data = mode_ri ? diff : b_r; wire [11:0] mul_result; wire mul_valid; barrett_mul_pipe u_barrett ( .clk(clk), .rst_n(rst_n), - .valid_i(valid_i), - .a(zeta), + .valid_i(vld_r), + .a(zeta_r), .b(mul_data), .product(mul_result), .valid_o(mul_valid) ); + // a/b/mode delay lines (fed from the REGISTERED inputs a_r/b_r/mode_ri so + // a_d6 aligns with mul_valid = valid_i delayed 8). reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6; reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6; reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6; @@ -51,6 +62,7 @@ module butterfly_unit_pipe ( always @(posedge clk or negedge rst_n) begin if (!rst_n) begin + a_r <= 12'd0; b_r <= 12'd0; zeta_r <= 12'd0; mode_ri <= 1'b0; vld_r <= 1'b0; a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0; b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0; mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0; @@ -58,9 +70,12 @@ module butterfly_unit_pipe ( b_out_r <= 12'd0; valid_r <= 1'b0; end else begin - a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5; - b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5; - mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1; + // input register stage + a_r <= a; b_r <= b; zeta_r <= zeta; mode_ri <= mode; vld_r <= valid_i; + + a_d0 <= a_r; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5; + b_d0 <= b_r; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5; + mode_d0 <= mode_ri; mode_d1 <= mode_d0; mode_d2 <= mode_d1; mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5; valid_r <= mul_valid; diff --git a/rtl/ip/mlkem/ntt/ntt_core.v b/rtl/ip/mlkem/ntt/ntt_core.v index 681c1b8..8e46be2 100644 --- a/rtl/ip/mlkem/ntt/ntt_core.v +++ b/rtl/ip/mlkem/ntt/ntt_core.v @@ -53,8 +53,8 @@ module ntt_core ( reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed // 9-deep writeback address delay lines (BRAM read + 8-stage butterfly). - reg [7:0] a0d[0:8]; reg [7:0] b0d[0:8]; - reg [7:0] a1d[0:8]; reg [7:0] b1d[0:8]; + reg [7:0] a0d[0:9]; reg [7:0] b0d[0:9]; + reg [7:0] a1d[0:9]; reg [7:0] b1d[0:9]; integer k; reg [DW-1:0] coeff_out_r; @@ -128,10 +128,10 @@ module ntt_core ( wire wr_en_1 = bf0_valid & ~loading_we; wire wr_en_2 = bf0_valid & ~loading_we; wire wr_en_3 = bf0_valid & ~loading_we; - wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[8]; - wire [7:0] wr_addr_1 = b0d[8]; - wire [7:0] wr_addr_2 = a1d[8]; - wire [7:0] wr_addr_3 = b1d[8]; + wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[9]; + wire [7:0] wr_addr_1 = b0d[9]; + wire [7:0] wr_addr_2 = a1d[9]; + wire [7:0] wr_addr_3 = b1d[9]; wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out; wire [11:0] wr_data_1 = bf0_b_out; wire [11:0] wr_data_2 = bf1_a_out; @@ -185,7 +185,7 @@ module ntt_core ( layer <= 3'd0; mode_r <= 1'b0; layer_issue_done <= 1'b0; bf_inflight <= 6'd0; rd_fire_r <= 1'b0; - for (k = 0; k <= 8; k = k + 1) begin + for (k = 0; k <= 9; k = k + 1) begin a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0; end coeff_out_r <= 12'd0; valid_o_r <= 1'b0; @@ -200,7 +200,7 @@ module ntt_core ( // writeback address delay lines: sample j-derived addrs at read cycle a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1; - for (k = 1; k <= 8; k = k + 1) begin + for (k = 1; k <= 9; k = k + 1) begin a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1]; a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1]; end From e30a65bd72c63fe17ce82f5454da3a141bef80f4 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 07:42:35 +0800 Subject: [PATCH 8/8] perf(mlkem): pipeline CBD->bank write to cut 100 MHz critical path At 100 MHz the worst path (WNS -1.58 ns) was CBD sample: squeeze_buf[buf_ptr] extraction -> CBD add/sub -> mod-Q -> bank_t/bank_se write, all combinational in one cycle and routing-dominated (squeeze_buf and the banks place far apart). Register the CBD write (data/addr/enable) so the bank write commits one cycle later; the registered write takes priority in the bank_se/bank_t mux so the 1-cycle spillover into the next state uses the correct addr/data. No consumer reads these coeffs until the following stage (>1 cycle later), so no RAW hazard. Verified byte-exact vs NIST KAT: Encaps 9/9, Decaps 9/9, KeyGen 11/11, hello_world e2e -- all PASS. Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/top/mlkem_top.v | 44 +++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 306df7a..04362d8 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -312,15 +312,15 @@ module mlkem_top #( // D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT]. wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo; wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx; - assign bt_we = ((st == ST_M) && pm_vo) || e2_we || + assign bt_we = ((st == ST_M) && pm_vo) || cbdw_bt_en || u_psum_we || u_intt_we || u_v_we || u_w_we; - assign bt_wa = e2_we ? (c_widx & ((1< [0,Q): add Q when negative wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff; + // Pipeline the CBD -> bank write by one cycle. The squeeze_buf[buf_ptr] + // extraction -> CBD add/sub -> mod-Q -> BRAM-write route is long and + // routing-dominated (the mlkem_clk critical path at 100 MHz). Register the + // write data/addr/enable here; the bank write commits one cycle later. No + // consumer reads these coeffs until the following stage, which begins more + // than one cycle after the last CBD coeff, so there is no read-after-write + // hazard. The registered write takes priority in the bank_se/bank_t mux so + // the 1-cycle spillover into the next state uses the correct addr/data. + reg cbdw_se_en, cbdw_bt_en; + reg [4:0] cbdw_poly; + reg [7:0] cbdw_widx; + reg [11:0] cbdw_data; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + cbdw_se_en <= 1'b0; cbdw_bt_en <= 1'b0; + cbdw_poly <= 5'd0; cbdw_widx <= 8'd0; cbdw_data <= 12'd0; + end else begin + cbdw_se_en <= ((st == ST_C) && c_busy && cbd_vo && cbd_ack) || + ((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack && + (c_poly < {1'b0, k_r, 1'b0})); + cbdw_bt_en <= (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack && + (c_poly == {1'b0, k_r, 1'b0}); + cbdw_poly <= c_poly; + cbdw_widx <= c_widx; + cbdw_data <= cbd_modq; + end + end + // ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ---- // N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot). reg [4:0] n_slot; // 0..2K (process s_hat[0..K-1] then e_hat[0..K-1])