From ce40a6df42791b48ff4a123b26b7a48f9ba18325 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 02:00:03 +0800 Subject: [PATCH] =?UTF-8?q?perf(mlkem):=20stream=20compress=20feed=20(B1)?= =?UTF-8?q?=20=E2=80=94=20~27%=20fewer=20Encaps=20cycles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decouple the ENC_C1/C2 compress micro-sequence into a streaming producer/collector instead of feeding one coeff and draining the whole 6-stage comp_decomp pipeline per coeff. - comp_decomp_sync: ready_o = 1'b1 (fully pipelined, consumer never backpressures since ready_i is tied high), so feeders can issue a new coeff every cycle. - mlkem_top ENC_C1/C2: producer feeds one coeff every 2 cycles (<=5.5 bit/cyc, under the single-port ct drain of 8 bit/cyc so the bit buffer stays bounded); a per-cycle collector appends compressed bits and drains one ct byte/cycle. Verified byte-exact against NIST KAT (Vivado 2019.2 XSIM) for KeyGen, Encaps (9/9), Decaps (9/9 incl. reject path + DECOMP), hello_world e2e. K=2 Encaps: ENC_C1 6273->1035, ENC_C2 2945->522, total 27851->20190. Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v | 10 +- rtl/ip/mlkem/top/mlkem_top.v | 164 ++++++++++++-------- 2 files changed, 103 insertions(+), 71 deletions(-) diff --git a/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v index 2690bf1..9861dd1 100644 --- a/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v +++ b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v @@ -75,9 +75,13 @@ module comp_decomp_sync ( wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5; - // Accept a new command only when the pipeline and output slot are both free. - // Current users issue one coefficient at a time and wait for valid_o. - assign ready_o = !pipe_busy && (!valid_r || ready_i); + // Fully pipelined: accept one coefficient per cycle. Every stage advances + // unconditionally (valid_sN <= valid_s(N-1)), and the sole consumer ties + // ready_i high, so the output register never needs to backpressure the + // pipeline. Streaming feeders (Encaps C1/C2, Decaps DECOMP) can therefore + // issue a new coeff every cycle instead of draining the 6-stage pipe each + // time. pipe_busy retained only for the legacy drain-based users' info. + assign ready_o = 1'b1; assign coeff_out = data_r; assign valid_o = valid_r; diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v index 293188d..306df7a 100644 --- a/rtl/ip/mlkem/top/mlkem_top.v +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -1041,13 +1041,18 @@ module mlkem_top #( .ready_i(1'b1) ); // bit-packer / ct walk bookkeeping - reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v - reg [7:0] cp_idx; // coeff 0..255 within poly - reg [2:0] cp_ph; // micro-phase - reg [24:0] cp_buf; // bit accumulator (LSB-first) - reg [5:0] cp_nbits; // valid bits in cp_buf + reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v (feed side) + reg [7:0] cp_idx; // coeff 0..255 within poly (feed side, drives bank addr) + reg [2:0] cp_ph; // feed micro-phase toggle (0=addr settle, 1=feed) + reg [63:0] cp_buf; // bit accumulator (LSB-first); widened for streaming + reg [6:0] cp_nbits; // valid bits in cp_buf reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2) reg cp_done; // serialization complete (this region) + reg cp_fed_done; // all coeffs for this region have been fed in + reg [10:0] cp_fcnt; // coeffs fed into comp_decomp this region + reg [10:0] cp_ccnt; // compressed coeffs collected this region + // total coeffs this region: c1 = K*256 (u rows), c2 = 256 (single v) + wire [10:0] cp_ntot = (st == ST_ENC_C2) ? 11'd256 : ({8'd0, k_r} << 8); // coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly); // c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0). wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly] @@ -1260,10 +1265,13 @@ module mlkem_top #( cp_poly <= 3'd0; cp_idx <= 8'd0; cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; cp_wa <= 12'd0; cp_done <= 1'b0; + cp_fed_done<= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; ct_we <= 1'b0; ct_wa <= 11'd0; ct_wd <= 8'd0; @@ -1695,69 +1703,86 @@ module mlkem_top #( // Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)). // Walk K polys * 256 coeffs; reset bit-packer + ct write pointer. if (st == ST_ENC_U && st_next == ST_ENC_C1) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_wa <= 12'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_wa <= 12'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; end // ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ---- // C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes). // C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes). - // Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram plus - // variable-latency comp_decomp), then a drain sub-phase emitting whole bytes: - // ph0: present coeff addr (cp_se_full / cp_bt_full by state). - // ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid. - // ph2: drop cd_valid (1-cyc pulse); comp_decomp captures. - // ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first. - // ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance. - // Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at - // each poly boundary (no carry across polys / regions). + // + // Streaming (B1): the comp_decomp pipeline is fully pipelined and its + // sole output consumer never backpressures (ready_i=1'b1), so instead + // of feeding one coeff and draining the whole pipe per coeff (~12 cyc), + // we run a decoupled producer + collector: + // PRODUCER (2-cycle cadence): ph0 lets the bank read address settle + // (1-cyc sd_bram latency), ph1 latches the coeff and pulses + // cd_valid, then advances the address. One coeff enters the pipe + // every 2 cycles = <=5.5 bits/cyc (du<=11), strictly under the + // single-port ct drain rate (8 bits/cyc), so cp_buf stays bounded. + // COLLECTOR (every cycle): when cd_vo, append the compressed cp_d + // bits LSB-first; independently emit one ct byte/cycle whenever + // >=8 bits are buffered. Coeffs come out in-order, one per pipe + // output. Each poly is 256*d bits (multiple of 8), so the buffer + // empties to 0 at region end (cp_done when all collected+drained). if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin - case (cp_ph) - 3'd0: cp_ph <= 3'd1; // addr presented; wait read - 3'd1: begin - cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) - cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse) - cp_ph <= 3'd2; - end - 3'd2: begin - cd_valid <= 1'b0; // comp_decomp captured this cyc - cp_ph <= 3'd3; - end - 3'd3: begin - if (cd_vo) begin - // Append compressed low cp_d bits LSB-first at bit cp_nbits. - cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits); - cp_nbits <= cp_nbits + {1'b0, cp_d}; - cp_ph <= 3'd4; - end - end - default: begin // 3'd4: drain whole bytes - if (cp_nbits >= 6'd8) begin - ct_we <= 1'b1; - ct_wa <= cp_wa; - ct_wd <= cp_buf[7:0]; - cp_wa <= cp_wa + 12'd1; - cp_buf <= cp_buf >> 8; - cp_nbits <= cp_nbits - 6'd8; + // ---- PRODUCER: feed one coeff every 2 cycles ---- + cd_valid <= 1'b0; + if (!cp_fed_done) begin + if (cp_ph == 3'd0) begin + cp_ph <= 3'd1; // address settling + end else begin + cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) + cd_valid <= 1'b1; // 1-cyc feed pulse + cp_fcnt <= cp_fcnt + 11'd1; + if (cp_fcnt + 11'd1 >= cp_ntot) cp_fed_done <= 1'b1; + if (cp_idx == 8'd255) begin + cp_idx <= 8'd0; + cp_poly <= cp_poly + 3'd1; // next u row (c1); c2 unused end else begin - // coeff fully packed; advance coeff / poly - if (cp_idx == 8'd255) begin - cp_idx <= 8'd0; - if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1; - else cp_done <= 1'b1; // region complete - end else begin - cp_idx <= cp_idx + 8'd1; - end - cp_ph <= 3'd0; + cp_idx <= cp_idx + 8'd1; end + cp_ph <= 3'd0; end - endcase + end + + // ---- COLLECTOR: append compressed bits + drain ct bytes ---- + if (cd_vo) begin + // append cp_d bits at cp_nbits, then drain one byte if >=8. + if ((cp_nbits + {1'b0, cp_d}) >= 7'd8) begin + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) & 64'hFF; + cp_wa <= cp_wa + 12'd1; + cp_buf <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) >> 8; + cp_nbits <= (cp_nbits + {1'b0, cp_d}) - 7'd8; + end else begin + cp_buf <= cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits); + cp_nbits <= cp_nbits + {1'b0, cp_d}; + end + cp_ccnt <= cp_ccnt + 11'd1; // one compressed coeff collected + end else if (cp_nbits >= 7'd8) begin + // no new coeff this cycle: keep draining buffered whole bytes. + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= cp_buf[7:0]; + cp_wa <= cp_wa + 12'd1; + cp_buf <= cp_buf >> 8; + cp_nbits <= cp_nbits - 7'd8; + end + + // region complete once all coeffs collected and buffer drained. + if (cp_ccnt >= cp_ntot && cp_nbits < 7'd8) + cp_done <= 1'b1; end // Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)). @@ -1765,13 +1790,16 @@ module mlkem_top #( // from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits // are 0 here (c1 ended on a poly/byte boundary) but reset for safety. if (st == ST_ENC_V && st_next == ST_ENC_C2) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; // cp_wa intentionally preserved (= c1_bytes_rt from C1). end