fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(mlkem): stream compress feed (B1) — ~27% fewer Encaps cycles

Browse Source 
Decouple the ENC_C1/C2 compress micro-sequence into a streaming
producer/collector instead of feeding one coeff and draining the whole
6-stage comp_decomp pipeline per coeff.

- comp_decomp_sync: ready_o = 1'b1 (fully pipelined, consumer never
  backpressures since ready_i is tied high), so feeders can issue a new
  coeff every cycle.
- mlkem_top ENC_C1/C2: producer feeds one coeff every 2 cycles
  (<=5.5 bit/cyc, under the single-port ct drain of 8 bit/cyc so the bit
  buffer stays bounded); a per-cycle collector appends compressed bits
  and drains one ct byte/cycle.

Verified byte-exact against NIST KAT (Vivado 2019.2 XSIM) for KeyGen,
Encaps (9/9), Decaps (9/9 incl. reject path + DECOMP), hello_world e2e.

K=2 Encaps: ENC_C1 6273->1035, ENC_C2 2945->522, total 27851->20190.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
FallenSigh
2026-07-11 02:00:03 +08:00
parent 6a1b374bc1
commit ce40a6df42
2 changed files with 103 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v
View File

@@ -75,9 +75,13 @@ module comp_decomp_sync (
wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5; wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5;
// Accept a new command only when the pipeline and output slot are both free. // Fully pipelined: accept one coefficient per cycle. Every stage advances
// Current users issue one coefficient at a time and wait for valid_o. // unconditionally (valid_sN <= valid_s(N-1)), and the sole consumer ties
assign ready_o = !pipe_busy && (!valid_r || ready_i); // ready_i high, so the output register never needs to backpressure the
// pipeline. Streaming feeders (Encaps C1/C2, Decaps DECOMP) can therefore
// issue a new coeff every cycle instead of draining the 6-stage pipe each
// time. pipe_busy retained only for the legacy drain-based users' info.
assign ready_o = 1'b1;
assign coeff_out = data_r; assign coeff_out = data_r;
assign valid_o = valid_r; assign valid_o = valid_r;

164
rtl/ip/mlkem/top/mlkem_top.v
View File

@@ -1041,13 +1041,18 @@ module mlkem_top #(
.ready_i(1'b1) .ready_i(1'b1)
); );
// bit-packer / ct walk bookkeeping // bit-packer / ct walk bookkeeping
reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v (feed side)
reg [7:0] cp_idx; // coeff 0..255 within poly reg [7:0] cp_idx; // coeff 0..255 within poly (feed side, drives bank addr)
reg [2:0] cp_ph; // micro-phase reg [2:0] cp_ph; // feed micro-phase toggle (0=addr settle, 1=feed)
reg [24:0] cp_buf; // bit accumulator (LSB-first) reg [63:0] cp_buf; // bit accumulator (LSB-first); widened for streaming
reg [5:0] cp_nbits; // valid bits in cp_buf reg [6:0] cp_nbits; // valid bits in cp_buf
reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2) reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2)
reg cp_done; // serialization complete (this region) reg cp_done; // serialization complete (this region)
reg cp_fed_done; // all coeffs for this region have been fed in
reg [10:0] cp_fcnt; // coeffs fed into comp_decomp this region
reg [10:0] cp_ccnt; // compressed coeffs collected this region
// total coeffs this region: c1 = K*256 (u rows), c2 = 256 (single v)
wire [10:0] cp_ntot = (st == ST_ENC_C2) ? 11'd256 : ({8'd0, k_r} << 8);
// coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly); // coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly);
// c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0). // c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0).
wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly] wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly]
@@ -1260,10 +1265,13 @@ module mlkem_top #(
cp_poly <= 3'd0; cp_poly <= 3'd0;
cp_idx <= 8'd0; cp_idx <= 8'd0;
cp_ph <= 3'd0; cp_ph <= 3'd0;
cp_buf <= 25'd0; cp_buf <= 64'd0;
cp_nbits <= 6'd0; cp_nbits <= 7'd0;
cp_wa <= 12'd0; cp_wa <= 12'd0;
cp_done <= 1'b0; cp_done <= 1'b0;
cp_fed_done<= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
ct_we <= 1'b0; ct_we <= 1'b0;
ct_wa <= 11'd0; ct_wa <= 11'd0;
ct_wd <= 8'd0; ct_wd <= 8'd0;
@@ -1695,69 +1703,86 @@ module mlkem_top #(
// Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)). // Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)).
// Walk K polys * 256 coeffs; reset bit-packer + ct write pointer. // Walk K polys * 256 coeffs; reset bit-packer + ct write pointer.
if (st == ST_ENC_U && st_next == ST_ENC_C1) begin if (st == ST_ENC_U && st_next == ST_ENC_C1) begin
cp_poly <= 3'd0; cp_poly <= 3'd0;
cp_idx <= 8'd0; cp_idx <= 8'd0;
cp_ph <= 3'd0; cp_ph <= 3'd0;
cp_buf <= 25'd0; cp_buf <= 64'd0;
cp_nbits <= 6'd0; cp_nbits <= 7'd0;
cp_wa <= 12'd0; cp_wa <= 12'd0;
cp_done <= 1'b0; cp_done <= 1'b0;
cd_valid <= 1'b0; cp_fed_done <= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
cd_valid <= 1'b0;
end end
// ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ---- // ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ----
// C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes). // C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes).
// C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes). // C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes).
// Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram plus //
// variable-latency comp_decomp), then a drain sub-phase emitting whole bytes: // Streaming (B1): the comp_decomp pipeline is fully pipelined and its
// ph0: present coeff addr (cp_se_full / cp_bt_full by state). // sole output consumer never backpressures (ready_i=1'b1), so instead
// ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid. // of feeding one coeff and draining the whole pipe per coeff (~12 cyc),
// ph2: drop cd_valid (1-cyc pulse); comp_decomp captures. // we run a decoupled producer + collector:
// ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first. // PRODUCER (2-cycle cadence): ph0 lets the bank read address settle
// ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance. // (1-cyc sd_bram latency), ph1 latches the coeff and pulses
// Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at // cd_valid, then advances the address. One coeff enters the pipe
// each poly boundary (no carry across polys / regions). // every 2 cycles = <=5.5 bits/cyc (du<=11), strictly under the
// single-port ct drain rate (8 bits/cyc), so cp_buf stays bounded.
// COLLECTOR (every cycle): when cd_vo, append the compressed cp_d
// bits LSB-first; independently emit one ct byte/cycle whenever
// >=8 bits are buffered. Coeffs come out in-order, one per pipe
// output. Each poly is 256*d bits (multiple of 8), so the buffer
// empties to 0 at region end (cp_done when all collected+drained).
if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin
case (cp_ph) // ---- PRODUCER: feed one coeff every 2 cycles ----
3'd0: cp_ph <= 3'd1; // addr presented; wait read cd_valid <= 1'b0;
3'd1: begin if (!cp_fed_done) begin
cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) if (cp_ph == 3'd0) begin
cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse) cp_ph <= 3'd1; // address settling
cp_ph <= 3'd2; end else begin
end cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t)
3'd2: begin cd_valid <= 1'b1; // 1-cyc feed pulse
cd_valid <= 1'b0; // comp_decomp captured this cyc cp_fcnt <= cp_fcnt + 11'd1;
cp_ph <= 3'd3; if (cp_fcnt + 11'd1 >= cp_ntot) cp_fed_done <= 1'b1;
end if (cp_idx == 8'd255) begin
3'd3: begin cp_idx <= 8'd0;
if (cd_vo) begin cp_poly <= cp_poly + 3'd1; // next u row (c1); c2 unused
// Append compressed low cp_d bits LSB-first at bit cp_nbits.
cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits);
cp_nbits <= cp_nbits + {1'b0, cp_d};
cp_ph <= 3'd4;
end
end
default: begin // 3'd4: drain whole bytes
if (cp_nbits >= 6'd8) begin
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= cp_buf[7:0];
cp_wa <= cp_wa + 12'd1;
cp_buf <= cp_buf >> 8;
cp_nbits <= cp_nbits - 6'd8;
end else begin end else begin
// coeff fully packed; advance coeff / poly cp_idx <= cp_idx + 8'd1;
if (cp_idx == 8'd255) begin
cp_idx <= 8'd0;
if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1;
else cp_done <= 1'b1; // region complete
end else begin
cp_idx <= cp_idx + 8'd1;
end
cp_ph <= 3'd0;
end end
cp_ph <= 3'd0;
end end
endcase end
// ---- COLLECTOR: append compressed bits + drain ct bytes ----
if (cd_vo) begin
// append cp_d bits at cp_nbits, then drain one byte if >=8.
if ((cp_nbits + {1'b0, cp_d}) >= 7'd8) begin
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) & 64'hFF;
cp_wa <= cp_wa + 12'd1;
cp_buf <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) >> 8;
cp_nbits <= (cp_nbits + {1'b0, cp_d}) - 7'd8;
end else begin
cp_buf <= cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits);
cp_nbits <= cp_nbits + {1'b0, cp_d};
end
cp_ccnt <= cp_ccnt + 11'd1; // one compressed coeff collected
end else if (cp_nbits >= 7'd8) begin
// no new coeff this cycle: keep draining buffered whole bytes.
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= cp_buf[7:0];
cp_wa <= cp_wa + 12'd1;
cp_buf <= cp_buf >> 8;
cp_nbits <= cp_nbits - 7'd8;
end
// region complete once all coeffs collected and buffer drained.
if (cp_ccnt >= cp_ntot && cp_nbits < 7'd8)
cp_done <= 1'b1;
end end
// Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)). // Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)).
@@ -1765,13 +1790,16 @@ module mlkem_top #(
// from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits // from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits
// are 0 here (c1 ended on a poly/byte boundary) but reset for safety. // are 0 here (c1 ended on a poly/byte boundary) but reset for safety.
if (st == ST_ENC_V && st_next == ST_ENC_C2) begin if (st == ST_ENC_V && st_next == ST_ENC_C2) begin
cp_poly <= 3'd0; cp_poly <= 3'd0;
cp_idx <= 8'd0; cp_idx <= 8'd0;
cp_ph <= 3'd0; cp_ph <= 3'd0;
cp_buf <= 25'd0; cp_buf <= 64'd0;
cp_nbits <= 6'd0; cp_nbits <= 7'd0;
cp_done <= 1'b0; cp_done <= 1'b0;
cd_valid <= 1'b0; cp_fed_done <= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
cd_valid <= 1'b0;
// cp_wa intentionally preserved (= c1_bytes_rt from C1). // cp_wa intentionally preserved (= c1_bytes_rt from C1).
end end