fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dec): Decaps D1 - byteDecode_d + Decompress -> u'/v'

Browse Source 
K-PKE.Decrypt step 1 (FIPS 203 Alg 15): decode+decompress the ciphertext.

- comp_decomp_sync instance made mode-selectable: Encaps C1/C2 still compress
  (mode 0), Decaps ST_DEC_DECOMP decompresses (mode 1) with d=du/dv.
- New ST_DEC_DECOMP state with an inline byteDecode_d walker (reverse of the
  C1/C2 bit-packer): walks c_in_bram bytes, accumulates LSB-first into a bit
  buffer, extracts d-bit symbols, feeds comp_decomp, writes each decompressed
  coeff (mod q) to a bank.
    c1 = K polys, d=du -> u'[i] in bank_se rel slot i (0..K-1)
    c2 = 1 poly,  d=dv -> v'   in bank_t rel slot DEC_VSLOT=2 (avoids UPSUM=1)
- dbg_slot_i widened 4->6 bits so the TB can read v' (abs slot 26 at K=4).
- bse/bt write muxes gain the DECOMP writeback paths (dec_u_we / dec_v_we).

Verification: examples/dump_decaps.rs (ml-kem-r worktree) emits per-stage
golden (u'/v'/s_hat/u_hat/w/m') into vectors/decgold/. TB verify_d1 reads back
u'[i] and v' and compares all 256 coeffs each.

Bring-up note: dbg coeff readback latency is bank(1)+dbg_coeff_r(1); the TB's
rdcoeff initially waited 2 cyc and saw data shifted by one index -> fixed to 3.

Verified: dec D1 K=2/3/4 all cases PASS; KeyGen + Encaps unregressed.
This commit is contained in:
FallenSigh
2026-06-29 17:32:03 +08:00
parent e46d2258d9
commit ecc00d6dd5
111 changed files with 25841 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
sync_rtl/top/mlkem_top.v
View File

@@ -64,7 +64,7 @@ module mlkem_top #(
// Debug readback tap: read one stored coefficient by (poly slot, index).
// Lets stage TBs verify intermediates without wide buses.
input [3:0] dbg_slot_i, // poly slot (see localparams below)
input [5:0] dbg_slot_i, // poly slot (see localparams below)
input [7:0] dbg_idx_i, // coefficient index 0..255
output [11:0] dbg_coeff_o,
// Debug byte readback: ek (sel=0, 0..799) / dk_pke (sel=1, 0..767)
@@ -192,9 +192,6 @@ module mlkem_top #(
.wr_en(c_in_we), .wr_addr(c_in_addr), .wr_data(c_in_byte)
);
assign cin_rd_addr = cin_rd_addr_r;
/* verilator lint_off UNUSEDSIGNAL */
wire [7:0] cin_rd_data_unused = cin_rd_data; // consumed in D5/D7
/* verilator lint_on UNUSEDSIGNAL */
// ================================================================
// Polynomial storage, sized for KMAX (worst case). Runtime k uses a
@@ -291,17 +288,22 @@ module mlkem_top #(
// the V-ADD writeback (psum + e2 + mu) at UPSUM via u_v_we below.
wire u_psum_we = (st == ST_ENC_U || st == ST_ENC_V) && (u_sub == 2'd0) && pm_vo;
wire u_intt_we = (st == ST_ENC_U || st == ST_ENC_V) && (u_sub == 2'd1) && ntt_vo;
// D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_t[DEC_VSLOT].
wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo;
wire [13:0] dec_v_wr = DEC_VSLOT*256 + dec_widx;
assign bt_we = ((st == ST_M) && pm_vo) || e2_we ||
u_psum_we || u_intt_we || u_v_we;
u_psum_we || u_intt_we || u_v_we || dec_v_we;
assign bt_wa = e2_we ? (c_widx & ((1<<PT_AW)-1)) :
u_psum_we ? ((UPSUM*256 + u_oidx) & ((1<<PT_AW)-1)) :
u_intt_we ? (u_intt_wr[PT_AW-1:0]) :
u_v_we ? (u_vadd_wr[PT_AW-1:0]) :
dec_v_we ? (dec_v_wr[PT_AW-1:0]) :
((m_i*256 + m_oidx) & ((1<<PT_AW)-1));
assign bt_wd = e2_we ? cbd_modq :
u_psum_we ? u_accq :
u_intt_we ? ntt_coeff :
u_v_we ? u_vq :
dec_v_we ? cd_out :
m_accq;
// Debug readback (registered for timing)
@@ -349,15 +351,21 @@ module mlkem_top #(
// Encaps E4 ADD u-writeback (u_sub==2).
// Encaps E4 ADD: write u[u_row] over e1 in bank_se rel slot (K+u_row).
wire u_add_we = (st == ST_ENC_U) && (u_sub == 2'd2) && u_avalid;
// D1 DECOMP writeback: cd_vo while decoding c1 writes u'[dec_poly] to
// bank_se rel slot dec_poly (0..K-1), at coeff index dec_widx.
wire dec_u_we = (st == ST_DEC_DECOMP) && !dec_in_c2 && cd_vo;
wire [13:0] dec_u_wr = {2'b0, dec_poly}*256 + dec_widx;
assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly < {1'b0, k_r, 1'b0})) ||
(((st == ST_N) || (st == ST_ENC_N)) && ntt_vo) ||
u_add_we;
u_add_we || dec_u_we;
assign bse_wa = u_add_we ? u_add_uwr[PSE_AW-1:0] :
dec_u_we ? dec_u_wr[PSE_AW-1:0] :
(st == ST_N || st == ST_ENC_N) ? ((n_slot*256 + n_widx) & ((1<<PSE_AW)-1))
: ((c_poly*256 + c_widx) & ((1<<PSE_AW)-1));
assign bse_wd = u_add_we ? u_uq :
dec_u_we ? cd_out :
(st == ST_N || st == ST_ENC_N) ? ntt_coeff : cbd_modq;
always @(posedge clk) begin
if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram)
@@ -455,7 +463,8 @@ module mlkem_top #(
localparam ST_ENC_C2 = 5'd18; // Compress_dv + byteEncode_dv -> ct c2
localparam ST_ENC_E2MV = 5'd19; // relocate e2 bank_t[0] -> bank_a[E2_ASLOT]
// ---- Decaps states ----
localparam ST_DEC_LOAD = 5'd20; // dk/c already streamed in; parse/settle (D0)
localparam ST_DEC_LOAD = 5'd20; // dk/c already streamed in; parse/settle (D0)
localparam ST_DEC_DECOMP = 5'd21; // D1: byteDecode_d + Decompress c1->u', c2->v'
localparam ST_DONE = 5'd31;
reg [4:0] st, st_next;
@@ -892,12 +901,19 @@ module mlkem_top #(
wire [11:0] cd_out; // compressed value (low d bits valid)
wire cd_vo;
wire [4:0] cp_d = (st == ST_ENC_C2) ? dv_rt : du_rt; // compress width
// comp_decomp is shared: Encaps C1/C2 compress (mode 0); Decaps DECOMP
// decompresses (mode 1). In DECOMP the decoder feeds dec_dc_coeff/valid/d.
wire cd_dec = (st == ST_DEC_DECOMP);
wire cd_mode = cd_dec ? 1'b1 : 1'b0; // 1=decompress
wire [4:0] cd_d_mux = cd_dec ? dec_dc_d : cp_d;
wire [11:0] cd_in_mux= cd_dec ? dec_dc_coeff : cd_coeff;
wire cd_vi_mux= cd_dec ? dec_dc_valid : cd_valid;
comp_decomp_sync u_comp (
.clk(clk), .rst_n(rst_n),
.coeff_in(cd_coeff),
.d(cp_d),
.mode(1'b0), // compress
.valid_i(cd_valid),
.coeff_in(cd_in_mux),
.d(cd_d_mux),
.mode(cd_mode),
.valid_i(cd_vi_mux),
.ready_o(cd_ready),
.coeff_out(cd_out),
.valid_o(cd_vo),
@@ -922,6 +938,34 @@ module mlkem_top #(
wire [11:0] cp_coeff_src = (st == ST_ENC_C2) ? bt_rd_data : bse_rd_data;
wire [2:0] cp_poly_max = (st == ST_ENC_C2) ? 3'd1 : k_r;
// ================================================================
// D1: K-PKE byteDecode_d + Decompress (decrypt c1/c2 -> u'/v').
// Reverse of the C1/C2 packer: walk c_in_bram bytes, accumulate LSB-first
// into a bit buffer, extract d-bit symbols, feed comp_decomp (mode=1
// decompress), and write each result (mod q) to a bank.
// c1 = K polys, d=du -> u'[i] in bank_se rel i (0..K-1)
// c2 = 1 poly, d=dv -> v' in bank_t rel slot DEC_VSLOT
// micro-phase dec_ph:
// 0 present c_in byte addr; 1 capture byte, append 8 bits to buf;
// while (nbits>=d): extract d-bit symbol -> comp_decomp (decompress);
// 2 wait pipe; 3 cd_vo -> write coeff to bank, advance coeff index.
// ================================================================
localparam DEC_VSLOT = 10'd2; // bank_t rel slot holding v' (avoid UPSUM=1)
reg [2:0] dec_poly; // c1: 0..K-1; c2: single (=K marker)
reg [7:0] dec_cidx; // coeff 0..255 within poly (feed side)
reg [7:0] dec_widx; // coeff 0..255 within poly (writeback side, lags pipe)
reg [1:0] dec_ph; // micro-phase
reg [11:0] dec_ba; // c_in_bram byte read address (c1 then c2)
reg [24:0] dec_buf; // bit accumulator (LSB-first)
reg [5:0] dec_nbits; // valid bits in dec_buf
reg dec_dc_done; // D1 complete (both c1 and c2 decoded)
reg dec_in_c2; // 0 = decoding c1 (u'), 1 = decoding c2 (v')
// comp_decomp feed (decompress): the extracted d-bit symbol.
reg [11:0] dec_dc_coeff;
reg dec_dc_valid;
wire [4:0] dec_dc_d = dec_in_c2 ? dv_rt : du_rt; // c2 uses dv, c1 uses du
// (c_in_bram read addr driven by cin_rd_addr_r, set to dec_ba in the walker)
reg pm_valid;
wire pm_ready;
wire [11:0] pm_coeff;
@@ -970,9 +1014,10 @@ module mlkem_top #(
case (st)
ST_IDLE: if (start_i) st_next = (op_i == 2'd2) ? ST_DEC_LOAD :
(op_i == 2'd1) ? ST_ENC_H : ST_G;
// D0: settle after dk/c parse, then (D1+) proceed to Decrypt. For now
// ST_DEC_LOAD just lands in DONE so the load/parse can be dbg-checked.
ST_DEC_LOAD: st_next = ST_DONE;
// D0: settle after dk/c parse, then D1 Decompress. D1 lands in DONE
// once u'/v' are computed so the stage can be dbg-checked.
ST_DEC_LOAD: st_next = ST_DEC_DECOMP;
ST_DEC_DECOMP: if (dec_dc_done) st_next = ST_DONE;
ST_G: if (sha3_vo) st_next = ST_A;
ST_A: if (a_pair >= kk_rt) st_next = ST_C;
ST_C: if (c_poly >= {1'b0, k_r, 1'b0}) st_next = ST_N;
@@ -1102,6 +1147,17 @@ module mlkem_top #(
z_r <= 256'd0;
kbar_r <= 256'd0;
cin_rd_addr_r <= 11'd0;
dec_poly <= 3'd0;
dec_cidx <= 8'd0;
dec_widx <= 8'd0;
dec_ph <= 2'd0;
dec_ba <= 12'd0;
dec_buf <= 25'd0;
dec_nbits <= 6'd0;
dec_in_c2 <= 1'b0;
dec_dc_coeff <= 12'd0;
dec_dc_valid <= 1'b0;
dec_dc_done <= 1'b0;
h_blk <= 3'd0;
h_byte <= 8'd0;
h_phase <= 2'd0;
@@ -1554,6 +1610,83 @@ module mlkem_top #(
// cp_wa intentionally preserved (= c1_bytes_rt from C1).
end
// Arm D1 DECOMP when load settles: decode c1 (u') first.
if (st == ST_DEC_LOAD && st_next == ST_DEC_DECOMP) begin
dec_poly <= 3'd0;
dec_cidx <= 8'd0;
dec_widx <= 8'd0;
dec_ph <= 2'd0;
dec_ba <= 12'd0;
dec_buf <= 25'd0;
dec_nbits <= 6'd0;
dec_in_c2 <= 1'b0;
dec_dc_valid <= 1'b0;
dec_dc_done <= 1'b0;
cin_rd_addr_r<= 11'd0; // present byte 0 address
end
// ---- ST_DEC_DECOMP (D1): byteDecode_d + Decompress c1/c2 ----
// Reverse of the C1/C2 packer. Per coeff:
// ph0: if nbits<d, need a byte (addr already presented) -> ph1;
// else extract low d bits -> comp_decomp (decompress) -> ph2.
// ph1: append cin_rd_data (8 bits) at dec_nbits; advance dec_ba;
// present next byte addr; back to ph0.
// ph2: drop dc_valid (pulse captured) -> ph3.
// ph3: cd_vo -> dec_u_we/dec_v_we writes cd_out; advance coeff.
// c1 = K polys (d=du) -> u'[poly] bank_se; then c2 = 1 poly (d=dv)
// -> v' bank_t[DEC_VSLOT]. Byte stream is contiguous in c_in_bram.
if (st == ST_DEC_DECOMP && !dec_dc_done) begin
case (dec_ph)
2'd0: begin
if (dec_nbits >= {1'b0, dec_dc_d}) begin
// enough bits: extract low d as the decoded symbol
dec_dc_coeff <= dec_buf[11:0] & ((12'd1 << dec_dc_d) - 12'd1);
dec_dc_valid <= 1'b1;
dec_buf <= dec_buf >> dec_dc_d;
dec_nbits <= dec_nbits - {1'b0, dec_dc_d};
dec_ph <= 2'd2;
end else begin
// need another byte (cin_rd_data reflects dec_ba)
dec_ph <= 2'd1;
end
end
2'd1: begin
dec_buf <= dec_buf | ({17'd0, cin_rd_data} << dec_nbits);
dec_nbits <= dec_nbits + 6'd8;
dec_ba <= dec_ba + 12'd1;
cin_rd_addr_r<= dec_ba[10:0] + 11'd1; // present next byte
dec_ph <= 2'd0;
end
2'd2: begin
dec_dc_valid <= 1'b0; // comp_decomp captured
dec_ph <= 2'd3;
end
default: begin // 2'd3: capture cd_vo
if (cd_vo) begin
// dec_u_we/dec_v_we commit cd_out at dec_widx this cycle.
dec_widx <= dec_widx + 8'd1;
if (dec_cidx == 8'd255) begin
dec_cidx <= 8'd0;
dec_widx <= 8'd0;
if (!dec_in_c2) begin
if (dec_poly + 3'd1 < k_r) begin
dec_poly <= dec_poly + 3'd1; // next u' row
end else begin
dec_in_c2 <= 1'b1; // switch to c2 (v')
dec_poly <= 3'd0;
end
end else begin
dec_dc_done <= 1'b1; // v' done -> D1 complete
end
end else begin
dec_cidx <= dec_cidx + 8'd1;
end
dec_ph <= 2'd0;
end
end
endcase
end
// Arm E stage when M finishes
if (st == ST_M && st_next == ST_E) begin
e_poly <= 3'd0;