fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(mlkem): pipeline CBD->bank write to cut 100 MHz critical path

Browse Source 
At 100 MHz the worst path (WNS -1.58 ns) was CBD sample: squeeze_buf[buf_ptr]
extraction -> CBD add/sub -> mod-Q -> bank_t/bank_se write, all combinational
in one cycle and routing-dominated (squeeze_buf and the banks place far apart).
Register the CBD write (data/addr/enable) so the bank write commits one cycle
later; the registered write takes priority in the bank_se/bank_t mux so the
1-cycle spillover into the next state uses the correct addr/data. No consumer
reads these coeffs until the following stage (>1 cycle later), so no RAW hazard.

Verified byte-exact vs NIST KAT: Encaps 9/9, Decaps 9/9, KeyGen 11/11,
hello_world e2e -- all PASS.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
FallenSigh
2026-07-11 07:42:35 +08:00
parent 4538ec496f
commit e30a65bd72
1 changed files with 36 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
rtl/ip/mlkem/top/mlkem_top.v
View File

@@ -312,15 +312,15 @@ module mlkem_top #(
// D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT].
wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo;
wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx;
assign bt_we = ((st == ST_M) && pm_vo) || e2_we ||
assign bt_we = ((st == ST_M) && pm_vo) || cbdw_bt_en ||
u_psum_we || u_intt_we || u_v_we || u_w_we;
assign bt_wa = e2_we ? (c_widx & ((1<<PT_AW)-1)) :
assign bt_wa = cbdw_bt_en ? ({6'd0, cbdw_widx} & ((1<<PT_AW)-1)) :
u_psum_we ? ((UPSUM*256 + u_oidx) & ((1<<PT_AW)-1)) :
u_intt_we ? (u_intt_wr[PT_AW-1:0]) :
u_v_we ? (u_vadd_wr[PT_AW-1:0]) :
u_w_we ? (u_wsub_wr[PT_AW-1:0]) :
((m_i*256 + m_oidx) & ((1<<PT_AW)-1));
assign bt_wd = e2_we ? cbd_modq :
assign bt_wd = cbdw_bt_en ? cbdw_data :
u_psum_we ? u_accq :
u_intt_we ? ntt_coeff :
u_v_we ? u_vq :
@@ -381,16 +381,16 @@ module mlkem_top #(
// bank_se rel slot dec_poly (0..K-1), at coeff index dec_widx.
wire dec_u_we = (st == ST_DEC_DECOMP) && !dec_in_c2 && cd_vo;
wire [13:0] dec_u_wr = {2'b0, dec_poly}*256 + dec_widx;
assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly < {1'b0, k_r, 1'b0})) ||
assign bse_we = cbdw_se_en ||
(((st == ST_N) || (st == ST_ENC_N) || (st == ST_DEC_NTT)) && ntt_vo) ||
u_add_we || dec_u_we;
assign bse_wa = u_add_we ? u_add_uwr[PSE_AW-1:0] :
assign bse_wa = cbdw_se_en ? (({1'b0, cbdw_poly}*256 + cbdw_widx) & ((1<<PSE_AW)-1)) :
u_add_we ? u_add_uwr[PSE_AW-1:0] :
dec_u_we ? dec_u_wr[PSE_AW-1:0] :
(st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ((n_slot*256 + n_widx) & ((1<<PSE_AW)-1))
: ((c_poly*256 + c_widx) & ((1<<PSE_AW)-1));
assign bse_wd = u_add_we ? u_uq :
assign bse_wd = cbdw_se_en ? cbdw_data :
u_add_we ? u_uq :
dec_u_we ? cd_out :
(st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ntt_coeff : cbd_modq;
always @(posedge clk) begin
@@ -797,6 +797,34 @@ module mlkem_top #(
// signed (two's complement) -> [0,Q): add Q when negative
wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff;
// Pipeline the CBD -> bank write by one cycle. The squeeze_buf[buf_ptr]
// extraction -> CBD add/sub -> mod-Q -> BRAM-write route is long and
// routing-dominated (the mlkem_clk critical path at 100 MHz). Register the
// write data/addr/enable here; the bank write commits one cycle later. No
// consumer reads these coeffs until the following stage, which begins more
// than one cycle after the last CBD coeff, so there is no read-after-write
// hazard. The registered write takes priority in the bank_se/bank_t mux so
// the 1-cycle spillover into the next state uses the correct addr/data.
reg cbdw_se_en, cbdw_bt_en;
reg [4:0] cbdw_poly;
reg [7:0] cbdw_widx;
reg [11:0] cbdw_data;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
cbdw_se_en <= 1'b0; cbdw_bt_en <= 1'b0;
cbdw_poly <= 5'd0; cbdw_widx <= 8'd0; cbdw_data <= 12'd0;
end else begin
cbdw_se_en <= ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly < {1'b0, k_r, 1'b0}));
cbdw_bt_en <= (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly == {1'b0, k_r, 1'b0});
cbdw_poly <= c_poly;
cbdw_widx <= c_widx;
cbdw_data <= cbd_modq;
end
end
// ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ----
// N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot).
reg [4:0] n_slot; // 0..2K (process s_hat[0..K-1] then e_hat[0..K-1])