From a38c41a1f5a0886ba7cce8dd8018e82d958166be Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sun, 28 Jun 2026 22:00:41 +0800 Subject: [PATCH] refactor(kg): bank_se -> sd_bram instance; Phase 2 complete (polymem all BRAM) Final bank promoted to sd_bram (the busiest: 5 read sites, 2 write sites). Read port phase-muxed: ST_N load / ST_M load (pm_b s_hat[j]) vs accumulate (e_hat, selected by m_loading) / ST_E dk-half / dbg. Write port combinational: ST_C CBD store vs ST_N NTT writeback (disjoint states). All explicit consumer read registers (n_rd_data, pm_b_rd, m_eacc_rd, e_se_rd) collapsed into the sd_bram internal read register; m_acc_src and e_rd_coeff now select between two registered sd_bram outputs (same 1-cycle latency). mlkem_top now contains ZERO behavioural RAM arrays: all coefficient storage is 3 sd_bram banks (a/se/t) + ek/dkp byte buffers = 5 sd_bram instances total, each inferring BRAM (ASIC: compiled SRAM). 11/11 KAT PASS, byte-exact. --- sync_rtl/top/mlkem_top.v | 131 +++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/sync_rtl/top/mlkem_top.v b/sync_rtl/top/mlkem_top.v index c26d8de..9c07daf 100644 --- a/sync_rtl/top/mlkem_top.v +++ b/sync_rtl/top/mlkem_top.v @@ -119,7 +119,20 @@ module mlkem_top #( assign ba_wa = (a_slot*256 + a_widx) & ((1<0 running t_hat) + ST_E ek-half + dbg. // Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the @@ -155,9 +168,23 @@ module mlkem_top #( assign bt_rd_addr = (st == ST_M) ? m_tacc_full[PT_AW-1:0] : (st == ST_E) ? e_rd_full[PT_AW-1:0] : dbg_t_addr[PT_AW-1:0]; + // bank_se read port: ST_N load (s/e NTT), ST_M load (pm_b s_hat[j]) vs + // accumulate (e_hat, mutually exclusive via m_loading), ST_E dk-half, dbg. + assign bse_rd_addr = (st == ST_N) ? ntt_rd_full[PSE_AW-1:0] : + (st == ST_M) ? (m_loading ? pm_b_full[PSE_AW-1:0] + : m_eacc_full[PSE_AW-1:0]) : + (st == ST_E) ? e_rd_full[PSE_AW-1:0] : + dbg_se_addr[PSE_AW-1:0]; + // bank_se write port: ST_C CBD store (rel slot c_poly), ST_N NTT writeback + // (rel slot n_slot). Disjoint states. + assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) || + ((st == ST_N) && ntt_vo); + assign bse_wa = (st == ST_N) ? ((n_slot*256 + n_widx) & ((1<= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram) - else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]]; + else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bse_rd_data; // bank_se (sd_bram) else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram) end assign dbg_coeff_o = dbg_coeff_r; @@ -455,17 +482,16 @@ module mlkem_top #( reg n_valid; // feeding coeffs to ntt_core (delayed 1 cyc vs n_ridx) reg n_loading; // 1 while presenting load addresses to bank_se reg n_pending; // waiting for ntt_core IDLE to start next slot - reg [11:0] n_rd_data; // registered bank_se read (== sd_bram timing) wire [SAW-1:0] n_slot_addr = slot_s_rt + n_slot; // s_hat then e_hat contiguous wire ntt_ready; wire [11:0] ntt_coeff; wire ntt_vo; wire ntt_done; - // bank_se read addr for the NTT load (relative slot = n_slot); registered - // into n_rd_data, which feeds ntt_core 1 cycle later. + // bank_se read addr for the NTT load (relative slot = n_slot); sd_bram + // registers it into bse_rd_data, which feeds ntt_core 1 cycle later. wire [13:0] ntt_rd_full = n_slot*256 + n_ridx[7:0]; - wire [11:0] ntt_in = n_rd_data; + wire [11:0] ntt_in = bse_rd_data; ntt_core u_ntt ( .clk(clk), .rst_n(rst_n), @@ -510,12 +536,11 @@ module mlkem_top #( reg [3:0] e_c0_hi; // saved c0[11:8] for b1 reg [7:0] e_c1_hi; // saved c1[11:4] for b2 wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1 - wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // == bt_rd_addr in ST_E - // dk half reads bank_se (registered here into e_se_rd); ek half reads - // bank_t via sd_bram (bt_rd_data) -- both have the same 1-cycle latency, - // so the coeff for the addr presented last cycle is selected here. - reg [11:0] e_se_rd; // registered bank_se read (dk-half) - wire [11:0] e_rd_coeff = e_is_dk ? e_se_rd : bt_rd_data; + wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // bse/bt rd addr in ST_E + // dk half reads bank_se (bse_rd_data), ek half reads bank_t (bt_rd_data); + // both registered inside their sd_bram with the same 1-cycle latency, so + // the coeff for the addr presented last cycle is selected here. + wire [11:0] e_rd_coeff = e_is_dk ? bse_rd_data : bt_rd_data; // byteEncode write byte offset within the target memory: pair*3 + byte index. wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3) wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3 @@ -535,13 +560,12 @@ module mlkem_top #( wire pm_vo; // pm_a: A_hat[i][j] in bank_a (abs slot m_aslot). pm_b: s_hat[j] in // bank_se (relative slot = m_sslot - slot_s_rt = m_j). m_ld is a read-ahead - // pointer; both bank reads are registered into pm_a_rd/pm_b_rd and fed to - // poly_mul one cycle later (pm_valid delayed 1 cyc), like ST_N. + // pointer; both bank reads come from their sd_bram (registered, 1-cycle + // latency) and feed poly_mul one cycle later (pm_valid delayed 1 cyc). wire [13:0] pm_a_full = m_aslot*256 + m_ld[7:0]; wire [13:0] pm_b_full = m_j*256 + m_ld[7:0]; - reg [11:0] pm_b_rd; // registered bank_se read (sd_bram timing) wire [11:0] pm_a_in = ba_rd_data; // bank_a sd_bram registered read - wire [11:0] pm_b_in = pm_b_rd; + wire [11:0] pm_b_in = bse_rd_data; // bank_se sd_bram registered read (load phase) poly_mul_sync u_pmul ( .clk(clk), .rst_n(rst_n), @@ -556,21 +580,19 @@ module mlkem_top #( // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]. // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i. - // t_hat[i] lives in bank_t at relative slot m_i (read via sd_bram bt_rd_data). + // t_hat[i] lives in bank_t at relative slot m_i. Both reads come from their + // sd_bram (bse_rd_data / bt_rd_data) with the same 1-cycle latency. // Registered read-ahead: present the index the NEXT pm_vo will consume // m_acc_radr = pm_vo ? m_oidx+1 : m_oidx - // The e_hat (bank_se) read is registered into m_eacc_rd here; the t_hat - // (bank_t) read is registered inside sd_bram (bt_rd_data). The j-select is - // applied on the registered outputs (m_jq = m_j delayed 1 cyc to align with - // the read latency). RMW read-old holds: read addr (m_oidx+1) leads write - // addr (m_oidx). Cadence CALC(vo=0)/C0(vo=1)/C1(vo=1). + // The j-select is applied on the registered outputs (m_jq = (m_j==0) + // delayed 1 cyc to align with the read latency). RMW read-old holds: read + // addr (m_oidx+1) leads write addr (m_oidx). Cadence CALC/C0/C1. wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx; - wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i + wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i (bse_rd_addr in acc) wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr - reg [11:0] m_eacc_rd; // registered bank_se (e_hat) read reg m_jq; // (m_j==0) delayed 1 cyc to match read latency // selected accumulator source aligned with pm_coeff - wire [11:0] m_acc_src = m_jq ? m_eacc_rd : bt_rd_data; + wire [11:0] m_acc_src = m_jq ? bse_rd_data : bt_rd_data; // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0]; @@ -616,7 +638,6 @@ module mlkem_top #( n_widx <= 8'd0; n_valid <= 1'b0; n_loading <= 1'b0; - n_rd_data <= 12'd0; n_pending <= 1'b0; m_i <= 2'd0; m_j <= 2'd0; @@ -625,13 +646,10 @@ module mlkem_top #( m_loading <= 1'b0; m_pending <= 1'b0; pm_valid <= 1'b0; - pm_b_rd <= 12'd0; - m_eacc_rd <= 12'd0; m_jq <= 1'b0; e_poly <= 3'd0; e_pair <= 8'd0; e_ph <= 2'd0; - e_se_rd <= 12'd0; e_c0_hi <= 4'd0; e_c1_hi <= 8'd0; e_rho <= 10'd0; @@ -734,8 +752,8 @@ module mlkem_top #( end if (c_busy && cbd_vo && cbd_ack) begin - // bank_se relative slot = c_slot - slot_s_rt = c_poly - bank_se[(c_poly*256 + c_widx) & ((1<0 after last end @@ -805,8 +822,8 @@ module mlkem_top #( end // Arm M stage when N finishes: prime first (i=0,j=0) poly_mul load. - // m_ld read-ahead pointer; pm_a_rd/pm_b_rd registered, pm_valid - // asserted one cycle after an address is presented (like ST_N). + // m_ld read-ahead pointer; bank_a/bank_se sd_bram reads land 1 cyc + // later, pm_valid asserted one cycle after an address is presented. if (st == ST_N && st_next == ST_M) begin m_i <= 2'd0; m_j <= 2'd0; @@ -819,15 +836,15 @@ module mlkem_top #( // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ---- if (st == ST_M) begin - // registered accumulator reads (sd_bram timing): e_hat from - // bank_se here, t_hat from bank_t via sd_bram (bt_rd_data). - // m_jq aligns the j-select with the 1-cycle read latency. - m_eacc_rd <= bank_se[m_eacc_full[PSE_AW-1:0]]; - m_jq <= (m_j == 2'd0); + // accumulator reads come from sd_bram (bse_rd_data e_hat / + // bt_rd_data t_hat). m_jq aligns the j-select with the 1-cycle + // read latency. (bse_rd_addr muxes load vs acc by m_loading.) + m_jq <= (m_j == 2'd0); - // LOAD: present read-ahead addr to bank_a/bank_se; the pair - // registered last cycle (pm_a_rd/pm_b_rd) is consumed by - // poly_mul this cycle (pm_valid). poly_mul holds ready high + // LOAD: present read-ahead addr to bank_a/bank_se via their + // sd_bram read ports (ba_rd_addr=pm_a_full, bse_rd_addr=pm_b_full + // while m_loading); ba_rd_data/bse_rd_data land next cycle and + // are consumed by poly_mul (pm_valid). poly_mul holds ready high // through LOAD, so a fixed 1-cycle skew suffices. if (m_loading) begin if (m_ld == 9'd256) begin @@ -836,9 +853,6 @@ module mlkem_top #( m_ld <= 9'd0; m_oidx <= 8'd0; end else begin - // bank_a read is via sd_bram (ba_rd_addr=pm_a_full); - // ba_rd_data lands next cycle == old pm_a_rd timing. - pm_b_rd <= bank_se[pm_b_full[PSE_AW-1:0]]; m_ld <= m_ld + 9'd1; pm_valid <= 1'b1; // pair presented last cycle is valid end @@ -886,11 +900,10 @@ module mlkem_top #( // Single registered bank read per cycle; 4-cycle micro-phase per // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1, // ph3 write b2). The coeff for the addr presented last cycle is - // e_rd_coeff (= e_se_rd for dk-half, bt_rd_data for ek-half). + // e_rd_coeff (= bse_rd_data for dk-half, bt_rd_data for ek-half), + // both registered inside their sd_bram (bse_rd_addr/bt_rd_addr = + // e_rd_full in ST_E). if (st == ST_E && !e_done) begin - // registered bank_se read (dk-half); ek-half uses bt_rd_data - e_se_rd <= bank_se[e_rd_full[PSE_AW-1:0]]; - if (e_poly < {1'b0, k_r, 1'b0}) begin // ph0: fetch only (prime). ph1..3: write one packed byte. if (e_ph != 2'd0) begin