fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(kg): bank_se -> sd_bram instance; Phase 2 complete (polymem all BRAM)

Browse Source 
Final bank promoted to sd_bram (the busiest: 5 read sites, 2 write sites).
Read port phase-muxed: ST_N load / ST_M load (pm_b s_hat[j]) vs accumulate
(e_hat, selected by m_loading) / ST_E dk-half / dbg. Write port combinational:
ST_C CBD store vs ST_N NTT writeback (disjoint states). All explicit consumer
read registers (n_rd_data, pm_b_rd, m_eacc_rd, e_se_rd) collapsed into the
sd_bram internal read register; m_acc_src and e_rd_coeff now select between
two registered sd_bram outputs (same 1-cycle latency).

mlkem_top now contains ZERO behavioural RAM arrays: all coefficient storage is
3 sd_bram banks (a/se/t) + ek/dkp byte buffers = 5 sd_bram instances total,
each inferring BRAM (ASIC: compiled SRAM). 11/11 KAT PASS, byte-exact.
This commit is contained in:
FallenSigh
2026-06-28 22:00:41 +08:00
parent 5cfe8c74ca
commit a38c41a1f5
1 changed files with 72 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
sync_rtl/top/mlkem_top.v
View File

@@ -119,7 +119,20 @@ module mlkem_top #(
assign ba_wa = (a_slot*256 + a_widx) & ((1<<PA_AW)-1);
assign ba_wd = snt_coeff;
reg [11:0] bank_se [0:(1<<PSE_AW)-1];
// bank_se: s_hat || e_hat. Readers = ST_N load, ST_M load (pm_b, s_hat[j]),
// ST_M acc (e_hat, j==0), ST_E dk-half, dbg. Writers = ST_C (CBD), ST_N
// writeback (NTT in place). All readers already registered; read port muxed
// by phase (within ST_M, load vs accumulate never overlap). Write port
// combinational (ST_C and ST_N are disjoint states).
wire [PSE_AW-1:0] bse_rd_addr;
wire [11:0] bse_rd_data;
wire bse_we;
wire [PSE_AW-1:0] bse_wa;
wire [11:0] bse_wd;
sd_bram #(.W(12), .D(1<<PSE_AW), .A(PSE_AW)) u_bank_se (
.clk(clk), .rd_addr(bse_rd_addr), .rd_data(bse_rd_data),
.wr_en(bse_we), .wr_addr(bse_wa), .wr_data(bse_wd)
);
// bank_t: t_hat. Readers = ST_M acc (j>0 running t_hat) + ST_E ek-half + dbg.
// Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the
@@ -155,9 +168,23 @@ module mlkem_top #(
assign bt_rd_addr = (st == ST_M) ? m_tacc_full[PT_AW-1:0] :
(st == ST_E) ? e_rd_full[PT_AW-1:0] :
dbg_t_addr[PT_AW-1:0];
// bank_se read port: ST_N load (s/e NTT), ST_M load (pm_b s_hat[j]) vs
// accumulate (e_hat, mutually exclusive via m_loading), ST_E dk-half, dbg.
assign bse_rd_addr = (st == ST_N) ? ntt_rd_full[PSE_AW-1:0] :
(st == ST_M) ? (m_loading ? pm_b_full[PSE_AW-1:0]
: m_eacc_full[PSE_AW-1:0]) :
(st == ST_E) ? e_rd_full[PSE_AW-1:0] :
dbg_se_addr[PSE_AW-1:0];
// bank_se write port: ST_C CBD store (rel slot c_poly), ST_N NTT writeback
// (rel slot n_slot). Disjoint states.
assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_N) && ntt_vo);
assign bse_wa = (st == ST_N) ? ((n_slot*256 + n_widx) & ((1<<PSE_AW)-1))
: ((c_poly*256 + c_widx) & ((1<<PSE_AW)-1));
assign bse_wd = (st == ST_N) ? ntt_coeff : cbd_modq;
always @(posedge clk) begin
if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram)
else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]];
else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bse_rd_data; // bank_se (sd_bram)
else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram)
end
assign dbg_coeff_o = dbg_coeff_r;
@@ -455,17 +482,16 @@ module mlkem_top #(
reg n_valid; // feeding coeffs to ntt_core (delayed 1 cyc vs n_ridx)
reg n_loading; // 1 while presenting load addresses to bank_se
reg n_pending; // waiting for ntt_core IDLE to start next slot
reg [11:0] n_rd_data; // registered bank_se read (== sd_bram timing)
wire [SAW-1:0] n_slot_addr = slot_s_rt + n_slot; // s_hat then e_hat contiguous
wire ntt_ready;
wire [11:0] ntt_coeff;
wire ntt_vo;
wire ntt_done;
// bank_se read addr for the NTT load (relative slot = n_slot); registered
// into n_rd_data, which feeds ntt_core 1 cycle later.
// bank_se read addr for the NTT load (relative slot = n_slot); sd_bram
// registers it into bse_rd_data, which feeds ntt_core 1 cycle later.
wire [13:0] ntt_rd_full = n_slot*256 + n_ridx[7:0];
wire [11:0] ntt_in = n_rd_data;
wire [11:0] ntt_in = bse_rd_data;
ntt_core u_ntt (
.clk(clk), .rst_n(rst_n),
@@ -510,12 +536,11 @@ module mlkem_top #(
reg [3:0] e_c0_hi; // saved c0[11:8] for b1
reg [7:0] e_c1_hi; // saved c1[11:4] for b2
wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // == bt_rd_addr in ST_E
// dk half reads bank_se (registered here into e_se_rd); ek half reads
// bank_t via sd_bram (bt_rd_data) -- both have the same 1-cycle latency,
// so the coeff for the addr presented last cycle is selected here.
reg [11:0] e_se_rd; // registered bank_se read (dk-half)
wire [11:0] e_rd_coeff = e_is_dk ? e_se_rd : bt_rd_data;
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // bse/bt rd addr in ST_E
// dk half reads bank_se (bse_rd_data), ek half reads bank_t (bt_rd_data);
// both registered inside their sd_bram with the same 1-cycle latency, so
// the coeff for the addr presented last cycle is selected here.
wire [11:0] e_rd_coeff = e_is_dk ? bse_rd_data : bt_rd_data;
// byteEncode write byte offset within the target memory: pair*3 + byte index.
wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3)
wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3
@@ -535,13 +560,12 @@ module mlkem_top #(
wire pm_vo;
// pm_a: A_hat[i][j] in bank_a (abs slot m_aslot). pm_b: s_hat[j] in
// bank_se (relative slot = m_sslot - slot_s_rt = m_j). m_ld is a read-ahead
// pointer; both bank reads are registered into pm_a_rd/pm_b_rd and fed to
// poly_mul one cycle later (pm_valid delayed 1 cyc), like ST_N.
// pointer; both bank reads come from their sd_bram (registered, 1-cycle
// latency) and feed poly_mul one cycle later (pm_valid delayed 1 cyc).
wire [13:0] pm_a_full = m_aslot*256 + m_ld[7:0];
wire [13:0] pm_b_full = m_j*256 + m_ld[7:0];
reg [11:0] pm_b_rd; // registered bank_se read (sd_bram timing)
wire [11:0] pm_a_in = ba_rd_data; // bank_a sd_bram registered read
wire [11:0] pm_b_in = pm_b_rd;
wire [11:0] pm_b_in = bse_rd_data; // bank_se sd_bram registered read (load phase)
poly_mul_sync u_pmul (
.clk(clk), .rst_n(rst_n),
@@ -556,21 +580,19 @@ module mlkem_top #(
// accumulator source: e_hat[i] for first term (j==0), else running t_hat[i].
// e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i.
// t_hat[i] lives in bank_t at relative slot m_i (read via sd_bram bt_rd_data).
// t_hat[i] lives in bank_t at relative slot m_i. Both reads come from their
// sd_bram (bse_rd_data / bt_rd_data) with the same 1-cycle latency.
// Registered read-ahead: present the index the NEXT pm_vo will consume
// m_acc_radr = pm_vo ? m_oidx+1 : m_oidx
// The e_hat (bank_se) read is registered into m_eacc_rd here; the t_hat
// (bank_t) read is registered inside sd_bram (bt_rd_data). The j-select is
// applied on the registered outputs (m_jq = m_j delayed 1 cyc to align with
// the read latency). RMW read-old holds: read addr (m_oidx+1) leads write
// addr (m_oidx). Cadence CALC(vo=0)/C0(vo=1)/C1(vo=1).
// The j-select is applied on the registered outputs (m_jq = (m_j==0)
// delayed 1 cyc to align with the read latency). RMW read-old holds: read
// addr (m_oidx+1) leads write addr (m_oidx). Cadence CALC/C0/C1.
wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx;
wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i
wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i (bse_rd_addr in acc)
wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr
reg [11:0] m_eacc_rd; // registered bank_se (e_hat) read
reg m_jq; // (m_j==0) delayed 1 cyc to match read latency
// selected accumulator source aligned with pm_coeff
wire [11:0] m_acc_src = m_jq ? m_eacc_rd : bt_rd_data;
wire [11:0] m_acc_src = m_jq ? bse_rd_data : bt_rd_data;
// (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract
wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff};
wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0];
@@ -616,7 +638,6 @@ module mlkem_top #(
n_widx <= 8'd0;
n_valid <= 1'b0;
n_loading <= 1'b0;
n_rd_data <= 12'd0;
n_pending <= 1'b0;
m_i <= 2'd0;
m_j <= 2'd0;
@@ -625,13 +646,10 @@ module mlkem_top #(
m_loading <= 1'b0;
m_pending <= 1'b0;
pm_valid <= 1'b0;
pm_b_rd <= 12'd0;
m_eacc_rd <= 12'd0;
m_jq <= 1'b0;
e_poly <= 3'd0;
e_pair <= 8'd0;
e_ph <= 2'd0;
e_se_rd <= 12'd0;
e_c0_hi <= 4'd0;
e_c1_hi <= 8'd0;
e_rho <= 10'd0;
@@ -734,8 +752,8 @@ module mlkem_top #(
end
if (c_busy && cbd_vo && cbd_ack) begin
// bank_se relative slot = c_slot - slot_s_rt = c_poly
bank_se[(c_poly*256 + c_widx) & ((1<<PSE_AW)-1)] <= cbd_modq;
// bank_se write is the combinational bse_we/bse_wa/bse_wd
// assigns (rel slot c_poly); here only advance counters.
if (cbd_last) begin
c_poly <= c_poly + 3'd1;
c_widx <= 8'd0;
@@ -748,8 +766,8 @@ module mlkem_top #(
end
// Arm N stage when C finishes: prime load of slot S0. n_ridx is a
// read-ahead pointer; bank_se read is registered into n_rd_data and
// fed to ntt_core one cycle later, so valid starts low (priming).
// read-ahead pointer; bank_se read is registered inside sd_bram (bse_rd_data)
// and fed to ntt_core one cycle later, so valid starts low (priming).
if (st == ST_C && st_next == ST_N) begin
n_slot <= 3'd0;
n_ridx <= 9'd0;
@@ -761,26 +779,25 @@ module mlkem_top #(
// ---- ST_N: forward NTT each of S0,S1,E0,E1 in place ----
if (st == ST_N) begin
// LOAD phase: present read-ahead addr to bank_se; the value
// registered last cycle (n_rd_data) is consumed by ntt_core
// this cycle (n_valid). Cores hold ready high through LOAD, so
// a fixed 1-cycle skew suffices (no backpressure gating).
// LOAD phase: present read-ahead addr to bank_se (bse_rd_addr);
// sd_bram registers it, so bse_rd_data is consumed by ntt_core
// one cycle later (n_valid). Cores hold ready high through LOAD,
// so a fixed 1-cycle skew suffices (no backpressure gating).
if (n_loading) begin
if (n_ridx == 9'd256) begin
// 256th coeff (bank_se[255]) consumed this cycle; stop
// 256th coeff consumed this cycle; stop presenting addr
n_loading <= 1'b0;
n_valid <= 1'b0;
end else begin
n_rd_data <= bank_se[ntt_rd_full[PSE_AW-1:0]];
n_ridx <= n_ridx + 9'd1;
n_valid <= 1'b1; // data presented last cycle is valid
end
end
// OUTPUT phase: collect 256 results, write back to same slot
// OUTPUT phase: collect 256 results, write back to same slot.
// The bank_se write is the combinational bse_we/bse_wa/bse_wd
// assigns (rel slot n_slot); here only advance the write index.
if (ntt_vo) begin
// bank_se relative slot = n_slot_addr - slot_s_rt = n_slot
bank_se[(n_slot*256 + n_widx) & ((1<<PSE_AW)-1)] <= ntt_coeff;
n_widx <= n_widx + 8'd1; // wraps 255->0 after last
end
@@ -805,8 +822,8 @@ module mlkem_top #(
end
// Arm M stage when N finishes: prime first (i=0,j=0) poly_mul load.
// m_ld read-ahead pointer; pm_a_rd/pm_b_rd registered, pm_valid
// asserted one cycle after an address is presented (like ST_N).
// m_ld read-ahead pointer; bank_a/bank_se sd_bram reads land 1 cyc
// later, pm_valid asserted one cycle after an address is presented.
if (st == ST_N && st_next == ST_M) begin
m_i <= 2'd0;
m_j <= 2'd0;
@@ -819,15 +836,15 @@ module mlkem_top #(
// ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ----
if (st == ST_M) begin
// registered accumulator reads (sd_bram timing): e_hat from
// bank_se here, t_hat from bank_t via sd_bram (bt_rd_data).
// m_jq aligns the j-select with the 1-cycle read latency.
m_eacc_rd <= bank_se[m_eacc_full[PSE_AW-1:0]];
m_jq <= (m_j == 2'd0);
// accumulator reads come from sd_bram (bse_rd_data e_hat /
// bt_rd_data t_hat). m_jq aligns the j-select with the 1-cycle
// read latency. (bse_rd_addr muxes load vs acc by m_loading.)
m_jq <= (m_j == 2'd0);
// LOAD: present read-ahead addr to bank_a/bank_se; the pair
// registered last cycle (pm_a_rd/pm_b_rd) is consumed by
// poly_mul this cycle (pm_valid). poly_mul holds ready high
// LOAD: present read-ahead addr to bank_a/bank_se via their
// sd_bram read ports (ba_rd_addr=pm_a_full, bse_rd_addr=pm_b_full
// while m_loading); ba_rd_data/bse_rd_data land next cycle and
// are consumed by poly_mul (pm_valid). poly_mul holds ready high
// through LOAD, so a fixed 1-cycle skew suffices.
if (m_loading) begin
if (m_ld == 9'd256) begin
@@ -836,9 +853,6 @@ module mlkem_top #(
m_ld <= 9'd0;
m_oidx <= 8'd0;
end else begin
// bank_a read is via sd_bram (ba_rd_addr=pm_a_full);
// ba_rd_data lands next cycle == old pm_a_rd timing.
pm_b_rd <= bank_se[pm_b_full[PSE_AW-1:0]];
m_ld <= m_ld + 9'd1;
pm_valid <= 1'b1; // pair presented last cycle is valid
end
@@ -886,11 +900,10 @@ module mlkem_top #(
// Single registered bank read per cycle; 4-cycle micro-phase per
// coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1,
// ph3 write b2). The coeff for the addr presented last cycle is
// e_rd_coeff (= e_se_rd for dk-half, bt_rd_data for ek-half).
// e_rd_coeff (= bse_rd_data for dk-half, bt_rd_data for ek-half),
// both registered inside their sd_bram (bse_rd_addr/bt_rd_addr =
// e_rd_full in ST_E).
if (st == ST_E && !e_done) begin
// registered bank_se read (dk-half); ek-half uses bt_rd_data
e_se_rd <= bank_se[e_rd_full[PSE_AW-1:0]];
if (e_poly < {1'b0, k_r, 1'b0}) begin
// ph0: fetch only (prime). ph1..3: write one packed byte.
if (e_ph != 2'd0) begin