fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(kg): bank_t -> sd_bram instance (1R+1W RMW, real BRAM)

Browse Source 
Second bank promoted to sd_bram. Read port muxed by phase (ST_M acc t_hat
addr / ST_E ek-half byteEncode addr / dbg). The shared mux-registers that fed
both bank_se and bank_t were split: the bank_se half stays a manual reg
(m_eacc_rd / e_se_rd), the bank_t half uses sd_bram's internal read register
(bt_rd_data) -- same 1-cycle latency, so the j-select (m_jq) and ST_E coeff
select (e_rd_coeff) just pick between the two registered outputs. RMW safe:
acc read addr (m_oidx+1) leads write addr (m_oidx), no same-cycle alias.
Write port combinational. 11/11 KAT PASS incl. K=3/4 deep accumulate.
This commit is contained in:
FallenSigh
2026-06-28 21:41:51 +08:00
parent d1b409f65f
commit 5cfe8c74ca
1 changed files with 63 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
sync_rtl/top/mlkem_top.v
View File

@@ -120,7 +120,25 @@ module mlkem_top #(
assign ba_wd = snt_coeff; assign ba_wd = snt_coeff;
reg [11:0] bank_se [0:(1<<PSE_AW)-1]; reg [11:0] bank_se [0:(1<<PSE_AW)-1];
reg [11:0] bank_t [0:(1<<PT_AW)-1];
// bank_t: t_hat. Readers = ST_M acc (j>0 running t_hat) + ST_E ek-half + dbg.
// Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the
// 1-cycle read register. RMW is safe: the acc read addr leads the write
// addr by 1 (m_oidx+1 vs m_oidx), so read/write never alias in a cycle and
// sd_bram write-first == reg-array read-old. Write port combinational.
wire [PT_AW-1:0] bt_rd_addr;
wire [11:0] bt_rd_data;
wire bt_we;
wire [PT_AW-1:0] bt_wa;
wire [11:0] bt_wd;
sd_bram #(.W(12), .D(1<<PT_AW), .A(PT_AW)) u_bank_t (
.clk(clk), .rd_addr(bt_rd_addr), .rd_data(bt_rd_data),
.wr_en(bt_we), .wr_addr(bt_wa), .wr_data(bt_wd)
);
// ST_M accumulate write: t_hat[m_i][m_oidx] <= (acc + product) mod Q when pm_vo.
assign bt_we = (st == ST_M) && pm_vo;
assign bt_wa = (m_i*256 + m_oidx) & ((1<<PT_AW)-1);
assign bt_wd = m_accq;
// Debug readback (registered for timing) // Debug readback (registered for timing)
reg [11:0] dbg_coeff_r; reg [11:0] dbg_coeff_r;
@@ -132,10 +150,15 @@ module mlkem_top #(
wire [13:0] dbg_t_addr = (dbg_slot_i - slot_t_rt)*256 + dbg_idx_i; wire [13:0] dbg_t_addr = (dbg_slot_i - slot_t_rt)*256 + dbg_idx_i;
// bank_a read port: ST_M load drives pm_a_full; otherwise debug index. // bank_a read port: ST_M load drives pm_a_full; otherwise debug index.
assign ba_rd_addr = (st == ST_M) ? pm_a_full[PA_AW-1:0] : dbg_a_addr[PA_AW-1:0]; assign ba_rd_addr = (st == ST_M) ? pm_a_full[PA_AW-1:0] : dbg_a_addr[PA_AW-1:0];
// bank_t read port: ST_M acc drives the t_hat accumulate addr; ST_E ek-half
// drives the byteEncode addr; otherwise debug index.
assign bt_rd_addr = (st == ST_M) ? m_tacc_full[PT_AW-1:0] :
(st == ST_E) ? e_rd_full[PT_AW-1:0] :
dbg_t_addr[PT_AW-1:0];
always @(posedge clk) begin always @(posedge clk) begin
if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bank_t [dbg_t_addr[PT_AW-1:0]]; if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram)
else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]]; else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]];
else dbg_coeff_r <= ba_rd_data; // bank_a (registered in sd_bram) else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram)
end end
assign dbg_coeff_o = dbg_coeff_r; assign dbg_coeff_o = dbg_coeff_r;
@@ -487,16 +510,18 @@ module mlkem_top #(
reg [3:0] e_c0_hi; // saved c0[11:8] for b1 reg [3:0] e_c0_hi; // saved c0[11:8] for b1
reg [7:0] e_c1_hi; // saved c1[11:4] for b2 reg [7:0] e_c1_hi; // saved c1[11:4] for b2
wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1 wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // == bt_rd_addr in ST_E
wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]] // dk half reads bank_se (registered here into e_se_rd); ek half reads
: bank_t [e_rd_full[PT_AW-1:0]]; // bank_t via sd_bram (bt_rd_data) -- both have the same 1-cycle latency,
reg [11:0] e_rd_data_r; // registered read: value for addr presented last cyc // so the coeff for the addr presented last cycle is selected here.
reg [11:0] e_se_rd; // registered bank_se read (dk-half)
wire [11:0] e_rd_coeff = e_is_dk ? e_se_rd : bt_rd_data;
// byteEncode write byte offset within the target memory: pair*3 + byte index. // byteEncode write byte offset within the target memory: pair*3 + byte index.
wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3) wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3)
wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3 wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3
wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2 wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2
wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0] // b0 = c0[7:0] wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_coeff[7:0] // b0 = c0[7:0]
: (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]} : (e_ph == 2'd2) ? {e_rd_coeff[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]}
: e_c1_hi; // b2 = c1[11:4] : e_c1_hi; // b2 = c1[11:4]
wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j
@@ -531,21 +556,21 @@ module mlkem_top #(
// accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]. // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i].
// e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i. // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i.
// t_hat[i] lives in bank_t at relative slot m_i. // t_hat[i] lives in bank_t at relative slot m_i (read via sd_bram bt_rd_data).
// Registered read-ahead: present the index that the NEXT pm_vo will consume // Registered read-ahead: present the index the NEXT pm_vo will consume
// m_acc_radr = pm_vo ? m_oidx+1 : m_oidx // m_acc_radr = pm_vo ? m_oidx+1 : m_oidx
// and register the selected bank into m_acc_rd, consumed one cycle later when // The e_hat (bank_se) read is registered into m_eacc_rd here; the t_hat
// pm_vo is high. The same-index read-modify-write on bank_t is read-old: // (bank_t) read is registered inside sd_bram (bt_rd_data). The j-select is
// the running t_hat source was written by the previous (j-1) term, long // applied on the registered outputs (m_jq = m_j delayed 1 cyc to align with
// settled; the value consumed this cycle was registered before this cycle's // the read latency). RMW read-old holds: read addr (m_oidx+1) leads write
// write to m_oidx. Cadence is CALC(vo=0)/C0(vo=1)/C1(vo=1). // addr (m_oidx). Cadence CALC(vo=0)/C0(vo=1)/C1(vo=1).
wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx; wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx;
wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i
wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr
wire [11:0] m_acc_rd_next = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]] reg [11:0] m_eacc_rd; // registered bank_se (e_hat) read
: bank_t [m_tacc_full[PT_AW-1:0]]; reg m_jq; // (m_j==0) delayed 1 cyc to match read latency
reg [11:0] m_acc_rd; // selected accumulator source aligned with pm_coeff
wire [11:0] m_acc_src = m_acc_rd; wire [11:0] m_acc_src = m_jq ? m_eacc_rd : bt_rd_data;
// (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract
wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff};
wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0]; wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0];
@@ -601,11 +626,12 @@ module mlkem_top #(
m_pending <= 1'b0; m_pending <= 1'b0;
pm_valid <= 1'b0; pm_valid <= 1'b0;
pm_b_rd <= 12'd0; pm_b_rd <= 12'd0;
m_acc_rd <= 12'd0; m_eacc_rd <= 12'd0;
m_jq <= 1'b0;
e_poly <= 3'd0; e_poly <= 3'd0;
e_pair <= 8'd0; e_pair <= 8'd0;
e_ph <= 2'd0; e_ph <= 2'd0;
e_rd_data_r<= 12'd0; e_se_rd <= 12'd0;
e_c0_hi <= 4'd0; e_c0_hi <= 4'd0;
e_c1_hi <= 8'd0; e_c1_hi <= 8'd0;
e_rho <= 10'd0; e_rho <= 10'd0;
@@ -793,10 +819,11 @@ module mlkem_top #(
// ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ---- // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ----
if (st == ST_M) begin if (st == ST_M) begin
// registered accumulator read (sd_bram timing): captures the // registered accumulator reads (sd_bram timing): e_hat from
// index the next pm_vo will consume. Free-running within ST_M; // bank_se here, t_hat from bank_t via sd_bram (bt_rd_data).
// primed during LOAD (m_oidx held at 0). // m_jq aligns the j-select with the 1-cycle read latency.
m_acc_rd <= m_acc_rd_next; m_eacc_rd <= bank_se[m_eacc_full[PSE_AW-1:0]];
m_jq <= (m_j == 2'd0);
// LOAD: present read-ahead addr to bank_a/bank_se; the pair // LOAD: present read-ahead addr to bank_a/bank_se; the pair
// registered last cycle (pm_a_rd/pm_b_rd) is consumed by // registered last cycle (pm_a_rd/pm_b_rd) is consumed by
@@ -817,10 +844,10 @@ module mlkem_top #(
end end
end end
// ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat // ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat.
// The bank_t write is the combinational bt_we/bt_wa/bt_wd assigns
// below; here we only advance the accumulate index / (i,j).
if (pm_vo) begin if (pm_vo) begin
// bank_t relative slot = m_tslot - slot_t_rt = m_i
bank_t[(m_i*256 + m_oidx) & ((1<<PT_AW)-1)] <= m_accq;
if (m_oidx == 8'd255) begin if (m_oidx == 8'd255) begin
// finished this (i,j) term; advance // finished this (i,j) term; advance
if (m_j + 2'd1 < k_r) begin if (m_j + 2'd1 < k_r) begin
@@ -858,10 +885,11 @@ module mlkem_top #(
// ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ---- // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ----
// Single registered bank read per cycle; 4-cycle micro-phase per // Single registered bank read per cycle; 4-cycle micro-phase per
// coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1, // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1,
// ph3 write b2). e_rd_data_r holds the value read last cycle. // ph3 write b2). The coeff for the addr presented last cycle is
// e_rd_coeff (= e_se_rd for dk-half, bt_rd_data for ek-half).
if (st == ST_E && !e_done) begin if (st == ST_E && !e_done) begin
// registered read (sd_bram timing): capture addr presented now // registered bank_se read (dk-half); ek-half uses bt_rd_data
e_rd_data_r <= e_rd_async; e_se_rd <= bank_se[e_rd_full[PSE_AW-1:0]];
if (e_poly < {1'b0, k_r, 1'b0}) begin if (e_poly < {1'b0, k_r, 1'b0}) begin
// ph0: fetch only (prime). ph1..3: write one packed byte. // ph0: fetch only (prime). ph1..3: write one packed byte.
@@ -877,8 +905,8 @@ module mlkem_top #(
end end
end end
// save coeff high nibbles as they become available // save coeff high nibbles as they become available
if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8]; // c0[11:8] if (e_ph == 2'd1) e_c0_hi <= e_rd_coeff[11:8]; // c0[11:8]
if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4]; // c1[11:4] if (e_ph == 2'd2) e_c1_hi <= e_rd_coeff[11:4]; // c1[11:4]
// advance micro-phase / pair / poly // advance micro-phase / pair / poly
if (e_ph == 2'd3) begin if (e_ph == 2'd3) begin