fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(kg): bank_t -> sd_bram instance (1R+1W RMW, real BRAM)

Browse Source 
Second bank promoted to sd_bram. Read port muxed by phase (ST_M acc t_hat
addr / ST_E ek-half byteEncode addr / dbg). The shared mux-registers that fed
both bank_se and bank_t were split: the bank_se half stays a manual reg
(m_eacc_rd / e_se_rd), the bank_t half uses sd_bram's internal read register
(bt_rd_data) -- same 1-cycle latency, so the j-select (m_jq) and ST_E coeff
select (e_rd_coeff) just pick between the two registered outputs. RMW safe:
acc read addr (m_oidx+1) leads write addr (m_oidx), no same-cycle alias.
Write port combinational. 11/11 KAT PASS incl. K=3/4 deep accumulate.
This commit is contained in:
FallenSigh
2026-06-28 21:41:51 +08:00
parent d1b409f65f
commit 5cfe8c74ca
1 changed files with 63 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
sync_rtl/top/mlkem_top.v
View File

@@ -120,7 +120,25 @@ module mlkem_top #(
assign ba_wd = snt_coeff;
reg [11:0] bank_se [0:(1<<PSE_AW)-1];
reg [11:0] bank_t [0:(1<<PT_AW)-1];
// bank_t: t_hat. Readers = ST_M acc (j>0 running t_hat) + ST_E ek-half + dbg.
// Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the
// 1-cycle read register. RMW is safe: the acc read addr leads the write
// addr by 1 (m_oidx+1 vs m_oidx), so read/write never alias in a cycle and
// sd_bram write-first == reg-array read-old. Write port combinational.
wire [PT_AW-1:0] bt_rd_addr;
wire [11:0] bt_rd_data;
wire bt_we;
wire [PT_AW-1:0] bt_wa;
wire [11:0] bt_wd;
sd_bram #(.W(12), .D(1<<PT_AW), .A(PT_AW)) u_bank_t (
.clk(clk), .rd_addr(bt_rd_addr), .rd_data(bt_rd_data),
.wr_en(bt_we), .wr_addr(bt_wa), .wr_data(bt_wd)
);
// ST_M accumulate write: t_hat[m_i][m_oidx] <= (acc + product) mod Q when pm_vo.
assign bt_we = (st == ST_M) && pm_vo;
assign bt_wa = (m_i*256 + m_oidx) & ((1<<PT_AW)-1);
assign bt_wd = m_accq;
// Debug readback (registered for timing)
reg [11:0] dbg_coeff_r;
@@ -132,10 +150,15 @@ module mlkem_top #(
wire [13:0] dbg_t_addr = (dbg_slot_i - slot_t_rt)*256 + dbg_idx_i;
// bank_a read port: ST_M load drives pm_a_full; otherwise debug index.
assign ba_rd_addr = (st == ST_M) ? pm_a_full[PA_AW-1:0] : dbg_a_addr[PA_AW-1:0];
// bank_t read port: ST_M acc drives the t_hat accumulate addr; ST_E ek-half
// drives the byteEncode addr; otherwise debug index.
assign bt_rd_addr = (st == ST_M) ? m_tacc_full[PT_AW-1:0] :
(st == ST_E) ? e_rd_full[PT_AW-1:0] :
dbg_t_addr[PT_AW-1:0];
always @(posedge clk) begin
if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bank_t [dbg_t_addr[PT_AW-1:0]];
if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram)
else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]];
else dbg_coeff_r <= ba_rd_data; // bank_a (registered in sd_bram)
else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram)
end
assign dbg_coeff_o = dbg_coeff_r;
@@ -487,16 +510,18 @@ module mlkem_top #(
reg [3:0] e_c0_hi; // saved c0[11:8] for b1
reg [7:0] e_c1_hi; // saved c1[11:4] for b2
wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half};
wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]]
: bank_t [e_rd_full[PT_AW-1:0]];
reg [11:0] e_rd_data_r; // registered read: value for addr presented last cyc
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // == bt_rd_addr in ST_E
// dk half reads bank_se (registered here into e_se_rd); ek half reads
// bank_t via sd_bram (bt_rd_data) -- both have the same 1-cycle latency,
// so the coeff for the addr presented last cycle is selected here.
reg [11:0] e_se_rd; // registered bank_se read (dk-half)
wire [11:0] e_rd_coeff = e_is_dk ? e_se_rd : bt_rd_data;
// byteEncode write byte offset within the target memory: pair*3 + byte index.
wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3)
wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3
wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2
wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0] // b0 = c0[7:0]
: (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]}
wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_coeff[7:0] // b0 = c0[7:0]
: (e_ph == 2'd2) ? {e_rd_coeff[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]}
: e_c1_hi; // b2 = c1[11:4]
wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j
@@ -531,21 +556,21 @@ module mlkem_top #(
// accumulator source: e_hat[i] for first term (j==0), else running t_hat[i].
// e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i.
// t_hat[i] lives in bank_t at relative slot m_i.
// Registered read-ahead: present the index that the NEXT pm_vo will consume
// t_hat[i] lives in bank_t at relative slot m_i (read via sd_bram bt_rd_data).
// Registered read-ahead: present the index the NEXT pm_vo will consume
// m_acc_radr = pm_vo ? m_oidx+1 : m_oidx
// and register the selected bank into m_acc_rd, consumed one cycle later when
// pm_vo is high. The same-index read-modify-write on bank_t is read-old:
// the running t_hat source was written by the previous (j-1) term, long
// settled; the value consumed this cycle was registered before this cycle's
// write to m_oidx. Cadence is CALC(vo=0)/C0(vo=1)/C1(vo=1).
// The e_hat (bank_se) read is registered into m_eacc_rd here; the t_hat
// (bank_t) read is registered inside sd_bram (bt_rd_data). The j-select is
// applied on the registered outputs (m_jq = m_j delayed 1 cyc to align with
// the read latency). RMW read-old holds: read addr (m_oidx+1) leads write
// addr (m_oidx). Cadence CALC(vo=0)/C0(vo=1)/C1(vo=1).
wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx;
wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i
wire [13:0] m_tacc_full = m_i*256 + m_acc_radr;
wire [11:0] m_acc_rd_next = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]]
: bank_t [m_tacc_full[PT_AW-1:0]];
reg [11:0] m_acc_rd;
wire [11:0] m_acc_src = m_acc_rd;
wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr
reg [11:0] m_eacc_rd; // registered bank_se (e_hat) read
reg m_jq; // (m_j==0) delayed 1 cyc to match read latency
// selected accumulator source aligned with pm_coeff
wire [11:0] m_acc_src = m_jq ? m_eacc_rd : bt_rd_data;
// (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract
wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff};
wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0];
@@ -601,11 +626,12 @@ module mlkem_top #(
m_pending <= 1'b0;
pm_valid <= 1'b0;
pm_b_rd <= 12'd0;
m_acc_rd <= 12'd0;
m_eacc_rd <= 12'd0;
m_jq <= 1'b0;
e_poly <= 3'd0;
e_pair <= 8'd0;
e_ph <= 2'd0;
e_rd_data_r<= 12'd0;
e_se_rd <= 12'd0;
e_c0_hi <= 4'd0;
e_c1_hi <= 8'd0;
e_rho <= 10'd0;
@@ -793,10 +819,11 @@ module mlkem_top #(
// ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ----
if (st == ST_M) begin
// registered accumulator read (sd_bram timing): captures the
// index the next pm_vo will consume. Free-running within ST_M;
// primed during LOAD (m_oidx held at 0).
m_acc_rd <= m_acc_rd_next;
// registered accumulator reads (sd_bram timing): e_hat from
// bank_se here, t_hat from bank_t via sd_bram (bt_rd_data).
// m_jq aligns the j-select with the 1-cycle read latency.
m_eacc_rd <= bank_se[m_eacc_full[PSE_AW-1:0]];
m_jq <= (m_j == 2'd0);
// LOAD: present read-ahead addr to bank_a/bank_se; the pair
// registered last cycle (pm_a_rd/pm_b_rd) is consumed by
@@ -817,10 +844,10 @@ module mlkem_top #(
end
end
// ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat
// ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat.
// The bank_t write is the combinational bt_we/bt_wa/bt_wd assigns
// below; here we only advance the accumulate index / (i,j).
if (pm_vo) begin
// bank_t relative slot = m_tslot - slot_t_rt = m_i
bank_t[(m_i*256 + m_oidx) & ((1<<PT_AW)-1)] <= m_accq;
if (m_oidx == 8'd255) begin
// finished this (i,j) term; advance
if (m_j + 2'd1 < k_r) begin
@@ -858,10 +885,11 @@ module mlkem_top #(
// ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ----
// Single registered bank read per cycle; 4-cycle micro-phase per
// coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1,
// ph3 write b2). e_rd_data_r holds the value read last cycle.
// ph3 write b2). The coeff for the addr presented last cycle is
// e_rd_coeff (= e_se_rd for dk-half, bt_rd_data for ek-half).
if (st == ST_E && !e_done) begin
// registered read (sd_bram timing): capture addr presented now
e_rd_data_r <= e_rd_async;
// registered bank_se read (dk-half); ek-half uses bt_rd_data
e_se_rd <= bank_se[e_rd_full[PSE_AW-1:0]];
if (e_poly < {1'b0, k_r, 1'b0}) begin
// ph0: fetch only (prime). ph1..3: write one packed byte.
@@ -877,8 +905,8 @@ module mlkem_top #(
end
end
// save coeff high nibbles as they become available
if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8]; // c0[11:8]
if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4]; // c1[11:4]
if (e_ph == 2'd1) e_c0_hi <= e_rd_coeff[11:8]; // c0[11:8]
if (e_ph == 2'd2) e_c1_hi <= e_rd_coeff[11:4]; // c1[11:4]
// advance micro-phase / pair / poly
if (e_ph == 2'd3) begin