fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(kg): registered single-port read for ST_E byteEncode (bank_se/bank_t)

Browse Source 
Final per-consumer step of stage 2b. ST_E read two coeffs of a pair from one
bank combinationally; now serialized through a single registered read port
(e_rd_data_r) over a 4-cycle micro-phase e_ph: ph0 fetch c0, ph1 write b0 +
fetch c1, ph2 write b1, ph3 write b2. Matches sd_bram timing; +512 cyc on K=2
(one extra fetch per pair-stream). All polymem read consumers (ST_N, ST_M
load, ST_M acc, ST_E) are now registered-read. 11/11 KAT PASS, byte-exact.
This commit is contained in:
FallenSigh
2026-06-28 21:13:39 +08:00
parent 0fa7f48ed4
commit 4afa3a4998
1 changed files with 52 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
sync_rtl/top/mlkem_top.v
View File

@@ -450,28 +450,33 @@ module mlkem_top #(
// bytes || rho; dk_pke = s_hat[0..K-1] bytes. Walk coeff pairs per poly.
reg [4:0] e_poly; // 0..2K-1: [0,K) = t_hat -> ek; [K,2K) = s_hat -> dk_pke
reg [7:0] e_pair; // 0..127 coeff-pair within poly
reg [1:0] e_byte; // 0..2 byte within the current pair (BRAM: 1 write/cycle)
reg [9:0] e_rho; // 0..31 rho byte copy index (ek tail)
reg e_done; // serialization complete
// source poly slot: t_hat[e_poly] for ek half, s_hat[e_poly-K] for dk half
wire e_is_dk = (e_poly >= {1'b0, k_r});
wire [4:0] e_pidx = e_is_dk ? (e_poly - {1'b0, k_r}) : e_poly; // index within target
wire [SAW-1:0] e_slot = e_is_dk ? (slot_s_rt + e_pidx) : (slot_t_rt + e_pidx);
// two coeffs of the current pair: ek half reads t_hat (bank_t), dk half
// reads s_hat (bank_se). Relative index within the bank = e_pidx.
wire [13:0] e_rd0_full = e_pidx*256 + {e_pair, 1'b0};
wire [13:0] e_rd1_full = e_pidx*256 + {e_pair, 1'b1};
wire [11:0] e_c0 = e_is_dk ? bank_se[e_rd0_full[PSE_AW-1:0]] : bank_t[e_rd0_full[PT_AW-1:0]];
wire [11:0] e_c1 = e_is_dk ? bank_se[e_rd1_full[PSE_AW-1:0]] : bank_t[e_rd1_full[PT_AW-1:0]];
// 3 packed bytes
wire [7:0] e_b0 = e_c0[7:0];
wire [7:0] e_b1 = {e_c1[3:0], e_c0[11:8]};
wire [7:0] e_b2 = e_c1[11:4];
// byte base offset within target memory: poly index *384 (= 128 pairs *3)
wire [11:0] e_base = e_pidx * 12'd384;
// Registered single-port read (sd_bram timing). One bank read per cycle:
// ek half reads bank_t (t_hat), dk half reads bank_se (s_hat). The two
// coeffs of a pair are serialized across a 4-cycle micro-phase e_ph:
// 0 = fetch c0
// 1 = c0 ready: write b0, save c0[11:8], fetch c1
// 2 = c1 ready: write b1, save c1[11:4]
// 3 = write b2, advance pair
reg [1:0] e_ph; // per-pair micro-phase 0..3
reg [3:0] e_c0_hi; // saved c0[11:8] for b1
reg [7:0] e_c1_hi; // saved c1[11:4] for b2
wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1
wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half};
wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]]
: bank_t [e_rd_full[PT_AW-1:0]];
reg [11:0] e_rd_data_r; // registered read: value for addr presented last cyc
// byteEncode write byte offset within the target memory: pair*3 + byte index.
wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3)
wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3
// BRAM write is 1 byte/cycle: select packed byte by e_byte sub-counter
wire [7:0] e_byte_d = (e_byte == 2'd0) ? e_b0 : (e_byte == 2'd1) ? e_b1 : e_b2;
wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2
wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0] // b0 = c0[7:0]
: (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]}
: e_c1_hi; // b2 = c1[11:4]
wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j
wire [SAW-1:0] m_sslot = slot_s_rt + m_j; // s_hat[j]
@@ -579,7 +584,10 @@ module mlkem_top #(
m_acc_rd <= 12'd0;
e_poly <= 3'd0;
e_pair <= 8'd0;
e_byte <= 2'd0;
e_ph <= 2'd0;
e_rd_data_r<= 12'd0;
e_c0_hi <= 4'd0;
e_c1_hi <= 8'd0;
e_rho <= 10'd0;
e_done <= 1'b0;
ek_we <= 1'b0;
@@ -819,27 +827,39 @@ module mlkem_top #(
if (st == ST_M && st_next == ST_E) begin
e_poly <= 3'd0;
e_pair <= 8'd0;
e_byte <= 2'd0;
e_ph <= 2'd0; // start at fetch-c0 of first pair
e_rho <= 10'd0;
e_done <= 1'b0;
end
// ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke, 1 byte/cycle ----
// ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ----
// Single registered bank read per cycle; 4-cycle micro-phase per
// coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1,
// ph3 write b2). e_rd_data_r holds the value read last cycle.
if (st == ST_E && !e_done) begin
// registered read (sd_bram timing): capture addr presented now
e_rd_data_r <= e_rd_async;
if (e_poly < {1'b0, k_r, 1'b0}) begin
// write current byte (e_byte_d) to target BRAM
if (!e_is_dk) begin
ek_we <= 1'b1;
ek_wa <= e_boff[10:0] + {9'd0, e_byte};
ek_wd <= e_byte_d;
end else begin
dkp_we <= 1'b1;
dkp_wa <= e_boff[10:0] + {9'd0, e_byte};
dkp_wd <= e_byte_d;
// ph0: fetch only (prime). ph1..3: write one packed byte.
if (e_ph != 2'd0) begin
if (!e_is_dk) begin
ek_we <= 1'b1;
ek_wa <= e_boff[10:0] + {9'd0, e_wb};
ek_wd <= e_wbyte;
end else begin
dkp_we <= 1'b1;
dkp_wa <= e_boff[10:0] + {9'd0, e_wb};
dkp_wd <= e_wbyte;
end
end
// advance sub-counters
if (e_byte == 2'd2) begin
e_byte <= 2'd0;
// save coeff high nibbles as they become available
if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8]; // c0[11:8]
if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4]; // c1[11:4]
// advance micro-phase / pair / poly
if (e_ph == 2'd3) begin
e_ph <= 2'd0;
if (e_pair == 8'd127) begin
e_pair <= 8'd0;
e_poly <= e_poly + 5'd1; // next poly or -> rho phase
@@ -847,7 +867,7 @@ module mlkem_top #(
e_pair <= e_pair + 8'd1;
end
end else begin
e_byte <= e_byte + 2'd1;
e_ph <= e_ph + 2'd1;
end
end else begin
// rho copy: ek[384*K + r] = rho byte r (r = 0..31), 1 byte/cycle