diff --git a/sync_rtl/top/mlkem_top.v b/sync_rtl/top/mlkem_top.v index d926c02..c26d8de 100644 --- a/sync_rtl/top/mlkem_top.v +++ b/sync_rtl/top/mlkem_top.v @@ -120,7 +120,25 @@ module mlkem_top #( assign ba_wd = snt_coeff; reg [11:0] bank_se [0:(1<0 running t_hat) + ST_E ek-half + dbg. + // Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the + // 1-cycle read register. RMW is safe: the acc read addr leads the write + // addr by 1 (m_oidx+1 vs m_oidx), so read/write never alias in a cycle and + // sd_bram write-first == reg-array read-old. Write port combinational. + wire [PT_AW-1:0] bt_rd_addr; + wire [11:0] bt_rd_data; + wire bt_we; + wire [PT_AW-1:0] bt_wa; + wire [11:0] bt_wd; + sd_bram #(.W(12), .D(1<= slot_t_rt) dbg_coeff_r <= bank_t [dbg_t_addr[PT_AW-1:0]]; + if (dbg_slot_i >= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram) else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bank_se[dbg_se_addr[PSE_AW-1:0]]; - else dbg_coeff_r <= ba_rd_data; // bank_a (registered in sd_bram) + else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram) end assign dbg_coeff_o = dbg_coeff_r; @@ -487,16 +510,18 @@ module mlkem_top #( reg [3:0] e_c0_hi; // saved c0[11:8] for b1 reg [7:0] e_c1_hi; // saved c1[11:4] for b2 wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1 - wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; - wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]] - : bank_t [e_rd_full[PT_AW-1:0]]; - reg [11:0] e_rd_data_r; // registered read: value for addr presented last cyc + wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // == bt_rd_addr in ST_E + // dk half reads bank_se (registered here into e_se_rd); ek half reads + // bank_t via sd_bram (bt_rd_data) -- both have the same 1-cycle latency, + // so the coeff for the addr presented last cycle is selected here. + reg [11:0] e_se_rd; // registered bank_se read (dk-half) + wire [11:0] e_rd_coeff = e_is_dk ? e_se_rd : bt_rd_data; // byteEncode write byte offset within the target memory: pair*3 + byte index. wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3) wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3 wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2 - wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0] // b0 = c0[7:0] - : (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]} + wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_coeff[7:0] // b0 = c0[7:0] + : (e_ph == 2'd2) ? {e_rd_coeff[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]} : e_c1_hi; // b2 = c1[11:4] wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j @@ -531,21 +556,21 @@ module mlkem_top #( // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]. // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i. - // t_hat[i] lives in bank_t at relative slot m_i. - // Registered read-ahead: present the index that the NEXT pm_vo will consume + // t_hat[i] lives in bank_t at relative slot m_i (read via sd_bram bt_rd_data). + // Registered read-ahead: present the index the NEXT pm_vo will consume // m_acc_radr = pm_vo ? m_oidx+1 : m_oidx - // and register the selected bank into m_acc_rd, consumed one cycle later when - // pm_vo is high. The same-index read-modify-write on bank_t is read-old: - // the running t_hat source was written by the previous (j-1) term, long - // settled; the value consumed this cycle was registered before this cycle's - // write to m_oidx. Cadence is CALC(vo=0)/C0(vo=1)/C1(vo=1). + // The e_hat (bank_se) read is registered into m_eacc_rd here; the t_hat + // (bank_t) read is registered inside sd_bram (bt_rd_data). The j-select is + // applied on the registered outputs (m_jq = m_j delayed 1 cyc to align with + // the read latency). RMW read-old holds: read addr (m_oidx+1) leads write + // addr (m_oidx). Cadence CALC(vo=0)/C0(vo=1)/C1(vo=1). wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx; wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i - wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; - wire [11:0] m_acc_rd_next = (m_j == 2'd0) ? bank_se[m_eacc_full[PSE_AW-1:0]] - : bank_t [m_tacc_full[PT_AW-1:0]]; - reg [11:0] m_acc_rd; - wire [11:0] m_acc_src = m_acc_rd; + wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr + reg [11:0] m_eacc_rd; // registered bank_se (e_hat) read + reg m_jq; // (m_j==0) delayed 1 cyc to match read latency + // selected accumulator source aligned with pm_coeff + wire [11:0] m_acc_src = m_jq ? m_eacc_rd : bt_rd_data; // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0]; @@ -601,11 +626,12 @@ module mlkem_top #( m_pending <= 1'b0; pm_valid <= 1'b0; pm_b_rd <= 12'd0; - m_acc_rd <= 12'd0; + m_eacc_rd <= 12'd0; + m_jq <= 1'b0; e_poly <= 3'd0; e_pair <= 8'd0; e_ph <= 2'd0; - e_rd_data_r<= 12'd0; + e_se_rd <= 12'd0; e_c0_hi <= 4'd0; e_c1_hi <= 8'd0; e_rho <= 10'd0; @@ -793,10 +819,11 @@ module mlkem_top #( // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ---- if (st == ST_M) begin - // registered accumulator read (sd_bram timing): captures the - // index the next pm_vo will consume. Free-running within ST_M; - // primed during LOAD (m_oidx held at 0). - m_acc_rd <= m_acc_rd_next; + // registered accumulator reads (sd_bram timing): e_hat from + // bank_se here, t_hat from bank_t via sd_bram (bt_rd_data). + // m_jq aligns the j-select with the 1-cycle read latency. + m_eacc_rd <= bank_se[m_eacc_full[PSE_AW-1:0]]; + m_jq <= (m_j == 2'd0); // LOAD: present read-ahead addr to bank_a/bank_se; the pair // registered last cycle (pm_a_rd/pm_b_rd) is consumed by @@ -817,10 +844,10 @@ module mlkem_top #( end end - // ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat + // ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat. + // The bank_t write is the combinational bt_we/bt_wa/bt_wd assigns + // below; here we only advance the accumulate index / (i,j). if (pm_vo) begin - // bank_t relative slot = m_tslot - slot_t_rt = m_i - bank_t[(m_i*256 + m_oidx) & ((1< ek, s_hat -> dk_pke ---- // Single registered bank read per cycle; 4-cycle micro-phase per // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1, - // ph3 write b2). e_rd_data_r holds the value read last cycle. + // ph3 write b2). The coeff for the addr presented last cycle is + // e_rd_coeff (= e_se_rd for dk-half, bt_rd_data for ek-half). if (st == ST_E && !e_done) begin - // registered read (sd_bram timing): capture addr presented now - e_rd_data_r <= e_rd_async; + // registered bank_se read (dk-half); ek-half uses bt_rd_data + e_se_rd <= bank_se[e_rd_full[PSE_AW-1:0]]; if (e_poly < {1'b0, k_r, 1'b0}) begin // ph0: fetch only (prime). ph1..3: write one packed byte. @@ -877,8 +905,8 @@ module mlkem_top #( end end // save coeff high nibbles as they become available - if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8]; // c0[11:8] - if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4]; // c1[11:4] + if (e_ph == 2'd1) e_c0_hi <= e_rd_coeff[11:8]; // c0[11:8] + if (e_ph == 2'd2) e_c1_hi <= e_rd_coeff[11:4]; // c1[11:4] // advance micro-phase / pair / poly if (e_ph == 2'd3) begin