diff --git a/sync_rtl/top/mlkem_top.v b/sync_rtl/top/mlkem_top.v index f17a1ed..61463d1 100644 --- a/sync_rtl/top/mlkem_top.v +++ b/sync_rtl/top/mlkem_top.v @@ -450,28 +450,33 @@ module mlkem_top #( // bytes || rho; dk_pke = s_hat[0..K-1] bytes. Walk coeff pairs per poly. reg [4:0] e_poly; // 0..2K-1: [0,K) = t_hat -> ek; [K,2K) = s_hat -> dk_pke reg [7:0] e_pair; // 0..127 coeff-pair within poly - reg [1:0] e_byte; // 0..2 byte within the current pair (BRAM: 1 write/cycle) reg [9:0] e_rho; // 0..31 rho byte copy index (ek tail) reg e_done; // serialization complete // source poly slot: t_hat[e_poly] for ek half, s_hat[e_poly-K] for dk half wire e_is_dk = (e_poly >= {1'b0, k_r}); wire [4:0] e_pidx = e_is_dk ? (e_poly - {1'b0, k_r}) : e_poly; // index within target - wire [SAW-1:0] e_slot = e_is_dk ? (slot_s_rt + e_pidx) : (slot_t_rt + e_pidx); - // two coeffs of the current pair: ek half reads t_hat (bank_t), dk half - // reads s_hat (bank_se). Relative index within the bank = e_pidx. - wire [13:0] e_rd0_full = e_pidx*256 + {e_pair, 1'b0}; - wire [13:0] e_rd1_full = e_pidx*256 + {e_pair, 1'b1}; - wire [11:0] e_c0 = e_is_dk ? bank_se[e_rd0_full[PSE_AW-1:0]] : bank_t[e_rd0_full[PT_AW-1:0]]; - wire [11:0] e_c1 = e_is_dk ? bank_se[e_rd1_full[PSE_AW-1:0]] : bank_t[e_rd1_full[PT_AW-1:0]]; - // 3 packed bytes - wire [7:0] e_b0 = e_c0[7:0]; - wire [7:0] e_b1 = {e_c1[3:0], e_c0[11:8]}; - wire [7:0] e_b2 = e_c1[11:4]; - // byte base offset within target memory: poly index *384 (= 128 pairs *3) - wire [11:0] e_base = e_pidx * 12'd384; + // Registered single-port read (sd_bram timing). One bank read per cycle: + // ek half reads bank_t (t_hat), dk half reads bank_se (s_hat). The two + // coeffs of a pair are serialized across a 4-cycle micro-phase e_ph: + // 0 = fetch c0 + // 1 = c0 ready: write b0, save c0[11:8], fetch c1 + // 2 = c1 ready: write b1, save c1[11:4] + // 3 = write b2, advance pair + reg [1:0] e_ph; // per-pair micro-phase 0..3 + reg [3:0] e_c0_hi; // saved c0[11:8] for b1 + reg [7:0] e_c1_hi; // saved c1[11:4] for b2 + wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1 + wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; + wire [11:0] e_rd_async = e_is_dk ? bank_se[e_rd_full[PSE_AW-1:0]] + : bank_t [e_rd_full[PT_AW-1:0]]; + reg [11:0] e_rd_data_r; // registered read: value for addr presented last cyc + // byteEncode write byte offset within the target memory: pair*3 + byte index. + wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3) wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3 - // BRAM write is 1 byte/cycle: select packed byte by e_byte sub-counter - wire [7:0] e_byte_d = (e_byte == 2'd0) ? e_b0 : (e_byte == 2'd1) ? e_b1 : e_b2; + wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2 + wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_data_r[7:0] // b0 = c0[7:0] + : (e_ph == 2'd2) ? {e_rd_data_r[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]} + : e_c1_hi; // b2 = c1[11:4] wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j wire [SAW-1:0] m_sslot = slot_s_rt + m_j; // s_hat[j] @@ -579,7 +584,10 @@ module mlkem_top #( m_acc_rd <= 12'd0; e_poly <= 3'd0; e_pair <= 8'd0; - e_byte <= 2'd0; + e_ph <= 2'd0; + e_rd_data_r<= 12'd0; + e_c0_hi <= 4'd0; + e_c1_hi <= 8'd0; e_rho <= 10'd0; e_done <= 1'b0; ek_we <= 1'b0; @@ -819,27 +827,39 @@ module mlkem_top #( if (st == ST_M && st_next == ST_E) begin e_poly <= 3'd0; e_pair <= 8'd0; - e_byte <= 2'd0; + e_ph <= 2'd0; // start at fetch-c0 of first pair e_rho <= 10'd0; e_done <= 1'b0; end - // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke, 1 byte/cycle ---- + // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ---- + // Single registered bank read per cycle; 4-cycle micro-phase per + // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1, + // ph3 write b2). e_rd_data_r holds the value read last cycle. if (st == ST_E && !e_done) begin + // registered read (sd_bram timing): capture addr presented now + e_rd_data_r <= e_rd_async; + if (e_poly < {1'b0, k_r, 1'b0}) begin - // write current byte (e_byte_d) to target BRAM - if (!e_is_dk) begin - ek_we <= 1'b1; - ek_wa <= e_boff[10:0] + {9'd0, e_byte}; - ek_wd <= e_byte_d; - end else begin - dkp_we <= 1'b1; - dkp_wa <= e_boff[10:0] + {9'd0, e_byte}; - dkp_wd <= e_byte_d; + // ph0: fetch only (prime). ph1..3: write one packed byte. + if (e_ph != 2'd0) begin + if (!e_is_dk) begin + ek_we <= 1'b1; + ek_wa <= e_boff[10:0] + {9'd0, e_wb}; + ek_wd <= e_wbyte; + end else begin + dkp_we <= 1'b1; + dkp_wa <= e_boff[10:0] + {9'd0, e_wb}; + dkp_wd <= e_wbyte; + end end - // advance sub-counters - if (e_byte == 2'd2) begin - e_byte <= 2'd0; + // save coeff high nibbles as they become available + if (e_ph == 2'd1) e_c0_hi <= e_rd_data_r[11:8]; // c0[11:8] + if (e_ph == 2'd2) e_c1_hi <= e_rd_data_r[11:4]; // c1[11:4] + + // advance micro-phase / pair / poly + if (e_ph == 2'd3) begin + e_ph <= 2'd0; if (e_pair == 8'd127) begin e_pair <= 8'd0; e_poly <= e_poly + 5'd1; // next poly or -> rho phase @@ -847,7 +867,7 @@ module mlkem_top #( e_pair <= e_pair + 8'd1; end end else begin - e_byte <= e_byte + 2'd1; + e_ph <= e_ph + 2'd1; end end else begin // rho copy: ek[384*K + r] = rho byte r (r = 0..31), 1 byte/cycle