perf(mlkem): ping-pong poly_mul + decoupled MAC load/accumulate (B2)
Double-buffer poly_mul_sync so the next MAC term's 256-cycle load overlaps the current term's accumulate. Decouple the top-level MAC load index (u_ldj) from the accumulate index (u_j) in the shared ENC_U/ENC_V/DEC_W engine; load reads bank_a/bank_se while accumulate reads bank_t (disjoint ports, no conflict). Accumulate stays strictly serial (psum order-dependent); only the load is hidden. Verified byte-exact vs NIST KAT in ~/Dev/mlkem: Encaps 9/9, Decaps 9/9 (incl. implicit-reject + DECOMP), KeyGen all, hello_world end-to-end. ENC_U 5585->5073, ENC_V 2793->2537 (K=2). Larger relative win at K=4 (K^2 terms). poly_mul contract unchanged (standalone TB 11/11).
This commit is contained in:
@@ -1,30 +1,32 @@
|
||||
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier
|
||||
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (B2 ping-pong)
|
||||
//
|
||||
// Computes pointwise (Karatsuba-like base-case) multiplication of two
|
||||
// 256-coefficient NTT-domain polynomials.
|
||||
//
|
||||
// Operation flow:
|
||||
// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
|
||||
// Double-buffered (ping-pong) load so the caller can stream the NEXT operation's
|
||||
// 256 A+B pairs into the idle buffer WHILE the current operation's results are
|
||||
// still draining out. This hides the 256-cycle load behind the ~256-cycle
|
||||
// compute/output of the previous multiply: back-to-back multiplies cost ~256
|
||||
// cycles each instead of 512 (load THEN run).
|
||||
//
|
||||
// RUN issues one base-case multiply every two cycles, matching the two-cycle
|
||||
// c0/c1 output bandwidth after the basecase pipeline fills.
|
||||
// Contract (backward compatible with the single-shot testbench):
|
||||
// - ready_o is high whenever a buffer is free to accept a load. A fresh op is
|
||||
// accepted (load_active started) as soon as the first pair arrives.
|
||||
// - Each op streams 256 pairs on coeff_a_in/coeff_b_in (one per accepted cyc).
|
||||
// - When a buffer is full it is handed to the compute engine, which issues one
|
||||
// base-case every two cycles and emits 256 result coeffs on coeff_out via
|
||||
// valid_o/ready_i (c0 then c1 per pair).
|
||||
// - A caller that only ever loads-then-drains (never loads during RUN) sees
|
||||
// the exact same behaviour as before, just with an extra buffer.
|
||||
//
|
||||
// The LOAD phase accepts both A and B coefficients simultaneously
|
||||
// (one pair per cycle) on coeff_a_in/coeff_b_in.
|
||||
//
|
||||
// The COMPUTE phase outputs the 256 result coefficients one per cycle
|
||||
// via valid/ready handshake on coeff_out/valid_o/ready_i.
|
||||
//
|
||||
// Memory: 256×12-bit register arrays for A and B coefficients.
|
||||
//
|
||||
// Interface:
|
||||
// clk, rst_n - Clock, active-low reset
|
||||
// coeff_a_in[11:0]- Polynomial A coefficient input
|
||||
// coeff_b_in[11:0]- Polynomial B coefficient input
|
||||
// Interface (unchanged ports):
|
||||
// clk, rst_n - Clock, active-low reset
|
||||
// coeff_a_in[11:0]- Polynomial A coefficient (index = pairs accepted this load)
|
||||
// coeff_b_in[11:0]- Polynomial B coefficient
|
||||
// valid_i - Input valid
|
||||
// ready_o - Ready to accept input (high in IDLE/LOAD)
|
||||
// ready_o - Ready to accept a load pair (a buffer is free)
|
||||
// coeff_out[11:0] - Result coefficient output
|
||||
// valid_o - Output valid during RUN output phase
|
||||
// valid_o - Output valid
|
||||
// ready_i - Output consumer ready
|
||||
|
||||
module poly_mul_sync (
|
||||
@@ -38,150 +40,133 @@ module poly_mul_sync (
|
||||
input ready_i
|
||||
);
|
||||
|
||||
// State definitions
|
||||
localparam S_IDLE = 3'd0;
|
||||
localparam S_LOAD = 3'd1;
|
||||
localparam S_RUN = 3'd2;
|
||||
// Two coefficient buffers (ping-pong).
|
||||
reg [11:0] mem_A0 [0:255];
|
||||
reg [11:0] mem_B0 [0:255];
|
||||
reg [11:0] mem_A1 [0:255];
|
||||
reg [11:0] mem_B1 [0:255];
|
||||
|
||||
reg [2:0] state, next_state;
|
||||
// ---- Load side ----
|
||||
reg ld_active; // currently accepting a load into ld_ptr
|
||||
reg ld_ptr; // FIFO write buffer (toggles each completed load)
|
||||
reg [8:0] load_cnt; // 0..256 pairs written this load
|
||||
reg buf_full [0:1]; // buffer holds a complete (unconsumed) operand set
|
||||
|
||||
// Coefficient storage (register arrays)
|
||||
reg [11:0] mem_A [0:255];
|
||||
reg [11:0] mem_B [0:255];
|
||||
// ---- Compute side ----
|
||||
reg run_active; // compute engine busy on rn_ptr
|
||||
reg rn_ptr; // FIFO read buffer (toggles each completed run)
|
||||
reg [7:0] issue_cnt; // 0..128 base-cases issued
|
||||
reg [7:0] out_cnt; // 0..255 output coeffs consumed
|
||||
reg issue_gap; // 2-cycle issue spacing
|
||||
reg out_phase; // 0=c0, 1=c1
|
||||
reg out_valid_r;
|
||||
|
||||
// Counters
|
||||
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
|
||||
reg [7:0] issue_cnt; // 0..128, issued base-cases
|
||||
reg [7:0] out_cnt; // 0..255, consumed output coefficients
|
||||
reg issue_gap; // spaces base-case requests to match c0/c1 output
|
||||
reg out_phase; // 0=c0, 1=c1
|
||||
reg out_valid_r;
|
||||
|
||||
// Registered basecase_mul inputs/results
|
||||
// Registered basecase inputs/results
|
||||
reg [11:0] bc_a0_reg, bc_a1_reg;
|
||||
reg [11:0] bc_b0_reg, bc_b1_reg;
|
||||
reg [11:0] bc_zeta_reg;
|
||||
reg bc_valid_reg;
|
||||
reg [11:0] c0_reg, c1_reg;
|
||||
|
||||
// Combinational read signals for the next base-case request.
|
||||
// Compute-side memory read (from the buffer being consumed).
|
||||
wire [6:0] issue_k = issue_cnt[6:0];
|
||||
wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2
|
||||
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1
|
||||
wire [11:0] mem_a0 = mem_A[addr_even];
|
||||
wire [11:0] mem_a1 = mem_A[addr_odd];
|
||||
wire [11:0] mem_b0 = mem_B[addr_even];
|
||||
wire [11:0] mem_b1 = mem_B[addr_odd];
|
||||
wire [7:0] addr_even = {issue_k, 1'b0};
|
||||
wire [7:0] addr_odd = {issue_k, 1'b1};
|
||||
wire [11:0] mem_a0 = rn_ptr ? mem_A1[addr_even] : mem_A0[addr_even];
|
||||
wire [11:0] mem_a1 = rn_ptr ? mem_A1[addr_odd] : mem_A0[addr_odd];
|
||||
wire [11:0] mem_b0 = rn_ptr ? mem_B1[addr_even] : mem_B0[addr_even];
|
||||
wire [11:0] mem_b1 = rn_ptr ? mem_B1[addr_odd] : mem_B0[addr_odd];
|
||||
|
||||
// Zeta ROM
|
||||
wire [11:0] zeta;
|
||||
poly_mul_zeta_rom u_zeta (
|
||||
.addr (issue_k),
|
||||
.zeta (zeta)
|
||||
);
|
||||
poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta));
|
||||
|
||||
// Pipelined basecase multiply. Requests are issued every two cycles; inputs
|
||||
// are registered locally so issue_cnt does not directly drive DSP input muxes.
|
||||
wire [11:0] bc_c0, bc_c1;
|
||||
wire bc_vo;
|
||||
basecase_mul_pipe u_bc (
|
||||
.clk (clk),
|
||||
.rst_n(rst_n),
|
||||
.clk(clk), .rst_n(rst_n),
|
||||
.valid_i(bc_valid_reg),
|
||||
.a0 (bc_a0_reg),
|
||||
.a1 (bc_a1_reg),
|
||||
.b0 (bc_b0_reg),
|
||||
.b1 (bc_b1_reg),
|
||||
.a0(bc_a0_reg), .a1(bc_a1_reg),
|
||||
.b0(bc_b0_reg), .b1(bc_b1_reg),
|
||||
.zeta(bc_zeta_reg),
|
||||
.c0 (bc_c0),
|
||||
.c1 (bc_c1),
|
||||
.c0(bc_c0), .c1(bc_c1),
|
||||
.valid_o(bc_vo)
|
||||
);
|
||||
|
||||
// Output interface
|
||||
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
|
||||
assign valid_o = out_valid_r;
|
||||
// Accept a load pair whenever the FIFO write buffer (ld_ptr) is free. Once a
|
||||
// load is in progress it always finishes (ready stays high for its 256).
|
||||
assign ready_o = ld_active ? 1'b1 : !buf_full[ld_ptr];
|
||||
|
||||
assign valid_o = out_valid_r;
|
||||
assign coeff_out = out_phase ? c1_reg : c0_reg;
|
||||
|
||||
// State transition logic (combinational)
|
||||
always @* begin
|
||||
next_state = state;
|
||||
case (state)
|
||||
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
|
||||
S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o)
|
||||
next_state = S_RUN;
|
||||
S_RUN: if (valid_o && ready_i && out_cnt == 8'd255)
|
||||
next_state = S_IDLE;
|
||||
default: next_state = S_IDLE;
|
||||
endcase
|
||||
end
|
||||
wire ld_fire = valid_i && ready_o;
|
||||
|
||||
// Sequential logic
|
||||
integer i;
|
||||
always @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
state <= S_IDLE;
|
||||
load_cnt <= 8'd0;
|
||||
issue_cnt <= 8'd0;
|
||||
out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0;
|
||||
bc_a0_reg <= 12'd0;
|
||||
bc_a1_reg <= 12'd0;
|
||||
bc_b0_reg <= 12'd0;
|
||||
bc_b1_reg <= 12'd0;
|
||||
bc_zeta_reg <= 12'd0;
|
||||
bc_valid_reg <= 1'b0;
|
||||
c0_reg <= 12'd0;
|
||||
c1_reg <= 12'd0;
|
||||
ld_active <= 1'b0; ld_ptr <= 1'b0; load_cnt <= 9'd0;
|
||||
buf_full[0] <= 1'b0; buf_full[1] <= 1'b0;
|
||||
run_active <= 1'b0; rn_ptr <= 1'b0;
|
||||
issue_cnt <= 8'd0; out_cnt <= 8'd0; issue_gap <= 1'b0;
|
||||
out_phase <= 1'b0; out_valid_r <= 1'b0;
|
||||
bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0;
|
||||
bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0;
|
||||
bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0;
|
||||
c0_reg <= 12'd0; c1_reg <= 12'd0;
|
||||
for (i = 0; i < 256; i = i + 1) begin
|
||||
mem_A[i] = 12'd0;
|
||||
mem_B[i] = 12'd0;
|
||||
mem_A0[i] = 12'd0; mem_B0[i] = 12'd0;
|
||||
mem_A1[i] = 12'd0; mem_B1[i] = 12'd0;
|
||||
end
|
||||
end else begin
|
||||
state <= next_state;
|
||||
bc_valid_reg <= 1'b0;
|
||||
|
||||
// ---- LOAD phase ----
|
||||
// First coefficient captured on IDLE → LOAD transition
|
||||
if (state == S_IDLE && valid_i && ready_o) begin
|
||||
mem_A[0] <= coeff_a_in;
|
||||
mem_B[0] <= coeff_b_in;
|
||||
load_cnt <= 8'd1;
|
||||
issue_cnt <= 8'd0;
|
||||
out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0;
|
||||
end
|
||||
|
||||
// Subsequent coefficients in LOAD state
|
||||
if (state == S_LOAD && valid_i && ready_o) begin
|
||||
mem_A[load_cnt] <= coeff_a_in;
|
||||
mem_B[load_cnt] <= coeff_b_in;
|
||||
load_cnt <= load_cnt + 8'd1;
|
||||
if (load_cnt >= 8'd255) begin
|
||||
issue_cnt <= 8'd0;
|
||||
out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0;
|
||||
// ================= LOAD SIDE (FIFO write into ld_ptr) =================
|
||||
if (!ld_active) begin
|
||||
if (ld_fire) begin
|
||||
if (!ld_ptr) begin
|
||||
mem_A0[0] <= coeff_a_in; mem_B0[0] <= coeff_b_in;
|
||||
end else begin
|
||||
mem_A1[0] <= coeff_a_in; mem_B1[0] <= coeff_b_in;
|
||||
end
|
||||
load_cnt <= 9'd1;
|
||||
ld_active <= 1'b1;
|
||||
end
|
||||
end else begin
|
||||
if (ld_fire) begin
|
||||
if (!ld_ptr) begin
|
||||
mem_A0[load_cnt[7:0]] <= coeff_a_in;
|
||||
mem_B0[load_cnt[7:0]] <= coeff_b_in;
|
||||
end else begin
|
||||
mem_A1[load_cnt[7:0]] <= coeff_a_in;
|
||||
mem_B1[load_cnt[7:0]] <= coeff_b_in;
|
||||
end
|
||||
if (load_cnt == 9'd255) begin
|
||||
buf_full[ld_ptr] <= 1'b1; // buffer complete
|
||||
ld_ptr <= ~ld_ptr; // FIFO advance
|
||||
ld_active <= 1'b0;
|
||||
load_cnt <= 9'd0;
|
||||
end else begin
|
||||
load_cnt <= load_cnt + 9'd1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ---- COMPUTE phase ----
|
||||
// RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
|
||||
// regs, then launch requests every other cycle. The two-cycle issue
|
||||
// interval matches the c0/c1 output bandwidth, so no deep result FIFO
|
||||
// is needed for this always-ready system.
|
||||
if (state == S_RUN) begin
|
||||
// ================= COMPUTE SIDE (FIFO read from rn_ptr) =============
|
||||
if (!run_active) begin
|
||||
// Start compute on the FIFO head buffer once it is full (and not
|
||||
// still mid-load, which cannot happen for rn_ptr==ld_ptr while
|
||||
// ld_active, since buf_full is set only at load completion).
|
||||
if (buf_full[rn_ptr]) begin
|
||||
run_active <= 1'b1;
|
||||
issue_cnt <= 8'd0; out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0;
|
||||
end
|
||||
end else begin
|
||||
// Issue one base-case every two cycles.
|
||||
if (issue_cnt < 8'd128) begin
|
||||
if (!issue_gap) begin
|
||||
bc_a0_reg <= mem_a0;
|
||||
bc_a1_reg <= mem_a1;
|
||||
bc_b0_reg <= mem_b0;
|
||||
bc_b1_reg <= mem_b1;
|
||||
bc_a0_reg <= mem_a0; bc_a1_reg <= mem_a1;
|
||||
bc_b0_reg <= mem_b0; bc_b1_reg <= mem_b1;
|
||||
bc_zeta_reg <= zeta;
|
||||
bc_valid_reg <= 1'b1;
|
||||
issue_cnt <= issue_cnt + 8'd1;
|
||||
@@ -194,32 +179,27 @@ module poly_mul_sync (
|
||||
if (out_valid_r && ready_i) begin
|
||||
out_cnt <= out_cnt + 8'd1;
|
||||
if (out_cnt == 8'd255) begin
|
||||
out_valid_r <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0; out_phase <= 1'b0;
|
||||
end else if (!out_phase) begin
|
||||
out_phase <= 1'b1;
|
||||
end else begin
|
||||
out_valid_r <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0; out_phase <= 1'b0;
|
||||
end
|
||||
end
|
||||
|
||||
if (bc_vo) begin
|
||||
c0_reg <= bc_c0;
|
||||
c1_reg <= bc_c1;
|
||||
out_valid_r <= 1'b1;
|
||||
out_phase <= 1'b0;
|
||||
c0_reg <= bc_c0; c1_reg <= bc_c1;
|
||||
out_valid_r <= 1'b1; out_phase <= 1'b0;
|
||||
end
|
||||
end
|
||||
|
||||
// ---- final output consumed: reset counters before accepting a new op ----
|
||||
if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
|
||||
load_cnt <= 8'd0;
|
||||
issue_cnt <= 8'd0;
|
||||
out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0;
|
||||
out_phase <= 1'b0;
|
||||
out_valid_r <= 1'b0;
|
||||
// Final output consumed: free this buffer, advance FIFO read ptr.
|
||||
if (out_valid_r && ready_i && out_cnt == 8'd255) begin
|
||||
buf_full[rn_ptr] <= 1'b0;
|
||||
rn_ptr <= ~rn_ptr;
|
||||
run_active <= 1'b0;
|
||||
issue_cnt <= 8'd0; out_cnt <= 8'd0;
|
||||
issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0;
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user