fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mlkem test diagnostics and allocator updates

Browse Source
This commit is contained in:
FallenSigh
2026-07-08 02:35:56 +08:00
parent 65607badd2
commit 0c85a20b4e
40 changed files with 37168 additions and 2093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
rtl/ip/mlkem/poly_mul/basecase_mul_pipe.v Normal file
View File

@@ -0,0 +1,143 @@
// basecase_mul_pipe.v - pipelined NTT-domain degree-1 multiplication
//
// Fixed-latency replacement for the combinational basecase_mul hot path.
// The shared ntt/barrett_mul remains combinational for NTT users; this module
// keeps the extra registers local to poly_mul_sync.
module basecase_mul_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a0, a1,
input [11:0] b0, b1,
input [11:0] zeta,
output [11:0] c0,
output [11:0] c1,
output valid_o
);
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] K13 = 13'd5039;
localparam [24:0] Q25 = 25'd3329;
function [12:0] barrett_q;
input [23:0] p;
reg [36:0] prod;
begin
prod = {13'd0, p} * K13;
barrett_q = prod[36:24];
end
endfunction
function [11:0] barrett_reduce;
input [23:0] p;
input [12:0] qe;
reg [24:0] q_approx;
reg [24:0] r0;
reg [24:0] r1;
reg [24:0] r2;
begin
q_approx = qe * Q12;
r0 = {1'b0, p} - q_approx;
r1 = (r0 >= Q25) ? (r0 - Q25) : r0;
r2 = (r1 >= Q25) ? (r1 - Q25) : r1;
barrett_reduce = r2[11:0];
end
endfunction
function [11:0] mod_add;
input [11:0] x;
input [11:0] y;
reg [12:0] sum;
begin
sum = {1'b0, x} + {1'b0, y};
mod_add = (sum >= {1'b0, Q12}) ? (sum[11:0] - Q12) : sum[11:0];
end
endfunction
// Stage 1: first-level scalar products.
reg [23:0] p10, p11, p12, p13;
reg [11:0] zeta_s1;
// Stage 2: Barrett quotient estimates for first-level products.
reg [23:0] p20, p21, p22, p23;
reg [12:0] q20, q21, q22, q23;
reg [11:0] zeta_s2;
// Stage 3: first-level reduced products.
reg [11:0] t1_s3, t2_s3, t3_s3, t4_s3;
reg [11:0] zeta_s3;
// Stage 4: zeta product for c0's second Barrett multiply.
reg [11:0] t1_s4, t3_s4, t4_s4;
reg [23:0] pz_s4;
// Stage 5: Barrett quotient estimate for zeta product.
reg [11:0] t1_s5, t3_s5, t4_s5;
reg [23:0] pz_s5;
reg [12:0] qz_s5;
// Stage 6: reduced zeta product.
reg [11:0] t1_s6, t3_s6, t4_s6, tz_s6;
// Stage 7: final modular additions.
reg [11:0] c0_r, c1_r;
reg [6:0] valid_sr;
assign c0 = c0_r;
assign c1 = c1_r;
assign valid_o = valid_sr[6];
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
p10 <= 24'd0; p11 <= 24'd0; p12 <= 24'd0; p13 <= 24'd0; zeta_s1 <= 12'd0;
p20 <= 24'd0; p21 <= 24'd0; p22 <= 24'd0; p23 <= 24'd0;
q20 <= 13'd0; q21 <= 13'd0; q22 <= 13'd0; q23 <= 13'd0; zeta_s2 <= 12'd0;
t1_s3 <= 12'd0; t2_s3 <= 12'd0; t3_s3 <= 12'd0; t4_s3 <= 12'd0; zeta_s3 <= 12'd0;
t1_s4 <= 12'd0; t3_s4 <= 12'd0; t4_s4 <= 12'd0; pz_s4 <= 24'd0;
t1_s5 <= 12'd0; t3_s5 <= 12'd0; t4_s5 <= 12'd0; pz_s5 <= 24'd0; qz_s5 <= 13'd0;
t1_s6 <= 12'd0; t3_s6 <= 12'd0; t4_s6 <= 12'd0; tz_s6 <= 12'd0;
c0_r <= 12'd0; c1_r <= 12'd0; valid_sr <= 7'd0;
end else begin
valid_sr <= {valid_sr[5:0], valid_i};
p10 <= {12'd0, a0} * b0;
p11 <= {12'd0, a1} * b1;
p12 <= {12'd0, a1} * b0;
p13 <= {12'd0, a0} * b1;
zeta_s1 <= zeta;
p20 <= p10; p21 <= p11; p22 <= p12; p23 <= p13;
q20 <= barrett_q(p10);
q21 <= barrett_q(p11);
q22 <= barrett_q(p12);
q23 <= barrett_q(p13);
zeta_s2 <= zeta_s1;
t1_s3 <= barrett_reduce(p20, q20);
t2_s3 <= barrett_reduce(p21, q21);
t3_s3 <= barrett_reduce(p22, q22);
t4_s3 <= barrett_reduce(p23, q23);
zeta_s3 <= zeta_s2;
t1_s4 <= t1_s3;
t3_s4 <= t3_s3;
t4_s4 <= t4_s3;
pz_s4 <= {12'd0, t2_s3} * zeta_s3;
t1_s5 <= t1_s4;
t3_s5 <= t3_s4;
t4_s5 <= t4_s4;
pz_s5 <= pz_s4;
qz_s5 <= barrett_q(pz_s4);
t1_s6 <= t1_s5;
t3_s6 <= t3_s5;
t4_s6 <= t4_s5;
tz_s6 <= barrett_reduce(pz_s5, qz_s5);
c0_r <= mod_add(t1_s6, tz_s6);
c1_r <= mod_add(t3_s6, t4_s6);
end
end
endmodule

169
rtl/ip/mlkem/poly_mul/poly_mul_sync.v
View File

@@ -4,8 +4,10 @@
// 256-coefficient NTT-domain polynomials.
//
// Operation flow:
// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle)
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE
// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
//
// RUN issues one base-case multiply every two cycles, matching the two-cycle
// c0/c1 output bandwidth after the basecase pipeline fills.
//
// The LOAD phase accepts both A and B coefficients simultaneously
// (one pair per cycle) on coeff_a_in/coeff_b_in.
@@ -22,7 +24,7 @@
// valid_i - Input valid
// ready_o - Ready to accept input (high in IDLE/LOAD)
// coeff_out[11:0] - Result coefficient output
// valid_o - Output valid (high in COMP_C0/COMP_C1)
// valid_o - Output valid during RUN output phase
// ready_i - Output consumer ready
module poly_mul_sync (
@@ -37,12 +39,9 @@ module poly_mul_sync (
);
// State definitions
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_COMP_CALC = 3'd2;
localparam S_COMP_C0 = 3'd3;
localparam S_COMP_C1 = 3'd4;
localparam S_DONE = 3'd5;
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_RUN = 3'd2;
reg [2:0] state, next_state;
@@ -52,14 +51,23 @@ module poly_mul_sync (
// Counters
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [6:0] comp_k; // 0..127, current base-case index
reg [7:0] issue_cnt; // 0..128, issued base-cases
reg [7:0] out_cnt; // 0..255, consumed output coefficients
reg issue_gap; // spaces base-case requests to match c0/c1 output
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Registered basecase_mul results
// Registered basecase_mul inputs/results
reg [11:0] bc_a0_reg, bc_a1_reg;
reg [11:0] bc_b0_reg, bc_b1_reg;
reg [11:0] bc_zeta_reg;
reg bc_valid_reg;
reg [11:0] c0_reg, c1_reg;
// Combinational read signals for COMP_CALC
wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits)
wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1
// Combinational read signals for the next base-case request.
wire [6:0] issue_k = issue_cnt[6:0];
wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1
wire [11:0] mem_a0 = mem_A[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even];
@@ -68,41 +76,42 @@ module poly_mul_sync (
// Zeta ROM
wire [11:0] zeta;
poly_mul_zeta_rom u_zeta (
.addr (comp_k),
.addr (issue_k),
.zeta (zeta)
);
// Basecase multiply
// Pipelined basecase multiply. Requests are issued every two cycles; inputs
// are registered locally so issue_cnt does not directly drive DSP input muxes.
wire [11:0] bc_c0, bc_c1;
basecase_mul u_bc (
.a0 (mem_a0),
.a1 (mem_a1),
.b0 (mem_b0),
.b1 (mem_b1),
.zeta(zeta),
wire bc_vo;
basecase_mul_pipe u_bc (
.clk (clk),
.rst_n(rst_n),
.valid_i(bc_valid_reg),
.a0 (bc_a0_reg),
.a1 (bc_a1_reg),
.b0 (bc_b0_reg),
.b1 (bc_b1_reg),
.zeta(bc_zeta_reg),
.c0 (bc_c0),
.c1 (bc_c1)
.c1 (bc_c1),
.valid_o(bc_vo)
);
// Output interface
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1);
assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg;
assign valid_o = out_valid_r;
assign coeff_out = out_phase ? c1_reg : c0_reg;
// State transition logic (combinational)
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o)
next_state = S_COMP_CALC;
S_COMP_CALC: next_state = S_COMP_C0;
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
S_COMP_C1: if (valid_o && ready_i) begin
if (comp_k >= 127) next_state = S_DONE;
else next_state = S_COMP_CALC;
end
S_DONE: next_state = S_IDLE;
S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o)
next_state = S_RUN;
S_RUN: if (valid_o && ready_i && out_cnt == 8'd255)
next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
@@ -113,15 +122,26 @@ module poly_mul_sync (
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
comp_k <= 7'd0;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
bc_a0_reg <= 12'd0;
bc_a1_reg <= 12'd0;
bc_b0_reg <= 12'd0;
bc_b1_reg <= 12'd0;
bc_zeta_reg <= 12'd0;
bc_valid_reg <= 1'b0;
c0_reg <= 12'd0;
c1_reg <= 12'd0;
for (i = 0; i < 256; i = i + 1) begin
mem_A[i] = 12'd0;
mem_B[i] = 12'd0;
mem_A[i] <= 12'd0;
mem_B[i] <= 12'd0;
end
end else begin
state <= next_state;
bc_valid_reg <= 1'b0;
// ---- LOAD phase ----
// First coefficient captured on IDLE → LOAD transition
@@ -129,6 +149,11 @@ module poly_mul_sync (
mem_A[0] <= coeff_a_in;
mem_B[0] <= coeff_b_in;
load_cnt <= 8'd1;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
// Subsequent coefficients in LOAD state
@@ -136,29 +161,65 @@ module poly_mul_sync (
mem_A[load_cnt] <= coeff_a_in;
mem_B[load_cnt] <= coeff_b_in;
load_cnt <= load_cnt + 8'd1;
if (load_cnt >= 8'd255) begin
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end
// ---- COMPUTE phase ----
// COMP_CALC: capture basecase_mul results
if (state == S_COMP_CALC) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
// RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
// regs, then launch requests every other cycle. The two-cycle issue
// interval matches the c0/c1 output bandwidth, so no deep result FIFO
// is needed for this always-ready system.
if (state == S_RUN) begin
if (issue_cnt < 8'd128) begin
if (!issue_gap) begin
bc_a0_reg <= mem_a0;
bc_a1_reg <= mem_a1;
bc_b0_reg <= mem_b0;
bc_b1_reg <= mem_b1;
bc_zeta_reg <= zeta;
bc_valid_reg <= 1'b1;
issue_cnt <= issue_cnt + 8'd1;
issue_gap <= 1'b1;
end else begin
issue_gap <= 1'b0;
end
end
if (out_valid_r && ready_i) begin
out_cnt <= out_cnt + 8'd1;
if (out_cnt == 8'd255) begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end else if (!out_phase) begin
out_phase <= 1'b1;
end else begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end
end
if (bc_vo) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
out_valid_r <= 1'b1;
out_phase <= 1'b0;
end
end
// COMP_C0 → COMP_C1: c0 was consumed, increment comp_k
if (state == S_COMP_C0 && valid_o && ready_i) begin
// comp_k stays same, c1 still to output
end
// COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair
if (state == S_COMP_C1 && valid_o && ready_i) begin
comp_k <= comp_k + 7'd1;
end
// ---- DONE → IDLE: reset counters ----
if (state == S_DONE) begin
// ---- final output consumed: reset counters before accepting a new op ----
if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
load_cnt <= 8'd0;
comp_k <= 7'd0;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end
end