Fix ML-KEM arithmetic timing paths
This commit is contained in:
@@ -38,8 +38,9 @@ xvlog -sv ${NTT_DIR}/barrett_mul.v
|
||||
# ================================================================
|
||||
puts "=== Compiling PolyMul RTL sources ==="
|
||||
|
||||
# Basecase multiplier (instantiates barrett_mul)
|
||||
# Basecase multipliers
|
||||
xvlog -sv ${PM_DIR}/basecase_mul.v
|
||||
xvlog -sv ${PM_DIR}/basecase_mul_pipe.v
|
||||
|
||||
# PolyMul zeta ROM
|
||||
xvlog -sv ${PM_DIR}/poly_mul_zeta_rom.v
|
||||
|
||||
144
sync_rtl/poly_mul/basecase_mul_pipe.v
Normal file
144
sync_rtl/poly_mul/basecase_mul_pipe.v
Normal file
@@ -0,0 +1,144 @@
|
||||
// basecase_mul_pipe.v - pipelined NTT-domain degree-1 multiplication
|
||||
//
|
||||
// Fixed-latency replacement for the combinational basecase_mul hot path.
|
||||
// The shared ntt/barrett_mul remains combinational for NTT users; this module
|
||||
// keeps the extra registers local to poly_mul_sync.
|
||||
|
||||
(* use_dsp = "no" *)
|
||||
module basecase_mul_pipe (
|
||||
input clk,
|
||||
input rst_n,
|
||||
input valid_i,
|
||||
input [11:0] a0, a1,
|
||||
input [11:0] b0, b1,
|
||||
input [11:0] zeta,
|
||||
output [11:0] c0,
|
||||
output [11:0] c1,
|
||||
output valid_o
|
||||
);
|
||||
localparam [11:0] Q12 = 12'd3329;
|
||||
localparam [12:0] K13 = 13'd5039;
|
||||
localparam [24:0] Q25 = 25'd3329;
|
||||
|
||||
function [12:0] barrett_q;
|
||||
input [23:0] p;
|
||||
reg [36:0] prod;
|
||||
begin
|
||||
prod = {13'd0, p} * K13;
|
||||
barrett_q = prod[36:24];
|
||||
end
|
||||
endfunction
|
||||
|
||||
function [11:0] barrett_reduce;
|
||||
input [23:0] p;
|
||||
input [12:0] qe;
|
||||
reg [24:0] q_approx;
|
||||
reg [24:0] r0;
|
||||
reg [24:0] r1;
|
||||
reg [24:0] r2;
|
||||
begin
|
||||
q_approx = qe * Q12;
|
||||
r0 = {1'b0, p} - q_approx;
|
||||
r1 = (r0 >= Q25) ? (r0 - Q25) : r0;
|
||||
r2 = (r1 >= Q25) ? (r1 - Q25) : r1;
|
||||
barrett_reduce = r2[11:0];
|
||||
end
|
||||
endfunction
|
||||
|
||||
function [11:0] mod_add;
|
||||
input [11:0] x;
|
||||
input [11:0] y;
|
||||
reg [12:0] sum;
|
||||
begin
|
||||
sum = {1'b0, x} + {1'b0, y};
|
||||
mod_add = (sum >= {1'b0, Q12}) ? (sum[11:0] - Q12) : sum[11:0];
|
||||
end
|
||||
endfunction
|
||||
|
||||
// Stage 1: first-level scalar products.
|
||||
reg [23:0] p10, p11, p12, p13;
|
||||
reg [11:0] zeta_s1;
|
||||
|
||||
// Stage 2: Barrett quotient estimates for first-level products.
|
||||
reg [23:0] p20, p21, p22, p23;
|
||||
reg [12:0] q20, q21, q22, q23;
|
||||
reg [11:0] zeta_s2;
|
||||
|
||||
// Stage 3: first-level reduced products.
|
||||
reg [11:0] t1_s3, t2_s3, t3_s3, t4_s3;
|
||||
reg [11:0] zeta_s3;
|
||||
|
||||
// Stage 4: zeta product for c0's second Barrett multiply.
|
||||
reg [11:0] t1_s4, t3_s4, t4_s4;
|
||||
reg [23:0] pz_s4;
|
||||
|
||||
// Stage 5: Barrett quotient estimate for zeta product.
|
||||
reg [11:0] t1_s5, t3_s5, t4_s5;
|
||||
reg [23:0] pz_s5;
|
||||
reg [12:0] qz_s5;
|
||||
|
||||
// Stage 6: reduced zeta product.
|
||||
reg [11:0] t1_s6, t3_s6, t4_s6, tz_s6;
|
||||
|
||||
// Stage 7: final modular additions.
|
||||
reg [11:0] c0_r, c1_r;
|
||||
reg [6:0] valid_sr;
|
||||
|
||||
assign c0 = c0_r;
|
||||
assign c1 = c1_r;
|
||||
assign valid_o = valid_sr[6];
|
||||
|
||||
always @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
p10 <= 24'd0; p11 <= 24'd0; p12 <= 24'd0; p13 <= 24'd0; zeta_s1 <= 12'd0;
|
||||
p20 <= 24'd0; p21 <= 24'd0; p22 <= 24'd0; p23 <= 24'd0;
|
||||
q20 <= 13'd0; q21 <= 13'd0; q22 <= 13'd0; q23 <= 13'd0; zeta_s2 <= 12'd0;
|
||||
t1_s3 <= 12'd0; t2_s3 <= 12'd0; t3_s3 <= 12'd0; t4_s3 <= 12'd0; zeta_s3 <= 12'd0;
|
||||
t1_s4 <= 12'd0; t3_s4 <= 12'd0; t4_s4 <= 12'd0; pz_s4 <= 24'd0;
|
||||
t1_s5 <= 12'd0; t3_s5 <= 12'd0; t4_s5 <= 12'd0; pz_s5 <= 24'd0; qz_s5 <= 13'd0;
|
||||
t1_s6 <= 12'd0; t3_s6 <= 12'd0; t4_s6 <= 12'd0; tz_s6 <= 12'd0;
|
||||
c0_r <= 12'd0; c1_r <= 12'd0; valid_sr <= 7'd0;
|
||||
end else begin
|
||||
valid_sr <= {valid_sr[5:0], valid_i};
|
||||
|
||||
p10 <= {12'd0, a0} * b0;
|
||||
p11 <= {12'd0, a1} * b1;
|
||||
p12 <= {12'd0, a1} * b0;
|
||||
p13 <= {12'd0, a0} * b1;
|
||||
zeta_s1 <= zeta;
|
||||
|
||||
p20 <= p10; p21 <= p11; p22 <= p12; p23 <= p13;
|
||||
q20 <= barrett_q(p10);
|
||||
q21 <= barrett_q(p11);
|
||||
q22 <= barrett_q(p12);
|
||||
q23 <= barrett_q(p13);
|
||||
zeta_s2 <= zeta_s1;
|
||||
|
||||
t1_s3 <= barrett_reduce(p20, q20);
|
||||
t2_s3 <= barrett_reduce(p21, q21);
|
||||
t3_s3 <= barrett_reduce(p22, q22);
|
||||
t4_s3 <= barrett_reduce(p23, q23);
|
||||
zeta_s3 <= zeta_s2;
|
||||
|
||||
t1_s4 <= t1_s3;
|
||||
t3_s4 <= t3_s3;
|
||||
t4_s4 <= t4_s3;
|
||||
pz_s4 <= {12'd0, t2_s3} * zeta_s3;
|
||||
|
||||
t1_s5 <= t1_s4;
|
||||
t3_s5 <= t3_s4;
|
||||
t4_s5 <= t4_s4;
|
||||
pz_s5 <= pz_s4;
|
||||
qz_s5 <= barrett_q(pz_s4);
|
||||
|
||||
t1_s6 <= t1_s5;
|
||||
t3_s6 <= t3_s5;
|
||||
t4_s6 <= t4_s5;
|
||||
tz_s6 <= barrett_reduce(pz_s5, qz_s5);
|
||||
|
||||
c0_r <= mod_add(t1_s6, tz_s6);
|
||||
c1_r <= mod_add(t3_s6, t4_s6);
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -4,7 +4,7 @@
|
||||
// 256-coefficient NTT-domain polynomials.
|
||||
//
|
||||
// Operation flow:
|
||||
// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle)
|
||||
// IDLE → LOAD (256× A+B pairs) → COMP_ISSUE → COMP_WAIT
|
||||
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE
|
||||
//
|
||||
// The LOAD phase accepts both A and B coefficients simultaneously
|
||||
@@ -39,10 +39,11 @@ module poly_mul_sync (
|
||||
// State definitions
|
||||
localparam S_IDLE = 3'd0;
|
||||
localparam S_LOAD = 3'd1;
|
||||
localparam S_COMP_CALC = 3'd2;
|
||||
localparam S_COMP_C0 = 3'd3;
|
||||
localparam S_COMP_C1 = 3'd4;
|
||||
localparam S_DONE = 3'd5;
|
||||
localparam S_COMP_ISSUE = 3'd2;
|
||||
localparam S_COMP_WAIT = 3'd3;
|
||||
localparam S_COMP_C0 = 3'd4;
|
||||
localparam S_COMP_C1 = 3'd5;
|
||||
localparam S_DONE = 3'd6;
|
||||
|
||||
reg [2:0] state, next_state;
|
||||
|
||||
@@ -72,16 +73,22 @@ module poly_mul_sync (
|
||||
.zeta (zeta)
|
||||
);
|
||||
|
||||
// Basecase multiply
|
||||
// Pipelined basecase multiply. One request is issued at a time; the
|
||||
// output interface is unchanged for top-level consumers.
|
||||
wire [11:0] bc_c0, bc_c1;
|
||||
basecase_mul u_bc (
|
||||
wire bc_vo;
|
||||
basecase_mul_pipe u_bc (
|
||||
.clk (clk),
|
||||
.rst_n(rst_n),
|
||||
.valid_i(state == S_COMP_ISSUE),
|
||||
.a0 (mem_a0),
|
||||
.a1 (mem_a1),
|
||||
.b0 (mem_b0),
|
||||
.b1 (mem_b1),
|
||||
.zeta(zeta),
|
||||
.c0 (bc_c0),
|
||||
.c1 (bc_c1)
|
||||
.c1 (bc_c1),
|
||||
.valid_o(bc_vo)
|
||||
);
|
||||
|
||||
// Output interface
|
||||
@@ -95,12 +102,13 @@ module poly_mul_sync (
|
||||
case (state)
|
||||
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
|
||||
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o)
|
||||
next_state = S_COMP_CALC;
|
||||
S_COMP_CALC: next_state = S_COMP_C0;
|
||||
next_state = S_COMP_ISSUE;
|
||||
S_COMP_ISSUE: next_state = S_COMP_WAIT;
|
||||
S_COMP_WAIT: if (bc_vo) next_state = S_COMP_C0;
|
||||
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
|
||||
S_COMP_C1: if (valid_o && ready_i) begin
|
||||
if (comp_k >= 127) next_state = S_DONE;
|
||||
else next_state = S_COMP_CALC;
|
||||
else next_state = S_COMP_ISSUE;
|
||||
end
|
||||
S_DONE: next_state = S_IDLE;
|
||||
default: next_state = S_IDLE;
|
||||
@@ -139,8 +147,8 @@ module poly_mul_sync (
|
||||
end
|
||||
|
||||
// ---- COMPUTE phase ----
|
||||
// COMP_CALC: capture basecase_mul results
|
||||
if (state == S_COMP_CALC) begin
|
||||
// COMP_WAIT: capture pipelined basecase_mul results when ready.
|
||||
if (state == S_COMP_WAIT && bc_vo) begin
|
||||
c0_reg <= bc_c0;
|
||||
c1_reg <= bc_c1;
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user