fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pipeline ML-KEM datapath bottlenecks

Browse Source
This commit is contained in:
FallenSigh
2026-07-08 00:23:46 +08:00
parent 8c3f4317f5
commit 372a90e601
17 changed files with 776 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
create_project.tcl
View File

@@ -34,8 +34,10 @@ read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_cbd/sample_cbd_sync_shared.v
# ── NTT ──
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul_pipe.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/zeta_rom.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit_pipe.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/ntt_core.v
# ── 多项式乘法 ──

30
run_tb.sh
View File

@@ -12,8 +12,8 @@
# ./run_tb.sh enc 4 1 # Encaps K=4, only CASE=1
# ./run_tb.sh dec # ML-KEM Decaps: all K, all cases (0..2)
# ./run_tb.sh dec 2 0 # Decaps K=2, only CASE=0
# ./run_tb.sh hello # hello_world end-to-end (single instance)
# ./run_tb.sh hello two # hello_world end-to-end (two instances)
# ./run_tb.sh hello [K] # hello_world end-to-end (single instance; K=2/3/4)
# ./run_tb.sh hello two [K] # hello_world end-to-end (two instances; K=2/3/4)
# ./run_tb.sh --list
#
# 'top' (KeyGen), 'enc' (Encaps) and 'dec' (Decaps) share the same RTL datapath;
@@ -41,7 +41,7 @@ if [ "$1" = "--list" ]; then
done
echo " enc (ML-KEM Encaps; shares the 'top' RTL/tcl, enc testbench)"
echo " dec (ML-KEM Decaps; shares the 'top' RTL/tcl, dec testbench)"
echo " hello (ML-KEM hello_world end-to-end; 'hello two' for two instances)"
echo " hello (ML-KEM hello_world end-to-end; optional K=2/3/4, 'hello two [K]' for two instances)"
exit 0
fi
@@ -286,12 +286,24 @@ fi
# ML-KEM hello_world end-to-end. Reuses the 'top' RTL compile list, then runs a
# self-checking protocol TB (no KAT vectors): single instance by default, or two
# instances (genenc + dec) with 'hello two'. Prints each step's inputs/outputs.
# instances (genenc + dec) with 'hello two'. Optional K selects ML-KEM-512/768/1024.
run_hello_selected() {
local tcl_file="$1" variant="$2"
local tcl_file="$1" arg1="$2" arg2="$3"
set +e
rm -rf xsim.dir .Xil
local variant="" ksel="2"
if [ "$arg1" = "two" ]; then
variant="two"
[ -n "$arg2" ] && ksel="$arg2"
elif [ -n "$arg1" ]; then
ksel="$arg1"
fi
case "$ksel" in
2|3|4) ;;
*) echo "ERROR: hello K must be 2, 3, or 4 (got '$ksel')"; return 1 ;;
esac
local tb snap
if [ "$variant" = "two" ]; then
tb=tb_mlkem_two_inst_xsim; snap=mlkem_two
@@ -308,14 +320,14 @@ run_hello_selected() {
echo " xvlog -sv --relax sync_rtl/top/TB/$tb.v"
xvlog -sv --relax "sync_rtl/top/TB/$tb.v" || { echo "HELLO TB COMPILE FAILED"; return 1; }
echo " xelab $tb -s $snap --timescale 1ns/1ps"
xelab "$tb" -s "$snap" --timescale 1ns/1ps || { echo "ELAB FAILED"; return 1; }
echo " xelab $tb -generic_top KP=$ksel -s ${snap}_k${ksel} --timescale 1ns/1ps"
xelab "$tb" -generic_top "KP=$ksel" -s "${snap}_k${ksel}" --timescale 1ns/1ps || { echo "ELAB FAILED"; return 1; }
xsim "$snap" -R
xsim "${snap}_k${ksel}" -R
}
if [ "$MODULE" = "hello" ]; then
run_hello_selected "$TCL_FILE" "$SEL_K" # SEL_K position carries the variant ('two')
run_hello_selected "$TCL_FILE" "$SEL_K" "$SEL_CASE" # args are [K] or [two] [K]
exit $?
fi

202
sync_rtl/comp_decomp/comp_decomp_sync.v
View File

@@ -1,7 +1,7 @@
// comp_decomp_sync.v - ML-KEM coefficient compression/decompression
//
// Streaming valid/ready interface. Compression is intentionally iterative to
// avoid inferring a long combinational divider for /Q.
// Streaming valid/ready interface. The arithmetic is fixed-latency pipelined
// so compression does not infer a long combinational divider/reducer.
// mode=0: compress — round((2^d * x) / Q) mod 2^d
// mode=1: decompress — round((Q * x) / 2^d)
// Uses round-half-up (round(2.5)=3, round(3.5)=4).
@@ -11,7 +11,7 @@
`include "sync_rtl/common/defines.vh"
(* use_dsp = "no" *)
(* use_dsp = "yes" *)
module comp_decomp_sync (
input clk,
input rst_n,
@@ -25,79 +25,169 @@ module comp_decomp_sync (
input ready_i
);
localparam [11:0] Q_VAL = 12'(`Q);
localparam [4:0] COMP_MSB = 5'd22; // max ((Q-1)<<11)+Q/2 is 23 bits
localparam [24:0] Q25 = 25'd`Q;
localparam [12:0] K13 = 13'd5039; // floor(2^24 / 3329)
reg busy_r;
reg [4:0] bit_idx_r;
reg [4:0] d_r;
reg [23:0] dividend_r;
reg [23:0] quotient_r;
reg [12:0] rem_r;
reg [11:0] data_r;
reg valid_r;
reg mode_s0;
reg [4:0] d_s0;
reg [11:0] coeff_s0;
reg valid_s0;
reg mode_s1;
reg [4:0] d_s1;
reg [23:0] comp_dividend_s1;
reg [23:0] decomp_rounded_s1;
reg valid_s1;
reg mode_s2;
reg [4:0] d_s2;
reg [23:0] comp_dividend_s2;
reg [36:0] comp_qprod_est_s2;
reg [23:0] decomp_rounded_s2;
reg valid_s2;
reg mode_s3;
reg [4:0] d_s3;
reg [23:0] comp_dividend_s3;
reg [12:0] comp_q_est_s3;
reg [11:0] decomp_result_s3;
reg valid_s3;
reg mode_s4;
reg [4:0] d_s4;
reg [23:0] comp_dividend_s4;
reg [12:0] comp_q_est_s4;
reg [24:0] comp_q_mul_q_s4;
reg [11:0] decomp_result_s4;
reg valid_s4;
reg mode_s5;
reg [4:0] d_s5;
reg [12:0] comp_q_est_s5;
reg [24:0] comp_rem_s5;
reg [11:0] decomp_result_s5;
reg valid_s5;
wire fire_i = valid_i && ready_o;
// Accept a new command only when the iterative divider and output slot are
// both free. Current users tie ready_i high and issue one coefficient at a
// time, but this keeps the module from accepting work it cannot retain.
assign ready_o = !busy_r && (!valid_r || ready_i);
wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5;
// Accept a new command only when the pipeline and output slot are both free.
// Current users issue one coefficient at a time and wait for valid_o.
assign ready_o = !pipe_busy && (!valid_r || ready_i);
assign coeff_out = data_r;
assign valid_o = valid_r;
wire [23:0] comp_dividend = ({12'd0, coeff_in} << d) + 24'd1664;
wire [23:0] decomp_product =
{12'd0, coeff_in} +
({12'd0, coeff_in} << 8) +
({12'd0, coeff_in} << 10) +
({12'd0, coeff_in} << 11);
wire [23:0] decomp_rounded = decomp_product + (24'd1 << (d - 1'b1));
wire [11:0] decomp_result = decomp_rounded >> d;
wire [23:0] coeff_ext_s0 = {12'd0, coeff_s0};
wire [23:0] comp_dividend_next = (coeff_ext_s0 << d_s0) + 24'd1664;
wire [23:0] decomp_product_next = coeff_s0 * 12'd`Q;
wire [23:0] decomp_round_next =
decomp_product_next + ((d_s0 == 5'd0) ? 24'd0 : (24'd1 << (d_s0 - 5'd1)));
wire [12:0] rem_shift = {rem_r[11:0], dividend_r[bit_idx_r]};
wire rem_ge_q = rem_shift >= {1'b0, Q_VAL};
wire [12:0] rem_sub_q = rem_shift - {1'b0, Q_VAL};
wire [12:0] rem_next = rem_ge_q ? rem_sub_q : rem_shift;
wire [23:0] quot_next = quotient_r | (rem_ge_q ? (24'd1 << bit_idx_r) : 24'd0);
wire [11:0] comp_mask = (12'd1 << d_r) - 12'd1;
wire [12:0] comp_q_corr1_s5 =
comp_q_est_s5 + ((comp_rem_s5 >= Q25) ? 13'd1 : 13'd0);
wire [24:0] comp_rem_corr1_s5 =
(comp_rem_s5 >= Q25) ? (comp_rem_s5 - Q25) : comp_rem_s5;
wire [12:0] comp_q_corr2_s5 =
comp_q_corr1_s5 + ((comp_rem_corr1_s5 >= Q25) ? 13'd1 : 13'd0);
wire [11:0] comp_mask_s5 =
(d_s5 == 5'd0) ? 12'd0 : (12'd1 << d_s5) - 12'd1;
wire [11:0] comp_result_s5 = comp_q_corr2_s5[11:0] & comp_mask_s5;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
busy_r <= 1'b0;
bit_idx_r <= 5'd0;
d_r <= 5'd0;
dividend_r <= 24'd0;
quotient_r <= 24'd0;
rem_r <= 13'd0;
data_r <= 12'd0;
valid_r <= 1'b0;
mode_s0 <= 1'b0;
d_s0 <= 5'd0;
coeff_s0 <= 12'd0;
valid_s0 <= 1'b0;
mode_s1 <= 1'b0;
d_s1 <= 5'd0;
comp_dividend_s1 <= 24'd0;
decomp_rounded_s1 <= 24'd0;
valid_s1 <= 1'b0;
mode_s2 <= 1'b0;
d_s2 <= 5'd0;
comp_dividend_s2 <= 24'd0;
comp_qprod_est_s2 <= 37'd0;
decomp_rounded_s2 <= 24'd0;
valid_s2 <= 1'b0;
mode_s3 <= 1'b0;
d_s3 <= 5'd0;
comp_dividend_s3 <= 24'd0;
comp_q_est_s3 <= 13'd0;
decomp_result_s3 <= 12'd0;
valid_s3 <= 1'b0;
mode_s4 <= 1'b0;
d_s4 <= 5'd0;
comp_dividend_s4 <= 24'd0;
comp_q_est_s4 <= 13'd0;
comp_q_mul_q_s4 <= 25'd0;
decomp_result_s4 <= 12'd0;
valid_s4 <= 1'b0;
mode_s5 <= 1'b0;
d_s5 <= 5'd0;
comp_q_est_s5 <= 13'd0;
comp_rem_s5 <= 25'd0;
decomp_result_s5 <= 12'd0;
valid_s5 <= 1'b0;
end else begin
if (valid_r && ready_i)
valid_r <= 1'b0;
if (busy_r) begin
rem_r <= rem_next;
quotient_r <= quot_next;
if (bit_idx_r == 5'd0) begin
busy_r <= 1'b0;
data_r <= quot_next[11:0] & comp_mask;
mode_s0 <= mode;
d_s0 <= d;
coeff_s0 <= coeff_in;
valid_s0 <= fire_i;
mode_s1 <= mode_s0;
d_s1 <= d_s0;
comp_dividend_s1 <= comp_dividend_next;
decomp_rounded_s1 <= decomp_round_next;
valid_s1 <= valid_s0;
mode_s2 <= mode_s1;
d_s2 <= d_s1;
comp_dividend_s2 <= comp_dividend_s1;
comp_qprod_est_s2 <= {13'd0, comp_dividend_s1} * K13;
decomp_rounded_s2 <= decomp_rounded_s1;
valid_s2 <= valid_s1;
mode_s3 <= mode_s2;
d_s3 <= d_s2;
comp_dividend_s3 <= comp_dividend_s2;
comp_q_est_s3 <= comp_qprod_est_s2[36:24];
decomp_result_s3 <= decomp_rounded_s2 >> d_s2;
valid_s3 <= valid_s2;
mode_s4 <= mode_s3;
d_s4 <= d_s3;
comp_dividend_s4 <= comp_dividend_s3;
comp_q_est_s4 <= comp_q_est_s3;
comp_q_mul_q_s4 <= comp_q_est_s3 * Q25;
decomp_result_s4 <= decomp_result_s3;
valid_s4 <= valid_s3;
mode_s5 <= mode_s4;
d_s5 <= d_s4;
comp_q_est_s5 <= comp_q_est_s4;
comp_rem_s5 <= {1'b0, comp_dividend_s4} - comp_q_mul_q_s4;
decomp_result_s5 <= decomp_result_s4;
valid_s5 <= valid_s4;
if (valid_s5) begin
data_r <= mode_s5 ? decomp_result_s5 : comp_result_s5;
valid_r <= 1'b1;
end else begin
bit_idx_r <= bit_idx_r - 5'd1;
end
end else if (fire_i) begin
if (mode) begin
data_r <= decomp_result;
valid_r <= 1'b1;
end else begin
busy_r <= 1'b1;
bit_idx_r <= COMP_MSB;
d_r <= d;
dividend_r <= comp_dividend;
quotient_r <= 24'd0;
rem_r <= 13'd0;
end
end
end
end

2
sync_rtl/ntt/TB/xsim_run.tcl
View File

@@ -30,12 +30,14 @@ puts "=== Compiling RTL sources ==="
# Barrett modular multiplier (combinational)
xvlog -sv ${SRC_DIR}/barrett_mul.v
xvlog -sv ${SRC_DIR}/barrett_mul_pipe.v
# Zeta ROM (combinational)
xvlog -sv ${SRC_DIR}/zeta_rom.v
# Butterfly unit (combinational, instantiates barrett_mul)
xvlog -sv ${SRC_DIR}/butterfly_unit.v
xvlog -sv ${SRC_DIR}/butterfly_unit_pipe.v
# NTT core (FSM-based, instantiates butterfly_unit + zeta_rom + barrett_mul)
xvlog -sv ${SRC_DIR}/ntt_core.v

72
sync_rtl/ntt/barrett_mul_pipe.v Normal file
View File

@@ -0,0 +1,72 @@
// barrett_mul_pipe.v - DSP-friendly pipelined Barrett modular multiply
//
// Computes product = a * b mod Q with fixed latency. This module is used on
// timing-critical NTT paths; the legacy barrett_mul remains combinational for
// older users.
module barrett_mul_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a,
input [11:0] b,
output [11:0] product,
output valid_o
);
localparam [24:0] Q25 = 25'd3329;
localparam [12:0] K13 = 13'd5039;
reg [23:0] p_s1;
reg [23:0] p_s2;
reg [36:0] tk_s2;
reg [23:0] p_s3;
reg [12:0] q_s3;
reg [24:0] p_s4;
reg [24:0] qprod_s4;
reg [24:0] r0_s5;
reg [24:0] r1_s6;
reg [11:0] product_r;
reg [6:0] valid_sr;
assign product = product_r;
assign valid_o = valid_sr[6];
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
p_s1 <= 24'd0;
p_s2 <= 24'd0; tk_s2 <= 37'd0;
p_s3 <= 24'd0; q_s3 <= 13'd0;
p_s4 <= 25'd0; qprod_s4 <= 25'd0;
r0_s5 <= 25'd0; r1_s6 <= 25'd0;
product_r <= 12'd0;
valid_sr <= 7'd0;
end else begin
valid_sr <= {valid_sr[5:0], valid_i};
// S1: full product.
p_s1 <= {12'd0, a} * b;
// S2: quotient-estimate product.
p_s2 <= p_s1;
tk_s2 <= {13'd0, p_s1} * K13;
// S3: quotient estimate.
p_s3 <= p_s2;
q_s3 <= tk_s2[36:24];
// S4: q * Q.
p_s4 <= {1'b0, p_s3};
qprod_s4 <= q_s3 * Q25;
// S5/S6/S7: subtract q*Q and conditionally subtract Q twice.
r0_s5 <= p_s4 - qprod_s4;
r1_s6 <= (r0_s5 >= Q25) ? (r0_s5 - Q25) : r0_s5;
product_r <= (r1_s6 >= Q25) ? (r1_s6 - Q25) : r1_s6[11:0];
end
end
endmodule

74
sync_rtl/ntt/butterfly_unit_pipe.v Normal file
View File

@@ -0,0 +1,74 @@
// butterfly_unit_pipe.v - pipelined Cooley-Tukey / Gentleman-Sande butterfly
//
// Fixed-latency wrapper around barrett_mul_pipe. All arithmetic that used to
// sit in one NTT cycle is now split by registers.
module butterfly_unit_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a,
input [11:0] b,
input [11:0] zeta,
input mode, // 0 = forward NTT, 1 = inverse NTT
output [11:0] a_out,
output [11:0] b_out,
output valid_o
);
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] Q13 = 13'd3329;
wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12);
wire [11:0] mul_data = mode ? diff : b;
wire [11:0] mul_result;
wire mul_valid;
barrett_mul_pipe u_barrett (
.clk(clk),
.rst_n(rst_n),
.valid_i(valid_i),
.a(zeta),
.b(mul_data),
.product(mul_result),
.valid_o(mul_valid)
);
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5;
reg [11:0] a_out_r, b_out_r;
reg valid_r;
wire [12:0] a_sum = {1'b0, a_d5} + {1'b0, (mode_d5 ? b_d5 : mul_result)};
wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0];
wire [11:0] b_sub = (a_d5 >= mul_result) ? (a_d5 - mul_result) :
(a_d5 - mul_result + Q12);
assign a_out = a_out_r;
assign b_out = b_out_r;
assign valid_o = valid_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0;
a_out_r <= 12'd0;
b_out_r <= 12'd0;
valid_r <= 1'b0;
end else begin
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4;
mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4;
valid_r <= mul_valid;
if (mul_valid) begin
a_out_r <= a_mod;
b_out_r <= mode_d5 ? mul_result : b_sub;
end
end
end
endmodule

216
sync_rtl/ntt/ntt_core.v
View File

@@ -1,8 +1,10 @@
// ntt_core.v - NTT core with individual coefficient registers
//
// Uses 256 individual 12-bit registers and generate-based muxing
// to avoid any part-select simulation issues.
// 3-cycle butterfly: SetAddr -> Read -> Compute+Write
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
// The arithmetic hot path is split into:
// address -> operand/zeta register -> pipelined Barrett butterfly -> writeback
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
module ntt_core (
input clk, rst_n,
@@ -17,54 +19,91 @@ module ntt_core (
);
localparam N = 256, LAYERS = 7, DW = 12;
// Individual coefficient registers
reg [DW-1:0] cr [0:N-1];
integer ci;
// State machine
localparam S_IDLE=3'd0, S_LOAD=3'd1, S_CMP_A=3'd2, S_CMP_B=3'd3,
S_CMP_C=3'd4, S_OUTPUT=3'd5, S_DONE=3'd6;
reg [2:0] state, next_state;
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_A = 4'd2;
localparam S_CMP_B = 4'd3;
localparam S_CMP_ISSUE = 4'd4;
localparam S_CMP_WAIT = 4'd5;
localparam S_CMP_WB = 4'd6;
localparam S_OUT_PREP = 4'd7;
localparam S_OUTPUT = 4'd8;
localparam S_OUT_SCALE = 4'd9;
localparam S_DONE = 4'd10;
reg [7:0] load_cnt, out_cnt;
reg [3:0] state, next_state;
reg [7:0] load_cnt;
reg [7:0] out_cnt;
reg [8:0] scale_issue_cnt;
reg [8:0] scale_emit_cnt;
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
reg mode_r;
// Pipeline registers
reg [DW-1:0] r_a, r_b;
reg [DW-1:0] r_a, r_b, r_zeta;
reg [7:0] r_wa, r_wb;
reg [DW-1:0] wr_a_data, wr_b_data;
reg [7:0] wr_wa, wr_wb;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
reg scale_valid_i;
reg [DW-1:0] scale_a_i;
// Zeta
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
// Butterfly
wire [DW-1:0] bf_a_out, bf_b_out;
butterfly_unit u_bf (
.a(r_a), .b(r_b), .zeta(zeta), .mode(mode),
.a_out(bf_a_out), .b_out(bf_b_out));
// Output scaling
wire [DW-1:0] coeff_scaled;
barrett_mul u_scl (.a(cr[out_cnt]), .b(12'd3303), .product(coeff_scaled));
assign coeff_out = mode ? coeff_scaled : cr[out_cnt];
wire bf_valid;
butterfly_unit_pipe u_bf (
.clk(clk),
.rst_n(rst_n),
.valid_i(state == S_CMP_ISSUE),
.a(r_a),
.b(r_b),
.zeta(r_zeta),
.mode(mode_r),
.a_out(bf_a_out),
.b_out(bf_b_out),
.valid_o(bf_valid)
);
wire [DW-1:0] scale_product;
wire scale_valid_o;
barrett_mul_pipe u_scl (
.clk(clk),
.rst_n(rst_n),
.valid_i(scale_valid_i),
.a(scale_a_i),
.b(12'd3303),
.product(scale_product),
.valid_o(scale_valid_o)
);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_OUTPUT);
assign coeff_out = coeff_out_r;
assign valid_o = valid_o_r;
assign done_o = (state == S_DONE);
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i) next_state = S_CMP_A;
S_CMP_A: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_B;
S_CMP_B: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_C;
S_CMP_C: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_A;
S_OUTPUT:if (out_cnt >= 255 && ready_i) next_state = S_DONE;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_A;
S_CMP_A: next_state = bf_done ? S_OUT_PREP : S_CMP_B;
S_CMP_B: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE;
S_CMP_ISSUE: next_state = S_CMP_WAIT;
S_CMP_WAIT: if (bf_valid) next_state = S_CMP_WB;
S_CMP_WB: next_state = S_CMP_A;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUT_SCALE: if (scale_valid_o && scale_emit_cnt >= 9'd255) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
@@ -72,17 +111,57 @@ module ntt_core (
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state<=S_IDLE; load_cnt<=0; out_cnt<=0; j<=0; start<=0; layer_len<=0;
zeta_idx<=0; layer<=0; bf_done<=0; r_a<=0; r_b<=0; r_wa<=0; r_wb<=0;
for (ci=0; ci<N; ci=ci+1) cr[ci] <= 0;
state <= S_IDLE;
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer_len <= 8'd0;
zeta_idx <= 7'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
r_a <= 12'd0;
r_b <= 12'd0;
r_zeta <= 12'd0;
r_wa <= 8'd0;
r_wb <= 8'd0;
wr_a_data <= 12'd0;
wr_b_data <= 12'd0;
wr_wa <= 8'd0;
wr_wb <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
scale_a_i <= 12'd0;
for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0;
end else begin
state <= next_state;
scale_valid_i <= 1'b0;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt<=1; out_cnt<=0; j<=0; start<=0; layer<=0; bf_done<=0;
if (!mode) begin layer_len<=128; zeta_idx<=1; end
else begin layer_len<=2; zeta_idx<=127; end
load_cnt <= 8'd1;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
zeta_idx <= 7'd1;
end else begin
layer_len <= 8'd2;
zeta_idx <= 7'd127;
end
end
if (state == S_LOAD && valid_i) begin
@@ -90,32 +169,38 @@ module ntt_core (
load_cnt <= load_cnt + 8'd1;
end
// S_CMP_A: set read addresses (j, j+len)
if (state == S_CMP_A) begin
r_wa <= j;
r_wb <= j + layer_len;
end
// S_CMP_B: capture read data
if (state == S_CMP_B) begin
r_a <= cr[j];
r_b <= cr[j + layer_len];
r_zeta <= zeta;
end
// S_CMP_C: write butterfly results, advance counters
if (state == S_CMP_C) begin
cr[r_wa] <= bf_a_out;
cr[r_wb] <= bf_b_out;
if (state == S_CMP_WAIT && bf_valid) begin
wr_a_data <= bf_a_out;
wr_b_data <= bf_b_out;
wr_wa <= r_wa;
wr_wb <= r_wb;
end
if (state == S_CMP_WB) begin
cr[wr_wa] <= wr_a_data;
cr[wr_wb] <= wr_b_data;
j <= j + 8'd1;
if (j + 8'd1 >= start + layer_len) begin
if (!mode) zeta_idx <= zeta_idx + 7'd1;
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 256) begin
if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 9'd256) begin
layer <= layer + 3'd1;
layer_len <= mode ? (layer_len<<1) : (layer_len>>1);
start <= 0; j <= 0;
layer_len <= mode_r ? (layer_len << 1) : (layer_len >> 1);
start <= 8'd0;
j <= 8'd0;
if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1;
end else begin
start <= start + layer_len + layer_len;
@@ -124,8 +209,49 @@ module ntt_core (
end
end
if (state == S_OUTPUT && ready_i)
out_cnt <= (out_cnt>=255) ? 0 : (out_cnt+8'd1);
if (state == S_OUT_PREP) begin
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
if (!mode_r) begin
coeff_out_r <= cr[0];
valid_o_r <= 1'b1;
end
end
if (state == S_OUTPUT && valid_o_r && ready_i) begin
if (out_cnt < 8'd255) begin
out_cnt <= out_cnt + 8'd1;
coeff_out_r <= cr[out_cnt + 8'd1];
valid_o_r <= 1'b1;
end else begin
out_cnt <= 8'd0;
valid_o_r <= 1'b0;
end
end
if (state == S_OUT_SCALE) begin
if (scale_issue_cnt < 9'd256) begin
scale_valid_i <= 1'b1;
scale_a_i <= cr[scale_issue_cnt[7:0]];
scale_issue_cnt <= scale_issue_cnt + 9'd1;
end
valid_o_r <= 1'b0;
if (scale_valid_o) begin
coeff_out_r <= scale_product;
valid_o_r <= 1'b1;
scale_emit_cnt <= scale_emit_cnt + 9'd1;
end
end
if (state == S_DONE) begin
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
valid_o_r <= 1'b0;
end
end
end

2
sync_rtl/poly_mul/basecase_mul_pipe.v
View File

@@ -4,7 +4,7 @@
// The shared ntt/barrett_mul remains combinational for NTT users; this module
// keeps the extra registers local to poly_mul_sync.
(* use_dsp = "no" *)
(* use_dsp = "yes" *)
module basecase_mul_pipe (
input clk,
input rst_n,

20
sync_rtl/sha3/sha3_top.v
View File

@@ -100,6 +100,16 @@ module sha3_top (
reg mb_last_r; // captured last-block flag
reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE)
// ================================================================
// Keccak core
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire kc_valid_o;
// XOR the incoming block into the low 1088 bits (rate) of the state.
wire [1599:0] mb_xored;
assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i};
@@ -119,16 +129,6 @@ module sha3_top (
endcase
end
// ================================================================
// Keccak core
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire kc_valid_o;
// Keccak input: multi-block xored state when mb_en, else single-block absorb.
wire [1599:0] kc_state_i_mux;
assign kc_state_i_mux = mb_en ? mb_xored : absorb_state;

73
sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v
View File

@@ -13,6 +13,8 @@ module tb_mlkem_dec_katK_xsim;
localparam EKB = 384*KP + 32; // ek_pke bytes within dk
localparam DKPB = 384*KP; // dk_pke bytes
localparam CTB = (KP==4) ? 1568 : (32*(10*KP+4)); // ct bytes: 768/1088/1568
localparam integer CLK_PERIOD_NS = 100; // 10 MHz
localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2;
reg clk=0, rst_n=0, start_i=0;
reg [2:0] k_i;
@@ -47,7 +49,7 @@ module tb_mlkem_dec_katK_xsim;
.dbg_mprime_o(dbg_mprime_o), .dbg_kbar_o(dbg_kbar_o),
.dbg_decz_o(dbg_decz_o), .dbg_dech_o(dbg_dech_o)
);
always #5 clk = ~clk;
always #(CLK_HALF_NS) clk = ~clk;
reg [7:0] dk_b [0:DKB-1];
reg [7:0] ct_b [0:CTB-1];
@@ -55,8 +57,72 @@ module tb_mlkem_dec_katK_xsim;
reg [7:0] ss_b [0:31];
reg [7:0] ssn_b [0:31];
integer c, i, j, errors, casenum;
integer state_cyc [0:31];
integer si;
reg [8*80-1:0] tag, dkfile, ctfile, ssfile, ctnfile, ssnfile;
function [8*16-1:0] state_name;
input [4:0] s;
begin
case (s)
5'd0: state_name = "IDLE";
5'd1: state_name = "G";
5'd2: state_name = "A";
5'd3: state_name = "C";
5'd4: state_name = "N";
5'd5: state_name = "M";
5'd6: state_name = "E";
5'd7: state_name = "H";
5'd8: state_name = "ENC_LOAD";
5'd9: state_name = "ENC_H";
5'd10: state_name = "ENC_G";
5'd11: state_name = "ENC_A";
5'd12: state_name = "ENC_TDEC";
5'd13: state_name = "ENC_C";
5'd14: state_name = "ENC_N";
5'd15: state_name = "ENC_U";
5'd16: state_name = "ENC_C1";
5'd17: state_name = "ENC_V";
5'd18: state_name = "ENC_C2";
5'd19: state_name = "ENC_E2MV";
5'd20: state_name = "DEC_LOAD";
5'd21: state_name = "DEC_DECOMP";
5'd22: state_name = "DEC_SDEC";
5'd23: state_name = "DEC_NTT";
5'd24: state_name = "DEC_W";
5'd25: state_name = "DEC_MENC";
5'd26: state_name = "DEC_G";
5'd27: state_name = "DEC_J";
5'd28: state_name = "DEC_CMP";
5'd31: state_name = "DONE";
default: state_name = "UNKNOWN";
endcase
end
endfunction
task reset_timing;
begin
for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0;
end
endtask
task print_timing;
input [8*16-1:0] op_name;
input integer cycles;
integer runtime_ns;
begin
runtime_ns = cycles * CLK_PERIOD_NS;
$display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)",
KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns,
runtime_ns/1000, runtime_ns%1000,
runtime_ns/1000000, (runtime_ns%1000000)/1000);
$display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name);
for (si = 0; si < 32; si = si + 1)
if (state_cyc[si] != 0)
$display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS);
end
endtask
initial begin
if (!$value$plusargs("CASE=%d", casenum)) casenum = 0;
$sformat(tag, "k%0d", KP);
@@ -130,9 +196,12 @@ module tb_mlkem_dec_katK_xsim;
end
c_in_we = 1'b0; @(posedge clk);
start_i=1; @(posedge clk); start_i=0;
c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end
#1;
reset_timing;
c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end
if(!done_o) begin $display("FAIL K=%0d case %0d: timeout (sel=%0d)", KP, casenum, sel); $finish; end
$display("=== Decaps run (sel=%0d) done in %0d cyc ===", sel, c);
print_timing(sel ? "DecapsReject" : "DecapsAccept", c);
// settle so dbg taps reflect the finished run
repeat(2) @(posedge clk);
end

73
sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v
View File

@@ -12,6 +12,8 @@ module tb_mlkem_enc_katK_xsim;
localparam CTB = (KP==4) ? 1568 : (32*(10*KP+4)); // ct bytes: K2 768,K3 1088,K4 1568
localparam DU = (KP==4) ? 11 : 10; // compression du
localparam C1B = 32*DU*KP; // c1 byte count: K2 640,K3 960,K4 1408
localparam integer CLK_PERIOD_NS = 100; // 10 MHz
localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2;
reg clk=0, rst_n=0, start_i=0;
reg [2:0] k_i;
@@ -41,15 +43,79 @@ module tb_mlkem_enc_katK_xsim;
.dbg_r_o(dbg_r_o), .dbg_hek_o(dbg_hek_o),
.dbg_mprime_o(), .dbg_kbar_o(), .dbg_decz_o(), .dbg_dech_o()
);
always #5 clk = ~clk;
always #(CLK_HALF_NS) clk = ~clk;
reg [7:0] ek_b [0:EKB-1];
reg [7:0] m_b [0:31];
reg [7:0] ss_b [0:31];
reg [7:0] ct_b [0:CTB-1];
integer c, i, errors, casenum, j;
integer state_cyc [0:31];
integer si;
reg [8*80-1:0] tag, ekfile, mfile, ssfile, ctfile;
function [8*16-1:0] state_name;
input [4:0] s;
begin
case (s)
5'd0: state_name = "IDLE";
5'd1: state_name = "G";
5'd2: state_name = "A";
5'd3: state_name = "C";
5'd4: state_name = "N";
5'd5: state_name = "M";
5'd6: state_name = "E";
5'd7: state_name = "H";
5'd8: state_name = "ENC_LOAD";
5'd9: state_name = "ENC_H";
5'd10: state_name = "ENC_G";
5'd11: state_name = "ENC_A";
5'd12: state_name = "ENC_TDEC";
5'd13: state_name = "ENC_C";
5'd14: state_name = "ENC_N";
5'd15: state_name = "ENC_U";
5'd16: state_name = "ENC_C1";
5'd17: state_name = "ENC_V";
5'd18: state_name = "ENC_C2";
5'd19: state_name = "ENC_E2MV";
5'd20: state_name = "DEC_LOAD";
5'd21: state_name = "DEC_DECOMP";
5'd22: state_name = "DEC_SDEC";
5'd23: state_name = "DEC_NTT";
5'd24: state_name = "DEC_W";
5'd25: state_name = "DEC_MENC";
5'd26: state_name = "DEC_G";
5'd27: state_name = "DEC_J";
5'd28: state_name = "DEC_CMP";
5'd31: state_name = "DONE";
default: state_name = "UNKNOWN";
endcase
end
endfunction
task reset_timing;
begin
for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0;
end
endtask
task print_timing;
input [8*16-1:0] op_name;
input integer cycles;
integer runtime_ns;
begin
runtime_ns = cycles * CLK_PERIOD_NS;
$display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)",
KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns,
runtime_ns/1000, runtime_ns%1000,
runtime_ns/1000000, (runtime_ns%1000000)/1000);
$display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name);
for (si = 0; si < 32; si = si + 1)
if (state_cyc[si] != 0)
$display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS);
end
endtask
initial begin
if (!$value$plusargs("CASE=%d", casenum)) casenum = 0;
$sformat(tag, "k%0d", KP);
@@ -81,9 +147,12 @@ module tb_mlkem_enc_katK_xsim;
// ---- run Encaps ----
start_i=1; @(posedge clk); start_i=0;
c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end
#1;
reset_timing;
c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end
if(!done_o) begin $display("FAIL K=%0d case %0d: timeout", KP, casenum); $finish; end
$display("=== Encaps E0 done in %0d cyc ===", c);
print_timing("Encaps", c);
$write(" H(ek) = "); for (j=0;j<32;j=j+1) $write("%02x", dbg_hek_o[8*j +: 8]); $write("\n");
$write(" r = "); for (j=0;j<32;j=j+1) $write("%02x", dbg_r_o[8*j +: 8]); $write("\n");

20
sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v
View File

@@ -1,8 +1,8 @@
// tb_mlkem_hello_world_xsim.v - Hardware run of the ML-KEM hello_world demo.
//
// Mirrors the Rust example end-to-end on the mlkem_top DUT (ML-KEM-512, K=2):
// 1. Alice: KeyGen(d=0x42..., z=0x77...) -> (ek 800B, dk 1632B)
// 2. Bob: Encaps(ek, m=0xDE...) -> (shared_key, kem_ct 768B)
// Mirrors the Rust example end-to-end on the mlkem_top DUT (default ML-KEM-512, K=2):
// 1. Alice: KeyGen(d=0x42..., z=0x77...) -> (ek, dk)
// 2. Bob: Encaps(ek, m=0xDE...) -> (shared_key, kem_ct)
// 3. Bob: XOR-encrypt "hello world" with key -> encrypted
// 4. Alice: Decaps(dk, kem_ct) -> recovered_key
// 5. Alice: XOR-decrypt encrypted with key -> "hello world"
@@ -17,10 +17,12 @@
// xelab tb_mlkem_hello_world_xsim ; xsim
`timescale 1ns/1ps
module tb_mlkem_hello_world_xsim;
localparam KP = 2; // ML-KEM-512
localparam EKB = 384*KP + 32; // 800
localparam DKB = 768*KP + 96; // 1632
localparam CTB = 32*(10*KP + 4); // 768
parameter KP = 2; // 2/3/4 = ML-KEM-512/768/1024
localparam DU = (KP == 4) ? 11 : 10;
localparam DV = (KP == 4) ? 5 : 4;
localparam EKB = 384*KP + 32;
localparam DKB = 768*KP + 96;
localparam CTB = 32*(DU*KP + DV);
localparam MLEN = 11; // "hello world"
reg clk=0, rst_n=0, start_i=0;
@@ -93,7 +95,7 @@ module tb_mlkem_hello_world_xsim;
msg_b[0]="h"; msg_b[1]="e"; msg_b[2]="l"; msg_b[3]="l"; msg_b[4]="o";
msg_b[5]=" "; msg_b[6]="w"; msg_b[7]="o"; msg_b[8]="r"; msg_b[9]="l"; msg_b[10]="d";
$display("=== ML-KEM hello_world (hardware, ML-KEM-512) ===");
$display("=== ML-KEM hello_world (hardware, K=%0d) ===", KP);
$write("Original: \"");
for (i = 0; i < MLEN; i = i + 1) $write("%c", msg_b[i]);
$display("\"\n");
@@ -189,7 +191,7 @@ module tb_mlkem_hello_world_xsim;
if (dec_b[i] !== msg_b[i]) errors = errors + 1;
if (errors == 0)
$display("Success: keys match, message recovered. (hardware ML-KEM-512)");
$display("Success: keys match, message recovered. (hardware K=%0d)", KP);
else
$display("FAILURE: %0d mismatches", errors);
$finish;

73
sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v
View File

@@ -7,6 +7,8 @@ module tb_mlkem_kg_katK_xsim;
parameter KP = 2;
localparam EKB = 384*KP + 32;
localparam DKB = 768*KP + 96;
localparam integer CLK_PERIOD_NS = 100; // 10 MHz
localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2;
reg clk=0, rst_n=0, start_i=0;
reg [2:0] k_i;
@@ -33,7 +35,7 @@ module tb_mlkem_kg_katK_xsim;
.dbg_r_o(), .dbg_hek_o(),
.dbg_mprime_o(), .dbg_kbar_o(), .dbg_decz_o(), .dbg_dech_o()
);
always #5 clk = ~clk;
always #(CLK_HALF_NS) clk = ~clk;
reg [255:0] dmem [0:0];
reg [255:0] zmem [0:0];
@@ -42,8 +44,72 @@ module tb_mlkem_kg_katK_xsim;
reg [7:0] ek_got [0:EKB-1]; // ek bytes read back from DUT
reg [7:0] dk_got [0:DKB-1]; // dk bytes read back from DUT
integer c, i, errors, casenum, j;
integer state_cyc [0:31];
integer si;
reg [8*80-1:0] tag, dfile, zfile, ekfile, dkfile;
function [8*16-1:0] state_name;
input [4:0] s;
begin
case (s)
5'd0: state_name = "IDLE";
5'd1: state_name = "G";
5'd2: state_name = "A";
5'd3: state_name = "C";
5'd4: state_name = "N";
5'd5: state_name = "M";
5'd6: state_name = "E";
5'd7: state_name = "H";
5'd8: state_name = "ENC_LOAD";
5'd9: state_name = "ENC_H";
5'd10: state_name = "ENC_G";
5'd11: state_name = "ENC_A";
5'd12: state_name = "ENC_TDEC";
5'd13: state_name = "ENC_C";
5'd14: state_name = "ENC_N";
5'd15: state_name = "ENC_U";
5'd16: state_name = "ENC_C1";
5'd17: state_name = "ENC_V";
5'd18: state_name = "ENC_C2";
5'd19: state_name = "ENC_E2MV";
5'd20: state_name = "DEC_LOAD";
5'd21: state_name = "DEC_DECOMP";
5'd22: state_name = "DEC_SDEC";
5'd23: state_name = "DEC_NTT";
5'd24: state_name = "DEC_W";
5'd25: state_name = "DEC_MENC";
5'd26: state_name = "DEC_G";
5'd27: state_name = "DEC_J";
5'd28: state_name = "DEC_CMP";
5'd31: state_name = "DONE";
default: state_name = "UNKNOWN";
endcase
end
endfunction
task reset_timing;
begin
for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0;
end
endtask
task print_timing;
input [8*16-1:0] op_name;
input integer cycles;
integer runtime_ns;
begin
runtime_ns = cycles * CLK_PERIOD_NS;
$display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)",
KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns,
runtime_ns/1000, runtime_ns%1000,
runtime_ns/1000000, (runtime_ns%1000000)/1000);
$display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name);
for (si = 0; si < 32; si = si + 1)
if (state_cyc[si] != 0)
$display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS);
end
endtask
// Dump a byte array as offset-prefixed hex, 32 bytes/line.
task dump_bytes(input [8*16-1:0] name, input integer n);
integer a, b;
@@ -80,9 +146,12 @@ module tb_mlkem_kg_katK_xsim;
rst_n=0; repeat(4) @(posedge clk); rst_n=1; @(posedge clk);
start_i=1; @(posedge clk); start_i=0;
c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end
#1;
reset_timing;
c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end
if(!done_o) begin $display("FAIL K=%0d case %0d: timeout", KP, casenum); $finish; end
$display("=== ML-KEM K=%0d KAT case %0d: KeyGen done in %0d cyc ===", KP, casenum, c);
print_timing("KeyGen", c);
errors = 0;
dbg_byte_sel_i = 1'b0;

12
sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v
View File

@@ -14,10 +14,12 @@
// xelab tb_mlkem_two_inst_xsim ; xsim
`timescale 1ns/1ps
module tb_mlkem_two_inst_xsim;
localparam KP = 2; // ML-KEM-512
localparam EKB = 384*KP + 32; // 800
localparam DKB = 768*KP + 96; // 1632
localparam CTB = 32*(10*KP + 4); // 768
parameter KP = 2; // 2/3/4 = ML-KEM-512/768/1024
localparam DU = (KP == 4) ? 11 : 10;
localparam DV = (KP == 4) ? 5 : 4;
localparam EKB = 384*KP + 32;
localparam DKB = 768*KP + 96;
localparam CTB = 32*(DU*KP + DV);
localparam MLEN = 11; // "hello world"
reg clk=0;
@@ -100,7 +102,7 @@ module tb_mlkem_two_inst_xsim;
msg_b[0]="h"; msg_b[1]="e"; msg_b[2]="l"; msg_b[3]="l"; msg_b[4]="o";
msg_b[5]=" "; msg_b[6]="w"; msg_b[7]="o"; msg_b[8]="r"; msg_b[9]="l"; msg_b[10]="d";
$display("=== ML-KEM hello_world (TWO instances: genenc + dec, ML-KEM-512) ===");
$display("=== ML-KEM hello_world (TWO instances: genenc + dec, K=%0d) ===", KP);
$write("Original: \""); for(i=0;i<MLEN;i=i+1) $write("%c", msg_b[i]); $display("\"\n");
a_rst_n=0; b_rst_n=0; repeat(4) @(posedge clk); a_rst_n=1; b_rst_n=1; @(posedge clk);

2
sync_rtl/top/TB/xsim_run.tcl
View File

@@ -13,8 +13,10 @@ xvlog -sv --relax -i . sync_rtl/sha3/sha3_top_shared.v
xvlog -sv --relax -i . sync_rtl/sample_ntt/sample_ntt_sync_shared.v
xvlog -sv --relax -i . sync_rtl/sample_cbd/sample_cbd_sync_shared.v
xvlog -sv --relax -i . sync_rtl/ntt/barrett_mul.v
xvlog -sv --relax -i . sync_rtl/ntt/barrett_mul_pipe.v
xvlog -sv --relax -i . sync_rtl/ntt/zeta_rom.v
xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit.v
xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit_pipe.v
xvlog -sv --relax -i . sync_rtl/ntt/ntt_core.v
xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul.v
xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul_pipe.v

53
sync_rtl/top/mlkem_top.v
View File

@@ -22,7 +22,7 @@
`include "sync_rtl/common/defines.vh"
(* use_dsp = "no" *)
(* use_dsp = "yes" *)
module mlkem_top #(
parameter KMAX = 4 // storage sizing (worst case = ML-KEM-1024)
) (
@@ -194,6 +194,7 @@ module mlkem_top #(
wire [10:0] cin_rd_addr;
wire [7:0] cin_rd_data;
reg [10:0] cin_rd_addr_r; // D5/D7 read address (tied 0 until then)
reg dj_active; // background Decaps J(z||c) owns c_in read port
sd_bram #(.W(8), .D(2048), .A(11)) u_c_in_bram (
.clk(clk),
.rd_addr(cin_rd_addr), .rd_data(cin_rd_data),
@@ -203,7 +204,7 @@ module mlkem_top #(
// stage) so the registered BRAM read yields the byte 1 cycle later, matching
// the assemble/writeback pipeline. Otherwise the registered cin_rd_addr_r
// (D1 walker / idle) drives it.
assign cin_rd_addr = (st == ST_DEC_J) ? dj_c_idx[10:0] :
assign cin_rd_addr = dj_active ? dj_c_idx[10:0] :
(st == ST_DEC_CMP) ? cmp_idx : cin_rd_addr_r;
// ================================================================
@@ -567,7 +568,7 @@ module mlkem_top #(
// ST_ENC_A (snt), ST_ENC_C (cbd).
wire sel_sha3 = (st == ST_G) || (st == ST_H) ||
(st == ST_ENC_H) || (st == ST_ENC_G) ||
(st == ST_DEC_G) || (st == ST_DEC_J);
(st == ST_DEC_G) || dj_active;
wire sel_snt = (st == ST_A) || (st == ST_ENC_A);
wire sel_cbd = (st == ST_C) || (st == ST_ENC_C);
@@ -597,14 +598,14 @@ module mlkem_top #(
// G/H/J share one sha3_top. Single-block (mb_en=0): KeyGen G (ST_G, mode 00),
// Encaps G (ST_ENC_G, mode 11), Decaps D5 G (ST_DEC_G, mode 11). Multi-block
// absorb (mb_en=1): H(ek) (ST_H/ST_ENC_H, SHA3-256 pad) and Decaps D5 J
// (ST_DEC_J, SHAKE256 pad over z||c). mb_* inputs are muxed by phase.
wire sha3_mb_en = (st == ST_H) || (st == ST_ENC_H) || (st == ST_DEC_J);
// (background dj_active, SHAKE256 pad over z||c). mb_* inputs are muxed by phase.
wire sha3_mb_en = (st == ST_H) || (st == ST_ENC_H) || dj_active;
wire [1:0] sha3_mode = (st == ST_ENC_G || st == ST_DEC_G) ? 2'b11 : 2'b00;
// multi-block feed mux: J (ST_DEC_J) drives dj_*, else H drives h_*.
wire [1087:0] mb_block_mux = (st == ST_DEC_J) ? dj_block_r : h_block_r;
wire mb_valid_mux = (st == ST_DEC_J) ? dj_mbvalid : h_mbvalid;
wire mb_last_mux = (st == ST_DEC_J) ? dj_mblast : h_mblast;
wire mb_ack_mux = (st == ST_DEC_J) ? dj_ack : h_ack;
// multi-block feed mux: background J drives dj_*, else H drives h_*.
wire [1087:0] mb_block_mux = dj_active ? dj_block_r : h_block_r;
wire mb_valid_mux = dj_active ? dj_mbvalid : h_mbvalid;
wire mb_last_mux = dj_active ? dj_mblast : h_mblast;
wire mb_ack_mux = dj_active ? dj_ack : h_ack;
sha3_top_shared u_sha3 (
.clk(clk), .rst_n(rst_n),
.mode(sha3_mode), // G = SHA3-512 (only used when mb_en=0)
@@ -1145,12 +1146,12 @@ module mlkem_top #(
// D3: w = v' - INTT(sum_j s_hat[j] o u_hat[j]). Single output poly
// (u_row 0..0); MAC->INTT->SUB, then D4 encodes m'.
ST_DEC_W: if (u_row >= 3'd1) st_next = ST_DEC_MENC;
ST_DEC_MENC: if (men_done) st_next = ST_DEC_G;
// D5: (K',r') = G(m'||h) single-block, then K-bar = J(z||c) multi-block.
ST_DEC_G: if (sha3_vo) st_next = ST_DEC_J;
// D5 J done -> D6 re-encrypt: c' = K-PKE.Encrypt(ek_pke, m', r').
// Reuse the entire Encaps pipeline (rho load -> A -> C -> ... -> C2).
// r' is in r_r (CBD seed), m' in m_r (V/mu), ek_pke in ek_bram.
ST_DEC_MENC: if (men_done && dj_done) st_next = ST_DEC_G;
// D5: (K',r') = G(m'||h) single-block. K-bar = J(z||c) runs in a
// background SHAKE256 walker after D1, so the main FSM only waits
// for it at the D4->D5 boundary if it somehow has not finished.
ST_DEC_G: if (sha3_vo) st_next = ST_ENC_LOAD;
// Legacy state kept unused; background J normally skips it.
ST_DEC_J: if (dj_done) st_next = ST_ENC_LOAD;
ST_G: if (sha3_vo) st_next = ST_A;
ST_A: if (a_pair >= kk_rt) st_next = ST_C;
@@ -1303,6 +1304,7 @@ module mlkem_top #(
cmp_neq <= 1'b0;
cmp_done <= 1'b0;
dec_reject <= 1'b0;
dj_active <= 1'b0;
dj_blk <= 4'd0;
dj_byte <= 8'd0;
dj_phase <= 2'd0;
@@ -1349,6 +1351,8 @@ module mlkem_top #(
end else if (op_i == 2'd2) begin
// Decaps: dk_pke/ek_pke already in BRAM; z/H(ek) captured into
// z_r/hek_r during load (below). Nothing else to arm in D0.
dj_active <= 1'b0;
dj_done <= 1'b0;
end else begin
sha3_valid <= 1'b1;
sha3_ack <= 1'b1;
@@ -1993,9 +1997,11 @@ module mlkem_top #(
sha3_ack <= 1'b0;
end
// Arm D5 J(z||c) when G completes (ST_DEC_G -> ST_DEC_J): assemble the
// first 136-byte block. dj_ack high to consume the final digest.
if (st == ST_DEC_G && st_next == ST_DEC_J) begin
// Arm D5 J(z||c) as soon as D1 has stopped reading c_in_bram. It
// then runs in the background during D2/D3/D4 using the otherwise
// idle SHAKE256 path and c_in read port.
if (st == ST_DEC_DECOMP && st_next == ST_DEC_SDEC) begin
dj_active <= 1'b1;
dj_blk <= 4'd0;
dj_byte <= 8'd0;
dj_phase <= 2'd0;
@@ -2007,14 +2013,14 @@ module mlkem_top #(
cin_rd_addr_r <= 11'd0; // present c byte 0 (g=32 -> c_idx 0)
end
// ---- ST_DEC_J: K-bar = J(z||c) multi-block SHAKE256 ----
// ---- Background Decaps J: K-bar = J(z||c) multi-block SHAKE256 ----
// Mirror of the H(ek) multi-block machine. Byte source by global g:
// g < 32 -> z_r byte g
// 32 <= g < msglen -> c_in_bram byte (g-32), registered read
// else -> SHAKE256 pad constant
// c_in_bram read is registered: present c_idx for dj_byte this cycle,
// write back the byte that arrived for the addr presented last cycle.
if (st == ST_DEC_J && !dj_done) begin
if (dj_active && !dj_done) begin
case (dj_phase)
2'd0: begin
// writeback the byte read for the previous address
@@ -2034,7 +2040,7 @@ module mlkem_top #(
dj_wb_zidx <= dj_g[4:0];
dj_wb_pad <= dj_padconst(dj_g);
// c_in addr is presented combinationally (cin_rd_addr
// mux uses dj_c_idx during ST_DEC_J); data lands next
// mux uses dj_c_idx while dj_active); data lands next
// cycle, matching this byte's writeback.
dj_byte <= dj_byte + 8'd1;
end else begin
@@ -2056,6 +2062,7 @@ module mlkem_top #(
if (sha3_vo) begin
kbar_r <= sha3_hash[255:0];
dj_done <= 1'b1;
dj_active <= 1'b0;
dj_phase<= 2'd3;
end else if (h_mbready) begin
dj_blk <= dj_blk + 4'd1;
@@ -2095,7 +2102,7 @@ module mlkem_top #(
end
if (st_next == ST_ENC_LOAD &&
(st == ST_ENC_G || st == ST_DEC_J)) begin
(st == ST_ENC_G || st == ST_DEC_G || st == ST_DEC_J)) begin
rl_idx <= 6'd0;
rl_widx <= 6'd0;
rl_vld <= 1'b0;

2
synth_timing.tcl
View File

@@ -10,8 +10,10 @@ read_verilog -sv ${PROJECT_DIR}/sync_rtl/sha3/sha3_top_shared.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_ntt/sample_ntt_sync_shared.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_cbd/sample_cbd_sync_shared.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul_pipe.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/zeta_rom.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit_pipe.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/ntt_core.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/basecase_mul.v
read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/basecase_mul_pipe.v