From 372a90e60154ea714d586e4428491680fb5adeb8 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Wed, 8 Jul 2026 00:23:46 +0800 Subject: [PATCH] Pipeline ML-KEM datapath bottlenecks --- create_project.tcl | 2 + run_tb.sh | 32 ++- sync_rtl/comp_decomp/comp_decomp_sync.v | 208 +++++++++++++----- sync_rtl/ntt/TB/xsim_run.tcl | 2 + sync_rtl/ntt/barrett_mul_pipe.v | 72 ++++++ sync_rtl/ntt/butterfly_unit_pipe.v | 74 +++++++ sync_rtl/ntt/ntt_core.v | 232 +++++++++++++++----- sync_rtl/poly_mul/basecase_mul_pipe.v | 2 +- sync_rtl/sha3/sha3_top.v | 20 +- sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v | 73 +++++- sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v | 73 +++++- sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v | 20 +- sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v | 73 +++++- sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v | 12 +- sync_rtl/top/TB/xsim_run.tcl | 2 + sync_rtl/top/mlkem_top.v | 53 +++-- synth_timing.tcl | 2 + 17 files changed, 776 insertions(+), 176 deletions(-) create mode 100644 sync_rtl/ntt/barrett_mul_pipe.v create mode 100644 sync_rtl/ntt/butterfly_unit_pipe.v diff --git a/create_project.tcl b/create_project.tcl index c315eb0..5accb7a 100644 --- a/create_project.tcl +++ b/create_project.tcl @@ -34,8 +34,10 @@ read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_cbd/sample_cbd_sync_shared.v # ── NTT ── read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul.v +read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul_pipe.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/zeta_rom.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit.v +read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit_pipe.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/ntt_core.v # ── 多项式乘法 ── diff --git a/run_tb.sh b/run_tb.sh index 402b2e6..6a45154 100755 --- a/run_tb.sh +++ b/run_tb.sh @@ -12,8 +12,8 @@ # ./run_tb.sh enc 4 1 # Encaps K=4, only CASE=1 # ./run_tb.sh dec # ML-KEM Decaps: all K, all cases (0..2) # ./run_tb.sh dec 2 0 # Decaps K=2, only CASE=0 -# ./run_tb.sh hello # hello_world end-to-end (single instance) -# ./run_tb.sh hello two # hello_world end-to-end (two instances) +# ./run_tb.sh hello [K] # hello_world end-to-end (single instance; K=2/3/4) +# ./run_tb.sh hello two [K] # hello_world end-to-end (two instances; K=2/3/4) # ./run_tb.sh --list # # 'top' (KeyGen), 'enc' (Encaps) and 'dec' (Decaps) share the same RTL datapath; @@ -41,7 +41,7 @@ if [ "$1" = "--list" ]; then done echo " enc (ML-KEM Encaps; shares the 'top' RTL/tcl, enc testbench)" echo " dec (ML-KEM Decaps; shares the 'top' RTL/tcl, dec testbench)" - echo " hello (ML-KEM hello_world end-to-end; 'hello two' for two instances)" + echo " hello (ML-KEM hello_world end-to-end; optional K=2/3/4, 'hello two [K]' for two instances)" exit 0 fi @@ -286,12 +286,24 @@ fi # ML-KEM hello_world end-to-end. Reuses the 'top' RTL compile list, then runs a # self-checking protocol TB (no KAT vectors): single instance by default, or two -# instances (genenc + dec) with 'hello two'. Prints each step's inputs/outputs. +# instances (genenc + dec) with 'hello two'. Optional K selects ML-KEM-512/768/1024. run_hello_selected() { - local tcl_file="$1" variant="$2" + local tcl_file="$1" arg1="$2" arg2="$3" set +e rm -rf xsim.dir .Xil + local variant="" ksel="2" + if [ "$arg1" = "two" ]; then + variant="two" + [ -n "$arg2" ] && ksel="$arg2" + elif [ -n "$arg1" ]; then + ksel="$arg1" + fi + case "$ksel" in + 2|3|4) ;; + *) echo "ERROR: hello K must be 2, 3, or 4 (got '$ksel')"; return 1 ;; + esac + local tb snap if [ "$variant" = "two" ]; then tb=tb_mlkem_two_inst_xsim; snap=mlkem_two @@ -308,15 +320,15 @@ run_hello_selected() { echo " xvlog -sv --relax sync_rtl/top/TB/$tb.v" xvlog -sv --relax "sync_rtl/top/TB/$tb.v" || { echo "HELLO TB COMPILE FAILED"; return 1; } - echo " xelab $tb -s $snap --timescale 1ns/1ps" - xelab "$tb" -s "$snap" --timescale 1ns/1ps || { echo "ELAB FAILED"; return 1; } + echo " xelab $tb -generic_top KP=$ksel -s ${snap}_k${ksel} --timescale 1ns/1ps" + xelab "$tb" -generic_top "KP=$ksel" -s "${snap}_k${ksel}" --timescale 1ns/1ps || { echo "ELAB FAILED"; return 1; } - xsim "$snap" -R + xsim "${snap}_k${ksel}" -R } if [ "$MODULE" = "hello" ]; then - run_hello_selected "$TCL_FILE" "$SEL_K" # SEL_K position carries the variant ('two') + run_hello_selected "$TCL_FILE" "$SEL_K" "$SEL_CASE" # args are [K] or [two] [K] exit $? fi -execute_tcl "$TCL_FILE" \ No newline at end of file +execute_tcl "$TCL_FILE" diff --git a/sync_rtl/comp_decomp/comp_decomp_sync.v b/sync_rtl/comp_decomp/comp_decomp_sync.v index a31b5e2..3ba4cbd 100644 --- a/sync_rtl/comp_decomp/comp_decomp_sync.v +++ b/sync_rtl/comp_decomp/comp_decomp_sync.v @@ -1,7 +1,7 @@ // comp_decomp_sync.v - ML-KEM coefficient compression/decompression // -// Streaming valid/ready interface. Compression is intentionally iterative to -// avoid inferring a long combinational divider for /Q. +// Streaming valid/ready interface. The arithmetic is fixed-latency pipelined +// so compression does not infer a long combinational divider/reducer. // mode=0: compress — round((2^d * x) / Q) mod 2^d // mode=1: decompress — round((Q * x) / 2^d) // Uses round-half-up (round(2.5)=3, round(3.5)=4). @@ -11,7 +11,7 @@ `include "sync_rtl/common/defines.vh" -(* use_dsp = "no" *) +(* use_dsp = "yes" *) module comp_decomp_sync ( input clk, input rst_n, @@ -25,79 +25,169 @@ module comp_decomp_sync ( input ready_i ); - localparam [11:0] Q_VAL = 12'(`Q); - localparam [4:0] COMP_MSB = 5'd22; // max ((Q-1)<<11)+Q/2 is 23 bits + localparam [24:0] Q25 = 25'd`Q; + localparam [12:0] K13 = 13'd5039; // floor(2^24 / 3329) - reg busy_r; - reg [4:0] bit_idx_r; - reg [4:0] d_r; - reg [23:0] dividend_r; - reg [23:0] quotient_r; - reg [12:0] rem_r; reg [11:0] data_r; reg valid_r; + reg mode_s0; + reg [4:0] d_s0; + reg [11:0] coeff_s0; + reg valid_s0; + + reg mode_s1; + reg [4:0] d_s1; + reg [23:0] comp_dividend_s1; + reg [23:0] decomp_rounded_s1; + reg valid_s1; + + reg mode_s2; + reg [4:0] d_s2; + reg [23:0] comp_dividend_s2; + reg [36:0] comp_qprod_est_s2; + reg [23:0] decomp_rounded_s2; + reg valid_s2; + + reg mode_s3; + reg [4:0] d_s3; + reg [23:0] comp_dividend_s3; + reg [12:0] comp_q_est_s3; + reg [11:0] decomp_result_s3; + reg valid_s3; + + reg mode_s4; + reg [4:0] d_s4; + reg [23:0] comp_dividend_s4; + reg [12:0] comp_q_est_s4; + reg [24:0] comp_q_mul_q_s4; + reg [11:0] decomp_result_s4; + reg valid_s4; + + reg mode_s5; + reg [4:0] d_s5; + reg [12:0] comp_q_est_s5; + reg [24:0] comp_rem_s5; + reg [11:0] decomp_result_s5; + reg valid_s5; + wire fire_i = valid_i && ready_o; - // Accept a new command only when the iterative divider and output slot are - // both free. Current users tie ready_i high and issue one coefficient at a - // time, but this keeps the module from accepting work it cannot retain. - assign ready_o = !busy_r && (!valid_r || ready_i); + wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5; + + // Accept a new command only when the pipeline and output slot are both free. + // Current users issue one coefficient at a time and wait for valid_o. + assign ready_o = !pipe_busy && (!valid_r || ready_i); assign coeff_out = data_r; assign valid_o = valid_r; - wire [23:0] comp_dividend = ({12'd0, coeff_in} << d) + 24'd1664; - wire [23:0] decomp_product = - {12'd0, coeff_in} + - ({12'd0, coeff_in} << 8) + - ({12'd0, coeff_in} << 10) + - ({12'd0, coeff_in} << 11); - wire [23:0] decomp_rounded = decomp_product + (24'd1 << (d - 1'b1)); - wire [11:0] decomp_result = decomp_rounded >> d; + wire [23:0] coeff_ext_s0 = {12'd0, coeff_s0}; + wire [23:0] comp_dividend_next = (coeff_ext_s0 << d_s0) + 24'd1664; + wire [23:0] decomp_product_next = coeff_s0 * 12'd`Q; + wire [23:0] decomp_round_next = + decomp_product_next + ((d_s0 == 5'd0) ? 24'd0 : (24'd1 << (d_s0 - 5'd1))); - wire [12:0] rem_shift = {rem_r[11:0], dividend_r[bit_idx_r]}; - wire rem_ge_q = rem_shift >= {1'b0, Q_VAL}; - wire [12:0] rem_sub_q = rem_shift - {1'b0, Q_VAL}; - wire [12:0] rem_next = rem_ge_q ? rem_sub_q : rem_shift; - wire [23:0] quot_next = quotient_r | (rem_ge_q ? (24'd1 << bit_idx_r) : 24'd0); - wire [11:0] comp_mask = (12'd1 << d_r) - 12'd1; + wire [12:0] comp_q_corr1_s5 = + comp_q_est_s5 + ((comp_rem_s5 >= Q25) ? 13'd1 : 13'd0); + wire [24:0] comp_rem_corr1_s5 = + (comp_rem_s5 >= Q25) ? (comp_rem_s5 - Q25) : comp_rem_s5; + wire [12:0] comp_q_corr2_s5 = + comp_q_corr1_s5 + ((comp_rem_corr1_s5 >= Q25) ? 13'd1 : 13'd0); + wire [11:0] comp_mask_s5 = + (d_s5 == 5'd0) ? 12'd0 : (12'd1 << d_s5) - 12'd1; + wire [11:0] comp_result_s5 = comp_q_corr2_s5[11:0] & comp_mask_s5; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - busy_r <= 1'b0; - bit_idx_r <= 5'd0; - d_r <= 5'd0; - dividend_r <= 24'd0; - quotient_r <= 24'd0; - rem_r <= 13'd0; - data_r <= 12'd0; - valid_r <= 1'b0; + data_r <= 12'd0; + valid_r <= 1'b0; + + mode_s0 <= 1'b0; + d_s0 <= 5'd0; + coeff_s0 <= 12'd0; + valid_s0 <= 1'b0; + + mode_s1 <= 1'b0; + d_s1 <= 5'd0; + comp_dividend_s1 <= 24'd0; + decomp_rounded_s1 <= 24'd0; + valid_s1 <= 1'b0; + + mode_s2 <= 1'b0; + d_s2 <= 5'd0; + comp_dividend_s2 <= 24'd0; + comp_qprod_est_s2 <= 37'd0; + decomp_rounded_s2 <= 24'd0; + valid_s2 <= 1'b0; + + mode_s3 <= 1'b0; + d_s3 <= 5'd0; + comp_dividend_s3 <= 24'd0; + comp_q_est_s3 <= 13'd0; + decomp_result_s3 <= 12'd0; + valid_s3 <= 1'b0; + + mode_s4 <= 1'b0; + d_s4 <= 5'd0; + comp_dividend_s4 <= 24'd0; + comp_q_est_s4 <= 13'd0; + comp_q_mul_q_s4 <= 25'd0; + decomp_result_s4 <= 12'd0; + valid_s4 <= 1'b0; + + mode_s5 <= 1'b0; + d_s5 <= 5'd0; + comp_q_est_s5 <= 13'd0; + comp_rem_s5 <= 25'd0; + decomp_result_s5 <= 12'd0; + valid_s5 <= 1'b0; end else begin if (valid_r && ready_i) valid_r <= 1'b0; - if (busy_r) begin - rem_r <= rem_next; - quotient_r <= quot_next; - if (bit_idx_r == 5'd0) begin - busy_r <= 1'b0; - data_r <= quot_next[11:0] & comp_mask; - valid_r <= 1'b1; - end else begin - bit_idx_r <= bit_idx_r - 5'd1; - end - end else if (fire_i) begin - if (mode) begin - data_r <= decomp_result; - valid_r <= 1'b1; - end else begin - busy_r <= 1'b1; - bit_idx_r <= COMP_MSB; - d_r <= d; - dividend_r <= comp_dividend; - quotient_r <= 24'd0; - rem_r <= 13'd0; - end + mode_s0 <= mode; + d_s0 <= d; + coeff_s0 <= coeff_in; + valid_s0 <= fire_i; + + mode_s1 <= mode_s0; + d_s1 <= d_s0; + comp_dividend_s1 <= comp_dividend_next; + decomp_rounded_s1 <= decomp_round_next; + valid_s1 <= valid_s0; + + mode_s2 <= mode_s1; + d_s2 <= d_s1; + comp_dividend_s2 <= comp_dividend_s1; + comp_qprod_est_s2 <= {13'd0, comp_dividend_s1} * K13; + decomp_rounded_s2 <= decomp_rounded_s1; + valid_s2 <= valid_s1; + + mode_s3 <= mode_s2; + d_s3 <= d_s2; + comp_dividend_s3 <= comp_dividend_s2; + comp_q_est_s3 <= comp_qprod_est_s2[36:24]; + decomp_result_s3 <= decomp_rounded_s2 >> d_s2; + valid_s3 <= valid_s2; + + mode_s4 <= mode_s3; + d_s4 <= d_s3; + comp_dividend_s4 <= comp_dividend_s3; + comp_q_est_s4 <= comp_q_est_s3; + comp_q_mul_q_s4 <= comp_q_est_s3 * Q25; + decomp_result_s4 <= decomp_result_s3; + valid_s4 <= valid_s3; + + mode_s5 <= mode_s4; + d_s5 <= d_s4; + comp_q_est_s5 <= comp_q_est_s4; + comp_rem_s5 <= {1'b0, comp_dividend_s4} - comp_q_mul_q_s4; + decomp_result_s5 <= decomp_result_s4; + valid_s5 <= valid_s4; + + if (valid_s5) begin + data_r <= mode_s5 ? decomp_result_s5 : comp_result_s5; + valid_r <= 1'b1; end end end diff --git a/sync_rtl/ntt/TB/xsim_run.tcl b/sync_rtl/ntt/TB/xsim_run.tcl index b6f79b7..b099a71 100644 --- a/sync_rtl/ntt/TB/xsim_run.tcl +++ b/sync_rtl/ntt/TB/xsim_run.tcl @@ -30,12 +30,14 @@ puts "=== Compiling RTL sources ===" # Barrett modular multiplier (combinational) xvlog -sv ${SRC_DIR}/barrett_mul.v +xvlog -sv ${SRC_DIR}/barrett_mul_pipe.v # Zeta ROM (combinational) xvlog -sv ${SRC_DIR}/zeta_rom.v # Butterfly unit (combinational, instantiates barrett_mul) xvlog -sv ${SRC_DIR}/butterfly_unit.v +xvlog -sv ${SRC_DIR}/butterfly_unit_pipe.v # NTT core (FSM-based, instantiates butterfly_unit + zeta_rom + barrett_mul) xvlog -sv ${SRC_DIR}/ntt_core.v diff --git a/sync_rtl/ntt/barrett_mul_pipe.v b/sync_rtl/ntt/barrett_mul_pipe.v new file mode 100644 index 0000000..253ae52 --- /dev/null +++ b/sync_rtl/ntt/barrett_mul_pipe.v @@ -0,0 +1,72 @@ +// barrett_mul_pipe.v - DSP-friendly pipelined Barrett modular multiply +// +// Computes product = a * b mod Q with fixed latency. This module is used on +// timing-critical NTT paths; the legacy barrett_mul remains combinational for +// older users. + +module barrett_mul_pipe ( + input clk, + input rst_n, + input valid_i, + input [11:0] a, + input [11:0] b, + output [11:0] product, + output valid_o +); + localparam [24:0] Q25 = 25'd3329; + localparam [12:0] K13 = 13'd5039; + + reg [23:0] p_s1; + + reg [23:0] p_s2; + reg [36:0] tk_s2; + + reg [23:0] p_s3; + reg [12:0] q_s3; + + reg [24:0] p_s4; + reg [24:0] qprod_s4; + + reg [24:0] r0_s5; + reg [24:0] r1_s6; + reg [11:0] product_r; + reg [6:0] valid_sr; + + assign product = product_r; + assign valid_o = valid_sr[6]; + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + p_s1 <= 24'd0; + p_s2 <= 24'd0; tk_s2 <= 37'd0; + p_s3 <= 24'd0; q_s3 <= 13'd0; + p_s4 <= 25'd0; qprod_s4 <= 25'd0; + r0_s5 <= 25'd0; r1_s6 <= 25'd0; + product_r <= 12'd0; + valid_sr <= 7'd0; + end else begin + valid_sr <= {valid_sr[5:0], valid_i}; + + // S1: full product. + p_s1 <= {12'd0, a} * b; + + // S2: quotient-estimate product. + p_s2 <= p_s1; + tk_s2 <= {13'd0, p_s1} * K13; + + // S3: quotient estimate. + p_s3 <= p_s2; + q_s3 <= tk_s2[36:24]; + + // S4: q * Q. + p_s4 <= {1'b0, p_s3}; + qprod_s4 <= q_s3 * Q25; + + // S5/S6/S7: subtract q*Q and conditionally subtract Q twice. + r0_s5 <= p_s4 - qprod_s4; + r1_s6 <= (r0_s5 >= Q25) ? (r0_s5 - Q25) : r0_s5; + product_r <= (r1_s6 >= Q25) ? (r1_s6 - Q25) : r1_s6[11:0]; + end + end + +endmodule diff --git a/sync_rtl/ntt/butterfly_unit_pipe.v b/sync_rtl/ntt/butterfly_unit_pipe.v new file mode 100644 index 0000000..1c8736b --- /dev/null +++ b/sync_rtl/ntt/butterfly_unit_pipe.v @@ -0,0 +1,74 @@ +// butterfly_unit_pipe.v - pipelined Cooley-Tukey / Gentleman-Sande butterfly +// +// Fixed-latency wrapper around barrett_mul_pipe. All arithmetic that used to +// sit in one NTT cycle is now split by registers. + +module butterfly_unit_pipe ( + input clk, + input rst_n, + input valid_i, + input [11:0] a, + input [11:0] b, + input [11:0] zeta, + input mode, // 0 = forward NTT, 1 = inverse NTT + output [11:0] a_out, + output [11:0] b_out, + output valid_o +); + localparam [11:0] Q12 = 12'd3329; + localparam [12:0] Q13 = 13'd3329; + + wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12); + wire [11:0] mul_data = mode ? diff : b; + + wire [11:0] mul_result; + wire mul_valid; + barrett_mul_pipe u_barrett ( + .clk(clk), + .rst_n(rst_n), + .valid_i(valid_i), + .a(zeta), + .b(mul_data), + .product(mul_result), + .valid_o(mul_valid) + ); + + reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5; + reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5; + reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5; + + reg [11:0] a_out_r, b_out_r; + reg valid_r; + + wire [12:0] a_sum = {1'b0, a_d5} + {1'b0, (mode_d5 ? b_d5 : mul_result)}; + wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0]; + wire [11:0] b_sub = (a_d5 >= mul_result) ? (a_d5 - mul_result) : + (a_d5 - mul_result + Q12); + + assign a_out = a_out_r; + assign b_out = b_out_r; + assign valid_o = valid_r; + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; + b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; + mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; + a_out_r <= 12'd0; + b_out_r <= 12'd0; + valid_r <= 1'b0; + end else begin + a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; + b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; + mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1; + mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; + + valid_r <= mul_valid; + if (mul_valid) begin + a_out_r <= a_mod; + b_out_r <= mode_d5 ? mul_result : b_sub; + end + end + end + +endmodule diff --git a/sync_rtl/ntt/ntt_core.v b/sync_rtl/ntt/ntt_core.v index f5146ce..35dfe0f 100644 --- a/sync_rtl/ntt/ntt_core.v +++ b/sync_rtl/ntt/ntt_core.v @@ -1,8 +1,10 @@ // ntt_core.v - NTT core with individual coefficient registers // -// Uses 256 individual 12-bit registers and generate-based muxing -// to avoid any part-select simulation issues. -// 3-cycle butterfly: SetAddr -> Read -> Compute+Write +// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path. +// The arithmetic hot path is split into: +// address -> operand/zeta register -> pipelined Barrett butterfly -> writeback +// In inverse mode, final x3303 output scaling also uses a pipelined Barrett +// multiplier so the output path does not reintroduce a combinational reducer. module ntt_core ( input clk, rst_n, @@ -17,72 +19,149 @@ module ntt_core ( ); localparam N = 256, LAYERS = 7, DW = 12; - // Individual coefficient registers reg [DW-1:0] cr [0:N-1]; integer ci; - // State machine - localparam S_IDLE=3'd0, S_LOAD=3'd1, S_CMP_A=3'd2, S_CMP_B=3'd3, - S_CMP_C=3'd4, S_OUTPUT=3'd5, S_DONE=3'd6; - reg [2:0] state, next_state; + localparam S_IDLE = 4'd0; + localparam S_LOAD = 4'd1; + localparam S_CMP_A = 4'd2; + localparam S_CMP_B = 4'd3; + localparam S_CMP_ISSUE = 4'd4; + localparam S_CMP_WAIT = 4'd5; + localparam S_CMP_WB = 4'd6; + localparam S_OUT_PREP = 4'd7; + localparam S_OUTPUT = 4'd8; + localparam S_OUT_SCALE = 4'd9; + localparam S_DONE = 4'd10; - reg [7:0] load_cnt, out_cnt; + reg [3:0] state, next_state; + + reg [7:0] load_cnt; + reg [7:0] out_cnt; + reg [8:0] scale_issue_cnt; + reg [8:0] scale_emit_cnt; reg [7:0] j, start, layer_len; reg [6:0] zeta_idx; reg [2:0] layer; - reg bf_done; + reg bf_done; + reg mode_r; - // Pipeline registers - reg [DW-1:0] r_a, r_b; - reg [7:0] r_wa, r_wb; + reg [DW-1:0] r_a, r_b, r_zeta; + reg [7:0] r_wa, r_wb; + reg [DW-1:0] wr_a_data, wr_b_data; + reg [7:0] wr_wa, wr_wb; + + reg [DW-1:0] coeff_out_r; + reg valid_o_r; + reg scale_valid_i; + reg [DW-1:0] scale_a_i; - // Zeta wire [DW-1:0] zeta; zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); - // Butterfly wire [DW-1:0] bf_a_out, bf_b_out; - butterfly_unit u_bf ( - .a(r_a), .b(r_b), .zeta(zeta), .mode(mode), - .a_out(bf_a_out), .b_out(bf_b_out)); + wire bf_valid; + butterfly_unit_pipe u_bf ( + .clk(clk), + .rst_n(rst_n), + .valid_i(state == S_CMP_ISSUE), + .a(r_a), + .b(r_b), + .zeta(r_zeta), + .mode(mode_r), + .a_out(bf_a_out), + .b_out(bf_b_out), + .valid_o(bf_valid) + ); - // Output scaling - wire [DW-1:0] coeff_scaled; - barrett_mul u_scl (.a(cr[out_cnt]), .b(12'd3303), .product(coeff_scaled)); - assign coeff_out = mode ? coeff_scaled : cr[out_cnt]; + wire [DW-1:0] scale_product; + wire scale_valid_o; + barrett_mul_pipe u_scl ( + .clk(clk), + .rst_n(rst_n), + .valid_i(scale_valid_i), + .a(scale_a_i), + .b(12'd3303), + .product(scale_product), + .valid_o(scale_valid_o) + ); - - assign ready_o = (state == S_IDLE) || (state == S_LOAD); - assign valid_o = (state == S_OUTPUT); - assign done_o = (state == S_DONE); + assign ready_o = (state == S_IDLE) || (state == S_LOAD); + assign coeff_out = coeff_out_r; + assign valid_o = valid_o_r; + assign done_o = (state == S_DONE); always @* begin next_state = state; case (state) - S_IDLE: if (valid_i) next_state = S_LOAD; - S_LOAD: if (load_cnt >= 255 && valid_i) next_state = S_CMP_A; - S_CMP_A: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_B; - S_CMP_B: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_C; - S_CMP_C: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_A; - S_OUTPUT:if (out_cnt >= 255 && ready_i) next_state = S_DONE; - S_DONE: next_state = S_IDLE; - default: next_state = S_IDLE; + S_IDLE: if (valid_i) next_state = S_LOAD; + S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_A; + S_CMP_A: next_state = bf_done ? S_OUT_PREP : S_CMP_B; + S_CMP_B: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE; + S_CMP_ISSUE: next_state = S_CMP_WAIT; + S_CMP_WAIT: if (bf_valid) next_state = S_CMP_WB; + S_CMP_WB: next_state = S_CMP_A; + S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; + S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; + S_OUT_SCALE: if (scale_valid_o && scale_emit_cnt >= 9'd255) next_state = S_DONE; + S_DONE: next_state = S_IDLE; + default: next_state = S_IDLE; endcase end always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - state<=S_IDLE; load_cnt<=0; out_cnt<=0; j<=0; start<=0; layer_len<=0; - zeta_idx<=0; layer<=0; bf_done<=0; r_a<=0; r_b<=0; r_wa<=0; r_wb<=0; - for (ci=0; ci= start + layer_len) begin - if (!mode) zeta_idx <= zeta_idx + 7'd1; - else zeta_idx <= zeta_idx - 7'd1; + if (!mode_r) zeta_idx <= zeta_idx + 7'd1; + else zeta_idx <= zeta_idx - 7'd1; - if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 256) begin + if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 9'd256) begin layer <= layer + 3'd1; - layer_len <= mode ? (layer_len<<1) : (layer_len>>1); - start <= 0; j <= 0; + layer_len <= mode_r ? (layer_len << 1) : (layer_len >> 1); + start <= 8'd0; + j <= 8'd0; if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1; end else begin start <= start + layer_len + layer_len; @@ -124,8 +209,49 @@ module ntt_core ( end end - if (state == S_OUTPUT && ready_i) - out_cnt <= (out_cnt>=255) ? 0 : (out_cnt+8'd1); + if (state == S_OUT_PREP) begin + out_cnt <= 8'd0; + scale_issue_cnt <= 9'd0; + scale_emit_cnt <= 9'd0; + if (!mode_r) begin + coeff_out_r <= cr[0]; + valid_o_r <= 1'b1; + end + end + + if (state == S_OUTPUT && valid_o_r && ready_i) begin + if (out_cnt < 8'd255) begin + out_cnt <= out_cnt + 8'd1; + coeff_out_r <= cr[out_cnt + 8'd1]; + valid_o_r <= 1'b1; + end else begin + out_cnt <= 8'd0; + valid_o_r <= 1'b0; + end + end + + if (state == S_OUT_SCALE) begin + if (scale_issue_cnt < 9'd256) begin + scale_valid_i <= 1'b1; + scale_a_i <= cr[scale_issue_cnt[7:0]]; + scale_issue_cnt <= scale_issue_cnt + 9'd1; + end + + valid_o_r <= 1'b0; + if (scale_valid_o) begin + coeff_out_r <= scale_product; + valid_o_r <= 1'b1; + scale_emit_cnt <= scale_emit_cnt + 9'd1; + end + end + + if (state == S_DONE) begin + load_cnt <= 8'd0; + out_cnt <= 8'd0; + scale_issue_cnt <= 9'd0; + scale_emit_cnt <= 9'd0; + valid_o_r <= 1'b0; + end end end diff --git a/sync_rtl/poly_mul/basecase_mul_pipe.v b/sync_rtl/poly_mul/basecase_mul_pipe.v index d573931..7297b2a 100644 --- a/sync_rtl/poly_mul/basecase_mul_pipe.v +++ b/sync_rtl/poly_mul/basecase_mul_pipe.v @@ -4,7 +4,7 @@ // The shared ntt/barrett_mul remains combinational for NTT users; this module // keeps the extra registers local to poly_mul_sync. -(* use_dsp = "no" *) +(* use_dsp = "yes" *) module basecase_mul_pipe ( input clk, input rst_n, diff --git a/sync_rtl/sha3/sha3_top.v b/sync_rtl/sha3/sha3_top.v index 56411e3..8d22b47 100644 --- a/sync_rtl/sha3/sha3_top.v +++ b/sync_rtl/sha3/sha3_top.v @@ -100,6 +100,16 @@ module sha3_top ( reg mb_last_r; // captured last-block flag reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE) + // ================================================================ + // Keccak core + // ================================================================ + wire kc_valid_i; + /* verilator lint_off UNUSEDSIGNAL */ + wire [1599:0] kc_state_o; + wire kc_ready_o; + /* verilator lint_on UNUSEDSIGNAL */ + wire kc_valid_o; + // XOR the incoming block into the low 1088 bits (rate) of the state. wire [1599:0] mb_xored; assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i}; @@ -119,16 +129,6 @@ module sha3_top ( endcase end - // ================================================================ - // Keccak core - // ================================================================ - wire kc_valid_i; - /* verilator lint_off UNUSEDSIGNAL */ - wire [1599:0] kc_state_o; - wire kc_ready_o; - /* verilator lint_on UNUSEDSIGNAL */ - wire kc_valid_o; - // Keccak input: multi-block xored state when mb_en, else single-block absorb. wire [1599:0] kc_state_i_mux; assign kc_state_i_mux = mb_en ? mb_xored : absorb_state; diff --git a/sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v b/sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v index f208bcb..397e7b7 100644 --- a/sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v +++ b/sync_rtl/top/TB/tb_mlkem_dec_katK_xsim.v @@ -13,6 +13,8 @@ module tb_mlkem_dec_katK_xsim; localparam EKB = 384*KP + 32; // ek_pke bytes within dk localparam DKPB = 384*KP; // dk_pke bytes localparam CTB = (KP==4) ? 1568 : (32*(10*KP+4)); // ct bytes: 768/1088/1568 + localparam integer CLK_PERIOD_NS = 100; // 10 MHz + localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2; reg clk=0, rst_n=0, start_i=0; reg [2:0] k_i; @@ -47,7 +49,7 @@ module tb_mlkem_dec_katK_xsim; .dbg_mprime_o(dbg_mprime_o), .dbg_kbar_o(dbg_kbar_o), .dbg_decz_o(dbg_decz_o), .dbg_dech_o(dbg_dech_o) ); - always #5 clk = ~clk; + always #(CLK_HALF_NS) clk = ~clk; reg [7:0] dk_b [0:DKB-1]; reg [7:0] ct_b [0:CTB-1]; @@ -55,8 +57,72 @@ module tb_mlkem_dec_katK_xsim; reg [7:0] ss_b [0:31]; reg [7:0] ssn_b [0:31]; integer c, i, j, errors, casenum; + integer state_cyc [0:31]; + integer si; reg [8*80-1:0] tag, dkfile, ctfile, ssfile, ctnfile, ssnfile; + function [8*16-1:0] state_name; + input [4:0] s; + begin + case (s) + 5'd0: state_name = "IDLE"; + 5'd1: state_name = "G"; + 5'd2: state_name = "A"; + 5'd3: state_name = "C"; + 5'd4: state_name = "N"; + 5'd5: state_name = "M"; + 5'd6: state_name = "E"; + 5'd7: state_name = "H"; + 5'd8: state_name = "ENC_LOAD"; + 5'd9: state_name = "ENC_H"; + 5'd10: state_name = "ENC_G"; + 5'd11: state_name = "ENC_A"; + 5'd12: state_name = "ENC_TDEC"; + 5'd13: state_name = "ENC_C"; + 5'd14: state_name = "ENC_N"; + 5'd15: state_name = "ENC_U"; + 5'd16: state_name = "ENC_C1"; + 5'd17: state_name = "ENC_V"; + 5'd18: state_name = "ENC_C2"; + 5'd19: state_name = "ENC_E2MV"; + 5'd20: state_name = "DEC_LOAD"; + 5'd21: state_name = "DEC_DECOMP"; + 5'd22: state_name = "DEC_SDEC"; + 5'd23: state_name = "DEC_NTT"; + 5'd24: state_name = "DEC_W"; + 5'd25: state_name = "DEC_MENC"; + 5'd26: state_name = "DEC_G"; + 5'd27: state_name = "DEC_J"; + 5'd28: state_name = "DEC_CMP"; + 5'd31: state_name = "DONE"; + default: state_name = "UNKNOWN"; + endcase + end + endfunction + + task reset_timing; + begin + for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0; + end + endtask + + task print_timing; + input [8*16-1:0] op_name; + input integer cycles; + integer runtime_ns; + begin + runtime_ns = cycles * CLK_PERIOD_NS; + $display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)", + KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns, + runtime_ns/1000, runtime_ns%1000, + runtime_ns/1000000, (runtime_ns%1000000)/1000); + $display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name); + for (si = 0; si < 32; si = si + 1) + if (state_cyc[si] != 0) + $display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS); + end + endtask + initial begin if (!$value$plusargs("CASE=%d", casenum)) casenum = 0; $sformat(tag, "k%0d", KP); @@ -130,9 +196,12 @@ module tb_mlkem_dec_katK_xsim; end c_in_we = 1'b0; @(posedge clk); start_i=1; @(posedge clk); start_i=0; - c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end + #1; + reset_timing; + c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end if(!done_o) begin $display("FAIL K=%0d case %0d: timeout (sel=%0d)", KP, casenum, sel); $finish; end $display("=== Decaps run (sel=%0d) done in %0d cyc ===", sel, c); + print_timing(sel ? "DecapsReject" : "DecapsAccept", c); // settle so dbg taps reflect the finished run repeat(2) @(posedge clk); end diff --git a/sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v b/sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v index f214c6b..9dba17a 100644 --- a/sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v +++ b/sync_rtl/top/TB/tb_mlkem_enc_katK_xsim.v @@ -12,6 +12,8 @@ module tb_mlkem_enc_katK_xsim; localparam CTB = (KP==4) ? 1568 : (32*(10*KP+4)); // ct bytes: K2 768,K3 1088,K4 1568 localparam DU = (KP==4) ? 11 : 10; // compression du localparam C1B = 32*DU*KP; // c1 byte count: K2 640,K3 960,K4 1408 + localparam integer CLK_PERIOD_NS = 100; // 10 MHz + localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2; reg clk=0, rst_n=0, start_i=0; reg [2:0] k_i; @@ -41,15 +43,79 @@ module tb_mlkem_enc_katK_xsim; .dbg_r_o(dbg_r_o), .dbg_hek_o(dbg_hek_o), .dbg_mprime_o(), .dbg_kbar_o(), .dbg_decz_o(), .dbg_dech_o() ); - always #5 clk = ~clk; + always #(CLK_HALF_NS) clk = ~clk; reg [7:0] ek_b [0:EKB-1]; reg [7:0] m_b [0:31]; reg [7:0] ss_b [0:31]; reg [7:0] ct_b [0:CTB-1]; integer c, i, errors, casenum, j; + integer state_cyc [0:31]; + integer si; reg [8*80-1:0] tag, ekfile, mfile, ssfile, ctfile; + function [8*16-1:0] state_name; + input [4:0] s; + begin + case (s) + 5'd0: state_name = "IDLE"; + 5'd1: state_name = "G"; + 5'd2: state_name = "A"; + 5'd3: state_name = "C"; + 5'd4: state_name = "N"; + 5'd5: state_name = "M"; + 5'd6: state_name = "E"; + 5'd7: state_name = "H"; + 5'd8: state_name = "ENC_LOAD"; + 5'd9: state_name = "ENC_H"; + 5'd10: state_name = "ENC_G"; + 5'd11: state_name = "ENC_A"; + 5'd12: state_name = "ENC_TDEC"; + 5'd13: state_name = "ENC_C"; + 5'd14: state_name = "ENC_N"; + 5'd15: state_name = "ENC_U"; + 5'd16: state_name = "ENC_C1"; + 5'd17: state_name = "ENC_V"; + 5'd18: state_name = "ENC_C2"; + 5'd19: state_name = "ENC_E2MV"; + 5'd20: state_name = "DEC_LOAD"; + 5'd21: state_name = "DEC_DECOMP"; + 5'd22: state_name = "DEC_SDEC"; + 5'd23: state_name = "DEC_NTT"; + 5'd24: state_name = "DEC_W"; + 5'd25: state_name = "DEC_MENC"; + 5'd26: state_name = "DEC_G"; + 5'd27: state_name = "DEC_J"; + 5'd28: state_name = "DEC_CMP"; + 5'd31: state_name = "DONE"; + default: state_name = "UNKNOWN"; + endcase + end + endfunction + + task reset_timing; + begin + for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0; + end + endtask + + task print_timing; + input [8*16-1:0] op_name; + input integer cycles; + integer runtime_ns; + begin + runtime_ns = cycles * CLK_PERIOD_NS; + $display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)", + KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns, + runtime_ns/1000, runtime_ns%1000, + runtime_ns/1000000, (runtime_ns%1000000)/1000); + $display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name); + for (si = 0; si < 32; si = si + 1) + if (state_cyc[si] != 0) + $display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS); + end + endtask + initial begin if (!$value$plusargs("CASE=%d", casenum)) casenum = 0; $sformat(tag, "k%0d", KP); @@ -81,9 +147,12 @@ module tb_mlkem_enc_katK_xsim; // ---- run Encaps ---- start_i=1; @(posedge clk); start_i=0; - c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end + #1; + reset_timing; + c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end if(!done_o) begin $display("FAIL K=%0d case %0d: timeout", KP, casenum); $finish; end $display("=== Encaps E0 done in %0d cyc ===", c); + print_timing("Encaps", c); $write(" H(ek) = "); for (j=0;j<32;j=j+1) $write("%02x", dbg_hek_o[8*j +: 8]); $write("\n"); $write(" r = "); for (j=0;j<32;j=j+1) $write("%02x", dbg_r_o[8*j +: 8]); $write("\n"); diff --git a/sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v b/sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v index 6008ff0..3bf83ff 100644 --- a/sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v +++ b/sync_rtl/top/TB/tb_mlkem_hello_world_xsim.v @@ -1,8 +1,8 @@ // tb_mlkem_hello_world_xsim.v - Hardware run of the ML-KEM hello_world demo. // -// Mirrors the Rust example end-to-end on the mlkem_top DUT (ML-KEM-512, K=2): -// 1. Alice: KeyGen(d=0x42..., z=0x77...) -> (ek 800B, dk 1632B) -// 2. Bob: Encaps(ek, m=0xDE...) -> (shared_key, kem_ct 768B) +// Mirrors the Rust example end-to-end on the mlkem_top DUT (default ML-KEM-512, K=2): +// 1. Alice: KeyGen(d=0x42..., z=0x77...) -> (ek, dk) +// 2. Bob: Encaps(ek, m=0xDE...) -> (shared_key, kem_ct) // 3. Bob: XOR-encrypt "hello world" with key -> encrypted // 4. Alice: Decaps(dk, kem_ct) -> recovered_key // 5. Alice: XOR-decrypt encrypted with key -> "hello world" @@ -17,10 +17,12 @@ // xelab tb_mlkem_hello_world_xsim ; xsim `timescale 1ns/1ps module tb_mlkem_hello_world_xsim; - localparam KP = 2; // ML-KEM-512 - localparam EKB = 384*KP + 32; // 800 - localparam DKB = 768*KP + 96; // 1632 - localparam CTB = 32*(10*KP + 4); // 768 + parameter KP = 2; // 2/3/4 = ML-KEM-512/768/1024 + localparam DU = (KP == 4) ? 11 : 10; + localparam DV = (KP == 4) ? 5 : 4; + localparam EKB = 384*KP + 32; + localparam DKB = 768*KP + 96; + localparam CTB = 32*(DU*KP + DV); localparam MLEN = 11; // "hello world" reg clk=0, rst_n=0, start_i=0; @@ -93,7 +95,7 @@ module tb_mlkem_hello_world_xsim; msg_b[0]="h"; msg_b[1]="e"; msg_b[2]="l"; msg_b[3]="l"; msg_b[4]="o"; msg_b[5]=" "; msg_b[6]="w"; msg_b[7]="o"; msg_b[8]="r"; msg_b[9]="l"; msg_b[10]="d"; - $display("=== ML-KEM hello_world (hardware, ML-KEM-512) ==="); + $display("=== ML-KEM hello_world (hardware, K=%0d) ===", KP); $write("Original: \""); for (i = 0; i < MLEN; i = i + 1) $write("%c", msg_b[i]); $display("\"\n"); @@ -189,7 +191,7 @@ module tb_mlkem_hello_world_xsim; if (dec_b[i] !== msg_b[i]) errors = errors + 1; if (errors == 0) - $display("Success: keys match, message recovered. (hardware ML-KEM-512)"); + $display("Success: keys match, message recovered. (hardware K=%0d)", KP); else $display("FAILURE: %0d mismatches", errors); $finish; diff --git a/sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v b/sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v index cf0ac64..70e2b28 100644 --- a/sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v +++ b/sync_rtl/top/TB/tb_mlkem_kg_katK_xsim.v @@ -7,6 +7,8 @@ module tb_mlkem_kg_katK_xsim; parameter KP = 2; localparam EKB = 384*KP + 32; localparam DKB = 768*KP + 96; + localparam integer CLK_PERIOD_NS = 100; // 10 MHz + localparam integer CLK_HALF_NS = CLK_PERIOD_NS / 2; reg clk=0, rst_n=0, start_i=0; reg [2:0] k_i; @@ -33,7 +35,7 @@ module tb_mlkem_kg_katK_xsim; .dbg_r_o(), .dbg_hek_o(), .dbg_mprime_o(), .dbg_kbar_o(), .dbg_decz_o(), .dbg_dech_o() ); - always #5 clk = ~clk; + always #(CLK_HALF_NS) clk = ~clk; reg [255:0] dmem [0:0]; reg [255:0] zmem [0:0]; @@ -42,8 +44,72 @@ module tb_mlkem_kg_katK_xsim; reg [7:0] ek_got [0:EKB-1]; // ek bytes read back from DUT reg [7:0] dk_got [0:DKB-1]; // dk bytes read back from DUT integer c, i, errors, casenum, j; + integer state_cyc [0:31]; + integer si; reg [8*80-1:0] tag, dfile, zfile, ekfile, dkfile; + function [8*16-1:0] state_name; + input [4:0] s; + begin + case (s) + 5'd0: state_name = "IDLE"; + 5'd1: state_name = "G"; + 5'd2: state_name = "A"; + 5'd3: state_name = "C"; + 5'd4: state_name = "N"; + 5'd5: state_name = "M"; + 5'd6: state_name = "E"; + 5'd7: state_name = "H"; + 5'd8: state_name = "ENC_LOAD"; + 5'd9: state_name = "ENC_H"; + 5'd10: state_name = "ENC_G"; + 5'd11: state_name = "ENC_A"; + 5'd12: state_name = "ENC_TDEC"; + 5'd13: state_name = "ENC_C"; + 5'd14: state_name = "ENC_N"; + 5'd15: state_name = "ENC_U"; + 5'd16: state_name = "ENC_C1"; + 5'd17: state_name = "ENC_V"; + 5'd18: state_name = "ENC_C2"; + 5'd19: state_name = "ENC_E2MV"; + 5'd20: state_name = "DEC_LOAD"; + 5'd21: state_name = "DEC_DECOMP"; + 5'd22: state_name = "DEC_SDEC"; + 5'd23: state_name = "DEC_NTT"; + 5'd24: state_name = "DEC_W"; + 5'd25: state_name = "DEC_MENC"; + 5'd26: state_name = "DEC_G"; + 5'd27: state_name = "DEC_J"; + 5'd28: state_name = "DEC_CMP"; + 5'd31: state_name = "DONE"; + default: state_name = "UNKNOWN"; + endcase + end + endfunction + + task reset_timing; + begin + for (si = 0; si < 32; si = si + 1) state_cyc[si] = 0; + end + endtask + + task print_timing; + input [8*16-1:0] op_name; + input integer cycles; + integer runtime_ns; + begin + runtime_ns = cycles * CLK_PERIOD_NS; + $display("TIME K=%0d CASE=%0d OP=%0s cycles=%0d clk=10MHz period=%0dns runtime=%0d ns (%0d.%03d us, %0d.%03d ms)", + KP, casenum, op_name, cycles, CLK_PERIOD_NS, runtime_ns, + runtime_ns/1000, runtime_ns%1000, + runtime_ns/1000000, (runtime_ns%1000000)/1000); + $display("TIME_BREAKDOWN K=%0d CASE=%0d OP=%0s", KP, casenum, op_name); + for (si = 0; si < 32; si = si + 1) + if (state_cyc[si] != 0) + $display(" STATE %-12s cycles=%0d time_ns=%0d", state_name(si[4:0]), state_cyc[si], state_cyc[si]*CLK_PERIOD_NS); + end + endtask + // Dump a byte array as offset-prefixed hex, 32 bytes/line. task dump_bytes(input [8*16-1:0] name, input integer n); integer a, b; @@ -80,9 +146,12 @@ module tb_mlkem_kg_katK_xsim; rst_n=0; repeat(4) @(posedge clk); rst_n=1; @(posedge clk); start_i=1; @(posedge clk); start_i=0; - c=0; while(!done_o && c<2000000) begin @(posedge clk); c=c+1; end + #1; + reset_timing; + c=0; while(!done_o && c<2000000) begin state_cyc[dut.st] = state_cyc[dut.st] + 1; @(posedge clk); c=c+1; end if(!done_o) begin $display("FAIL K=%0d case %0d: timeout", KP, casenum); $finish; end $display("=== ML-KEM K=%0d KAT case %0d: KeyGen done in %0d cyc ===", KP, casenum, c); + print_timing("KeyGen", c); errors = 0; dbg_byte_sel_i = 1'b0; diff --git a/sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v b/sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v index 3f19a44..476371c 100644 --- a/sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v +++ b/sync_rtl/top/TB/tb_mlkem_two_inst_xsim.v @@ -14,10 +14,12 @@ // xelab tb_mlkem_two_inst_xsim ; xsim `timescale 1ns/1ps module tb_mlkem_two_inst_xsim; - localparam KP = 2; // ML-KEM-512 - localparam EKB = 384*KP + 32; // 800 - localparam DKB = 768*KP + 96; // 1632 - localparam CTB = 32*(10*KP + 4); // 768 + parameter KP = 2; // 2/3/4 = ML-KEM-512/768/1024 + localparam DU = (KP == 4) ? 11 : 10; + localparam DV = (KP == 4) ? 5 : 4; + localparam EKB = 384*KP + 32; + localparam DKB = 768*KP + 96; + localparam CTB = 32*(DU*KP + DV); localparam MLEN = 11; // "hello world" reg clk=0; @@ -100,7 +102,7 @@ module tb_mlkem_two_inst_xsim; msg_b[0]="h"; msg_b[1]="e"; msg_b[2]="l"; msg_b[3]="l"; msg_b[4]="o"; msg_b[5]=" "; msg_b[6]="w"; msg_b[7]="o"; msg_b[8]="r"; msg_b[9]="l"; msg_b[10]="d"; - $display("=== ML-KEM hello_world (TWO instances: genenc + dec, ML-KEM-512) ==="); + $display("=== ML-KEM hello_world (TWO instances: genenc + dec, K=%0d) ===", KP); $write("Original: \""); for(i=0;iINTT->SUB, then D4 encodes m'. ST_DEC_W: if (u_row >= 3'd1) st_next = ST_DEC_MENC; - ST_DEC_MENC: if (men_done) st_next = ST_DEC_G; - // D5: (K',r') = G(m'||h) single-block, then K-bar = J(z||c) multi-block. - ST_DEC_G: if (sha3_vo) st_next = ST_DEC_J; - // D5 J done -> D6 re-encrypt: c' = K-PKE.Encrypt(ek_pke, m', r'). - // Reuse the entire Encaps pipeline (rho load -> A -> C -> ... -> C2). - // r' is in r_r (CBD seed), m' in m_r (V/mu), ek_pke in ek_bram. + ST_DEC_MENC: if (men_done && dj_done) st_next = ST_DEC_G; + // D5: (K',r') = G(m'||h) single-block. K-bar = J(z||c) runs in a + // background SHAKE256 walker after D1, so the main FSM only waits + // for it at the D4->D5 boundary if it somehow has not finished. + ST_DEC_G: if (sha3_vo) st_next = ST_ENC_LOAD; + // Legacy state kept unused; background J normally skips it. ST_DEC_J: if (dj_done) st_next = ST_ENC_LOAD; ST_G: if (sha3_vo) st_next = ST_A; ST_A: if (a_pair >= kk_rt) st_next = ST_C; @@ -1303,6 +1304,7 @@ module mlkem_top #( cmp_neq <= 1'b0; cmp_done <= 1'b0; dec_reject <= 1'b0; + dj_active <= 1'b0; dj_blk <= 4'd0; dj_byte <= 8'd0; dj_phase <= 2'd0; @@ -1349,6 +1351,8 @@ module mlkem_top #( end else if (op_i == 2'd2) begin // Decaps: dk_pke/ek_pke already in BRAM; z/H(ek) captured into // z_r/hek_r during load (below). Nothing else to arm in D0. + dj_active <= 1'b0; + dj_done <= 1'b0; end else begin sha3_valid <= 1'b1; sha3_ack <= 1'b1; @@ -1993,9 +1997,11 @@ module mlkem_top #( sha3_ack <= 1'b0; end - // Arm D5 J(z||c) when G completes (ST_DEC_G -> ST_DEC_J): assemble the - // first 136-byte block. dj_ack high to consume the final digest. - if (st == ST_DEC_G && st_next == ST_DEC_J) begin + // Arm D5 J(z||c) as soon as D1 has stopped reading c_in_bram. It + // then runs in the background during D2/D3/D4 using the otherwise + // idle SHAKE256 path and c_in read port. + if (st == ST_DEC_DECOMP && st_next == ST_DEC_SDEC) begin + dj_active <= 1'b1; dj_blk <= 4'd0; dj_byte <= 8'd0; dj_phase <= 2'd0; @@ -2007,14 +2013,14 @@ module mlkem_top #( cin_rd_addr_r <= 11'd0; // present c byte 0 (g=32 -> c_idx 0) end - // ---- ST_DEC_J: K-bar = J(z||c) multi-block SHAKE256 ---- + // ---- Background Decaps J: K-bar = J(z||c) multi-block SHAKE256 ---- // Mirror of the H(ek) multi-block machine. Byte source by global g: // g < 32 -> z_r byte g // 32 <= g < msglen -> c_in_bram byte (g-32), registered read // else -> SHAKE256 pad constant // c_in_bram read is registered: present c_idx for dj_byte this cycle, // write back the byte that arrived for the addr presented last cycle. - if (st == ST_DEC_J && !dj_done) begin + if (dj_active && !dj_done) begin case (dj_phase) 2'd0: begin // writeback the byte read for the previous address @@ -2034,7 +2040,7 @@ module mlkem_top #( dj_wb_zidx <= dj_g[4:0]; dj_wb_pad <= dj_padconst(dj_g); // c_in addr is presented combinationally (cin_rd_addr - // mux uses dj_c_idx during ST_DEC_J); data lands next + // mux uses dj_c_idx while dj_active); data lands next // cycle, matching this byte's writeback. dj_byte <= dj_byte + 8'd1; end else begin @@ -2056,6 +2062,7 @@ module mlkem_top #( if (sha3_vo) begin kbar_r <= sha3_hash[255:0]; dj_done <= 1'b1; + dj_active <= 1'b0; dj_phase<= 2'd3; end else if (h_mbready) begin dj_blk <= dj_blk + 4'd1; @@ -2095,7 +2102,7 @@ module mlkem_top #( end if (st_next == ST_ENC_LOAD && - (st == ST_ENC_G || st == ST_DEC_J)) begin + (st == ST_ENC_G || st == ST_DEC_G || st == ST_DEC_J)) begin rl_idx <= 6'd0; rl_widx <= 6'd0; rl_vld <= 1'b0; diff --git a/synth_timing.tcl b/synth_timing.tcl index d5747cd..2a9e82a 100644 --- a/synth_timing.tcl +++ b/synth_timing.tcl @@ -10,8 +10,10 @@ read_verilog -sv ${PROJECT_DIR}/sync_rtl/sha3/sha3_top_shared.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_ntt/sample_ntt_sync_shared.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/sample_cbd/sample_cbd_sync_shared.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul.v +read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/barrett_mul_pipe.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/zeta_rom.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit.v +read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/butterfly_unit_pipe.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/ntt/ntt_core.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/basecase_mul.v read_verilog -sv ${PROJECT_DIR}/sync_rtl/poly_mul/basecase_mul_pipe.v