diff --git a/rtl/ip/mlkem/common/defines.vh b/rtl/ip/mlkem/common/defines.vh new file mode 100644 index 0000000..1431644 --- /dev/null +++ b/rtl/ip/mlkem/common/defines.vh @@ -0,0 +1,6 @@ +`ifndef DEFINES_VH +`define DEFINES_VH +`define CLK_PERIOD 20.0 // 50MHz +`define Q 3329 // ML-KEM prime modulus +`define N 256 // polynomial degree +`endif diff --git a/rtl/ip/mlkem/common/pipeline_reg.v b/rtl/ip/mlkem/common/pipeline_reg.v new file mode 100644 index 0000000..ba6b5ea --- /dev/null +++ b/rtl/ip/mlkem/common/pipeline_reg.v @@ -0,0 +1,53 @@ +// pipeline_reg.v - Single pipeline stage with valid/ready handshake +// +// valid_i && ready_o -> register data -> valid_o next cycle. +// ready_o = !valid_o || ready_i. +// +// Parameters: +// DW = data width (default 12) +// +// Interface: +// clk, rst_n - clock, active-low reset +// data_i, valid_i, ready_o - input side +// data_o, valid_o, ready_i - output side + +module pipeline_reg #(parameter DW = 12) ( + input clk, + input rst_n, + input [DW-1:0] data_i, + input valid_i, + output ready_o, + output [DW-1:0] data_o, + output valid_o, + input ready_i +); + + reg valid_r; + reg [DW-1:0] data_r; + + // ready_o: accept new data when output slot is free + assign ready_o = !valid_r || ready_i; + + // Drive outputs from internal register + assign valid_o = valid_r; + assign data_o = data_r; + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + valid_r <= 1'b0; + data_r <= {DW{1'b0}}; + end else begin + if (valid_r && ready_i) begin + // Output consumed, clear valid + valid_r <= 1'b0; + end + + if (valid_i && ready_o) begin + // Capture new data + data_r <= data_i; + valid_r <= 1'b1; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/common/skid_buffer.v b/rtl/ip/mlkem/common/skid_buffer.v new file mode 100644 index 0000000..791725a --- /dev/null +++ b/rtl/ip/mlkem/common/skid_buffer.v @@ -0,0 +1,73 @@ +// skid_buffer.v - 2-entry skid buffer with valid/ready handshake +// +// When valid_i && ready_o, capture data_i. Output via valid_o && ready_i. +// Stores one extra entry in skid register. Pure combinational ready_o path. +// +// Parameters: +// DW = data width (default 12) +// +// Interface: +// clk, rst_n - clock, active-low reset +// data_i, valid_i, ready_o - input side +// data_o, valid_o, ready_i - output side + +module skid_buffer #(parameter DW = 12) ( + input clk, + input rst_n, + input [DW-1:0] data_i, + input valid_i, + output ready_o, + output [DW-1:0] data_o, + output valid_o, + input ready_i +); + + // Internal state + reg skid_valid; // skid register has valid data + reg [DW-1:0] skid_data; // skid register data + + // Output register (pipeline stage) + reg out_valid; + reg [DW-1:0] out_data; + + // ready_o is combinational: ready when output slot is empty + // (either out_valid is 0, or ready_i is high and we can accept new data) + assign ready_o = !out_valid || ready_i; + + // valid_o drives from output register + assign valid_o = out_valid; + assign data_o = out_data; + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + skid_valid <= 1'b0; + skid_data <= {DW{1'b0}}; + out_valid <= 1'b0; + out_data <= {DW{1'b0}}; + end else begin + if (out_valid && ready_i) begin + // Output consumed: load from skid if available + if (skid_valid) begin + out_data <= skid_data; + out_valid <= 1'b1; + skid_valid <= 1'b0; + end else begin + out_valid <= 1'b0; + end + end + + if (valid_i && ready_o) begin + if (!out_valid || (out_valid && ready_i && !skid_valid)) begin + // Output slot free: go directly to output + out_data <= data_i; + out_valid <= 1'b1; + end else begin + // Output slot occupied: store in skid + skid_data <= data_i; + skid_valid <= 1'b1; + end + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v new file mode 100644 index 0000000..629fb4a --- /dev/null +++ b/rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v @@ -0,0 +1,96 @@ +// comp_decomp_sync.v - ML-KEM coefficient compression/decompression +// +// Streaming: one coefficient per cycle through pipeline_reg. +// mode=0: compress - round((2^d * x) / Q) mod 2^d +// mode=1: decompress - round((Q * x) / 2^d) mod Q +// Uses round-half-up (round(2.5)=3, round(3.5)=4). +// Integer arithmetic: +// compress: (x * 2^d + Q/2) / Q, lower d bits as result +// decompress: (x * Q + 2^(d-1)) / 2^d, result mod Q +// +// Division by Q=3329 replaced with Barrett multiplication + correction: +// floor(x / Q) ≈ (x * 5039) >> 24 (5039 = floor(2^24 / 3329)) +// Correction: if x - q_approx * Q >= Q, add 1 to quotient. +// Q*5039 = 16,774,831 < 2^24, so Barrett may under-estimate by 1. +// Q = 3329 = 2048+512+256+1, so q*Q uses 3 shifts + add (no 2nd multiplier). + +`include "defines.vh" + +(* use_dsp = "no" *) +module comp_decomp_sync ( + input clk, + input rst_n, + input [11:0] coeff_in, + input [4:0] d, + input mode, // 0=compress, 1=decompress + input valid_i, + output ready_o, + output [11:0] coeff_out, + output valid_o, + input ready_i +); + + wire [11:0] two_pow_d = 12'd1 << d; + + // product: 12b*12b = max 2048*3328 = 6,815,744 → 23b, pad to 24b + wire [23:0] product = mode ? {12'b0, coeff_in} * {12'b0, 12'd3329} + : {12'b0, coeff_in} * {12'b0, two_pow_d}; + + wire [11:0] round_off = mode ? (two_pow_d >> 1) : 12'd1664; + + wire [23:0] dividend = product + {12'b0, round_off}; + + // =========================================================== + // Barrett: floor(x / Q) ≈ (x * 5039) >> 24 + // M = floor(2^24 / Q) = 5039 + // M*Q = 16,774,831 < 2^24 → floor(x/Q) may be 1 too low + // Correction: remainder = x - q_approx*Q; if rem >= Q, q→q+1 + // Q*5039 fails at: k*2385 >= 2^24 → k >= 7033; max k ≈ 2047 → safe + // =========================================================== + wire [36:0] barrett_prod = dividend * 13'd5039; // 24×13=37b + wire [11:0] q_approx = barrett_prod[35:24]; // quotient estimate + // q*Q = q*2048 + q*512 + q*256 + q (3329 = 2048+512+256+1) + wire [23:0] q_times_Q = ({q_approx, 11'd0}) + + ({q_approx, 9'd0}) + + ({q_approx, 8'd0}) + + q_approx; + wire [24:0] delta = {1'b0, dividend} - {1'b0, q_times_Q}; // 25b + wire [11:0] comp_q = q_approx + (delta >= {13'b0, 12'd3329}); + // lower d bits = compressed result + wire [11:0] comp_res = comp_q & ({12'b0, two_pow_d} - 1'b1); + + // =========================================================== + // Decompress: dividend >> d (free), then % Q via Barrett + // =========================================================== + wire [23:0] dec_sh = dividend >> d; + wire [36:0] dm_prod = dec_sh * 13'd5039; + wire [11:0] dm_qe = dm_prod[35:24]; + wire [23:0] dm_qQ = ({dm_qe, 11'd0}) + + ({dm_qe, 9'd0}) + + ({dm_qe, 8'd0}) + + dm_qe; + wire [24:0] dm_del = {1'b0, dec_sh} - {1'b0, dm_qQ}; + // divide by Q, take remainder: r = x - q*Q + wire [12:0] dm_q = dm_qe + (dm_del >= {13'b0, 12'd3329}); // corr quotient + // actual remainder = dec_sh - dm_q*Q + wire [23:0] dm_corrQ = ({dm_q, 11'd0}) + ({dm_q, 9'd0}) + ({dm_q, 8'd0}) + dm_q; + wire [24:0] dm_rem = {1'b0, dec_sh} - {1'b0, dm_corrQ}; + // dm_rem < Q*2 (Barrett gives exact quotient after correction), one cond-sub + wire [11:0] dec_mod = (dm_rem >= {13'b0, 12'd3329}) + ? (dm_rem[11:0] - 12'd3329) + : dm_rem[11:0]; + + wire [11:0] mod_result = mode ? dec_mod : comp_res; + + pipeline_reg #(.DW(12)) u_pipe ( + .clk (clk), + .rst_n (rst_n), + .data_i (mod_result), + .valid_i(valid_i), + .ready_o(ready_o), + .data_o (coeff_out), + .valid_o(valid_o), + .ready_i(ready_i) + ); + +endmodule diff --git a/rtl/ip/mlkem/ntt/barrett_mul.v b/rtl/ip/mlkem/ntt/barrett_mul.v new file mode 100644 index 0000000..6d5249a --- /dev/null +++ b/rtl/ip/mlkem/ntt/barrett_mul.v @@ -0,0 +1,54 @@ +// barrett_mul.v - Barrett modular multiplication (a * b mod Q) +// +// Computes product = a * b mod Q using Barrett reduction. +// Q = 3329, k = floor(2^24 / Q) = 5039 +// +// Pure combinational, single-cycle latency. +// All multiplication widths explicitly controlled to avoid Verilog truncation. + +(* use_dsp = "no" *) +module barrett_mul ( + input [11:0] a, + input [11:0] b, + output [11:0] product +); + localparam Q = 3329; + localparam K = 5039; // floor(2^24 / 3329) + localparam R = 24; + + // Full product: a * b (both < 3329, product < 11,082,241 < 2^24) + // Force 24-bit evaluation by extending operand + wire [23:0] p; + assign p = {12'd0, a} * b; + + // Extend p to 37 bits for multiplication with K + wire [36:0] p_ext; + assign p_ext = {13'd0, p}; + + // Compute t_shifted = (p * K) >> 24 + // Use explicit wire for the product to control width + /* verilator lint_off UNUSEDSIGNAL */ + wire [36:0] t_product; + /* verilator lint_on UNUSEDSIGNAL */ + assign t_product = p_ext * K; + wire [12:0] t_shifted; + assign t_shifted = t_product[36:R]; + + // q_approx = t_shifted * Q + wire [24:0] q_approx; + assign q_approx = t_shifted * Q; + + // r = p - q_approx + wire [24:0] r0; + assign r0 = {1'b0, p} - q_approx; + + // Conditional subtract Q (at most twice) + wire [24:0] r1; + assign r1 = (r0 >= Q) ? (r0 - Q) : r0; + + wire [11:0] r2; + assign r2 = (r1[11:0] >= Q) ? (r1[11:0] - Q) : r1[11:0]; + + assign product = (r1 >= Q) ? r2 : r1[11:0]; + +endmodule diff --git a/rtl/ip/mlkem/ntt/butterfly_unit.v b/rtl/ip/mlkem/ntt/butterfly_unit.v new file mode 100644 index 0000000..8f616a8 --- /dev/null +++ b/rtl/ip/mlkem/ntt/butterfly_unit.v @@ -0,0 +1,67 @@ +// butterfly_unit.v - Cooley-Tukey / Gentleman-Sande butterfly +// +// Computes one butterfly operation for NTT or inverse NTT. +// +// Parameters: +// Q = 3329 (prime modulus) +// +// Forward NTT (mode=0): +// t = zeta * b mod Q (via barrett_mul) +// a_out = (a + t) mod Q +// b_out = (a - t) mod Q +// +// Inverse NTT (mode=1): +// a_out = (a + b) mod Q +// diff = (b - a) mod Q (handled as: if b >= a: b-a; else: b-a+Q) +// b_out = zeta * diff mod Q (via barrett_mul) +// +// Pure combinational. + +module butterfly_unit ( + input [11:0] a, + input [11:0] b, + input [11:0] zeta, + input mode, // 0 = forward NTT, 1 = inverse NTT + output [11:0] a_out, + output [11:0] b_out +); + localparam Q = 3329; + + // Barrett modular multiplication: zeta * mul_data mod Q + wire [11:0] mul_data; // what to multiply with zeta + wire [11:0] mul_result; // result of barrett_mul(zeta, mul_data) + + // Forward: mul_data = b, t = zeta * b mod Q + // Inverse: mul_data = (b - a) mod Q positive + assign mul_data = (mode == 1'b0) ? b : ((b >= a) ? (b - a) : (b - a + Q)); + + barrett_mul u_barrett ( + .a (zeta), + .b (mul_data), + .product (mul_result) + ); + + // ---- a_out computation ---- + // Forward: a_out = (a + t) mod Q + // Inverse: a_out = (a + b) mod Q + wire [12:0] a_sum; + wire [11:0] add_val; + assign add_val = (mode == 1'b0) ? mul_result : b; + assign a_sum = {1'b0, a} + {1'b0, add_val}; + // a_sum - Q produces 13-bit result; we only need lower 12 bits since + // a_sum >= Q guarantees the result < Q < 2^12 + wire [11:0] a_sub_12; + assign a_sub_12 = a_sum[11:0] - Q[11:0]; + wire [11:0] a_result = (a_sum >= Q) ? a_sub_12 : a_sum[11:0]; + assign a_out = a_result; + + // ---- b_out computation ---- + // Forward: b_out = (a - t) mod Q → if a >= t: a-t; else: a-t+Q + // Inverse: b_out = t (mul_result, which is zeta * (b-a) mod Q) + wire [11:0] sub_val; + assign sub_val = mul_result; + wire [11:0] sub_result; + assign sub_result = (a >= sub_val) ? (a - sub_val) : (a - sub_val + Q); + assign b_out = (mode == 1'b0) ? sub_result : mul_result; + +endmodule diff --git a/rtl/ip/mlkem/ntt/ntt_core.v b/rtl/ip/mlkem/ntt/ntt_core.v new file mode 100644 index 0000000..f5146ce --- /dev/null +++ b/rtl/ip/mlkem/ntt/ntt_core.v @@ -0,0 +1,132 @@ +// ntt_core.v - NTT core with individual coefficient registers +// +// Uses 256 individual 12-bit registers and generate-based muxing +// to avoid any part-select simulation issues. +// 3-cycle butterfly: SetAddr -> Read -> Compute+Write + +module ntt_core ( + input clk, rst_n, + input [11:0] coeff_in, + input valid_i, + output ready_o, + input mode, + output [11:0] coeff_out, + output valid_o, + input ready_i, + output done_o +); + localparam N = 256, LAYERS = 7, DW = 12; + + // Individual coefficient registers + reg [DW-1:0] cr [0:N-1]; + integer ci; + + // State machine + localparam S_IDLE=3'd0, S_LOAD=3'd1, S_CMP_A=3'd2, S_CMP_B=3'd3, + S_CMP_C=3'd4, S_OUTPUT=3'd5, S_DONE=3'd6; + reg [2:0] state, next_state; + + reg [7:0] load_cnt, out_cnt; + reg [7:0] j, start, layer_len; + reg [6:0] zeta_idx; + reg [2:0] layer; + reg bf_done; + + // Pipeline registers + reg [DW-1:0] r_a, r_b; + reg [7:0] r_wa, r_wb; + + // Zeta + wire [DW-1:0] zeta; + zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); + + // Butterfly + wire [DW-1:0] bf_a_out, bf_b_out; + butterfly_unit u_bf ( + .a(r_a), .b(r_b), .zeta(zeta), .mode(mode), + .a_out(bf_a_out), .b_out(bf_b_out)); + + // Output scaling + wire [DW-1:0] coeff_scaled; + barrett_mul u_scl (.a(cr[out_cnt]), .b(12'd3303), .product(coeff_scaled)); + assign coeff_out = mode ? coeff_scaled : cr[out_cnt]; + + + assign ready_o = (state == S_IDLE) || (state == S_LOAD); + assign valid_o = (state == S_OUTPUT); + assign done_o = (state == S_DONE); + + always @* begin + next_state = state; + case (state) + S_IDLE: if (valid_i) next_state = S_LOAD; + S_LOAD: if (load_cnt >= 255 && valid_i) next_state = S_CMP_A; + S_CMP_A: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_B; + S_CMP_B: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_C; + S_CMP_C: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_A; + S_OUTPUT:if (out_cnt >= 255 && ready_i) next_state = S_DONE; + S_DONE: next_state = S_IDLE; + default: next_state = S_IDLE; + endcase + end + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state<=S_IDLE; load_cnt<=0; out_cnt<=0; j<=0; start<=0; layer_len<=0; + zeta_idx<=0; layer<=0; bf_done<=0; r_a<=0; r_b<=0; r_wa<=0; r_wb<=0; + for (ci=0; ci= start + layer_len) begin + if (!mode) zeta_idx <= zeta_idx + 7'd1; + else zeta_idx <= zeta_idx - 7'd1; + + if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 256) begin + layer <= layer + 3'd1; + layer_len <= mode ? (layer_len<<1) : (layer_len>>1); + start <= 0; j <= 0; + if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1; + end else begin + start <= start + layer_len + layer_len; + j <= start + layer_len + layer_len; + end + end + end + + if (state == S_OUTPUT && ready_i) + out_cnt <= (out_cnt>=255) ? 0 : (out_cnt+8'd1); + end + end + +endmodule diff --git a/rtl/ip/mlkem/ntt/ntt_sync.v b/rtl/ip/mlkem/ntt/ntt_sync.v new file mode 100644 index 0000000..90f2329 --- /dev/null +++ b/rtl/ip/mlkem/ntt/ntt_sync.v @@ -0,0 +1,19 @@ +module ntt_sync ( + input clk, rst_n, + input [11:0] coeff_in, + input valid_i, + output ready_o, + output [11:0] coeff_out, + output valid_o, + input ready_i, + input mode, + output done_o +); + ntt_core u_ntt_core ( + .clk(clk), .rst_n(rst_n), + .coeff_in(coeff_in), .valid_i(valid_i), .ready_o(ready_o), + .mode(mode), + .coeff_out(coeff_out), .valid_o(valid_o), .ready_i(ready_i), + .done_o(done_o) + ); +endmodule diff --git a/rtl/ip/mlkem/ntt/zeta_rom.v b/rtl/ip/mlkem/ntt/zeta_rom.v new file mode 100644 index 0000000..09eeafd --- /dev/null +++ b/rtl/ip/mlkem/ntt/zeta_rom.v @@ -0,0 +1,141 @@ +// zeta_rom.v - Precomputed Zeta ROM for ML-KEM NTT +// +// Stores 128 bit-reversed roots of unity for Z_q (q=3329). +// Address 0..127, each entry is a 12-bit value. +// Pure combinational, single continuous assignment. + +module zeta_rom ( + input [6:0] addr, // 0..127 + output [11:0] zeta +); + assign zeta = + (addr == 7'd0) ? 12'd1 : + (addr == 7'd1) ? 12'd1729 : + (addr == 7'd2) ? 12'd2580 : + (addr == 7'd3) ? 12'd3289 : + (addr == 7'd4) ? 12'd2642 : + (addr == 7'd5) ? 12'd630 : + (addr == 7'd6) ? 12'd1897 : + (addr == 7'd7) ? 12'd848 : + (addr == 7'd8) ? 12'd1062 : + (addr == 7'd9) ? 12'd1919 : + (addr == 7'd10) ? 12'd193 : + (addr == 7'd11) ? 12'd797 : + (addr == 7'd12) ? 12'd2786 : + (addr == 7'd13) ? 12'd3260 : + (addr == 7'd14) ? 12'd569 : + (addr == 7'd15) ? 12'd1746 : + (addr == 7'd16) ? 12'd296 : + (addr == 7'd17) ? 12'd2447 : + (addr == 7'd18) ? 12'd1339 : + (addr == 7'd19) ? 12'd1476 : + (addr == 7'd20) ? 12'd3046 : + (addr == 7'd21) ? 12'd56 : + (addr == 7'd22) ? 12'd2240 : + (addr == 7'd23) ? 12'd1333 : + (addr == 7'd24) ? 12'd1426 : + (addr == 7'd25) ? 12'd2094 : + (addr == 7'd26) ? 12'd535 : + (addr == 7'd27) ? 12'd2882 : + (addr == 7'd28) ? 12'd2393 : + (addr == 7'd29) ? 12'd2879 : + (addr == 7'd30) ? 12'd1974 : + (addr == 7'd31) ? 12'd821 : + (addr == 7'd32) ? 12'd289 : + (addr == 7'd33) ? 12'd331 : + (addr == 7'd34) ? 12'd3253 : + (addr == 7'd35) ? 12'd1756 : + (addr == 7'd36) ? 12'd1197 : + (addr == 7'd37) ? 12'd2304 : + (addr == 7'd38) ? 12'd2277 : + (addr == 7'd39) ? 12'd2055 : + (addr == 7'd40) ? 12'd650 : + (addr == 7'd41) ? 12'd1977 : + (addr == 7'd42) ? 12'd2513 : + (addr == 7'd43) ? 12'd632 : + (addr == 7'd44) ? 12'd2865 : + (addr == 7'd45) ? 12'd33 : + (addr == 7'd46) ? 12'd1320 : + (addr == 7'd47) ? 12'd1915 : + (addr == 7'd48) ? 12'd2319 : + (addr == 7'd49) ? 12'd1435 : + (addr == 7'd50) ? 12'd807 : + (addr == 7'd51) ? 12'd452 : + (addr == 7'd52) ? 12'd1438 : + (addr == 7'd53) ? 12'd2868 : + (addr == 7'd54) ? 12'd1534 : + (addr == 7'd55) ? 12'd2402 : + (addr == 7'd56) ? 12'd2647 : + (addr == 7'd57) ? 12'd2617 : + (addr == 7'd58) ? 12'd1481 : + (addr == 7'd59) ? 12'd648 : + (addr == 7'd60) ? 12'd2474 : + (addr == 7'd61) ? 12'd3110 : + (addr == 7'd62) ? 12'd1227 : + (addr == 7'd63) ? 12'd910 : + (addr == 7'd64) ? 12'd17 : + (addr == 7'd65) ? 12'd2761 : + (addr == 7'd66) ? 12'd583 : + (addr == 7'd67) ? 12'd2649 : + (addr == 7'd68) ? 12'd1637 : + (addr == 7'd69) ? 12'd723 : + (addr == 7'd70) ? 12'd2288 : + (addr == 7'd71) ? 12'd1100 : + (addr == 7'd72) ? 12'd1409 : + (addr == 7'd73) ? 12'd2662 : + (addr == 7'd74) ? 12'd3281 : + (addr == 7'd75) ? 12'd233 : + (addr == 7'd76) ? 12'd756 : + (addr == 7'd77) ? 12'd2156 : + (addr == 7'd78) ? 12'd3015 : + (addr == 7'd79) ? 12'd3050 : + (addr == 7'd80) ? 12'd1703 : + (addr == 7'd81) ? 12'd1651 : + (addr == 7'd82) ? 12'd2789 : + (addr == 7'd83) ? 12'd1789 : + (addr == 7'd84) ? 12'd1847 : + (addr == 7'd85) ? 12'd952 : + (addr == 7'd86) ? 12'd1461 : + (addr == 7'd87) ? 12'd2687 : + (addr == 7'd88) ? 12'd939 : + (addr == 7'd89) ? 12'd2308 : + (addr == 7'd90) ? 12'd2437 : + (addr == 7'd91) ? 12'd2388 : + (addr == 7'd92) ? 12'd733 : + (addr == 7'd93) ? 12'd2337 : + (addr == 7'd94) ? 12'd268 : + (addr == 7'd95) ? 12'd641 : + (addr == 7'd96) ? 12'd1584 : + (addr == 7'd97) ? 12'd2298 : + (addr == 7'd98) ? 12'd2037 : + (addr == 7'd99) ? 12'd3220 : + (addr == 7'd100) ? 12'd375 : + (addr == 7'd101) ? 12'd2549 : + (addr == 7'd102) ? 12'd2090 : + (addr == 7'd103) ? 12'd1645 : + (addr == 7'd104) ? 12'd1063 : + (addr == 7'd105) ? 12'd319 : + (addr == 7'd106) ? 12'd2773 : + (addr == 7'd107) ? 12'd757 : + (addr == 7'd108) ? 12'd2099 : + (addr == 7'd109) ? 12'd561 : + (addr == 7'd110) ? 12'd2466 : + (addr == 7'd111) ? 12'd2594 : + (addr == 7'd112) ? 12'd2804 : + (addr == 7'd113) ? 12'd1092 : + (addr == 7'd114) ? 12'd403 : + (addr == 7'd115) ? 12'd1026 : + (addr == 7'd116) ? 12'd1143 : + (addr == 7'd117) ? 12'd2150 : + (addr == 7'd118) ? 12'd2775 : + (addr == 7'd119) ? 12'd886 : + (addr == 7'd120) ? 12'd1722 : + (addr == 7'd121) ? 12'd1212 : + (addr == 7'd122) ? 12'd1874 : + (addr == 7'd123) ? 12'd1029 : + (addr == 7'd124) ? 12'd2110 : + (addr == 7'd125) ? 12'd2935 : + (addr == 7'd126) ? 12'd885 : + 12'd2154; + +endmodule diff --git a/rtl/ip/mlkem/poly_arith/poly_arith_sync.v b/rtl/ip/mlkem/poly_arith/poly_arith_sync.v new file mode 100644 index 0000000..5e6cb2c --- /dev/null +++ b/rtl/ip/mlkem/poly_arith/poly_arith_sync.v @@ -0,0 +1,63 @@ +// poly_arith_sync.v - Polynomial modular add/sub for ML-KEM +// +// Performs element-wise modular addition or subtraction on two streaming +// 256-coefficient polynomials over Z_q (q=3329). +// +// mode=0: coeff_out = (coeff_a_in + coeff_b_in) mod Q +// mode=1: coeff_out = (coeff_a_in - coeff_b_in) mod Q +// +// Each cycle processes one coefficient pair. Pure streaming — no internal +// storage; the caller sequences all 256 coefficients in order. + +// `include "sync_rtl/common/defines.vh +`define Q 3329 // ML-KEM prime modulus +`define N 256 // polynomial degree" + +module poly_arith_sync ( + input clk, + input rst_n, + input [11:0] coeff_a_in, + input [11:0] coeff_b_in, + input mode, // 0=add, 1=sub + input valid_i, + output ready_o, + output [11:0] coeff_out, + output valid_o, + input ready_i +); + + //-------------------------------------------------------------- + // Combinational modular arithmetic + //-------------------------------------------------------------- + wire [12:0] add_raw; // a + b (13-bit to catch overflow) + wire [11:0] add_sub_q; // (a + b) - Q + wire [11:0] add_result; // (a + b) mod Q + + assign add_raw = {1'b0, coeff_a_in} + {1'b0, coeff_b_in}; + assign add_sub_q = add_raw[11:0] - `Q; + assign add_result = (add_raw < `Q) ? add_raw[11:0] : add_sub_q; + + wire [11:0] sub_result; // (a - b) mod Q + // When a >= b: diff = a - b (already in [0, Q-2]) + // When a < b: diff = a + Q - b (borrow correction) + assign sub_result = (coeff_a_in < coeff_b_in) + ? (coeff_a_in + `Q - coeff_b_in) + : (coeff_a_in - coeff_b_in); + + wire [11:0] mod_result = mode ? sub_result : add_result; + + //-------------------------------------------------------------- + // Pipeline the result through pipeline_reg for valid/ready + //-------------------------------------------------------------- + pipeline_reg #(.DW(12)) u_pipe ( + .clk (clk), + .rst_n (rst_n), + .data_i (mod_result), + .valid_i(valid_i), + .ready_o(ready_o), + .data_o (coeff_out), + .valid_o(valid_o), + .ready_i(ready_i) + ); + +endmodule diff --git a/rtl/ip/mlkem/poly_mul/basecase_mul.v b/rtl/ip/mlkem/poly_mul/basecase_mul.v new file mode 100644 index 0000000..420704c --- /dev/null +++ b/rtl/ip/mlkem/poly_mul/basecase_mul.v @@ -0,0 +1,45 @@ +// basecase_mul.v - NTT-domain degree-1 polynomial multiplication +// +// Computes pointwise product of two degree-1 NTT-domain polynomials: +// c0 = (a0*b0 + a1*b1*zeta) mod Q +// c1 = (a0*b1 + a1*b0) mod Q +// +// Uses three barrett_mul instances for modular multiplications +// and pure combinational modular addition for the final sums. +// +// All inputs/outputs are 12-bit values in [0, Q-1] where Q = 3329. + +module basecase_mul ( + input [11:0] a0, a1, // first polynomial: degree-0 and degree-1 coeffs + input [11:0] b0, b1, // second polynomial: degree-0 and degree-1 coeffs + input [11:0] zeta, // precomputed twiddle factor (zeta^2 * 17 mod Q) + output [11:0] c0, c1 // result polynomial coefficients +); + + localparam Q = 3329; + + // Barrett modular multiplication outputs + wire [11:0] t1; // a0 * b0 mod Q + wire [11:0] t2; // a1 * b1 mod Q + wire [11:0] t3; // a1 * b0 mod Q + wire [11:0] t4; // a0 * b1 mod Q + wire [11:0] t2_zeta; // (a1 * b1) * zeta mod Q + + // Four Barrett multiplications for the scalar products + barrett_mul u_mul1 (.a(a0), .b(b0), .product(t1)); + barrett_mul u_mul2 (.a(a1), .b(b1), .product(t2)); + barrett_mul u_mul3 (.a(a1), .b(b0), .product(t3)); + barrett_mul u_mul4 (.a(a0), .b(b1), .product(t4)); + + // Multiply t2 by zeta for the zeta term in c0 + barrett_mul u_mul_zeta (.a(t2), .b(zeta), .product(t2_zeta)); + + // Combinational modular addition: each sum is in [0, 2Q-1) + // Subtract Q once if >= Q + wire [12:0] sum_c0 = {1'b0, t1} + {1'b0, t2_zeta}; + wire [12:0] sum_c1 = {1'b0, t3} + {1'b0, t4}; + + assign c0 = (sum_c0 >= Q) ? sum_c0[11:0] - Q[11:0] : sum_c0[11:0]; + assign c1 = (sum_c1 >= Q) ? sum_c1[11:0] - Q[11:0] : sum_c1[11:0]; + +endmodule diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v new file mode 100644 index 0000000..59d02b1 --- /dev/null +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -0,0 +1,166 @@ +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier +// +// Computes pointwise (Karatsuba-like base-case) multiplication of two +// 256-coefficient NTT-domain polynomials. +// +// Operation flow: +// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle) +// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE +// +// The LOAD phase accepts both A and B coefficients simultaneously +// (one pair per cycle) on coeff_a_in/coeff_b_in. +// +// The COMPUTE phase outputs the 256 result coefficients one per cycle +// via valid/ready handshake on coeff_out/valid_o/ready_i. +// +// Memory: 256×12-bit register arrays for A and B coefficients. +// +// Interface: +// clk, rst_n - Clock, active-low reset +// coeff_a_in[11:0]- Polynomial A coefficient input +// coeff_b_in[11:0]- Polynomial B coefficient input +// valid_i - Input valid +// ready_o - Ready to accept input (high in IDLE/LOAD) +// coeff_out[11:0] - Result coefficient output +// valid_o - Output valid (high in COMP_C0/COMP_C1) +// ready_i - Output consumer ready + +module poly_mul_sync ( + input clk, rst_n, + input [11:0] coeff_a_in, + input [11:0] coeff_b_in, + input valid_i, + output ready_o, + output [11:0] coeff_out, + output valid_o, + input ready_i +); + + // State definitions + localparam S_IDLE = 3'd0; + localparam S_LOAD = 3'd1; + localparam S_COMP_CALC = 3'd2; + localparam S_COMP_C0 = 3'd3; + localparam S_COMP_C1 = 3'd4; + localparam S_DONE = 3'd5; + + reg [2:0] state, next_state; + + // Coefficient storage (register arrays) + reg [11:0] mem_A [0:255]; + reg [11:0] mem_B [0:255]; + + // Counters + reg [7:0] load_cnt; // 0..256 for loading 256 pairs + reg [6:0] comp_k; // 0..127, current base-case index + + // Registered basecase_mul results + reg [11:0] c0_reg, c1_reg; + + // Combinational read signals for COMP_CALC + wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits) + wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1 + wire [11:0] mem_a0 = mem_A[addr_even]; + wire [11:0] mem_a1 = mem_A[addr_odd]; + wire [11:0] mem_b0 = mem_B[addr_even]; + wire [11:0] mem_b1 = mem_B[addr_odd]; + + // Zeta ROM + wire [11:0] zeta; + poly_mul_zeta_rom u_zeta ( + .addr (comp_k), + .zeta (zeta) + ); + + // Basecase multiply + wire [11:0] bc_c0, bc_c1; + basecase_mul u_bc ( + .a0 (mem_a0), + .a1 (mem_a1), + .b0 (mem_b0), + .b1 (mem_b1), + .zeta(zeta), + .c0 (bc_c0), + .c1 (bc_c1) + ); + + // Output interface + assign ready_o = (state == S_IDLE) || (state == S_LOAD); + assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1); + assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg; + + // State transition logic (combinational) + always @* begin + next_state = state; + case (state) + S_IDLE: if (valid_i && ready_o) next_state = S_LOAD; + S_LOAD: if (load_cnt >= 255 && valid_i && ready_o) + next_state = S_COMP_CALC; + S_COMP_CALC: next_state = S_COMP_C0; + S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1; + S_COMP_C1: if (valid_o && ready_i) begin + if (comp_k >= 127) next_state = S_DONE; + else next_state = S_COMP_CALC; + end + S_DONE: next_state = S_IDLE; + default: next_state = S_IDLE; + endcase + end + + // Sequential logic + integer i; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state <= S_IDLE; + load_cnt <= 8'd0; + comp_k <= 7'd0; + c0_reg <= 12'd0; + c1_reg <= 12'd0; + for (i = 0; i < 256; i = i + 1) begin + mem_A[i] <= 12'd0; + mem_B[i] <= 12'd0; + end + end else begin + state <= next_state; + + // ---- LOAD phase ---- + // First coefficient captured on IDLE → LOAD transition + if (state == S_IDLE && valid_i && ready_o) begin + mem_A[0] <= coeff_a_in; + mem_B[0] <= coeff_b_in; + load_cnt <= 8'd1; + end + + // Subsequent coefficients in LOAD state + if (state == S_LOAD && valid_i && ready_o) begin + mem_A[load_cnt] <= coeff_a_in; + mem_B[load_cnt] <= coeff_b_in; + load_cnt <= load_cnt + 8'd1; + end + + // ---- COMPUTE phase ---- + // COMP_CALC: capture basecase_mul results + if (state == S_COMP_CALC) begin + c0_reg <= bc_c0; + c1_reg <= bc_c1; + end + + // COMP_C0 → COMP_C1: c0 was consumed, increment comp_k + if (state == S_COMP_C0 && valid_o && ready_i) begin + // comp_k stays same, c1 still to output + end + + // COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair + if (state == S_COMP_C1 && valid_o && ready_i) begin + comp_k <= comp_k + 7'd1; + end + + // ---- DONE → IDLE: reset counters ---- + if (state == S_DONE) begin + load_cnt <= 8'd0; + comp_k <= 7'd0; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v b/rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v new file mode 100644 index 0000000..e25b9bb --- /dev/null +++ b/rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v @@ -0,0 +1,58 @@ +// poly_mul_zeta_rom.v - Read-only memory for PolyMul zeta values +// +// Stores precomputed zeta values for base-case NTT-domain polynomial +// multiplication. Each entry zeta[k] = (zeta_bitRev[k]^2 * 17) mod Q. +// +// 128 entries × 12 bits. Pure combinational read. +// +// zeta_bitRev values are the same bit-reversed twiddle factors used +// in NTT. The PolyMul formula squares them and multiplies by 17 mod Q. + +module poly_mul_zeta_rom ( + input [6:0] addr, // 0..127, selects which zeta to read + output [11:0] zeta // zeta = (zeta_bitRev[addr]^2 * 17) mod Q +); + + reg [11:0] rom [0:127]; + + // Initialize ROM with precomputed zeta values + // Generated from Python: zeta = (zeta_bitRev[i]**2 * 17) % 3329 + initial begin + rom[ 0] = 12'd17; rom[ 1] = 12'd3312; rom[ 2] = 12'd2761; rom[ 3] = 12'd568; + rom[ 4] = 12'd583; rom[ 5] = 12'd2746; rom[ 6] = 12'd2649; rom[ 7] = 12'd680; + rom[ 8] = 12'd1637; rom[ 9] = 12'd1692; rom[ 10] = 12'd723; rom[ 11] = 12'd2606; + rom[ 12] = 12'd2288; rom[ 13] = 12'd1041; rom[ 14] = 12'd1100; rom[ 15] = 12'd2229; + rom[ 16] = 12'd1409; rom[ 17] = 12'd1920; rom[ 18] = 12'd2662; rom[ 19] = 12'd667; + rom[ 20] = 12'd3281; rom[ 21] = 12'd48; rom[ 22] = 12'd233; rom[ 23] = 12'd3096; + rom[ 24] = 12'd756; rom[ 25] = 12'd2573; rom[ 26] = 12'd2156; rom[ 27] = 12'd1173; + rom[ 28] = 12'd3015; rom[ 29] = 12'd314; rom[ 30] = 12'd3050; rom[ 31] = 12'd279; + rom[ 32] = 12'd1703; rom[ 33] = 12'd1626; rom[ 34] = 12'd1651; rom[ 35] = 12'd1678; + rom[ 36] = 12'd2789; rom[ 37] = 12'd540; rom[ 38] = 12'd1789; rom[ 39] = 12'd1540; + rom[ 40] = 12'd1847; rom[ 41] = 12'd1482; rom[ 42] = 12'd952; rom[ 43] = 12'd2377; + rom[ 44] = 12'd1461; rom[ 45] = 12'd1868; rom[ 46] = 12'd2687; rom[ 47] = 12'd642; + rom[ 48] = 12'd939; rom[ 49] = 12'd2390; rom[ 50] = 12'd2308; rom[ 51] = 12'd1021; + rom[ 52] = 12'd2437; rom[ 53] = 12'd892; rom[ 54] = 12'd2388; rom[ 55] = 12'd941; + rom[ 56] = 12'd733; rom[ 57] = 12'd2596; rom[ 58] = 12'd2337; rom[ 59] = 12'd992; + rom[ 60] = 12'd268; rom[ 61] = 12'd3061; rom[ 62] = 12'd641; rom[ 63] = 12'd2688; + rom[ 64] = 12'd1584; rom[ 65] = 12'd1745; rom[ 66] = 12'd2298; rom[ 67] = 12'd1031; + rom[ 68] = 12'd2037; rom[ 69] = 12'd1292; rom[ 70] = 12'd3220; rom[ 71] = 12'd109; + rom[ 72] = 12'd375; rom[ 73] = 12'd2954; rom[ 74] = 12'd2549; rom[ 75] = 12'd780; + rom[ 76] = 12'd2090; rom[ 77] = 12'd1239; rom[ 78] = 12'd1645; rom[ 79] = 12'd1684; + rom[ 80] = 12'd1063; rom[ 81] = 12'd2266; rom[ 82] = 12'd319; rom[ 83] = 12'd3010; + rom[ 84] = 12'd2773; rom[ 85] = 12'd556; rom[ 86] = 12'd757; rom[ 87] = 12'd2572; + rom[ 88] = 12'd2099; rom[ 89] = 12'd1230; rom[ 90] = 12'd561; rom[ 91] = 12'd2768; + rom[ 92] = 12'd2466; rom[ 93] = 12'd863; rom[ 94] = 12'd2594; rom[ 95] = 12'd735; + rom[ 96] = 12'd2804; rom[ 97] = 12'd525; rom[ 98] = 12'd1092; rom[ 99] = 12'd2237; + rom[100] = 12'd403; rom[101] = 12'd2926; rom[102] = 12'd1026; rom[103] = 12'd2303; + rom[104] = 12'd1143; rom[105] = 12'd2186; rom[106] = 12'd2150; rom[107] = 12'd1179; + rom[108] = 12'd2775; rom[109] = 12'd554; rom[110] = 12'd886; rom[111] = 12'd2443; + rom[112] = 12'd1722; rom[113] = 12'd1607; rom[114] = 12'd1212; rom[115] = 12'd2117; + rom[116] = 12'd1874; rom[117] = 12'd1455; rom[118] = 12'd1029; rom[119] = 12'd2300; + rom[120] = 12'd2110; rom[121] = 12'd1219; rom[122] = 12'd2935; rom[123] = 12'd394; + rom[124] = 12'd885; rom[125] = 12'd2444; rom[126] = 12'd2154; rom[127] = 12'd1175; + end + + // Combinational read + assign zeta = rom[addr]; + +endmodule diff --git a/rtl/ip/mlkem/rng/rng_sync.v b/rtl/ip/mlkem/rng/rng_sync.v new file mode 100644 index 0000000..b4fc49e --- /dev/null +++ b/rtl/ip/mlkem/rng/rng_sync.v @@ -0,0 +1,50 @@ +// rng_sync.v - 256-bit Galois LFSR PRNG (taps: 255,253,252,247,0) +module rng_sync #( + parameter [255:0] SEED = 256'hDEADBEEFCAFEBABEFEEDFACEDECAFBAD1234567887654321ABCDEF010FEDCBA9 +) ( + input clk, + input rst_n, + input valid_i, + output ready_o, + output [255:0] data_o, + output valid_o, + input ready_i +); + + reg [255:0] state; + reg valid_r; + reg [255:0] lfsr_next; + wire feedback; + integer i; + + assign ready_o = 1'b1; + assign valid_o = valid_r; + assign data_o = state; + assign feedback = state[0]; + + always @(*) begin + for (i = 0; i < 255; i = i + 1) + lfsr_next[i] = state[i+1]; + lfsr_next[255] = feedback; + lfsr_next[254] = lfsr_next[254] ^ feedback; + lfsr_next[252] = lfsr_next[252] ^ feedback; + lfsr_next[251] = lfsr_next[251] ^ feedback; + lfsr_next[246] = lfsr_next[246] ^ feedback; + end + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state <= SEED; + valid_r <= 1'b0; + end else begin + if (valid_r && ready_i) + valid_r <= 1'b0; + + if (valid_i) begin + state <= lfsr_next; + valid_r <= 1'b1; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sample_cbd/sample_cbd_sync.v b/rtl/ip/mlkem/sample_cbd/sample_cbd_sync.v new file mode 100644 index 0000000..0122008 --- /dev/null +++ b/rtl/ip/mlkem/sample_cbd/sample_cbd_sync.v @@ -0,0 +1,305 @@ +// sample_cbd_sync.v - Centered Binomial Distribution sampling via SHAKE-256 PRF +// +// Generates 256 polynomial coefficients from a 256-bit seed and 8-bit nonce +// using SHAKE-256 PRF followed by Centered Binomial Distribution (CBD). +// +// Algorithm (FIPS 203 / ML-KEM): +// PRF(sigma, N) = SHAKE-256(sigma || N) → squeeze eta*64 bytes +// For each of 256 coefficients: +// eta=2: read 4 bits, coeff = (b0+b1) - (b2+b3) +// eta=3: read 6 bits, coeff = (b0+b1+b2) - (b3+b4+b5) +// Each coefficient in range [-eta, eta], stored as 12-bit signed. +// +// SHAKE-256 parameters: +// rate = 1088 bits, capacity = 512 bits +// suffix = 4'b1111 +// pad10*1 padding +// +// Multi-squeeze (eta=3): +// SHAKE-256 squeezes 1088-bit blocks. For eta=3 we need 1536 bits. +// First squeeze provides bits [0:1087], second provides [1088:1535]. +// The 1536-bit squeeze_buf accumulates both blocks contiguously: +// After 1st: buf[1087:0] = squeeze1, buf[1535:1088] = 0 +// After 2nd: buf[1535:1088] = squeeze2[447:0] (remapped) +// Bit ordering matches Python reference PRF output. +// +// Interface: +// clk, rst_n - clock, active-low reset +// seed_i [255:0] - sigma (256-bit seed) +// nonce_i [7:0] - N counter byte +// eta_i [1:0] - 2'd2 or 2'd3 +// valid_i - input valid (start sampling) +// ready_o - module can accept new input +// coeff_o [11:0] - one coefficient per cycle, 12-bit signed +// valid_o - output valid +// ready_i - downstream accepts output +// last_o - high when last coefficient (255th, 0-indexed) + +module sample_cbd_sync ( + input clk, + input rst_n, + input [255:0] seed_i, + input [7:0] nonce_i, + input [1:0] eta_i, // 2'd2 or 2'd3 + input valid_i, + output ready_o, + output [11:0] coeff_o, // 12-bit signed + output valid_o, + input ready_i, + output last_o +); + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 2'd0; + localparam ST_PERMUTE = 2'd1; + localparam ST_SQUEEZE = 2'd2; + + reg [1:0] state_r, state_next; + + // ================================================================ + // SHAKE-256 pad10*1 construction (combinational) + // + // Message: {nonce_i, seed_i} = 264 bits + // In FIPS 202 bit ordering: message[0] = seed_i[0], message[263] = nonce_i[7] + // Padded block (1088 rate bits): + // {1'b1, 818'b0, 1'b1, 4'b1111, nonce_i, seed_i} + // + // Matches Python: PRF(sigma, N) = shake256(sigma||N, d) + // where sigma bits come first (LSB), then N bits, then suffix+padding. + // ================================================================ + wire [263:0] message_264; + wire [1087:0] padded_block; + + assign message_264 = {nonce_i, seed_i}; + assign padded_block = {1'b1, {818{1'b0}}, 1'b1, 4'b1111, message_264}; + + // Absorb state: capacity (512 bits of zero) || padded rate block + wire [1599:0] absorb_state; + assign absorb_state = {{(1600-1088){1'b0}}, padded_block}; + + // ================================================================ + // Registered inputs (captured on valid_i && ready_o) + // ================================================================ + reg [2:0] eta_r; // 2 or 3 + reg [7:0] coeff_cnt; // 0..255, number of coeffs output so far + reg [1599:0] keccak_state_r; // current 1600-bit keccak state (for re-permutation) + reg perm_done; // 1 after first keccak permutation completes + + // Combinational mux select: absorb_state BEFORE first perm finishes, + // keccak_state_r AFTER. Must be combinational to avoid NBA race + // with kc_valid_i on the IDLE→PERMUTE transition edge. + wire first_perm_sel; + assign first_perm_sel = !perm_done; + + // ================================================================ + // Squeeze buffer (accumulated across multiple squeezes) + // ================================================================ + reg [1535:0] squeeze_buf; // accumulated squeeze data + reg [10:0] buf_fill; // valid bits in squeeze_buf (0, 1088, or 1536) + reg [10:0] buf_ptr; // read position within squeeze_buf + reg kc_done_d1; // kc_valid_o delayed by 1 cycle (gate for valid_o) + + // ================================================================ + // keccak_core instantiation + // ================================================================ + wire kc_valid_i; + /* verilator lint_off UNUSEDSIGNAL */ + wire kc_ready_o; + /* verilator lint_on UNUSEDSIGNAL */ + wire [1599:0] kc_state_o; + wire kc_valid_o; + + // Mux: absorb_state for first perm, keccak_state_r for re-permutation + wire [1599:0] kc_state_i; + assign kc_state_i = first_perm_sel ? absorb_state : keccak_state_r; + + keccak_core #(.ROUNDS(24)) u_keccak ( + .clk (clk), + .rst_n (rst_n), + .state_i (kc_state_i), + .valid_i (kc_valid_i), + .ready_o (kc_ready_o), + .state_o (kc_state_o), + .valid_o (kc_valid_o), + .ready_i (1'b1) // always accept keccak output + ); + + // ================================================================ + // Bits per coefficient + // ================================================================ + wire [3:0] bits_per_coeff; + assign bits_per_coeff = (eta_r == 3'd2) ? 4'd4 : 4'd6; + + // ================================================================ + // CBD computation (combinational) + // + // Extract bits_per_coeff bits from squeeze_buf starting at buf_ptr. + // For eta=2: b0,b1,b2,b3 → coeff = (b0+b1) - (b2+b3) + // For eta=3: b0,b1,b2,b3,b4,b5 → coeff = (b0+b1+b2) - (b3+b4+b5) + // ================================================================ + wire [5:0] cbd_bits; + assign cbd_bits = squeeze_buf[buf_ptr +: 6]; + + wire [2:0] sum_pos, sum_neg; + + // eta=2: sum_pos = b0+b1, sum_neg = b2+b3 + wire [2:0] sp2, sn2; + assign sp2 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]}; + assign sn2 = {2'b00, cbd_bits[2]} + {2'b00, cbd_bits[3]}; + + // eta=3: sum_pos = b0+b1+b2, sum_neg = b3+b4+b5 + wire [2:0] sp3, sn3; + assign sp3 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]} + {2'b00, cbd_bits[2]}; + assign sn3 = {2'b00, cbd_bits[3]} + {2'b00, cbd_bits[4]} + {2'b00, cbd_bits[5]}; + + assign sum_pos = (eta_r == 3'd2) ? sp2 : sp3; + assign sum_neg = (eta_r == 3'd2) ? sn2 : sn3; + + wire signed [3:0] coeff_raw; + assign coeff_raw = $signed({1'b0, sum_pos}) - $signed({1'b0, sum_neg}); + + wire [11:0] coeff_signed; + assign coeff_signed = {{8{coeff_raw[3]}}, coeff_raw[3:0]}; // sign-extend to 12 bits + + assign coeff_o = coeff_signed; + // valid_o: suppress for 1 cycle after keccak finishes to avoid + // buf_ptr race between squeeze capture and SQUEEZE advancement. + assign valid_o = (state_r == ST_SQUEEZE) && (buf_fill >= {7'b0, bits_per_coeff}) && !kc_done_d1; + assign last_o = valid_o && (coeff_cnt == 8'd255); + + // ================================================================ + // FSM: ready_o + // ================================================================ + assign ready_o = (state_r == ST_IDLE); + + // ================================================================ + // kc_valid_i: start keccak_core when transitioning to PERMUTE + // ================================================================ + assign kc_valid_i = (state_next == ST_PERMUTE) && (state_r != ST_PERMUTE); + + // ================================================================ + // Buffer exhaustion detection + // ================================================================ + wire [11:0] next_buf_ptr; + assign next_buf_ptr = {1'b0, buf_ptr} + {8'b0, bits_per_coeff}; + + // buffer will be exhausted after outputting NEXT coefficient + wire buffer_exhaust_next; + assign buffer_exhaust_next = (next_buf_ptr + {8'b0, bits_per_coeff}) > {1'b0, buf_fill}; + + // Coefficients remaining AFTER the current one + wire [8:0] coeffs_remaining; + assign coeffs_remaining = 9'd256 - {1'b0, coeff_cnt} - 9'd1; + + // ================================================================ + // FSM combinational next-state logic + // ================================================================ + always @(*) begin + state_next = state_r; + case (state_r) + ST_IDLE: begin + if (valid_i && ready_o) + state_next = ST_PERMUTE; + end + + ST_PERMUTE: begin + // Wait for keccak_core to finish + if (kc_valid_o) + state_next = ST_SQUEEZE; + end + + ST_SQUEEZE: begin + // Output one coeff per cycle when ready + if (valid_o && ready_i) begin + if (coeff_cnt == 8'd255) begin + // Last coefficient was output + state_next = ST_IDLE; + end else if (buffer_exhaust_next && (coeffs_remaining > 9'd0)) begin + // Need more squeeze data: start another keccak permutation + state_next = ST_PERMUTE; + end + // else: stay in SQUEEZE for next coefficient + end + end + + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // Sequential logic + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + eta_r <= 3'd0; + coeff_cnt <= 8'd0; + keccak_state_r <= 1600'd0; + perm_done <= 1'b0; + squeeze_buf <= 1536'd0; + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + kc_done_d1 <= 1'b0; + end else begin + state_r <= state_next; + + // Delay kc_valid_o by 1 cycle to gate valid_o + kc_done_d1 <= kc_valid_o; + + // ---- Capture inputs on IDLE → PERMUTE transition ---- + if (state_r == ST_IDLE && valid_i && ready_o) begin + // Determine eta: 2 or 3 + if (eta_i == 2'd2) + eta_r <= 3'd2; + else if (eta_i == 2'd3) + eta_r <= 3'd3; + else + eta_r <= 3'd2; // default + + coeff_cnt <= 8'd0; + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + end + + // ---- On keccak_core valid_o: latch squeeze data ---- + if (kc_valid_o) begin + // Save full 1600-bit state for potential re-permutation + keccak_state_r <= kc_state_o; + + if (!perm_done) begin + // First squeeze: fill lower 1088 bits of squeeze_buf + // buf[1087:0] = squeeze data, buf[1535:1088] = 0 + squeeze_buf <= {{(1536 - 1088){1'b0}}, kc_state_o[1087:0]}; + buf_fill <= 11'd1088; + buf_ptr <= 11'd0; + perm_done <= 1'b1; + end else begin + // Second squeeze: fill upper 448 bits of squeeze_buf + // buf[1535:1088] = squeeze2[447:0], buf[1087:0] preserved + // This remaps: squeeze2[0] → buf[1088], matching Python's contiguous output + squeeze_buf <= {kc_state_o[447:0], squeeze_buf[1087:0]}; + buf_fill <= 11'd1536; + // buf_ptr stays at current position (kept from SQUEEZE) + end + end + + // ---- SQUEEZE: advance on output ---- + if (state_r == ST_SQUEEZE && valid_o && ready_i) begin + buf_ptr <= buf_ptr + {7'b0, bits_per_coeff}; + coeff_cnt <= coeff_cnt + 8'd1; + + // Clear state on last coeff + if (coeff_cnt == 8'd255) begin + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + // Reset perm_done here (before next IDLE→PERMUTE) + // so the mux selects absorb_state for the next vector. + perm_done <= 1'b0; + end + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sample_cbd/sample_cbd_sync_shared.v b/rtl/ip/mlkem/sample_cbd/sample_cbd_sync_shared.v new file mode 100644 index 0000000..0b5a417 --- /dev/null +++ b/rtl/ip/mlkem/sample_cbd/sample_cbd_sync_shared.v @@ -0,0 +1,311 @@ +// sample_cbd_sync_shared.v - Centered Binomial Distribution sampling via SHAKE-256 PRF +// (shared keccak_core variant — external instance) +// +// Generates 256 polynomial coefficients from a 256-bit seed and 8-bit nonce +// using SHAKE-256 PRF followed by Centered Binomial Distribution (CBD). +// +// Algorithm (FIPS 203 / ML-KEM): +// PRF(sigma, N) = SHAKE-256(sigma || N) → squeeze eta*64 bytes +// For each of 256 coefficients: +// eta=2: read 4 bits, coeff = (b0+b1) - (b2+b3) +// eta=3: read 6 bits, coeff = (b0+b1+b2) - (b3+b4+b5) +// Each coefficient in range [-eta, eta], stored as 12-bit signed. +// +// SHAKE-256 parameters: +// rate = 1088 bits, capacity = 512 bits +// suffix = 4'b1111 +// pad10*1 padding +// +// Multi-squeeze (eta=3): +// SHAKE-256 squeezes 1088-bit blocks. For eta=3 we need 1536 bits. +// First squeeze provides bits [0:1087], second provides [1088:1535]. +// The 1536-bit squeeze_buf accumulates both blocks contiguously: +// After 1st: buf[1087:0] = squeeze1, buf[1535:1088] = 0 +// After 2nd: buf[1535:1088] = squeeze2[447:0] (remapped) +// Bit ordering matches Python reference PRF output. +// +// Interface: +// clk, rst_n - clock, active-low reset +// seed_i [255:0] - sigma (256-bit seed) +// nonce_i [7:0] - N counter byte +// eta_i [1:0] - 2'd2 or 2'd3 +// valid_i - input valid (start sampling) +// ready_o - module can accept new input +// coeff_o [11:0] - one coefficient per cycle, 12-bit signed +// valid_o - output valid +// ready_i - downstream accepts output +// last_o - high when last coefficient (255th, 0-indexed) +// kc_state_o - keccak result (from external keccak_core.state_o) +// kc_valid_o - keccak done (from external keccak_core.valid_o) +// kc_ready_i - always ready to accept keccak output (to keccak_core.ready_i) +// kc_state_i - keccak input state (to external keccak_core.state_i) +// kc_valid_i - request keccak permutation (to external keccak_core.valid_i) +// kc_ready_o - keccak ready to accept (from external keccak_core.ready_o) + +module sample_cbd_sync_shared ( + input clk, + input rst_n, + input [255:0] seed_i, + input [7:0] nonce_i, + input [1:0] eta_i, // 2'd2 or 2'd3 + input valid_i, + output ready_o, + output [11:0] coeff_o, // 12-bit signed + output valid_o, + input ready_i, + output last_o, + + // External keccak_core interface + input [1599:0] kc_state_o, // keccak result (from keccak_core.state_o) + input kc_valid_o, // keccak done (from keccak_core.valid_o) + output kc_ready_i, // always ready to accept (to keccak_core.ready_i) + output [1599:0] kc_state_i, // keccak input state (to keccak_core.state_i) + output kc_valid_i, // request keccak permutation (to keccak_core.valid_i) + input kc_ready_o // keccak ready to accept (from keccak_core.ready_o) +); + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 2'd0; + localparam ST_PERMUTE = 2'd1; + localparam ST_SQUEEZE = 2'd2; + + reg [1:0] state_r, state_next; + + // ================================================================ + // SHAKE-256 pad10*1 construction (combinational) + // + // Message: {nonce_i, seed_i} = 264 bits + // In FIPS 202 bit ordering: message[0] = seed_i[0], message[263] = nonce_i[7] + // Padded block (1088 rate bits): + // {1'b1, 818'b0, 1'b1, 4'b1111, nonce_i, seed_i} + // + // Matches Python: PRF(sigma, N) = shake256(sigma||N, d) + // where sigma bits come first (LSB), then N bits, then suffix+padding. + // ================================================================ + wire [263:0] message_264; + wire [1087:0] padded_block; + + assign message_264 = {nonce_i, seed_i}; + assign padded_block = {1'b1, {818{1'b0}}, 1'b1, 4'b1111, message_264}; + + // Absorb state: capacity (512 bits of zero) || padded rate block + wire [1599:0] absorb_state; + assign absorb_state = {{(1600-1088){1'b0}}, padded_block}; + + // ================================================================ + // Registered inputs (captured on valid_i && ready_o) + // ================================================================ + reg [2:0] eta_r; // 2 or 3 + reg [7:0] coeff_cnt; // 0..255, number of coeffs output so far + reg [1599:0] keccak_state_r; // current 1600-bit keccak state (for re-permutation) + reg perm_done; // 1 after first keccak permutation completes + + // Combinational mux select: absorb_state BEFORE first perm finishes, + // keccak_state_r AFTER. Must be combinational to avoid NBA race + // with kc_valid_i on the IDLE→PERMUTE transition edge. + wire first_perm_sel; + assign first_perm_sel = !perm_done; + + // ================================================================ + // Squeeze buffer (accumulated across multiple squeezes) + // ================================================================ + reg [1535:0] squeeze_buf; // accumulated squeeze data + reg [10:0] buf_fill; // valid bits in squeeze_buf (0, 1088, or 1536) + reg [10:0] buf_ptr; // read position within squeeze_buf + reg kc_done_d1; // kc_valid_o delayed by 1 cycle (gate for valid_o) + + // ================================================================ + // External keccak_core interface connections + // + // The external keccak_core is instantiated at the top level. + // This module drives kc_valid_i and kc_state_i, receives + // kc_valid_o and kc_state_o, and provides kc_ready_i = 1'b1 + // (always ready to accept keccak output). kc_ready_o is exposed + // for the top-level arbiter to check keccak availability. + // ================================================================ + + // Mux: absorb_state for first perm, keccak_state_r for re-permutation + assign kc_state_i = first_perm_sel ? absorb_state : keccak_state_r; + + // Always ready to accept keccak output + assign kc_ready_i = 1'b1; + + // ================================================================ + // Bits per coefficient + // ================================================================ + wire [3:0] bits_per_coeff; + assign bits_per_coeff = (eta_r == 3'd2) ? 4'd4 : 4'd6; + + // ================================================================ + // CBD computation (combinational) + // + // Extract bits_per_coeff bits from squeeze_buf starting at buf_ptr. + // For eta=2: b0,b1,b2,b3 → coeff = (b0+b1) - (b2+b3) + // For eta=3: b0,b1,b2,b3,b4,b5 → coeff = (b0+b1+b2) - (b3+b4+b5) + // ================================================================ + wire [5:0] cbd_bits; + assign cbd_bits = squeeze_buf[buf_ptr +: 6]; + + wire [2:0] sum_pos, sum_neg; + + // eta=2: sum_pos = b0+b1, sum_neg = b2+b3 + wire [2:0] sp2, sn2; + assign sp2 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]}; + assign sn2 = {2'b00, cbd_bits[2]} + {2'b00, cbd_bits[3]}; + + // eta=3: sum_pos = b0+b1+b2, sum_neg = b3+b4+b5 + wire [2:0] sp3, sn3; + assign sp3 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]} + {2'b00, cbd_bits[2]}; + assign sn3 = {2'b00, cbd_bits[3]} + {2'b00, cbd_bits[4]} + {2'b00, cbd_bits[5]}; + + assign sum_pos = (eta_r == 3'd2) ? sp2 : sp3; + assign sum_neg = (eta_r == 3'd2) ? sn2 : sn3; + + wire signed [3:0] coeff_raw; + assign coeff_raw = $signed({1'b0, sum_pos}) - $signed({1'b0, sum_neg}); + + wire [11:0] coeff_signed; + assign coeff_signed = {{8{coeff_raw[3]}}, coeff_raw[3:0]}; // sign-extend to 12 bits + + assign coeff_o = coeff_signed; + // valid_o: suppress for 1 cycle after keccak finishes to avoid + // buf_ptr race between squeeze capture and SQUEEZE advancement. + assign valid_o = (state_r == ST_SQUEEZE) && (buf_fill >= {7'b0, bits_per_coeff}) && !kc_done_d1; + assign last_o = valid_o && (coeff_cnt == 8'd255); + + // ================================================================ + // FSM: ready_o + // ================================================================ + assign ready_o = (state_r == ST_IDLE); + + // ================================================================ + // kc_valid_i: start external keccak_core when transitioning to PERMUTE + // ================================================================ + assign kc_valid_i = (state_next == ST_PERMUTE) && (state_r != ST_PERMUTE); + + // ================================================================ + // Buffer exhaustion detection + // ================================================================ + wire [11:0] next_buf_ptr; + assign next_buf_ptr = {1'b0, buf_ptr} + {8'b0, bits_per_coeff}; + + // buffer will be exhausted after outputting NEXT coefficient + wire buffer_exhaust_next; + assign buffer_exhaust_next = (next_buf_ptr + {8'b0, bits_per_coeff}) > {1'b0, buf_fill}; + + // Coefficients remaining AFTER the current one + wire [8:0] coeffs_remaining; + assign coeffs_remaining = 9'd256 - {1'b0, coeff_cnt} - 9'd1; + + // ================================================================ + // FSM combinational next-state logic + // ================================================================ + always @(*) begin + state_next = state_r; + case (state_r) + ST_IDLE: begin + if (valid_i && ready_o) + state_next = ST_PERMUTE; + end + + ST_PERMUTE: begin + // Wait for external keccak_core to finish + if (kc_valid_o) + state_next = ST_SQUEEZE; + end + + ST_SQUEEZE: begin + // Output one coeff per cycle when ready + if (valid_o && ready_i) begin + if (coeff_cnt == 8'd255) begin + // Last coefficient was output + state_next = ST_IDLE; + end else if (buffer_exhaust_next && (coeffs_remaining > 9'd0)) begin + // Need more squeeze data: start another keccak permutation + state_next = ST_PERMUTE; + end + // else: stay in SQUEEZE for next coefficient + end + end + + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // Sequential logic + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + eta_r <= 3'd0; + coeff_cnt <= 8'd0; + keccak_state_r <= 1600'd0; + perm_done <= 1'b0; + squeeze_buf <= 1536'd0; + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + kc_done_d1 <= 1'b0; + end else begin + state_r <= state_next; + + // Delay kc_valid_o by 1 cycle to gate valid_o + kc_done_d1 <= kc_valid_o; + + // ---- Capture inputs on IDLE → PERMUTE transition ---- + if (state_r == ST_IDLE && valid_i && ready_o) begin + // Determine eta: 2 or 3 + if (eta_i == 2'd2) + eta_r <= 3'd2; + else if (eta_i == 2'd3) + eta_r <= 3'd3; + else + eta_r <= 3'd2; // default + + coeff_cnt <= 8'd0; + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + end + + // ---- On keccak_core valid_o: latch squeeze data ---- + if (kc_valid_o) begin + // Save full 1600-bit state for potential re-permutation + keccak_state_r <= kc_state_o; + + if (!perm_done) begin + // First squeeze: fill lower 1088 bits of squeeze_buf + // buf[1087:0] = squeeze data, buf[1535:1088] = 0 + squeeze_buf <= {{(1536 - 1088){1'b0}}, kc_state_o[1087:0]}; + buf_fill <= 11'd1088; + buf_ptr <= 11'd0; + perm_done <= 1'b1; + end else begin + // Second squeeze: fill upper 448 bits of squeeze_buf + // buf[1535:1088] = squeeze2[447:0], buf[1087:0] preserved + // This remaps: squeeze2[0] → buf[1088], matching Python's contiguous output + squeeze_buf <= {kc_state_o[447:0], squeeze_buf[1087:0]}; + buf_fill <= 11'd1536; + // buf_ptr stays at current position (kept from SQUEEZE) + end + end + + // ---- SQUEEZE: advance on output ---- + if (state_r == ST_SQUEEZE && valid_o && ready_i) begin + buf_ptr <= buf_ptr + {7'b0, bits_per_coeff}; + coeff_cnt <= coeff_cnt + 8'd1; + + // Clear state on last coeff + if (coeff_cnt == 8'd255) begin + buf_fill <= 11'd0; + buf_ptr <= 11'd0; + // Reset perm_done here (before next IDLE→PERMUTE) + // so the mux selects absorb_state for the next vector. + perm_done <= 1'b0; + end + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v new file mode 100644 index 0000000..6089c99 --- /dev/null +++ b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v @@ -0,0 +1,404 @@ +// sample_ntt_sync.v - Synchronous SampleNTT for ML-KEM A matrix generation +// +// Generates one k×k polynomial (256 coefficients) via SHAKE-128 XOF +// rejection sampling from seed rho || j || i. +// +// FIPS 202/203 conformant SHAKE-128 squeeze: +// - Absorb: S = Keccak-p(padded(rho || j || i)) +// - Squeeze the full 1344-bit (168-byte) rate as 56 groups of 3 bytes, +// each group g read from S[24*g +: 24]; extract d1[11:0], d2[23:12] +// - Accept d if d < Q=3329 +// - Only after all 56 groups of the block are consumed: S = Keccak-p(S) +// (re-permute once per rate block, NOT per 3-byte group) +// - Repeat until 256 coefficients collected (~3 blocks) +// +// Parameters: +// K = 4 (ML-KEM parameter) +// +// Interface: +// clk, rst_n - clock, active-low reset +// rho_i[255:0] - 256-bit seed rho (32 bytes) +// k_i[2:0] - actual k value (2/3/4) +// i_idx[1:0] - row index (0..k-1) +// j_idx[1:0] - column index (0..k-1) +// valid_i - start generation for this (i,j) pair +// ready_o - module can accept request +// coeff_o[11:0] - one coefficient output per cycle +// valid_o - coefficient output valid +// ready_i - consumer accepts coefficient +// last_o - high when 256th coefficient is output + +`include "defines.vh" + +/* verilator lint_off UNUSEDPARAM */ +(* use_dsp = "no" *) +module sample_ntt_sync #(parameter K = 4) ( +/* verilator lint_on UNUSEDPARAM */ + input clk, + input rst_n, + input [255:0] rho_i, + /* verilator lint_off UNUSEDSIGNAL */ + input [2:0] k_i, + /* verilator lint_on UNUSEDSIGNAL */ + input [1:0] i_idx, + input [1:0] j_idx, + input valid_i, + output ready_o, + output [11:0] coeff_o, + output valid_o, + input ready_i, + output last_o +); + + // ================================================================ + // Local parameters + // ================================================================ + localparam Q = `Q; // 3329 + + // SHAKE-128 rate = 1344 bits = 168 bytes = 56 groups of 3 bytes. + // After consuming all 56 groups of a block, re-permute the state. + localparam GRP_MAX = 6'd55; + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 3'd0; + localparam ST_ABSORB = 3'd1; // keccak running on absorb_state + localparam ST_SQUEEZE = 3'd2; // extract d1/d2, output coefficients + localparam ST_WAIT = 3'd3; // wait for next keccak to finish + localparam ST_DONE = 3'd4; // all 256 coefficients output + + reg [2:0] state_r, state_next; + + // ================================================================ + // Registered inputs (captured on valid_i) + // ================================================================ + /* verilator lint_off UNUSEDSIGNAL */ + reg [255:0] rho_r; + reg [2:0] k_r; + reg [1:0] i_r; + reg [1:0] j_r; + /* verilator lint_on UNUSEDSIGNAL */ + + // ================================================================ + // Coefficient counter (0..256). 9 bits to avoid overflow when + // reaching 256 after the 256th output. + // ================================================================ + reg [8:0] coeff_cnt_r; + + // ================================================================ + // Squeeze state register (1600-bit result of keccak_p) + // ================================================================ + reg [1599:0] squeeze_state_r; + + // ================================================================ + // Group pointer: which 3-byte group within the 1344-bit rate + // block is currently being consumed (0..GRP_MAX). Re-permute when + // it would exceed GRP_MAX. + // ================================================================ + reg [5:0] grp_ptr_r; + + // ================================================================ + // Registered d1, d2 and acceptance flags + // ================================================================ + reg [11:0] d1_r, d2_r; + reg d1_acc_r, d2_acc_r; // d1/d2 accepted? + + // ================================================================ + // Squeeze sub-phase (for multi-cycle coefficient output) + // 0 → latch d1/d2, start keccak → 1 + // 1 → output d1 if accepted → 2 + // 2 → output d2 if accepted → WAIT + // ================================================================ + reg [1:0] sq_phase_r; + + // ================================================================ + // Comb: build absorb state from rho, i, j INPUT PORTS directly + // (not registered copies — avoids NBA race on IDLE→ABSORB edge) + // ================================================================ + wire [7:0] abs_i_byte, abs_j_byte; + assign abs_i_byte = {6'b0, i_idx}; + assign abs_j_byte = {6'b0, j_idx}; + + wire [271:0] msg_bytes; + assign msg_bytes = {abs_i_byte, abs_j_byte, rho_i}; + + // SHAKE-128 absorb block: capacity(256b) | pad10*1 | suffix(1111) | msg(272b) + wire [1599:0] absorb_state; + assign absorb_state = { + 256'b0, // capacity [1599:1344] + 1'b1, // pad10*1 final 1 [1343] + {1066{1'b0}}, // pad10*1 zeros [1342:277] + 1'b1, // pad10*1 first 1 [276] + 4'b1111, // SHAKE suffix [275:272] + msg_bytes // message [271:0] + }; + + // ================================================================ + // Comb: extract d1,d2 from the current 3-byte group of squeeze state + // ================================================================ + // group g occupies bits [24*g +: 24]: c0=byte0, c1=byte1, c2=byte2 + // d1 = {c1[3:0], c0} + // d2 = {c2, c1[7:4]} + wire [10:0] grp_bit_off; + assign grp_bit_off = grp_ptr_r * 11'd24; // 0..1320, +24 ≤ 1344 (rate) + + wire [23:0] grp_bits; + assign grp_bits = squeeze_state_r[ grp_bit_off +: 24 ]; + + wire [7:0] c0, c1, c2; + assign c0 = grp_bits[7:0]; + assign c1 = grp_bits[15:8]; + assign c2 = grp_bits[23:16]; + + wire [11:0] d1_comb, d2_comb; + assign d1_comb = {c1[3:0], c0}; + assign d2_comb = {c2, c1[7:4]}; + + wire d1_ok, d2_ok; + assign d1_ok = (d1_comb < Q); + assign d2_ok = (d2_comb < Q); + + // ================================================================ + // Keccak core instantiation + // ================================================================ + wire kc_valid_i; + wire [1599:0] kc_state_i; + /* verilator lint_off UNUSEDSIGNAL */ + wire kc_ready_o; + /* verilator lint_on UNUSEDSIGNAL */ + wire [1599:0] kc_state_o; + wire kc_valid_o; + + keccak_core #(.ROUNDS(24)) u_keccak ( + .clk (clk), + .rst_n (rst_n), + .state_i (kc_state_i), + .valid_i (kc_valid_i), + .ready_o (kc_ready_o), + .state_o (kc_state_o), + .valid_o (kc_valid_o), + .ready_i (1'b1) + ); + + // kc_valid_i: asserted on the ABSORB load, and for one cycle when the + // squeeze block is exhausted (SQUEEZE → WAIT) to re-permute the state. + assign kc_valid_i = (state_next == ST_ABSORB) || + (state_r == ST_SQUEEZE && state_next == ST_WAIT); + + // kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise + assign kc_state_i = (state_next == ST_ABSORB) ? absorb_state : squeeze_state_r; + + // ================================================================ + // Output signals + // ================================================================ + reg [11:0] coeff_o_r; + reg valid_o_r; + reg last_o_r; + + assign coeff_o = coeff_o_r; + assign valid_o = valid_o_r; + assign last_o = last_o_r; + + // ================================================================ + // ready_o: accept new request in IDLE + // ================================================================ + assign ready_o = (state_r == ST_IDLE); + wire need_more = (coeff_cnt_r < 9'd256); + + // grp_done: current 3-byte group fully consumed (phase 2 resolved): + // d2 rejected, no longer need coefficients, or d2 was just accepted out. + wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) && + (!d2_acc_r || !need_more || (valid_o_r && ready_i)); + + // ================================================================ + // FSM: state_next (combinational) + // ================================================================ + always @(*) begin + state_next = state_r; + + case (state_r) + ST_IDLE: begin + if (valid_i && ready_o) + state_next = ST_ABSORB; + end + + ST_ABSORB: begin + // Wait for keccak to finish the absorb permutation + if (kc_valid_o) + state_next = ST_SQUEEZE; + end + + ST_SQUEEZE: begin + // A group is fully consumed once phase 2 resolves (d2 output, + // rejected, or no longer needed). Then either advance to the + // next group in this block, re-permute (block exhausted), or + // finish. + if (grp_done) begin + if (!need_more) + state_next = ST_DONE; + else if (grp_ptr_r < GRP_MAX) + state_next = ST_SQUEEZE; // next group, no re-permute + else + state_next = ST_WAIT; // block exhausted: re-permute + end + end + + ST_WAIT: begin + // Wait for keccak to finish squeeze permutation + if (kc_valid_o) begin + if (!need_more) + state_next = ST_DONE; + else + state_next = ST_SQUEEZE; + end + end + + ST_DONE: begin + state_next = ST_IDLE; + end + + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // FSM: sequential logic (registered) + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + sq_phase_r <= 2'd0; + coeff_cnt_r <= 9'd0; + squeeze_state_r <= 1600'd0; + grp_ptr_r <= 6'd0; + d1_r <= 12'd0; + d2_r <= 12'd0; + d1_acc_r <= 1'b0; + d2_acc_r <= 1'b0; + coeff_o_r <= 12'd0; + valid_o_r <= 1'b0; + last_o_r <= 1'b0; + rho_r <= 256'd0; + k_r <= 3'd0; + i_r <= 2'd0; + j_r <= 2'd0; + end else begin + state_r <= state_next; + + // --------------------------------------------------------- + // Capture inputs on valid_i (IDLE → ABSORB transition) + // --------------------------------------------------------- + if (state_r == ST_IDLE && valid_i && ready_o) begin + rho_r <= rho_i; + k_r <= k_i; + i_r <= i_idx; + j_r <= j_idx; + coeff_cnt_r <= 9'd0; + end + + // --------------------------------------------------------- + // Latch keccak output when valid_o fires. A fresh block + // starts at group 0 (ABSORB load or WAIT re-permute result). + // --------------------------------------------------------- + if (kc_valid_o) begin + squeeze_state_r <= kc_state_o; + grp_ptr_r <= 6'd0; + end + + // --------------------------------------------------------- + // SQ_PHASE = 0: latch d1/d2 acceptance + // --------------------------------------------------------- + if (state_r == ST_SQUEEZE && sq_phase_r == 2'd0) begin + d1_r <= d1_comb; + d2_r <= d2_comb; + d1_acc_r <= d1_ok; + d2_acc_r <= d2_ok; + end + + // --------------------------------------------------------- + // SQ_PHASE management and coefficient output + // + // Handshake pattern (matching pipeline_reg convention): + // Cycle N: valid_o_r ← 1, coeff_o_r ← value + // Cycle N+1: if ready_i: consume, valid_o_r ← 0, advance + // + // The testbench reads coeff_o after Cycle N and consumes + // with the next posedge (Cycle N+1). + // --------------------------------------------------------- + if (state_r == ST_SQUEEZE) begin + case (sq_phase_r) + // ----- Phase 0: latch, advance to phase 1 ----- + 2'd0: begin + sq_phase_r <= 2'd1; + end + + // ----- Phase 1: output d1 ----- + 2'd1: begin + if (d1_acc_r && need_more) begin + if (!valid_o_r) begin + // First cycle: assert output + coeff_o_r <= d1_r; + valid_o_r <= 1'b1; + last_o_r <= (coeff_cnt_r == 9'd255); + end else begin + // valid_o is high; wait for consumer + if (ready_i) begin + coeff_cnt_r <= coeff_cnt_r + 9'd1; + valid_o_r <= 1'b0; + sq_phase_r <= 2'd2; + end + end + end else begin + // d1 rejected or no longer needed: skip to phase 2 + valid_o_r <= 1'b0; + sq_phase_r <= 2'd2; + end + end + + // ----- Phase 2: output d2 ----- + 2'd2: begin + if (d2_acc_r && need_more) begin + if (!valid_o_r) begin + // First cycle: assert output + coeff_o_r <= d2_r; + valid_o_r <= 1'b1; + last_o_r <= (coeff_cnt_r == 9'd255); + end else begin + // valid_o is high; wait for consumer + if (ready_i) begin + coeff_cnt_r <= coeff_cnt_r + 9'd1; + valid_o_r <= 1'b0; + // state_next → ST_WAIT (combinational) + end + end + end else begin + // d2 rejected or done + valid_o_r <= 1'b0; + end + end + + default: begin + valid_o_r <= 1'b0; + end + endcase + + // Group consumed but block not exhausted: advance to the + // next 3-byte group within the same block (no re-permute). + if (grp_done && need_more && grp_ptr_r < GRP_MAX) begin + grp_ptr_r <= grp_ptr_r + 6'd1; + sq_phase_r <= 2'd0; + end + end else if (state_r != ST_SQUEEZE && state_next == ST_SQUEEZE) begin + // About to enter SQUEEZE: reset phase and output + sq_phase_r <= 2'd0; + valid_o_r <= 1'b0; + end else begin + // Not in SQUEEZE: clear + valid_o_r <= 1'b0; + sq_phase_r <= 2'd0; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v new file mode 100644 index 0000000..fd0242d --- /dev/null +++ b/rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v @@ -0,0 +1,418 @@ +// sample_ntt_sync_shared.v - Synchronous SampleNTT for ML-KEM A matrix generation +// with external Keccak interface ports. +// +// Identical to sample_ntt_sync.v except the internal keccak_core instance +// is replaced by explicit kc_* ports. This allows multiple SampleNTT +// instances to share a single keccak_core (via a round-robin arbiter). +// +// Generates one k×k polynomial (256 coefficients) via SHAKE-128 XOF +// rejection sampling from seed rho || j || i. +// +// FIPS 202/203 conformant SHAKE-128 squeeze: +// - Absorb: S = Keccak-p(padded(rho || j || i)) +// - Squeeze the full 1344-bit (168-byte) rate as 56 groups of 3 bytes, +// each group g read from S[24*g +: 24]; extract d1[11:0], d2[23:12] +// - Accept d if d < Q=3329 +// - Only after all 56 groups of the block are consumed: S = Keccak-p(S) +// (re-permute once per rate block, NOT per 3-byte group) +// - Repeat until 256 coefficients collected (~3 blocks) +// +// Parameters: +// K = 4 (ML-KEM parameter) +// +// Interface: +// clk, rst_n - clock, active-low reset +// rho_i[255:0] - 256-bit seed rho (32 bytes) +// k_i[2:0] - actual k value (2/3/4) +// i_idx[1:0] - row index (0..k-1) +// j_idx[1:0] - column index (0..k-1) +// valid_i - start generation for this (i,j) pair +// ready_o - module can accept request +// coeff_o[11:0] - one coefficient output per cycle +// valid_o - coefficient output valid +// ready_i - consumer accepts coefficient +// last_o - high when 256th coefficient is output +// +// --- Keccak interface (external, shared) --- +// kc_state_o[1599:0] - keccak result (from keccak_core) +// kc_valid_o - keccak output valid (from keccak_core) +// kc_ready_i - sample_ntt ready to accept keccak output (always 1'b1) +// kc_state_i[1599:0] - keccak input state (to keccak_core) +// kc_valid_i - request keccak permutation (to keccak_core) +// kc_ready_o - keccak ready to accept new request (from keccak_core) + +`include "defines.vh" + +/* verilator lint_off UNUSEDPARAM */ +(* use_dsp = "no" *) +module sample_ntt_sync_shared #(parameter K = 4) ( +/* verilator lint_on UNUSEDPARAM */ + input clk, + input rst_n, + input [255:0] rho_i, + /* verilator lint_off UNUSEDSIGNAL */ + input [2:0] k_i, + /* verilator lint_on UNUSEDSIGNAL */ + input [1:0] i_idx, + input [1:0] j_idx, + input valid_i, + output ready_o, + output [11:0] coeff_o, + output valid_o, + input ready_i, + output last_o, + + // --- Keccak interface ports (external) --- + input [1599:0] kc_state_o, + input kc_valid_o, + output kc_ready_i, + output [1599:0] kc_state_i, + output kc_valid_i, + /* verilator lint_off UNUSEDSIGNAL */ + input kc_ready_o + /* verilator lint_on UNUSEDSIGNAL */ +); + + // ================================================================ + // Local parameters + // ================================================================ + localparam Q = `Q; // 3329 + + // SHAKE-128 rate = 1344 bits = 168 bytes = 56 groups of 3 bytes. + // After consuming all 56 groups of a block, re-permute the state. + localparam GRP_MAX = 6'd55; + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 3'd0; + localparam ST_ABSORB = 3'd1; // keccak running on absorb_state + localparam ST_SQUEEZE = 3'd2; // extract d1/d2, output coefficients + localparam ST_WAIT = 3'd3; // wait for next keccak to finish + localparam ST_DONE = 3'd4; // all 256 coefficients output + + reg [2:0] state_r, state_next; + + // ================================================================ + // Registered inputs (captured on valid_i) + // ================================================================ + /* verilator lint_off UNUSEDSIGNAL */ + reg [255:0] rho_r; + reg [2:0] k_r; + reg [1:0] i_r; + reg [1:0] j_r; + /* verilator lint_on UNUSEDSIGNAL */ + + // ================================================================ + // Coefficient counter (0..256). 9 bits to avoid overflow when + // reaching 256 after the 256th output. + // ================================================================ + reg [8:0] coeff_cnt_r; + + // ================================================================ + // Squeeze state register (1600-bit result of keccak_p) + // ================================================================ + reg [1599:0] squeeze_state_r; + + // ================================================================ + // Group pointer: which 3-byte group within the 1344-bit rate + // block is currently being consumed (0..GRP_MAX). + // ================================================================ + reg [5:0] grp_ptr_r; + + // ================================================================ + // Registered d1, d2 and acceptance flags + // ================================================================ + reg [11:0] d1_r, d2_r; + reg d1_acc_r, d2_acc_r; // d1/d2 accepted? + + // ================================================================ + // Squeeze sub-phase (for multi-cycle coefficient output) + // 0 → latch d1/d2, start keccak → 1 + // 1 → output d1 if accepted → 2 + // 2 → output d2 if accepted → WAIT + // ================================================================ + reg [1:0] sq_phase_r; + + // ================================================================ + // Comb: build absorb state from rho, i, j INPUT PORTS directly + // (not registered copies — avoids NBA race on IDLE→ABSORB edge) + // ================================================================ + wire [7:0] abs_i_byte, abs_j_byte; + assign abs_i_byte = {6'b0, i_idx}; + assign abs_j_byte = {6'b0, j_idx}; + + wire [271:0] msg_bytes; + assign msg_bytes = {abs_i_byte, abs_j_byte, rho_i}; + + // SHAKE-128 absorb block: capacity(256b) | pad10*1 | suffix(1111) | msg(272b) + wire [1599:0] absorb_state; + assign absorb_state = { + 256'b0, // capacity [1599:1344] + 1'b1, // pad10*1 final 1 [1343] + {1066{1'b0}}, // pad10*1 zeros [1342:277] + 1'b1, // pad10*1 first 1 [276] + 4'b1111, // SHAKE suffix [275:272] + msg_bytes // message [271:0] + }; + + // ================================================================ + // Comb: extract d1,d2 from the current 3-byte group of squeeze state + // ================================================================ + // group g occupies bits [24*g +: 24]: c0=byte0, c1=byte1, c2=byte2 + // d1 = {c1[3:0], c0} + // d2 = {c2, c1[7:4]} + wire [10:0] grp_bit_off; + assign grp_bit_off = grp_ptr_r * 11'd24; // 0..1320, +24 ≤ 1344 (rate) + + wire [23:0] grp_bits; + assign grp_bits = squeeze_state_r[ grp_bit_off +: 24 ]; + + wire [7:0] c0, c1, c2; + assign c0 = grp_bits[7:0]; + assign c1 = grp_bits[15:8]; + assign c2 = grp_bits[23:16]; + + wire [11:0] d1_comb, d2_comb; + assign d1_comb = {c1[3:0], c0}; + assign d2_comb = {c2, c1[7:4]}; + + wire d1_ok, d2_ok; + assign d1_ok = (d1_comb < Q); + assign d2_ok = (d2_comb < Q); + + // ================================================================ + // Keccak interface: external ports instead of internal keccak_core. + // + // kc_valid_i: asserted during ABSORB and first phase of SQUEEZE. + // kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise. + // kc_ready_i: always 1'b1 — this module always accepts keccak output. + // + // These signals are output ports driven by the combinational logic + // below. The upstream arbiter connects them to a shared keccak_core. + // ================================================================ + + // kc_ready_i: always ready to accept keccak output + assign kc_ready_i = 1'b1; + + // kc_valid_i: asserted on the ABSORB load, and for one cycle when the + // squeeze block is exhausted (SQUEEZE → WAIT) to re-permute the state. + assign kc_valid_i = (state_next == ST_ABSORB) || + (state_r == ST_SQUEEZE && state_next == ST_WAIT); + + // kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise + assign kc_state_i = (state_next == ST_ABSORB) ? absorb_state : squeeze_state_r; + + // ================================================================ + // Output signals + // ================================================================ + reg [11:0] coeff_o_r; + reg valid_o_r; + reg last_o_r; + + assign coeff_o = coeff_o_r; + assign valid_o = valid_o_r; + assign last_o = last_o_r; + + // ================================================================ + // ready_o: accept new request in IDLE + // ================================================================ + assign ready_o = (state_r == ST_IDLE); + wire need_more = (coeff_cnt_r < 9'd256); + + // grp_done: current 3-byte group fully consumed (phase 2 resolved): + // d2 rejected, no longer need coefficients, or d2 was just accepted out. + wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) && + (!d2_acc_r || !need_more || (valid_o_r && ready_i)); + + // ================================================================ + // FSM: state_next (combinational) + // ================================================================ + always @(*) begin + state_next = state_r; + + case (state_r) + ST_IDLE: begin + if (valid_i && ready_o) + state_next = ST_ABSORB; + end + + ST_ABSORB: begin + // Wait for keccak to finish the absorb permutation + if (kc_valid_o) + state_next = ST_SQUEEZE; + end + + ST_SQUEEZE: begin + // A group is fully consumed once phase 2 resolves (d2 output, + // rejected, or no longer needed). Then either advance to the + // next group in this block, re-permute (block exhausted), or + // finish. + if (grp_done) begin + if (!need_more) + state_next = ST_DONE; + else if (grp_ptr_r < GRP_MAX) + state_next = ST_SQUEEZE; // next group, no re-permute + else + state_next = ST_WAIT; // block exhausted: re-permute + end + end + + ST_WAIT: begin + // Wait for keccak to finish squeeze permutation + if (kc_valid_o) begin + if (!need_more) + state_next = ST_DONE; + else + state_next = ST_SQUEEZE; + end + end + + ST_DONE: begin + state_next = ST_IDLE; + end + + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // FSM: sequential logic (registered) + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + sq_phase_r <= 2'd0; + coeff_cnt_r <= 9'd0; + squeeze_state_r <= 1600'd0; + grp_ptr_r <= 6'd0; + d1_r <= 12'd0; + d2_r <= 12'd0; + d1_acc_r <= 1'b0; + d2_acc_r <= 1'b0; + coeff_o_r <= 12'd0; + valid_o_r <= 1'b0; + last_o_r <= 1'b0; + rho_r <= 256'd0; + k_r <= 3'd0; + i_r <= 2'd0; + j_r <= 2'd0; + end else begin + state_r <= state_next; + + // --------------------------------------------------------- + // Capture inputs on valid_i (IDLE → ABSORB transition) + // --------------------------------------------------------- + if (state_r == ST_IDLE && valid_i && ready_o) begin + rho_r <= rho_i; + k_r <= k_i; + i_r <= i_idx; + j_r <= j_idx; + coeff_cnt_r <= 9'd0; + end + + // --------------------------------------------------------- + // Latch keccak output when valid_o fires. A fresh block + // starts at group 0 (ABSORB load or WAIT re-permute result). + // --------------------------------------------------------- + if (kc_valid_o) begin + squeeze_state_r <= kc_state_o; + grp_ptr_r <= 6'd0; + end + + // --------------------------------------------------------- + // SQ_PHASE = 0: latch d1/d2 acceptance + // --------------------------------------------------------- + if (state_r == ST_SQUEEZE && sq_phase_r == 2'd0) begin + d1_r <= d1_comb; + d2_r <= d2_comb; + d1_acc_r <= d1_ok; + d2_acc_r <= d2_ok; + end + + // --------------------------------------------------------- + // SQ_PHASE management and coefficient output + // + // Handshake pattern (matching pipeline_reg convention): + // Cycle N: valid_o_r ← 1, coeff_o_r ← value + // Cycle N+1: if ready_i: consume, valid_o_r ← 0, advance + // + // The testbench reads coeff_o after Cycle N and consumes + // with the next posedge (Cycle N+1). + // --------------------------------------------------------- + if (state_r == ST_SQUEEZE) begin + case (sq_phase_r) + // ----- Phase 0: latch, advance to phase 1 ----- + 2'd0: begin + sq_phase_r <= 2'd1; + end + + // ----- Phase 1: output d1 ----- + 2'd1: begin + if (d1_acc_r && need_more) begin + if (!valid_o_r) begin + // First cycle: assert output + coeff_o_r <= d1_r; + valid_o_r <= 1'b1; + last_o_r <= (coeff_cnt_r == 9'd255); + end else begin + // valid_o is high; wait for consumer + if (ready_i) begin + coeff_cnt_r <= coeff_cnt_r + 9'd1; + valid_o_r <= 1'b0; + sq_phase_r <= 2'd2; + end + end + end else begin + // d1 rejected or no longer needed: skip to phase 2 + valid_o_r <= 1'b0; + sq_phase_r <= 2'd2; + end + end + + // ----- Phase 2: output d2 ----- + 2'd2: begin + if (d2_acc_r && need_more) begin + if (!valid_o_r) begin + // First cycle: assert output + coeff_o_r <= d2_r; + valid_o_r <= 1'b1; + last_o_r <= (coeff_cnt_r == 9'd255); + end else begin + // valid_o is high; wait for consumer + if (ready_i) begin + coeff_cnt_r <= coeff_cnt_r + 9'd1; + valid_o_r <= 1'b0; + // state_next → ST_WAIT (combinational) + end + end + end else begin + // d2 rejected or done + valid_o_r <= 1'b0; + end + end + + default: begin + valid_o_r <= 1'b0; + end + endcase + + // Group consumed but block not exhausted: advance to the + // next 3-byte group within the same block (no re-permute). + if (grp_done && need_more && grp_ptr_r < GRP_MAX) begin + grp_ptr_r <= grp_ptr_r + 6'd1; + sq_phase_r <= 2'd0; + end + end else if (state_r != ST_SQUEEZE && state_next == ST_SQUEEZE) begin + // About to enter SQUEEZE: reset phase and output + sq_phase_r <= 2'd0; + valid_o_r <= 1'b0; + end else begin + // Not in SQUEEZE: clear + valid_o_r <= 1'b0; + sq_phase_r <= 2'd0; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sha3/keccak_core.v b/rtl/ip/mlkem/sha3/keccak_core.v new file mode 100644 index 0000000..6cf5c7d --- /dev/null +++ b/rtl/ip/mlkem/sha3/keccak_core.v @@ -0,0 +1,99 @@ +// keccak_core.v - Synchronous Keccak-f[1600] core with valid/ready +// +// Runs 24 rounds of keccak_round on the input state. +// valid_i high with ready_o high → load state_i, start 24-round sequence. +// Each round takes 1 cycle. After round 23, valid_o asserted with state_o. +// +// Parameter: +// ROUNDS = 24 (default) +// +// Interface: +// clk, rst_n - clock, active-low reset +// state_i - 1600-bit input state +// valid_i - start permutation +// ready_o - core can accept new input +// state_o - 1600-bit output state (after permutation) +// valid_o - output is valid +// ready_i - consumer accepts output + +module keccak_core #(parameter ROUNDS = 24) ( + input clk, + input rst_n, + input [1599:0] state_i, + input valid_i, + output ready_o, + output [1599:0] state_o, + output valid_o, + input ready_i +); + + // ================================================================ + // Internal registers + // ================================================================ + reg busy_r; // 1 while running permutation + reg [4:0] cnt_r; // round counter (0 to ROUNDS-1) + reg [1599:0] state_r; // current state + + // ================================================================ + // Next-state logic + // ================================================================ + wire cnt_is_last; // cnt_r == ROUNDS-1 ? + wire [1599:0] state_next; // combinational round output + + assign cnt_is_last = (cnt_r == (ROUNDS - 1)); + + // keccak_round: apply one round to current state + keccak_round u_round ( + .state_i(state_r), + .round_i(cnt_r), + .state_o(state_next) + ); + + // ================================================================ + // Valid/ready handshake + // ================================================================ + // ready_o: accept new input when not busy (or just finishing) + assign ready_o = !busy_r; + + // valid_o: output valid when we just finished the last round + // (the final state is already in state_r at the output cycle) + assign valid_o = busy_r && cnt_is_last; + + // state_o: output the final state + assign state_o = state_next; // final state from the last round + + // ================================================================ + // Sequential logic + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + busy_r <= 1'b0; + cnt_r <= 5'd0; + state_r <= 1600'd0; + end else begin + if (!busy_r) begin + // IDLE: wait for valid_i + if (valid_i) begin + state_r <= state_i; + cnt_r <= 5'd0; + busy_r <= 1'b1; + end + end else begin + // BUSY: running permutations + if (cnt_is_last) begin + // Final round: check if output can be accepted + if (ready_i) begin + busy_r <= 1'b0; + cnt_r <= 5'd0; + end + // else: hold until ready_i + end else begin + // Intermediate round: advance state + state_r <= state_next; + cnt_r <= cnt_r + 5'd1; + end + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sha3/keccak_round.v b/rtl/ip/mlkem/sha3/keccak_round.v new file mode 100644 index 0000000..4ba1a25 --- /dev/null +++ b/rtl/ip/mlkem/sha3/keccak_round.v @@ -0,0 +1,248 @@ +// keccak_round.v - Single Keccak-f round (theta/rho/pi/chi/iota) +// +// Pure combinational: no clock, no valid/ready. +// Input: 1600-bit state, 5-bit round index (0-23). +// Output: transformed 1600-bit state. +// +// State layout: s[64*(5*y + x) + z] = A[x][y][z] +// Lane(x,y) = s[64*(5*y + x) +: 64] + +module keccak_round ( + input [1599:0] state_i, + input [4:0] round_i, + output [1599:0] state_o +); + + // ================================================================ + // Round constants (24 entries, FIPS 202) + // ================================================================ + wire [63:0] RC [0:23]; + assign RC[ 0] = 64'h0000000000000001; + assign RC[ 1] = 64'h0000000000008082; + assign RC[ 2] = 64'h800000000000808A; + assign RC[ 3] = 64'h8000000080008000; + assign RC[ 4] = 64'h000000000000808B; + assign RC[ 5] = 64'h0000000080000001; + assign RC[ 6] = 64'h8000000080008081; + assign RC[ 7] = 64'h8000000000008009; + assign RC[ 8] = 64'h000000000000008A; + assign RC[ 9] = 64'h0000000000000088; + assign RC[10] = 64'h0000000080008009; + assign RC[11] = 64'h000000008000000A; + assign RC[12] = 64'h000000008000808B; + assign RC[13] = 64'h800000000000008B; + assign RC[14] = 64'h8000000000008089; + assign RC[15] = 64'h8000000000008003; + assign RC[16] = 64'h8000000000008002; + assign RC[17] = 64'h8000000000000080; + assign RC[18] = 64'h000000000000800A; + assign RC[19] = 64'h800000008000000A; + assign RC[20] = 64'h8000000080008081; + assign RC[21] = 64'h8000000000008080; + assign RC[22] = 64'h0000000080000001; + assign RC[23] = 64'h8000000080008008; + + // ================================================================ + // Rho rotation offsets (FIPS 202, Table 2) + // Inlined as generate-if chains for tool compatibility + // ================================================================ + + // ================================================================ + // Unpack flat 1600-bit input into 25 lanes (each 64-bit) + // Lane index: 5*y + x + // ================================================================ + wire [63:0] A [0:4][0:4]; + genvar gx, gy; + generate + for (gx = 0; gx < 5; gx = gx + 1) begin : up_x + for (gy = 0; gy < 5; gy = gy + 1) begin : up_y + assign A[gx][gy] = state_i[((5 * gy + gx) * 64) +: 64]; + end + end + endgenerate + + // ================================================================ + // Theta step + // C[x] = XOR over y of A[x][y] + // D[x] = C[x-1] ^ ROTL(C[x+1], 1) + // A_theta[x][y] = A[x][y] ^ D[x] + // ================================================================ + wire [63:0] C [0:4]; + wire [63:0] D [0:4]; + wire [63:0] A_t [0:4][0:4]; + + generate + for (gx = 0; gx < 5; gx = gx + 1) begin : th_x + // C[x] = XOR of the 5 lanes in column x + assign C[gx] = A[gx][0] ^ A[gx][1] ^ A[gx][2] ^ A[gx][3] ^ A[gx][4]; + + // D[x] = C[(x-1) mod 5] ^ ROTL(C[(x+1) mod 5], 1) + localparam [2:0] xp1 = (gx == 4) ? 3'd0 : (gx + 3'd1); + localparam [2:0] xm1 = (gx == 0) ? 3'd4 : (gx - 3'd1); + assign D[gx] = C[xm1] ^ {C[xp1][62:0], C[xp1][63]}; + + for (gy = 0; gy < 5; gy = gy + 1) begin : th_y + assign A_t[gx][gy] = A[gx][gy] ^ D[gx]; + end + end + endgenerate + + // ================================================================ + // Rho step (rotation of each lane by precomputed offset) + // ================================================================ + wire [63:0] A_r [0:4][0:4]; + generate + for (gx = 0; gx < 5; gx = gx + 1) begin : rh_x + for (gy = 0; gy < 5; gy = gy + 1) begin : rh_y + if (gx == 0 && gy == 0) begin : rho_00 + assign A_r[gx][gy] = A_t[gx][gy]; + end else + if (gy == 0 && gx == 1) begin : rho_10 + assign A_r[gx][gy] = {A_t[gx][gy][62:0], A_t[gx][gy][63]}; + end else + if (gy == 0 && gx == 2) begin : rho_20 + assign A_r[gx][gy] = {A_t[gx][gy][1:0], A_t[gx][gy][63:2]}; + end else + if (gy == 0 && gx == 3) begin : rho_30 + assign A_r[gx][gy] = {A_t[gx][gy][35:0], A_t[gx][gy][63:36]}; + end else + if (gy == 0 && gx == 4) begin : rho_40 + assign A_r[gx][gy] = {A_t[gx][gy][36:0], A_t[gx][gy][63:37]}; + end else + if (gy == 1 && gx == 0) begin : rho_01 + assign A_r[gx][gy] = {A_t[gx][gy][27:0], A_t[gx][gy][63:28]}; + end else + if (gy == 1 && gx == 1) begin : rho_11 + assign A_r[gx][gy] = {A_t[gx][gy][19:0], A_t[gx][gy][63:20]}; + end else + if (gy == 1 && gx == 2) begin : rho_21 + assign A_r[gx][gy] = {A_t[gx][gy][57:0], A_t[gx][gy][63:58]}; + end else + if (gy == 1 && gx == 3) begin : rho_31 + assign A_r[gx][gy] = {A_t[gx][gy][8:0], A_t[gx][gy][63:9]}; + end else + if (gy == 1 && gx == 4) begin : rho_41 + assign A_r[gx][gy] = {A_t[gx][gy][43:0], A_t[gx][gy][63:44]}; + end else + if (gy == 2 && gx == 0) begin : rho_02 + assign A_r[gx][gy] = {A_t[gx][gy][60:0], A_t[gx][gy][63:61]}; + end else + if (gy == 2 && gx == 1) begin : rho_12 + assign A_r[gx][gy] = {A_t[gx][gy][53:0], A_t[gx][gy][63:54]}; + end else + if (gy == 2 && gx == 2) begin : rho_22 + assign A_r[gx][gy] = {A_t[gx][gy][20:0], A_t[gx][gy][63:21]}; + end else + if (gy == 2 && gx == 3) begin : rho_32 + assign A_r[gx][gy] = {A_t[gx][gy][38:0], A_t[gx][gy][63:39]}; + end else + if (gy == 2 && gx == 4) begin : rho_42 + assign A_r[gx][gy] = {A_t[gx][gy][24:0], A_t[gx][gy][63:25]}; + end else + if (gy == 3 && gx == 0) begin : rho_03 + assign A_r[gx][gy] = {A_t[gx][gy][22:0], A_t[gx][gy][63:23]}; + end else + if (gy == 3 && gx == 1) begin : rho_13 + assign A_r[gx][gy] = {A_t[gx][gy][18:0], A_t[gx][gy][63:19]}; + end else + if (gy == 3 && gx == 2) begin : rho_23 + assign A_r[gx][gy] = {A_t[gx][gy][48:0], A_t[gx][gy][63:49]}; + end else + if (gy == 3 && gx == 3) begin : rho_33 + assign A_r[gx][gy] = {A_t[gx][gy][42:0], A_t[gx][gy][63:43]}; + end else + if (gy == 3 && gx == 4) begin : rho_43 + assign A_r[gx][gy] = {A_t[gx][gy][55:0], A_t[gx][gy][63:56]}; + end else + if (gy == 4 && gx == 0) begin : rho_04 + assign A_r[gx][gy] = {A_t[gx][gy][45:0], A_t[gx][gy][63:46]}; + end else + if (gy == 4 && gx == 1) begin : rho_14 + assign A_r[gx][gy] = {A_t[gx][gy][61:0], A_t[gx][gy][63:62]}; + end else + if (gy == 4 && gx == 2) begin : rho_24 + assign A_r[gx][gy] = {A_t[gx][gy][2:0], A_t[gx][gy][63:3]}; + end else + if (gy == 4 && gx == 3) begin : rho_34 + assign A_r[gx][gy] = {A_t[gx][gy][7:0], A_t[gx][gy][63:8]}; + end else + if (gy == 4 && gx == 4) begin : rho_44 + assign A_r[gx][gy] = {A_t[gx][gy][49:0], A_t[gx][gy][63:50]}; + end + end + end + endgenerate + + // ================================================================ + // Pi step (permute lanes) + // A_pi[x][y] = A_rho[(x + 3*y) % 5][x] + // ================================================================ + wire [63:0] A_p [0:4][0:4]; + generate + for (gx = 0; gx < 5; gx = gx + 1) begin : pi_x + for (gy = 0; gy < 5; gy = gy + 1) begin : pi_y + localparam [2:0] src_x = (gx + 3 * gy) % 5; + assign A_p[gx][gy] = A_r[src_x][gx]; + end + end + endgenerate + + // ================================================================ + // Chi step + // A_chi[x][y] = A_pi[x][y] ^ (~A_pi[x+1][y] & A_pi[x+2][y]) + // ================================================================ + wire [63:0] A_c [0:4][0:4]; + generate + for (gx = 0; gx < 5; gx = gx + 1) begin : ch_x + localparam [2:0] xp1 = (gx == 4) ? 3'd0 : (gx + 3'd1); + localparam [2:0] xp2 = (gx == 3) ? 3'd0 : + (gx == 4) ? 3'd1 : (gx + 3'd2); + for (gy = 0; gy < 5; gy = gy + 1) begin : ch_y + assign A_c[gx][gy] = A_p[gx][gy] ^ + ((~A_p[xp1][gy]) & A_p[xp2][gy]); + end + end + endgenerate + + // ================================================================ + // Iota step + // A_iota[0][0] = A_chi[0][0] ^ RC[round_i] + // Other lanes unchanged + // ================================================================ + wire [63:0] A_io [0:4][0:4]; + assign A_io[0][0] = A_c[0][0] ^ RC[round_i]; + assign A_io[0][1] = A_c[0][1]; + assign A_io[0][2] = A_c[0][2]; + assign A_io[0][3] = A_c[0][3]; + assign A_io[0][4] = A_c[0][4]; + assign A_io[1][0] = A_c[1][0]; + assign A_io[1][1] = A_c[1][1]; + assign A_io[1][2] = A_c[1][2]; + assign A_io[1][3] = A_c[1][3]; + assign A_io[1][4] = A_c[1][4]; + assign A_io[2][0] = A_c[2][0]; + assign A_io[2][1] = A_c[2][1]; + assign A_io[2][2] = A_c[2][2]; + assign A_io[2][3] = A_c[2][3]; + assign A_io[2][4] = A_c[2][4]; + assign A_io[3][0] = A_c[3][0]; + assign A_io[3][1] = A_c[3][1]; + assign A_io[3][2] = A_c[3][2]; + assign A_io[3][3] = A_c[3][3]; + assign A_io[3][4] = A_c[3][4]; + assign A_io[4][0] = A_c[4][0]; + assign A_io[4][1] = A_c[4][1]; + assign A_io[4][2] = A_c[4][2]; + assign A_io[4][3] = A_c[4][3]; + assign A_io[4][4] = A_c[4][4]; + + // ================================================================ + // Pack 25 lanes back into 1600-bit flat output + // ================================================================ + assign state_o = {A_io[4][4], A_io[3][4], A_io[2][4], A_io[1][4], A_io[0][4], + A_io[4][3], A_io[3][3], A_io[2][3], A_io[1][3], A_io[0][3], + A_io[4][2], A_io[3][2], A_io[2][2], A_io[1][2], A_io[0][2], + A_io[4][1], A_io[3][1], A_io[2][1], A_io[1][1], A_io[0][1], + A_io[4][0], A_io[3][0], A_io[2][0], A_io[1][0], A_io[0][0]}; + +endmodule diff --git a/rtl/ip/mlkem/sha3/sha3_top.v b/rtl/ip/mlkem/sha3/sha3_top.v new file mode 100644 index 0000000..56411e3 --- /dev/null +++ b/rtl/ip/mlkem/sha3/sha3_top.v @@ -0,0 +1,215 @@ +// sha3_top.v - SHA3/SHAKE top wrapper with valid/ready interface +// +// Implements SHA3-512 (G), SHA3-256 (H), SHAKE-256 (J) over a single +// Keccak-f[1600] core. Supports single-block absorption, plus a separate +// multi-block SHA3-256 streaming absorb path for long inputs (e.g. H(ek)). +// +// Single-block modes (mb_en=0): +// 00 = G (SHA3-512): rate=576, suffix=01, msg_len=264, out=512 +// 01 = H (SHA3-256): rate=1088, suffix=01, msg_len=256, out=256 +// 10 = J (SHAKE-256): rate=1088, suffix=1111,msg_len=512,out=256 +// +// Multi-block SHA3-256 (mb_en=1): +// Streams pre-padded 1088-bit rate blocks. The CALLER applies SHA3-256 +// padding (suffix 0x06 ... 0x80) to the final block so this module only +// does the absorb loop: state ^= block; state = Keccak-p(state); repeat. +// The single-block paths above are bit-identical when mb_en=0. +// +// Interface (single-block): +// mode[1:0] - 00=G, 01=H, 10=J +// data_i - 512-bit message input +// valid_i / ready_o / hash_o[511:0] / valid_o / ready_i +// +// Interface (multi-block SHA3-256, active when mb_en=1): +// mb_en - 1 selects the multi-block absorb path +// mb_block_i - one pre-padded 1088-bit (136-byte) rate block, byte 0 in [7:0] +// mb_valid_i - block valid +// mb_ready_o - module can accept a block +// mb_last_i - asserted with the final (already-padded) block +// result reuses hash_o[255:0] / valid_o / ready_i + +module sha3_top ( + input clk, + input rst_n, + input [1:0] mode, + input [511:0] data_i, + input valid_i, + output ready_o, + output [511:0] hash_o, + output valid_o, + input ready_i, + // --- multi-block SHA3-256 absorb (tie mb_en=0 to disable) --- + input mb_en, + input [1087:0] mb_block_i, + input mb_valid_i, + input mb_last_i, + output mb_ready_o +); + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 2'd0; + localparam ST_PERMUTE = 2'd1; + localparam ST_SQUEEZE = 2'd2; + + reg [1:0] state_r, state_next; + + // ================================================================ + // Absorb state: message || suffix || pad10*1 into rate bits + // Built combinationally from data_i and mode. + // + // G: padded_block = {1, 308'b0, 1, 2'b01, data_i[263:0]} + // absorb_state = {1024'b0, padded_block_576} + // + // H: padded_block = {1, 828'b0, 1, 2'b01, data_i[255:0]} + // absorb_state = {512'b0, padded_block_1088} + // + // J: padded_block = {1, 570'b0, 1, 4'b1111, data_i[511:0]} + // absorb_state = {512'b0, padded_block_1088} + // ================================================================ + wire [575:0] g_pad; + wire [1087:0] h_pad; + wire [1087:0] j_pad; + + assign g_pad = {1'b1, {308{1'b0}}, 1'b1, 2'b10, data_i[263:0]}; + assign h_pad = {1'b1, {828{1'b0}}, 1'b1, 2'b10, data_i[255:0]}; + // J: SHAKE suffix is "1111" — all ones, order irrelevant + assign j_pad = {1'b1, {570{1'b0}}, 1'b1, 4'b1111, data_i[511:0]}; + + wire [1599:0] absorb_state; + assign absorb_state = (mode == 2'b00) ? {{(1600-576){1'b0}}, g_pad} : + (mode == 2'b01) ? {{(1600-1088){1'b0}}, h_pad} : + (mode == 2'b10) ? {{(1600-1088){1'b0}}, j_pad} : + 1600'd0; + + // ================================================================ + // Multi-block SHA3-256 absorb FSM (active only when mb_en=1) + // + // Running sponge state mb_state_r (init 0). For each pre-padded + // 1088-bit rate block: mb_state_r ^= block; mb_state_r = Keccak-p(...). + // After the last block, squeeze 256 bits. The caller pads the final + // block (SHA3-256 suffix 0x06 ... 0x80), so this FSM is pure absorb. + // ================================================================ + localparam MB_IDLE = 2'd0; // ready for a block (or first block) + localparam MB_PERMUTE = 2'd1; // keccak running on xored state + localparam MB_DONE = 2'd2; // squeeze: present 256-bit digest + + reg [1:0] mb_state, mb_state_next; + reg [1599:0] mb_state_r; // running sponge state + reg mb_last_r; // captured last-block flag + reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE) + + // XOR the incoming block into the low 1088 bits (rate) of the state. + wire [1599:0] mb_xored; + assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i}; + + // Accept a block only in MB_IDLE while enabled. + assign mb_ready_o = mb_en && (mb_state == MB_IDLE); + wire mb_accept = mb_en && (mb_state == MB_IDLE) && mb_valid_i; + wire mb_kc_valid = mb_accept; // start keccak on the accept cycle + + always @(*) begin + mb_state_next = mb_state; + case (mb_state) + MB_IDLE: if (mb_accept) mb_state_next = MB_PERMUTE; + MB_PERMUTE: if (kc_valid_o) mb_state_next = mb_last_r ? MB_DONE : MB_IDLE; + MB_DONE: if (ready_i) mb_state_next = MB_IDLE; + default: mb_state_next = MB_IDLE; + endcase + end + + // ================================================================ + // Keccak core + // ================================================================ + wire kc_valid_i; + /* verilator lint_off UNUSEDSIGNAL */ + wire [1599:0] kc_state_o; + wire kc_ready_o; + /* verilator lint_on UNUSEDSIGNAL */ + wire kc_valid_o; + + // Keccak input: multi-block xored state when mb_en, else single-block absorb. + wire [1599:0] kc_state_i_mux; + assign kc_state_i_mux = mb_en ? mb_xored : absorb_state; + + keccak_core #(.ROUNDS(24)) u_keccak ( + .clk (clk), + .rst_n (rst_n), + .state_i (kc_state_i_mux), + .valid_i (kc_valid_i), + .ready_o (kc_ready_o), // unused but must connect + .state_o (kc_state_o), + .valid_o (kc_valid_o), + .ready_i (1'b1) // always accept output + ); + + // ================================================================ + // FSM combinational logic + // ================================================================ + assign ready_o = !mb_en && (state_r == ST_IDLE); + + // kc_valid_i: single-block start (state_next==PERMUTE) OR multi-block accept. + assign kc_valid_i = mb_en ? mb_kc_valid : (state_next == ST_PERMUTE); + + always @(*) begin + state_next = state_r; + case (state_r) + ST_IDLE: if (valid_i && ready_o) state_next = ST_PERMUTE; + ST_PERMUTE: if (kc_valid_o) state_next = ST_SQUEEZE; + ST_SQUEEZE: if (valid_o && ready_i) state_next = ST_IDLE; + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // Output + // ================================================================ + // Register for squeezed output (only 512 bits needed) + reg [511:0] squeezed_state_r; + + // valid_o / hash_o serve both paths, selected by mb_en. + assign valid_o = mb_en ? (mb_state == MB_DONE) : (state_r == ST_SQUEEZE); + assign hash_o = mb_en ? {256'b0, mb_digest_r} : squeezed_state_r; + + // ================================================================ + // Sequential logic + // ================================================================ + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + squeezed_state_r <= 512'd0; + mb_state <= MB_IDLE; + mb_state_r <= 1600'd0; + mb_last_r <= 1'b0; + mb_digest_r <= 256'd0; + end else begin + state_r <= state_next; + mb_state <= mb_state_next; + + // --- single-block: latch squeezed output --- + if (state_r == ST_PERMUTE && kc_valid_o) begin + squeezed_state_r <= kc_state_o[511:0]; + end + + // --- multi-block: capture last flag on accept --- + if (mb_accept) begin + mb_last_r <= mb_last_i; + end + + // --- multi-block: latch permuted state when keccak finishes --- + if (mb_state == MB_PERMUTE && kc_valid_o) begin + mb_state_r <= kc_state_o; + // On the final block, latch the 256-bit digest (sticky for MB_DONE). + if (mb_last_r) mb_digest_r <= kc_state_o[255:0]; + end + + // --- multi-block: clear running state after digest consumed --- + if (mb_state == MB_DONE && ready_i) begin + mb_state_r <= 1600'd0; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/sha3/sha3_top_shared.v b/rtl/ip/mlkem/sha3/sha3_top_shared.v new file mode 100644 index 0000000..101aff2 --- /dev/null +++ b/rtl/ip/mlkem/sha3/sha3_top_shared.v @@ -0,0 +1,187 @@ +// sha3_top_shared.v - SHA3/SHAKE wrapper with EXTERNAL (shared) keccak_core. +// +// Identical to sha3_top.v except the internal keccak_core instance is +// replaced by explicit kc_* ports, so several Keccak consumers can share a +// single keccak_core via a phase mux (see mlkem_top). The sponge state +// (mb_state_r) and all FSM/squeeze logic stay here; only the 24-round +// permutation is external. Bit-identical to sha3_top when wired to a +// dedicated keccak_core. +// +// Single-block modes (mb_en=0): +// 00 = G (SHA3-512): rate=576, suffix=01, msg_len=264, out=512 +// 01 = H (SHA3-256): rate=1088, suffix=01, msg_len=256, out=256 +// 10 = J (SHAKE-256): rate=1088, suffix=1111,msg_len=512,out=256 +// +// Multi-block SHA3-256 (mb_en=1): streams pre-padded 1088-bit rate blocks; +// the CALLER applies SHA3-256 padding to the final block. +// +// Keccak interface (port names mirror the keccak_core port each connects to): +// kc_state_o[1599:0] - keccak result (input, from keccak_core.state_o) +// kc_valid_o - keccak output valid (input, from keccak_core.valid_o) +// kc_ready_o - keccak ready for input (input, from keccak_core.ready_o) +// kc_state_i[1599:0] - keccak input state (output, to keccak_core.state_i) +// kc_valid_i - request permutation (output, to keccak_core.valid_i) +// kc_ready_i - accept keccak output (output, to keccak_core.ready_i, always 1) + +module sha3_top_shared ( + input clk, + input rst_n, + input [1:0] mode, + input [511:0] data_i, + input valid_i, + output ready_o, + output [511:0] hash_o, + output valid_o, + input ready_i, + // --- multi-block SHA3-256 absorb (tie mb_en=0 to disable) --- + input mb_en, + input [1087:0] mb_block_i, + input mb_valid_i, + input mb_last_i, + output mb_ready_o, + // --- external (shared) keccak_core interface --- + input [1599:0] kc_state_o, + input kc_valid_o, + /* verilator lint_off UNUSEDSIGNAL */ + input kc_ready_o, + /* verilator lint_on UNUSEDSIGNAL */ + output [1599:0] kc_state_i, + output kc_valid_i, + output kc_ready_i +); + + // ================================================================ + // FSM state encoding + // ================================================================ + localparam ST_IDLE = 2'd0; + localparam ST_PERMUTE = 2'd1; + localparam ST_SQUEEZE = 2'd2; + + reg [1:0] state_r, state_next; + + // ================================================================ + // Absorb state: message || suffix || pad10*1 into rate bits + // ================================================================ + wire [575:0] g_pad; + wire [575:0] g512_pad; + wire [1087:0] h_pad; + wire [1087:0] j_pad; + + assign g_pad = {1'b1, {308{1'b0}}, 1'b1, 2'b10, data_i[263:0]}; + assign h_pad = {1'b1, {828{1'b0}}, 1'b1, 2'b10, data_i[255:0]}; + // J: SHAKE suffix is "1111" — all ones, order irrelevant + assign j_pad = {1'b1, {570{1'b0}}, 1'b1, 4'b1111, data_i[511:0]}; + // mode 11 = G over a full 64-byte (512-bit) message (Encaps: G(m||H(ek))). + // SHA3-512 rate=576; suffix 01 + pad10*1: 1 + 60 zeros + 1 + 2'b10 + 512 = 576. + assign g512_pad = {1'b1, {60{1'b0}}, 1'b1, 2'b10, data_i[511:0]}; + + wire [1599:0] absorb_state; + assign absorb_state = (mode == 2'b00) ? {{(1600-576){1'b0}}, g_pad} : + (mode == 2'b01) ? {{(1600-1088){1'b0}}, h_pad} : + (mode == 2'b10) ? {{(1600-1088){1'b0}}, j_pad} : + {{(1600-576){1'b0}}, g512_pad}; + + // ================================================================ + // Multi-block SHA3-256 absorb FSM (active only when mb_en=1) + // ================================================================ + localparam MB_IDLE = 2'd0; // ready for a block (or first block) + localparam MB_PERMUTE = 2'd1; // keccak running on xored state + localparam MB_DONE = 2'd2; // squeeze: present 256-bit digest + + reg [1:0] mb_state, mb_state_next; + reg [1599:0] mb_state_r; // running sponge state + reg mb_last_r; // captured last-block flag + reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE) + + // XOR the incoming block into the low 1088 bits (rate) of the state. + wire [1599:0] mb_xored; + assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i}; + + // Accept a block only in MB_IDLE while enabled. + assign mb_ready_o = mb_en && (mb_state == MB_IDLE); + wire mb_accept = mb_en && (mb_state == MB_IDLE) && mb_valid_i; + wire mb_kc_valid = mb_accept; // start keccak on the accept cycle + + always @(*) begin + mb_state_next = mb_state; + case (mb_state) + MB_IDLE: if (mb_accept) mb_state_next = MB_PERMUTE; + MB_PERMUTE: if (kc_valid_o) mb_state_next = mb_last_r ? MB_DONE : MB_IDLE; + MB_DONE: if (ready_i) mb_state_next = MB_IDLE; + default: mb_state_next = MB_IDLE; + endcase + end + + // ================================================================ + // External keccak_core interface (was internal keccak_core) + // ================================================================ + // Keccak input: multi-block xored state when mb_en, else single-block absorb. + assign kc_state_i = mb_en ? mb_xored : absorb_state; + // Always accept keccak output (matches the dedicated-core ready_i=1'b1). + assign kc_ready_i = 1'b1; + // kc_valid_i: single-block start (state_next==PERMUTE) OR multi-block accept. + assign kc_valid_i = mb_en ? mb_kc_valid : (state_next == ST_PERMUTE); + + // ================================================================ + // FSM combinational logic + // ================================================================ + assign ready_o = !mb_en && (state_r == ST_IDLE); + + always @(*) begin + state_next = state_r; + case (state_r) + ST_IDLE: if (valid_i && ready_o) state_next = ST_PERMUTE; + ST_PERMUTE: if (kc_valid_o) state_next = ST_SQUEEZE; + ST_SQUEEZE: if (valid_o && ready_i) state_next = ST_IDLE; + default: state_next = ST_IDLE; + endcase + end + + // ================================================================ + // Output + // ================================================================ + reg [511:0] squeezed_state_r; + + // valid_o / hash_o serve both paths, selected by mb_en. + assign valid_o = mb_en ? (mb_state == MB_DONE) : (state_r == ST_SQUEEZE); + assign hash_o = mb_en ? {256'b0, mb_digest_r} : squeezed_state_r; + + // ================================================================ + // Sequential logic + // ================================================================ + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_r <= ST_IDLE; + squeezed_state_r <= 512'd0; + mb_state <= MB_IDLE; + mb_state_r <= 1600'd0; + mb_last_r <= 1'b0; + mb_digest_r <= 256'd0; + end else begin + state_r <= state_next; + mb_state <= mb_state_next; + + // --- single-block: latch squeezed output --- + if (state_r == ST_PERMUTE && kc_valid_o) begin + squeezed_state_r <= kc_state_o[511:0]; + end + + // --- multi-block: capture last flag on accept --- + if (mb_accept) begin + mb_last_r <= mb_last_i; + end + + // --- multi-block: latch permuted state when keccak finishes --- + if (mb_state == MB_PERMUTE && kc_valid_o) begin + mb_state_r <= kc_state_o; + if (mb_last_r) mb_digest_r <= kc_state_o[255:0]; + end + + // --- multi-block: clear running state after digest consumed --- + if (mb_state == MB_DONE && ready_i) begin + mb_state_r <= 1600'd0; + end + end + end + +endmodule diff --git a/rtl/ip/mlkem/storage/s_bram.v b/rtl/ip/mlkem/storage/s_bram.v new file mode 100644 index 0000000..972b9b8 --- /dev/null +++ b/rtl/ip/mlkem/storage/s_bram.v @@ -0,0 +1,31 @@ +// s_bram.v - Single-port behavioral RAM (shared read/write port) +// +// One port: either read or write per cycle (+1 cycle read latency). +// When both rd_en and wr_en are high, write takes priority. +// Read data is registered (1-cycle latency after rd_en). +// No valid/ready handshake — pure memory, handshake managed at higher level. + +module s_bram #( + parameter W = 48, + parameter D = 512, + parameter A = 9 +) ( + input wire clk, + input wire rd_en, + input wire [A-1:0] rd_addr, + output reg [W-1:0] rd_data, + input wire wr_en, + input wire [A-1:0] wr_addr, + input wire [W-1:0] wr_data +); + + reg [W-1:0] mem [0:D-1]; + + always @(posedge clk) begin + if (wr_en) + mem[wr_addr] <= wr_data; + else if (rd_en) + rd_data <= mem[rd_addr]; + end + +endmodule diff --git a/rtl/ip/mlkem/storage/sd_bram.v b/rtl/ip/mlkem/storage/sd_bram.v new file mode 100644 index 0000000..5bcb7c6 --- /dev/null +++ b/rtl/ip/mlkem/storage/sd_bram.v @@ -0,0 +1,32 @@ +// sd_bram.v - Simple dual-port behavioral RAM (1 read + 1 write port) +// +// Vivado auto-infers this as BRAM when W >= 12 and D >= 64. +// Read: rd_addr sampled at posedge → rd_addr_r → rd_data = mem[rd_addr_r] +// Write: wr_data written to mem[wr_addr] at posedge when wr_en=1 +// No valid/ready handshake — pure combinational-read / registered-write memory. + +module sd_bram #( + parameter W = 48, + parameter D = 64, + parameter A = 6 +) ( + input wire clk, + input wire [A-1:0] rd_addr, + output wire [W-1:0] rd_data, + input wire wr_en, + input wire [A-1:0] wr_addr, + input wire [W-1:0] wr_data +); + + reg [W-1:0] mem [0:D-1]; + reg [A-1:0] rd_addr_r; + + always @(posedge clk) begin + rd_addr_r <= rd_addr; + if (wr_en) + mem[wr_addr] <= wr_data; + end + + assign rd_data = mem[rd_addr_r]; + +endmodule diff --git a/rtl/ip/mlkem/top/mlkem_axi_slave.v b/rtl/ip/mlkem/top/mlkem_axi_slave.v new file mode 100644 index 0000000..b58d407 --- /dev/null +++ b/rtl/ip/mlkem/top/mlkem_axi_slave.v @@ -0,0 +1,524 @@ +// mlkem_axi_slave.v — AXI4 (burst) slave wrapper for the ML-KEM core. +// Makes the ML-KEM accelerator (KeyGen / Encaps / Decaps) CPU-controllable. +// +// The bus runs at sys_clk (50 MHz). The ML-KEM core cannot close timing at +// 50 MHz, so it is clocked by mlkem_clk = sys_clk / 2 (25 MHz), produced by a +// toggle flip-flop in this wrapper. mlkem_clk is a phase-aligned synchronous +// divide of sys_clk; every signal crossing between the two domains still goes +// through an explicit synchronizer or toggle handshake. +// NOTE (FPGA): for real silicon route mlkem_clk through a BUFG/BUFGCE rather +// than a logic-generated net; kept as a plain toggle here for portability. +// +// Register map (byte offsets from base, e.g. virtual 0xbf400000): +// 0x000 CTRL W bit0 = start (launches op selected in CONFIG) +// 0x004 STATUS R bit0 busy, bit1 done, bit2 preload_busy +// W1C write bit1=1 to clear the latched done flag +// 0x008 CONFIG RW [2:0] k (2/3/4), [5:4] op (0=KeyGen 1=Encaps 2=Decaps) +// 0x010..0x02C D RW KeyGen seed d (8 words; d byte0 in word0[7:0]) +// 0x030..0x04C Z RW implicit-reject seed z (8 words) +// 0x050..0x06C MSG RW Encaps message m (8 words) +// 0x070..0x08C SS R shared secret out (8 words) +// 0x100 PRELOAD_SEL RW 0=ek_in 1=dk_in 2=c_in +// 0x104 PRELOAD_ADDR RW byte address (auto-increments after each data write) +// 0x108 PRELOAD_DATA W write low byte -> streams to selected port (wait-stated) +// 0x10C RDBK_SEL RW 0=ek 1=dk_pke 2=full-dk 3=ciphertext +// 0x110 RDBK_ADDR RW output byte index +// 0x114 RDBK_DATA R selected readback-tap byte (low 8 bits) +module mlkem_axi_slave ( + input clk, // sys_clk (AXI bus clock, 50 MHz) + input resetn, // sys_resetn (active low, already synchronized) + + // AXI Write Address Channel + input [4:0] s_axi_awid, + input [31:0] s_axi_awaddr, + input [7:0] s_axi_awlen, + input [2:0] s_axi_awsize, + input [1:0] s_axi_awburst, + input s_axi_awlock, + input [3:0] s_axi_awcache, + input [2:0] s_axi_awprot, + input s_axi_awvalid, + output reg s_axi_awready, + + // AXI Write Data Channel + input [4:0] s_axi_wid, + input [31:0] s_axi_wdata, + input [3:0] s_axi_wstrb, + input s_axi_wlast, + input s_axi_wvalid, + output reg s_axi_wready, + + // AXI Write Response Channel + output [4:0] s_axi_bid, + output [1:0] s_axi_bresp, + output reg s_axi_bvalid, + input s_axi_bready, + + // AXI Read Address Channel + input [4:0] s_axi_arid, + input [31:0] s_axi_araddr, + input [7:0] s_axi_arlen, + input [2:0] s_axi_arsize, + input [1:0] s_axi_arburst, + input s_axi_arlock, + input [3:0] s_axi_arcache, + input [2:0] s_axi_arprot, + input s_axi_arvalid, + output reg s_axi_arready, + + // AXI Read Data Channel + output reg [4:0] s_axi_rid, + output reg [31:0] s_axi_rdata, + output [1:0] s_axi_rresp, + output reg s_axi_rlast, + output reg s_axi_rvalid, + input s_axi_rready, + + // Interrupt output (active high level, asserted while done latched) + output intr +); + + // ========================================== + // Address map (byte offsets within the 1 MB slave window) + // ========================================== + localparam ADDR_CTRL = 12'h000; + localparam ADDR_STATUS = 12'h004; + localparam ADDR_CONFIG = 12'h008; + localparam ADDR_D = 12'h010; // 0x010..0x02C (8 words) + localparam ADDR_Z = 12'h030; // 0x030..0x04C + localparam ADDR_MSG = 12'h050; // 0x050..0x06C + localparam ADDR_SS = 12'h070; // 0x070..0x08C (read-only) + localparam ADDR_PRE_SEL = 12'h100; + localparam ADDR_PRE_ADR = 12'h104; + localparam ADDR_PRE_DAT = 12'h108; + localparam ADDR_RDB_SEL = 12'h10C; + localparam ADDR_RDB_ADR = 12'h110; + localparam ADDR_RDB_DAT = 12'h114; + + assign s_axi_bresp = 2'b00; + assign s_axi_rresp = 2'b00; + + function [31:0] apply_wstrb; + input [31:0] old_value; + input [31:0] new_value; + input [3:0] strobe; + begin + apply_wstrb = old_value; + if (strobe[0]) apply_wstrb[7:0] = new_value[7:0]; + if (strobe[1]) apply_wstrb[15:8] = new_value[15:8]; + if (strobe[2]) apply_wstrb[23:16] = new_value[23:16]; + if (strobe[3]) apply_wstrb[31:24] = new_value[31:24]; + end + endfunction + + // ========================================== + // mlkem_clk = sys_clk / 2 (25 MHz) via toggle FF. + // ========================================== + reg mlkem_clk; + always @(posedge clk or negedge resetn) begin + if (!resetn) mlkem_clk <= 1'b0; + else mlkem_clk <= ~mlkem_clk; + end + // Reset for the mlkem domain: reuse sys_resetn (async-assert, and the core's + // own rst_n is sampled on mlkem_clk edges). + wire mlkem_rstn = resetn; + + // ========================================== + // sys_clk-domain config / data registers (quasi-static; fed to the core + // combinationally — written long before start, stable across the op). + // ========================================== + reg [2:0] cfg_k; // 2/3/4 + reg [1:0] cfg_op; // 0=KeyGen 1=Encaps 2=Decaps + reg [31:0] d_word [0:7]; + reg [31:0] z_word [0:7]; + reg [31:0] msg_word [0:7]; + + wire [255:0] d_bus = {d_word[7], d_word[6], d_word[5], d_word[4], + d_word[3], d_word[2], d_word[1], d_word[0]}; + wire [255:0] z_bus = {z_word[7], z_word[6], z_word[5], z_word[4], + z_word[3], z_word[2], z_word[1], z_word[0]}; + wire [255:0] msg_bus = {msg_word[7],msg_word[6],msg_word[5],msg_word[4], + msg_word[3],msg_word[2],msg_word[1],msg_word[0]}; + + // Preload / readback index registers (sys domain). + reg [1:0] pre_sel; // 0=ek 1=dk 2=c + reg [11:0] pre_addr; // byte address into the selected port + reg [1:0] rdb_sel; // 0=ek 1=dk_pke 2=full-dk 3=ciphertext + reg [11:0] rdb_addr; // output byte index + + // ========================================== + // CDC: start pulse (sys toggle -> mlkem edge-detect -> 1-cyc start_i) + // ========================================== + reg start_tg; // sys-domain toggle, flips on CTRL.start write + reg [2:0] start_sync; // mlkem-domain 2FF sync (+1 for edge) + wire mlkem_start = start_sync[2] ^ start_sync[1]; // 1 mlkem_clk pulse + + // ========================================== + // CDC: busy / done back to sys + // ========================================== + wire core_busy; // busy_o (mlkem domain) + wire core_done; // done_o pulse (mlkem domain) + reg done_latch; // sticky done in mlkem domain + reg [1:0] busy_sync; // -> sys + reg [1:0] done_sync; // -> sys + // done-clear: sys toggle -> mlkem edge -> clears done_latch + reg dclr_tg; // sys-domain toggle (STATUS W1C of done) + reg [2:0] dclr_sync; // mlkem-domain 2FF sync (+1 for edge) + wire mlkem_dclr = dclr_sync[2] ^ dclr_sync[1]; + + // ========================================== + // CDC: preload byte write handshake. + // sys: latch {sel,addr,byte}, flip pre_req_tg, hold wready low. + // mlkem: 2FF sync pre_req_tg, on edge pulse the selected *_in_we 1 cycle, + // flip pre_ack_tg. + // sys: 2FF sync pre_ack_tg, on edge release the bus (addr++). + // ========================================== + reg pre_req_tg; // sys toggle: a byte is pending + reg [2:0] pre_req_sync; // mlkem 2FF (+1 for edge) + wire mlkem_pre_we = pre_req_sync[2] ^ pre_req_sync[1]; + reg pre_ack_tg; // mlkem toggle: byte consumed + reg [2:0] pre_ack_sync; // sys 2FF (+1 for edge) + wire pre_ack_edge = pre_ack_sync[2] ^ pre_ack_sync[1]; + reg pre_busy; // sys: a preload handshake is in flight + // latched byte/port for the in-flight preload (sys, stable while pre_busy) + reg [1:0] pre_sel_l; + reg [11:0] pre_addr_l; + reg [7:0] pre_byte_l; + + // ========================================== + // mlkem_top instance + // ========================================== + wire [7:0] dbg_byte_o, dbg_dk_o, dbg_ct_o; + wire [255:0] ss_o; + wire dbg_byte_sel = (rdb_sel == 2'd1); // 0=ek, 1=dk_pke + + // one-cycle write-enable pulses into the core (combinational off the synced + // edge; pre_sel_l/pre_addr_l/pre_byte_l are stable in sys, sampled by core). + wire ek_in_we = mlkem_pre_we && (pre_sel_l == 2'd0); + wire dk_in_we = mlkem_pre_we && (pre_sel_l == 2'd1); + wire c_in_we = mlkem_pre_we && (pre_sel_l == 2'd2); + + mlkem_top #(.KMAX(4)) u_mlkem ( + .clk (mlkem_clk), + .rst_n (mlkem_rstn), + .k_i (cfg_k), + .op_i (cfg_op), + .d_i (d_bus), + .z_i (z_bus), + .msg_i (msg_bus), + .start_i (mlkem_start), + .busy_o (core_busy), + .done_o (core_done), + // ek preload + .ek_in_we (ek_in_we), + .ek_in_addr (pre_addr_l[10:0]), + .ek_in_byte (pre_byte_l), + // dk preload + .dk_in_we (dk_in_we), + .dk_in_addr (pre_addr_l), + .dk_in_byte (pre_byte_l), + // ciphertext preload + .c_in_we (c_in_we), + .c_in_addr (pre_addr_l[10:0]), + .c_in_byte (pre_byte_l), + // shared secret out + .ss_o (ss_o), + // ciphertext readback tap + .dbg_ct_idx_i (rdb_addr[10:0]), + .dbg_ct_o (dbg_ct_o), + // coefficient debug tap (unused by CPU path; tie inputs off) + .dbg_slot_i (6'd0), + .dbg_idx_i (8'd0), + .dbg_coeff_o (/* unused */), + // ek / dk_pke byte readback + .dbg_byte_sel_i (dbg_byte_sel), + .dbg_byte_idx_i (rdb_addr[10:0]), + .dbg_byte_o (dbg_byte_o), + // full-dk byte readback + .dbg_dk_idx_i (rdb_addr), + .dbg_dk_o (dbg_dk_o), + // remaining debug taps unused + .dbg_rho_o (/* unused */), + .dbg_sigma_o (/* unused */), + .dbg_r_o (/* unused */), + .dbg_hek_o (/* unused */), + .dbg_mprime_o (/* unused */), + .dbg_kbar_o (/* unused */), + .dbg_decz_o (/* unused */), + .dbg_dech_o (/* unused */) + ); + + // ========================================== + // mlkem_clk domain: synchronizers, done latch, preload-ack toggle. + // ========================================== + always @(posedge mlkem_clk or negedge mlkem_rstn) begin + if (!mlkem_rstn) begin + start_sync <= 3'd0; + dclr_sync <= 3'd0; + pre_req_sync <= 3'd0; + pre_ack_tg <= 1'b0; + done_latch <= 1'b0; + end else begin + // 2FF (+edge) synchronizers of sys-domain toggles + start_sync <= {start_sync[1:0], start_tg}; + dclr_sync <= {dclr_sync[1:0], dclr_tg}; + pre_req_sync <= {pre_req_sync[1:0], pre_req_tg}; + + // acknowledge a consumed preload byte (write-enable pulsed this cycle) + if (mlkem_pre_we) pre_ack_tg <= ~pre_ack_tg; + + // sticky done: set on the core's done_o pulse, cleared by synced W1C. + // (mlkem_start also clears it so a new run starts with done low.) + if (core_done) done_latch <= 1'b1; + else if (mlkem_dclr) done_latch <= 1'b0; + else if (mlkem_start) done_latch <= 1'b0; + end + end + + // ========================================== + // sys_clk domain: sync busy/done/preload-ack back from mlkem. + // ========================================== + always @(posedge clk or negedge resetn) begin + if (!resetn) begin + busy_sync <= 2'd0; + done_sync <= 2'd0; + pre_ack_sync <= 3'd0; + end else begin + busy_sync <= {busy_sync[0], core_busy}; + done_sync <= {done_sync[0], done_latch}; + pre_ack_sync <= {pre_ack_sync[1:0], pre_ack_tg}; + end + end + wire status_busy = busy_sync[1]; + wire status_done = done_sync[1]; + + assign intr = status_done; + + // ========================================== + // AXI Write channel (INCR burst). Preload-data beats are wait-stated until + // the cross-domain byte handshake completes (W_PRELOAD). + // ========================================== + localparam W_IDLE = 2'd0; + localparam W_DATA = 2'd1; + localparam W_PRELOAD = 2'd2; + localparam W_RESP = 2'd3; + + reg [1:0] w_state; + reg [31:0] w_addr; + reg [4:0] w_id; + reg [1:0] w_burst; + reg w_last_l; // saved wlast across a preload wait + wire [11:0] w_offset = w_addr[11:0]; + + assign s_axi_bid = w_id; + + integer wi; + always @(posedge clk or negedge resetn) begin + if (!resetn) begin + w_state <= W_IDLE; + s_axi_awready <= 1'b1; + s_axi_wready <= 1'b0; + s_axi_bvalid <= 1'b0; + cfg_k <= 3'd4; // default ML-KEM-1024 + cfg_op <= 2'd0; + pre_sel <= 2'd0; + pre_addr <= 12'd0; + rdb_sel <= 2'd0; + rdb_addr <= 12'd0; + start_tg <= 1'b0; + dclr_tg <= 1'b0; + pre_req_tg <= 1'b0; + pre_busy <= 1'b0; + pre_sel_l <= 2'd0; + pre_addr_l <= 12'd0; + pre_byte_l <= 8'd0; + w_last_l <= 1'b0; + for (wi = 0; wi < 8; wi = wi + 1) begin + d_word[wi] <= 32'd0; + z_word[wi] <= 32'd0; + msg_word[wi] <= 32'd0; + end + end else begin + // clear the preload-busy flag when the mlkem domain acks the byte + if (pre_ack_edge) begin + pre_busy <= 1'b0; + pre_addr <= pre_addr + 12'd1; // auto-increment for next byte + end + + case (w_state) + W_IDLE: begin + s_axi_awready <= 1'b1; + if (s_axi_awvalid && s_axi_awready) begin + s_axi_awready <= 1'b0; + s_axi_wready <= 1'b1; + w_addr <= s_axi_awaddr; + w_id <= s_axi_awid; + w_burst <= s_axi_awburst; + w_state <= W_DATA; + end + end + + W_DATA: begin + if (s_axi_wvalid && s_axi_wready) begin + if (w_offset == ADDR_PRE_DAT) begin + // launch cross-domain byte write; stall the beat. + pre_sel_l <= pre_sel; + pre_addr_l <= pre_addr; + pre_byte_l <= s_axi_wdata[7:0]; + pre_req_tg <= ~pre_req_tg; + pre_busy <= 1'b1; + w_last_l <= s_axi_wlast; + s_axi_wready <= 1'b0; + w_state <= W_PRELOAD; + end else begin + // normal single-cycle register write + case (1'b1) + (w_offset == ADDR_CTRL): + if (s_axi_wdata[0]) start_tg <= ~start_tg; + (w_offset == ADDR_STATUS): + if (s_axi_wdata[1]) dclr_tg <= ~dclr_tg; // clear done + (w_offset == ADDR_CONFIG): begin + cfg_k <= s_axi_wdata[2:0]; + cfg_op <= s_axi_wdata[5:4]; + end + (w_offset == ADDR_PRE_SEL): pre_sel <= s_axi_wdata[1:0]; + (w_offset == ADDR_PRE_ADR): pre_addr <= s_axi_wdata[11:0]; + (w_offset == ADDR_RDB_SEL): rdb_sel <= s_axi_wdata[1:0]; + (w_offset == ADDR_RDB_ADR): rdb_addr <= s_axi_wdata[11:0]; + (w_offset >= ADDR_D && w_offset < ADDR_D + 12'h20): + d_word[(w_offset-ADDR_D)>>2] <= apply_wstrb(d_word[(w_offset-ADDR_D)>>2], s_axi_wdata, s_axi_wstrb); + (w_offset >= ADDR_Z && w_offset < ADDR_Z + 12'h20): + z_word[(w_offset-ADDR_Z)>>2] <= apply_wstrb(z_word[(w_offset-ADDR_Z)>>2], s_axi_wdata, s_axi_wstrb); + (w_offset >= ADDR_MSG && w_offset < ADDR_MSG + 12'h20): + msg_word[(w_offset-ADDR_MSG)>>2] <= apply_wstrb(msg_word[(w_offset-ADDR_MSG)>>2], s_axi_wdata, s_axi_wstrb); + default: ; // SS / RDB_DAT / unmapped: ignore + endcase + + if (w_burst == 2'b01) w_addr <= w_addr + 32'd4; + if (s_axi_wlast) begin + s_axi_wready <= 1'b0; + s_axi_bvalid <= 1'b1; + w_state <= W_RESP; + end + end + end + end + + W_PRELOAD: begin + // wait for the synced ack (pre_ack_edge clears pre_busy above) + if (!pre_busy) begin + if (w_burst == 2'b01) w_addr <= w_addr + 32'd4; + if (w_last_l) begin + s_axi_bvalid <= 1'b1; + w_state <= W_RESP; + end else begin + s_axi_wready <= 1'b1; + w_state <= W_DATA; + end + end + end + + W_RESP: begin + if (s_axi_bvalid && s_axi_bready) begin + s_axi_bvalid <= 1'b0; + s_axi_awready <= 1'b1; + w_state <= W_IDLE; + end + end + default: w_state <= W_IDLE; + endcase + end + end + + // ========================================== + // AXI Read channel (INCR burst). + // ========================================== + localparam R_IDLE = 1'b0; + localparam R_DATA = 1'b1; + + reg r_state; + reg [31:0] r_addr; + reg [7:0] r_cnt; + reg [7:0] r_len; + reg [1:0] r_burst; + wire [11:0] r_offset = r_addr[11:0]; + + always @(posedge clk or negedge resetn) begin + if (!resetn) begin + r_state <= R_IDLE; + s_axi_arready <= 1'b1; + s_axi_rvalid <= 1'b0; + s_axi_rlast <= 1'b0; + s_axi_rid <= 5'd0; + r_addr <= 32'd0; + r_cnt <= 8'd0; + r_len <= 8'd0; + r_burst <= 2'd0; + end else begin + case (r_state) + R_IDLE: begin + s_axi_arready <= 1'b1; + if (s_axi_arvalid && s_axi_arready) begin + s_axi_arready <= 1'b0; + r_addr <= s_axi_araddr; + r_len <= s_axi_arlen; + r_burst <= s_axi_arburst; + r_cnt <= 8'd0; + s_axi_rvalid <= 1'b1; + s_axi_rlast <= (s_axi_arlen == 8'd0); + s_axi_rid <= s_axi_arid; + r_state <= R_DATA; + end + end + R_DATA: begin + if (s_axi_rvalid && s_axi_rready) begin + if (r_cnt == r_len) begin + s_axi_rvalid <= 1'b0; + s_axi_rlast <= 1'b0; + s_axi_arready <= 1'b1; + r_state <= R_IDLE; + end else begin + r_cnt <= r_cnt + 8'd1; + s_axi_rlast <= (r_cnt + 8'd1 == r_len); + if (r_burst == 2'b01) r_addr <= r_addr + 32'd4; + end + end + end + default: r_state <= R_IDLE; + endcase + end + end + + // Read-data mux. Use the live address while in R_IDLE accepting AR so the + // first beat is valid the cycle rvalid asserts (matches matmul style). + wire [11:0] cur_off = (r_state == R_DATA) ? r_offset : s_axi_araddr[11:0]; + // Selected readback byte (taps are combinational over the core's BRAMs). + wire [7:0] rdb_byte = (rdb_sel == 2'd3) ? dbg_ct_o : + (rdb_sel == 2'd2) ? dbg_dk_o : dbg_byte_o; + + always @(*) begin + s_axi_rdata = 32'd0; + if (r_state == R_DATA || (r_state == R_IDLE && s_axi_arvalid)) begin + case (1'b1) + (cur_off == ADDR_CTRL): s_axi_rdata = 32'd0; + (cur_off == ADDR_STATUS): s_axi_rdata = {29'd0, pre_busy, status_done, status_busy}; + (cur_off == ADDR_CONFIG): s_axi_rdata = {26'd0, cfg_op, 1'b0, cfg_k}; + (cur_off == ADDR_PRE_SEL): s_axi_rdata = {30'd0, pre_sel}; + (cur_off == ADDR_PRE_ADR): s_axi_rdata = {20'd0, pre_addr}; + (cur_off == ADDR_RDB_SEL): s_axi_rdata = {30'd0, rdb_sel}; + (cur_off == ADDR_RDB_ADR): s_axi_rdata = {20'd0, rdb_addr}; + (cur_off == ADDR_RDB_DAT): s_axi_rdata = {24'd0, rdb_byte}; + (cur_off >= ADDR_D && cur_off < ADDR_D + 12'h20): + s_axi_rdata = d_word[(cur_off-ADDR_D)>>2]; + (cur_off >= ADDR_Z && cur_off < ADDR_Z + 12'h20): + s_axi_rdata = z_word[(cur_off-ADDR_Z)>>2]; + (cur_off >= ADDR_MSG && cur_off < ADDR_MSG + 12'h20): + s_axi_rdata = msg_word[(cur_off-ADDR_MSG)>>2]; + (cur_off >= ADDR_SS && cur_off < ADDR_SS + 12'h20): + s_axi_rdata = ss_o[((cur_off-ADDR_SS)>>2)*32 +: 32]; + default: s_axi_rdata = 32'd0; + endcase + end + end + +endmodule diff --git a/rtl/ip/mlkem/top/mlkem_top.v b/rtl/ip/mlkem/top/mlkem_top.v new file mode 100644 index 0000000..440c5d7 --- /dev/null +++ b/rtl/ip/mlkem/top/mlkem_top.v @@ -0,0 +1,2252 @@ +// mlkem_top.v - ML-KEM KeyGen top-level integration. Runtime-selectable +// parameter set via k_i: k=2 (ML-KEM-512, eta1=3), k=3 (768), k=4 (1024). +// Storage is sized for KMAX (worst case = ML-KEM-1024); k_i picks the +// active sub-range at start_i. +// +// Streaming valid/ready interface. Given seeds d and z, computes the +// ML-KEM key pair per FIPS 203 Algorithm 16 (KeyGen_internal): +// (rho,sigma) = G(d || K) +// A_hat[i][j] = SampleNTT(rho || j || i) i,j in 0..K-1 +// s[i] = CBD3(PRF(sigma, i)), e[i] = CBD3(PRF(sigma, K+i)) +// s_hat[i] = NTT(s[i]); e_hat[i] = NTT(e[i]) +// t_hat[i] = e_hat[i] + sum_j A_hat[i][j] o s_hat[j] +// ek = byteEncode12(t_hat[0..K-1]) || rho +// dk = byteEncode12(s_hat[0..K-1]) || ek || H(ek) || z +// +// Built incrementally and verified stage-by-stage against reference golden +// vectors (test_framework/modules/mlkem_keygen/golden) and NIST KAT. +// +// Uses independent (verified) leaf modules over a shared keccak_core: +// sha3_top_shared, sample_ntt_sync_shared, sample_cbd_sync_shared, ntt_core, +// poly_mul_sync, comp_decomp_sync. + +`include "defines.vh" + +(* use_dsp = "no" *) +module mlkem_top #( + parameter KMAX = 4 // storage sizing (worst case = ML-KEM-1024) +) ( + input clk, + input rst_n, + input [2:0] k_i, // RUNTIME ML-KEM param: 2=512, 3=768, 4=1024 + input [1:0] op_i, // 0=KeyGen, 1=Encaps, 2=Decaps (captured at start_i) + input [255:0] d_i, // KeyGen seed d (byte 0 in d_i[7:0]) + input [255:0] z_i, // implicit-rejection seed z + input [255:0] msg_i, // Encaps message m (byte 0 in msg_i[7:0]) + input start_i, // pulse to begin KeyGen + output busy_o, // high while running + output done_o, // pulse when ek/dk ready + + // Encaps ek input: stream ek bytes (=pk) into ek_bram before start_i with + // op_i=1. ek_in_we writes byte ek_in_byte at address ek_in_addr. + input ek_in_we, // write one ek byte (only honored in ST_IDLE for Encaps preload) + input [10:0] ek_in_addr, // ek byte address 0..ek_bytes-1 + input [7:0] ek_in_byte, // ek byte value + // Decaps dk input: stream dk bytes (=sk, 768K+96) into the design before + // start_i with op_i=2. The load logic routes each byte by region: + // [0,384K) -> dk_pke (dkp_bram) = s_hat encoding + // [384K, 768K+32) -> ek_pke (ek_bram) = t_hat encoding || rho + // [768K+32, 768K+64) -> H(ek) (hek_r) + // [768K+64, 768K+96) -> z (z_r) + input dk_in_we, // write one dk byte (ST_IDLE Decaps preload) + input [11:0] dk_in_addr, // dk byte address 0..(768K+96)-1 + input [7:0] dk_in_byte, // dk byte value + // Decaps ciphertext input: stream c bytes (=ct) into c_in_bram before + // start_i. Kept separate from ct_bram so the computed c' can be compared + // against the original c (and J(z||c) can read c) without aliasing. + input c_in_we, // write one c byte (ST_IDLE Decaps preload) + input [10:0] c_in_addr, // c byte address 0..ct_bytes-1 + input [7:0] c_in_byte, // c byte value + // Encaps shared secret output (= K), valid at done_o + output [255:0] ss_o, + // Encaps ciphertext readback tap: ct byte by index 0..ct_bytes-1 + input [10:0] dbg_ct_idx_i, + output [7:0] dbg_ct_o, + + // Debug readback tap: read one stored coefficient by (poly slot, index). + // Lets stage TBs verify intermediates without wide buses. + input [5:0] dbg_slot_i, // poly slot (see localparams below) + input [7:0] dbg_idx_i, // coefficient index 0..255 + output [11:0] dbg_coeff_o, + // Debug byte readback: ek (sel=0, 0..799) / dk_pke (sel=1, 0..767) + input dbg_byte_sel_i, + input [10:0] dbg_byte_idx_i, + output [7:0] dbg_byte_o, + // Debug full-dk readback: dk = dk_pke(768) || ek(800) || H(ek)(32) || z(32) + // = 1632 bytes. Index 0..1631. + input [11:0] dbg_dk_idx_i, + output [7:0] dbg_dk_o, + // Debug taps for hash outputs + output [255:0] dbg_rho_o, + output [255:0] dbg_sigma_o, + // Encaps debug taps: r (G high half) and H(ek) + output [255:0] dbg_r_o, + output [255:0] dbg_hek_o, + // Decaps debug taps: m' (Decrypt output), K-bar (J output), parsed z/h. + output [255:0] dbg_mprime_o, + output [255:0] dbg_kbar_o, + output [255:0] dbg_decz_o, // parsed z (dk[768K+64:+32]) + output [255:0] dbg_dech_o // parsed H(ek) (dk[768K+32:+32]) +); + + localparam Q = `Q; // 3329 + + // Runtime ML-KEM parameter, captured at start_i. + reg [2:0] k_r; + // FIPS 203: eta1 = 3 for ML-KEM-512 (k=2), else 2 (k=3/4). + wire [1:0] eta1_rt = (k_r == 3'd2) ? 2'd3 : 2'd2; + + // Runtime-derived sizes (k_r in {2,3,4}). Small multiplies are cheap. + wire [5:0] kk_rt = k_r * k_r; // 4/9/16 + wire [5:0] slot_s_rt = kk_rt; // s_hat base slot + wire [5:0] slot_e_rt = kk_rt + k_r; // e_hat base slot + wire [5:0] slot_t_rt = kk_rt + {1'b0, k_r} + {1'b0, k_r}; // t_hat base = kk+2k + wire [11:0] ek_bytes_rt = 12'd384 * {9'b0, k_r} + 12'd32; // 800/1184/1568 + wire [11:0] dk_bytes_rt = 12'd384 * {9'b0, k_r}; // 768/1152/1536 + // full dk = dk_pke(dk_bytes) || ek(ek_bytes) || H(ek)(32) || z(32). Region + // boundaries (used by Decaps dk load routing and dk readback). + wire [11:0] dk_ek_end = dk_bytes_rt + ek_bytes_rt; // ek_pke region end + wire [11:0] dk_hek_end = dk_ek_end + 12'd32; // H(ek) region end + // LOAD-TIME boundaries: dk is streamed in BEFORE start_i, so k_r is not yet + // captured. Route the dk load by the LIVE k_i input instead of k_r. + wire [11:0] dkp_bytes_ld = 12'd384 * {9'b0, k_i}; // 768/1152/1536 + wire [11:0] ek_bytes_ld = 12'd384 * {9'b0, k_i} + 12'd32; // 800/1184/1568 + wire [11:0] dk_ek_end_ld = dkp_bytes_ld + ek_bytes_ld; // ek_pke region end + wire [11:0] dk_hek_end_ld= dk_ek_end_ld + 12'd32; // H(ek) region end + // H(ek) block count = ceil((ek_bytes+1)/136): 6/9/12 for k=2/3/4 (table) + wire [3:0] h_nblk_rt = (k_r == 3'd2) ? 4'd6 : (k_r == 3'd3) ? 4'd9 : 4'd12; + wire [11:0] h_last_rt = {6'b0, h_nblk_rt} * 12'd136 - 12'd1; // final padded byte index + + // ---- Encaps runtime params ---- + reg [1:0] op_r; // 0=KeyGen 1=Encaps 2=Decaps (captured at start) + reg [255:0] m_r; // Encaps message m (captured at start); Decaps reuses for m' + reg [255:0] ss_r; // Encaps shared secret K (= G output low half) + reg [255:0] r_r; // Encaps PRF seed r (= G output high half) + // ---- Decaps runtime params (parsed from dk during load) ---- + reg [255:0] z_r; // implicit-rejection seed z (dk tail), captured at load + reg [255:0] kbar_r; // K-bar = J(z||c) (D5) + // FIPS 203: eta2 = 2 for all parameter sets. + wire [1:0] eta2_rt = 2'd2; + // Compression params: (du,dv) = (10,4) for k=2/3, (11,5) for k=4. + wire [4:0] du_rt = (k_r == 3'd4) ? 5'd11 : 5'd10; + wire [4:0] dv_rt = (k_r == 3'd4) ? 5'd5 : 5'd4; + // Ciphertext byte sizes: c1 = 32*du*k, c2 = 32*dv, ct = c1+c2. + wire [11:0] c1_bytes_rt = 12'd32 * {7'b0, du_rt} * {9'b0, k_r}; // 640/960/1408 + wire [11:0] c2_bytes_rt = 12'd32 * {7'b0, dv_rt}; // 128/128/160 + wire [11:0] ct_bytes_rt = c1_bytes_rt + c2_bytes_rt; // 768/1088/1568 + // Shared secret out: Encaps drives ss_r = K directly. Decaps implicit reject: + // ss = (c' == c) ? K' : K-bar. K' is in ss_r, K-bar in kbar_r; dec_reject is + // the latched c' != c flag from D7 (0 for KeyGen/Encaps, so ss_r passes through). + assign ss_o = dec_reject ? kbar_r : ss_r; + + // ---- E1: rho-load + byteDecode12 bookkeeping ---- + // rho load: read 32 bytes from ek_bram at offset 384*k into rho_r. + reg [5:0] rl_idx; // 0..33 read-ahead pointer over rho bytes + reg [5:0] rl_widx; // 0..31 byte being written (1 cyc behind rl_idx) + reg rl_vld; // a byte is arriving this cycle + wire [11:0] rl_ek_off = dk_bytes_rt + {6'b0, rl_idx}; // ek addr = 384k + idx + // byteDecode12: read 3 ek bytes -> 2 coeffs into bank_t. Walk K polys * 128 + // triples. 5-cycle micro-phase per triple: ph0..2 fetch b0,b1,b2 (read-ahead, + // 1-cycle bram latency), ph3 write c0, ph4 write c1. + reg [2:0] td_poly; // 0..k-1 t_hat poly index + reg [7:0] td_trip; // 0..127 triple within poly + reg [2:0] td_ph; // 0..4 micro-phase + reg [7:0] td_b0, td_b1, td_b2; // captured ek bytes + reg td_done; + // ek byte address for the triple: poly*384 + trip*3 + (ph for ph<3) + wire [11:0] td_base = {td_poly, 7'd0} * 12'd3; // poly*384 (=poly*256*1.5)... see below + // poly*384 + trip*3 + byteidx; byteidx = ph (0,1,2) during fetch phases + wire [11:0] td_byteidx = {2'd0, td_trip, 1'b0} + {3'd0, td_trip}; // trip*3 + wire [11:0] td_ekaddr = ({4'd0,td_poly}*12'd384) + td_byteidx + {9'd0, td_ph}; + // decoded coeffs: c0 = b0 | ((b1&0xF)<<8); c1 = (b1>>4) | (b2<<4) + wire [11:0] td_c0 = {td_b1[3:0], td_b0}; + wire [11:0] td_c1 = {td_b2, td_b1[7:4]}; + // TDEC byte source: Encaps reads ek (ek_rd_data); Decaps D2 (ST_DEC_SDEC) + // reuses the same machine to byteDecode12 dk_pke -> s_hat, reading dkp_bram. + wire td_dec_s = (st == ST_DEC_SDEC); + wire [7:0] td_byte = td_dec_s ? dkp_rd_data : ek_rd_data; + // bank_a write for TDEC (registered); t_hat[td_poly] -> bank_a slot + // td_poly*K. td_wa is a bank_a byte address (PA_AW=12). + reg td_we; + reg [11:0] td_wa; // bank_a address (PA_AW=12) + reg [11:0] td_wd; + + // ---- ct_bram: ciphertext byte buffer (<=1568 B). Written by E5/E7 + // (compress + byteEncode_du/dv), read back via dbg_ct tap. ---- + wire [10:0] ct_rd_addr; + wire [7:0] ct_rd_data; + reg ct_we; + reg [10:0] ct_wa; + reg [7:0] ct_wd; + sd_bram #(.W(8), .D(2048), .A(11)) u_ct_bram ( + .clk(clk), + .rd_addr(ct_rd_addr), .rd_data(ct_rd_data), + .wr_en(ct_we), .wr_addr(ct_wa), .wr_data(ct_wd) + ); + // ct_bram read addr: D7 compare walks c' via cmp_idx; else debug readback. + assign ct_rd_addr = (st == ST_DEC_CMP) ? cmp_idx : dbg_ct_idx_i; + assign dbg_ct_o = ct_rd_data; + + // ---- c_in_bram: Decaps input ciphertext c (<=1568 B). Preloaded via + // c_in_* in ST_IDLE. Kept separate from ct_bram (which holds the computed + // c') so D7 can compare c' vs c and D5 J(z||c) can read c. Read port is + // muxed (D5 J-feed / D7 compare); for D0 only the load write path exists. + wire [10:0] cin_rd_addr; + wire [7:0] cin_rd_data; + reg [10:0] cin_rd_addr_r; // D5/D7 read address (tied 0 until then) + sd_bram #(.W(8), .D(2048), .A(11)) u_c_in_bram ( + .clk(clk), + .rd_addr(cin_rd_addr), .rd_data(cin_rd_data), + .wr_en(c_in_we), .wr_addr(c_in_addr), .wr_data(c_in_byte) + ); + // c_in read addr: D5 J presents dj_c_idx combinationally (no extra register + // stage) so the registered BRAM read yields the byte 1 cycle later, matching + // the assemble/writeback pipeline. Otherwise the registered cin_rd_addr_r + // (D1 walker / idle) drives it. + assign cin_rd_addr = (st == ST_DEC_J) ? dj_c_idx[10:0] : + (st == ST_DEC_CMP) ? cmp_idx : cin_rd_addr_r; + + // ================================================================ + // Polynomial storage, sized for KMAX (worst case). Runtime k uses a + // sub-range. Slot layout (each slot = 256 coeffs): + // A_hat[i][j] : slots 0 .. k*k-1 at index i*k + j + // s_hat[i] : slots slot_s_rt .. +k-1 + // e_hat[i] : slots slot_e_rt .. +k-1 + // t_hat[i] : slots slot_t_rt .. +k-1 + // NUM_SLOTS = KMAX*KMAX + 3*KMAX = 28 for KMAX=4. + // ================================================================ + localparam NUM_SLOTS = KMAX*KMAX + 3*KMAX; + localparam SAW = 5; // slot-address width (>=clog2(28)) + + // ================================================================ + // Coefficient storage as 3 banks (Phase 2). Was one async-read reg + // array polymem[0:28*256-1]; split by polynomial so each becomes a + // registered-read sd_bram (infers BRAM / ASIC SRAM) in the final step. + // bank_a : A_hat[i][j] slots 0..K^2-1 -> D=KMAX*KMAX*256=4096 + // bank_se: s_hat||e_hat slots slot_s.. -> D=2*KMAX*256=2048 (rel 0..2K-1) + // bank_t : t_hat[i] slots slot_t.. -> D=KMAX*256=1024 (rel 0..K-1) + // Addresses are base-relative: bank index = abs_slot - base_slot. + // Stage 2b (in progress): per-consumer registered read-ahead. Banks are + // still async reg arrays here; converted consumers read via their own + // 1-cycle pipeline reg (== sd_bram timing). Once every consumer of a + // bank is registered, the array is replaced by an sd_bram with its read + // port muxed across the (phase-disjoint) consumers. + // ================================================================ + localparam PA_AW = 12; // bank_a addr width (4096) + localparam PSE_AW = 11; // bank_se addr width (2048) + localparam PT_AW = 10; // bank_t addr width (1024) + + // bank_a: A_hat. Reader = ST_M load (pm_a) + dbg. Writer = ST_A. + // Read port muxed by phase; sd_bram's internal rd_addr_r register is the + // 1-cycle read pipeline (replaces the explicit pm_a_rd reg). Write port is + // driven combinationally (sd_bram registers the write at posedge, matching + // the old reg-array nonblocking write timing). + wire [PA_AW-1:0] ba_rd_addr; + wire [11:0] ba_rd_data; + wire ba_we; + wire [PA_AW-1:0] ba_wa; + wire [11:0] ba_wd; + sd_bram #(.W(12), .D(1<0 running t_hat) + ST_E ek-half + dbg. + // Writer = ST_M acc. Read port muxed by phase; sd_bram's rd_addr_r is the + // 1-cycle read register. RMW is safe: the acc read addr leads the write + // addr by 1 (m_oidx+1 vs m_oidx), so read/write never alias in a cycle and + // sd_bram write-first == reg-array read-old. Write port combinational. + wire [PT_AW-1:0] bt_rd_addr; + wire [11:0] bt_rd_data; + wire bt_we; + wire [PT_AW-1:0] bt_wa; + wire [11:0] bt_wd; + sd_bram #(.W(12), .D(1< bank_t rel slot 0, free during C/N/U). (TDEC writes bank_a now.) + wire e2_we = (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack && + (c_poly == {1'b0, k_r, 1'b0}); // c_poly == 2K + // Encaps E4/E6 bank_t writes: MAC psum (u_sub0, on pm_vo) and INTT in-place + // (u_sub1, on ntt_vo). Both target bank_t rel slot UPSUM. ST_ENC_V (E6) adds + // the V-ADD writeback (psum + e2 + mu) at UPSUM via u_v_we below. + wire u_psum_we = (st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && (u_sub == 2'd0) && pm_vo; + wire u_intt_we = (st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && (u_sub == 2'd1) && ntt_vo; + // D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT]. + wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo; + wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx; + assign bt_we = ((st == ST_M) && pm_vo) || e2_we || + u_psum_we || u_intt_we || u_v_we || u_w_we; + assign bt_wa = e2_we ? (c_widx & ((1<= slot_t_rt) dbg_coeff_r <= bt_rd_data; // bank_t (sd_bram) + else if (dbg_slot_i >= slot_s_rt) dbg_coeff_r <= bse_rd_data; // bank_se (sd_bram) + else dbg_coeff_r <= ba_rd_data; // bank_a (sd_bram) + end + assign dbg_coeff_o = dbg_coeff_r; + + // ek and dk_pke byte memories sized for KMAX. + localparam EK_MAX = 384*KMAX + 32; // 1568 + localparam DK_MAX = 384*KMAX; // 1536 + + // ek / dk_pke byte storage in BRAM (sd_bram: 1R/1W, 1-cycle read latency). + // Each single read port is shared time-disjointly: ST_H assemble reads ek + // (in ST_H), debug readback reads ek/dk (in ST_DONE). Write port driven by + // ST_E (1 byte/cycle) and the rho-tail copy. + wire [10:0] ek_rd_addr, dkp_rd_addr; + wire [7:0] ek_rd_data, dkp_rd_data; + reg ek_we, dkp_we; + reg [10:0] ek_wa, dkp_wa; + reg [7:0] ek_wd, dkp_wd; + + // ek BRAM write port: KeyGen ST_E drives ek_we/ek_wa/ek_wd; Encaps preloads + // ek from the external ek_in_* port (TB streams ek=pk before start_i). Decaps + // routes the dk's ek_pke region (dk[384K:768K+32]) here at load time. The + // three never overlap (preloads happen in ST_IDLE before a run). + wire dk_ld_ekpke = dk_in_we && (dk_in_addr >= dkp_bytes_ld) + && (dk_in_addr < dk_ek_end_ld); // ek_pke region + wire dk_ld_dkpke = dk_in_we && (dk_in_addr < dkp_bytes_ld); // dk_pke region + wire [10:0] dk_ek_off = (dk_in_addr - dkp_bytes_ld); // offset within ek + wire ek_we_mux = ek_in_we ? 1'b1 : + dk_ld_ekpke ? 1'b1 : ek_we; + wire [10:0] ek_wa_mux = ek_in_we ? ek_in_addr : + dk_ld_ekpke ? dk_ek_off : ek_wa; + wire [7:0] ek_wd_mux = ek_in_we ? ek_in_byte : + dk_ld_ekpke ? dk_in_byte : ek_wd; + + sd_bram #(.W(8), .D(2048), .A(11)) u_ek_bram ( + .clk(clk), + .rd_addr(ek_rd_addr), .rd_data(ek_rd_data), + .wr_en(ek_we_mux), .wr_addr(ek_wa_mux), .wr_data(ek_wd_mux) + ); + // dk_pke BRAM write port: KeyGen ST_E (s_hat byteEncode) drives dkp_we/wa/wd; + // Decaps routes the dk's dk_pke region (dk[0:384K]) here at load time. The + // two never overlap (preload in ST_IDLE before a Decaps run). + wire dkp_we_mux = dk_ld_dkpke ? 1'b1 : dkp_we; + wire [10:0] dkp_wa_mux = dk_ld_dkpke ? dk_in_addr[10:0] : dkp_wa; + wire [7:0] dkp_wd_mux = dk_ld_dkpke ? dk_in_byte : dkp_wd; + sd_bram #(.W(8), .D(2048), .A(11)) u_dkp_bram ( + .clk(clk), + .rd_addr(dkp_rd_addr), .rd_data(dkp_rd_data), + .wr_en(dkp_we_mux), .wr_addr(dkp_wa_mux), .wr_data(dkp_wd_mux) + ); + + // full dk = dk_pke(dk_bytes) || ek(ek_bytes) || H(ek)(32) || z(32). + // (dk_ek_end / dk_hek_end declared near the top with the other size wires.) + + // Debug-region selects for dk readback (combinational region decode). + wire dbgdk_in_dkp = (dbg_dk_idx_i < dk_bytes_rt); + wire dbgdk_in_ek = (dbg_dk_idx_i >= dk_bytes_rt) && (dbg_dk_idx_i < dk_ek_end); + + // Read-address muxes (declared here; depend on ST_H h_g defined below via + // the function-free expression h_blk*136 + h_byte). See assigns after the + // H-stage register declarations. + // dbg_byte_o / dbg_dk_o are combinational taps over the (registered) BRAM + // read data; net 1-cycle latency, well within the TB's 2-cycle read wait. + assign dbg_byte_o = dbg_byte_sel_i ? dkp_rd_data : ek_rd_data; + assign dbg_dk_o = dbgdk_in_dkp ? dkp_rd_data : + dbgdk_in_ek ? ek_rd_data : + (dbg_dk_idx_i < dk_hek_end) ? hek_r[(dbg_dk_idx_i - dk_ek_end)*8 +: 8] + : z_i[(dbg_dk_idx_i - dk_hek_end)*8 +: 8]; + + // ================================================================ + // Top-level FSM (built incrementally). Stage 2a: G only. + // ================================================================ + localparam ST_IDLE = 5'd0; + localparam ST_G = 5'd1; // run G(d||K), capture rho/sigma + localparam ST_A = 5'd2; // generate A_hat[i][j] via SampleNTT + localparam ST_C = 5'd3; // generate s[i],e[i] via CBD + localparam ST_N = 5'd4; // forward NTT of s[i],e[i] in place + localparam ST_M = 5'd5; // matrix accumulate t_hat = e_hat + sum A o s_hat + localparam ST_E = 5'd6; // byteEncode12 -> ek/dk BRAM + localparam ST_H = 5'd7; // H(ek) via multi-block SHA3-256 + // ---- Encaps states ---- + localparam ST_ENC_LOAD = 5'd8; // ek already preloaded; placeholder/settle + localparam ST_ENC_H = 5'd9; // H(ek) via multi-block SHA3-256 (reuses ST_H logic) + localparam ST_ENC_G = 5'd10; // (K,r) = G(m || H(ek)), 64-byte single block + localparam ST_ENC_A = 5'd11; // regenerate A_hat (transpose used in U) + localparam ST_ENC_TDEC = 5'd12; // byteDecode12: ek -> t_hat (bank_a slot j*K) + localparam ST_ENC_C = 5'd13; // sample y,e1,e2 via CBD (eta1/eta2) + localparam ST_ENC_N = 5'd14; // forward NTT of y in place + localparam ST_ENC_U = 5'd15; // u = INTT(sum A^T o y_hat) + e1 + localparam ST_ENC_C1 = 5'd16; // Compress_du + byteEncode_du -> ct c1 + localparam ST_ENC_V = 5'd17; // v = INTT(sum t_hat o y_hat) + e2 + mu + localparam ST_ENC_C2 = 5'd18; // Compress_dv + byteEncode_dv -> ct c2 + localparam ST_ENC_E2MV = 5'd19; // relocate e2 bank_t[0] -> bank_a[E2_ASLOT] + // ---- Decaps states ---- + localparam ST_DEC_LOAD = 5'd20; // dk/c already streamed in; parse/settle (D0) + localparam ST_DEC_DECOMP = 5'd21; // D1: byteDecode_d + Decompress c1->u', c2->v' + localparam ST_DEC_SDEC = 5'd22; // D2: byteDecode12 dk_pke -> s_hat (bank_a slot j*K) + localparam ST_DEC_NTT = 5'd23; // D2: u_hat[i] = NTT(u'[i]) in place (bank_se rel i) + localparam ST_DEC_W = 5'd24; // D3: w = v' - INTT(sum_j s_hat[j] o u_hat[j]) + localparam ST_DEC_MENC = 5'd25; // D4: m' = byteEncode_1(Compress_1(w)) + localparam ST_DEC_G = 5'd26; // D5: (K',r') = G(m' || h), SHA3-512 single block + localparam ST_DEC_J = 5'd27; // D5: K-bar = J(z || c), multi-block SHAKE256 + localparam ST_DEC_CMP = 5'd28; // D7: c' == c compare + implicit-reject ss select + localparam ST_DONE = 5'd31; + + reg [4:0] st, st_next; + reg [255:0] rho_r, sigma_r; + + // A-generation bookkeeping: explicit i/j counters (avoid runtime divide) + reg [2:0] a_i; // row 0..k-1 + reg [2:0] a_j; // col 0..k-1 + reg [4:0] a_pair; // 0..k*k pairs done (for done test) + reg [7:0] a_widx; // write index 0..255 within current poly + reg a_busy; // 1 once current pair's request accepted (gates collect) + wire [SAW-1:0] a_slot = a_i*k_r + a_j; // A_hat[i][j] slot = i*k + j + + // C-generation bookkeeping: 2*k polys (s[0..k-1] then e[0..k-1]) + reg [4:0] c_poly; // 0..2k + reg [7:0] c_widx; + reg c_busy; + wire [7:0] c_nonce = {3'b0, c_poly}; // s:0..k-1 e:k..2k-1 == nonce + // slot: c_poly < k -> s_hat[c_poly], else e_hat[c_poly-k] + wire [SAW-1:0] c_slot = (c_poly < {2'b0, k_r}) ? (slot_s_rt + c_poly) + : (slot_e_rt + (c_poly - {2'b0, k_r})); + + assign busy_o = (st != ST_IDLE); + assign done_o = (st == ST_DONE); + assign dbg_rho_o = rho_r; + assign dbg_sigma_o = sigma_r; + assign dbg_r_o = r_r; + assign dbg_hek_o = hek_r; + // Decaps taps: m' from D4 (written into m_r, reused by V/mu in D6 re-encrypt); + // z/h parsed from dk at load. + assign dbg_mprime_o = m_r; + assign dbg_kbar_o = kbar_r; + assign dbg_decz_o = z_r; + assign dbg_dech_o = hek_r; // Decaps parses dk's H(ek) into hek_r at load + + // ---- sha3_top in G mode: data_i = {K_byte, d} (d byte0 in [7:0]) ---- + reg sha3_valid; + wire sha3_ready; + wire [511:0] sha3_hash; + wire sha3_vo; + reg sha3_ack; // consumer ready for hash + wire [511:0] kg_g_data = {248'b0, 5'b0, k_r, d_i}; // KeyGen G: [263:256]=k, [255:0]=d + wire [511:0] enc_g_data = {hek_r, m_r}; // Encaps G: m || H(ek), 64 bytes + wire [511:0] dec_g_data = {hek_r, m_r}; // Decaps D5 G: m' || h, 64 bytes + wire [511:0] g_data = (st == ST_ENC_G) ? enc_g_data : + (st == ST_DEC_G) ? dec_g_data : kg_g_data; + + // ================================================================ + // Shared keccak_core + phase mux (3 consumers -> 1 core) + // + // G/H (u_sha3), SampleNTT (u_snt) and CBD (u_cbd) each need a Keccak + // permutation, but they run in DISJOINT top-FSM phases (ST_G/ST_H, ST_A, + // ST_C respectively), so at most one is ever active. One keccak_core + // serves all three: kc_state_i/kc_valid_i are muxed by phase; kc_valid_o + // is broadcast but GATED per consumer so an inactive consumer never + // latches a permutation result meant for another (the samplers latch + // squeeze state unconditionally on their kc_valid_o input). + // ================================================================ + wire [1599:0] kc_state_o; // shared core output (to all consumers) + wire kc_valid_o; // shared core output-valid + wire kc_ready_o; // shared core input-ready + + // per-consumer drives toward the core + wire [1599:0] sha3_kc_state_i, snt_kc_state_i, cbd_kc_state_i; + wire sha3_kc_valid_i, snt_kc_valid_i, cbd_kc_valid_i; + /* verilator lint_off UNUSEDSIGNAL */ + wire sha3_kc_ready_i, snt_kc_ready_i, cbd_kc_ready_i; // all 1'b1 + /* verilator lint_on UNUSEDSIGNAL */ + + // phase selects (mutually exclusive). Encaps adds ST_ENC_H/G (sha3), + // ST_ENC_A (snt), ST_ENC_C (cbd). + wire sel_sha3 = (st == ST_G) || (st == ST_H) || + (st == ST_ENC_H) || (st == ST_ENC_G) || + (st == ST_DEC_G) || (st == ST_DEC_J); + wire sel_snt = (st == ST_A) || (st == ST_ENC_A); + wire sel_cbd = (st == ST_C) || (st == ST_ENC_C); + + // gated output-valid: only the active consumer sees kc_valid_o + wire kc_valid_o_sha3 = kc_valid_o & sel_sha3; + wire kc_valid_o_snt = kc_valid_o & sel_snt; + wire kc_valid_o_cbd = kc_valid_o & sel_cbd; + + // input mux: route the active consumer's request to the core + wire [1599:0] kc_state_i_mux = sel_snt ? snt_kc_state_i : + sel_cbd ? cbd_kc_state_i : + sha3_kc_state_i; + wire kc_valid_i_mux = sel_snt ? snt_kc_valid_i : + sel_cbd ? cbd_kc_valid_i : + sel_sha3 ? sha3_kc_valid_i : 1'b0; + + keccak_core #(.ROUNDS(24)) u_keccak ( + .clk(clk), .rst_n(rst_n), + .state_i(kc_state_i_mux), + .valid_i(kc_valid_i_mux), + .ready_o(kc_ready_o), + .state_o(kc_state_o), + .valid_o(kc_valid_o), + .ready_i(1'b1) // consumers always accept (kc_ready_i=1) + ); + + // G/H/J share one sha3_top. Single-block (mb_en=0): KeyGen G (ST_G, mode 00), + // Encaps G (ST_ENC_G, mode 11), Decaps D5 G (ST_DEC_G, mode 11). Multi-block + // absorb (mb_en=1): H(ek) (ST_H/ST_ENC_H, SHA3-256 pad) and Decaps D5 J + // (ST_DEC_J, SHAKE256 pad over z||c). mb_* inputs are muxed by phase. + wire sha3_mb_en = (st == ST_H) || (st == ST_ENC_H) || (st == ST_DEC_J); + wire [1:0] sha3_mode = (st == ST_ENC_G || st == ST_DEC_G) ? 2'b11 : 2'b00; + // multi-block feed mux: J (ST_DEC_J) drives dj_*, else H drives h_*. + wire [1087:0] mb_block_mux = (st == ST_DEC_J) ? dj_block_r : h_block_r; + wire mb_valid_mux = (st == ST_DEC_J) ? dj_mbvalid : h_mbvalid; + wire mb_last_mux = (st == ST_DEC_J) ? dj_mblast : h_mblast; + wire mb_ack_mux = (st == ST_DEC_J) ? dj_ack : h_ack; + sha3_top_shared u_sha3 ( + .clk(clk), .rst_n(rst_n), + .mode(sha3_mode), // G = SHA3-512 (only used when mb_en=0) + .data_i(g_data), + .valid_i(sha3_valid), + .ready_o(sha3_ready), + .hash_o(sha3_hash), + .valid_o(sha3_vo), + .ready_i(sha3_mb_en ? mb_ack_mux : sha3_ack), + .mb_en(sha3_mb_en), + .mb_block_i(mb_block_mux), + .mb_valid_i(mb_valid_mux), + .mb_last_i(mb_last_mux), + .mb_ready_o(h_mbready), + // shared keccak_core interface (gated by phase in the mux below) + .kc_state_o(kc_state_o), + .kc_valid_o(kc_valid_o_sha3), + .kc_ready_o(kc_ready_o), + .kc_state_i(sha3_kc_state_i), + .kc_valid_i(sha3_kc_valid_i), + .kc_ready_i(sha3_kc_ready_i) + ); + + // ---- multi-block H(ek) state (SHA3-256, 6/9/12 blocks); fed to shared u_sha3 ---- + reg [1087:0] h_block_r; // current pre-padded rate block + reg h_mbvalid; + reg h_mblast; + wire h_mbready; + wire [511:0] h_hash; + wire h_vo; + reg h_ack; + reg [255:0] hek_r; // captured H(ek) + reg [3:0] h_blk; // 0..H_NBLK-1 block index (up to 11 for K=4) + reg [7:0] h_byte; // 0..136 byte address being presented (assemble) + reg [1:0] h_phase; // 0=assemble 1=feed 2=wait-perm 3=done + // ek BRAM read is registered (1-cycle latency); assemble presents the + // address for h_byte this cycle and writes the byte that arrived (for the + // address presented last cycle) into h_block_r. Writeback pipeline regs: + reg h_wb_vld; // a byte is arriving this cycle + reg [7:0] h_wb_idx; // its position within the 136-byte block + reg [11:0] h_wb_g; // its global ek byte index + reg [7:0] h_wb_pad; // pad constant to use if g is out of ek range + reg h_wb_inek; // 1 if g is within ek range (use BRAM data) + + // ---- Decaps D5 J(z||c) multi-block SHAKE256 state ---- + // Message = z (32 B, from z_r) || c (ct_bytes_rt B, from c_in_bram). Rate=136. + // SHAKE256 pad: byte at msg_len = 0x1F, last byte of last block |= 0x80. + // Reuses the shared u_sha3 mb_* port (muxed by ST_DEC_J above). + reg [1087:0] dj_block_r; // current pre-padded rate block + reg dj_mbvalid; + reg dj_mblast; + reg dj_ack; + reg [3:0] dj_blk; // block index 0..DJ_NBLK-1 (z||c up to ~1600B -> <=12 blks) + reg [7:0] dj_byte; // 0..136 byte being assembled + reg [1:0] dj_phase; // 0=assemble 1=feed 2=wait-perm 3=done + reg dj_done; + // writeback pipeline (c_in_bram registered read, 1-cyc latency) + reg dj_wb_vld; + reg [7:0] dj_wb_idx; // position within 136-byte block + reg dj_wb_inc; // 1 if global byte g is within c range (use BRAM) + reg dj_wb_inz; // 1 if g < 32 (use z_r byte) + reg [7:0] dj_wb_pad; // pad constant if g out of msg range + reg [4:0] dj_wb_zidx; // z byte index (g) when inz + // J message length = 32 (z) + ct_bytes_rt; total padded blocks. + wire [11:0] dj_msglen = 12'd32 + {1'b0, ct_bytes_rt}; + wire [11:0] dj_last_byte = (dj_msglen / 12'd136) * 12'd136 + 12'd135; // last byte of final block + wire [3:0] dj_nblk = (dj_msglen / 12'd136) + 4'd1; + // global byte index for the address presented this cycle (assemble) + wire [11:0] dj_g = {4'd0, dj_blk} * 12'd136 + {4'd0, dj_byte}; + // c_in_bram byte index (when g>=32): g-32 + wire [11:0] dj_c_idx = dj_g - 12'd32; + // SHAKE256 pad constant for global byte g + function [7:0] dj_padconst(input [11:0] g); + begin + if (g == dj_last_byte && g == dj_msglen) dj_padconst = 8'h9f; // 0x1F|0x80 + else if (g == dj_msglen) dj_padconst = 8'h1f; + else if (g == dj_last_byte) dj_padconst = 8'h80; + else dj_padconst = 8'h00; + end + endfunction + + // h_hash / h_vo are now served by the shared u_sha3 above (mb_en=1 during + // ST_H). They alias the single core's outputs; the H consumer logic below + // already gates on st==ST_H, and u_sha3's valid_o/hash_o are mb-selected. + assign h_hash = sha3_hash; + assign h_vo = sha3_vo; + + // SHA3-256 over ek (ek_bytes_rt bytes): rate=136. Padded length = h_nblk_rt*136. + // pad: byte ek_bytes_rt = 0x06 (domain + first pad bit), last byte |= 0x80. + // byte b (0..135) of block blk: global g = blk*136 + b. + // ek bytes come from the ek BRAM (registered read); only the pad/zero bytes + // are constants, returned by h_padconst. Reads runtime ek_bytes_rt/h_last_rt. + function [7:0] h_padconst(input [3:0] blk, input [7:0] b); + integer g; + begin + g = blk*136 + b; + if (g == h_last_rt && g == ek_bytes_rt) h_padconst = 8'h86; // 0x06|0x80 + else if (g == ek_bytes_rt) h_padconst = 8'h06; + else if (g == h_last_rt) h_padconst = 8'h80; + else h_padconst = 8'h00; + end + endfunction + + // Global ek byte index for the address currently presented in ST_H assemble. + wire [11:0] h_g_addr = {8'd0, h_blk} * 12'd136 + {4'd0, h_byte}; + + // ek BRAM read-address mux: ST_H assemble drives it; otherwise debug readback. + // dbg_dk ek-region takes priority (its loop runs after the ek loop), so the + // single read port serves dbg_byte_o (ek loop) then dbg_dk_o (dk loop). + wire [11:0] dbgdk_ek_off = dbg_dk_idx_i - dk_bytes_rt; // offset within ek region + assign ek_rd_addr = (st == ST_H || st == ST_ENC_H) ? h_g_addr[10:0] : + (st == ST_ENC_LOAD) ? rl_ek_off[10:0] : // rho load + (st == ST_ENC_TDEC) ? td_ekaddr[10:0] : // byteDecode12 + dbgdk_in_ek ? dbgdk_ek_off[10:0] + : dbg_byte_idx_i; + // dkp BRAM read-address mux: D2 byteDecode12 (ST_DEC_SDEC) walks dk_pke via + // td_ekaddr (poly*384 + trip*3 + ph); else dbg_byte (sel=1) / dbg_dk. + assign dkp_rd_addr = (st == ST_DEC_SDEC) ? td_ekaddr[10:0] : + dbg_byte_sel_i ? dbg_byte_idx_i : dbg_dk_idx_i[10:0]; + + // ---- sample_ntt_sync: Â[i][j] = SampleNTT(rho || j || i) ---- + reg snt_valid; + wire snt_ready; + wire [11:0] snt_coeff; + wire snt_vo; + wire snt_last; + reg snt_ack; // we accept coeffs + + sample_ntt_sync_shared #(.K(KMAX)) u_snt ( + .clk(clk), .rst_n(rst_n), + .rho_i(rho_r), + .k_i(k_r), + .i_idx(a_i[1:0]), + .j_idx(a_j[1:0]), + .valid_i(snt_valid), + .ready_o(snt_ready), + .coeff_o(snt_coeff), + .valid_o(snt_vo), + .ready_i(snt_ack), + .last_o(snt_last), + // shared keccak_core interface + .kc_state_o(kc_state_o), + .kc_valid_o(kc_valid_o_snt), + .kc_ready_o(kc_ready_o), + .kc_state_i(snt_kc_state_i), + .kc_valid_i(snt_kc_valid_i), + .kc_ready_i(snt_kc_ready_i) + ); + + // ---- sample_cbd_sync: s[i]=CBD3(PRF(sigma,i)), e[i]=CBD3(PRF(sigma,K+i)) ---- + reg cbd_valid; + wire cbd_ready; + wire [11:0] cbd_coeff; // 12-bit signed (two's complement) + wire cbd_vo; + wire cbd_last; + reg cbd_ack; + + // CBD inputs muxed KeyGen vs Encaps: + // KeyGen ST_C: seed=sigma, eta=eta1 (all polys), nonce=c_poly. + // Encaps ST_ENC_C: seed=r, nonce=c_poly; eta = eta1 for y (c_poly=K). nonce sequence 0..2K matches FIPS. + wire cbd_enc = (st == ST_ENC_C); + wire [255:0] cbd_seed = cbd_enc ? r_r : sigma_r; + wire [1:0] cbd_eta = cbd_enc ? ((c_poly < {2'b0, k_r}) ? eta1_rt : eta2_rt) + : eta1_rt; + sample_cbd_sync_shared u_cbd ( + .clk(clk), .rst_n(rst_n), + .seed_i(cbd_seed), + .nonce_i(c_nonce), + .eta_i(cbd_eta), + .valid_i(cbd_valid), + .ready_o(cbd_ready), + .coeff_o(cbd_coeff), + .valid_o(cbd_vo), + .ready_i(cbd_ack), + .last_o(cbd_last), + // shared keccak_core interface + .kc_state_o(kc_state_o), + .kc_valid_o(kc_valid_o_cbd), + .kc_ready_o(kc_ready_o), + .kc_state_i(cbd_kc_state_i), + .kc_valid_i(cbd_kc_valid_i), + .kc_ready_i(cbd_kc_ready_i) + ); + + // signed (two's complement) -> [0,Q): add Q when negative + wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff; + + // ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ---- + // N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot). + reg [4:0] n_slot; // 0..2K (process s_hat[0..K-1] then e_hat[0..K-1]) + reg [8:0] n_ridx; // load read-AHEAD pointer 0..256 (leads consume by 1) + reg [7:0] n_widx; // output write index 0..255 + reg n_valid; // feeding coeffs to ntt_core (delayed 1 cyc vs n_ridx) + reg n_loading; // 1 while presenting load addresses to bank_se + reg n_pending; // waiting for ntt_core IDLE to start next slot + wire [SAW-1:0] n_slot_addr = slot_s_rt + n_slot; // s_hat then e_hat contiguous + // NTT slot count: KeyGen processes 2K (s,e); Encaps processes K (y only). + wire [4:0] n_slot_max = (st == ST_ENC_N || st == ST_DEC_NTT) ? {2'b0, k_r} : {1'b0, k_r, 1'b0}; + + wire ntt_ready; + wire [11:0] ntt_coeff; + wire ntt_vo; + wire ntt_done; + // bank_se read addr for the NTT load (relative slot = n_slot); sd_bram + // registers it into bse_rd_data, which feeds ntt_core 1 cycle later. + wire [13:0] ntt_rd_full = n_slot*256 + n_ridx[7:0]; + // ntt_core inputs muxed: KeyGen/Encaps fwd NTT feeds bse_rd_data (mode 0); + // Encaps E4/E6 INTT sub-phase feeds psum (bt_rd_data) with mode 1. + wire u_intt_act = (st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && (u_sub == 2'd1); + wire [11:0] ntt_in = u_intt_act ? bt_rd_data : bse_rd_data; + wire ntt_vin = u_intt_act ? u_nvalid : n_valid; + wire ntt_mode = u_intt_act ? 1'b1 : 1'b0; + + ntt_core u_ntt ( + .clk(clk), .rst_n(rst_n), + .coeff_in(ntt_in), + .valid_i(ntt_vin), + .ready_o(ntt_ready), + .mode(ntt_mode), // 0 = fwd (NTT), 1 = inv (INTT, x3303) + .coeff_out(ntt_coeff), + .valid_o(ntt_vo), + .ready_i(1'b1), // always accept output + .done_o(ntt_done) + ); + + // ---- poly_mul_sync: t_hat[i] = e_hat[i] + sum_j A_hat[i][j] o s_hat[j] ---- + // M-stage bookkeeping. For each (i,j): LOAD 256 (A,shat) pairs, then accumulate + // 256 products into T_i (init from E_i when j==0, else from running T_i). + reg [2:0] m_i; // row 0..K (needs to reach K to exit) + reg [2:0] m_j; // col 0..K-1 + reg [8:0] m_ld; // load index 0..256 + reg [7:0] m_oidx; // output/accum index 0..255 + reg m_loading; // 1 while streaming pairs into poly_mul + reg m_pending; // wait for poly_mul IDLE before next (i,j) + + // ---- Stage 2f: byteEncode12 serializer ---- + // Pack each poly (2 coeffs -> 3 bytes, LSB-first 12-bit). ek = t_hat[0..K-1] + // bytes || rho; dk_pke = s_hat[0..K-1] bytes. Walk coeff pairs per poly. + reg [4:0] e_poly; // 0..2K-1: [0,K) = t_hat -> ek; [K,2K) = s_hat -> dk_pke + reg [7:0] e_pair; // 0..127 coeff-pair within poly + reg [9:0] e_rho; // 0..31 rho byte copy index (ek tail) + reg e_done; // serialization complete + // source poly slot: t_hat[e_poly] for ek half, s_hat[e_poly-K] for dk half + wire e_is_dk = (e_poly >= {1'b0, k_r}); + wire [4:0] e_pidx = e_is_dk ? (e_poly - {1'b0, k_r}) : e_poly; // index within target + // Registered single-port read (sd_bram timing). One bank read per cycle: + // ek half reads bank_t (t_hat), dk half reads bank_se (s_hat). The two + // coeffs of a pair are serialized across a 4-cycle micro-phase e_ph: + // 0 = fetch c0 + // 1 = c0 ready: write b0, save c0[11:8], fetch c1 + // 2 = c1 ready: write b1, save c1[11:4] + // 3 = write b2, advance pair + reg [1:0] e_ph; // per-pair micro-phase 0..3 + reg [3:0] e_c0_hi; // saved c0[11:8] for b1 + reg [7:0] e_c1_hi; // saved c1[11:4] for b2 + wire e_rd_half = (e_ph == 2'd1); // ph0 -> c0, ph1 -> c1 + wire [13:0] e_rd_full = e_pidx*256 + {e_pair, e_rd_half}; // bse/bt rd addr in ST_E + // dk half reads bank_se (bse_rd_data), ek half reads bank_t (bt_rd_data); + // both registered inside their sd_bram with the same 1-cycle latency, so + // the coeff for the addr presented last cycle is selected here. + wire [11:0] e_rd_coeff = e_is_dk ? bse_rd_data : bt_rd_data; + // byteEncode write byte offset within the target memory: pair*3 + byte index. + wire [11:0] e_base = e_pidx * 12'd384; // poly index *384 (=128 pairs*3) + wire [11:0] e_boff = e_base + {e_pair, 1'b0} + {2'b0, e_pair}; // pair*3 + wire [1:0] e_wb = e_ph - 2'd1; // ph1->byte0, ph2->byte1, ph3->byte2 + wire [7:0] e_wbyte = (e_ph == 2'd1) ? e_rd_coeff[7:0] // b0 = c0[7:0] + : (e_ph == 2'd2) ? {e_rd_coeff[3:0], e_c0_hi} // b1 = {c1[3:0],c0[11:8]} + : e_c1_hi; // b2 = c1[11:4] + + wire [SAW-1:0] m_aslot = m_i*k_r + m_j; // A_hat[i][j] slot = i*k + j + wire [SAW-1:0] m_sslot = slot_s_rt + m_j; // s_hat[j] + wire [SAW-1:0] m_eslot = slot_e_rt + m_i; // e_hat[i] + wire [SAW-1:0] m_tslot = slot_t_rt + m_i; // t_hat[i] + + // ================================================================ + // E4: u[i] = INTT(sum_j A^T[i][j] o y_hat[j]) + e1[i] (Encaps ST_ENC_U) + // Per row i, three sub-phases (u_sub): + // 0 MAC : sum_j A_hat[j][i] o y_hat[j] (transpose) -> psum bank_t[UPSUM] + // 1 INTT: INTT(psum) mode=1 in place in bank_t[UPSUM] + // 2 ADD : u[i][w] = psum[w] + e1[i][w] mod Q -> bank_se rel (K+i) (over e1) + // y_hat (bank_se 0..K-1) survives for V. Reuses shared u_pmul + u_ntt. + // ================================================================ + localparam UPSUM = 10'd1; // bank_t rel slot for the NTT-domain psum + reg [2:0] u_row; // output row i 0..K-1 + reg [1:0] u_sub; // 0=MAC 1=INTT 2=ADD + // MAC bookkeeping (mirrors ST_M) + reg [2:0] u_j; // term col j 0..K-1 + reg [8:0] u_ld; // load read-ahead 0..256 + reg [7:0] u_oidx; // accumulate index 0..255 + reg u_loading; // streaming pairs into poly_mul + reg u_pending; // wait poly_mul IDLE before next j + reg u_j0q; // (u_j==0) delayed 1 cyc (first-term=0 select) + // INTT bookkeeping (mirrors ST_N) + reg [8:0] u_ridx; // INTT load read-ahead 0..256 + reg [7:0] u_widx; // INTT output index 0..255 + reg u_nvalid; // feed psum to ntt_core (mode=1) + reg u_nloading; // presenting INTT load addresses + reg u_npending; // (unused placeholder for symmetry) + // ADD bookkeeping + reg [8:0] u_aidx; // ADD read-ahead 0..256 + reg [7:0] u_awidx; // ADD write index 0..255 + reg u_avalid; // ADD write enable pipeline + // MAC addresses: A^T -> A_hat[u_j][u_row] slot = u_j*k+u_row; y_hat[u_j] rel u_j + wire [SAW-1:0] u_aslot = u_j*k_r + u_row; + wire [13:0] u_pm_a_full = u_aslot*256 + u_ld[7:0]; // bank_a + wire [13:0] u_pm_b_full = u_j*256 + u_ld[7:0]; // bank_se y_hat[u_j] + wire [7:0] u_acc_radr = pm_vo ? (u_oidx + 8'd1) : u_oidx; + wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) + wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 + wire [12:0] u_sum = {1'b0, u_acc_src} + {1'b0, pm_coeff}; + wire [11:0] u_accq = (u_sum >= 13'd3329) ? (u_sum - 13'd3329) : u_sum[11:0]; + // INTT load addr (psum read) + in-place writeback to bank_t[UPSUM] + wire [13:0] u_intt_rd = UPSUM*256 + u_ridx[7:0]; + wire [13:0] u_intt_wr = UPSUM*256 + u_widx; + // ADD: read psum (bank_t) + e1 (bank_se K+u_row), write u over e1 (read-ahead 1) + wire [13:0] u_add_prd = UPSUM*256 + u_aidx[7:0]; // bank_t psum + wire [13:0] u_add_e1rd = ({2'b0,k_r}+{2'b0,u_row})*256 + u_aidx[7:0]; // bank_se e1 (lead) + wire [13:0] u_add_uwr = ({2'b0,k_r}+{2'b0,u_row})*256 + u_awidx; // bank_se u (write) + wire [12:0] u_usum = {1'b0, bt_rd_data} + {1'b0, bse_rd_data}; // psum + e1 + wire [11:0] u_uq = (u_usum >= 13'd3329) ? (u_usum - 13'd3329) : u_usum[11:0]; + + // ================================================================ + // E6: v = INTT(sum_j t_hat[j] o y_hat[j]) + e2 + mu (Encaps ST_ENC_V). + // Reuses the u_* MAC/INTT/ADD machine with u_row tied to 0 (single output + // poly). Differences from E4: + // - MAC reads t_hat[j] from bank_a slot j*K (== u_aslot=u_j*K, u_row=0), + // so the bank_a MAC address is unchanged. + // - ADD computes psum + e2 + mu (not psum + e1); e2 lives in bank_a slot + // E2_ASLOT (relocated from bank_t[0] in ST_ENC_E2MV), v writes back to + // bank_t[UPSUM] in place. + // - mu[w] = Decompress_1(m bit w) = m_r[w] ? 1665 : 0 (1665 = round(Q/2)). + // u_row_max bounds the row loop: K for U, 1 for V. + localparam E2_ASLOT = 12'd1; // bank_a slot holding relocated e2 (never a t_hat slot, K>=2) + wire [2:0] u_row_max = (st == ST_ENC_V || st == ST_DEC_W) ? 3'd1 : k_r; + // V-ADD: read psum (bank_t UPSUM) + e2 (bank_a E2_ASLOT) at u_aidx (lead), + // add mu (by write index u_awidx), write v to bank_t[UPSUM] at u_awidx. + wire [13:0] u_v_e2rd = E2_ASLOT*256 + u_aidx[7:0]; // bank_a e2 (lead) + wire [13:0] u_vadd_wr = UPSUM*256 + u_awidx; // bank_t v (write) + wire [11:0] u_mu = m_r[u_awidx] ? 12'd1665 : 12'd0; // mu by msg bit + wire [13:0] u_vsum = {2'b0, bt_rd_data} + {2'b0, ba_rd_data} + {2'b0, u_mu}; // psum+e2+mu + wire [13:0] u_vsub1 = (u_vsum >= 14'd3329) ? (u_vsum - 14'd3329) : u_vsum; + wire [11:0] u_vq = (u_vsub1 >= 14'd3329) ? (u_vsub1 - 14'd3329) : u_vsub1[11:0]; + wire u_v_we = (st == ST_ENC_V) && (u_sub == 2'd2) && u_avalid; + + // ================================================================ + // D3: w = v' - INTT(sum_j s_hat[j] o u_hat[j]) (Decaps ST_DEC_W). + // Reuses the u_* MAC/INTT machine (u_row tied to 0, single poly): + // MAC : s_hat[j] (bank_a slot j*K, == u_aslot with u_row=0) o u_hat[j] + // (bank_se rel j, == u_pm_b_full) -> psum bank_t[UPSUM]. (same + // addressing as Encaps V MAC, which reads t_hat[j] o y_hat[j].) + // INTT: INTT(psum) in place in bank_t[UPSUM]. + // SUB : w[i] = v'[i] - psum[i] mod Q. v' is in bank_a[DEC_VASLOT] (D1), + // psum in bank_t[UPSUM]; read both in parallel (like V-ADD reads + // psum + e2). w written back to bank_t[UPSUM] in place. + wire [13:0] u_w_vrd = DEC_VASLOT*256 + u_aidx[7:0]; // bank_a v' (lead) + wire [13:0] u_wsub_wr = UPSUM*256 + u_awidx; // bank_t w (write) + wire [12:0] u_wdiff = {1'b0, ba_rd_data} - {1'b0, bt_rd_data}; // v' - psum + wire [11:0] u_wq = u_wdiff[12] ? (u_wdiff[11:0] + 12'd3329) : u_wdiff[11:0]; + wire u_w_we = (st == ST_DEC_W) && (u_sub == 2'd2) && u_avalid; + + // ================================================================ + // D4: m' = byteEncode_1(Compress_1(w)) (Decaps ST_DEC_MENC). + // Compress_1(x) = round(2x/Q) mod 2 = 1 iff x in (Q/4, 3Q/4], i.e. + // 832 < x <= 2496 for Q=3329. Pack 256 bits LSB-first into 32 bytes. + // w is read from bank_t[UPSUM] (in place from D3), 1 coeff per 2 cycles. + reg [7:0] men_idx; // coeff 0..255 + reg [1:0] men_ph; // micro-phase + reg men_done; + // D4 packs the recovered message bits directly into m_r (reused by D6's + // V/mu re-encrypt and exposed on dbg_mprime_o). + + // ================================================================ + // D7: implicit-reject compare (Decaps ST_DEC_CMP). + // Walk all ct bytes, reading c' (ct_bram) and c (c_in_bram) in parallel; + // OR every byte difference into cmp_neq. After the last byte, ss = cmp_neq + // ? K-bar : K'. The walk visits every byte regardless of early differences + // (constant work / no early-out), matching the constant-time spec intent. + // ph0: present both addrs; ph1: compare registered reads, accumulate. + reg [10:0] cmp_idx; // byte index 0..ct_bytes + reg cmp_ph; // 0=present addr, 1=compare + reg cmp_neq; // sticky: any byte differed + reg cmp_done; + reg dec_reject; // latched cmp_neq at end (drives ss select) + wire cmp_byte_neq = (ct_rd_data != cin_rd_data); + wire [13:0] men_rd = UPSUM*256 + men_idx; // bank_t w read addr + wire men_w_bit = (bt_rd_data > 12'd832) && (bt_rd_data <= 12'd2496); // Compress_1 + + // ST_ENC_E2MV: copy e2 (bank_t rel slot 0, 256 coeffs) into bank_a[E2_ASLOT]. + // bank_t read leads (em_ridx), bank_a write trails 1 cycle (em_we/em_widx). + reg [8:0] em_ridx; // 0..256 read-ahead over e2 + reg [7:0] em_widx; // bank_a write index + reg em_we; // a coeff is being written this cycle + reg em_done; + wire [13:0] em_rd = 14'd0 + em_ridx[7:0]; // bank_t rel slot 0 (e2) + wire [11:0] em_wa = E2_ASLOT*256 + em_widx; // bank_a E2_ASLOT + + // ================================================================ + // E5/E7: Compress_d + byteEncode_d -> ciphertext (Encaps ST_ENC_C1/C2). + // Per coeff: read poly coeff -> comp_decomp (mode 0 compress, d=du/dv) -> + // bit-packer (LSB-first) -> emit bytes to ct_bram. c1 = K polys of u + // (d=du), then c2 = 1 poly of v (d=dv). Per poly = 256 coeffs -> 32*d + // bytes (whole), so the bit buffer empties at each poly boundary. + // micro-phase cp_ph: 0 present coeff addr; 1 feed comp_decomp (cd_valid); + // 2 wait pipe; 3 capture compressed + accumulate bits; 4..n drain bytes. + // ================================================================ + wire cd_active = (st == ST_ENC_C1) || (st == ST_ENC_C2); + reg [11:0] cd_coeff; // coeff presented to comp_decomp + reg cd_valid; // 1-cyc pulse to comp_decomp + wire cd_ready; + wire [11:0] cd_out; // compressed value (low d bits valid) + wire cd_vo; + wire [4:0] cp_d = (st == ST_ENC_C2) ? dv_rt : du_rt; // compress width + // comp_decomp is shared: Encaps C1/C2 compress (mode 0); Decaps DECOMP + // decompresses (mode 1). In DECOMP the decoder feeds dec_dc_coeff/valid/d. + wire cd_dec = (st == ST_DEC_DECOMP); + wire cd_mode = cd_dec ? 1'b1 : 1'b0; // 1=decompress + wire [4:0] cd_d_mux = cd_dec ? dec_dc_d : cp_d; + wire [11:0] cd_in_mux= cd_dec ? dec_dc_coeff : cd_coeff; + wire cd_vi_mux= cd_dec ? dec_dc_valid : cd_valid; + comp_decomp_sync u_comp ( + .clk(clk), .rst_n(rst_n), + .coeff_in(cd_in_mux), + .d(cd_d_mux), + .mode(cd_mode), + .valid_i(cd_vi_mux), + .ready_o(cd_ready), + .coeff_out(cd_out), + .valid_o(cd_vo), + .ready_i(1'b1) + ); + // bit-packer / ct walk bookkeeping + reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v + reg [7:0] cp_idx; // coeff 0..255 within poly + reg [2:0] cp_ph; // micro-phase + reg [24:0] cp_buf; // bit accumulator (LSB-first) + reg [5:0] cp_nbits; // valid bits in cp_buf + reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2) + reg cp_done; // serialization complete (this region) + // coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly); + // c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0). + wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly] + wire [13:0] cp_bt_full = UPSUM*256 + cp_idx; // bank_t v (UPSUM) + // packer generalization (E5 c1 vs E7 c2): + // coeff source: c1 = bank_se (u), c2 = bank_t (v) + // poly count: c1 = K (u rows), c2 = 1 (single v) + // bit width cp_d already = dv_rt for C2 else du_rt (declared above). + wire [11:0] cp_coeff_src = (st == ST_ENC_C2) ? bt_rd_data : bse_rd_data; + wire [2:0] cp_poly_max = (st == ST_ENC_C2) ? 3'd1 : k_r; + + // ================================================================ + // D1: K-PKE byteDecode_d + Decompress (decrypt c1/c2 -> u'/v'). + // Reverse of the C1/C2 packer: walk c_in_bram bytes, accumulate LSB-first + // into a bit buffer, extract d-bit symbols, feed comp_decomp (mode=1 + // decompress), and write each result (mod q) to a bank. + // c1 = K polys, d=du -> u'[i] in bank_se rel i (0..K-1) + // c2 = 1 poly, d=dv -> v' in bank_t rel slot DEC_VSLOT + // c1 = K polys, d=du -> u'[i] in bank_se rel i (0..K-1) + // c2 = 1 poly, d=dv -> v' in bank_a slot DEC_VASLOT (so D3's w-SUB can + // read psum (bank_t) and v' (bank_a) in parallel, like Encaps V-ADD). + // micro-phase dec_ph: + // 0 present c_in byte addr; 1 capture byte, append 8 bits to buf; + // while (nbits>=d): extract d-bit symbol -> comp_decomp (decompress); + // 2 wait pipe; 3 cd_vo -> write coeff to bank, advance coeff index. + // ================================================================ + localparam DEC_VASLOT = 12'd1; // bank_a slot for v' (free: s_hat at j*K, slot 1 unused K>=2) + reg [2:0] dec_poly; // c1: 0..K-1; c2: single (=K marker) + reg [7:0] dec_cidx; // coeff 0..255 within poly (feed side) + reg [7:0] dec_widx; // coeff 0..255 within poly (writeback side, lags pipe) + reg [1:0] dec_ph; // micro-phase + reg [11:0] dec_ba; // c_in_bram byte read address (c1 then c2) + reg [24:0] dec_buf; // bit accumulator (LSB-first) + reg [5:0] dec_nbits; // valid bits in dec_buf + reg dec_dc_done; // D1 complete (both c1 and c2 decoded) + reg dec_in_c2; // 0 = decoding c1 (u'), 1 = decoding c2 (v') + // comp_decomp feed (decompress): the extracted d-bit symbol. + reg [11:0] dec_dc_coeff; + reg dec_dc_valid; + wire [4:0] dec_dc_d = dec_in_c2 ? dv_rt : du_rt; // c2 uses dv, c1 uses du + // (c_in_bram read addr driven by cin_rd_addr_r, set to dec_ba in the walker) + + reg pm_valid; + wire pm_ready; + wire [11:0] pm_coeff; + wire pm_vo; + // pm_a: A_hat[i][j] in bank_a (abs slot m_aslot). pm_b: s_hat[j] in + // bank_se (relative slot = m_sslot - slot_s_rt = m_j). m_ld is a read-ahead + // pointer; both bank reads come from their sd_bram (registered, 1-cycle + // latency) and feed poly_mul one cycle later (pm_valid delayed 1 cyc). + wire [13:0] pm_a_full = m_aslot*256 + m_ld[7:0]; + wire [13:0] pm_b_full = m_j*256 + m_ld[7:0]; + wire [11:0] pm_a_in = ba_rd_data; // bank_a sd_bram registered read + wire [11:0] pm_b_in = bse_rd_data; // bank_se sd_bram registered read (load phase) + + poly_mul_sync u_pmul ( + .clk(clk), .rst_n(rst_n), + .coeff_a_in(pm_a_in), + .coeff_b_in(pm_b_in), + .valid_i(pm_valid), + .ready_o(pm_ready), + .coeff_out(pm_coeff), + .valid_o(pm_vo), + .ready_i(1'b1) + ); + + // accumulator source: e_hat[i] for first term (j==0), else running t_hat[i]. + // e_hat[i] lives in bank_se at relative slot (slot_e_rt-slot_s_rt + m_i) = K+m_i. + // t_hat[i] lives in bank_t at relative slot m_i. Both reads come from their + // sd_bram (bse_rd_data / bt_rd_data) with the same 1-cycle latency. + // Registered read-ahead: present the index the NEXT pm_vo will consume + // m_acc_radr = pm_vo ? m_oidx+1 : m_oidx + // The j-select is applied on the registered outputs (m_jq = (m_j==0) + // delayed 1 cyc to align with the read latency). RMW read-old holds: read + // addr (m_oidx+1) leads write addr (m_oidx). Cadence CALC/C0/C1. + wire [7:0] m_acc_radr = pm_vo ? (m_oidx + 8'd1) : m_oidx; + wire [13:0] m_eacc_full = ({2'b0, k_r} + {2'b0, m_i})*256 + m_acc_radr; // K+m_i (bse_rd_addr in acc) + wire [13:0] m_tacc_full = m_i*256 + m_acc_radr; // bt_rd_addr + reg m_jq; // (m_j==0) delayed 1 cyc to match read latency + // selected accumulator source aligned with pm_coeff + wire [11:0] m_acc_src = m_jq ? bse_rd_data : bt_rd_data; + // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract + wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; + wire [11:0] m_accq = (m_sum >= 13'd3329) ? (m_sum - 13'd3329) : m_sum[11:0]; + + always @(*) begin + st_next = st; + case (st) + ST_IDLE: if (start_i) st_next = (op_i == 2'd2) ? ST_DEC_LOAD : + (op_i == 2'd1) ? ST_ENC_H : ST_G; + // D0: settle after dk/c parse, then D1 Decompress. D1 lands in DONE + // once u'/v' are computed so the stage can be dbg-checked. + ST_DEC_LOAD: st_next = ST_DEC_DECOMP; + ST_DEC_DECOMP: if (dec_dc_done) st_next = ST_DEC_SDEC; + // D2: s_hat decode (reuse TDEC machine), then u_hat = NTT(u'). D2 + // lands in DONE once u_hat is ready so the stage can be dbg-checked. + ST_DEC_SDEC: if (td_done) st_next = ST_DEC_NTT; + ST_DEC_NTT: if (n_slot >= {2'b0, k_r}) st_next = ST_DEC_W; + // D3: w = v' - INTT(sum_j s_hat[j] o u_hat[j]). Single output poly + // (u_row 0..0); MAC->INTT->SUB, then D4 encodes m'. + ST_DEC_W: if (u_row >= 3'd1) st_next = ST_DEC_MENC; + ST_DEC_MENC: if (men_done) st_next = ST_DEC_G; + // D5: (K',r') = G(m'||h) single-block, then K-bar = J(z||c) multi-block. + ST_DEC_G: if (sha3_vo) st_next = ST_DEC_J; + // D5 J done -> D6 re-encrypt: c' = K-PKE.Encrypt(ek_pke, m', r'). + // Reuse the entire Encaps pipeline (rho load -> A -> C -> ... -> C2). + // r' is in r_r (CBD seed), m' in m_r (V/mu), ek_pke in ek_bram. + ST_DEC_J: if (dj_done) st_next = ST_ENC_LOAD; + ST_G: if (sha3_vo) st_next = ST_A; + ST_A: if (a_pair >= kk_rt) st_next = ST_C; + ST_C: if (c_poly >= {1'b0, k_r, 1'b0}) st_next = ST_N; + ST_N: if (n_slot >= {1'b0, k_r, 1'b0}) st_next = ST_M; + ST_M: if (m_i >= k_r) st_next = ST_E; + ST_E: if (e_done) st_next = ST_H; + ST_H: if (h_phase == 2'd3) st_next = ST_DONE; + // ---- Encaps ---- + ST_ENC_H: if (h_phase == 2'd3) st_next = ST_ENC_G; + ST_ENC_G: if (sha3_vo) st_next = ST_ENC_LOAD; // (K,r) captured + ST_ENC_LOAD: if (rl_idx >= 6'd33) st_next = ST_ENC_A; // rho loaded + ST_ENC_A: if (a_pair >= kk_rt) st_next = ST_ENC_C; + ST_ENC_C: if (c_poly >= {k_r, 1'b1}) st_next = ST_ENC_N; // 2K+1 polys done + ST_ENC_N: if (n_slot >= {2'b0, k_r}) st_next = ST_ENC_U; // K slots (y_hat) + ST_ENC_U: if (u_row >= k_r) st_next = ST_ENC_C1; // u[0..K-1] done + ST_ENC_C1: if (cp_done) st_next = ST_ENC_TDEC; // E5 done -> E6 + ST_ENC_TDEC: if (td_done) st_next = ST_ENC_E2MV; // t_hat decoded -> relocate e2 + ST_ENC_E2MV: if (em_done) st_next = ST_ENC_V; // e2 relocated -> compute v + ST_ENC_V: if (u_row >= 3'd1) st_next = ST_ENC_C2; // E6: v done -> E7 + ST_ENC_C2: if (cp_done) st_next = (op_r == 2'd2) ? ST_DEC_CMP : ST_DONE; + // D7: walk ct bytes comparing c' (ct_bram) vs c (c_in_bram); when the + // last byte is compared, select ss = (c'==c) ? K' : K-bar and finish. + ST_DEC_CMP: if (cmp_done) st_next = ST_DONE; + ST_DONE: st_next = ST_IDLE; + default: st_next = ST_IDLE; + endcase + end + + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + st <= ST_IDLE; + k_r <= 3'd0; + op_r <= 2'd0; + m_r <= 256'd0; + ss_r <= 256'd0; + r_r <= 256'd0; + rl_idx <= 6'd0; + rl_widx <= 6'd0; + rl_vld <= 1'b0; + td_poly <= 3'd0; + td_trip <= 8'd0; + td_ph <= 3'd0; + td_b0 <= 8'd0; + td_b1 <= 8'd0; + td_b2 <= 8'd0; + td_done <= 1'b0; + td_we <= 1'b0; + td_wa <= 12'd0; + td_wd <= 12'd0; + rho_r <= 256'd0; + sigma_r <= 256'd0; + sha3_valid <= 1'b0; + sha3_ack <= 1'b0; + snt_valid <= 1'b0; + snt_ack <= 1'b0; + a_pair <= 5'd0; + a_i <= 3'd0; + a_j <= 3'd0; + a_widx <= 8'd0; + a_busy <= 1'b0; + cbd_valid <= 1'b0; + cbd_ack <= 1'b0; + c_poly <= 3'd0; + c_widx <= 8'd0; + c_busy <= 1'b0; + n_slot <= 3'd0; + n_ridx <= 9'd0; + n_widx <= 8'd0; + n_valid <= 1'b0; + n_loading <= 1'b0; + n_pending <= 1'b0; + m_i <= 2'd0; + m_j <= 2'd0; + m_ld <= 9'd0; + m_oidx <= 8'd0; + m_loading <= 1'b0; + m_pending <= 1'b0; + pm_valid <= 1'b0; + m_jq <= 1'b0; + u_row <= 3'd0; + u_sub <= 2'd0; + u_j <= 3'd0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + u_loading <= 1'b0; + u_pending <= 1'b0; + u_j0q <= 1'b0; + u_ridx <= 9'd0; + u_widx <= 8'd0; + u_nvalid <= 1'b0; + u_nloading <= 1'b0; + u_npending <= 1'b0; + u_aidx <= 9'd0; + u_awidx <= 8'd0; + u_avalid <= 1'b0; + em_ridx <= 9'd0; + em_widx <= 8'd0; + em_we <= 1'b0; + em_done <= 1'b0; + cd_coeff <= 12'd0; + cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 25'd0; + cp_nbits <= 6'd0; + cp_wa <= 12'd0; + cp_done <= 1'b0; + ct_we <= 1'b0; + ct_wa <= 11'd0; + ct_wd <= 8'd0; + e_poly <= 3'd0; + e_pair <= 8'd0; + e_ph <= 2'd0; + e_c0_hi <= 4'd0; + e_c1_hi <= 8'd0; + e_rho <= 10'd0; + e_done <= 1'b0; + ek_we <= 1'b0; + dkp_we <= 1'b0; + ek_wa <= 11'd0; + dkp_wa <= 11'd0; + ek_wd <= 8'd0; + dkp_wd <= 8'd0; + h_block_r <= 1088'd0; + h_mbvalid <= 1'b0; + h_mblast <= 1'b0; + h_ack <= 1'b0; + hek_r <= 256'd0; + z_r <= 256'd0; + kbar_r <= 256'd0; + cin_rd_addr_r <= 11'd0; + dec_poly <= 3'd0; + dec_cidx <= 8'd0; + dec_widx <= 8'd0; + dec_ph <= 2'd0; + dec_ba <= 12'd0; + dec_buf <= 25'd0; + dec_nbits <= 6'd0; + dec_in_c2 <= 1'b0; + dec_dc_coeff <= 12'd0; + dec_dc_valid <= 1'b0; + dec_dc_done <= 1'b0; + men_idx <= 8'd0; + men_ph <= 2'd0; + men_done <= 1'b0; + cmp_idx <= 11'd0; + cmp_ph <= 1'b0; + cmp_neq <= 1'b0; + cmp_done <= 1'b0; + dec_reject <= 1'b0; + dj_blk <= 4'd0; + dj_byte <= 8'd0; + dj_phase <= 2'd0; + dj_mbvalid <= 1'b0; + dj_mblast <= 1'b0; + dj_ack <= 1'b0; + dj_done <= 1'b0; + dj_wb_vld <= 1'b0; + dj_block_r <= 1088'd0; + h_blk <= 3'd0; + h_byte <= 8'd0; + h_phase <= 2'd0; + h_wb_vld <= 1'b0; + h_wb_idx <= 8'd0; + h_wb_g <= 12'd0; + h_wb_pad <= 8'd0; + h_wb_inek <= 1'b0; + end else begin + st <= st_next; + + // BRAM write-enables default low; pulsed where a byte is written. + ek_we <= 1'b0; + dkp_we <= 1'b0; + td_we <= 1'b0; // TDEC bank_a write default low + ct_we <= 1'b0; // ct_bram byte write default low (E5/E7) + em_we <= 1'b0; // e2-relocate bank_a write default low (E6) + + // Kick off when entering from IDLE: KeyGen starts G; Encaps captures + // op/m and arms the H(ek) machinery (ST_ENC_H reuses the ST_H FSM); + // Decaps captures op (dk/c already streamed; z/h captured at load). + if (st == ST_IDLE && start_i) begin + k_r <= k_i; // capture runtime ML-KEM param + op_r <= op_i; + if (op_i == 2'd1) begin + m_r <= msg_i; // capture Encaps message + // arm H(ek) (same fields the ST_E->ST_H arming sets) + h_blk <= 3'd0; + h_byte <= 8'd0; + h_phase <= 2'd0; // assemble + h_mbvalid<= 1'b0; + h_mblast <= 1'b0; + h_ack <= 1'b1; // ready to consume final digest + h_wb_vld <= 1'b0; + end else if (op_i == 2'd2) begin + // Decaps: dk_pke/ek_pke already in BRAM; z/H(ek) captured into + // z_r/hek_r during load (below). Nothing else to arm in D0. + end else begin + sha3_valid <= 1'b1; + sha3_ack <= 1'b1; + end + end + + // Decaps dk load: capture the H(ek) and z byte regions into registers + // as they stream in (the dk_pke/ek_pke regions go to BRAM via the + // write muxes). Byte i within a region lands in [8i +: 8]. Uses the + // LIVE k_i boundaries (k_r not yet captured during preload). + if (dk_in_we) begin + if (dk_in_addr >= dk_ek_end_ld && dk_in_addr < dk_hek_end_ld) + hek_r[(dk_in_addr - dk_ek_end_ld)*8 +: 8] <= dk_in_byte; // H(ek) + else if (dk_in_addr >= dk_hek_end_ld) + z_r[(dk_in_addr - dk_hek_end_ld)*8 +: 8] <= dk_in_byte; // z + end + // Drop valid once accepted + if (sha3_valid && sha3_ready) sha3_valid <= 1'b0; + + // Capture rho/sigma when G completes; arm A stage + if (st == ST_G && sha3_vo) begin + rho_r <= sha3_hash[255:0]; // rho = G output bytes 0..31 + sigma_r <= sha3_hash[511:256]; // sigma = bytes 32..63 + sha3_ack <= 1'b0; + snt_valid <= 1'b1; // start first SampleNTT + snt_ack <= 1'b1; + a_pair <= 5'd0; + a_i <= 3'd0; + a_j <= 3'd0; + a_widx <= 8'd0; + a_busy <= 1'b0; + end + + // ---- ST_A / ST_ENC_A: drive SampleNTT, store 256 coeffs per pair ---- + if (st == ST_A || st == ST_ENC_A) begin + // mark busy once this pair's request accepted + if (snt_valid && snt_ready) begin + snt_valid <= 1'b0; + a_busy <= 1'b1; + end + + // store each output coefficient only while busy (ignore stale + // last coeff from prior poly). The bank_a write itself is the + // combinational ba_we/ba_wa/ba_wd assigns above; here we only + // advance the write index and (i,j) bookkeeping. + if (a_busy && snt_vo && snt_ack) begin + if (snt_last) begin + // finished this poly; advance (i,j) in row-major order + a_pair <= a_pair + 5'd1; + a_widx <= 8'd0; + a_busy <= 1'b0; + if (a_j + 3'd1 < k_r) begin + a_j <= a_j + 3'd1; + end else begin + a_j <= 3'd0; + a_i <= a_i + 3'd1; + end + // start next SampleNTT if more pairs remain + if (a_pair + 5'd1 < kk_rt) snt_valid <= 1'b1; + end else begin + a_widx <= a_widx + 8'd1; + end + end + end + + // Arm C stage when A finishes (KeyGen ST_A or Encaps ST_ENC_A) + if ((st == ST_A && st_next == ST_C) || + (st == ST_ENC_A && st_next == ST_ENC_C)) begin + cbd_valid <= 1'b1; + cbd_ack <= 1'b1; + c_poly <= 3'd0; + c_widx <= 8'd0; + c_busy <= 1'b0; + end + + // ---- ST_C / ST_ENC_C: drive CBD, store 256 mod-q coeffs per poly ---- + // KeyGen: 2K polys (s,e). Encaps: 2K+1 polys (y,e1,e2). Loop bound + // differs: KeyGen restarts while c_poly+1 < 2K; Encaps while < 2K+1. + if (st == ST_C || st == ST_ENC_C) begin + if (cbd_valid && cbd_ready) begin + cbd_valid <= 1'b0; + c_busy <= 1'b1; + end + + if (c_busy && cbd_vo && cbd_ack) begin + if (cbd_last) begin + c_poly <= c_poly + 3'd1; + c_widx <= 8'd0; + c_busy <= 1'b0; + // restart for next poly while more remain (state-dependent bound) + if (st == ST_ENC_C) begin + if (c_poly + 3'd1 < {k_r, 1'b1}) cbd_valid <= 1'b1; // < 2K+1 + end else begin + if (c_poly + 3'd1 < {1'b0, k_r, 1'b0}) cbd_valid <= 1'b1; // < 2K + end + end else begin + c_widx <= c_widx + 8'd1; + end + end + end + + // Arm N stage when C finishes: prime load of slot S0. n_ridx is a + // read-ahead pointer; bank_se read is registered inside sd_bram (bse_rd_data) + // and fed to ntt_core one cycle later, so valid starts low (priming). + if ((st == ST_C && st_next == ST_N) || + (st == ST_ENC_C && st_next == ST_ENC_N) || + (st == ST_DEC_SDEC && st_next == ST_DEC_NTT)) begin + n_slot <= 3'd0; + n_ridx <= 9'd0; + n_widx <= 8'd0; + n_valid <= 1'b0; + n_loading <= 1'b1; // begin presenting load addresses + n_pending <= 1'b0; + end + + // ---- ST_N / ST_ENC_N / ST_DEC_NTT: forward NTT in place. KeyGen: 2K + // slots (s,e). Encaps: K slots (y). Decaps D2: K slots (u' -> u_hat). ---- + if (st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) begin + // slot-count bound: 2K for KeyGen, K for Encaps + // (n_slot_max below); same LOAD/OUTPUT cadence either way. + if (n_loading) begin + if (n_ridx == 9'd256) begin + // 256th coeff consumed this cycle; stop presenting addr + n_loading <= 1'b0; + n_valid <= 1'b0; + end else begin + n_ridx <= n_ridx + 9'd1; + n_valid <= 1'b1; // data presented last cycle is valid + end + end + + // OUTPUT phase: collect 256 results, write back to same slot. + if (ntt_vo) begin + n_widx <= n_widx + 8'd1; // wraps 255->0 after last + end + + // Slot complete when ntt_core returns to DONE + if (ntt_done) begin + if (n_slot + 3'd1 < n_slot_max) begin + n_slot <= n_slot + 3'd1; + n_widx <= 8'd0; + n_pending <= 1'b1; // wait one cycle for core IDLE + end else begin + n_slot <= n_slot + 3'd1; // == n_slot_max -> exit + end + end + + // Kick next slot's load once core is back IDLE (re-prime) + if (n_pending && ntt_ready && !ntt_done) begin + n_ridx <= 9'd0; + n_valid <= 1'b0; + n_loading <= 1'b1; + n_pending <= 1'b0; + end + end + + // Arm M stage when N finishes: prime first (i=0,j=0) poly_mul load. + // m_ld read-ahead pointer; bank_a/bank_se sd_bram reads land 1 cyc + // later, pm_valid asserted one cycle after an address is presented. + if (st == ST_N && st_next == ST_M) begin + m_i <= 2'd0; + m_j <= 2'd0; + m_ld <= 9'd0; + m_oidx <= 8'd0; + m_loading <= 1'b1; + m_pending <= 1'b0; + pm_valid <= 1'b0; + end + + // ---- ST_M: t_hat[i] = e_hat[i] + sum_j A[i][j] o s_hat[j] ---- + if (st == ST_M) begin + // accumulator reads come from sd_bram (bse_rd_data e_hat / + // bt_rd_data t_hat). m_jq aligns the j-select with the 1-cycle + // read latency. (bse_rd_addr muxes load vs acc by m_loading.) + m_jq <= (m_j == 2'd0); + + // LOAD: present read-ahead addr to bank_a/bank_se via their + // sd_bram read ports (ba_rd_addr=pm_a_full, bse_rd_addr=pm_b_full + // while m_loading); ba_rd_data/bse_rd_data land next cycle and + // are consumed by poly_mul (pm_valid). poly_mul holds ready high + // through LOAD, so a fixed 1-cycle skew suffices. + if (m_loading) begin + if (m_ld == 9'd256) begin + pm_valid <= 1'b0; // 256th pair consumed this cycle + m_loading <= 1'b0; + m_ld <= 9'd0; + m_oidx <= 8'd0; + end else begin + m_ld <= m_ld + 9'd1; + pm_valid <= 1'b1; // pair presented last cycle is valid + end + end + + // ACCUMULATE: each product coeff += e_hat (j==0) or running t_hat. + // The bank_t write is the combinational bt_we/bt_wa/bt_wd assigns + // below; here we only advance the accumulate index / (i,j). + if (pm_vo) begin + if (m_oidx == 8'd255) begin + // finished this (i,j) term; advance + if (m_j + 2'd1 < k_r) begin + m_j <= m_j + 2'd1; + m_pending <= 1'b1; // next term, same row + end else begin + m_j <= 2'd0; + m_i <= m_i + 2'd1; // next row (or == K -> DONE) + if (m_i + 2'd1 < k_r) m_pending <= 1'b1; + end + end else begin + m_oidx <= m_oidx + 8'd1; + end + end + + // Start next (i,j) poly_mul load once core is IDLE again (re-prime) + if (m_pending && pm_ready && !pm_vo) begin + m_loading <= 1'b1; + m_ld <= 9'd0; + m_oidx <= 8'd0; + pm_valid <= 1'b0; + m_pending <= 1'b0; + end + end + + // Arm Encaps U stage when N finishes: prime first row MAC (i=0,j=0). + if (st == ST_ENC_N && st_next == ST_ENC_U) begin + u_row <= 3'd0; + u_sub <= 2'd0; // MAC + u_j <= 3'd0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + u_loading <= 1'b1; + u_pending <= 1'b0; + pm_valid <= 1'b0; + end + + // ---- ST_ENC_U/V sub-phase 0: MAC sum_j (A^T or t_hat)[j] o y_hat[j] ---- + // psum (init 0 at j==0) accumulates into bank_t[UPSUM]. Mirrors ST_M + // load/accumulate cadence (read-ahead by 1, j-select via u_j0q). + // U: A^T[u_row][j] (bank_a u_j*K+u_row). V: t_hat[j] (bank_a u_j*K, u_row=0). + if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd0) begin + u_j0q <= (u_j == 3'd0); + + if (u_loading) begin + if (u_ld == 9'd256) begin + pm_valid <= 1'b0; + u_loading <= 1'b0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + end else begin + u_ld <= u_ld + 9'd1; + pm_valid <= 1'b1; + end + end + + if (pm_vo) begin + // bank_t psum write is combinational (u_psum_we); advance idx + if (u_oidx == 8'd255) begin + if (u_j + 3'd1 < k_r) begin + u_j <= u_j + 3'd1; + u_pending <= 1'b1; // next term, same row + end else begin + // row's MAC done -> INTT sub-phase + u_sub <= 2'd1; + u_ridx <= 9'd0; + u_widx <= 8'd0; + u_nvalid <= 1'b0; + u_nloading <= 1'b1; + end + end else begin + u_oidx <= u_oidx + 8'd1; + end + end + + // re-prime next term's poly_mul load + if (u_pending && pm_ready && !pm_vo) begin + u_loading <= 1'b1; + u_ld <= 9'd0; + u_oidx <= 8'd0; + pm_valid <= 1'b0; + u_pending <= 1'b0; + end + end + + // ---- ST_ENC_U/V sub-phase 1: INTT(psum) mode=1 in place (bank_t) ---- + if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd1) begin + if (u_nloading) begin + if (u_ridx == 9'd256) begin + u_nloading <= 1'b0; + u_nvalid <= 1'b0; + end else begin + u_ridx <= u_ridx + 9'd1; + u_nvalid <= 1'b1; + end + end + if (ntt_vo) u_widx <= u_widx + 8'd1; // in-place write (u_intt_we) + if (ntt_done) begin + // INTT done -> ADD sub-phase (psum + e1[u_row]) + u_sub <= 2'd2; + u_aidx <= 9'd0; + u_awidx <= 8'd0; + u_avalid <= 1'b0; + end + end + + // ---- ST_ENC_U/V + ST_DEC_W sub-phase 2: ADD/SUB + writeback ---- + // U: u[u_row] = psum + e1[u_row] mod Q -> bank_se (u_add_we). + // V: v = psum + e2 + mu mod Q -> bank_t[UPSUM] (u_v_we). + // D3 w = v' - psum mod Q -> bank_t[UPSUM] (u_w_we). + // Read psum (bank_t) + e1/e2/v' at u_aidx; both arrive 1 cyc later, so + // register (valid,widx) and write next cycle. u_row_max bounds the + // row loop (K for U, 1 for V/D3). + if ((st == ST_ENC_U || st == ST_ENC_V || st == ST_DEC_W) && u_sub == 2'd2) begin + if (u_aidx < 9'd256) begin + u_aidx <= u_aidx + 9'd1; + u_avalid <= 1'b1; // addr presented this cyc -> write next + u_awidx <= u_aidx[7:0]; + end else begin + u_avalid <= 1'b0; // no more addresses + end + // writeback (u_add_we / u_v_we) commits index u_awidx this cycle. + // Advance row after the 256th coeff (u_awidx==255) is written. + if (u_avalid && u_awidx == 8'd255) begin + if (u_row + 3'd1 < u_row_max) begin + u_row <= u_row + 3'd1; + u_sub <= 2'd0; // next row MAC + u_j <= 3'd0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + u_loading <= 1'b1; + u_pending <= 1'b0; + pm_valid <= 1'b0; + end else begin + u_row <= u_row + 3'd1; // U: ==K ->C1; V: ==1 ->DONE + end + end + end + + // Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)). + // Walk K polys * 256 coeffs; reset bit-packer + ct write pointer. + if (st == ST_ENC_U && st_next == ST_ENC_C1) begin + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 25'd0; + cp_nbits <= 6'd0; + cp_wa <= 12'd0; + cp_done <= 1'b0; + cd_valid <= 1'b0; + end + + // ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ---- + // C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes). + // C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes). + // Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram + 1 cyc + // comp_decomp pipe), then a drain sub-phase emitting whole bytes: + // ph0: present coeff addr (cp_se_full / cp_bt_full by state). + // ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid. + // ph2: drop cd_valid (1-cyc pulse); comp_decomp captures. + // ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first. + // ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance. + // Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at + // each poly boundary (no carry across polys / regions). + if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin + case (cp_ph) + 3'd0: cp_ph <= 3'd1; // addr presented; wait read + 3'd1: begin + cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) + cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse) + cp_ph <= 3'd2; + end + 3'd2: begin + cd_valid <= 1'b0; // comp_decomp captured this cyc + cp_ph <= 3'd3; + end + 3'd3: begin + // cd_out valid (cd_vo): append cp_d bits LSB-first at bit cp_nbits + cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits); + cp_nbits <= cp_nbits + {1'b0, cp_d}; + cp_ph <= 3'd4; + end + default: begin // 3'd4: drain whole bytes + if (cp_nbits >= 6'd8) begin + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= cp_buf[7:0]; + cp_wa <= cp_wa + 12'd1; + cp_buf <= cp_buf >> 8; + cp_nbits <= cp_nbits - 6'd8; + end else begin + // coeff fully packed; advance coeff / poly + if (cp_idx == 8'd255) begin + cp_idx <= 8'd0; + if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1; + else cp_done <= 1'b1; // region complete + end else begin + cp_idx <= cp_idx + 8'd1; + end + cp_ph <= 3'd0; + end + end + endcase + end + + // Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)). + // Single poly (v at bank_t UPSUM); ct write pointer cp_wa CONTINUES + // from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits + // are 0 here (c1 ended on a poly/byte boundary) but reset for safety. + if (st == ST_ENC_V && st_next == ST_ENC_C2) begin + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 25'd0; + cp_nbits <= 6'd0; + cp_done <= 1'b0; + cd_valid <= 1'b0; + // cp_wa intentionally preserved (= c1_bytes_rt from C1). + end + + // Arm D1 DECOMP when load settles: decode c1 (u') first. + if (st == ST_DEC_LOAD && st_next == ST_DEC_DECOMP) begin + dec_poly <= 3'd0; + dec_cidx <= 8'd0; + dec_widx <= 8'd0; + dec_ph <= 2'd0; + dec_ba <= 12'd0; + dec_buf <= 25'd0; + dec_nbits <= 6'd0; + dec_in_c2 <= 1'b0; + dec_dc_valid <= 1'b0; + dec_dc_done <= 1'b0; + cin_rd_addr_r<= 11'd0; // present byte 0 address + end + + // ---- ST_DEC_DECOMP (D1): byteDecode_d + Decompress c1/c2 ---- + // Reverse of the C1/C2 packer. Per coeff: + // ph0: if nbits ph1; + // else extract low d bits -> comp_decomp (decompress) -> ph2. + // ph1: append cin_rd_data (8 bits) at dec_nbits; advance dec_ba; + // present next byte addr; back to ph0. + // ph2: drop dc_valid (pulse captured) -> ph3. + // ph3: cd_vo -> dec_u_we/dec_v_we writes cd_out; advance coeff. + // c1 = K polys (d=du) -> u'[poly] bank_se; then c2 = 1 poly (d=dv) + // -> v' bank_a[DEC_VASLOT]. Byte stream is contiguous in c_in_bram. + if (st == ST_DEC_DECOMP && !dec_dc_done) begin + case (dec_ph) + 2'd0: begin + if (dec_nbits >= {1'b0, dec_dc_d}) begin + // enough bits: extract low d as the decoded symbol + dec_dc_coeff <= dec_buf[11:0] & ((12'd1 << dec_dc_d) - 12'd1); + dec_dc_valid <= 1'b1; + dec_buf <= dec_buf >> dec_dc_d; + dec_nbits <= dec_nbits - {1'b0, dec_dc_d}; + dec_ph <= 2'd2; + end else begin + // need another byte (cin_rd_data reflects dec_ba) + dec_ph <= 2'd1; + end + end + 2'd1: begin + dec_buf <= dec_buf | ({17'd0, cin_rd_data} << dec_nbits); + dec_nbits <= dec_nbits + 6'd8; + dec_ba <= dec_ba + 12'd1; + cin_rd_addr_r<= dec_ba[10:0] + 11'd1; // present next byte + dec_ph <= 2'd0; + end + 2'd2: begin + dec_dc_valid <= 1'b0; // comp_decomp captured + dec_ph <= 2'd3; + end + default: begin // 2'd3: capture cd_vo + if (cd_vo) begin + // dec_u_we/dec_v_we commit cd_out at dec_widx this cycle. + dec_widx <= dec_widx + 8'd1; + if (dec_cidx == 8'd255) begin + dec_cidx <= 8'd0; + dec_widx <= 8'd0; + if (!dec_in_c2) begin + if (dec_poly + 3'd1 < k_r) begin + dec_poly <= dec_poly + 3'd1; // next u' row + end else begin + dec_in_c2 <= 1'b1; // switch to c2 (v') + dec_poly <= 3'd0; + end + end else begin + dec_dc_done <= 1'b1; // v' done -> D1 complete + end + end else begin + dec_cidx <= dec_cidx + 8'd1; + end + dec_ph <= 2'd0; + end + end + endcase + end + + // Arm E stage when M finishes + if (st == ST_M && st_next == ST_E) begin + e_poly <= 3'd0; + e_pair <= 8'd0; + e_ph <= 2'd0; // start at fetch-c0 of first pair + e_rho <= 10'd0; + e_done <= 1'b0; + end + + // ---- ST_E: byteEncode12 t_hat -> ek, s_hat -> dk_pke ---- + // Single registered bank read per cycle; 4-cycle micro-phase per + // coeff pair (ph0 fetch c0, ph1 write b0 + fetch c1, ph2 write b1, + // ph3 write b2). The coeff for the addr presented last cycle is + // e_rd_coeff (= bse_rd_data for dk-half, bt_rd_data for ek-half), + // both registered inside their sd_bram (bse_rd_addr/bt_rd_addr = + // e_rd_full in ST_E). + if (st == ST_E && !e_done) begin + if (e_poly < {1'b0, k_r, 1'b0}) begin + // ph0: fetch only (prime). ph1..3: write one packed byte. + if (e_ph != 2'd0) begin + if (!e_is_dk) begin + ek_we <= 1'b1; + ek_wa <= e_boff[10:0] + {9'd0, e_wb}; + ek_wd <= e_wbyte; + end else begin + dkp_we <= 1'b1; + dkp_wa <= e_boff[10:0] + {9'd0, e_wb}; + dkp_wd <= e_wbyte; + end + end + // save coeff high nibbles as they become available + if (e_ph == 2'd1) e_c0_hi <= e_rd_coeff[11:8]; // c0[11:8] + if (e_ph == 2'd2) e_c1_hi <= e_rd_coeff[11:4]; // c1[11:4] + + // advance micro-phase / pair / poly + if (e_ph == 2'd3) begin + e_ph <= 2'd0; + if (e_pair == 8'd127) begin + e_pair <= 8'd0; + e_poly <= e_poly + 5'd1; // next poly or -> rho phase + end else begin + e_pair <= e_pair + 8'd1; + end + end else begin + e_ph <= e_ph + 2'd1; + end + end else begin + // rho copy: ek[384*K + r] = rho byte r (r = 0..31), 1 byte/cycle + ek_we <= 1'b1; + ek_wa <= dk_bytes_rt[10:0] + {1'b0, e_rho}; + ek_wd <= rho_r[e_rho*8 +: 8]; + if (e_rho == 10'd31) e_done <= 1'b1; + else e_rho <= e_rho + 10'd1; + end + end + + // Arm H stage when E finishes + if (st == ST_E && st_next == ST_H) begin + h_blk <= 3'd0; + h_byte <= 8'd0; + h_phase <= 2'd0; // assemble + h_mbvalid<= 1'b0; + h_mblast <= 1'b0; + h_ack <= 1'b1; // ready to consume final digest + h_wb_vld <= 1'b0; // no pending writeback yet + end + + // ---- ST_H / ST_ENC_H: H(ek) via multi-block SHA3-256 ---- + // Same logic for KeyGen (ek just written by ST_E) and Encaps (ek + // preloaded into ek_bram by the TB). hek_r captures H(ek) either way. + if (st == ST_H || st == ST_ENC_H) begin + case (h_phase) + // assemble 136 bytes of block h_blk into h_block_r. + // ek BRAM read is registered: present addr for h_byte this + // cycle, write back the byte that arrived for the addr we + // presented last cycle (h_wb_*). h_byte runs 0..136 (one + // extra cycle to flush the final writeback). + 2'd0: begin + // writeback the byte read for the previous address + if (h_wb_vld) + h_block_r[h_wb_idx*8 +: 8] <= h_wb_inek ? ek_rd_data : h_wb_pad; + + if (h_byte <= 8'd135) begin + // set up writeback for the address presented this cycle + h_wb_vld <= 1'b1; + h_wb_idx <= h_byte; + h_wb_g <= h_g_addr; + h_wb_inek <= (h_g_addr < ek_bytes_rt); + h_wb_pad <= h_padconst(h_blk, h_byte); + h_byte <= h_byte + 8'd1; + end else begin + // h_byte==136: final writeback (for byte 135) done above + h_wb_vld <= 1'b0; + h_byte <= 8'd0; + h_mbvalid <= 1'b1; + h_mblast <= (h_blk == h_nblk_rt - 4'd1); + h_phase <= 2'd1; // feed + end + end + // feed: hold valid until accepted (mb_ready drops) + 2'd1: begin + if (h_mbvalid && !h_mbready) begin + h_mbvalid <= 1'b0; + h_mblast <= 1'b0; + h_phase <= 2'd2; // wait permute + end + end + // wait permute done: ready again (more blocks) or digest valid (last) + 2'd2: begin + if (h_vo) begin + hek_r <= h_hash[255:0]; + h_phase <= 2'd3; // done + end else if (h_mbready) begin + h_blk <= h_blk + 3'd1; + h_phase <= 2'd0; // assemble next block + end + end + default: ; // 2'd3 done: hold + endcase + end + + // Arm Encaps G when H(ek) finishes: fire the 64-byte single-block + // G(m||H(ek)). hek_r now holds H(ek); enc_g_data = {hek_r, m_r}. + if (st == ST_ENC_H && st_next == ST_ENC_G) begin + sha3_valid <= 1'b1; + sha3_ack <= 1'b1; + end + // Capture (K, r) when Encaps G completes. ss = K = low half, r = high. + if (st == ST_ENC_G && sha3_vo) begin + ss_r <= sha3_hash[255:0]; // K (shared secret) = G bytes 0..31 + r_r <= sha3_hash[511:256]; // r = G bytes 32..63 + sha3_ack <= 1'b0; + end + + // Arm Decaps D5 G when m' is ready (ST_DEC_MENC -> ST_DEC_G): fire the + // 64-byte single-block G(m'||h). dec_g_data = {hek_r, m_r}. + if (st == ST_DEC_MENC && st_next == ST_DEC_G) begin + sha3_valid <= 1'b1; + sha3_ack <= 1'b1; + end + // Capture (K', r') when D5 G completes. K' = low half (candidate ss), + // r' = high half (PRF seed for D6 re-encrypt). Reuse ss_r / r_r. + if (st == ST_DEC_G && sha3_vo) begin + ss_r <= sha3_hash[255:0]; // K' = G bytes 0..31 + r_r <= sha3_hash[511:256]; // r' = G bytes 32..63 + sha3_ack <= 1'b0; + end + + // Arm D5 J(z||c) when G completes (ST_DEC_G -> ST_DEC_J): assemble the + // first 136-byte block. dj_ack high to consume the final digest. + if (st == ST_DEC_G && st_next == ST_DEC_J) begin + dj_blk <= 4'd0; + dj_byte <= 8'd0; + dj_phase <= 2'd0; + dj_mbvalid<= 1'b0; + dj_mblast <= 1'b0; + dj_ack <= 1'b1; + dj_done <= 1'b0; + dj_wb_vld <= 1'b0; + cin_rd_addr_r <= 11'd0; // present c byte 0 (g=32 -> c_idx 0) + end + + // ---- ST_DEC_J: K-bar = J(z||c) multi-block SHAKE256 ---- + // Mirror of the H(ek) multi-block machine. Byte source by global g: + // g < 32 -> z_r byte g + // 32 <= g < msglen -> c_in_bram byte (g-32), registered read + // else -> SHAKE256 pad constant + // c_in_bram read is registered: present c_idx for dj_byte this cycle, + // write back the byte that arrived for the addr presented last cycle. + if (st == ST_DEC_J && !dj_done) begin + case (dj_phase) + 2'd0: begin + // writeback the byte read for the previous address + if (dj_wb_vld) begin + if (dj_wb_inz) + dj_block_r[dj_wb_idx*8 +: 8] <= z_r[dj_wb_zidx*8 +: 8]; + else if (dj_wb_inc) + dj_block_r[dj_wb_idx*8 +: 8] <= cin_rd_data; + else + dj_block_r[dj_wb_idx*8 +: 8] <= dj_wb_pad; + end + if (dj_byte <= 8'd135) begin + dj_wb_vld <= 1'b1; + dj_wb_idx <= dj_byte; + dj_wb_inz <= (dj_g < 12'd32); + dj_wb_inc <= (dj_g >= 12'd32) && (dj_g < dj_msglen); + dj_wb_zidx <= dj_g[4:0]; + dj_wb_pad <= dj_padconst(dj_g); + // c_in addr is presented combinationally (cin_rd_addr + // mux uses dj_c_idx during ST_DEC_J); data lands next + // cycle, matching this byte's writeback. + dj_byte <= dj_byte + 8'd1; + end else begin + dj_wb_vld <= 1'b0; + dj_byte <= 8'd0; + dj_mbvalid <= 1'b1; + dj_mblast <= (dj_blk == dj_nblk - 4'd1); + dj_phase <= 2'd1; + end + end + 2'd1: begin + if (dj_mbvalid && !h_mbready) begin + dj_mbvalid <= 1'b0; + dj_mblast <= 1'b0; + dj_phase <= 2'd2; + end + end + 2'd2: begin + if (sha3_vo) begin + kbar_r <= sha3_hash[255:0]; + dj_done <= 1'b1; + dj_phase<= 2'd3; + end else if (h_mbready) begin + dj_blk <= dj_blk + 4'd1; + dj_phase <= 2'd0; + end + end + default: ; // done + endcase + end + + // Arm D7 compare when re-encrypt finishes (ST_ENC_C2 -> ST_DEC_CMP). + if (st == ST_ENC_C2 && st_next == ST_DEC_CMP) begin + cmp_idx <= 11'd0; + cmp_ph <= 1'b0; + cmp_neq <= 1'b0; + cmp_done <= 1'b0; + end + + // ---- ST_DEC_CMP (D7): constant-time c' == c compare + ss select ---- + // ph0: present cmp_idx to both ct_bram (c') and c_in_bram (c). + // ph1: registered reads valid -> OR any difference into cmp_neq, + // advance. After the last byte, latch dec_reject and ss stays + // K' (ss_r) or switches to K-bar (kbar_r) via the ss_o mux. + if (st == ST_DEC_CMP && !cmp_done) begin + if (!cmp_ph) begin + cmp_ph <= 1'b1; // addr presented; wait for read + end else begin + cmp_neq <= cmp_neq | cmp_byte_neq; + if (cmp_idx == ct_bytes_rt - 12'd1) begin + dec_reject <= cmp_neq | cmp_byte_neq; // include final byte + cmp_done <= 1'b1; + end else begin + cmp_idx <= cmp_idx + 11'd1; + end + cmp_ph <= 1'b0; + end + end + + if (st_next == ST_ENC_LOAD && + (st == ST_ENC_G || st == ST_DEC_J)) begin + rl_idx <= 6'd0; + rl_widx <= 6'd0; + rl_vld <= 1'b0; + end + + // ---- ST_ENC_LOAD: stream 32 rho bytes from ek_bram into rho_r ---- + // ek read is registered (1-cyc latency): the data arriving this cycle + // is for the address presented last cycle (rl_idx-1). Present addr + // rl_idx (valid 0..31); write rho_r[rl_idx-1] when rl_idx in 1..32. + if (st == ST_ENC_LOAD) begin + if (rl_idx >= 6'd1 && rl_idx <= 6'd32) + rho_r[(rl_idx-6'd1)*8 +: 8] <= ek_rd_data; + rl_idx <= rl_idx + 6'd1; // exit when rl_idx >= 33 + end + + // Arm ST_ENC_A (regenerate A_hat via SampleNTT, same as ST_A). + if (st == ST_ENC_LOAD && st_next == ST_ENC_A) begin + snt_valid <= 1'b1; + snt_ack <= 1'b1; + a_pair <= 5'd0; + a_i <= 3'd0; + a_j <= 3'd0; + a_widx <= 8'd0; + a_busy <= 1'b0; + end + + // Arm ST_ENC_TDEC (byteDecode12: ek -> t_hat in bank_a). Now follows + // E5 (ST_ENC_C1) since e2 occupied bank_t during C/N/U and t_hat is + // only needed for V. (Was armed off ST_ENC_A in the E1 scaffold.) + if (st == ST_ENC_C1 && st_next == ST_ENC_TDEC) begin + td_poly <= 3'd0; + td_trip <= 8'd0; + td_ph <= 3'd0; + td_done <= 1'b0; + end + // Arm D2 s_hat decode (ST_DEC_SDEC) when DECOMP finishes: same TDEC + // machine, byteDecode12 dk_pke -> s_hat in bank_a slot j*K. + if (st == ST_DEC_DECOMP && st_next == ST_DEC_SDEC) begin + td_poly <= 3'd0; + td_trip <= 8'd0; + td_ph <= 3'd0; + td_done <= 1'b0; + end + + // Encaps: ek -> t_hat. Decaps D2: dk_pke -> s_hat. Same 5-cycle + // micro-phase per triple; only the byte source (td_byte) differs. + if ((st == ST_ENC_TDEC || st == ST_DEC_SDEC) && !td_done) begin + // capture the byte that arrived for the address presented last cycle + if (td_ph == 3'd1) td_b0 <= td_byte; + if (td_ph == 3'd2) td_b1 <= td_byte; + if (td_ph == 3'd3) td_b2 <= td_byte; + + // write decoded coeffs into bank_a at slot td_poly*K (so V's MAC + // reads t_hat[j] via the same u_aslot=u_j*K addressing, u_row=0). + if (td_ph == 3'd3) begin + td_we <= 1'b1; + td_wa <= (({4'd0,td_poly}*{9'd0,k_r})*256 + {td_trip, 1'b0}) & ((1< bank_a[E2_ASLOT], 256 coeffs ---- + // Present read addr em_ridx (bank_t rel 0); bank_t read is registered + // so bt_rd_data next cycle = e2[em_ridx]. Schedule the bank_a write + // (em_we/em_widx) for that next cycle with em_widx == em_ridx, so the + // write commits e2[R] into slot[R] (no off-by-one). + if (st == ST_ENC_E2MV && !em_done) begin + if (em_ridx <= 9'd255) begin // presenting a valid e2 addr this cycle + em_we <= 1'b1; // -> write next cycle (ba_wd = bt_rd_data) + em_widx <= em_ridx[7:0]; + end + if (em_ridx == 9'd256) em_done <= 1'b1; // last write (e2[255]) issued + else em_ridx <= em_ridx + 9'd1; + end + + // Arm ST_ENC_V when e2 relocated: prime first (only) row MAC (i=0,j=0). + // Reuses the u_* MAC/INTT/ADD machine with u_row tied to 0. + if (st == ST_ENC_E2MV && st_next == ST_ENC_V) begin + u_row <= 3'd0; + u_sub <= 2'd0; // MAC + u_j <= 3'd0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + u_loading <= 1'b1; + u_pending <= 1'b0; + pm_valid <= 1'b0; + end + + // Arm D3 (ST_DEC_W) when D2 NTT finishes: same u_* MAC/INTT machine, + // single row (u_row 0..0). MAC s_hat[j] o u_hat[j], INTT, then SUB. + if (st == ST_DEC_NTT && st_next == ST_DEC_W) begin + u_row <= 3'd0; + u_sub <= 2'd0; // MAC + u_j <= 3'd0; + u_ld <= 9'd0; + u_oidx <= 8'd0; + u_loading <= 1'b1; + u_pending <= 1'b0; + pm_valid <= 1'b0; + end + + // Arm D4 (ST_DEC_MENC) when D3 w done: byteEncode_1(Compress_1(w)). + if (st == ST_DEC_W && st_next == ST_DEC_MENC) begin + men_idx <= 8'd0; + men_ph <= 2'd0; + men_done <= 1'b0; + end + + // ---- ST_DEC_MENC (D4): m' = byteEncode_1(Compress_1(w)) ---- + // Compress_1(w)=1 iff 832 < w <= 2496 (Q=3329). Pack 256 bits + // LSB-first into m_r (bit men_idx). Read w from bank_t[UPSUM]. + // ph0: present w[men_idx] addr; ph1: bt_rd_data valid -> set bit. + if (st == ST_DEC_MENC && !men_done) begin + case (men_ph) + 2'd0: men_ph <= 2'd1; // addr presented; wait read + default: begin // ph1: bt_rd_data = w[men_idx] + m_r[men_idx] <= men_w_bit; + if (men_idx == 8'd255) men_done <= 1'b1; + else men_idx <= men_idx + 8'd1; + men_ph <= 2'd0; + end + endcase + end + end + end + +endmodule diff --git a/rtl/soc_top.v b/rtl/soc_top.v index f8f4c8f..8ca2c1a 100644 --- a/rtl/soc_top.v +++ b/rtl/soc_top.v @@ -604,6 +604,7 @@ wire axiOut_7_bready ; wire confreg_int; wire matmul_intr; +wire mlkem_intr; AxiCrossbar_2x8 u_AxiCrossbar_2x8 ( .clk ( sys_clk ), @@ -1035,8 +1036,8 @@ AxiCrossbar_2x8 u_AxiCrossbar_2x8 ( core_top u_cpu( - .intrpt ({5'd0, matmul_intr, dma_finish, confreg_int}), //high active - // intrpt[2]=matmul, intrpt[1]=dma, intrpt[0]=confreg + .intrpt ({4'd0, mlkem_intr, matmul_intr, dma_finish, confreg_int}), //high active + // intrpt[3]=mlkem, intrpt[2]=matmul, intrpt[1]=dma, intrpt[0]=confreg .aclk (cpu_clk ), .aresetn (cpu_resetn ), //low active @@ -1424,18 +1425,54 @@ axi_dvi u_axi_dvi ( .video_blue ( video_blue ) ); -// slave 6 (FFT) removed — tie off -assign fft_bid = 5'b0; -assign fft_bresp = 2'b0; -assign fft_bvalid = 1'b0; -assign fft_rid = 5'b0; -assign fft_rdata = 32'b0; -assign fft_rresp = 2'b0; -assign fft_rlast = 1'b0; -assign fft_rvalid = 1'b0; -assign fft_arready = 1'b1; -assign fft_awready = 1'b1; -assign fft_wready = 1'b1; +// slave 6 - ML-KEM accelerator (0x1f400000, virtual 0xbf400000) +// Bus runs at sys_clk; the core is internally clocked at sys_clk/2 (25 MHz). +mlkem_axi_slave u_mlkem_axi_slave ( + .clk (sys_clk), + .resetn (sys_resetn), + + .s_axi_awid (fft_awid ), + .s_axi_awaddr (fft_awaddr ), + .s_axi_awlen (fft_awlen ), + .s_axi_awsize (fft_awsize ), + .s_axi_awburst (fft_awburst), + .s_axi_awlock (fft_awlock ), + .s_axi_awcache (fft_awcache), + .s_axi_awprot (fft_awprot ), + .s_axi_awvalid (fft_awvalid), + .s_axi_awready (fft_awready), + + .s_axi_wid (fft_wid ), + .s_axi_wdata (fft_wdata ), + .s_axi_wstrb (fft_wstrb ), + .s_axi_wlast (fft_wlast ), + .s_axi_wvalid (fft_wvalid), + .s_axi_wready (fft_wready), + + .s_axi_bid (fft_bid ), + .s_axi_bresp (fft_bresp ), + .s_axi_bvalid (fft_bvalid), + .s_axi_bready (fft_bready), + + .s_axi_arid (fft_arid ), + .s_axi_araddr (fft_araddr ), + .s_axi_arlen (fft_arlen ), + .s_axi_arsize (fft_arsize ), + .s_axi_arburst (fft_arburst), + .s_axi_arlock (fft_arlock ), + .s_axi_arcache (fft_arcache), + .s_axi_arprot (fft_arprot ), + .s_axi_arvalid (fft_arvalid), + .s_axi_arready (fft_arready), + + .s_axi_rid (fft_rid ), + .s_axi_rdata (fft_rdata ), + .s_axi_rresp (fft_rresp ), + .s_axi_rlast (fft_rlast ), + .s_axi_rvalid (fft_rvalid), + .s_axi_rready (fft_rready), + .intr (mlkem_intr) +); //slave 7 - matmul accelerator matmul_axi_slave u_matmul_axi_slave ( diff --git a/sdk/software/examples/mlkem_test/Makefile b/sdk/software/examples/mlkem_test/Makefile new file mode 100644 index 0000000..d7c6f5d --- /dev/null +++ b/sdk/software/examples/mlkem_test/Makefile @@ -0,0 +1,11 @@ +TARGET = mlkem_test + +CFLAGS += -O3 -g + +C_SRCS := $(wildcard ./*.c ) + +OBJDIR = obj +COMMON_DIR = ../../bsp +GCC_DIR=../../../toolchains/loongson-gnu-toolchain-8.3-x86_64-loongarch32r-linux-gnusf-v2.0 +PICOLIBC_DIR=../../../toolchains/picolibc +include ../../bsp/common.mk diff --git a/sdk/software/examples/mlkem_test/main.c b/sdk/software/examples/mlkem_test/main.c new file mode 100644 index 0000000..5f3fa43 --- /dev/null +++ b/sdk/software/examples/mlkem_test/main.c @@ -0,0 +1,221 @@ +/** + * mlkem_test - ML-KEM hardware accelerator example (CIC-IEC 2026) + * + * Exercises the mlkem_axi_slave at 0xbf400000 by running a full + * KeyGen -> Encaps -> Decaps round trip on ML-KEM-512 (k=2) and verifying + * that the decapsulated shared secret matches the encapsulated one. This is a + * self-consistency check: it needs no external golden vectors, yet touches + * every register path (seeds, start/poll, byte preload, byte readback, ss). + * + * KeyGen : d,z -> ek (800 B), dk (1632 B) + * Encaps : ek + msg -> ss, ct (768 B) + * Decaps : dk + ct -> ss' (must equal ss) + * + * The core runs at sys_clk/2 (25 MHz); the wrapper hides the clock crossing, + * so software just does normal MMIO loads/stores (preload writes are + * automatically wait-stated on the bus). + */ + +#include +#include + +/* ─── BSP globals ─────────────────────────────────────────────────── */ +unsigned long UART_BASE = 0xbf000000; +unsigned long CONFREG_TIMER_BASE = 0xbf20f100; +unsigned long CONFREG_CLOCKS_PER_SEC = 50000000L; +unsigned long CORE_CLOCKS_PER_SEC = 33000000L; + +/* ─── ML-KEM IP register map (base 0xbf400000) ────────────────────── */ +#define MLKEM_BASE 0xbf400000 +#define MLKEM_CTRL (MLKEM_BASE + 0x000) /* W bit0 = start */ +#define MLKEM_STATUS (MLKEM_BASE + 0x004) /* R busy/done/preload_busy */ +#define MLKEM_CONFIG (MLKEM_BASE + 0x008) /* RW [2:0]=k [5:4]=op */ +#define MLKEM_D (MLKEM_BASE + 0x010) /* RW seed d (8 words) */ +#define MLKEM_Z (MLKEM_BASE + 0x030) /* RW seed z (8 words) */ +#define MLKEM_MSG (MLKEM_BASE + 0x050) /* RW message (8 words) */ +#define MLKEM_SS (MLKEM_BASE + 0x070) /* R shared secret (8 words) */ +#define MLKEM_PRE_SEL (MLKEM_BASE + 0x100) /* RW 0=ek 1=dk 2=c */ +#define MLKEM_PRE_ADR (MLKEM_BASE + 0x104) /* RW byte addr (auto-incr) */ +#define MLKEM_PRE_DAT (MLKEM_BASE + 0x108) /* W byte -> selected port */ +#define MLKEM_RDB_SEL (MLKEM_BASE + 0x10C) /* RW 0=ek 1=dk_pke 2=dk 3=ct */ +#define MLKEM_RDB_ADR (MLKEM_BASE + 0x110) /* RW output byte index */ +#define MLKEM_RDB_DAT (MLKEM_BASE + 0x114) /* R selected byte (low 8b) */ + +#define CTRL_START 0x1 +#define STATUS_BUSY 0x1 +#define STATUS_DONE 0x2 + +#define OP_KEYGEN 0 +#define OP_ENCAPS 1 +#define OP_DECAPS 2 + +/* preload select */ +#define PRE_EK 0 +#define PRE_DK 1 +#define PRE_C 2 +/* readback select */ +#define RDB_EK 0 +#define RDB_DKP 1 +#define RDB_DK 2 +#define RDB_CT 3 + +/* ─── ML-KEM-512 (k=2) byte sizes ─────────────────────────────────── */ +#define KVAL 2 +#define EK_BYTES (384 * KVAL + 32) /* 800 */ +#define DKP_BYTES (384 * KVAL) /* 768 */ +#define DK_BYTES (DKP_BYTES + EK_BYTES + 32 + 32) /* 1632 */ +#define CT_BYTES 768 + +/* ─── Convenience macros ──────────────────────────────────────────── */ +#define REG_WR(addr, val) (*(volatile uint32_t *)(addr) = (uint32_t)(val)) +#define REG_RD(addr) (*(volatile uint32_t *)(addr)) + +/* PLACEHOLDER */ + +/* ─── Wait until the core finishes the current op ─────────────────── */ +static void wait_done(void) +{ + while (!(REG_RD(MLKEM_STATUS) & STATUS_DONE)) + ; +} + +/* Clear the latched done flag (W1C) before launching a new op. */ +static void clear_done(void) +{ + REG_WR(MLKEM_STATUS, STATUS_DONE); +} + +/* Launch the op currently configured in CONFIG and block until done. */ +static void run_op(uint32_t op) +{ + REG_WR(MLKEM_CONFIG, ((op & 0x3) << 4) | (KVAL & 0x7)); + clear_done(); + REG_WR(MLKEM_CTRL, CTRL_START); + wait_done(); +} + +/* Write a 256-bit value (32 bytes) into one of the seed/msg windows. + * Byte i of the logical value lands in word[i>>2] byte (i&3) — i.e. the + * core sees byte 0 in D[0][7:0], matching mlkem_top's d_i[7:0] convention. */ +static void write_block32(uint32_t base, const uint8_t *bytes) +{ + int w; + for (w = 0; w < 8; w++) { + uint32_t v = (uint32_t)bytes[w*4 + 0] + | ((uint32_t)bytes[w*4 + 1] << 8) + | ((uint32_t)bytes[w*4 + 2] << 16) + | ((uint32_t)bytes[w*4 + 3] << 24); + REG_WR(base + w*4, v); + } +} + +/* Read a 256-bit value (32 bytes) back from a window (used for SS). */ +static void read_block32(uint32_t base, uint8_t *bytes) +{ + int w; + for (w = 0; w < 8; w++) { + uint32_t v = REG_RD(base + w*4); + bytes[w*4 + 0] = (uint8_t)(v & 0xFF); + bytes[w*4 + 1] = (uint8_t)((v >> 8) & 0xFF); + bytes[w*4 + 2] = (uint8_t)((v >> 16) & 0xFF); + bytes[w*4 + 3] = (uint8_t)((v >> 24) & 0xFF); + } +} + +/* Stream `n` bytes into a preload port. Each PRE_DAT write auto-increments + * the preload address; the bus wait-states each write until the cross-domain + * byte handshake completes, so a plain loop is correct. */ +static void preload(uint32_t sel, const uint8_t *buf, int n) +{ + int i; + REG_WR(MLKEM_PRE_SEL, sel); + REG_WR(MLKEM_PRE_ADR, 0); + for (i = 0; i < n; i++) + REG_WR(MLKEM_PRE_DAT, buf[i]); +} + +/* Read `n` bytes out of a readback tap. RDB_ADR has no auto-increment, so set + * the index for each byte (write addr, then read data). */ +static void readback(uint32_t sel, uint8_t *buf, int n) +{ + int i; + REG_WR(MLKEM_RDB_SEL, sel); + for (i = 0; i < n; i++) { + REG_WR(MLKEM_RDB_ADR, i); + buf[i] = (uint8_t)(REG_RD(MLKEM_RDB_DAT) & 0xFF); + } +} + +/* PLACEHOLDER2 */ + +/* Arbitrary but fixed test seeds / message (32 bytes each). */ +static const uint8_t SEED_D[32] = { + 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F +}; +static const uint8_t SEED_Z[32] = { + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0x2A,0x2B,0x2C,0x2D,0x2E,0x2F, + 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x3A,0x3B,0x3C,0x3D,0x3E,0x3F +}; +static const uint8_t MSG_M[32] = { + 0xA0,0xA1,0xA2,0xA3,0xA4,0xA5,0xA6,0xA7,0xA8,0xA9,0xAA,0xAB,0xAC,0xAD,0xAE,0xAF, + 0xB0,0xB1,0xB2,0xB3,0xB4,0xB5,0xB6,0xB7,0xB8,0xB9,0xBA,0xBB,0xBC,0xBD,0xBE,0xBF +}; + +static uint8_t ek[EK_BYTES]; +static uint8_t dk[DK_BYTES]; +static uint8_t ct[CT_BYTES]; +static uint8_t ss_enc[32]; +static uint8_t ss_dec[32]; + +static void print_hex(const char *label, const uint8_t *b, int n) +{ + int i; + printf("%s", label); + for (i = 0; i < n; i++) + printf("%02x", b[i]); + printf("\n"); +} + +int main(void) +{ + int i, mismatch; + + printf("MLKEM_START\n"); + + /* ── 1. KeyGen: d,z -> ek, dk ─────────────────────────────────── */ + write_block32(MLKEM_D, SEED_D); + write_block32(MLKEM_Z, SEED_Z); + run_op(OP_KEYGEN); + readback(RDB_EK, ek, EK_BYTES); + readback(RDB_DK, dk, DK_BYTES); + print_hex("EK32=", ek, 32); /* first 32 bytes as a fingerprint */ + + /* ── 2. Encaps: ek + msg -> ss, ct ────────────────────────────── */ + preload(PRE_EK, ek, EK_BYTES); + write_block32(MLKEM_MSG, MSG_M); + run_op(OP_ENCAPS); + read_block32(MLKEM_SS, ss_enc); + readback(RDB_CT, ct, CT_BYTES); + print_hex("SS_ENC=", ss_enc, 32); + + /* ── 3. Decaps: dk + ct -> ss' ────────────────────────────────── */ + preload(PRE_DK, dk, DK_BYTES); + preload(PRE_C, ct, CT_BYTES); + run_op(OP_DECAPS); + read_block32(MLKEM_SS, ss_dec); + print_hex("SS_DEC=", ss_dec, 32); + + /* ── 4. Verify shared secrets match ───────────────────────────── */ + mismatch = 0; + for (i = 0; i < 32; i++) + if (ss_enc[i] != ss_dec[i]) mismatch++; + + if (mismatch == 0) + printf("MLKEM_RESULT=PASS\n"); + else + printf("MLKEM_RESULT=FAIL (%d byte mismatch)\n", mismatch); + + printf("MLKEM_DONE\n"); + return 0; +}