fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ML-KEM accelerator with AXI slave wrapper

Browse Source 
Wrap mlkem_top in an AXI4 burst slave (mlkem_axi_slave) mapped to the
freed FFT slot 6 at 0x1f400000 (virtual 0xbf400000). The core runs on
sys_clk/2 (25 MHz); the wrapper handles the clock-domain crossing:
start toggle->pulse, wait-stated byte-preload handshake, busy/done
sync, and direct readback taps (ek/dk/ct/ss). CPU drives it via a
register map (CTRL/STATUS/CONFIG, d/z/msg, preload + readback windows).

Wire into soc_top on the existing axiOut_6 nets (no crossbar change),
add mlkem_intr on intrpt[3]. Add sdk mlkem_test example running a
KeyGen->Encaps->Decaps self-consistency round trip.
This commit is contained in:
FallenSigh
2026-06-30 01:39:35 +08:00
parent 95fe94063c
commit c5072d7170
29 changed files with 6332 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
rtl/ip/mlkem/common/defines.vh Normal file
View File

@@ -0,0 +1,6 @@
`ifndef DEFINES_VH
`define DEFINES_VH
`define CLK_PERIOD 20.0 // 50MHz
`define Q 3329 // ML-KEM prime modulus
`define N 256 // polynomial degree
`endif

53
rtl/ip/mlkem/common/pipeline_reg.v Normal file
View File

@@ -0,0 +1,53 @@
// pipeline_reg.v - Single pipeline stage with valid/ready handshake
//
// valid_i && ready_o -> register data -> valid_o next cycle.
// ready_o = !valid_o || ready_i.
//
// Parameters:
// DW = data width (default 12)
//
// Interface:
// clk, rst_n - clock, active-low reset
// data_i, valid_i, ready_o - input side
// data_o, valid_o, ready_i - output side
module pipeline_reg #(parameter DW = 12) (
input clk,
input rst_n,
input [DW-1:0] data_i,
input valid_i,
output ready_o,
output [DW-1:0] data_o,
output valid_o,
input ready_i
);
reg valid_r;
reg [DW-1:0] data_r;
// ready_o: accept new data when output slot is free
assign ready_o = !valid_r || ready_i;
// Drive outputs from internal register
assign valid_o = valid_r;
assign data_o = data_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
valid_r <= 1'b0;
data_r <= {DW{1'b0}};
end else begin
if (valid_r && ready_i) begin
// Output consumed, clear valid
valid_r <= 1'b0;
end
if (valid_i && ready_o) begin
// Capture new data
data_r <= data_i;
valid_r <= 1'b1;
end
end
end
endmodule

73
rtl/ip/mlkem/common/skid_buffer.v Normal file
View File

@@ -0,0 +1,73 @@
// skid_buffer.v - 2-entry skid buffer with valid/ready handshake
//
// When valid_i && ready_o, capture data_i. Output via valid_o && ready_i.
// Stores one extra entry in skid register. Pure combinational ready_o path.
//
// Parameters:
// DW = data width (default 12)
//
// Interface:
// clk, rst_n - clock, active-low reset
// data_i, valid_i, ready_o - input side
// data_o, valid_o, ready_i - output side
module skid_buffer #(parameter DW = 12) (
input clk,
input rst_n,
input [DW-1:0] data_i,
input valid_i,
output ready_o,
output [DW-1:0] data_o,
output valid_o,
input ready_i
);
// Internal state
reg skid_valid; // skid register has valid data
reg [DW-1:0] skid_data; // skid register data
// Output register (pipeline stage)
reg out_valid;
reg [DW-1:0] out_data;
// ready_o is combinational: ready when output slot is empty
// (either out_valid is 0, or ready_i is high and we can accept new data)
assign ready_o = !out_valid || ready_i;
// valid_o drives from output register
assign valid_o = out_valid;
assign data_o = out_data;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
skid_valid <= 1'b0;
skid_data <= {DW{1'b0}};
out_valid <= 1'b0;
out_data <= {DW{1'b0}};
end else begin
if (out_valid && ready_i) begin
// Output consumed: load from skid if available
if (skid_valid) begin
out_data <= skid_data;
out_valid <= 1'b1;
skid_valid <= 1'b0;
end else begin
out_valid <= 1'b0;
end
end
if (valid_i && ready_o) begin
if (!out_valid || (out_valid && ready_i && !skid_valid)) begin
// Output slot free: go directly to output
out_data <= data_i;
out_valid <= 1'b1;
end else begin
// Output slot occupied: store in skid
skid_data <= data_i;
skid_valid <= 1'b1;
end
end
end
end
endmodule

96
rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v Normal file
View File

@@ -0,0 +1,96 @@
// comp_decomp_sync.v - ML-KEM coefficient compression/decompression
//
// Streaming: one coefficient per cycle through pipeline_reg.
// mode=0: compress - round((2^d * x) / Q) mod 2^d
// mode=1: decompress - round((Q * x) / 2^d) mod Q
// Uses round-half-up (round(2.5)=3, round(3.5)=4).
// Integer arithmetic:
// compress: (x * 2^d + Q/2) / Q, lower d bits as result
// decompress: (x * Q + 2^(d-1)) / 2^d, result mod Q
//
// Division by Q=3329 replaced with Barrett multiplication + correction:
// floor(x / Q) ≈ (x * 5039) >> 24 (5039 = floor(2^24 / 3329))
// Correction: if x - q_approx * Q >= Q, add 1 to quotient.
// Q*5039 = 16,774,831 < 2^24, so Barrett may under-estimate by 1.
// Q = 3329 = 2048+512+256+1, so q*Q uses 3 shifts + add (no 2nd multiplier).
`include "defines.vh"
(* use_dsp = "no" *)
module comp_decomp_sync (
input clk,
input rst_n,
input [11:0] coeff_in,
input [4:0] d,
input mode, // 0=compress, 1=decompress
input valid_i,
output ready_o,
output [11:0] coeff_out,
output valid_o,
input ready_i
);
wire [11:0] two_pow_d = 12'd1 << d;
// product: 12b*12b = max 2048*3328 = 6,815,744 → 23b, pad to 24b
wire [23:0] product = mode ? {12'b0, coeff_in} * {12'b0, 12'd3329}
: {12'b0, coeff_in} * {12'b0, two_pow_d};
wire [11:0] round_off = mode ? (two_pow_d >> 1) : 12'd1664;
wire [23:0] dividend = product + {12'b0, round_off};
// ===========================================================
// Barrett: floor(x / Q) ≈ (x * 5039) >> 24
// M = floor(2^24 / Q) = 5039
// M*Q = 16,774,831 < 2^24 → floor(x/Q) may be 1 too low
// Correction: remainder = x - q_approx*Q; if rem >= Q, q→q+1
// Q*5039 fails at: k*2385 >= 2^24 → k >= 7033; max k ≈ 2047 → safe
// ===========================================================
wire [36:0] barrett_prod = dividend * 13'd5039; // 24×13=37b
wire [11:0] q_approx = barrett_prod[35:24]; // quotient estimate
// q*Q = q*2048 + q*512 + q*256 + q (3329 = 2048+512+256+1)
wire [23:0] q_times_Q = ({q_approx, 11'd0})
+ ({q_approx, 9'd0})
+ ({q_approx, 8'd0})
+ q_approx;
wire [24:0] delta = {1'b0, dividend} - {1'b0, q_times_Q}; // 25b
wire [11:0] comp_q = q_approx + (delta >= {13'b0, 12'd3329});
// lower d bits = compressed result
wire [11:0] comp_res = comp_q & ({12'b0, two_pow_d} - 1'b1);
// ===========================================================
// Decompress: dividend >> d (free), then % Q via Barrett
// ===========================================================
wire [23:0] dec_sh = dividend >> d;
wire [36:0] dm_prod = dec_sh * 13'd5039;
wire [11:0] dm_qe = dm_prod[35:24];
wire [23:0] dm_qQ = ({dm_qe, 11'd0})
+ ({dm_qe, 9'd0})
+ ({dm_qe, 8'd0})
+ dm_qe;
wire [24:0] dm_del = {1'b0, dec_sh} - {1'b0, dm_qQ};
// divide by Q, take remainder: r = x - q*Q
wire [12:0] dm_q = dm_qe + (dm_del >= {13'b0, 12'd3329}); // corr quotient
// actual remainder = dec_sh - dm_q*Q
wire [23:0] dm_corrQ = ({dm_q, 11'd0}) + ({dm_q, 9'd0}) + ({dm_q, 8'd0}) + dm_q;
wire [24:0] dm_rem = {1'b0, dec_sh} - {1'b0, dm_corrQ};
// dm_rem < Q*2 (Barrett gives exact quotient after correction), one cond-sub
wire [11:0] dec_mod = (dm_rem >= {13'b0, 12'd3329})
? (dm_rem[11:0] - 12'd3329)
: dm_rem[11:0];
wire [11:0] mod_result = mode ? dec_mod : comp_res;
pipeline_reg #(.DW(12)) u_pipe (
.clk (clk),
.rst_n (rst_n),
.data_i (mod_result),
.valid_i(valid_i),
.ready_o(ready_o),
.data_o (coeff_out),
.valid_o(valid_o),
.ready_i(ready_i)
);
endmodule

54
rtl/ip/mlkem/ntt/barrett_mul.v Normal file
View File

@@ -0,0 +1,54 @@
// barrett_mul.v - Barrett modular multiplication (a * b mod Q)
//
// Computes product = a * b mod Q using Barrett reduction.
// Q = 3329, k = floor(2^24 / Q) = 5039
//
// Pure combinational, single-cycle latency.
// All multiplication widths explicitly controlled to avoid Verilog truncation.
(* use_dsp = "no" *)
module barrett_mul (
input [11:0] a,
input [11:0] b,
output [11:0] product
);
localparam Q = 3329;
localparam K = 5039; // floor(2^24 / 3329)
localparam R = 24;
// Full product: a * b (both < 3329, product < 11,082,241 < 2^24)
// Force 24-bit evaluation by extending operand
wire [23:0] p;
assign p = {12'd0, a} * b;
// Extend p to 37 bits for multiplication with K
wire [36:0] p_ext;
assign p_ext = {13'd0, p};
// Compute t_shifted = (p * K) >> 24
// Use explicit wire for the product to control width
/* verilator lint_off UNUSEDSIGNAL */
wire [36:0] t_product;
/* verilator lint_on UNUSEDSIGNAL */
assign t_product = p_ext * K;
wire [12:0] t_shifted;
assign t_shifted = t_product[36:R];
// q_approx = t_shifted * Q
wire [24:0] q_approx;
assign q_approx = t_shifted * Q;
// r = p - q_approx
wire [24:0] r0;
assign r0 = {1'b0, p} - q_approx;
// Conditional subtract Q (at most twice)
wire [24:0] r1;
assign r1 = (r0 >= Q) ? (r0 - Q) : r0;
wire [11:0] r2;
assign r2 = (r1[11:0] >= Q) ? (r1[11:0] - Q) : r1[11:0];
assign product = (r1 >= Q) ? r2 : r1[11:0];
endmodule

67
rtl/ip/mlkem/ntt/butterfly_unit.v Normal file
View File

@@ -0,0 +1,67 @@
// butterfly_unit.v - Cooley-Tukey / Gentleman-Sande butterfly
//
// Computes one butterfly operation for NTT or inverse NTT.
//
// Parameters:
// Q = 3329 (prime modulus)
//
// Forward NTT (mode=0):
// t = zeta * b mod Q (via barrett_mul)
// a_out = (a + t) mod Q
// b_out = (a - t) mod Q
//
// Inverse NTT (mode=1):
// a_out = (a + b) mod Q
// diff = (b - a) mod Q (handled as: if b >= a: b-a; else: b-a+Q)
// b_out = zeta * diff mod Q (via barrett_mul)
//
// Pure combinational.
module butterfly_unit (
input [11:0] a,
input [11:0] b,
input [11:0] zeta,
input mode, // 0 = forward NTT, 1 = inverse NTT
output [11:0] a_out,
output [11:0] b_out
);
localparam Q = 3329;
// Barrett modular multiplication: zeta * mul_data mod Q
wire [11:0] mul_data; // what to multiply with zeta
wire [11:0] mul_result; // result of barrett_mul(zeta, mul_data)
// Forward: mul_data = b, t = zeta * b mod Q
// Inverse: mul_data = (b - a) mod Q positive
assign mul_data = (mode == 1'b0) ? b : ((b >= a) ? (b - a) : (b - a + Q));
barrett_mul u_barrett (
.a (zeta),
.b (mul_data),
.product (mul_result)
);
// ---- a_out computation ----
// Forward: a_out = (a + t) mod Q
// Inverse: a_out = (a + b) mod Q
wire [12:0] a_sum;
wire [11:0] add_val;
assign add_val = (mode == 1'b0) ? mul_result : b;
assign a_sum = {1'b0, a} + {1'b0, add_val};
// a_sum - Q produces 13-bit result; we only need lower 12 bits since
// a_sum >= Q guarantees the result < Q < 2^12
wire [11:0] a_sub_12;
assign a_sub_12 = a_sum[11:0] - Q[11:0];
wire [11:0] a_result = (a_sum >= Q) ? a_sub_12 : a_sum[11:0];
assign a_out = a_result;
// ---- b_out computation ----
// Forward: b_out = (a - t) mod Q → if a >= t: a-t; else: a-t+Q
// Inverse: b_out = t (mul_result, which is zeta * (b-a) mod Q)
wire [11:0] sub_val;
assign sub_val = mul_result;
wire [11:0] sub_result;
assign sub_result = (a >= sub_val) ? (a - sub_val) : (a - sub_val + Q);
assign b_out = (mode == 1'b0) ? sub_result : mul_result;
endmodule

132
rtl/ip/mlkem/ntt/ntt_core.v Normal file
View File

@@ -0,0 +1,132 @@
// ntt_core.v - NTT core with individual coefficient registers
//
// Uses 256 individual 12-bit registers and generate-based muxing
// to avoid any part-select simulation issues.
// 3-cycle butterfly: SetAddr -> Read -> Compute+Write
module ntt_core (
input clk, rst_n,
input [11:0] coeff_in,
input valid_i,
output ready_o,
input mode,
output [11:0] coeff_out,
output valid_o,
input ready_i,
output done_o
);
localparam N = 256, LAYERS = 7, DW = 12;
// Individual coefficient registers
reg [DW-1:0] cr [0:N-1];
integer ci;
// State machine
localparam S_IDLE=3'd0, S_LOAD=3'd1, S_CMP_A=3'd2, S_CMP_B=3'd3,
S_CMP_C=3'd4, S_OUTPUT=3'd5, S_DONE=3'd6;
reg [2:0] state, next_state;
reg [7:0] load_cnt, out_cnt;
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
// Pipeline registers
reg [DW-1:0] r_a, r_b;
reg [7:0] r_wa, r_wb;
// Zeta
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
// Butterfly
wire [DW-1:0] bf_a_out, bf_b_out;
butterfly_unit u_bf (
.a(r_a), .b(r_b), .zeta(zeta), .mode(mode),
.a_out(bf_a_out), .b_out(bf_b_out));
// Output scaling
wire [DW-1:0] coeff_scaled;
barrett_mul u_scl (.a(cr[out_cnt]), .b(12'd3303), .product(coeff_scaled));
assign coeff_out = mode ? coeff_scaled : cr[out_cnt];
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_OUTPUT);
assign done_o = (state == S_DONE);
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i) next_state = S_CMP_A;
S_CMP_A: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_B;
S_CMP_B: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_C;
S_CMP_C: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_A;
S_OUTPUT:if (out_cnt >= 255 && ready_i) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state<=S_IDLE; load_cnt<=0; out_cnt<=0; j<=0; start<=0; layer_len<=0;
zeta_idx<=0; layer<=0; bf_done<=0; r_a<=0; r_b<=0; r_wa<=0; r_wb<=0;
for (ci=0; ci<N; ci=ci+1) cr[ci] <= 0;
end else begin
state <= next_state;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt<=1; out_cnt<=0; j<=0; start<=0; layer<=0; bf_done<=0;
if (!mode) begin layer_len<=128; zeta_idx<=1; end
else begin layer_len<=2; zeta_idx<=127; end
end
if (state == S_LOAD && valid_i) begin
cr[load_cnt] <= coeff_in;
load_cnt <= load_cnt + 8'd1;
end
// S_CMP_A: set read addresses (j, j+len)
if (state == S_CMP_A) begin
r_wa <= j;
r_wb <= j + layer_len;
end
// S_CMP_B: capture read data
if (state == S_CMP_B) begin
r_a <= cr[j];
r_b <= cr[j + layer_len];
end
// S_CMP_C: write butterfly results, advance counters
if (state == S_CMP_C) begin
cr[r_wa] <= bf_a_out;
cr[r_wb] <= bf_b_out;
j <= j + 8'd1;
if (j + 8'd1 >= start + layer_len) begin
if (!mode) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 256) begin
layer <= layer + 3'd1;
layer_len <= mode ? (layer_len<<1) : (layer_len>>1);
start <= 0; j <= 0;
if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1;
end else begin
start <= start + layer_len + layer_len;
j <= start + layer_len + layer_len;
end
end
end
if (state == S_OUTPUT && ready_i)
out_cnt <= (out_cnt>=255) ? 0 : (out_cnt+8'd1);
end
end
endmodule

19
rtl/ip/mlkem/ntt/ntt_sync.v Normal file
View File

@@ -0,0 +1,19 @@
module ntt_sync (
input clk, rst_n,
input [11:0] coeff_in,
input valid_i,
output ready_o,
output [11:0] coeff_out,
output valid_o,
input ready_i,
input mode,
output done_o
);
ntt_core u_ntt_core (
.clk(clk), .rst_n(rst_n),
.coeff_in(coeff_in), .valid_i(valid_i), .ready_o(ready_o),
.mode(mode),
.coeff_out(coeff_out), .valid_o(valid_o), .ready_i(ready_i),
.done_o(done_o)
);
endmodule

141
rtl/ip/mlkem/ntt/zeta_rom.v Normal file
View File

@@ -0,0 +1,141 @@
// zeta_rom.v - Precomputed Zeta ROM for ML-KEM NTT
//
// Stores 128 bit-reversed roots of unity for Z_q (q=3329).
// Address 0..127, each entry is a 12-bit value.
// Pure combinational, single continuous assignment.
module zeta_rom (
input [6:0] addr, // 0..127
output [11:0] zeta
);
assign zeta =
(addr == 7'd0) ? 12'd1 :
(addr == 7'd1) ? 12'd1729 :
(addr == 7'd2) ? 12'd2580 :
(addr == 7'd3) ? 12'd3289 :
(addr == 7'd4) ? 12'd2642 :
(addr == 7'd5) ? 12'd630 :
(addr == 7'd6) ? 12'd1897 :
(addr == 7'd7) ? 12'd848 :
(addr == 7'd8) ? 12'd1062 :
(addr == 7'd9) ? 12'd1919 :
(addr == 7'd10) ? 12'd193 :
(addr == 7'd11) ? 12'd797 :
(addr == 7'd12) ? 12'd2786 :
(addr == 7'd13) ? 12'd3260 :
(addr == 7'd14) ? 12'd569 :
(addr == 7'd15) ? 12'd1746 :
(addr == 7'd16) ? 12'd296 :
(addr == 7'd17) ? 12'd2447 :
(addr == 7'd18) ? 12'd1339 :
(addr == 7'd19) ? 12'd1476 :
(addr == 7'd20) ? 12'd3046 :
(addr == 7'd21) ? 12'd56 :
(addr == 7'd22) ? 12'd2240 :
(addr == 7'd23) ? 12'd1333 :
(addr == 7'd24) ? 12'd1426 :
(addr == 7'd25) ? 12'd2094 :
(addr == 7'd26) ? 12'd535 :
(addr == 7'd27) ? 12'd2882 :
(addr == 7'd28) ? 12'd2393 :
(addr == 7'd29) ? 12'd2879 :
(addr == 7'd30) ? 12'd1974 :
(addr == 7'd31) ? 12'd821 :
(addr == 7'd32) ? 12'd289 :
(addr == 7'd33) ? 12'd331 :
(addr == 7'd34) ? 12'd3253 :
(addr == 7'd35) ? 12'd1756 :
(addr == 7'd36) ? 12'd1197 :
(addr == 7'd37) ? 12'd2304 :
(addr == 7'd38) ? 12'd2277 :
(addr == 7'd39) ? 12'd2055 :
(addr == 7'd40) ? 12'd650 :
(addr == 7'd41) ? 12'd1977 :
(addr == 7'd42) ? 12'd2513 :
(addr == 7'd43) ? 12'd632 :
(addr == 7'd44) ? 12'd2865 :
(addr == 7'd45) ? 12'd33 :
(addr == 7'd46) ? 12'd1320 :
(addr == 7'd47) ? 12'd1915 :
(addr == 7'd48) ? 12'd2319 :
(addr == 7'd49) ? 12'd1435 :
(addr == 7'd50) ? 12'd807 :
(addr == 7'd51) ? 12'd452 :
(addr == 7'd52) ? 12'd1438 :
(addr == 7'd53) ? 12'd2868 :
(addr == 7'd54) ? 12'd1534 :
(addr == 7'd55) ? 12'd2402 :
(addr == 7'd56) ? 12'd2647 :
(addr == 7'd57) ? 12'd2617 :
(addr == 7'd58) ? 12'd1481 :
(addr == 7'd59) ? 12'd648 :
(addr == 7'd60) ? 12'd2474 :
(addr == 7'd61) ? 12'd3110 :
(addr == 7'd62) ? 12'd1227 :
(addr == 7'd63) ? 12'd910 :
(addr == 7'd64) ? 12'd17 :
(addr == 7'd65) ? 12'd2761 :
(addr == 7'd66) ? 12'd583 :
(addr == 7'd67) ? 12'd2649 :
(addr == 7'd68) ? 12'd1637 :
(addr == 7'd69) ? 12'd723 :
(addr == 7'd70) ? 12'd2288 :
(addr == 7'd71) ? 12'd1100 :
(addr == 7'd72) ? 12'd1409 :
(addr == 7'd73) ? 12'd2662 :
(addr == 7'd74) ? 12'd3281 :
(addr == 7'd75) ? 12'd233 :
(addr == 7'd76) ? 12'd756 :
(addr == 7'd77) ? 12'd2156 :
(addr == 7'd78) ? 12'd3015 :
(addr == 7'd79) ? 12'd3050 :
(addr == 7'd80) ? 12'd1703 :
(addr == 7'd81) ? 12'd1651 :
(addr == 7'd82) ? 12'd2789 :
(addr == 7'd83) ? 12'd1789 :
(addr == 7'd84) ? 12'd1847 :
(addr == 7'd85) ? 12'd952 :
(addr == 7'd86) ? 12'd1461 :
(addr == 7'd87) ? 12'd2687 :
(addr == 7'd88) ? 12'd939 :
(addr == 7'd89) ? 12'd2308 :
(addr == 7'd90) ? 12'd2437 :
(addr == 7'd91) ? 12'd2388 :
(addr == 7'd92) ? 12'd733 :
(addr == 7'd93) ? 12'd2337 :
(addr == 7'd94) ? 12'd268 :
(addr == 7'd95) ? 12'd641 :
(addr == 7'd96) ? 12'd1584 :
(addr == 7'd97) ? 12'd2298 :
(addr == 7'd98) ? 12'd2037 :
(addr == 7'd99) ? 12'd3220 :
(addr == 7'd100) ? 12'd375 :
(addr == 7'd101) ? 12'd2549 :
(addr == 7'd102) ? 12'd2090 :
(addr == 7'd103) ? 12'd1645 :
(addr == 7'd104) ? 12'd1063 :
(addr == 7'd105) ? 12'd319 :
(addr == 7'd106) ? 12'd2773 :
(addr == 7'd107) ? 12'd757 :
(addr == 7'd108) ? 12'd2099 :
(addr == 7'd109) ? 12'd561 :
(addr == 7'd110) ? 12'd2466 :
(addr == 7'd111) ? 12'd2594 :
(addr == 7'd112) ? 12'd2804 :
(addr == 7'd113) ? 12'd1092 :
(addr == 7'd114) ? 12'd403 :
(addr == 7'd115) ? 12'd1026 :
(addr == 7'd116) ? 12'd1143 :
(addr == 7'd117) ? 12'd2150 :
(addr == 7'd118) ? 12'd2775 :
(addr == 7'd119) ? 12'd886 :
(addr == 7'd120) ? 12'd1722 :
(addr == 7'd121) ? 12'd1212 :
(addr == 7'd122) ? 12'd1874 :
(addr == 7'd123) ? 12'd1029 :
(addr == 7'd124) ? 12'd2110 :
(addr == 7'd125) ? 12'd2935 :
(addr == 7'd126) ? 12'd885 :
12'd2154;
endmodule

63
rtl/ip/mlkem/poly_arith/poly_arith_sync.v Normal file
View File

@@ -0,0 +1,63 @@
// poly_arith_sync.v - Polynomial modular add/sub for ML-KEM
//
// Performs element-wise modular addition or subtraction on two streaming
// 256-coefficient polynomials over Z_q (q=3329).
//
// mode=0: coeff_out = (coeff_a_in + coeff_b_in) mod Q
// mode=1: coeff_out = (coeff_a_in - coeff_b_in) mod Q
//
// Each cycle processes one coefficient pair. Pure streaming — no internal
// storage; the caller sequences all 256 coefficients in order.
// `include "sync_rtl/common/defines.vh
`define Q 3329 // ML-KEM prime modulus
`define N 256 // polynomial degree"
module poly_arith_sync (
input clk,
input rst_n,
input [11:0] coeff_a_in,
input [11:0] coeff_b_in,
input mode, // 0=add, 1=sub
input valid_i,
output ready_o,
output [11:0] coeff_out,
output valid_o,
input ready_i
);
//--------------------------------------------------------------
// Combinational modular arithmetic
//--------------------------------------------------------------
wire [12:0] add_raw; // a + b (13-bit to catch overflow)
wire [11:0] add_sub_q; // (a + b) - Q
wire [11:0] add_result; // (a + b) mod Q
assign add_raw = {1'b0, coeff_a_in} + {1'b0, coeff_b_in};
assign add_sub_q = add_raw[11:0] - `Q;
assign add_result = (add_raw < `Q) ? add_raw[11:0] : add_sub_q;
wire [11:0] sub_result; // (a - b) mod Q
// When a >= b: diff = a - b (already in [0, Q-2])
// When a < b: diff = a + Q - b (borrow correction)
assign sub_result = (coeff_a_in < coeff_b_in)
? (coeff_a_in + `Q - coeff_b_in)
: (coeff_a_in - coeff_b_in);
wire [11:0] mod_result = mode ? sub_result : add_result;
//--------------------------------------------------------------
// Pipeline the result through pipeline_reg for valid/ready
//--------------------------------------------------------------
pipeline_reg #(.DW(12)) u_pipe (
.clk (clk),
.rst_n (rst_n),
.data_i (mod_result),
.valid_i(valid_i),
.ready_o(ready_o),
.data_o (coeff_out),
.valid_o(valid_o),
.ready_i(ready_i)
);
endmodule

45
rtl/ip/mlkem/poly_mul/basecase_mul.v Normal file
View File

@@ -0,0 +1,45 @@
// basecase_mul.v - NTT-domain degree-1 polynomial multiplication
//
// Computes pointwise product of two degree-1 NTT-domain polynomials:
// c0 = (a0*b0 + a1*b1*zeta) mod Q
// c1 = (a0*b1 + a1*b0) mod Q
//
// Uses three barrett_mul instances for modular multiplications
// and pure combinational modular addition for the final sums.
//
// All inputs/outputs are 12-bit values in [0, Q-1] where Q = 3329.
module basecase_mul (
input [11:0] a0, a1, // first polynomial: degree-0 and degree-1 coeffs
input [11:0] b0, b1, // second polynomial: degree-0 and degree-1 coeffs
input [11:0] zeta, // precomputed twiddle factor (zeta^2 * 17 mod Q)
output [11:0] c0, c1 // result polynomial coefficients
);
localparam Q = 3329;
// Barrett modular multiplication outputs
wire [11:0] t1; // a0 * b0 mod Q
wire [11:0] t2; // a1 * b1 mod Q
wire [11:0] t3; // a1 * b0 mod Q
wire [11:0] t4; // a0 * b1 mod Q
wire [11:0] t2_zeta; // (a1 * b1) * zeta mod Q
// Four Barrett multiplications for the scalar products
barrett_mul u_mul1 (.a(a0), .b(b0), .product(t1));
barrett_mul u_mul2 (.a(a1), .b(b1), .product(t2));
barrett_mul u_mul3 (.a(a1), .b(b0), .product(t3));
barrett_mul u_mul4 (.a(a0), .b(b1), .product(t4));
// Multiply t2 by zeta for the zeta term in c0
barrett_mul u_mul_zeta (.a(t2), .b(zeta), .product(t2_zeta));
// Combinational modular addition: each sum is in [0, 2Q-1)
// Subtract Q once if >= Q
wire [12:0] sum_c0 = {1'b0, t1} + {1'b0, t2_zeta};
wire [12:0] sum_c1 = {1'b0, t3} + {1'b0, t4};
assign c0 = (sum_c0 >= Q) ? sum_c0[11:0] - Q[11:0] : sum_c0[11:0];
assign c1 = (sum_c1 >= Q) ? sum_c1[11:0] - Q[11:0] : sum_c1[11:0];
endmodule

166
rtl/ip/mlkem/poly_mul/poly_mul_sync.v Normal file
View File

@@ -0,0 +1,166 @@
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier
//
// Computes pointwise (Karatsuba-like base-case) multiplication of two
// 256-coefficient NTT-domain polynomials.
//
// Operation flow:
// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle)
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE
//
// The LOAD phase accepts both A and B coefficients simultaneously
// (one pair per cycle) on coeff_a_in/coeff_b_in.
//
// The COMPUTE phase outputs the 256 result coefficients one per cycle
// via valid/ready handshake on coeff_out/valid_o/ready_i.
//
// Memory: 256×12-bit register arrays for A and B coefficients.
//
// Interface:
// clk, rst_n - Clock, active-low reset
// coeff_a_in[11:0]- Polynomial A coefficient input
// coeff_b_in[11:0]- Polynomial B coefficient input
// valid_i - Input valid
// ready_o - Ready to accept input (high in IDLE/LOAD)
// coeff_out[11:0] - Result coefficient output
// valid_o - Output valid (high in COMP_C0/COMP_C1)
// ready_i - Output consumer ready
module poly_mul_sync (
input clk, rst_n,
input [11:0] coeff_a_in,
input [11:0] coeff_b_in,
input valid_i,
output ready_o,
output [11:0] coeff_out,
output valid_o,
input ready_i
);
// State definitions
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_COMP_CALC = 3'd2;
localparam S_COMP_C0 = 3'd3;
localparam S_COMP_C1 = 3'd4;
localparam S_DONE = 3'd5;
reg [2:0] state, next_state;
// Coefficient storage (register arrays)
reg [11:0] mem_A [0:255];
reg [11:0] mem_B [0:255];
// Counters
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [6:0] comp_k; // 0..127, current base-case index
// Registered basecase_mul results
reg [11:0] c0_reg, c1_reg;
// Combinational read signals for COMP_CALC
wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits)
wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1
wire [11:0] mem_a0 = mem_A[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even];
wire [11:0] mem_b1 = mem_B[addr_odd];
// Zeta ROM
wire [11:0] zeta;
poly_mul_zeta_rom u_zeta (
.addr (comp_k),
.zeta (zeta)
);
// Basecase multiply
wire [11:0] bc_c0, bc_c1;
basecase_mul u_bc (
.a0 (mem_a0),
.a1 (mem_a1),
.b0 (mem_b0),
.b1 (mem_b1),
.zeta(zeta),
.c0 (bc_c0),
.c1 (bc_c1)
);
// Output interface
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1);
assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg;
// State transition logic (combinational)
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o)
next_state = S_COMP_CALC;
S_COMP_CALC: next_state = S_COMP_C0;
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
S_COMP_C1: if (valid_o && ready_i) begin
if (comp_k >= 127) next_state = S_DONE;
else next_state = S_COMP_CALC;
end
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
// Sequential logic
integer i;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
comp_k <= 7'd0;
c0_reg <= 12'd0;
c1_reg <= 12'd0;
for (i = 0; i < 256; i = i + 1) begin
mem_A[i] <= 12'd0;
mem_B[i] <= 12'd0;
end
end else begin
state <= next_state;
// ---- LOAD phase ----
// First coefficient captured on IDLE → LOAD transition
if (state == S_IDLE && valid_i && ready_o) begin
mem_A[0] <= coeff_a_in;
mem_B[0] <= coeff_b_in;
load_cnt <= 8'd1;
end
// Subsequent coefficients in LOAD state
if (state == S_LOAD && valid_i && ready_o) begin
mem_A[load_cnt] <= coeff_a_in;
mem_B[load_cnt] <= coeff_b_in;
load_cnt <= load_cnt + 8'd1;
end
// ---- COMPUTE phase ----
// COMP_CALC: capture basecase_mul results
if (state == S_COMP_CALC) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
end
// COMP_C0 → COMP_C1: c0 was consumed, increment comp_k
if (state == S_COMP_C0 && valid_o && ready_i) begin
// comp_k stays same, c1 still to output
end
// COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair
if (state == S_COMP_C1 && valid_o && ready_i) begin
comp_k <= comp_k + 7'd1;
end
// ---- DONE → IDLE: reset counters ----
if (state == S_DONE) begin
load_cnt <= 8'd0;
comp_k <= 7'd0;
end
end
end
endmodule

58
rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v Normal file
View File

@@ -0,0 +1,58 @@
// poly_mul_zeta_rom.v - Read-only memory for PolyMul zeta values
//
// Stores precomputed zeta values for base-case NTT-domain polynomial
// multiplication. Each entry zeta[k] = (zeta_bitRev[k]^2 * 17) mod Q.
//
// 128 entries × 12 bits. Pure combinational read.
//
// zeta_bitRev values are the same bit-reversed twiddle factors used
// in NTT. The PolyMul formula squares them and multiplies by 17 mod Q.
module poly_mul_zeta_rom (
input [6:0] addr, // 0..127, selects which zeta to read
output [11:0] zeta // zeta = (zeta_bitRev[addr]^2 * 17) mod Q
);
reg [11:0] rom [0:127];
// Initialize ROM with precomputed zeta values
// Generated from Python: zeta = (zeta_bitRev[i]**2 * 17) % 3329
initial begin
rom[ 0] = 12'd17; rom[ 1] = 12'd3312; rom[ 2] = 12'd2761; rom[ 3] = 12'd568;
rom[ 4] = 12'd583; rom[ 5] = 12'd2746; rom[ 6] = 12'd2649; rom[ 7] = 12'd680;
rom[ 8] = 12'd1637; rom[ 9] = 12'd1692; rom[ 10] = 12'd723; rom[ 11] = 12'd2606;
rom[ 12] = 12'd2288; rom[ 13] = 12'd1041; rom[ 14] = 12'd1100; rom[ 15] = 12'd2229;
rom[ 16] = 12'd1409; rom[ 17] = 12'd1920; rom[ 18] = 12'd2662; rom[ 19] = 12'd667;
rom[ 20] = 12'd3281; rom[ 21] = 12'd48; rom[ 22] = 12'd233; rom[ 23] = 12'd3096;
rom[ 24] = 12'd756; rom[ 25] = 12'd2573; rom[ 26] = 12'd2156; rom[ 27] = 12'd1173;
rom[ 28] = 12'd3015; rom[ 29] = 12'd314; rom[ 30] = 12'd3050; rom[ 31] = 12'd279;
rom[ 32] = 12'd1703; rom[ 33] = 12'd1626; rom[ 34] = 12'd1651; rom[ 35] = 12'd1678;
rom[ 36] = 12'd2789; rom[ 37] = 12'd540; rom[ 38] = 12'd1789; rom[ 39] = 12'd1540;
rom[ 40] = 12'd1847; rom[ 41] = 12'd1482; rom[ 42] = 12'd952; rom[ 43] = 12'd2377;
rom[ 44] = 12'd1461; rom[ 45] = 12'd1868; rom[ 46] = 12'd2687; rom[ 47] = 12'd642;
rom[ 48] = 12'd939; rom[ 49] = 12'd2390; rom[ 50] = 12'd2308; rom[ 51] = 12'd1021;
rom[ 52] = 12'd2437; rom[ 53] = 12'd892; rom[ 54] = 12'd2388; rom[ 55] = 12'd941;
rom[ 56] = 12'd733; rom[ 57] = 12'd2596; rom[ 58] = 12'd2337; rom[ 59] = 12'd992;
rom[ 60] = 12'd268; rom[ 61] = 12'd3061; rom[ 62] = 12'd641; rom[ 63] = 12'd2688;
rom[ 64] = 12'd1584; rom[ 65] = 12'd1745; rom[ 66] = 12'd2298; rom[ 67] = 12'd1031;
rom[ 68] = 12'd2037; rom[ 69] = 12'd1292; rom[ 70] = 12'd3220; rom[ 71] = 12'd109;
rom[ 72] = 12'd375; rom[ 73] = 12'd2954; rom[ 74] = 12'd2549; rom[ 75] = 12'd780;
rom[ 76] = 12'd2090; rom[ 77] = 12'd1239; rom[ 78] = 12'd1645; rom[ 79] = 12'd1684;
rom[ 80] = 12'd1063; rom[ 81] = 12'd2266; rom[ 82] = 12'd319; rom[ 83] = 12'd3010;
rom[ 84] = 12'd2773; rom[ 85] = 12'd556; rom[ 86] = 12'd757; rom[ 87] = 12'd2572;
rom[ 88] = 12'd2099; rom[ 89] = 12'd1230; rom[ 90] = 12'd561; rom[ 91] = 12'd2768;
rom[ 92] = 12'd2466; rom[ 93] = 12'd863; rom[ 94] = 12'd2594; rom[ 95] = 12'd735;
rom[ 96] = 12'd2804; rom[ 97] = 12'd525; rom[ 98] = 12'd1092; rom[ 99] = 12'd2237;
rom[100] = 12'd403; rom[101] = 12'd2926; rom[102] = 12'd1026; rom[103] = 12'd2303;
rom[104] = 12'd1143; rom[105] = 12'd2186; rom[106] = 12'd2150; rom[107] = 12'd1179;
rom[108] = 12'd2775; rom[109] = 12'd554; rom[110] = 12'd886; rom[111] = 12'd2443;
rom[112] = 12'd1722; rom[113] = 12'd1607; rom[114] = 12'd1212; rom[115] = 12'd2117;
rom[116] = 12'd1874; rom[117] = 12'd1455; rom[118] = 12'd1029; rom[119] = 12'd2300;
rom[120] = 12'd2110; rom[121] = 12'd1219; rom[122] = 12'd2935; rom[123] = 12'd394;
rom[124] = 12'd885; rom[125] = 12'd2444; rom[126] = 12'd2154; rom[127] = 12'd1175;
end
// Combinational read
assign zeta = rom[addr];
endmodule

50
rtl/ip/mlkem/rng/rng_sync.v Normal file
View File

@@ -0,0 +1,50 @@
// rng_sync.v - 256-bit Galois LFSR PRNG (taps: 255,253,252,247,0)
module rng_sync #(
parameter [255:0] SEED = 256'hDEADBEEFCAFEBABEFEEDFACEDECAFBAD1234567887654321ABCDEF010FEDCBA9
) (
input clk,
input rst_n,
input valid_i,
output ready_o,
output [255:0] data_o,
output valid_o,
input ready_i
);
reg [255:0] state;
reg valid_r;
reg [255:0] lfsr_next;
wire feedback;
integer i;
assign ready_o = 1'b1;
assign valid_o = valid_r;
assign data_o = state;
assign feedback = state[0];
always @(*) begin
for (i = 0; i < 255; i = i + 1)
lfsr_next[i] = state[i+1];
lfsr_next[255] = feedback;
lfsr_next[254] = lfsr_next[254] ^ feedback;
lfsr_next[252] = lfsr_next[252] ^ feedback;
lfsr_next[251] = lfsr_next[251] ^ feedback;
lfsr_next[246] = lfsr_next[246] ^ feedback;
end
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state <= SEED;
valid_r <= 1'b0;
end else begin
if (valid_r && ready_i)
valid_r <= 1'b0;
if (valid_i) begin
state <= lfsr_next;
valid_r <= 1'b1;
end
end
end
endmodule

305
rtl/ip/mlkem/sample_cbd/sample_cbd_sync.v Normal file
View File

@@ -0,0 +1,305 @@
// sample_cbd_sync.v - Centered Binomial Distribution sampling via SHAKE-256 PRF
//
// Generates 256 polynomial coefficients from a 256-bit seed and 8-bit nonce
// using SHAKE-256 PRF followed by Centered Binomial Distribution (CBD).
//
// Algorithm (FIPS 203 / ML-KEM):
// PRF(sigma, N) = SHAKE-256(sigma || N) → squeeze eta*64 bytes
// For each of 256 coefficients:
// eta=2: read 4 bits, coeff = (b0+b1) - (b2+b3)
// eta=3: read 6 bits, coeff = (b0+b1+b2) - (b3+b4+b5)
// Each coefficient in range [-eta, eta], stored as 12-bit signed.
//
// SHAKE-256 parameters:
// rate = 1088 bits, capacity = 512 bits
// suffix = 4'b1111
// pad10*1 padding
//
// Multi-squeeze (eta=3):
// SHAKE-256 squeezes 1088-bit blocks. For eta=3 we need 1536 bits.
// First squeeze provides bits [0:1087], second provides [1088:1535].
// The 1536-bit squeeze_buf accumulates both blocks contiguously:
// After 1st: buf[1087:0] = squeeze1, buf[1535:1088] = 0
// After 2nd: buf[1535:1088] = squeeze2[447:0] (remapped)
// Bit ordering matches Python reference PRF output.
//
// Interface:
// clk, rst_n - clock, active-low reset
// seed_i [255:0] - sigma (256-bit seed)
// nonce_i [7:0] - N counter byte
// eta_i [1:0] - 2'd2 or 2'd3
// valid_i - input valid (start sampling)
// ready_o - module can accept new input
// coeff_o [11:0] - one coefficient per cycle, 12-bit signed
// valid_o - output valid
// ready_i - downstream accepts output
// last_o - high when last coefficient (255th, 0-indexed)
module sample_cbd_sync (
input clk,
input rst_n,
input [255:0] seed_i,
input [7:0] nonce_i,
input [1:0] eta_i, // 2'd2 or 2'd3
input valid_i,
output ready_o,
output [11:0] coeff_o, // 12-bit signed
output valid_o,
input ready_i,
output last_o
);
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 2'd0;
localparam ST_PERMUTE = 2'd1;
localparam ST_SQUEEZE = 2'd2;
reg [1:0] state_r, state_next;
// ================================================================
// SHAKE-256 pad10*1 construction (combinational)
//
// Message: {nonce_i, seed_i} = 264 bits
// In FIPS 202 bit ordering: message[0] = seed_i[0], message[263] = nonce_i[7]
// Padded block (1088 rate bits):
// {1'b1, 818'b0, 1'b1, 4'b1111, nonce_i, seed_i}
//
// Matches Python: PRF(sigma, N) = shake256(sigma||N, d)
// where sigma bits come first (LSB), then N bits, then suffix+padding.
// ================================================================
wire [263:0] message_264;
wire [1087:0] padded_block;
assign message_264 = {nonce_i, seed_i};
assign padded_block = {1'b1, {818{1'b0}}, 1'b1, 4'b1111, message_264};
// Absorb state: capacity (512 bits of zero) || padded rate block
wire [1599:0] absorb_state;
assign absorb_state = {{(1600-1088){1'b0}}, padded_block};
// ================================================================
// Registered inputs (captured on valid_i && ready_o)
// ================================================================
reg [2:0] eta_r; // 2 or 3
reg [7:0] coeff_cnt; // 0..255, number of coeffs output so far
reg [1599:0] keccak_state_r; // current 1600-bit keccak state (for re-permutation)
reg perm_done; // 1 after first keccak permutation completes
// Combinational mux select: absorb_state BEFORE first perm finishes,
// keccak_state_r AFTER. Must be combinational to avoid NBA race
// with kc_valid_i on the IDLE→PERMUTE transition edge.
wire first_perm_sel;
assign first_perm_sel = !perm_done;
// ================================================================
// Squeeze buffer (accumulated across multiple squeezes)
// ================================================================
reg [1535:0] squeeze_buf; // accumulated squeeze data
reg [10:0] buf_fill; // valid bits in squeeze_buf (0, 1088, or 1536)
reg [10:0] buf_ptr; // read position within squeeze_buf
reg kc_done_d1; // kc_valid_o delayed by 1 cycle (gate for valid_o)
// ================================================================
// keccak_core instantiation
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_valid_o;
// Mux: absorb_state for first perm, keccak_state_r for re-permutation
wire [1599:0] kc_state_i;
assign kc_state_i = first_perm_sel ? absorb_state : keccak_state_r;
keccak_core #(.ROUNDS(24)) u_keccak (
.clk (clk),
.rst_n (rst_n),
.state_i (kc_state_i),
.valid_i (kc_valid_i),
.ready_o (kc_ready_o),
.state_o (kc_state_o),
.valid_o (kc_valid_o),
.ready_i (1'b1) // always accept keccak output
);
// ================================================================
// Bits per coefficient
// ================================================================
wire [3:0] bits_per_coeff;
assign bits_per_coeff = (eta_r == 3'd2) ? 4'd4 : 4'd6;
// ================================================================
// CBD computation (combinational)
//
// Extract bits_per_coeff bits from squeeze_buf starting at buf_ptr.
// For eta=2: b0,b1,b2,b3 → coeff = (b0+b1) - (b2+b3)
// For eta=3: b0,b1,b2,b3,b4,b5 → coeff = (b0+b1+b2) - (b3+b4+b5)
// ================================================================
wire [5:0] cbd_bits;
assign cbd_bits = squeeze_buf[buf_ptr +: 6];
wire [2:0] sum_pos, sum_neg;
// eta=2: sum_pos = b0+b1, sum_neg = b2+b3
wire [2:0] sp2, sn2;
assign sp2 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]};
assign sn2 = {2'b00, cbd_bits[2]} + {2'b00, cbd_bits[3]};
// eta=3: sum_pos = b0+b1+b2, sum_neg = b3+b4+b5
wire [2:0] sp3, sn3;
assign sp3 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]} + {2'b00, cbd_bits[2]};
assign sn3 = {2'b00, cbd_bits[3]} + {2'b00, cbd_bits[4]} + {2'b00, cbd_bits[5]};
assign sum_pos = (eta_r == 3'd2) ? sp2 : sp3;
assign sum_neg = (eta_r == 3'd2) ? sn2 : sn3;
wire signed [3:0] coeff_raw;
assign coeff_raw = $signed({1'b0, sum_pos}) - $signed({1'b0, sum_neg});
wire [11:0] coeff_signed;
assign coeff_signed = {{8{coeff_raw[3]}}, coeff_raw[3:0]}; // sign-extend to 12 bits
assign coeff_o = coeff_signed;
// valid_o: suppress for 1 cycle after keccak finishes to avoid
// buf_ptr race between squeeze capture and SQUEEZE advancement.
assign valid_o = (state_r == ST_SQUEEZE) && (buf_fill >= {7'b0, bits_per_coeff}) && !kc_done_d1;
assign last_o = valid_o && (coeff_cnt == 8'd255);
// ================================================================
// FSM: ready_o
// ================================================================
assign ready_o = (state_r == ST_IDLE);
// ================================================================
// kc_valid_i: start keccak_core when transitioning to PERMUTE
// ================================================================
assign kc_valid_i = (state_next == ST_PERMUTE) && (state_r != ST_PERMUTE);
// ================================================================
// Buffer exhaustion detection
// ================================================================
wire [11:0] next_buf_ptr;
assign next_buf_ptr = {1'b0, buf_ptr} + {8'b0, bits_per_coeff};
// buffer will be exhausted after outputting NEXT coefficient
wire buffer_exhaust_next;
assign buffer_exhaust_next = (next_buf_ptr + {8'b0, bits_per_coeff}) > {1'b0, buf_fill};
// Coefficients remaining AFTER the current one
wire [8:0] coeffs_remaining;
assign coeffs_remaining = 9'd256 - {1'b0, coeff_cnt} - 9'd1;
// ================================================================
// FSM combinational next-state logic
// ================================================================
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: begin
if (valid_i && ready_o)
state_next = ST_PERMUTE;
end
ST_PERMUTE: begin
// Wait for keccak_core to finish
if (kc_valid_o)
state_next = ST_SQUEEZE;
end
ST_SQUEEZE: begin
// Output one coeff per cycle when ready
if (valid_o && ready_i) begin
if (coeff_cnt == 8'd255) begin
// Last coefficient was output
state_next = ST_IDLE;
end else if (buffer_exhaust_next && (coeffs_remaining > 9'd0)) begin
// Need more squeeze data: start another keccak permutation
state_next = ST_PERMUTE;
end
// else: stay in SQUEEZE for next coefficient
end
end
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// Sequential logic
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
eta_r <= 3'd0;
coeff_cnt <= 8'd0;
keccak_state_r <= 1600'd0;
perm_done <= 1'b0;
squeeze_buf <= 1536'd0;
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
kc_done_d1 <= 1'b0;
end else begin
state_r <= state_next;
// Delay kc_valid_o by 1 cycle to gate valid_o
kc_done_d1 <= kc_valid_o;
// ---- Capture inputs on IDLE → PERMUTE transition ----
if (state_r == ST_IDLE && valid_i && ready_o) begin
// Determine eta: 2 or 3
if (eta_i == 2'd2)
eta_r <= 3'd2;
else if (eta_i == 2'd3)
eta_r <= 3'd3;
else
eta_r <= 3'd2; // default
coeff_cnt <= 8'd0;
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
end
// ---- On keccak_core valid_o: latch squeeze data ----
if (kc_valid_o) begin
// Save full 1600-bit state for potential re-permutation
keccak_state_r <= kc_state_o;
if (!perm_done) begin
// First squeeze: fill lower 1088 bits of squeeze_buf
// buf[1087:0] = squeeze data, buf[1535:1088] = 0
squeeze_buf <= {{(1536 - 1088){1'b0}}, kc_state_o[1087:0]};
buf_fill <= 11'd1088;
buf_ptr <= 11'd0;
perm_done <= 1'b1;
end else begin
// Second squeeze: fill upper 448 bits of squeeze_buf
// buf[1535:1088] = squeeze2[447:0], buf[1087:0] preserved
// This remaps: squeeze2[0] → buf[1088], matching Python's contiguous output
squeeze_buf <= {kc_state_o[447:0], squeeze_buf[1087:0]};
buf_fill <= 11'd1536;
// buf_ptr stays at current position (kept from SQUEEZE)
end
end
// ---- SQUEEZE: advance on output ----
if (state_r == ST_SQUEEZE && valid_o && ready_i) begin
buf_ptr <= buf_ptr + {7'b0, bits_per_coeff};
coeff_cnt <= coeff_cnt + 8'd1;
// Clear state on last coeff
if (coeff_cnt == 8'd255) begin
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
// Reset perm_done here (before next IDLE→PERMUTE)
// so the mux selects absorb_state for the next vector.
perm_done <= 1'b0;
end
end
end
end
endmodule

311
rtl/ip/mlkem/sample_cbd/sample_cbd_sync_shared.v Normal file
View File

@@ -0,0 +1,311 @@
// sample_cbd_sync_shared.v - Centered Binomial Distribution sampling via SHAKE-256 PRF
// (shared keccak_core variant — external instance)
//
// Generates 256 polynomial coefficients from a 256-bit seed and 8-bit nonce
// using SHAKE-256 PRF followed by Centered Binomial Distribution (CBD).
//
// Algorithm (FIPS 203 / ML-KEM):
// PRF(sigma, N) = SHAKE-256(sigma || N) → squeeze eta*64 bytes
// For each of 256 coefficients:
// eta=2: read 4 bits, coeff = (b0+b1) - (b2+b3)
// eta=3: read 6 bits, coeff = (b0+b1+b2) - (b3+b4+b5)
// Each coefficient in range [-eta, eta], stored as 12-bit signed.
//
// SHAKE-256 parameters:
// rate = 1088 bits, capacity = 512 bits
// suffix = 4'b1111
// pad10*1 padding
//
// Multi-squeeze (eta=3):
// SHAKE-256 squeezes 1088-bit blocks. For eta=3 we need 1536 bits.
// First squeeze provides bits [0:1087], second provides [1088:1535].
// The 1536-bit squeeze_buf accumulates both blocks contiguously:
// After 1st: buf[1087:0] = squeeze1, buf[1535:1088] = 0
// After 2nd: buf[1535:1088] = squeeze2[447:0] (remapped)
// Bit ordering matches Python reference PRF output.
//
// Interface:
// clk, rst_n - clock, active-low reset
// seed_i [255:0] - sigma (256-bit seed)
// nonce_i [7:0] - N counter byte
// eta_i [1:0] - 2'd2 or 2'd3
// valid_i - input valid (start sampling)
// ready_o - module can accept new input
// coeff_o [11:0] - one coefficient per cycle, 12-bit signed
// valid_o - output valid
// ready_i - downstream accepts output
// last_o - high when last coefficient (255th, 0-indexed)
// kc_state_o - keccak result (from external keccak_core.state_o)
// kc_valid_o - keccak done (from external keccak_core.valid_o)
// kc_ready_i - always ready to accept keccak output (to keccak_core.ready_i)
// kc_state_i - keccak input state (to external keccak_core.state_i)
// kc_valid_i - request keccak permutation (to external keccak_core.valid_i)
// kc_ready_o - keccak ready to accept (from external keccak_core.ready_o)
module sample_cbd_sync_shared (
input clk,
input rst_n,
input [255:0] seed_i,
input [7:0] nonce_i,
input [1:0] eta_i, // 2'd2 or 2'd3
input valid_i,
output ready_o,
output [11:0] coeff_o, // 12-bit signed
output valid_o,
input ready_i,
output last_o,
// External keccak_core interface
input [1599:0] kc_state_o, // keccak result (from keccak_core.state_o)
input kc_valid_o, // keccak done (from keccak_core.valid_o)
output kc_ready_i, // always ready to accept (to keccak_core.ready_i)
output [1599:0] kc_state_i, // keccak input state (to keccak_core.state_i)
output kc_valid_i, // request keccak permutation (to keccak_core.valid_i)
input kc_ready_o // keccak ready to accept (from keccak_core.ready_o)
);
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 2'd0;
localparam ST_PERMUTE = 2'd1;
localparam ST_SQUEEZE = 2'd2;
reg [1:0] state_r, state_next;
// ================================================================
// SHAKE-256 pad10*1 construction (combinational)
//
// Message: {nonce_i, seed_i} = 264 bits
// In FIPS 202 bit ordering: message[0] = seed_i[0], message[263] = nonce_i[7]
// Padded block (1088 rate bits):
// {1'b1, 818'b0, 1'b1, 4'b1111, nonce_i, seed_i}
//
// Matches Python: PRF(sigma, N) = shake256(sigma||N, d)
// where sigma bits come first (LSB), then N bits, then suffix+padding.
// ================================================================
wire [263:0] message_264;
wire [1087:0] padded_block;
assign message_264 = {nonce_i, seed_i};
assign padded_block = {1'b1, {818{1'b0}}, 1'b1, 4'b1111, message_264};
// Absorb state: capacity (512 bits of zero) || padded rate block
wire [1599:0] absorb_state;
assign absorb_state = {{(1600-1088){1'b0}}, padded_block};
// ================================================================
// Registered inputs (captured on valid_i && ready_o)
// ================================================================
reg [2:0] eta_r; // 2 or 3
reg [7:0] coeff_cnt; // 0..255, number of coeffs output so far
reg [1599:0] keccak_state_r; // current 1600-bit keccak state (for re-permutation)
reg perm_done; // 1 after first keccak permutation completes
// Combinational mux select: absorb_state BEFORE first perm finishes,
// keccak_state_r AFTER. Must be combinational to avoid NBA race
// with kc_valid_i on the IDLE→PERMUTE transition edge.
wire first_perm_sel;
assign first_perm_sel = !perm_done;
// ================================================================
// Squeeze buffer (accumulated across multiple squeezes)
// ================================================================
reg [1535:0] squeeze_buf; // accumulated squeeze data
reg [10:0] buf_fill; // valid bits in squeeze_buf (0, 1088, or 1536)
reg [10:0] buf_ptr; // read position within squeeze_buf
reg kc_done_d1; // kc_valid_o delayed by 1 cycle (gate for valid_o)
// ================================================================
// External keccak_core interface connections
//
// The external keccak_core is instantiated at the top level.
// This module drives kc_valid_i and kc_state_i, receives
// kc_valid_o and kc_state_o, and provides kc_ready_i = 1'b1
// (always ready to accept keccak output). kc_ready_o is exposed
// for the top-level arbiter to check keccak availability.
// ================================================================
// Mux: absorb_state for first perm, keccak_state_r for re-permutation
assign kc_state_i = first_perm_sel ? absorb_state : keccak_state_r;
// Always ready to accept keccak output
assign kc_ready_i = 1'b1;
// ================================================================
// Bits per coefficient
// ================================================================
wire [3:0] bits_per_coeff;
assign bits_per_coeff = (eta_r == 3'd2) ? 4'd4 : 4'd6;
// ================================================================
// CBD computation (combinational)
//
// Extract bits_per_coeff bits from squeeze_buf starting at buf_ptr.
// For eta=2: b0,b1,b2,b3 → coeff = (b0+b1) - (b2+b3)
// For eta=3: b0,b1,b2,b3,b4,b5 → coeff = (b0+b1+b2) - (b3+b4+b5)
// ================================================================
wire [5:0] cbd_bits;
assign cbd_bits = squeeze_buf[buf_ptr +: 6];
wire [2:0] sum_pos, sum_neg;
// eta=2: sum_pos = b0+b1, sum_neg = b2+b3
wire [2:0] sp2, sn2;
assign sp2 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]};
assign sn2 = {2'b00, cbd_bits[2]} + {2'b00, cbd_bits[3]};
// eta=3: sum_pos = b0+b1+b2, sum_neg = b3+b4+b5
wire [2:0] sp3, sn3;
assign sp3 = {2'b00, cbd_bits[0]} + {2'b00, cbd_bits[1]} + {2'b00, cbd_bits[2]};
assign sn3 = {2'b00, cbd_bits[3]} + {2'b00, cbd_bits[4]} + {2'b00, cbd_bits[5]};
assign sum_pos = (eta_r == 3'd2) ? sp2 : sp3;
assign sum_neg = (eta_r == 3'd2) ? sn2 : sn3;
wire signed [3:0] coeff_raw;
assign coeff_raw = $signed({1'b0, sum_pos}) - $signed({1'b0, sum_neg});
wire [11:0] coeff_signed;
assign coeff_signed = {{8{coeff_raw[3]}}, coeff_raw[3:0]}; // sign-extend to 12 bits
assign coeff_o = coeff_signed;
// valid_o: suppress for 1 cycle after keccak finishes to avoid
// buf_ptr race between squeeze capture and SQUEEZE advancement.
assign valid_o = (state_r == ST_SQUEEZE) && (buf_fill >= {7'b0, bits_per_coeff}) && !kc_done_d1;
assign last_o = valid_o && (coeff_cnt == 8'd255);
// ================================================================
// FSM: ready_o
// ================================================================
assign ready_o = (state_r == ST_IDLE);
// ================================================================
// kc_valid_i: start external keccak_core when transitioning to PERMUTE
// ================================================================
assign kc_valid_i = (state_next == ST_PERMUTE) && (state_r != ST_PERMUTE);
// ================================================================
// Buffer exhaustion detection
// ================================================================
wire [11:0] next_buf_ptr;
assign next_buf_ptr = {1'b0, buf_ptr} + {8'b0, bits_per_coeff};
// buffer will be exhausted after outputting NEXT coefficient
wire buffer_exhaust_next;
assign buffer_exhaust_next = (next_buf_ptr + {8'b0, bits_per_coeff}) > {1'b0, buf_fill};
// Coefficients remaining AFTER the current one
wire [8:0] coeffs_remaining;
assign coeffs_remaining = 9'd256 - {1'b0, coeff_cnt} - 9'd1;
// ================================================================
// FSM combinational next-state logic
// ================================================================
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: begin
if (valid_i && ready_o)
state_next = ST_PERMUTE;
end
ST_PERMUTE: begin
// Wait for external keccak_core to finish
if (kc_valid_o)
state_next = ST_SQUEEZE;
end
ST_SQUEEZE: begin
// Output one coeff per cycle when ready
if (valid_o && ready_i) begin
if (coeff_cnt == 8'd255) begin
// Last coefficient was output
state_next = ST_IDLE;
end else if (buffer_exhaust_next && (coeffs_remaining > 9'd0)) begin
// Need more squeeze data: start another keccak permutation
state_next = ST_PERMUTE;
end
// else: stay in SQUEEZE for next coefficient
end
end
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// Sequential logic
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
eta_r <= 3'd0;
coeff_cnt <= 8'd0;
keccak_state_r <= 1600'd0;
perm_done <= 1'b0;
squeeze_buf <= 1536'd0;
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
kc_done_d1 <= 1'b0;
end else begin
state_r <= state_next;
// Delay kc_valid_o by 1 cycle to gate valid_o
kc_done_d1 <= kc_valid_o;
// ---- Capture inputs on IDLE → PERMUTE transition ----
if (state_r == ST_IDLE && valid_i && ready_o) begin
// Determine eta: 2 or 3
if (eta_i == 2'd2)
eta_r <= 3'd2;
else if (eta_i == 2'd3)
eta_r <= 3'd3;
else
eta_r <= 3'd2; // default
coeff_cnt <= 8'd0;
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
end
// ---- On keccak_core valid_o: latch squeeze data ----
if (kc_valid_o) begin
// Save full 1600-bit state for potential re-permutation
keccak_state_r <= kc_state_o;
if (!perm_done) begin
// First squeeze: fill lower 1088 bits of squeeze_buf
// buf[1087:0] = squeeze data, buf[1535:1088] = 0
squeeze_buf <= {{(1536 - 1088){1'b0}}, kc_state_o[1087:0]};
buf_fill <= 11'd1088;
buf_ptr <= 11'd0;
perm_done <= 1'b1;
end else begin
// Second squeeze: fill upper 448 bits of squeeze_buf
// buf[1535:1088] = squeeze2[447:0], buf[1087:0] preserved
// This remaps: squeeze2[0] → buf[1088], matching Python's contiguous output
squeeze_buf <= {kc_state_o[447:0], squeeze_buf[1087:0]};
buf_fill <= 11'd1536;
// buf_ptr stays at current position (kept from SQUEEZE)
end
end
// ---- SQUEEZE: advance on output ----
if (state_r == ST_SQUEEZE && valid_o && ready_i) begin
buf_ptr <= buf_ptr + {7'b0, bits_per_coeff};
coeff_cnt <= coeff_cnt + 8'd1;
// Clear state on last coeff
if (coeff_cnt == 8'd255) begin
buf_fill <= 11'd0;
buf_ptr <= 11'd0;
// Reset perm_done here (before next IDLE→PERMUTE)
// so the mux selects absorb_state for the next vector.
perm_done <= 1'b0;
end
end
end
end
endmodule

404
rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v Normal file
View File

@@ -0,0 +1,404 @@
// sample_ntt_sync.v - Synchronous SampleNTT for ML-KEM A matrix generation
//
// Generates one k×k polynomial (256 coefficients) via SHAKE-128 XOF
// rejection sampling from seed rho || j || i.
//
// FIPS 202/203 conformant SHAKE-128 squeeze:
// - Absorb: S = Keccak-p(padded(rho || j || i))
// - Squeeze the full 1344-bit (168-byte) rate as 56 groups of 3 bytes,
// each group g read from S[24*g +: 24]; extract d1[11:0], d2[23:12]
// - Accept d if d < Q=3329
// - Only after all 56 groups of the block are consumed: S = Keccak-p(S)
// (re-permute once per rate block, NOT per 3-byte group)
// - Repeat until 256 coefficients collected (~3 blocks)
//
// Parameters:
// K = 4 (ML-KEM parameter)
//
// Interface:
// clk, rst_n - clock, active-low reset
// rho_i[255:0] - 256-bit seed rho (32 bytes)
// k_i[2:0] - actual k value (2/3/4)
// i_idx[1:0] - row index (0..k-1)
// j_idx[1:0] - column index (0..k-1)
// valid_i - start generation for this (i,j) pair
// ready_o - module can accept request
// coeff_o[11:0] - one coefficient output per cycle
// valid_o - coefficient output valid
// ready_i - consumer accepts coefficient
// last_o - high when 256th coefficient is output
`include "defines.vh"
/* verilator lint_off UNUSEDPARAM */
(* use_dsp = "no" *)
module sample_ntt_sync #(parameter K = 4) (
/* verilator lint_on UNUSEDPARAM */
input clk,
input rst_n,
input [255:0] rho_i,
/* verilator lint_off UNUSEDSIGNAL */
input [2:0] k_i,
/* verilator lint_on UNUSEDSIGNAL */
input [1:0] i_idx,
input [1:0] j_idx,
input valid_i,
output ready_o,
output [11:0] coeff_o,
output valid_o,
input ready_i,
output last_o
);
// ================================================================
// Local parameters
// ================================================================
localparam Q = `Q; // 3329
// SHAKE-128 rate = 1344 bits = 168 bytes = 56 groups of 3 bytes.
// After consuming all 56 groups of a block, re-permute the state.
localparam GRP_MAX = 6'd55;
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 3'd0;
localparam ST_ABSORB = 3'd1; // keccak running on absorb_state
localparam ST_SQUEEZE = 3'd2; // extract d1/d2, output coefficients
localparam ST_WAIT = 3'd3; // wait for next keccak to finish
localparam ST_DONE = 3'd4; // all 256 coefficients output
reg [2:0] state_r, state_next;
// ================================================================
// Registered inputs (captured on valid_i)
// ================================================================
/* verilator lint_off UNUSEDSIGNAL */
reg [255:0] rho_r;
reg [2:0] k_r;
reg [1:0] i_r;
reg [1:0] j_r;
/* verilator lint_on UNUSEDSIGNAL */
// ================================================================
// Coefficient counter (0..256). 9 bits to avoid overflow when
// reaching 256 after the 256th output.
// ================================================================
reg [8:0] coeff_cnt_r;
// ================================================================
// Squeeze state register (1600-bit result of keccak_p)
// ================================================================
reg [1599:0] squeeze_state_r;
// ================================================================
// Group pointer: which 3-byte group within the 1344-bit rate
// block is currently being consumed (0..GRP_MAX). Re-permute when
// it would exceed GRP_MAX.
// ================================================================
reg [5:0] grp_ptr_r;
// ================================================================
// Registered d1, d2 and acceptance flags
// ================================================================
reg [11:0] d1_r, d2_r;
reg d1_acc_r, d2_acc_r; // d1/d2 accepted?
// ================================================================
// Squeeze sub-phase (for multi-cycle coefficient output)
// 0 → latch d1/d2, start keccak → 1
// 1 → output d1 if accepted → 2
// 2 → output d2 if accepted → WAIT
// ================================================================
reg [1:0] sq_phase_r;
// ================================================================
// Comb: build absorb state from rho, i, j INPUT PORTS directly
// (not registered copies — avoids NBA race on IDLE→ABSORB edge)
// ================================================================
wire [7:0] abs_i_byte, abs_j_byte;
assign abs_i_byte = {6'b0, i_idx};
assign abs_j_byte = {6'b0, j_idx};
wire [271:0] msg_bytes;
assign msg_bytes = {abs_i_byte, abs_j_byte, rho_i};
// SHAKE-128 absorb block: capacity(256b) | pad10*1 | suffix(1111) | msg(272b)
wire [1599:0] absorb_state;
assign absorb_state = {
256'b0, // capacity [1599:1344]
1'b1, // pad10*1 final 1 [1343]
{1066{1'b0}}, // pad10*1 zeros [1342:277]
1'b1, // pad10*1 first 1 [276]
4'b1111, // SHAKE suffix [275:272]
msg_bytes // message [271:0]
};
// ================================================================
// Comb: extract d1,d2 from the current 3-byte group of squeeze state
// ================================================================
// group g occupies bits [24*g +: 24]: c0=byte0, c1=byte1, c2=byte2
// d1 = {c1[3:0], c0}
// d2 = {c2, c1[7:4]}
wire [10:0] grp_bit_off;
assign grp_bit_off = grp_ptr_r * 11'd24; // 0..1320, +24 ≤ 1344 (rate)
wire [23:0] grp_bits;
assign grp_bits = squeeze_state_r[ grp_bit_off +: 24 ];
wire [7:0] c0, c1, c2;
assign c0 = grp_bits[7:0];
assign c1 = grp_bits[15:8];
assign c2 = grp_bits[23:16];
wire [11:0] d1_comb, d2_comb;
assign d1_comb = {c1[3:0], c0};
assign d2_comb = {c2, c1[7:4]};
wire d1_ok, d2_ok;
assign d1_ok = (d1_comb < Q);
assign d2_ok = (d2_comb < Q);
// ================================================================
// Keccak core instantiation
// ================================================================
wire kc_valid_i;
wire [1599:0] kc_state_i;
/* verilator lint_off UNUSEDSIGNAL */
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_valid_o;
keccak_core #(.ROUNDS(24)) u_keccak (
.clk (clk),
.rst_n (rst_n),
.state_i (kc_state_i),
.valid_i (kc_valid_i),
.ready_o (kc_ready_o),
.state_o (kc_state_o),
.valid_o (kc_valid_o),
.ready_i (1'b1)
);
// kc_valid_i: asserted on the ABSORB load, and for one cycle when the
// squeeze block is exhausted (SQUEEZE → WAIT) to re-permute the state.
assign kc_valid_i = (state_next == ST_ABSORB) ||
(state_r == ST_SQUEEZE && state_next == ST_WAIT);
// kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise
assign kc_state_i = (state_next == ST_ABSORB) ? absorb_state : squeeze_state_r;
// ================================================================
// Output signals
// ================================================================
reg [11:0] coeff_o_r;
reg valid_o_r;
reg last_o_r;
assign coeff_o = coeff_o_r;
assign valid_o = valid_o_r;
assign last_o = last_o_r;
// ================================================================
// ready_o: accept new request in IDLE
// ================================================================
assign ready_o = (state_r == ST_IDLE);
wire need_more = (coeff_cnt_r < 9'd256);
// grp_done: current 3-byte group fully consumed (phase 2 resolved):
// d2 rejected, no longer need coefficients, or d2 was just accepted out.
wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) &&
(!d2_acc_r || !need_more || (valid_o_r && ready_i));
// ================================================================
// FSM: state_next (combinational)
// ================================================================
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: begin
if (valid_i && ready_o)
state_next = ST_ABSORB;
end
ST_ABSORB: begin
// Wait for keccak to finish the absorb permutation
if (kc_valid_o)
state_next = ST_SQUEEZE;
end
ST_SQUEEZE: begin
// A group is fully consumed once phase 2 resolves (d2 output,
// rejected, or no longer needed). Then either advance to the
// next group in this block, re-permute (block exhausted), or
// finish.
if (grp_done) begin
if (!need_more)
state_next = ST_DONE;
else if (grp_ptr_r < GRP_MAX)
state_next = ST_SQUEEZE; // next group, no re-permute
else
state_next = ST_WAIT; // block exhausted: re-permute
end
end
ST_WAIT: begin
// Wait for keccak to finish squeeze permutation
if (kc_valid_o) begin
if (!need_more)
state_next = ST_DONE;
else
state_next = ST_SQUEEZE;
end
end
ST_DONE: begin
state_next = ST_IDLE;
end
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// FSM: sequential logic (registered)
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
sq_phase_r <= 2'd0;
coeff_cnt_r <= 9'd0;
squeeze_state_r <= 1600'd0;
grp_ptr_r <= 6'd0;
d1_r <= 12'd0;
d2_r <= 12'd0;
d1_acc_r <= 1'b0;
d2_acc_r <= 1'b0;
coeff_o_r <= 12'd0;
valid_o_r <= 1'b0;
last_o_r <= 1'b0;
rho_r <= 256'd0;
k_r <= 3'd0;
i_r <= 2'd0;
j_r <= 2'd0;
end else begin
state_r <= state_next;
// ---------------------------------------------------------
// Capture inputs on valid_i (IDLE → ABSORB transition)
// ---------------------------------------------------------
if (state_r == ST_IDLE && valid_i && ready_o) begin
rho_r <= rho_i;
k_r <= k_i;
i_r <= i_idx;
j_r <= j_idx;
coeff_cnt_r <= 9'd0;
end
// ---------------------------------------------------------
// Latch keccak output when valid_o fires. A fresh block
// starts at group 0 (ABSORB load or WAIT re-permute result).
// ---------------------------------------------------------
if (kc_valid_o) begin
squeeze_state_r <= kc_state_o;
grp_ptr_r <= 6'd0;
end
// ---------------------------------------------------------
// SQ_PHASE = 0: latch d1/d2 acceptance
// ---------------------------------------------------------
if (state_r == ST_SQUEEZE && sq_phase_r == 2'd0) begin
d1_r <= d1_comb;
d2_r <= d2_comb;
d1_acc_r <= d1_ok;
d2_acc_r <= d2_ok;
end
// ---------------------------------------------------------
// SQ_PHASE management and coefficient output
//
// Handshake pattern (matching pipeline_reg convention):
// Cycle N: valid_o_r ← 1, coeff_o_r ← value
// Cycle N+1: if ready_i: consume, valid_o_r ← 0, advance
//
// The testbench reads coeff_o after Cycle N and consumes
// with the next posedge (Cycle N+1).
// ---------------------------------------------------------
if (state_r == ST_SQUEEZE) begin
case (sq_phase_r)
// ----- Phase 0: latch, advance to phase 1 -----
2'd0: begin
sq_phase_r <= 2'd1;
end
// ----- Phase 1: output d1 -----
2'd1: begin
if (d1_acc_r && need_more) begin
if (!valid_o_r) begin
// First cycle: assert output
coeff_o_r <= d1_r;
valid_o_r <= 1'b1;
last_o_r <= (coeff_cnt_r == 9'd255);
end else begin
// valid_o is high; wait for consumer
if (ready_i) begin
coeff_cnt_r <= coeff_cnt_r + 9'd1;
valid_o_r <= 1'b0;
sq_phase_r <= 2'd2;
end
end
end else begin
// d1 rejected or no longer needed: skip to phase 2
valid_o_r <= 1'b0;
sq_phase_r <= 2'd2;
end
end
// ----- Phase 2: output d2 -----
2'd2: begin
if (d2_acc_r && need_more) begin
if (!valid_o_r) begin
// First cycle: assert output
coeff_o_r <= d2_r;
valid_o_r <= 1'b1;
last_o_r <= (coeff_cnt_r == 9'd255);
end else begin
// valid_o is high; wait for consumer
if (ready_i) begin
coeff_cnt_r <= coeff_cnt_r + 9'd1;
valid_o_r <= 1'b0;
// state_next → ST_WAIT (combinational)
end
end
end else begin
// d2 rejected or done
valid_o_r <= 1'b0;
end
end
default: begin
valid_o_r <= 1'b0;
end
endcase
// Group consumed but block not exhausted: advance to the
// next 3-byte group within the same block (no re-permute).
if (grp_done && need_more && grp_ptr_r < GRP_MAX) begin
grp_ptr_r <= grp_ptr_r + 6'd1;
sq_phase_r <= 2'd0;
end
end else if (state_r != ST_SQUEEZE && state_next == ST_SQUEEZE) begin
// About to enter SQUEEZE: reset phase and output
sq_phase_r <= 2'd0;
valid_o_r <= 1'b0;
end else begin
// Not in SQUEEZE: clear
valid_o_r <= 1'b0;
sq_phase_r <= 2'd0;
end
end
end
endmodule

418
rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v Normal file
View File

@@ -0,0 +1,418 @@
// sample_ntt_sync_shared.v - Synchronous SampleNTT for ML-KEM A matrix generation
// with external Keccak interface ports.
//
// Identical to sample_ntt_sync.v except the internal keccak_core instance
// is replaced by explicit kc_* ports. This allows multiple SampleNTT
// instances to share a single keccak_core (via a round-robin arbiter).
//
// Generates one k×k polynomial (256 coefficients) via SHAKE-128 XOF
// rejection sampling from seed rho || j || i.
//
// FIPS 202/203 conformant SHAKE-128 squeeze:
// - Absorb: S = Keccak-p(padded(rho || j || i))
// - Squeeze the full 1344-bit (168-byte) rate as 56 groups of 3 bytes,
// each group g read from S[24*g +: 24]; extract d1[11:0], d2[23:12]
// - Accept d if d < Q=3329
// - Only after all 56 groups of the block are consumed: S = Keccak-p(S)
// (re-permute once per rate block, NOT per 3-byte group)
// - Repeat until 256 coefficients collected (~3 blocks)
//
// Parameters:
// K = 4 (ML-KEM parameter)
//
// Interface:
// clk, rst_n - clock, active-low reset
// rho_i[255:0] - 256-bit seed rho (32 bytes)
// k_i[2:0] - actual k value (2/3/4)
// i_idx[1:0] - row index (0..k-1)
// j_idx[1:0] - column index (0..k-1)
// valid_i - start generation for this (i,j) pair
// ready_o - module can accept request
// coeff_o[11:0] - one coefficient output per cycle
// valid_o - coefficient output valid
// ready_i - consumer accepts coefficient
// last_o - high when 256th coefficient is output
//
// --- Keccak interface (external, shared) ---
// kc_state_o[1599:0] - keccak result (from keccak_core)
// kc_valid_o - keccak output valid (from keccak_core)
// kc_ready_i - sample_ntt ready to accept keccak output (always 1'b1)
// kc_state_i[1599:0] - keccak input state (to keccak_core)
// kc_valid_i - request keccak permutation (to keccak_core)
// kc_ready_o - keccak ready to accept new request (from keccak_core)
`include "defines.vh"
/* verilator lint_off UNUSEDPARAM */
(* use_dsp = "no" *)
module sample_ntt_sync_shared #(parameter K = 4) (
/* verilator lint_on UNUSEDPARAM */
input clk,
input rst_n,
input [255:0] rho_i,
/* verilator lint_off UNUSEDSIGNAL */
input [2:0] k_i,
/* verilator lint_on UNUSEDSIGNAL */
input [1:0] i_idx,
input [1:0] j_idx,
input valid_i,
output ready_o,
output [11:0] coeff_o,
output valid_o,
input ready_i,
output last_o,
// --- Keccak interface ports (external) ---
input [1599:0] kc_state_o,
input kc_valid_o,
output kc_ready_i,
output [1599:0] kc_state_i,
output kc_valid_i,
/* verilator lint_off UNUSEDSIGNAL */
input kc_ready_o
/* verilator lint_on UNUSEDSIGNAL */
);
// ================================================================
// Local parameters
// ================================================================
localparam Q = `Q; // 3329
// SHAKE-128 rate = 1344 bits = 168 bytes = 56 groups of 3 bytes.
// After consuming all 56 groups of a block, re-permute the state.
localparam GRP_MAX = 6'd55;
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 3'd0;
localparam ST_ABSORB = 3'd1; // keccak running on absorb_state
localparam ST_SQUEEZE = 3'd2; // extract d1/d2, output coefficients
localparam ST_WAIT = 3'd3; // wait for next keccak to finish
localparam ST_DONE = 3'd4; // all 256 coefficients output
reg [2:0] state_r, state_next;
// ================================================================
// Registered inputs (captured on valid_i)
// ================================================================
/* verilator lint_off UNUSEDSIGNAL */
reg [255:0] rho_r;
reg [2:0] k_r;
reg [1:0] i_r;
reg [1:0] j_r;
/* verilator lint_on UNUSEDSIGNAL */
// ================================================================
// Coefficient counter (0..256). 9 bits to avoid overflow when
// reaching 256 after the 256th output.
// ================================================================
reg [8:0] coeff_cnt_r;
// ================================================================
// Squeeze state register (1600-bit result of keccak_p)
// ================================================================
reg [1599:0] squeeze_state_r;
// ================================================================
// Group pointer: which 3-byte group within the 1344-bit rate
// block is currently being consumed (0..GRP_MAX).
// ================================================================
reg [5:0] grp_ptr_r;
// ================================================================
// Registered d1, d2 and acceptance flags
// ================================================================
reg [11:0] d1_r, d2_r;
reg d1_acc_r, d2_acc_r; // d1/d2 accepted?
// ================================================================
// Squeeze sub-phase (for multi-cycle coefficient output)
// 0 → latch d1/d2, start keccak → 1
// 1 → output d1 if accepted → 2
// 2 → output d2 if accepted → WAIT
// ================================================================
reg [1:0] sq_phase_r;
// ================================================================
// Comb: build absorb state from rho, i, j INPUT PORTS directly
// (not registered copies — avoids NBA race on IDLE→ABSORB edge)
// ================================================================
wire [7:0] abs_i_byte, abs_j_byte;
assign abs_i_byte = {6'b0, i_idx};
assign abs_j_byte = {6'b0, j_idx};
wire [271:0] msg_bytes;
assign msg_bytes = {abs_i_byte, abs_j_byte, rho_i};
// SHAKE-128 absorb block: capacity(256b) | pad10*1 | suffix(1111) | msg(272b)
wire [1599:0] absorb_state;
assign absorb_state = {
256'b0, // capacity [1599:1344]
1'b1, // pad10*1 final 1 [1343]
{1066{1'b0}}, // pad10*1 zeros [1342:277]
1'b1, // pad10*1 first 1 [276]
4'b1111, // SHAKE suffix [275:272]
msg_bytes // message [271:0]
};
// ================================================================
// Comb: extract d1,d2 from the current 3-byte group of squeeze state
// ================================================================
// group g occupies bits [24*g +: 24]: c0=byte0, c1=byte1, c2=byte2
// d1 = {c1[3:0], c0}
// d2 = {c2, c1[7:4]}
wire [10:0] grp_bit_off;
assign grp_bit_off = grp_ptr_r * 11'd24; // 0..1320, +24 ≤ 1344 (rate)
wire [23:0] grp_bits;
assign grp_bits = squeeze_state_r[ grp_bit_off +: 24 ];
wire [7:0] c0, c1, c2;
assign c0 = grp_bits[7:0];
assign c1 = grp_bits[15:8];
assign c2 = grp_bits[23:16];
wire [11:0] d1_comb, d2_comb;
assign d1_comb = {c1[3:0], c0};
assign d2_comb = {c2, c1[7:4]};
wire d1_ok, d2_ok;
assign d1_ok = (d1_comb < Q);
assign d2_ok = (d2_comb < Q);
// ================================================================
// Keccak interface: external ports instead of internal keccak_core.
//
// kc_valid_i: asserted during ABSORB and first phase of SQUEEZE.
// kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise.
// kc_ready_i: always 1'b1 — this module always accepts keccak output.
//
// These signals are output ports driven by the combinational logic
// below. The upstream arbiter connects them to a shared keccak_core.
// ================================================================
// kc_ready_i: always ready to accept keccak output
assign kc_ready_i = 1'b1;
// kc_valid_i: asserted on the ABSORB load, and for one cycle when the
// squeeze block is exhausted (SQUEEZE → WAIT) to re-permute the state.
assign kc_valid_i = (state_next == ST_ABSORB) ||
(state_r == ST_SQUEEZE && state_next == ST_WAIT);
// kc_state_i: absorb_state in ABSORB, squeeze_state_r otherwise
assign kc_state_i = (state_next == ST_ABSORB) ? absorb_state : squeeze_state_r;
// ================================================================
// Output signals
// ================================================================
reg [11:0] coeff_o_r;
reg valid_o_r;
reg last_o_r;
assign coeff_o = coeff_o_r;
assign valid_o = valid_o_r;
assign last_o = last_o_r;
// ================================================================
// ready_o: accept new request in IDLE
// ================================================================
assign ready_o = (state_r == ST_IDLE);
wire need_more = (coeff_cnt_r < 9'd256);
// grp_done: current 3-byte group fully consumed (phase 2 resolved):
// d2 rejected, no longer need coefficients, or d2 was just accepted out.
wire grp_done = (state_r == ST_SQUEEZE) && (sq_phase_r == 2'd2) &&
(!d2_acc_r || !need_more || (valid_o_r && ready_i));
// ================================================================
// FSM: state_next (combinational)
// ================================================================
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: begin
if (valid_i && ready_o)
state_next = ST_ABSORB;
end
ST_ABSORB: begin
// Wait for keccak to finish the absorb permutation
if (kc_valid_o)
state_next = ST_SQUEEZE;
end
ST_SQUEEZE: begin
// A group is fully consumed once phase 2 resolves (d2 output,
// rejected, or no longer needed). Then either advance to the
// next group in this block, re-permute (block exhausted), or
// finish.
if (grp_done) begin
if (!need_more)
state_next = ST_DONE;
else if (grp_ptr_r < GRP_MAX)
state_next = ST_SQUEEZE; // next group, no re-permute
else
state_next = ST_WAIT; // block exhausted: re-permute
end
end
ST_WAIT: begin
// Wait for keccak to finish squeeze permutation
if (kc_valid_o) begin
if (!need_more)
state_next = ST_DONE;
else
state_next = ST_SQUEEZE;
end
end
ST_DONE: begin
state_next = ST_IDLE;
end
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// FSM: sequential logic (registered)
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
sq_phase_r <= 2'd0;
coeff_cnt_r <= 9'd0;
squeeze_state_r <= 1600'd0;
grp_ptr_r <= 6'd0;
d1_r <= 12'd0;
d2_r <= 12'd0;
d1_acc_r <= 1'b0;
d2_acc_r <= 1'b0;
coeff_o_r <= 12'd0;
valid_o_r <= 1'b0;
last_o_r <= 1'b0;
rho_r <= 256'd0;
k_r <= 3'd0;
i_r <= 2'd0;
j_r <= 2'd0;
end else begin
state_r <= state_next;
// ---------------------------------------------------------
// Capture inputs on valid_i (IDLE → ABSORB transition)
// ---------------------------------------------------------
if (state_r == ST_IDLE && valid_i && ready_o) begin
rho_r <= rho_i;
k_r <= k_i;
i_r <= i_idx;
j_r <= j_idx;
coeff_cnt_r <= 9'd0;
end
// ---------------------------------------------------------
// Latch keccak output when valid_o fires. A fresh block
// starts at group 0 (ABSORB load or WAIT re-permute result).
// ---------------------------------------------------------
if (kc_valid_o) begin
squeeze_state_r <= kc_state_o;
grp_ptr_r <= 6'd0;
end
// ---------------------------------------------------------
// SQ_PHASE = 0: latch d1/d2 acceptance
// ---------------------------------------------------------
if (state_r == ST_SQUEEZE && sq_phase_r == 2'd0) begin
d1_r <= d1_comb;
d2_r <= d2_comb;
d1_acc_r <= d1_ok;
d2_acc_r <= d2_ok;
end
// ---------------------------------------------------------
// SQ_PHASE management and coefficient output
//
// Handshake pattern (matching pipeline_reg convention):
// Cycle N: valid_o_r ← 1, coeff_o_r ← value
// Cycle N+1: if ready_i: consume, valid_o_r ← 0, advance
//
// The testbench reads coeff_o after Cycle N and consumes
// with the next posedge (Cycle N+1).
// ---------------------------------------------------------
if (state_r == ST_SQUEEZE) begin
case (sq_phase_r)
// ----- Phase 0: latch, advance to phase 1 -----
2'd0: begin
sq_phase_r <= 2'd1;
end
// ----- Phase 1: output d1 -----
2'd1: begin
if (d1_acc_r && need_more) begin
if (!valid_o_r) begin
// First cycle: assert output
coeff_o_r <= d1_r;
valid_o_r <= 1'b1;
last_o_r <= (coeff_cnt_r == 9'd255);
end else begin
// valid_o is high; wait for consumer
if (ready_i) begin
coeff_cnt_r <= coeff_cnt_r + 9'd1;
valid_o_r <= 1'b0;
sq_phase_r <= 2'd2;
end
end
end else begin
// d1 rejected or no longer needed: skip to phase 2
valid_o_r <= 1'b0;
sq_phase_r <= 2'd2;
end
end
// ----- Phase 2: output d2 -----
2'd2: begin
if (d2_acc_r && need_more) begin
if (!valid_o_r) begin
// First cycle: assert output
coeff_o_r <= d2_r;
valid_o_r <= 1'b1;
last_o_r <= (coeff_cnt_r == 9'd255);
end else begin
// valid_o is high; wait for consumer
if (ready_i) begin
coeff_cnt_r <= coeff_cnt_r + 9'd1;
valid_o_r <= 1'b0;
// state_next → ST_WAIT (combinational)
end
end
end else begin
// d2 rejected or done
valid_o_r <= 1'b0;
end
end
default: begin
valid_o_r <= 1'b0;
end
endcase
// Group consumed but block not exhausted: advance to the
// next 3-byte group within the same block (no re-permute).
if (grp_done && need_more && grp_ptr_r < GRP_MAX) begin
grp_ptr_r <= grp_ptr_r + 6'd1;
sq_phase_r <= 2'd0;
end
end else if (state_r != ST_SQUEEZE && state_next == ST_SQUEEZE) begin
// About to enter SQUEEZE: reset phase and output
sq_phase_r <= 2'd0;
valid_o_r <= 1'b0;
end else begin
// Not in SQUEEZE: clear
valid_o_r <= 1'b0;
sq_phase_r <= 2'd0;
end
end
end
endmodule

99
rtl/ip/mlkem/sha3/keccak_core.v Normal file
View File

@@ -0,0 +1,99 @@
// keccak_core.v - Synchronous Keccak-f[1600] core with valid/ready
//
// Runs 24 rounds of keccak_round on the input state.
// valid_i high with ready_o high → load state_i, start 24-round sequence.
// Each round takes 1 cycle. After round 23, valid_o asserted with state_o.
//
// Parameter:
// ROUNDS = 24 (default)
//
// Interface:
// clk, rst_n - clock, active-low reset
// state_i - 1600-bit input state
// valid_i - start permutation
// ready_o - core can accept new input
// state_o - 1600-bit output state (after permutation)
// valid_o - output is valid
// ready_i - consumer accepts output
module keccak_core #(parameter ROUNDS = 24) (
input clk,
input rst_n,
input [1599:0] state_i,
input valid_i,
output ready_o,
output [1599:0] state_o,
output valid_o,
input ready_i
);
// ================================================================
// Internal registers
// ================================================================
reg busy_r; // 1 while running permutation
reg [4:0] cnt_r; // round counter (0 to ROUNDS-1)
reg [1599:0] state_r; // current state
// ================================================================
// Next-state logic
// ================================================================
wire cnt_is_last; // cnt_r == ROUNDS-1 ?
wire [1599:0] state_next; // combinational round output
assign cnt_is_last = (cnt_r == (ROUNDS - 1));
// keccak_round: apply one round to current state
keccak_round u_round (
.state_i(state_r),
.round_i(cnt_r),
.state_o(state_next)
);
// ================================================================
// Valid/ready handshake
// ================================================================
// ready_o: accept new input when not busy (or just finishing)
assign ready_o = !busy_r;
// valid_o: output valid when we just finished the last round
// (the final state is already in state_r at the output cycle)
assign valid_o = busy_r && cnt_is_last;
// state_o: output the final state
assign state_o = state_next; // final state from the last round
// ================================================================
// Sequential logic
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
busy_r <= 1'b0;
cnt_r <= 5'd0;
state_r <= 1600'd0;
end else begin
if (!busy_r) begin
// IDLE: wait for valid_i
if (valid_i) begin
state_r <= state_i;
cnt_r <= 5'd0;
busy_r <= 1'b1;
end
end else begin
// BUSY: running permutations
if (cnt_is_last) begin
// Final round: check if output can be accepted
if (ready_i) begin
busy_r <= 1'b0;
cnt_r <= 5'd0;
end
// else: hold until ready_i
end else begin
// Intermediate round: advance state
state_r <= state_next;
cnt_r <= cnt_r + 5'd1;
end
end
end
end
endmodule

248
rtl/ip/mlkem/sha3/keccak_round.v Normal file
View File

@@ -0,0 +1,248 @@
// keccak_round.v - Single Keccak-f round (theta/rho/pi/chi/iota)
//
// Pure combinational: no clock, no valid/ready.
// Input: 1600-bit state, 5-bit round index (0-23).
// Output: transformed 1600-bit state.
//
// State layout: s[64*(5*y + x) + z] = A[x][y][z]
// Lane(x,y) = s[64*(5*y + x) +: 64]
module keccak_round (
input [1599:0] state_i,
input [4:0] round_i,
output [1599:0] state_o
);
// ================================================================
// Round constants (24 entries, FIPS 202)
// ================================================================
wire [63:0] RC [0:23];
assign RC[ 0] = 64'h0000000000000001;
assign RC[ 1] = 64'h0000000000008082;
assign RC[ 2] = 64'h800000000000808A;
assign RC[ 3] = 64'h8000000080008000;
assign RC[ 4] = 64'h000000000000808B;
assign RC[ 5] = 64'h0000000080000001;
assign RC[ 6] = 64'h8000000080008081;
assign RC[ 7] = 64'h8000000000008009;
assign RC[ 8] = 64'h000000000000008A;
assign RC[ 9] = 64'h0000000000000088;
assign RC[10] = 64'h0000000080008009;
assign RC[11] = 64'h000000008000000A;
assign RC[12] = 64'h000000008000808B;
assign RC[13] = 64'h800000000000008B;
assign RC[14] = 64'h8000000000008089;
assign RC[15] = 64'h8000000000008003;
assign RC[16] = 64'h8000000000008002;
assign RC[17] = 64'h8000000000000080;
assign RC[18] = 64'h000000000000800A;
assign RC[19] = 64'h800000008000000A;
assign RC[20] = 64'h8000000080008081;
assign RC[21] = 64'h8000000000008080;
assign RC[22] = 64'h0000000080000001;
assign RC[23] = 64'h8000000080008008;
// ================================================================
// Rho rotation offsets (FIPS 202, Table 2)
// Inlined as generate-if chains for tool compatibility
// ================================================================
// ================================================================
// Unpack flat 1600-bit input into 25 lanes (each 64-bit)
// Lane index: 5*y + x
// ================================================================
wire [63:0] A [0:4][0:4];
genvar gx, gy;
generate
for (gx = 0; gx < 5; gx = gx + 1) begin : up_x
for (gy = 0; gy < 5; gy = gy + 1) begin : up_y
assign A[gx][gy] = state_i[((5 * gy + gx) * 64) +: 64];
end
end
endgenerate
// ================================================================
// Theta step
// C[x] = XOR over y of A[x][y]
// D[x] = C[x-1] ^ ROTL(C[x+1], 1)
// A_theta[x][y] = A[x][y] ^ D[x]
// ================================================================
wire [63:0] C [0:4];
wire [63:0] D [0:4];
wire [63:0] A_t [0:4][0:4];
generate
for (gx = 0; gx < 5; gx = gx + 1) begin : th_x
// C[x] = XOR of the 5 lanes in column x
assign C[gx] = A[gx][0] ^ A[gx][1] ^ A[gx][2] ^ A[gx][3] ^ A[gx][4];
// D[x] = C[(x-1) mod 5] ^ ROTL(C[(x+1) mod 5], 1)
localparam [2:0] xp1 = (gx == 4) ? 3'd0 : (gx + 3'd1);
localparam [2:0] xm1 = (gx == 0) ? 3'd4 : (gx - 3'd1);
assign D[gx] = C[xm1] ^ {C[xp1][62:0], C[xp1][63]};
for (gy = 0; gy < 5; gy = gy + 1) begin : th_y
assign A_t[gx][gy] = A[gx][gy] ^ D[gx];
end
end
endgenerate
// ================================================================
// Rho step (rotation of each lane by precomputed offset)
// ================================================================
wire [63:0] A_r [0:4][0:4];
generate
for (gx = 0; gx < 5; gx = gx + 1) begin : rh_x
for (gy = 0; gy < 5; gy = gy + 1) begin : rh_y
if (gx == 0 && gy == 0) begin : rho_00
assign A_r[gx][gy] = A_t[gx][gy];
end else
if (gy == 0 && gx == 1) begin : rho_10
assign A_r[gx][gy] = {A_t[gx][gy][62:0], A_t[gx][gy][63]};
end else
if (gy == 0 && gx == 2) begin : rho_20
assign A_r[gx][gy] = {A_t[gx][gy][1:0], A_t[gx][gy][63:2]};
end else
if (gy == 0 && gx == 3) begin : rho_30
assign A_r[gx][gy] = {A_t[gx][gy][35:0], A_t[gx][gy][63:36]};
end else
if (gy == 0 && gx == 4) begin : rho_40
assign A_r[gx][gy] = {A_t[gx][gy][36:0], A_t[gx][gy][63:37]};
end else
if (gy == 1 && gx == 0) begin : rho_01
assign A_r[gx][gy] = {A_t[gx][gy][27:0], A_t[gx][gy][63:28]};
end else
if (gy == 1 && gx == 1) begin : rho_11
assign A_r[gx][gy] = {A_t[gx][gy][19:0], A_t[gx][gy][63:20]};
end else
if (gy == 1 && gx == 2) begin : rho_21
assign A_r[gx][gy] = {A_t[gx][gy][57:0], A_t[gx][gy][63:58]};
end else
if (gy == 1 && gx == 3) begin : rho_31
assign A_r[gx][gy] = {A_t[gx][gy][8:0], A_t[gx][gy][63:9]};
end else
if (gy == 1 && gx == 4) begin : rho_41
assign A_r[gx][gy] = {A_t[gx][gy][43:0], A_t[gx][gy][63:44]};
end else
if (gy == 2 && gx == 0) begin : rho_02
assign A_r[gx][gy] = {A_t[gx][gy][60:0], A_t[gx][gy][63:61]};
end else
if (gy == 2 && gx == 1) begin : rho_12
assign A_r[gx][gy] = {A_t[gx][gy][53:0], A_t[gx][gy][63:54]};
end else
if (gy == 2 && gx == 2) begin : rho_22
assign A_r[gx][gy] = {A_t[gx][gy][20:0], A_t[gx][gy][63:21]};
end else
if (gy == 2 && gx == 3) begin : rho_32
assign A_r[gx][gy] = {A_t[gx][gy][38:0], A_t[gx][gy][63:39]};
end else
if (gy == 2 && gx == 4) begin : rho_42
assign A_r[gx][gy] = {A_t[gx][gy][24:0], A_t[gx][gy][63:25]};
end else
if (gy == 3 && gx == 0) begin : rho_03
assign A_r[gx][gy] = {A_t[gx][gy][22:0], A_t[gx][gy][63:23]};
end else
if (gy == 3 && gx == 1) begin : rho_13
assign A_r[gx][gy] = {A_t[gx][gy][18:0], A_t[gx][gy][63:19]};
end else
if (gy == 3 && gx == 2) begin : rho_23
assign A_r[gx][gy] = {A_t[gx][gy][48:0], A_t[gx][gy][63:49]};
end else
if (gy == 3 && gx == 3) begin : rho_33
assign A_r[gx][gy] = {A_t[gx][gy][42:0], A_t[gx][gy][63:43]};
end else
if (gy == 3 && gx == 4) begin : rho_43
assign A_r[gx][gy] = {A_t[gx][gy][55:0], A_t[gx][gy][63:56]};
end else
if (gy == 4 && gx == 0) begin : rho_04
assign A_r[gx][gy] = {A_t[gx][gy][45:0], A_t[gx][gy][63:46]};
end else
if (gy == 4 && gx == 1) begin : rho_14
assign A_r[gx][gy] = {A_t[gx][gy][61:0], A_t[gx][gy][63:62]};
end else
if (gy == 4 && gx == 2) begin : rho_24
assign A_r[gx][gy] = {A_t[gx][gy][2:0], A_t[gx][gy][63:3]};
end else
if (gy == 4 && gx == 3) begin : rho_34
assign A_r[gx][gy] = {A_t[gx][gy][7:0], A_t[gx][gy][63:8]};
end else
if (gy == 4 && gx == 4) begin : rho_44
assign A_r[gx][gy] = {A_t[gx][gy][49:0], A_t[gx][gy][63:50]};
end
end
end
endgenerate
// ================================================================
// Pi step (permute lanes)
// A_pi[x][y] = A_rho[(x + 3*y) % 5][x]
// ================================================================
wire [63:0] A_p [0:4][0:4];
generate
for (gx = 0; gx < 5; gx = gx + 1) begin : pi_x
for (gy = 0; gy < 5; gy = gy + 1) begin : pi_y
localparam [2:0] src_x = (gx + 3 * gy) % 5;
assign A_p[gx][gy] = A_r[src_x][gx];
end
end
endgenerate
// ================================================================
// Chi step
// A_chi[x][y] = A_pi[x][y] ^ (~A_pi[x+1][y] & A_pi[x+2][y])
// ================================================================
wire [63:0] A_c [0:4][0:4];
generate
for (gx = 0; gx < 5; gx = gx + 1) begin : ch_x
localparam [2:0] xp1 = (gx == 4) ? 3'd0 : (gx + 3'd1);
localparam [2:0] xp2 = (gx == 3) ? 3'd0 :
(gx == 4) ? 3'd1 : (gx + 3'd2);
for (gy = 0; gy < 5; gy = gy + 1) begin : ch_y
assign A_c[gx][gy] = A_p[gx][gy] ^
((~A_p[xp1][gy]) & A_p[xp2][gy]);
end
end
endgenerate
// ================================================================
// Iota step
// A_iota[0][0] = A_chi[0][0] ^ RC[round_i]
// Other lanes unchanged
// ================================================================
wire [63:0] A_io [0:4][0:4];
assign A_io[0][0] = A_c[0][0] ^ RC[round_i];
assign A_io[0][1] = A_c[0][1];
assign A_io[0][2] = A_c[0][2];
assign A_io[0][3] = A_c[0][3];
assign A_io[0][4] = A_c[0][4];
assign A_io[1][0] = A_c[1][0];
assign A_io[1][1] = A_c[1][1];
assign A_io[1][2] = A_c[1][2];
assign A_io[1][3] = A_c[1][3];
assign A_io[1][4] = A_c[1][4];
assign A_io[2][0] = A_c[2][0];
assign A_io[2][1] = A_c[2][1];
assign A_io[2][2] = A_c[2][2];
assign A_io[2][3] = A_c[2][3];
assign A_io[2][4] = A_c[2][4];
assign A_io[3][0] = A_c[3][0];
assign A_io[3][1] = A_c[3][1];
assign A_io[3][2] = A_c[3][2];
assign A_io[3][3] = A_c[3][3];
assign A_io[3][4] = A_c[3][4];
assign A_io[4][0] = A_c[4][0];
assign A_io[4][1] = A_c[4][1];
assign A_io[4][2] = A_c[4][2];
assign A_io[4][3] = A_c[4][3];
assign A_io[4][4] = A_c[4][4];
// ================================================================
// Pack 25 lanes back into 1600-bit flat output
// ================================================================
assign state_o = {A_io[4][4], A_io[3][4], A_io[2][4], A_io[1][4], A_io[0][4],
A_io[4][3], A_io[3][3], A_io[2][3], A_io[1][3], A_io[0][3],
A_io[4][2], A_io[3][2], A_io[2][2], A_io[1][2], A_io[0][2],
A_io[4][1], A_io[3][1], A_io[2][1], A_io[1][1], A_io[0][1],
A_io[4][0], A_io[3][0], A_io[2][0], A_io[1][0], A_io[0][0]};
endmodule

215
rtl/ip/mlkem/sha3/sha3_top.v Normal file
View File

@@ -0,0 +1,215 @@
// sha3_top.v - SHA3/SHAKE top wrapper with valid/ready interface
//
// Implements SHA3-512 (G), SHA3-256 (H), SHAKE-256 (J) over a single
// Keccak-f[1600] core. Supports single-block absorption, plus a separate
// multi-block SHA3-256 streaming absorb path for long inputs (e.g. H(ek)).
//
// Single-block modes (mb_en=0):
// 00 = G (SHA3-512): rate=576, suffix=01, msg_len=264, out=512
// 01 = H (SHA3-256): rate=1088, suffix=01, msg_len=256, out=256
// 10 = J (SHAKE-256): rate=1088, suffix=1111,msg_len=512,out=256
//
// Multi-block SHA3-256 (mb_en=1):
// Streams pre-padded 1088-bit rate blocks. The CALLER applies SHA3-256
// padding (suffix 0x06 ... 0x80) to the final block so this module only
// does the absorb loop: state ^= block; state = Keccak-p(state); repeat.
// The single-block paths above are bit-identical when mb_en=0.
//
// Interface (single-block):
// mode[1:0] - 00=G, 01=H, 10=J
// data_i - 512-bit message input
// valid_i / ready_o / hash_o[511:0] / valid_o / ready_i
//
// Interface (multi-block SHA3-256, active when mb_en=1):
// mb_en - 1 selects the multi-block absorb path
// mb_block_i - one pre-padded 1088-bit (136-byte) rate block, byte 0 in [7:0]
// mb_valid_i - block valid
// mb_ready_o - module can accept a block
// mb_last_i - asserted with the final (already-padded) block
// result reuses hash_o[255:0] / valid_o / ready_i
module sha3_top (
input clk,
input rst_n,
input [1:0] mode,
input [511:0] data_i,
input valid_i,
output ready_o,
output [511:0] hash_o,
output valid_o,
input ready_i,
// --- multi-block SHA3-256 absorb (tie mb_en=0 to disable) ---
input mb_en,
input [1087:0] mb_block_i,
input mb_valid_i,
input mb_last_i,
output mb_ready_o
);
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 2'd0;
localparam ST_PERMUTE = 2'd1;
localparam ST_SQUEEZE = 2'd2;
reg [1:0] state_r, state_next;
// ================================================================
// Absorb state: message || suffix || pad10*1 into rate bits
// Built combinationally from data_i and mode.
//
// G: padded_block = {1, 308'b0, 1, 2'b01, data_i[263:0]}
// absorb_state = {1024'b0, padded_block_576}
//
// H: padded_block = {1, 828'b0, 1, 2'b01, data_i[255:0]}
// absorb_state = {512'b0, padded_block_1088}
//
// J: padded_block = {1, 570'b0, 1, 4'b1111, data_i[511:0]}
// absorb_state = {512'b0, padded_block_1088}
// ================================================================
wire [575:0] g_pad;
wire [1087:0] h_pad;
wire [1087:0] j_pad;
assign g_pad = {1'b1, {308{1'b0}}, 1'b1, 2'b10, data_i[263:0]};
assign h_pad = {1'b1, {828{1'b0}}, 1'b1, 2'b10, data_i[255:0]};
// J: SHAKE suffix is "1111" — all ones, order irrelevant
assign j_pad = {1'b1, {570{1'b0}}, 1'b1, 4'b1111, data_i[511:0]};
wire [1599:0] absorb_state;
assign absorb_state = (mode == 2'b00) ? {{(1600-576){1'b0}}, g_pad} :
(mode == 2'b01) ? {{(1600-1088){1'b0}}, h_pad} :
(mode == 2'b10) ? {{(1600-1088){1'b0}}, j_pad} :
1600'd0;
// ================================================================
// Multi-block SHA3-256 absorb FSM (active only when mb_en=1)
//
// Running sponge state mb_state_r (init 0). For each pre-padded
// 1088-bit rate block: mb_state_r ^= block; mb_state_r = Keccak-p(...).
// After the last block, squeeze 256 bits. The caller pads the final
// block (SHA3-256 suffix 0x06 ... 0x80), so this FSM is pure absorb.
// ================================================================
localparam MB_IDLE = 2'd0; // ready for a block (or first block)
localparam MB_PERMUTE = 2'd1; // keccak running on xored state
localparam MB_DONE = 2'd2; // squeeze: present 256-bit digest
reg [1:0] mb_state, mb_state_next;
reg [1599:0] mb_state_r; // running sponge state
reg mb_last_r; // captured last-block flag
reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE)
// XOR the incoming block into the low 1088 bits (rate) of the state.
wire [1599:0] mb_xored;
assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i};
// Accept a block only in MB_IDLE while enabled.
assign mb_ready_o = mb_en && (mb_state == MB_IDLE);
wire mb_accept = mb_en && (mb_state == MB_IDLE) && mb_valid_i;
wire mb_kc_valid = mb_accept; // start keccak on the accept cycle
always @(*) begin
mb_state_next = mb_state;
case (mb_state)
MB_IDLE: if (mb_accept) mb_state_next = MB_PERMUTE;
MB_PERMUTE: if (kc_valid_o) mb_state_next = mb_last_r ? MB_DONE : MB_IDLE;
MB_DONE: if (ready_i) mb_state_next = MB_IDLE;
default: mb_state_next = MB_IDLE;
endcase
end
// ================================================================
// Keccak core
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire kc_valid_o;
// Keccak input: multi-block xored state when mb_en, else single-block absorb.
wire [1599:0] kc_state_i_mux;
assign kc_state_i_mux = mb_en ? mb_xored : absorb_state;
keccak_core #(.ROUNDS(24)) u_keccak (
.clk (clk),
.rst_n (rst_n),
.state_i (kc_state_i_mux),
.valid_i (kc_valid_i),
.ready_o (kc_ready_o), // unused but must connect
.state_o (kc_state_o),
.valid_o (kc_valid_o),
.ready_i (1'b1) // always accept output
);
// ================================================================
// FSM combinational logic
// ================================================================
assign ready_o = !mb_en && (state_r == ST_IDLE);
// kc_valid_i: single-block start (state_next==PERMUTE) OR multi-block accept.
assign kc_valid_i = mb_en ? mb_kc_valid : (state_next == ST_PERMUTE);
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: if (valid_i && ready_o) state_next = ST_PERMUTE;
ST_PERMUTE: if (kc_valid_o) state_next = ST_SQUEEZE;
ST_SQUEEZE: if (valid_o && ready_i) state_next = ST_IDLE;
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// Output
// ================================================================
// Register for squeezed output (only 512 bits needed)
reg [511:0] squeezed_state_r;
// valid_o / hash_o serve both paths, selected by mb_en.
assign valid_o = mb_en ? (mb_state == MB_DONE) : (state_r == ST_SQUEEZE);
assign hash_o = mb_en ? {256'b0, mb_digest_r} : squeezed_state_r;
// ================================================================
// Sequential logic
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
squeezed_state_r <= 512'd0;
mb_state <= MB_IDLE;
mb_state_r <= 1600'd0;
mb_last_r <= 1'b0;
mb_digest_r <= 256'd0;
end else begin
state_r <= state_next;
mb_state <= mb_state_next;
// --- single-block: latch squeezed output ---
if (state_r == ST_PERMUTE && kc_valid_o) begin
squeezed_state_r <= kc_state_o[511:0];
end
// --- multi-block: capture last flag on accept ---
if (mb_accept) begin
mb_last_r <= mb_last_i;
end
// --- multi-block: latch permuted state when keccak finishes ---
if (mb_state == MB_PERMUTE && kc_valid_o) begin
mb_state_r <= kc_state_o;
// On the final block, latch the 256-bit digest (sticky for MB_DONE).
if (mb_last_r) mb_digest_r <= kc_state_o[255:0];
end
// --- multi-block: clear running state after digest consumed ---
if (mb_state == MB_DONE && ready_i) begin
mb_state_r <= 1600'd0;
end
end
end
endmodule

187
rtl/ip/mlkem/sha3/sha3_top_shared.v Normal file
View File

@@ -0,0 +1,187 @@
// sha3_top_shared.v - SHA3/SHAKE wrapper with EXTERNAL (shared) keccak_core.
//
// Identical to sha3_top.v except the internal keccak_core instance is
// replaced by explicit kc_* ports, so several Keccak consumers can share a
// single keccak_core via a phase mux (see mlkem_top). The sponge state
// (mb_state_r) and all FSM/squeeze logic stay here; only the 24-round
// permutation is external. Bit-identical to sha3_top when wired to a
// dedicated keccak_core.
//
// Single-block modes (mb_en=0):
// 00 = G (SHA3-512): rate=576, suffix=01, msg_len=264, out=512
// 01 = H (SHA3-256): rate=1088, suffix=01, msg_len=256, out=256
// 10 = J (SHAKE-256): rate=1088, suffix=1111,msg_len=512,out=256
//
// Multi-block SHA3-256 (mb_en=1): streams pre-padded 1088-bit rate blocks;
// the CALLER applies SHA3-256 padding to the final block.
//
// Keccak interface (port names mirror the keccak_core port each connects to):
// kc_state_o[1599:0] - keccak result (input, from keccak_core.state_o)
// kc_valid_o - keccak output valid (input, from keccak_core.valid_o)
// kc_ready_o - keccak ready for input (input, from keccak_core.ready_o)
// kc_state_i[1599:0] - keccak input state (output, to keccak_core.state_i)
// kc_valid_i - request permutation (output, to keccak_core.valid_i)
// kc_ready_i - accept keccak output (output, to keccak_core.ready_i, always 1)
module sha3_top_shared (
input clk,
input rst_n,
input [1:0] mode,
input [511:0] data_i,
input valid_i,
output ready_o,
output [511:0] hash_o,
output valid_o,
input ready_i,
// --- multi-block SHA3-256 absorb (tie mb_en=0 to disable) ---
input mb_en,
input [1087:0] mb_block_i,
input mb_valid_i,
input mb_last_i,
output mb_ready_o,
// --- external (shared) keccak_core interface ---
input [1599:0] kc_state_o,
input kc_valid_o,
/* verilator lint_off UNUSEDSIGNAL */
input kc_ready_o,
/* verilator lint_on UNUSEDSIGNAL */
output [1599:0] kc_state_i,
output kc_valid_i,
output kc_ready_i
);
// ================================================================
// FSM state encoding
// ================================================================
localparam ST_IDLE = 2'd0;
localparam ST_PERMUTE = 2'd1;
localparam ST_SQUEEZE = 2'd2;
reg [1:0] state_r, state_next;
// ================================================================
// Absorb state: message || suffix || pad10*1 into rate bits
// ================================================================
wire [575:0] g_pad;
wire [575:0] g512_pad;
wire [1087:0] h_pad;
wire [1087:0] j_pad;
assign g_pad = {1'b1, {308{1'b0}}, 1'b1, 2'b10, data_i[263:0]};
assign h_pad = {1'b1, {828{1'b0}}, 1'b1, 2'b10, data_i[255:0]};
// J: SHAKE suffix is "1111" — all ones, order irrelevant
assign j_pad = {1'b1, {570{1'b0}}, 1'b1, 4'b1111, data_i[511:0]};
// mode 11 = G over a full 64-byte (512-bit) message (Encaps: G(m||H(ek))).
// SHA3-512 rate=576; suffix 01 + pad10*1: 1 + 60 zeros + 1 + 2'b10 + 512 = 576.
assign g512_pad = {1'b1, {60{1'b0}}, 1'b1, 2'b10, data_i[511:0]};
wire [1599:0] absorb_state;
assign absorb_state = (mode == 2'b00) ? {{(1600-576){1'b0}}, g_pad} :
(mode == 2'b01) ? {{(1600-1088){1'b0}}, h_pad} :
(mode == 2'b10) ? {{(1600-1088){1'b0}}, j_pad} :
{{(1600-576){1'b0}}, g512_pad};
// ================================================================
// Multi-block SHA3-256 absorb FSM (active only when mb_en=1)
// ================================================================
localparam MB_IDLE = 2'd0; // ready for a block (or first block)
localparam MB_PERMUTE = 2'd1; // keccak running on xored state
localparam MB_DONE = 2'd2; // squeeze: present 256-bit digest
reg [1:0] mb_state, mb_state_next;
reg [1599:0] mb_state_r; // running sponge state
reg mb_last_r; // captured last-block flag
reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE)
// XOR the incoming block into the low 1088 bits (rate) of the state.
wire [1599:0] mb_xored;
assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i};
// Accept a block only in MB_IDLE while enabled.
assign mb_ready_o = mb_en && (mb_state == MB_IDLE);
wire mb_accept = mb_en && (mb_state == MB_IDLE) && mb_valid_i;
wire mb_kc_valid = mb_accept; // start keccak on the accept cycle
always @(*) begin
mb_state_next = mb_state;
case (mb_state)
MB_IDLE: if (mb_accept) mb_state_next = MB_PERMUTE;
MB_PERMUTE: if (kc_valid_o) mb_state_next = mb_last_r ? MB_DONE : MB_IDLE;
MB_DONE: if (ready_i) mb_state_next = MB_IDLE;
default: mb_state_next = MB_IDLE;
endcase
end
// ================================================================
// External keccak_core interface (was internal keccak_core)
// ================================================================
// Keccak input: multi-block xored state when mb_en, else single-block absorb.
assign kc_state_i = mb_en ? mb_xored : absorb_state;
// Always accept keccak output (matches the dedicated-core ready_i=1'b1).
assign kc_ready_i = 1'b1;
// kc_valid_i: single-block start (state_next==PERMUTE) OR multi-block accept.
assign kc_valid_i = mb_en ? mb_kc_valid : (state_next == ST_PERMUTE);
// ================================================================
// FSM combinational logic
// ================================================================
assign ready_o = !mb_en && (state_r == ST_IDLE);
always @(*) begin
state_next = state_r;
case (state_r)
ST_IDLE: if (valid_i && ready_o) state_next = ST_PERMUTE;
ST_PERMUTE: if (kc_valid_o) state_next = ST_SQUEEZE;
ST_SQUEEZE: if (valid_o && ready_i) state_next = ST_IDLE;
default: state_next = ST_IDLE;
endcase
end
// ================================================================
// Output
// ================================================================
reg [511:0] squeezed_state_r;
// valid_o / hash_o serve both paths, selected by mb_en.
assign valid_o = mb_en ? (mb_state == MB_DONE) : (state_r == ST_SQUEEZE);
assign hash_o = mb_en ? {256'b0, mb_digest_r} : squeezed_state_r;
// ================================================================
// Sequential logic
// ================================================================
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state_r <= ST_IDLE;
squeezed_state_r <= 512'd0;
mb_state <= MB_IDLE;
mb_state_r <= 1600'd0;
mb_last_r <= 1'b0;
mb_digest_r <= 256'd0;
end else begin
state_r <= state_next;
mb_state <= mb_state_next;
// --- single-block: latch squeezed output ---
if (state_r == ST_PERMUTE && kc_valid_o) begin
squeezed_state_r <= kc_state_o[511:0];
end
// --- multi-block: capture last flag on accept ---
if (mb_accept) begin
mb_last_r <= mb_last_i;
end
// --- multi-block: latch permuted state when keccak finishes ---
if (mb_state == MB_PERMUTE && kc_valid_o) begin
mb_state_r <= kc_state_o;
if (mb_last_r) mb_digest_r <= kc_state_o[255:0];
end
// --- multi-block: clear running state after digest consumed ---
if (mb_state == MB_DONE && ready_i) begin
mb_state_r <= 1600'd0;
end
end
end
endmodule

31
rtl/ip/mlkem/storage/s_bram.v Normal file
View File

@@ -0,0 +1,31 @@
// s_bram.v - Single-port behavioral RAM (shared read/write port)
//
// One port: either read or write per cycle (+1 cycle read latency).
// When both rd_en and wr_en are high, write takes priority.
// Read data is registered (1-cycle latency after rd_en).
// No valid/ready handshake — pure memory, handshake managed at higher level.
module s_bram #(
parameter W = 48,
parameter D = 512,
parameter A = 9
) (
input wire clk,
input wire rd_en,
input wire [A-1:0] rd_addr,
output reg [W-1:0] rd_data,
input wire wr_en,
input wire [A-1:0] wr_addr,
input wire [W-1:0] wr_data
);
reg [W-1:0] mem [0:D-1];
always @(posedge clk) begin
if (wr_en)
mem[wr_addr] <= wr_data;
else if (rd_en)
rd_data <= mem[rd_addr];
end
endmodule

32
rtl/ip/mlkem/storage/sd_bram.v Normal file
View File

@@ -0,0 +1,32 @@
// sd_bram.v - Simple dual-port behavioral RAM (1 read + 1 write port)
//
// Vivado auto-infers this as BRAM when W >= 12 and D >= 64.
// Read: rd_addr sampled at posedge → rd_addr_r → rd_data = mem[rd_addr_r]
// Write: wr_data written to mem[wr_addr] at posedge when wr_en=1
// No valid/ready handshake — pure combinational-read / registered-write memory.
module sd_bram #(
parameter W = 48,
parameter D = 64,
parameter A = 6
) (
input wire clk,
input wire [A-1:0] rd_addr,
output wire [W-1:0] rd_data,
input wire wr_en,
input wire [A-1:0] wr_addr,
input wire [W-1:0] wr_data
);
reg [W-1:0] mem [0:D-1];
reg [A-1:0] rd_addr_r;
always @(posedge clk) begin
rd_addr_r <= rd_addr;
if (wr_en)
mem[wr_addr] <= wr_data;
end
assign rd_data = mem[rd_addr_r];
endmodule

524
rtl/ip/mlkem/top/mlkem_axi_slave.v Normal file
View File

@@ -0,0 +1,524 @@
// mlkem_axi_slave.v — AXI4 (burst) slave wrapper for the ML-KEM core.
// Makes the ML-KEM accelerator (KeyGen / Encaps / Decaps) CPU-controllable.
//
// The bus runs at sys_clk (50 MHz). The ML-KEM core cannot close timing at
// 50 MHz, so it is clocked by mlkem_clk = sys_clk / 2 (25 MHz), produced by a
// toggle flip-flop in this wrapper. mlkem_clk is a phase-aligned synchronous
// divide of sys_clk; every signal crossing between the two domains still goes
// through an explicit synchronizer or toggle handshake.
// NOTE (FPGA): for real silicon route mlkem_clk through a BUFG/BUFGCE rather
// than a logic-generated net; kept as a plain toggle here for portability.
//
// Register map (byte offsets from base, e.g. virtual 0xbf400000):
// 0x000 CTRL W bit0 = start (launches op selected in CONFIG)
// 0x004 STATUS R bit0 busy, bit1 done, bit2 preload_busy
// W1C write bit1=1 to clear the latched done flag
// 0x008 CONFIG RW [2:0] k (2/3/4), [5:4] op (0=KeyGen 1=Encaps 2=Decaps)
// 0x010..0x02C D RW KeyGen seed d (8 words; d byte0 in word0[7:0])
// 0x030..0x04C Z RW implicit-reject seed z (8 words)
// 0x050..0x06C MSG RW Encaps message m (8 words)
// 0x070..0x08C SS R shared secret out (8 words)
// 0x100 PRELOAD_SEL RW 0=ek_in 1=dk_in 2=c_in
// 0x104 PRELOAD_ADDR RW byte address (auto-increments after each data write)
// 0x108 PRELOAD_DATA W write low byte -> streams to selected port (wait-stated)
// 0x10C RDBK_SEL RW 0=ek 1=dk_pke 2=full-dk 3=ciphertext
// 0x110 RDBK_ADDR RW output byte index
// 0x114 RDBK_DATA R selected readback-tap byte (low 8 bits)
module mlkem_axi_slave (
input clk, // sys_clk (AXI bus clock, 50 MHz)
input resetn, // sys_resetn (active low, already synchronized)
// AXI Write Address Channel
input [4:0] s_axi_awid,
input [31:0] s_axi_awaddr,
input [7:0] s_axi_awlen,
input [2:0] s_axi_awsize,
input [1:0] s_axi_awburst,
input s_axi_awlock,
input [3:0] s_axi_awcache,
input [2:0] s_axi_awprot,
input s_axi_awvalid,
output reg s_axi_awready,
// AXI Write Data Channel
input [4:0] s_axi_wid,
input [31:0] s_axi_wdata,
input [3:0] s_axi_wstrb,
input s_axi_wlast,
input s_axi_wvalid,
output reg s_axi_wready,
// AXI Write Response Channel
output [4:0] s_axi_bid,
output [1:0] s_axi_bresp,
output reg s_axi_bvalid,
input s_axi_bready,
// AXI Read Address Channel
input [4:0] s_axi_arid,
input [31:0] s_axi_araddr,
input [7:0] s_axi_arlen,
input [2:0] s_axi_arsize,
input [1:0] s_axi_arburst,
input s_axi_arlock,
input [3:0] s_axi_arcache,
input [2:0] s_axi_arprot,
input s_axi_arvalid,
output reg s_axi_arready,
// AXI Read Data Channel
output reg [4:0] s_axi_rid,
output reg [31:0] s_axi_rdata,
output [1:0] s_axi_rresp,
output reg s_axi_rlast,
output reg s_axi_rvalid,
input s_axi_rready,
// Interrupt output (active high level, asserted while done latched)
output intr
);
// ==========================================
// Address map (byte offsets within the 1 MB slave window)
// ==========================================
localparam ADDR_CTRL = 12'h000;
localparam ADDR_STATUS = 12'h004;
localparam ADDR_CONFIG = 12'h008;
localparam ADDR_D = 12'h010; // 0x010..0x02C (8 words)
localparam ADDR_Z = 12'h030; // 0x030..0x04C
localparam ADDR_MSG = 12'h050; // 0x050..0x06C
localparam ADDR_SS = 12'h070; // 0x070..0x08C (read-only)
localparam ADDR_PRE_SEL = 12'h100;
localparam ADDR_PRE_ADR = 12'h104;
localparam ADDR_PRE_DAT = 12'h108;
localparam ADDR_RDB_SEL = 12'h10C;
localparam ADDR_RDB_ADR = 12'h110;
localparam ADDR_RDB_DAT = 12'h114;
assign s_axi_bresp = 2'b00;
assign s_axi_rresp = 2'b00;
function [31:0] apply_wstrb;
input [31:0] old_value;
input [31:0] new_value;
input [3:0] strobe;
begin
apply_wstrb = old_value;
if (strobe[0]) apply_wstrb[7:0] = new_value[7:0];
if (strobe[1]) apply_wstrb[15:8] = new_value[15:8];
if (strobe[2]) apply_wstrb[23:16] = new_value[23:16];
if (strobe[3]) apply_wstrb[31:24] = new_value[31:24];
end
endfunction
// ==========================================
// mlkem_clk = sys_clk / 2 (25 MHz) via toggle FF.
// ==========================================
reg mlkem_clk;
always @(posedge clk or negedge resetn) begin
if (!resetn) mlkem_clk <= 1'b0;
else mlkem_clk <= ~mlkem_clk;
end
// Reset for the mlkem domain: reuse sys_resetn (async-assert, and the core's
// own rst_n is sampled on mlkem_clk edges).
wire mlkem_rstn = resetn;
// ==========================================
// sys_clk-domain config / data registers (quasi-static; fed to the core
// combinationally — written long before start, stable across the op).
// ==========================================
reg [2:0] cfg_k; // 2/3/4
reg [1:0] cfg_op; // 0=KeyGen 1=Encaps 2=Decaps
reg [31:0] d_word [0:7];
reg [31:0] z_word [0:7];
reg [31:0] msg_word [0:7];
wire [255:0] d_bus = {d_word[7], d_word[6], d_word[5], d_word[4],
d_word[3], d_word[2], d_word[1], d_word[0]};
wire [255:0] z_bus = {z_word[7], z_word[6], z_word[5], z_word[4],
z_word[3], z_word[2], z_word[1], z_word[0]};
wire [255:0] msg_bus = {msg_word[7],msg_word[6],msg_word[5],msg_word[4],
msg_word[3],msg_word[2],msg_word[1],msg_word[0]};
// Preload / readback index registers (sys domain).
reg [1:0] pre_sel; // 0=ek 1=dk 2=c
reg [11:0] pre_addr; // byte address into the selected port
reg [1:0] rdb_sel; // 0=ek 1=dk_pke 2=full-dk 3=ciphertext
reg [11:0] rdb_addr; // output byte index
// ==========================================
// CDC: start pulse (sys toggle -> mlkem edge-detect -> 1-cyc start_i)
// ==========================================
reg start_tg; // sys-domain toggle, flips on CTRL.start write
reg [2:0] start_sync; // mlkem-domain 2FF sync (+1 for edge)
wire mlkem_start = start_sync[2] ^ start_sync[1]; // 1 mlkem_clk pulse
// ==========================================
// CDC: busy / done back to sys
// ==========================================
wire core_busy; // busy_o (mlkem domain)
wire core_done; // done_o pulse (mlkem domain)
reg done_latch; // sticky done in mlkem domain
reg [1:0] busy_sync; // -> sys
reg [1:0] done_sync; // -> sys
// done-clear: sys toggle -> mlkem edge -> clears done_latch
reg dclr_tg; // sys-domain toggle (STATUS W1C of done)
reg [2:0] dclr_sync; // mlkem-domain 2FF sync (+1 for edge)
wire mlkem_dclr = dclr_sync[2] ^ dclr_sync[1];
// ==========================================
// CDC: preload byte write handshake.
// sys: latch {sel,addr,byte}, flip pre_req_tg, hold wready low.
// mlkem: 2FF sync pre_req_tg, on edge pulse the selected *_in_we 1 cycle,
// flip pre_ack_tg.
// sys: 2FF sync pre_ack_tg, on edge release the bus (addr++).
// ==========================================
reg pre_req_tg; // sys toggle: a byte is pending
reg [2:0] pre_req_sync; // mlkem 2FF (+1 for edge)
wire mlkem_pre_we = pre_req_sync[2] ^ pre_req_sync[1];
reg pre_ack_tg; // mlkem toggle: byte consumed
reg [2:0] pre_ack_sync; // sys 2FF (+1 for edge)
wire pre_ack_edge = pre_ack_sync[2] ^ pre_ack_sync[1];
reg pre_busy; // sys: a preload handshake is in flight
// latched byte/port for the in-flight preload (sys, stable while pre_busy)
reg [1:0] pre_sel_l;
reg [11:0] pre_addr_l;
reg [7:0] pre_byte_l;
// ==========================================
// mlkem_top instance
// ==========================================
wire [7:0] dbg_byte_o, dbg_dk_o, dbg_ct_o;
wire [255:0] ss_o;
wire dbg_byte_sel = (rdb_sel == 2'd1); // 0=ek, 1=dk_pke
// one-cycle write-enable pulses into the core (combinational off the synced
// edge; pre_sel_l/pre_addr_l/pre_byte_l are stable in sys, sampled by core).
wire ek_in_we = mlkem_pre_we && (pre_sel_l == 2'd0);
wire dk_in_we = mlkem_pre_we && (pre_sel_l == 2'd1);
wire c_in_we = mlkem_pre_we && (pre_sel_l == 2'd2);
mlkem_top #(.KMAX(4)) u_mlkem (
.clk (mlkem_clk),
.rst_n (mlkem_rstn),
.k_i (cfg_k),
.op_i (cfg_op),
.d_i (d_bus),
.z_i (z_bus),
.msg_i (msg_bus),
.start_i (mlkem_start),
.busy_o (core_busy),
.done_o (core_done),
// ek preload
.ek_in_we (ek_in_we),
.ek_in_addr (pre_addr_l[10:0]),
.ek_in_byte (pre_byte_l),
// dk preload
.dk_in_we (dk_in_we),
.dk_in_addr (pre_addr_l),
.dk_in_byte (pre_byte_l),
// ciphertext preload
.c_in_we (c_in_we),
.c_in_addr (pre_addr_l[10:0]),
.c_in_byte (pre_byte_l),
// shared secret out
.ss_o (ss_o),
// ciphertext readback tap
.dbg_ct_idx_i (rdb_addr[10:0]),
.dbg_ct_o (dbg_ct_o),
// coefficient debug tap (unused by CPU path; tie inputs off)
.dbg_slot_i (6'd0),
.dbg_idx_i (8'd0),
.dbg_coeff_o (/* unused */),
// ek / dk_pke byte readback
.dbg_byte_sel_i (dbg_byte_sel),
.dbg_byte_idx_i (rdb_addr[10:0]),
.dbg_byte_o (dbg_byte_o),
// full-dk byte readback
.dbg_dk_idx_i (rdb_addr),
.dbg_dk_o (dbg_dk_o),
// remaining debug taps unused
.dbg_rho_o (/* unused */),
.dbg_sigma_o (/* unused */),
.dbg_r_o (/* unused */),
.dbg_hek_o (/* unused */),
.dbg_mprime_o (/* unused */),
.dbg_kbar_o (/* unused */),
.dbg_decz_o (/* unused */),
.dbg_dech_o (/* unused */)
);
// ==========================================
// mlkem_clk domain: synchronizers, done latch, preload-ack toggle.
// ==========================================
always @(posedge mlkem_clk or negedge mlkem_rstn) begin
if (!mlkem_rstn) begin
start_sync <= 3'd0;
dclr_sync <= 3'd0;
pre_req_sync <= 3'd0;
pre_ack_tg <= 1'b0;
done_latch <= 1'b0;
end else begin
// 2FF (+edge) synchronizers of sys-domain toggles
start_sync <= {start_sync[1:0], start_tg};
dclr_sync <= {dclr_sync[1:0], dclr_tg};
pre_req_sync <= {pre_req_sync[1:0], pre_req_tg};
// acknowledge a consumed preload byte (write-enable pulsed this cycle)
if (mlkem_pre_we) pre_ack_tg <= ~pre_ack_tg;
// sticky done: set on the core's done_o pulse, cleared by synced W1C.
// (mlkem_start also clears it so a new run starts with done low.)
if (core_done) done_latch <= 1'b1;
else if (mlkem_dclr) done_latch <= 1'b0;
else if (mlkem_start) done_latch <= 1'b0;
end
end
// ==========================================
// sys_clk domain: sync busy/done/preload-ack back from mlkem.
// ==========================================
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
busy_sync <= 2'd0;
done_sync <= 2'd0;
pre_ack_sync <= 3'd0;
end else begin
busy_sync <= {busy_sync[0], core_busy};
done_sync <= {done_sync[0], done_latch};
pre_ack_sync <= {pre_ack_sync[1:0], pre_ack_tg};
end
end
wire status_busy = busy_sync[1];
wire status_done = done_sync[1];
assign intr = status_done;
// ==========================================
// AXI Write channel (INCR burst). Preload-data beats are wait-stated until
// the cross-domain byte handshake completes (W_PRELOAD).
// ==========================================
localparam W_IDLE = 2'd0;
localparam W_DATA = 2'd1;
localparam W_PRELOAD = 2'd2;
localparam W_RESP = 2'd3;
reg [1:0] w_state;
reg [31:0] w_addr;
reg [4:0] w_id;
reg [1:0] w_burst;
reg w_last_l; // saved wlast across a preload wait
wire [11:0] w_offset = w_addr[11:0];
assign s_axi_bid = w_id;
integer wi;
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
w_state <= W_IDLE;
s_axi_awready <= 1'b1;
s_axi_wready <= 1'b0;
s_axi_bvalid <= 1'b0;
cfg_k <= 3'd4; // default ML-KEM-1024
cfg_op <= 2'd0;
pre_sel <= 2'd0;
pre_addr <= 12'd0;
rdb_sel <= 2'd0;
rdb_addr <= 12'd0;
start_tg <= 1'b0;
dclr_tg <= 1'b0;
pre_req_tg <= 1'b0;
pre_busy <= 1'b0;
pre_sel_l <= 2'd0;
pre_addr_l <= 12'd0;
pre_byte_l <= 8'd0;
w_last_l <= 1'b0;
for (wi = 0; wi < 8; wi = wi + 1) begin
d_word[wi] <= 32'd0;
z_word[wi] <= 32'd0;
msg_word[wi] <= 32'd0;
end
end else begin
// clear the preload-busy flag when the mlkem domain acks the byte
if (pre_ack_edge) begin
pre_busy <= 1'b0;
pre_addr <= pre_addr + 12'd1; // auto-increment for next byte
end
case (w_state)
W_IDLE: begin
s_axi_awready <= 1'b1;
if (s_axi_awvalid && s_axi_awready) begin
s_axi_awready <= 1'b0;
s_axi_wready <= 1'b1;
w_addr <= s_axi_awaddr;
w_id <= s_axi_awid;
w_burst <= s_axi_awburst;
w_state <= W_DATA;
end
end
W_DATA: begin
if (s_axi_wvalid && s_axi_wready) begin
if (w_offset == ADDR_PRE_DAT) begin
// launch cross-domain byte write; stall the beat.
pre_sel_l <= pre_sel;
pre_addr_l <= pre_addr;
pre_byte_l <= s_axi_wdata[7:0];
pre_req_tg <= ~pre_req_tg;
pre_busy <= 1'b1;
w_last_l <= s_axi_wlast;
s_axi_wready <= 1'b0;
w_state <= W_PRELOAD;
end else begin
// normal single-cycle register write
case (1'b1)
(w_offset == ADDR_CTRL):
if (s_axi_wdata[0]) start_tg <= ~start_tg;
(w_offset == ADDR_STATUS):
if (s_axi_wdata[1]) dclr_tg <= ~dclr_tg; // clear done
(w_offset == ADDR_CONFIG): begin
cfg_k <= s_axi_wdata[2:0];
cfg_op <= s_axi_wdata[5:4];
end
(w_offset == ADDR_PRE_SEL): pre_sel <= s_axi_wdata[1:0];
(w_offset == ADDR_PRE_ADR): pre_addr <= s_axi_wdata[11:0];
(w_offset == ADDR_RDB_SEL): rdb_sel <= s_axi_wdata[1:0];
(w_offset == ADDR_RDB_ADR): rdb_addr <= s_axi_wdata[11:0];
(w_offset >= ADDR_D && w_offset < ADDR_D + 12'h20):
d_word[(w_offset-ADDR_D)>>2] <= apply_wstrb(d_word[(w_offset-ADDR_D)>>2], s_axi_wdata, s_axi_wstrb);
(w_offset >= ADDR_Z && w_offset < ADDR_Z + 12'h20):
z_word[(w_offset-ADDR_Z)>>2] <= apply_wstrb(z_word[(w_offset-ADDR_Z)>>2], s_axi_wdata, s_axi_wstrb);
(w_offset >= ADDR_MSG && w_offset < ADDR_MSG + 12'h20):
msg_word[(w_offset-ADDR_MSG)>>2] <= apply_wstrb(msg_word[(w_offset-ADDR_MSG)>>2], s_axi_wdata, s_axi_wstrb);
default: ; // SS / RDB_DAT / unmapped: ignore
endcase
if (w_burst == 2'b01) w_addr <= w_addr + 32'd4;
if (s_axi_wlast) begin
s_axi_wready <= 1'b0;
s_axi_bvalid <= 1'b1;
w_state <= W_RESP;
end
end
end
end
W_PRELOAD: begin
// wait for the synced ack (pre_ack_edge clears pre_busy above)
if (!pre_busy) begin
if (w_burst == 2'b01) w_addr <= w_addr + 32'd4;
if (w_last_l) begin
s_axi_bvalid <= 1'b1;
w_state <= W_RESP;
end else begin
s_axi_wready <= 1'b1;
w_state <= W_DATA;
end
end
end
W_RESP: begin
if (s_axi_bvalid && s_axi_bready) begin
s_axi_bvalid <= 1'b0;
s_axi_awready <= 1'b1;
w_state <= W_IDLE;
end
end
default: w_state <= W_IDLE;
endcase
end
end
// ==========================================
// AXI Read channel (INCR burst).
// ==========================================
localparam R_IDLE = 1'b0;
localparam R_DATA = 1'b1;
reg r_state;
reg [31:0] r_addr;
reg [7:0] r_cnt;
reg [7:0] r_len;
reg [1:0] r_burst;
wire [11:0] r_offset = r_addr[11:0];
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
r_state <= R_IDLE;
s_axi_arready <= 1'b1;
s_axi_rvalid <= 1'b0;
s_axi_rlast <= 1'b0;
s_axi_rid <= 5'd0;
r_addr <= 32'd0;
r_cnt <= 8'd0;
r_len <= 8'd0;
r_burst <= 2'd0;
end else begin
case (r_state)
R_IDLE: begin
s_axi_arready <= 1'b1;
if (s_axi_arvalid && s_axi_arready) begin
s_axi_arready <= 1'b0;
r_addr <= s_axi_araddr;
r_len <= s_axi_arlen;
r_burst <= s_axi_arburst;
r_cnt <= 8'd0;
s_axi_rvalid <= 1'b1;
s_axi_rlast <= (s_axi_arlen == 8'd0);
s_axi_rid <= s_axi_arid;
r_state <= R_DATA;
end
end
R_DATA: begin
if (s_axi_rvalid && s_axi_rready) begin
if (r_cnt == r_len) begin
s_axi_rvalid <= 1'b0;
s_axi_rlast <= 1'b0;
s_axi_arready <= 1'b1;
r_state <= R_IDLE;
end else begin
r_cnt <= r_cnt + 8'd1;
s_axi_rlast <= (r_cnt + 8'd1 == r_len);
if (r_burst == 2'b01) r_addr <= r_addr + 32'd4;
end
end
end
default: r_state <= R_IDLE;
endcase
end
end
// Read-data mux. Use the live address while in R_IDLE accepting AR so the
// first beat is valid the cycle rvalid asserts (matches matmul style).
wire [11:0] cur_off = (r_state == R_DATA) ? r_offset : s_axi_araddr[11:0];
// Selected readback byte (taps are combinational over the core's BRAMs).
wire [7:0] rdb_byte = (rdb_sel == 2'd3) ? dbg_ct_o :
(rdb_sel == 2'd2) ? dbg_dk_o : dbg_byte_o;
always @(*) begin
s_axi_rdata = 32'd0;
if (r_state == R_DATA || (r_state == R_IDLE && s_axi_arvalid)) begin
case (1'b1)
(cur_off == ADDR_CTRL): s_axi_rdata = 32'd0;
(cur_off == ADDR_STATUS): s_axi_rdata = {29'd0, pre_busy, status_done, status_busy};
(cur_off == ADDR_CONFIG): s_axi_rdata = {26'd0, cfg_op, 1'b0, cfg_k};
(cur_off == ADDR_PRE_SEL): s_axi_rdata = {30'd0, pre_sel};
(cur_off == ADDR_PRE_ADR): s_axi_rdata = {20'd0, pre_addr};
(cur_off == ADDR_RDB_SEL): s_axi_rdata = {30'd0, rdb_sel};
(cur_off == ADDR_RDB_ADR): s_axi_rdata = {20'd0, rdb_addr};
(cur_off == ADDR_RDB_DAT): s_axi_rdata = {24'd0, rdb_byte};
(cur_off >= ADDR_D && cur_off < ADDR_D + 12'h20):
s_axi_rdata = d_word[(cur_off-ADDR_D)>>2];
(cur_off >= ADDR_Z && cur_off < ADDR_Z + 12'h20):
s_axi_rdata = z_word[(cur_off-ADDR_Z)>>2];
(cur_off >= ADDR_MSG && cur_off < ADDR_MSG + 12'h20):
s_axi_rdata = msg_word[(cur_off-ADDR_MSG)>>2];
(cur_off >= ADDR_SS && cur_off < ADDR_SS + 12'h20):
s_axi_rdata = ss_o[((cur_off-ADDR_SS)>>2)*32 +: 32];
default: s_axi_rdata = 32'd0;
endcase
end
end
endmodule

2252
rtl/ip/mlkem/top/mlkem_top.v Normal file
View File

File diff suppressed because it is too large Load Diff

65
rtl/soc_top.v
View File

@@ -604,6 +604,7 @@ wire axiOut_7_bready ;
wire confreg_int;
wire matmul_intr;
wire mlkem_intr;
AxiCrossbar_2x8 u_AxiCrossbar_2x8 (
.clk ( sys_clk ),
@@ -1035,8 +1036,8 @@ AxiCrossbar_2x8 u_AxiCrossbar_2x8 (
core_top u_cpu(
.intrpt ({5'd0, matmul_intr, dma_finish, confreg_int}), //high active
// intrpt[2]=matmul, intrpt[1]=dma, intrpt[0]=confreg
.intrpt ({4'd0, mlkem_intr, matmul_intr, dma_finish, confreg_int}), //high active
// intrpt[3]=mlkem, intrpt[2]=matmul, intrpt[1]=dma, intrpt[0]=confreg
.aclk (cpu_clk ),
.aresetn (cpu_resetn ), //low active
@@ -1424,18 +1425,54 @@ axi_dvi u_axi_dvi (
.video_blue ( video_blue )
);
// slave 6 (FFT) removed — tie off
assign fft_bid = 5'b0;
assign fft_bresp = 2'b0;
assign fft_bvalid = 1'b0;
assign fft_rid = 5'b0;
assign fft_rdata = 32'b0;
assign fft_rresp = 2'b0;
assign fft_rlast = 1'b0;
assign fft_rvalid = 1'b0;
assign fft_arready = 1'b1;
assign fft_awready = 1'b1;
assign fft_wready = 1'b1;
// slave 6 - ML-KEM accelerator (0x1f400000, virtual 0xbf400000)
// Bus runs at sys_clk; the core is internally clocked at sys_clk/2 (25 MHz).
mlkem_axi_slave u_mlkem_axi_slave (
.clk (sys_clk),
.resetn (sys_resetn),
.s_axi_awid (fft_awid ),
.s_axi_awaddr (fft_awaddr ),
.s_axi_awlen (fft_awlen ),
.s_axi_awsize (fft_awsize ),
.s_axi_awburst (fft_awburst),
.s_axi_awlock (fft_awlock ),
.s_axi_awcache (fft_awcache),
.s_axi_awprot (fft_awprot ),
.s_axi_awvalid (fft_awvalid),
.s_axi_awready (fft_awready),
.s_axi_wid (fft_wid ),
.s_axi_wdata (fft_wdata ),
.s_axi_wstrb (fft_wstrb ),
.s_axi_wlast (fft_wlast ),
.s_axi_wvalid (fft_wvalid),
.s_axi_wready (fft_wready),
.s_axi_bid (fft_bid ),
.s_axi_bresp (fft_bresp ),
.s_axi_bvalid (fft_bvalid),
.s_axi_bready (fft_bready),
.s_axi_arid (fft_arid ),
.s_axi_araddr (fft_araddr ),
.s_axi_arlen (fft_arlen ),
.s_axi_arsize (fft_arsize ),
.s_axi_arburst (fft_arburst),
.s_axi_arlock (fft_arlock ),
.s_axi_arcache (fft_arcache),
.s_axi_arprot (fft_arprot ),
.s_axi_arvalid (fft_arvalid),
.s_axi_arready (fft_arready),
.s_axi_rid (fft_rid ),
.s_axi_rdata (fft_rdata ),
.s_axi_rresp (fft_rresp ),
.s_axi_rlast (fft_rlast ),
.s_axi_rvalid (fft_rvalid),
.s_axi_rready (fft_rready),
.intr (mlkem_intr)
);
//slave 7 - matmul accelerator
matmul_axi_slave u_matmul_axi_slave (