fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mlkem test diagnostics and allocator updates

Browse Source
This commit is contained in:
FallenSigh
2026-07-08 02:35:56 +08:00
parent 65607badd2
commit 0c85a20b4e
40 changed files with 37168 additions and 2093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
rtl/ip/mlkem/comp_decomp/comp_decomp_sync.v
View File

@@ -1,8 +1,9 @@
// comp_decomp_sync.v - ML-KEM coefficient compression/decompression
//
// Streaming: one coefficient per cycle through pipeline_reg.
// Streaming valid/ready interface. The arithmetic is fixed-latency pipelined
// so compression does not infer a long combinational divider/reducer.
// mode=0: compress — round((2^d * x) / Q) mod 2^d
// mode=1: decompress — round((Q * x) / 2^d) mod Q
// mode=1: decompress — round((Q * x) / 2^d)
// Uses round-half-up (round(2.5)=3, round(3.5)=4).
// Integer arithmetic:
// compress: (x * 2^d + Q/2) / Q, lower d bits as result
@@ -10,7 +11,7 @@
`include "defines.vh"
(* use_dsp = "no" *)
(* use_dsp = "yes" *)
module comp_decomp_sync (
input clk,
input rst_n,
@@ -24,52 +25,171 @@ module comp_decomp_sync (
input ready_i
);
// 2^d for d in {4,5,10,11}; fits in 12 bits (max 2048)
wire [11:0] two_pow_d;
assign two_pow_d = 12'd1 << d;
localparam [24:0] Q25 = 25'd`Q;
localparam [12:0] K13 = 13'd5039; // floor(2^24 / 3329)
// Product: 12-bit * 12-bit = max 2048*3328 = 6,815,744 → 23 bits (use 24)
wire [23:0] product;
reg [11:0] data_r;
reg valid_r;
// compress: x * 2^d; decompress: x * Q
assign product = mode ? {12'b0, coeff_in} * {12'b0, 12'd3329}
: {12'b0, coeff_in} * {12'b0, two_pow_d};
reg mode_s0;
reg [4:0] d_s0;
reg [11:0] coeff_s0;
reg valid_s0;
// Rounding offset: compress→Q/2=1664; decompress→2^(d-1)
wire [11:0] round_off;
assign round_off = mode ? (two_pow_d >> 1) : 12'd1664;
reg mode_s1;
reg [4:0] d_s1;
reg [23:0] comp_dividend_s1;
reg [23:0] decomp_rounded_s1;
reg valid_s1;
// Dividend = product + round_off (max ~ 6,817,408 fits in 24 bits)
wire [23:0] dividend;
assign dividend = product + {12'b0, round_off};
reg mode_s2;
reg [4:0] d_s2;
reg [23:0] comp_dividend_s2;
reg [36:0] comp_qprod_est_s2;
reg [23:0] decomp_rounded_s2;
reg valid_s2;
// Divisor: compress→Q=3329; decompress→2^d
wire [11:0] divisor;
assign divisor = mode ? two_pow_d : 12'd3329;
reg mode_s3;
reg [4:0] d_s3;
reg [23:0] comp_dividend_s3;
reg [12:0] comp_q_est_s3;
reg [11:0] decomp_result_s3;
reg valid_s3;
// Integer division (both ops positive → floor)
// Quotient is computed as 24b (widest operand) but fits in 12b
/* verilator lint_off WIDTH */
wire [11:0] div_result;
assign div_result = dividend / {12'b0, divisor};
/* verilator lint_on WIDTH */
reg mode_s4;
reg [4:0] d_s4;
reg [23:0] comp_dividend_s4;
reg [12:0] comp_q_est_s4;
reg [24:0] comp_q_mul_q_s4;
reg [11:0] decomp_result_s4;
reg valid_s4;
// Final modular reduction
// compress: lower d bits (mask with 2^d - 1)
// decompress: result < Q always, but apply mod Q for safety
wire [11:0] mod_result;
assign mod_result = mode ? (div_result % 12'd3329) : (div_result & (two_pow_d - 1'b1));
reg mode_s5;
reg [4:0] d_s5;
reg [12:0] comp_q_est_s5;
reg [24:0] comp_rem_s5;
reg [11:0] decomp_result_s5;
reg valid_s5;
// Pipeline through valid/ready stage
pipeline_reg #(.DW(12)) u_pipe (
.clk (clk),
.rst_n (rst_n),
.data_i (mod_result),
.valid_i(valid_i),
.ready_o(ready_o),
.data_o (coeff_out),
.valid_o(valid_o),
.ready_i(ready_i)
);
wire fire_i = valid_i && ready_o;
wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5;
// Accept a new command only when the pipeline and output slot are both free.
// Current users issue one coefficient at a time and wait for valid_o.
assign ready_o = !pipe_busy && (!valid_r || ready_i);
assign coeff_out = data_r;
assign valid_o = valid_r;
wire [23:0] coeff_ext_s0 = {12'd0, coeff_s0};
wire [23:0] comp_dividend_next = (coeff_ext_s0 << d_s0) + 24'd1664;
wire [23:0] decomp_product_next = coeff_s0 * 12'd`Q;
wire [23:0] decomp_round_next =
decomp_product_next + ((d_s0 == 5'd0) ? 24'd0 : (24'd1 << (d_s0 - 5'd1)));
wire [12:0] comp_q_corr1_s5 =
comp_q_est_s5 + ((comp_rem_s5 >= Q25) ? 13'd1 : 13'd0);
wire [24:0] comp_rem_corr1_s5 =
(comp_rem_s5 >= Q25) ? (comp_rem_s5 - Q25) : comp_rem_s5;
wire [12:0] comp_q_corr2_s5 =
comp_q_corr1_s5 + ((comp_rem_corr1_s5 >= Q25) ? 13'd1 : 13'd0);
wire [11:0] comp_mask_s5 =
(d_s5 == 5'd0) ? 12'd0 : (12'd1 << d_s5) - 12'd1;
wire [11:0] comp_result_s5 = comp_q_corr2_s5[11:0] & comp_mask_s5;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
data_r <= 12'd0;
valid_r <= 1'b0;
mode_s0 <= 1'b0;
d_s0 <= 5'd0;
coeff_s0 <= 12'd0;
valid_s0 <= 1'b0;
mode_s1 <= 1'b0;
d_s1 <= 5'd0;
comp_dividend_s1 <= 24'd0;
decomp_rounded_s1 <= 24'd0;
valid_s1 <= 1'b0;
mode_s2 <= 1'b0;
d_s2 <= 5'd0;
comp_dividend_s2 <= 24'd0;
comp_qprod_est_s2 <= 37'd0;
decomp_rounded_s2 <= 24'd0;
valid_s2 <= 1'b0;
mode_s3 <= 1'b0;
d_s3 <= 5'd0;
comp_dividend_s3 <= 24'd0;
comp_q_est_s3 <= 13'd0;
decomp_result_s3 <= 12'd0;
valid_s3 <= 1'b0;
mode_s4 <= 1'b0;
d_s4 <= 5'd0;
comp_dividend_s4 <= 24'd0;
comp_q_est_s4 <= 13'd0;
comp_q_mul_q_s4 <= 25'd0;
decomp_result_s4 <= 12'd0;
valid_s4 <= 1'b0;
mode_s5 <= 1'b0;
d_s5 <= 5'd0;
comp_q_est_s5 <= 13'd0;
comp_rem_s5 <= 25'd0;
decomp_result_s5 <= 12'd0;
valid_s5 <= 1'b0;
end else begin
if (valid_r && ready_i)
valid_r <= 1'b0;
mode_s0 <= mode;
d_s0 <= d;
coeff_s0 <= coeff_in;
valid_s0 <= fire_i;
mode_s1 <= mode_s0;
d_s1 <= d_s0;
comp_dividend_s1 <= comp_dividend_next;
decomp_rounded_s1 <= decomp_round_next;
valid_s1 <= valid_s0;
mode_s2 <= mode_s1;
d_s2 <= d_s1;
comp_dividend_s2 <= comp_dividend_s1;
comp_qprod_est_s2 <= {13'd0, comp_dividend_s1} * K13;
decomp_rounded_s2 <= decomp_rounded_s1;
valid_s2 <= valid_s1;
mode_s3 <= mode_s2;
d_s3 <= d_s2;
comp_dividend_s3 <= comp_dividend_s2;
comp_q_est_s3 <= comp_qprod_est_s2[36:24];
decomp_result_s3 <= decomp_rounded_s2 >> d_s2;
valid_s3 <= valid_s2;
mode_s4 <= mode_s3;
d_s4 <= d_s3;
comp_dividend_s4 <= comp_dividend_s3;
comp_q_est_s4 <= comp_q_est_s3;
comp_q_mul_q_s4 <= comp_q_est_s3 * Q25;
decomp_result_s4 <= decomp_result_s3;
valid_s4 <= valid_s3;
mode_s5 <= mode_s4;
d_s5 <= d_s4;
comp_q_est_s5 <= comp_q_est_s4;
comp_rem_s5 <= {1'b0, comp_dividend_s4} - comp_q_mul_q_s4;
decomp_result_s5 <= decomp_result_s4;
valid_s5 <= valid_s4;
if (valid_s5) begin
data_r <= mode_s5 ? decomp_result_s5 : comp_result_s5;
valid_r <= 1'b1;
end
end
end
endmodule

2
rtl/ip/mlkem/ntt/barrett_mul.v
View File

@@ -6,7 +6,7 @@
// Pure combinational, single-cycle latency.
// All multiplication widths explicitly controlled to avoid Verilog truncation.
(* use_dsp = "no" *)
module barrett_mul (
input [11:0] a,
input [11:0] b,

72
rtl/ip/mlkem/ntt/barrett_mul_pipe.v Normal file
View File

@@ -0,0 +1,72 @@
// barrett_mul_pipe.v - DSP-friendly pipelined Barrett modular multiply
//
// Computes product = a * b mod Q with fixed latency. This module is used on
// timing-critical NTT paths; the legacy barrett_mul remains combinational for
// older users.
module barrett_mul_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a,
input [11:0] b,
output [11:0] product,
output valid_o
);
localparam [24:0] Q25 = 25'd3329;
localparam [12:0] K13 = 13'd5039;
reg [23:0] p_s1;
reg [23:0] p_s2;
reg [36:0] tk_s2;
reg [23:0] p_s3;
reg [12:0] q_s3;
reg [24:0] p_s4;
reg [24:0] qprod_s4;
reg [24:0] r0_s5;
reg [24:0] r1_s6;
reg [11:0] product_r;
reg [6:0] valid_sr;
assign product = product_r;
assign valid_o = valid_sr[6];
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
p_s1 <= 24'd0;
p_s2 <= 24'd0; tk_s2 <= 37'd0;
p_s3 <= 24'd0; q_s3 <= 13'd0;
p_s4 <= 25'd0; qprod_s4 <= 25'd0;
r0_s5 <= 25'd0; r1_s6 <= 25'd0;
product_r <= 12'd0;
valid_sr <= 7'd0;
end else begin
valid_sr <= {valid_sr[5:0], valid_i};
// S1: full product.
p_s1 <= {12'd0, a} * b;
// S2: quotient-estimate product.
p_s2 <= p_s1;
tk_s2 <= {13'd0, p_s1} * K13;
// S3: quotient estimate.
p_s3 <= p_s2;
q_s3 <= tk_s2[36:24];
// S4: q * Q.
p_s4 <= {1'b0, p_s3};
qprod_s4 <= q_s3 * Q25;
// S5/S6/S7: subtract q*Q and conditionally subtract Q twice.
r0_s5 <= p_s4 - qprod_s4;
r1_s6 <= (r0_s5 >= Q25) ? (r0_s5 - Q25) : r0_s5;
product_r <= (r1_s6 >= Q25) ? (r1_s6 - Q25) : r1_s6[11:0];
end
end
endmodule

74
rtl/ip/mlkem/ntt/butterfly_unit_pipe.v Normal file
View File

@@ -0,0 +1,74 @@
// butterfly_unit_pipe.v - pipelined Cooley-Tukey / Gentleman-Sande butterfly
//
// Fixed-latency wrapper around barrett_mul_pipe. All arithmetic that used to
// sit in one NTT cycle is now split by registers.
module butterfly_unit_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a,
input [11:0] b,
input [11:0] zeta,
input mode, // 0 = forward NTT, 1 = inverse NTT
output [11:0] a_out,
output [11:0] b_out,
output valid_o
);
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] Q13 = 13'd3329;
wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12);
wire [11:0] mul_data = mode ? diff : b;
wire [11:0] mul_result;
wire mul_valid;
barrett_mul_pipe u_barrett (
.clk(clk),
.rst_n(rst_n),
.valid_i(valid_i),
.a(zeta),
.b(mul_data),
.product(mul_result),
.valid_o(mul_valid)
);
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6;
reg [11:0] a_out_r, b_out_r;
reg valid_r;
wire [12:0] a_sum = {1'b0, a_d6} + {1'b0, (mode_d6 ? b_d6 : mul_result)};
wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0];
wire [11:0] b_sub = (a_d6 >= mul_result) ? (a_d6 - mul_result) :
(a_d6 - mul_result + Q12);
assign a_out = a_out_r;
assign b_out = b_out_r;
assign valid_o = valid_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0;
a_out_r <= 12'd0;
b_out_r <= 12'd0;
valid_r <= 1'b0;
end else begin
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5;
valid_r <= mul_valid;
if (mul_valid) begin
a_out_r <= a_mod;
b_out_r <= mode_d6 ? mul_result : b_sub;
end
end
end
endmodule

293
rtl/ip/mlkem/ntt/ntt_core.v
View File

@@ -1,8 +1,12 @@
// ntt_core.v - NTT core with individual coefficient registers
//
// Uses 256 individual 12-bit registers and generate-based muxing
// to avoid any part-select simulation issues.
// 3-cycle butterfly: SetAddr -> Read -> Compute+Write
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
// The arithmetic hot path is split into:
// address/operand/zeta register -> pipelined Barrett butterfly -> writeback
// Each NTT layer is issued continuously into the butterfly pipeline, then the
// core drains pending writebacks before starting the next dependent layer.
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
module ntt_core (
input clk, rst_n,
@@ -17,72 +21,170 @@ module ntt_core (
);
localparam N = 256, LAYERS = 7, DW = 12;
// Individual coefficient registers
reg [DW-1:0] cr [0:N-1];
integer ci;
// State machine
localparam S_IDLE=3'd0, S_LOAD=3'd1, S_CMP_A=3'd2, S_CMP_B=3'd3,
S_CMP_C=3'd4, S_OUTPUT=3'd5, S_DONE=3'd6;
reg [2:0] state, next_state;
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_RUN = 4'd2;
localparam S_CMP_DRAIN = 4'd3;
localparam S_OUT_PREP = 4'd7;
localparam S_OUTPUT = 4'd8;
localparam S_OUT_SCALE = 4'd9;
localparam S_DONE = 4'd10;
reg [7:0] load_cnt, out_cnt;
reg [3:0] state, next_state;
reg [7:0] load_cnt;
reg [7:0] out_cnt;
reg [8:0] scale_issue_cnt;
reg [8:0] scale_emit_cnt;
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
reg bf_done;
reg mode_r;
reg layer_issue_done;
reg [4:0] bf_inflight;
// Pipeline registers
reg [DW-1:0] r_a, r_b;
reg [7:0] r_wa, r_wb;
reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7;
reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7;
reg wb_valid_r;
reg [DW-1:0] wb_a_data_r, wb_b_data_r;
reg [7:0] wb_wa_r, wb_wb_r;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
reg scale_valid_i;
reg [DW-1:0] scale_a_i;
// Zeta
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
// Butterfly
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end);
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
wire [DW-1:0] issue_a = cr[j];
wire [DW-1:0] issue_b = cr[j + layer_len];
wire [DW-1:0] bf_a_out, bf_b_out;
butterfly_unit u_bf (
.a(r_a), .b(r_b), .zeta(zeta), .mode(mode),
.a_out(bf_a_out), .b_out(bf_b_out));
wire bf_valid;
butterfly_unit_pipe u_bf (
.clk(clk),
.rst_n(rst_n),
.valid_i(issue_fire),
.a(issue_a),
.b(issue_b),
.zeta(zeta),
.mode(mode_r),
.a_out(bf_a_out),
.b_out(bf_b_out),
.valid_o(bf_valid)
);
// Output scaling
wire [DW-1:0] coeff_scaled;
barrett_mul u_scl (.a(cr[out_cnt]), .b(12'd3303), .product(coeff_scaled));
assign coeff_out = mode ? coeff_scaled : cr[out_cnt];
wire [DW-1:0] scale_product;
wire scale_valid_o;
barrett_mul_pipe u_scl (
.clk(clk),
.rst_n(rst_n),
.valid_i(scale_valid_i),
.a(scale_a_i),
.b(12'd3303),
.product(scale_product),
.valid_o(scale_valid_o)
);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_OUTPUT);
assign done_o = (state == S_DONE);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign coeff_out = coeff_out_r;
assign valid_o = valid_o_r;
assign done_o = (state == S_DONE);
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i) next_state = S_CMP_A;
S_CMP_A: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_B;
S_CMP_B: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_C;
S_CMP_C: if (bf_done) next_state = S_OUTPUT; else next_state = S_CMP_A;
S_OUTPUT:if (out_cnt >= 255 && ready_i) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN;
S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN;
S_CMP_DRAIN: if (bf_inflight == 5'd0)
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state<=S_IDLE; load_cnt<=0; out_cnt<=0; j<=0; start<=0; layer_len<=0;
zeta_idx<=0; layer<=0; bf_done<=0; r_a<=0; r_b<=0; r_wa<=0; r_wb<=0;
for (ci=0; ci<N; ci=ci+1) cr[ci] = 0;
state <= S_IDLE;
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer_len <= 8'd0;
zeta_idx <= 7'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0;
wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0;
wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0;
wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0;
wb_valid_r <= 1'b0;
wb_a_data_r <= 12'd0;
wb_b_data_r <= 12'd0;
wb_wa_r <= 8'd0;
wb_wb_r <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
scale_a_i <= 12'd0;
for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0;
end else begin
state <= next_state;
scale_valid_i <= 1'b0;
wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2;
wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6;
wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2;
wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6;
if (issue_fire && !wb_valid_r)
bf_inflight <= bf_inflight + 5'd1;
else if (!issue_fire && wb_valid_r)
bf_inflight <= bf_inflight - 5'd1;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt<=1; out_cnt<=0; j<=0; start<=0; layer<=0; bf_done<=0;
if (!mode) begin layer_len<=128; zeta_idx<=1; end
else begin layer_len<=2; zeta_idx<=127; end
load_cnt <= 8'd1;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
zeta_idx <= 7'd1;
end else begin
layer_len <= 8'd2;
zeta_idx <= 7'd127;
end
end
if (state == S_LOAD && valid_i) begin
@@ -90,42 +192,93 @@ module ntt_core (
load_cnt <= load_cnt + 8'd1;
end
// S_CMP_A: set read addresses (j, j+len)
if (state == S_CMP_A) begin
r_wa <= j;
r_wb <= j + layer_len;
end
if (state == S_CMP_RUN && !layer_issue_done) begin
if (issue_group_last) begin
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
// S_CMP_B: capture read data
if (state == S_CMP_B) begin
r_a <= cr[j];
r_b <= cr[j + layer_len];
end
// S_CMP_C: write butterfly results, advance counters
if (state == S_CMP_C) begin
cr[r_wa] <= bf_a_out;
cr[r_wb] <= bf_b_out;
j <= j + 8'd1;
if (j + 8'd1 >= start + layer_len) begin
if (!mode) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 256) begin
layer <= layer + 3'd1;
layer_len <= mode ? (layer_len<<1) : (layer_len>>1);
start <= 0; j <= 0;
if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1;
if (issue_layer_last) begin
layer_issue_done <= 1'b1;
end else begin
start <= start + layer_len + layer_len;
j <= start + layer_len + layer_len;
start <= next_group_start[7:0];
j <= next_group_start[7:0];
end
end else begin
j <= j + 8'd1;
end
end
if (state == S_OUTPUT && ready_i)
out_cnt <= (out_cnt>=255) ? 0 : (out_cnt+8'd1);
if (wb_valid_r) begin
cr[wb_wa_r] <= wb_a_data_r;
cr[wb_wb_r] <= wb_b_data_r;
end
wb_valid_r <= bf_valid;
if (bf_valid) begin
wb_a_data_r <= bf_a_out;
wb_b_data_r <= bf_b_out;
wb_wa_r <= wa_d7;
wb_wb_r <= wb_d7;
end
if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin
if (layer + 3'd1 >= LAYERS) begin
bf_done <= 1'b1;
end else begin
layer <= layer + 3'd1;
layer_len <= next_layer_len;
start <= 8'd0;
j <= 8'd0;
layer_issue_done <= 1'b0;
end
end
if (state == S_OUT_PREP) begin
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
if (!mode_r) begin
coeff_out_r <= cr[0];
valid_o_r <= 1'b1;
end
end
if (state == S_OUTPUT && valid_o_r && ready_i) begin
if (out_cnt < 8'd255) begin
out_cnt <= out_cnt + 8'd1;
coeff_out_r <= cr[out_cnt + 8'd1];
valid_o_r <= 1'b1;
end else begin
out_cnt <= 8'd0;
valid_o_r <= 1'b0;
end
end
if (state == S_OUT_SCALE) begin
if (scale_issue_cnt < 9'd256) begin
scale_valid_i <= 1'b1;
scale_a_i <= cr[scale_issue_cnt[7:0]];
scale_issue_cnt <= scale_issue_cnt + 9'd1;
end
valid_o_r <= 1'b0;
if (scale_valid_o) begin
coeff_out_r <= scale_product;
valid_o_r <= 1'b1;
scale_emit_cnt <= scale_emit_cnt + 9'd1;
end
end
if (state == S_DONE) begin
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wb_valid_r <= 1'b0;
valid_o_r <= 1'b0;
end
end
end

143
rtl/ip/mlkem/poly_mul/basecase_mul_pipe.v Normal file
View File

@@ -0,0 +1,143 @@
// basecase_mul_pipe.v - pipelined NTT-domain degree-1 multiplication
//
// Fixed-latency replacement for the combinational basecase_mul hot path.
// The shared ntt/barrett_mul remains combinational for NTT users; this module
// keeps the extra registers local to poly_mul_sync.
module basecase_mul_pipe (
input clk,
input rst_n,
input valid_i,
input [11:0] a0, a1,
input [11:0] b0, b1,
input [11:0] zeta,
output [11:0] c0,
output [11:0] c1,
output valid_o
);
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] K13 = 13'd5039;
localparam [24:0] Q25 = 25'd3329;
function [12:0] barrett_q;
input [23:0] p;
reg [36:0] prod;
begin
prod = {13'd0, p} * K13;
barrett_q = prod[36:24];
end
endfunction
function [11:0] barrett_reduce;
input [23:0] p;
input [12:0] qe;
reg [24:0] q_approx;
reg [24:0] r0;
reg [24:0] r1;
reg [24:0] r2;
begin
q_approx = qe * Q12;
r0 = {1'b0, p} - q_approx;
r1 = (r0 >= Q25) ? (r0 - Q25) : r0;
r2 = (r1 >= Q25) ? (r1 - Q25) : r1;
barrett_reduce = r2[11:0];
end
endfunction
function [11:0] mod_add;
input [11:0] x;
input [11:0] y;
reg [12:0] sum;
begin
sum = {1'b0, x} + {1'b0, y};
mod_add = (sum >= {1'b0, Q12}) ? (sum[11:0] - Q12) : sum[11:0];
end
endfunction
// Stage 1: first-level scalar products.
reg [23:0] p10, p11, p12, p13;
reg [11:0] zeta_s1;
// Stage 2: Barrett quotient estimates for first-level products.
reg [23:0] p20, p21, p22, p23;
reg [12:0] q20, q21, q22, q23;
reg [11:0] zeta_s2;
// Stage 3: first-level reduced products.
reg [11:0] t1_s3, t2_s3, t3_s3, t4_s3;
reg [11:0] zeta_s3;
// Stage 4: zeta product for c0's second Barrett multiply.
reg [11:0] t1_s4, t3_s4, t4_s4;
reg [23:0] pz_s4;
// Stage 5: Barrett quotient estimate for zeta product.
reg [11:0] t1_s5, t3_s5, t4_s5;
reg [23:0] pz_s5;
reg [12:0] qz_s5;
// Stage 6: reduced zeta product.
reg [11:0] t1_s6, t3_s6, t4_s6, tz_s6;
// Stage 7: final modular additions.
reg [11:0] c0_r, c1_r;
reg [6:0] valid_sr;
assign c0 = c0_r;
assign c1 = c1_r;
assign valid_o = valid_sr[6];
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
p10 <= 24'd0; p11 <= 24'd0; p12 <= 24'd0; p13 <= 24'd0; zeta_s1 <= 12'd0;
p20 <= 24'd0; p21 <= 24'd0; p22 <= 24'd0; p23 <= 24'd0;
q20 <= 13'd0; q21 <= 13'd0; q22 <= 13'd0; q23 <= 13'd0; zeta_s2 <= 12'd0;
t1_s3 <= 12'd0; t2_s3 <= 12'd0; t3_s3 <= 12'd0; t4_s3 <= 12'd0; zeta_s3 <= 12'd0;
t1_s4 <= 12'd0; t3_s4 <= 12'd0; t4_s4 <= 12'd0; pz_s4 <= 24'd0;
t1_s5 <= 12'd0; t3_s5 <= 12'd0; t4_s5 <= 12'd0; pz_s5 <= 24'd0; qz_s5 <= 13'd0;
t1_s6 <= 12'd0; t3_s6 <= 12'd0; t4_s6 <= 12'd0; tz_s6 <= 12'd0;
c0_r <= 12'd0; c1_r <= 12'd0; valid_sr <= 7'd0;
end else begin
valid_sr <= {valid_sr[5:0], valid_i};
p10 <= {12'd0, a0} * b0;
p11 <= {12'd0, a1} * b1;
p12 <= {12'd0, a1} * b0;
p13 <= {12'd0, a0} * b1;
zeta_s1 <= zeta;
p20 <= p10; p21 <= p11; p22 <= p12; p23 <= p13;
q20 <= barrett_q(p10);
q21 <= barrett_q(p11);
q22 <= barrett_q(p12);
q23 <= barrett_q(p13);
zeta_s2 <= zeta_s1;
t1_s3 <= barrett_reduce(p20, q20);
t2_s3 <= barrett_reduce(p21, q21);
t3_s3 <= barrett_reduce(p22, q22);
t4_s3 <= barrett_reduce(p23, q23);
zeta_s3 <= zeta_s2;
t1_s4 <= t1_s3;
t3_s4 <= t3_s3;
t4_s4 <= t4_s3;
pz_s4 <= {12'd0, t2_s3} * zeta_s3;
t1_s5 <= t1_s4;
t3_s5 <= t3_s4;
t4_s5 <= t4_s4;
pz_s5 <= pz_s4;
qz_s5 <= barrett_q(pz_s4);
t1_s6 <= t1_s5;
t3_s6 <= t3_s5;
t4_s6 <= t4_s5;
tz_s6 <= barrett_reduce(pz_s5, qz_s5);
c0_r <= mod_add(t1_s6, tz_s6);
c1_r <= mod_add(t3_s6, t4_s6);
end
end
endmodule

169
rtl/ip/mlkem/poly_mul/poly_mul_sync.v
View File

@@ -4,8 +4,10 @@
// 256-coefficient NTT-domain polynomials.
//
// Operation flow:
// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle)
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE
// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
//
// RUN issues one base-case multiply every two cycles, matching the two-cycle
// c0/c1 output bandwidth after the basecase pipeline fills.
//
// The LOAD phase accepts both A and B coefficients simultaneously
// (one pair per cycle) on coeff_a_in/coeff_b_in.
@@ -22,7 +24,7 @@
// valid_i - Input valid
// ready_o - Ready to accept input (high in IDLE/LOAD)
// coeff_out[11:0] - Result coefficient output
// valid_o - Output valid (high in COMP_C0/COMP_C1)
// valid_o - Output valid during RUN output phase
// ready_i - Output consumer ready
module poly_mul_sync (
@@ -37,12 +39,9 @@ module poly_mul_sync (
);
// State definitions
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_COMP_CALC = 3'd2;
localparam S_COMP_C0 = 3'd3;
localparam S_COMP_C1 = 3'd4;
localparam S_DONE = 3'd5;
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_RUN = 3'd2;
reg [2:0] state, next_state;
@@ -52,14 +51,23 @@ module poly_mul_sync (
// Counters
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [6:0] comp_k; // 0..127, current base-case index
reg [7:0] issue_cnt; // 0..128, issued base-cases
reg [7:0] out_cnt; // 0..255, consumed output coefficients
reg issue_gap; // spaces base-case requests to match c0/c1 output
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Registered basecase_mul results
// Registered basecase_mul inputs/results
reg [11:0] bc_a0_reg, bc_a1_reg;
reg [11:0] bc_b0_reg, bc_b1_reg;
reg [11:0] bc_zeta_reg;
reg bc_valid_reg;
reg [11:0] c0_reg, c1_reg;
// Combinational read signals for COMP_CALC
wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits)
wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1
// Combinational read signals for the next base-case request.
wire [6:0] issue_k = issue_cnt[6:0];
wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1
wire [11:0] mem_a0 = mem_A[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even];
@@ -68,41 +76,42 @@ module poly_mul_sync (
// Zeta ROM
wire [11:0] zeta;
poly_mul_zeta_rom u_zeta (
.addr (comp_k),
.addr (issue_k),
.zeta (zeta)
);
// Basecase multiply
// Pipelined basecase multiply. Requests are issued every two cycles; inputs
// are registered locally so issue_cnt does not directly drive DSP input muxes.
wire [11:0] bc_c0, bc_c1;
basecase_mul u_bc (
.a0 (mem_a0),
.a1 (mem_a1),
.b0 (mem_b0),
.b1 (mem_b1),
.zeta(zeta),
wire bc_vo;
basecase_mul_pipe u_bc (
.clk (clk),
.rst_n(rst_n),
.valid_i(bc_valid_reg),
.a0 (bc_a0_reg),
.a1 (bc_a1_reg),
.b0 (bc_b0_reg),
.b1 (bc_b1_reg),
.zeta(bc_zeta_reg),
.c0 (bc_c0),
.c1 (bc_c1)
.c1 (bc_c1),
.valid_o(bc_vo)
);
// Output interface
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1);
assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg;
assign valid_o = out_valid_r;
assign coeff_out = out_phase ? c1_reg : c0_reg;
// State transition logic (combinational)
always @* begin
next_state = state;
case (state)
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o)
next_state = S_COMP_CALC;
S_COMP_CALC: next_state = S_COMP_C0;
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
S_COMP_C1: if (valid_o && ready_i) begin
if (comp_k >= 127) next_state = S_DONE;
else next_state = S_COMP_CALC;
end
S_DONE: next_state = S_IDLE;
S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o)
next_state = S_RUN;
S_RUN: if (valid_o && ready_i && out_cnt == 8'd255)
next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
@@ -113,15 +122,26 @@ module poly_mul_sync (
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
comp_k <= 7'd0;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
bc_a0_reg <= 12'd0;
bc_a1_reg <= 12'd0;
bc_b0_reg <= 12'd0;
bc_b1_reg <= 12'd0;
bc_zeta_reg <= 12'd0;
bc_valid_reg <= 1'b0;
c0_reg <= 12'd0;
c1_reg <= 12'd0;
for (i = 0; i < 256; i = i + 1) begin
mem_A[i] = 12'd0;
mem_B[i] = 12'd0;
mem_A[i] <= 12'd0;
mem_B[i] <= 12'd0;
end
end else begin
state <= next_state;
bc_valid_reg <= 1'b0;
// ---- LOAD phase ----
// First coefficient captured on IDLE → LOAD transition
@@ -129,6 +149,11 @@ module poly_mul_sync (
mem_A[0] <= coeff_a_in;
mem_B[0] <= coeff_b_in;
load_cnt <= 8'd1;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
// Subsequent coefficients in LOAD state
@@ -136,29 +161,65 @@ module poly_mul_sync (
mem_A[load_cnt] <= coeff_a_in;
mem_B[load_cnt] <= coeff_b_in;
load_cnt <= load_cnt + 8'd1;
if (load_cnt >= 8'd255) begin
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end
// ---- COMPUTE phase ----
// COMP_CALC: capture basecase_mul results
if (state == S_COMP_CALC) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
// RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
// regs, then launch requests every other cycle. The two-cycle issue
// interval matches the c0/c1 output bandwidth, so no deep result FIFO
// is needed for this always-ready system.
if (state == S_RUN) begin
if (issue_cnt < 8'd128) begin
if (!issue_gap) begin
bc_a0_reg <= mem_a0;
bc_a1_reg <= mem_a1;
bc_b0_reg <= mem_b0;
bc_b1_reg <= mem_b1;
bc_zeta_reg <= zeta;
bc_valid_reg <= 1'b1;
issue_cnt <= issue_cnt + 8'd1;
issue_gap <= 1'b1;
end else begin
issue_gap <= 1'b0;
end
end
if (out_valid_r && ready_i) begin
out_cnt <= out_cnt + 8'd1;
if (out_cnt == 8'd255) begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end else if (!out_phase) begin
out_phase <= 1'b1;
end else begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end
end
if (bc_vo) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
out_valid_r <= 1'b1;
out_phase <= 1'b0;
end
end
// COMP_C0 → COMP_C1: c0 was consumed, increment comp_k
if (state == S_COMP_C0 && valid_o && ready_i) begin
// comp_k stays same, c1 still to output
end
// COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair
if (state == S_COMP_C1 && valid_o && ready_i) begin
comp_k <= comp_k + 7'd1;
end
// ---- DONE → IDLE: reset counters ----
if (state == S_DONE) begin
// ---- final output consumed: reset counters before accepting a new op ----
if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
load_cnt <= 8'd0;
comp_k <= 7'd0;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end
end

50
rtl/ip/mlkem/rng/rng_sync.v
View File

@@ -1,50 +0,0 @@
// rng_sync.v - 256-bit Galois LFSR PRNG (taps: 255,253,252,247,0)
module rng_sync #(
parameter [255:0] SEED = 256'hDEADBEEFCAFEBABEFEEDFACEDECAFBAD1234567887654321ABCDEF010FEDCBA9
) (
input clk,
input rst_n,
input valid_i,
output ready_o,
output [255:0] data_o,
output valid_o,
input ready_i
);
reg [255:0] state;
reg valid_r;
reg [255:0] lfsr_next;
wire feedback;
integer i;
assign ready_o = 1'b1;
assign valid_o = valid_r;
assign data_o = state;
assign feedback = state[0];
always @(*) begin
for (i = 0; i < 255; i = i + 1)
lfsr_next[i] = state[i+1];
lfsr_next[255] = feedback;
lfsr_next[254] = lfsr_next[254] ^ feedback;
lfsr_next[252] = lfsr_next[252] ^ feedback;
lfsr_next[251] = lfsr_next[251] ^ feedback;
lfsr_next[246] = lfsr_next[246] ^ feedback;
end
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state <= SEED;
valid_r <= 1'b0;
end else begin
if (valid_r && ready_i)
valid_r <= 1'b0;
if (valid_i) begin
state <= lfsr_next;
valid_r <= 1'b1;
end
end
end
endmodule

1
rtl/ip/mlkem/sample_ntt/sample_ntt_sync.v
View File

@@ -31,7 +31,6 @@
`include "defines.vh"
/* verilator lint_off UNUSED */
(* use_dsp = "no" *)
module sample_ntt_sync #(parameter K = 4) (
/* verilator lint_on UNUSED */
input clk,

1
rtl/ip/mlkem/sample_ntt/sample_ntt_sync_shared.v
View File

@@ -44,7 +44,6 @@
`include "defines.vh"
/* verilator lint_off UNUSED */
(* use_dsp = "no" *)
module sample_ntt_sync_shared #(parameter K = 4) (
/* verilator lint_on UNUSED */
input clk,

20
rtl/ip/mlkem/sha3/sha3_top.v
View File

@@ -100,6 +100,16 @@ module sha3_top (
reg mb_last_r; // captured last-block flag
reg [255:0] mb_digest_r; // latched 256-bit digest (sticky in MB_DONE)
// ================================================================
// Keccak core
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
wire [1599:0] kc_state_o;
wire kc_ready_o;
/* verilator lint_on UNUSEDSIGNAL */
wire kc_valid_o;
// XOR the incoming block into the low 1088 bits (rate) of the state.
wire [1599:0] mb_xored;
assign mb_xored = mb_state_r ^ {{(1600-1088){1'b0}}, mb_block_i};
@@ -119,16 +129,6 @@ module sha3_top (
endcase
end
// ================================================================
// Keccak core
// ================================================================
wire kc_valid_i;
/* verilator lint_off UNUSED */
wire [1599:0] kc_state_o;
wire kc_ready_o;
/* verilator lint_on UNUSED */
wire kc_valid_o;
// Keccak input: multi-block xored state when mb_en, else single-block absorb.
wire [1599:0] kc_state_i_mux;
assign kc_state_i_mux = mb_en ? mb_xored : absorb_state;

286
rtl/ip/mlkem/top/fft_mlkem_axi_slave.v
View File

@@ -4,13 +4,15 @@
// maps do not overlap:
// ML-KEM: 0x0000..0x3fff
// FFT: 0x8000..0xcfff
// PRNG: 0xd000..0xd0ff
// Route the low ML-KEM block to ML-KEM, the high FFT block to FFT, and the
// following 0xdxxx block to PRNG.
// Route the low ML-KEM block to ML-KEM and the remaining accelerator window to
// FFT.
module fft_mlkem_axi_slave (
input clk,
input resetn,
input mlkem_clk,
input mlkem_resetn,
input [4:0] s_axi_awid,
input [31:0] s_axi_awaddr,
input [7:0] s_axi_awlen,
@@ -58,17 +60,12 @@ module fft_mlkem_axi_slave (
localparam SEL_MLKEM = 2'd0;
localparam SEL_FFT = 2'd1;
localparam SEL_PRNG = 2'd2;
wire aw_to_mlkem = (s_axi_awaddr[15:12] <= 4'h3);
wire aw_to_prng = (s_axi_awaddr[15:12] == 4'hd);
wire ar_to_mlkem = (s_axi_araddr[15:12] <= 4'h3);
wire ar_to_prng = (s_axi_araddr[15:12] == 4'hd);
wire [1:0] aw_sel = aw_to_mlkem ? SEL_MLKEM :
aw_to_prng ? SEL_PRNG : SEL_FFT;
wire [1:0] ar_sel = ar_to_mlkem ? SEL_MLKEM :
ar_to_prng ? SEL_PRNG : SEL_FFT;
wire [1:0] aw_sel = aw_to_mlkem ? SEL_MLKEM : SEL_FFT;
wire [1:0] ar_sel = ar_to_mlkem ? SEL_MLKEM : SEL_FFT;
reg [1:0] write_sel;
reg write_active;
@@ -83,6 +80,44 @@ module fft_mlkem_axi_slave (
wire [31:0] mlkem_rdata;
wire [1:0] mlkem_rresp;
wire mlkem_m_awvalid;
wire mlkem_m_awready;
wire [31:0] mlkem_m_awaddr;
wire [4:0] mlkem_m_awid;
wire [7:0] mlkem_m_awlen;
wire [2:0] mlkem_m_awsize;
wire [1:0] mlkem_m_awburst;
wire [0:0] mlkem_m_awlock;
wire [3:0] mlkem_m_awcache;
wire [2:0] mlkem_m_awprot;
wire mlkem_m_wvalid;
wire mlkem_m_wready;
wire [31:0] mlkem_m_wdata;
wire [3:0] mlkem_m_wstrb;
wire mlkem_m_wlast;
wire mlkem_m_bvalid;
wire mlkem_m_bready;
wire [4:0] mlkem_m_bid;
wire [1:0] mlkem_m_bresp;
wire mlkem_m_arvalid;
wire mlkem_m_arready;
wire [31:0] mlkem_m_araddr;
wire [4:0] mlkem_m_arid;
wire [7:0] mlkem_m_arlen;
wire [2:0] mlkem_m_arsize;
wire [1:0] mlkem_m_arburst;
wire [0:0] mlkem_m_arlock;
wire [3:0] mlkem_m_arcache;
wire [2:0] mlkem_m_arprot;
wire mlkem_m_rvalid;
wire mlkem_m_rready;
wire [31:0] mlkem_m_rdata;
wire [4:0] mlkem_m_rid;
wire [1:0] mlkem_m_rresp;
wire mlkem_m_rlast;
wire mlkem_intr_m;
(* async_reg = "true" *) reg [1:0] mlkem_intr_sync;
wire fft_awready, fft_wready, fft_bvalid;
wire [4:0] fft_bid;
wire [1:0] fft_bresp;
@@ -91,52 +126,33 @@ module fft_mlkem_axi_slave (
wire [31:0] fft_rdata;
wire [1:0] fft_rresp;
wire prng_awready, prng_wready, prng_bvalid;
wire [4:0] prng_bid;
wire [1:0] prng_bresp;
wire prng_arready, prng_rvalid, prng_rlast;
wire [4:0] prng_rid;
wire [31:0] prng_rdata;
wire [1:0] prng_rresp;
wire [1:0] sel_w = write_active ? write_sel : aw_sel;
wire [1:0] sel_r = read_active ? read_sel : ar_sel;
assign s_axi_awready = !write_active && (
(aw_sel == SEL_MLKEM) ? mlkem_awready :
(aw_sel == SEL_PRNG) ? prng_awready :
fft_awready);
assign s_axi_wready = (sel_w == SEL_MLKEM) ? mlkem_wready :
(sel_w == SEL_PRNG) ? prng_wready :
fft_wready;
assign s_axi_bvalid = (sel_w == SEL_MLKEM) ? mlkem_bvalid :
(sel_w == SEL_PRNG) ? prng_bvalid :
fft_bvalid;
assign s_axi_bid = (sel_w == SEL_MLKEM) ? mlkem_bid :
(sel_w == SEL_PRNG) ? prng_bid :
fft_bid;
assign s_axi_bresp = (sel_w == SEL_MLKEM) ? mlkem_bresp :
(sel_w == SEL_PRNG) ? prng_bresp :
fft_bresp;
assign s_axi_arready = !read_active && (
(ar_sel == SEL_MLKEM) ? mlkem_arready :
(ar_sel == SEL_PRNG) ? prng_arready :
fft_arready);
assign s_axi_rvalid = (sel_r == SEL_MLKEM) ? mlkem_rvalid :
(sel_r == SEL_PRNG) ? prng_rvalid :
fft_rvalid;
assign s_axi_rlast = (sel_r == SEL_MLKEM) ? mlkem_rlast :
(sel_r == SEL_PRNG) ? prng_rlast :
fft_rlast;
assign s_axi_rid = (sel_r == SEL_MLKEM) ? mlkem_rid :
(sel_r == SEL_PRNG) ? prng_rid :
fft_rid;
assign s_axi_rdata = (sel_r == SEL_MLKEM) ? mlkem_rdata :
(sel_r == SEL_PRNG) ? prng_rdata :
fft_rdata;
assign s_axi_rresp = (sel_r == SEL_MLKEM) ? mlkem_rresp :
(sel_r == SEL_PRNG) ? prng_rresp :
fft_rresp;
always @(posedge clk or negedge resetn) begin
@@ -162,52 +178,140 @@ module fft_mlkem_axi_slave (
end
end
always @(posedge clk or negedge resetn) begin
if (!resetn)
mlkem_intr_sync <= 2'b00;
else
mlkem_intr_sync <= {mlkem_intr_sync[0], mlkem_intr_m};
end
assign mlkem_intr = mlkem_intr_sync[1];
Axi_CDC u_mlkem_axi_cdc (
.axiInClk (clk),
.axiInRstn (resetn),
.axiOutClk (mlkem_clk),
.axiOutRstn (mlkem_resetn),
.axiIn_awvalid (s_axi_awvalid && !write_active && (aw_sel == SEL_MLKEM)),
.axiIn_awready (mlkem_awready),
.axiIn_awaddr (s_axi_awaddr),
.axiIn_awid (s_axi_awid),
.axiIn_awlen (s_axi_awlen),
.axiIn_awsize (s_axi_awsize),
.axiIn_awburst (s_axi_awburst),
.axiIn_awlock (s_axi_awlock),
.axiIn_awcache (s_axi_awcache),
.axiIn_awprot (s_axi_awprot),
.axiIn_wvalid (s_axi_wvalid && (sel_w == SEL_MLKEM)),
.axiIn_wready (mlkem_wready),
.axiIn_wdata (s_axi_wdata),
.axiIn_wstrb (s_axi_wstrb),
.axiIn_wlast (s_axi_wlast),
.axiIn_bvalid (mlkem_bvalid),
.axiIn_bready (s_axi_bready && (sel_w == SEL_MLKEM)),
.axiIn_bid (mlkem_bid),
.axiIn_bresp (mlkem_bresp),
.axiIn_arvalid (s_axi_arvalid && !read_active && (ar_sel == SEL_MLKEM)),
.axiIn_arready (mlkem_arready),
.axiIn_araddr (s_axi_araddr),
.axiIn_arid (s_axi_arid),
.axiIn_arlen (s_axi_arlen),
.axiIn_arsize (s_axi_arsize),
.axiIn_arburst (s_axi_arburst),
.axiIn_arlock (s_axi_arlock),
.axiIn_arcache (s_axi_arcache),
.axiIn_arprot (s_axi_arprot),
.axiIn_rvalid (mlkem_rvalid),
.axiIn_rready (s_axi_rready && (sel_r == SEL_MLKEM)),
.axiIn_rdata (mlkem_rdata),
.axiIn_rid (mlkem_rid),
.axiIn_rresp (mlkem_rresp),
.axiIn_rlast (mlkem_rlast),
.axiOut_awvalid (mlkem_m_awvalid),
.axiOut_awready (mlkem_m_awready),
.axiOut_awaddr (mlkem_m_awaddr),
.axiOut_awid (mlkem_m_awid),
.axiOut_awlen (mlkem_m_awlen),
.axiOut_awsize (mlkem_m_awsize),
.axiOut_awburst (mlkem_m_awburst),
.axiOut_awlock (mlkem_m_awlock),
.axiOut_awcache (mlkem_m_awcache),
.axiOut_awprot (mlkem_m_awprot),
.axiOut_wvalid (mlkem_m_wvalid),
.axiOut_wready (mlkem_m_wready),
.axiOut_wdata (mlkem_m_wdata),
.axiOut_wstrb (mlkem_m_wstrb),
.axiOut_wlast (mlkem_m_wlast),
.axiOut_bvalid (mlkem_m_bvalid),
.axiOut_bready (mlkem_m_bready),
.axiOut_bid (mlkem_m_bid),
.axiOut_bresp (mlkem_m_bresp),
.axiOut_arvalid (mlkem_m_arvalid),
.axiOut_arready (mlkem_m_arready),
.axiOut_araddr (mlkem_m_araddr),
.axiOut_arid (mlkem_m_arid),
.axiOut_arlen (mlkem_m_arlen),
.axiOut_arsize (mlkem_m_arsize),
.axiOut_arburst (mlkem_m_arburst),
.axiOut_arlock (mlkem_m_arlock),
.axiOut_arcache (mlkem_m_arcache),
.axiOut_arprot (mlkem_m_arprot),
.axiOut_rvalid (mlkem_m_rvalid),
.axiOut_rready (mlkem_m_rready),
.axiOut_rdata (mlkem_m_rdata),
.axiOut_rid (mlkem_m_rid),
.axiOut_rresp (mlkem_m_rresp),
.axiOut_rlast (mlkem_m_rlast)
);
mlkem_axi_slave u_mlkem_axi_slave (
.clk (clk),
.resetn (resetn),
.clk (mlkem_clk),
.resetn (mlkem_resetn),
.s_axi_awid (s_axi_awid),
.s_axi_awaddr (s_axi_awaddr),
.s_axi_awlen (s_axi_awlen),
.s_axi_awsize (s_axi_awsize),
.s_axi_awburst (s_axi_awburst),
.s_axi_awlock (s_axi_awlock),
.s_axi_awcache (s_axi_awcache),
.s_axi_awprot (s_axi_awprot),
.s_axi_awvalid (s_axi_awvalid && !write_active && (aw_sel == SEL_MLKEM)),
.s_axi_awready (mlkem_awready),
.s_axi_awid (mlkem_m_awid),
.s_axi_awaddr (mlkem_m_awaddr),
.s_axi_awlen (mlkem_m_awlen),
.s_axi_awsize (mlkem_m_awsize),
.s_axi_awburst (mlkem_m_awburst),
.s_axi_awlock (mlkem_m_awlock[0]),
.s_axi_awcache (mlkem_m_awcache),
.s_axi_awprot (mlkem_m_awprot),
.s_axi_awvalid (mlkem_m_awvalid),
.s_axi_awready (mlkem_m_awready),
.s_axi_wid (s_axi_wid),
.s_axi_wdata (s_axi_wdata),
.s_axi_wstrb (s_axi_wstrb),
.s_axi_wlast (s_axi_wlast),
.s_axi_wvalid (s_axi_wvalid && (sel_w == SEL_MLKEM)),
.s_axi_wready (mlkem_wready),
.s_axi_wid (5'd0),
.s_axi_wdata (mlkem_m_wdata),
.s_axi_wstrb (mlkem_m_wstrb),
.s_axi_wlast (mlkem_m_wlast),
.s_axi_wvalid (mlkem_m_wvalid),
.s_axi_wready (mlkem_m_wready),
.s_axi_bid (mlkem_bid),
.s_axi_bresp (mlkem_bresp),
.s_axi_bvalid (mlkem_bvalid),
.s_axi_bready (s_axi_bready && (sel_w == SEL_MLKEM)),
.s_axi_bid (mlkem_m_bid),
.s_axi_bresp (mlkem_m_bresp),
.s_axi_bvalid (mlkem_m_bvalid),
.s_axi_bready (mlkem_m_bready),
.s_axi_arid (s_axi_arid),
.s_axi_araddr (s_axi_araddr),
.s_axi_arlen (s_axi_arlen),
.s_axi_arsize (s_axi_arsize),
.s_axi_arburst (s_axi_arburst),
.s_axi_arlock (s_axi_arlock),
.s_axi_arcache (s_axi_arcache),
.s_axi_arprot (s_axi_arprot),
.s_axi_arvalid (s_axi_arvalid && !read_active && (ar_sel == SEL_MLKEM)),
.s_axi_arready (mlkem_arready),
.s_axi_arid (mlkem_m_arid),
.s_axi_araddr (mlkem_m_araddr),
.s_axi_arlen (mlkem_m_arlen),
.s_axi_arsize (mlkem_m_arsize),
.s_axi_arburst (mlkem_m_arburst),
.s_axi_arlock (mlkem_m_arlock[0]),
.s_axi_arcache (mlkem_m_arcache),
.s_axi_arprot (mlkem_m_arprot),
.s_axi_arvalid (mlkem_m_arvalid),
.s_axi_arready (mlkem_m_arready),
.s_axi_rid (mlkem_rid),
.s_axi_rdata (mlkem_rdata),
.s_axi_rresp (mlkem_rresp),
.s_axi_rlast (mlkem_rlast),
.s_axi_rvalid (mlkem_rvalid),
.s_axi_rready (s_axi_rready && (sel_r == SEL_MLKEM)),
.s_axi_rid (mlkem_m_rid),
.s_axi_rdata (mlkem_m_rdata),
.s_axi_rresp (mlkem_m_rresp),
.s_axi_rlast (mlkem_m_rlast),
.s_axi_rvalid (mlkem_m_rvalid),
.s_axi_rready (mlkem_m_rready),
.intr (mlkem_intr)
.intr (mlkem_intr_m)
);
fft_controller u_fft_controller (
@@ -258,50 +362,4 @@ module fft_mlkem_axi_slave (
.fft_finish (fft_finish)
);
prng_axi_slave u_prng_axi_slave (
.clk (clk),
.resetn (resetn),
.s_axi_awid (s_axi_awid),
.s_axi_awaddr (s_axi_awaddr),
.s_axi_awlen (s_axi_awlen),
.s_axi_awsize (s_axi_awsize),
.s_axi_awburst (s_axi_awburst),
.s_axi_awlock (s_axi_awlock),
.s_axi_awcache (s_axi_awcache),
.s_axi_awprot (s_axi_awprot),
.s_axi_awvalid (s_axi_awvalid && !write_active && (aw_sel == SEL_PRNG)),
.s_axi_awready (prng_awready),
.s_axi_wid (s_axi_wid),
.s_axi_wdata (s_axi_wdata),
.s_axi_wstrb (s_axi_wstrb),
.s_axi_wlast (s_axi_wlast),
.s_axi_wvalid (s_axi_wvalid && (sel_w == SEL_PRNG)),
.s_axi_wready (prng_wready),
.s_axi_bid (prng_bid),
.s_axi_bresp (prng_bresp),
.s_axi_bvalid (prng_bvalid),
.s_axi_bready (s_axi_bready && (sel_w == SEL_PRNG)),
.s_axi_arid (s_axi_arid),
.s_axi_araddr (s_axi_araddr),
.s_axi_arlen (s_axi_arlen),
.s_axi_arsize (s_axi_arsize),
.s_axi_arburst (s_axi_arburst),
.s_axi_arlock (s_axi_arlock),
.s_axi_arcache (s_axi_arcache),
.s_axi_arprot (s_axi_arprot),
.s_axi_arvalid (s_axi_arvalid && !read_active && (ar_sel == SEL_PRNG)),
.s_axi_arready (prng_arready),
.s_axi_rid (prng_rid),
.s_axi_rdata (prng_rdata),
.s_axi_rresp (prng_rresp),
.s_axi_rlast (prng_rlast),
.s_axi_rvalid (prng_rvalid),
.s_axi_rready (s_axi_rready && (sel_r == SEL_PRNG))
);
endmodule

217
rtl/ip/mlkem/top/mlkem_axi_slave.v
View File

@@ -1,16 +1,13 @@
// mlkem_axi_slave.v — AXI4 (burst) slave wrapper for the ML-KEM core.
// Makes the ML-KEM accelerator (KeyGen / Encaps / Decaps) CPU-controllable.
//
// The bus runs at sys_clk (50 MHz). The ML-KEM core cannot close timing at
// 50 MHz, so it is clocked by mlkem_clk = sys_clk / 2 (25 MHz), produced by a
// toggle flip-flop in this wrapper. mlkem_clk is a phase-aligned synchronous
// divide of sys_clk; every signal crossing between the two domains still goes
// through an explicit synchronizer or toggle handshake.
// NOTE (FPGA): for real silicon route mlkem_clk through a BUFG/BUFGCE rather
// than a logic-generated net; kept as a plain toggle here for portability.
// This slave is placed behind an AXI clock-domain converter. Its AXI side and
// mlkem_top both run in mlkem_clk; sys_clk masters reach it through Axi_CDC in
// fft_mlkem_axi_slave.
//
// Register map (byte offsets from base, e.g. virtual 0xbf400000):
// 0x000 CTRL W bit0 = start (launches op selected in CONFIG)
// bit1 = soft_reset (resets ML-KEM core state)
// 0x004 STATUS R bit0 busy, bit1 done, bit2 preload_busy
// W1C write bit1=1 to clear the latched done flag
// 0x008 CONFIG RW [2:0] k (2/3/4), [5:4] op (0=KeyGen 1=Encaps 2=Decaps)
@@ -27,9 +24,11 @@
// 0x1000..0x1FFF EK W: preload ek_in, R: read generated ek
// 0x2000..0x2FFF DK W: preload dk_in, R: read generated full dk
// 0x3000..0x3FFF CT W: preload c_in, R: read generated ciphertext
(* use_dsp = "no" *)
module mlkem_axi_slave (
input clk, // sys_clk (AXI bus clock, 50 MHz)
input resetn, // sys_resetn (active low, already synchronized)
input clk, // mlkem_clk AXI-side clock after Axi_CDC
input resetn, // mlkem_resetn, active low
// AXI Write Address Channel
input [4:0] s_axi_awid,
@@ -153,20 +152,7 @@ module mlkem_axi_slave (
endfunction
// ==========================================
// mlkem_clk = sys_clk / 2 (25 MHz) via toggle FF.
// ==========================================
reg mlkem_clk;
always @(posedge clk or negedge resetn) begin
if (!resetn) mlkem_clk <= 1'b0;
else mlkem_clk <= ~mlkem_clk;
end
// Reset for the mlkem domain: reuse sys_resetn (async-assert, and the core's
// own rst_n is sampled on mlkem_clk edges).
wire mlkem_rstn = resetn;
// ==========================================
// sys_clk-domain config / data registers (quasi-static; fed to the core
// combinationally — written long before start, stable across the op).
// AXI/core clock-domain config / data registers.
// ==========================================
reg [2:0] cfg_k; // 2/3/4
reg [1:0] cfg_op; // 0=KeyGen 1=Encaps 2=Decaps
@@ -188,55 +174,79 @@ module mlkem_axi_slave (
reg [11:0] rdb_addr; // output byte index
// ==========================================
// CDC: start pulse (sys toggle -> mlkem edge-detect -> 1-cyc start_i)
// Start pulse: CTRL.start flips a toggle, edge-detected into start_i.
// ==========================================
reg start_tg; // sys-domain toggle, flips on CTRL.start write
reg [2:0] start_sync; // mlkem-domain 2FF sync (+1 for edge)
wire mlkem_start = start_sync[2] ^ start_sync[1]; // 1 mlkem_clk pulse
wire mlkem_start = start_sync[2] ^ start_sync[1]; // 1 clk pulse
// ==========================================
// CDC: busy / done back to sys
// Software reset pulse: CTRL.soft_reset flips a toggle and stretches reset.
// ==========================================
wire core_busy; // busy_o (mlkem domain)
wire core_done; // done_o pulse (mlkem domain)
reg done_latch; // sticky done in mlkem domain
reg [1:0] busy_sync; // -> sys
reg [1:0] done_sync; // -> sys
// done-clear: sys toggle -> mlkem edge -> clears done_latch
reg dclr_tg; // sys-domain toggle (STATUS W1C of done)
reg [2:0] dclr_sync; // mlkem-domain 2FF sync (+1 for edge)
reg srst_tg; // flips on CTRL.soft_reset
reg [2:0] srst_sync; // edge detector shift register
reg [3:0] srst_cnt; // reset stretcher in clk domain
wire mlkem_srst_edge = srst_sync[2] ^ srst_sync[1];
wire mlkem_srst_active = (srst_cnt != 4'd0);
wire core_resetn = resetn && !mlkem_srst_active;
// ==========================================
// Busy / done status.
// ==========================================
wire core_busy;
wire core_done;
reg done_latch;
reg [1:0] busy_sync;
reg [1:0] done_sync;
// done-clear: STATUS W1C flips a toggle, edge-detected to clear done_latch.
reg dclr_tg;
reg [2:0] dclr_sync;
wire mlkem_dclr = dclr_sync[2] ^ dclr_sync[1];
// ==========================================
// CDC: preload byte write handshake.
// sys: latch {sel,addr,byte}, flip pre_req_tg, hold wready low.
// mlkem: 2FF sync pre_req_tg, on edge pulse the selected *_in_we 1 cycle,
// flip pre_ack_tg.
// sys: 2FF sync pre_ack_tg, on edge release the bus (addr++).
// Preload byte write handshake.
// Latch {sel,addr,byte}, flip pre_req_tg, pulse the selected *_in_we for
// one cycle, then flip pre_ack_tg to release the AXI write beat.
// ==========================================
reg pre_req_tg; // sys toggle: a byte is pending
reg [2:0] pre_req_sync; // mlkem 2FF (+1 for edge)
reg pre_req_tg; // a byte is pending
reg [2:0] pre_req_sync; // edge detector shift register
wire mlkem_pre_we = pre_req_sync[2] ^ pre_req_sync[1];
reg pre_ack_tg; // mlkem toggle: byte consumed
reg [2:0] pre_ack_sync; // sys 2FF (+1 for edge)
reg pre_ack_tg; // byte consumed
reg [2:0] pre_ack_sync; // edge detector shift register
wire pre_ack_edge = pre_ack_sync[2] ^ pre_ack_sync[1];
reg pre_busy; // sys: a preload handshake is in flight
// latched byte/port for the in-flight preload (sys, stable while pre_busy)
reg pre_busy; // a preload handshake is in flight
// latched byte/port for the in-flight preload, stable while pre_busy
reg [1:0] pre_sel_l;
reg [11:0] pre_addr_l;
reg [7:0] pre_byte_l;
reg pre_auto_inc_l;
// ==========================================
// Bulk readback byte handshake.
// Hold {sel,addr}, flip rdb_req_tg, let the BRAM read settle, latch one
// byte, then flip rdb_ack_tg.
// ==========================================
reg rdb_req_tg; // one readback byte request
reg [2:0] rdb_req_sync; // edge detector shift register
wire mlkem_rdb_req = rdb_req_sync[2] ^ rdb_req_sync[1];
reg rdb_ack_tg; // readback byte ready
reg [2:0] rdb_ack_sync; // edge detector shift register
wire rdb_ack_edge = rdb_ack_sync[2] ^ rdb_ack_sync[1];
reg [1:0] rdb_req_sel_l; // stable while request in flight
reg [11:0] rdb_req_addr_l; // stable while request in flight
reg [1:0] rdb_sel_m; // captured request
reg [11:0] rdb_addr_m; // captured request
reg [1:0] rdb_wait_m; // waits for BRAM registered read
reg rdb_busy_m;
reg [7:0] rdb_data_m; // held stable until next ack
// ==========================================
// mlkem_top instance
// ==========================================
wire [7:0] dbg_byte_o, dbg_dk_o, dbg_ct_o;
wire [255:0] ss_o;
wire bulk_read_active;
wire [1:0] bulk_rdb_sel;
wire [11:0] bulk_rdb_addr;
wire [1:0] eff_rdb_sel = bulk_read_active ? bulk_rdb_sel : rdb_sel;
wire [11:0] eff_rdb_addr = bulk_read_active ? bulk_rdb_addr : rdb_addr;
wire [1:0] eff_rdb_sel = rdb_busy_m ? rdb_sel_m : rdb_sel;
wire [11:0] eff_rdb_addr = rdb_busy_m ? rdb_addr_m : rdb_addr;
wire dbg_byte_sel = (eff_rdb_sel == 2'd1); // 0=ek, 1=dk_pke
// The core's ek/dkp read-address muxes give dbg_dk_idx_i priority over
// dbg_byte_idx_i whenever it lands in the ek (768..1567) or dk_pke (<768)
@@ -245,21 +255,20 @@ module mlkem_axi_slave (
// it out of range (0xFFF) so an ek/dk_pke readback isn't corrupted.
wire [11:0] dbg_dk_idx = (eff_rdb_sel == 2'd2) ? eff_rdb_addr : 12'hFFF;
// one-cycle write-enable pulses into the core (combinational off the synced
// edge; pre_sel_l/pre_addr_l/pre_byte_l are stable in sys, sampled by core).
wire ek_in_we = mlkem_pre_we && (pre_sel_l == 2'd0);
wire dk_in_we = mlkem_pre_we && (pre_sel_l == 2'd1);
wire c_in_we = mlkem_pre_we && (pre_sel_l == 2'd2);
// one-cycle write-enable pulses into the core.
wire ek_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd0);
wire dk_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd1);
wire c_in_we = core_resetn && mlkem_pre_we && (pre_sel_l == 2'd2);
mlkem_top #(.KMAX(4)) u_mlkem (
.clk (mlkem_clk),
.rst_n (mlkem_rstn),
.clk (clk),
.rst_n (core_resetn),
.k_i (cfg_k),
.op_i (cfg_op),
.d_i (d_bus),
.z_i (z_bus),
.msg_i (msg_bus),
.start_i (mlkem_start),
.start_i (core_resetn ? mlkem_start : 1'b0),
.busy_o (core_busy),
.done_o (core_done),
// ek preload
@@ -302,44 +311,82 @@ module mlkem_axi_slave (
);
// ==========================================
// mlkem_clk domain: synchronizers, done latch, preload-ack toggle.
// Core/AXI clock domain: edge detectors, done latch, preload/readback acks.
// ==========================================
always @(posedge mlkem_clk or negedge mlkem_rstn) begin
if (!mlkem_rstn) begin
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
start_sync <= 3'd0;
srst_sync <= 3'd0;
srst_cnt <= 4'd0;
dclr_sync <= 3'd0;
pre_req_sync <= 3'd0;
pre_ack_tg <= 1'b0;
rdb_req_sync <= 3'd0;
rdb_ack_tg <= 1'b0;
rdb_sel_m <= 2'd0;
rdb_addr_m <= 12'd0;
rdb_wait_m <= 2'd0;
rdb_busy_m <= 1'b0;
rdb_data_m <= 8'd0;
done_latch <= 1'b0;
end else begin
// 2FF (+edge) synchronizers of sys-domain toggles
// Edge-detect request toggles.
start_sync <= {start_sync[1:0], start_tg};
srst_sync <= {srst_sync[1:0], srst_tg};
dclr_sync <= {dclr_sync[1:0], dclr_tg};
pre_req_sync <= {pre_req_sync[1:0], pre_req_tg};
rdb_req_sync <= {rdb_req_sync[1:0], rdb_req_tg};
if (mlkem_srst_edge)
srst_cnt <= 4'd8;
else if (srst_cnt != 4'd0)
srst_cnt <= srst_cnt - 4'd1;
// acknowledge a consumed preload byte (write-enable pulsed this cycle)
if (mlkem_pre_we) pre_ack_tg <= ~pre_ack_tg;
if (core_resetn && mlkem_pre_we) pre_ack_tg <= ~pre_ack_tg;
if (!core_resetn) begin
rdb_wait_m <= 2'd0;
rdb_busy_m <= 1'b0;
end else if (mlkem_rdb_req) begin
rdb_sel_m <= rdb_req_sel_l;
rdb_addr_m <= rdb_req_addr_l;
rdb_wait_m <= 2'd2;
rdb_busy_m <= 1'b1;
end else if (rdb_wait_m != 2'd0) begin
if (rdb_wait_m == 2'd1) begin
rdb_data_m <= rdb_byte;
rdb_ack_tg <= ~rdb_ack_tg;
rdb_wait_m <= 2'd0;
rdb_busy_m <= 1'b0;
end else begin
rdb_wait_m <= rdb_wait_m - 2'd1;
end
end
// sticky done: set on the core's done_o pulse, cleared by synced W1C.
// (mlkem_start also clears it so a new run starts with done low.)
if (core_done) done_latch <= 1'b1;
if (mlkem_srst_active) done_latch <= 1'b0;
else if (core_done) done_latch <= 1'b1;
else if (mlkem_dclr) done_latch <= 1'b0;
else if (mlkem_start) done_latch <= 1'b0;
end
end
// ==========================================
// sys_clk domain: sync busy/done/preload-ack back from mlkem.
// AXI clock domain: register status and acknowledge toggles.
// ==========================================
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
busy_sync <= 2'd0;
done_sync <= 2'd0;
pre_ack_sync <= 3'd0;
rdb_ack_sync <= 3'd0;
end else begin
busy_sync <= {busy_sync[0], core_busy};
busy_sync <= {busy_sync[0], core_busy | mlkem_srst_active};
done_sync <= {done_sync[0], done_latch};
pre_ack_sync <= {pre_ack_sync[1:0], pre_ack_tg};
rdb_ack_sync <= {rdb_ack_sync[1:0], rdb_ack_tg};
end
end
wire status_busy = busy_sync[1];
@@ -389,6 +436,7 @@ module mlkem_axi_slave (
rdb_sel <= 2'd0;
rdb_addr <= 12'd0;
start_tg <= 1'b0;
srst_tg <= 1'b0;
dclr_tg <= 1'b0;
pre_req_tg <= 1'b0;
pre_busy <= 1'b0;
@@ -453,8 +501,10 @@ module mlkem_axi_slave (
end else begin
// normal single-cycle register write
case (1'b1)
(w_offset == ADDR_CTRL):
(w_offset == ADDR_CTRL): begin
if (s_axi_wdata[0]) start_tg <= ~start_tg;
if (s_axi_wdata[1]) srst_tg <= ~srst_tg;
end
(w_offset == ADDR_STATUS):
if (s_axi_wdata[1]) dclr_tg <= ~dclr_tg; // clear done
(w_offset == ADDR_CONFIG): begin
@@ -541,7 +591,6 @@ module mlkem_axi_slave (
localparam R_IDLE = 3'd0;
localparam R_DATA = 3'd1;
localparam R_BULK_WAIT = 3'd2;
localparam RDB_WAIT_CYCLES = 3'd4;
reg [2:0] r_state;
reg [31:0] r_addr;
@@ -557,13 +606,8 @@ module mlkem_axi_slave (
reg [1:0] bulk_sel_r;
reg [11:0] bulk_addr_r;
reg [1:0] bulk_lane;
reg [2:0] bulk_wait;
reg [31:0] bulk_word;
assign bulk_read_active = r_bulk && (r_state == R_BULK_WAIT);
assign bulk_rdb_sel = bulk_sel_r;
assign bulk_rdb_addr = bulk_addr_r;
always @(posedge clk or negedge resetn) begin
if (!resetn) begin
r_state <= R_IDLE;
@@ -579,8 +623,10 @@ module mlkem_axi_slave (
bulk_sel_r <= 2'd0;
bulk_addr_r <= 12'd0;
bulk_lane <= 2'd0;
bulk_wait <= 3'd0;
bulk_word <= 32'd0;
rdb_req_tg <= 1'b0;
rdb_req_sel_l <= 2'd0;
rdb_req_addr_l <= 12'd0;
end else begin
case (r_state)
R_IDLE: begin
@@ -597,8 +643,10 @@ module mlkem_axi_slave (
bulk_sel_r <= window_readback_sel(s_axi_araddr[15:12]);
bulk_addr_r <= s_axi_araddr[11:0];
bulk_lane <= 2'd0;
bulk_wait <= RDB_WAIT_CYCLES;
bulk_word <= 32'd0;
rdb_req_sel_l <= window_readback_sel(s_axi_araddr[15:12]);
rdb_req_addr_l <= s_axi_araddr[11:0];
rdb_req_tg <= ~rdb_req_tg;
s_axi_rvalid <= 1'b0;
s_axi_rlast <= 1'b0;
r_state <= R_BULK_WAIT;
@@ -610,14 +658,12 @@ module mlkem_axi_slave (
end
end
R_BULK_WAIT: begin
if (bulk_wait != 3'd0) begin
bulk_wait <= bulk_wait - 3'd1;
end else begin
if (rdb_ack_edge) begin
case (bulk_lane)
2'd0: bulk_word[7:0] <= rdb_byte;
2'd1: bulk_word[15:8] <= rdb_byte;
2'd2: bulk_word[23:16] <= rdb_byte;
default: bulk_word[31:24] <= rdb_byte;
2'd0: bulk_word[7:0] <= rdb_data_m;
2'd1: bulk_word[15:8] <= rdb_data_m;
2'd2: bulk_word[23:16] <= rdb_data_m;
default: bulk_word[31:24] <= rdb_data_m;
endcase
if (bulk_lane == 2'd3) begin
@@ -627,7 +673,8 @@ module mlkem_axi_slave (
end else begin
bulk_lane <= bulk_lane + 2'd1;
bulk_addr_r <= bulk_addr_r + 12'd1;
bulk_wait <= RDB_WAIT_CYCLES;
rdb_req_addr_l <= bulk_addr_r + 12'd1;
rdb_req_tg <= ~rdb_req_tg;
end
end
end
@@ -645,8 +692,10 @@ module mlkem_axi_slave (
if (r_bulk) begin
bulk_addr_r <= (r_burst == 2'b01) ? (r_offset + 12'd4) : r_offset;
bulk_lane <= 2'd0;
bulk_wait <= RDB_WAIT_CYCLES;
bulk_word <= 32'd0;
rdb_req_sel_l <= bulk_sel_r;
rdb_req_addr_l <= (r_burst == 2'b01) ? (r_offset + 12'd4) : r_offset;
rdb_req_tg <= ~rdb_req_tg;
s_axi_rvalid <= 1'b0;
s_axi_rlast <= 1'b0;
r_state <= R_BULK_WAIT;

1697
rtl/ip/mlkem/top/mlkem_top.v
View File

File diff suppressed because it is too large Load Diff