feat: add ML-KEM accelerator with AXI slave wrapper
Wrap mlkem_top in an AXI4 burst slave (mlkem_axi_slave) mapped to the freed FFT slot 6 at 0x1f400000 (virtual 0xbf400000). The core runs on sys_clk/2 (25 MHz); the wrapper handles the clock-domain crossing: start toggle->pulse, wait-stated byte-preload handshake, busy/done sync, and direct readback taps (ek/dk/ct/ss). CPU drives it via a register map (CTRL/STATUS/CONFIG, d/z/msg, preload + readback windows). Wire into soc_top on the existing axiOut_6 nets (no crossbar change), add mlkem_intr on intrpt[3]. Add sdk mlkem_test example running a KeyGen->Encaps->Decaps self-consistency round trip.
This commit is contained in:
45
rtl/ip/mlkem/poly_mul/basecase_mul.v
Normal file
45
rtl/ip/mlkem/poly_mul/basecase_mul.v
Normal file
@@ -0,0 +1,45 @@
|
||||
// basecase_mul.v - NTT-domain degree-1 polynomial multiplication
|
||||
//
|
||||
// Computes pointwise product of two degree-1 NTT-domain polynomials:
|
||||
// c0 = (a0*b0 + a1*b1*zeta) mod Q
|
||||
// c1 = (a0*b1 + a1*b0) mod Q
|
||||
//
|
||||
// Uses three barrett_mul instances for modular multiplications
|
||||
// and pure combinational modular addition for the final sums.
|
||||
//
|
||||
// All inputs/outputs are 12-bit values in [0, Q-1] where Q = 3329.
|
||||
|
||||
module basecase_mul (
|
||||
input [11:0] a0, a1, // first polynomial: degree-0 and degree-1 coeffs
|
||||
input [11:0] b0, b1, // second polynomial: degree-0 and degree-1 coeffs
|
||||
input [11:0] zeta, // precomputed twiddle factor (zeta^2 * 17 mod Q)
|
||||
output [11:0] c0, c1 // result polynomial coefficients
|
||||
);
|
||||
|
||||
localparam Q = 3329;
|
||||
|
||||
// Barrett modular multiplication outputs
|
||||
wire [11:0] t1; // a0 * b0 mod Q
|
||||
wire [11:0] t2; // a1 * b1 mod Q
|
||||
wire [11:0] t3; // a1 * b0 mod Q
|
||||
wire [11:0] t4; // a0 * b1 mod Q
|
||||
wire [11:0] t2_zeta; // (a1 * b1) * zeta mod Q
|
||||
|
||||
// Four Barrett multiplications for the scalar products
|
||||
barrett_mul u_mul1 (.a(a0), .b(b0), .product(t1));
|
||||
barrett_mul u_mul2 (.a(a1), .b(b1), .product(t2));
|
||||
barrett_mul u_mul3 (.a(a1), .b(b0), .product(t3));
|
||||
barrett_mul u_mul4 (.a(a0), .b(b1), .product(t4));
|
||||
|
||||
// Multiply t2 by zeta for the zeta term in c0
|
||||
barrett_mul u_mul_zeta (.a(t2), .b(zeta), .product(t2_zeta));
|
||||
|
||||
// Combinational modular addition: each sum is in [0, 2Q-1)
|
||||
// Subtract Q once if >= Q
|
||||
wire [12:0] sum_c0 = {1'b0, t1} + {1'b0, t2_zeta};
|
||||
wire [12:0] sum_c1 = {1'b0, t3} + {1'b0, t4};
|
||||
|
||||
assign c0 = (sum_c0 >= Q) ? sum_c0[11:0] - Q[11:0] : sum_c0[11:0];
|
||||
assign c1 = (sum_c1 >= Q) ? sum_c1[11:0] - Q[11:0] : sum_c1[11:0];
|
||||
|
||||
endmodule
|
||||
166
rtl/ip/mlkem/poly_mul/poly_mul_sync.v
Normal file
166
rtl/ip/mlkem/poly_mul/poly_mul_sync.v
Normal file
@@ -0,0 +1,166 @@
|
||||
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier
|
||||
//
|
||||
// Computes pointwise (Karatsuba-like base-case) multiplication of two
|
||||
// 256-coefficient NTT-domain polynomials.
|
||||
//
|
||||
// Operation flow:
|
||||
// IDLE → LOAD (256× A+B pairs) → COMP_CALC (read+compute, 1 cycle)
|
||||
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE
|
||||
//
|
||||
// The LOAD phase accepts both A and B coefficients simultaneously
|
||||
// (one pair per cycle) on coeff_a_in/coeff_b_in.
|
||||
//
|
||||
// The COMPUTE phase outputs the 256 result coefficients one per cycle
|
||||
// via valid/ready handshake on coeff_out/valid_o/ready_i.
|
||||
//
|
||||
// Memory: 256×12-bit register arrays for A and B coefficients.
|
||||
//
|
||||
// Interface:
|
||||
// clk, rst_n - Clock, active-low reset
|
||||
// coeff_a_in[11:0]- Polynomial A coefficient input
|
||||
// coeff_b_in[11:0]- Polynomial B coefficient input
|
||||
// valid_i - Input valid
|
||||
// ready_o - Ready to accept input (high in IDLE/LOAD)
|
||||
// coeff_out[11:0] - Result coefficient output
|
||||
// valid_o - Output valid (high in COMP_C0/COMP_C1)
|
||||
// ready_i - Output consumer ready
|
||||
|
||||
module poly_mul_sync (
|
||||
input clk, rst_n,
|
||||
input [11:0] coeff_a_in,
|
||||
input [11:0] coeff_b_in,
|
||||
input valid_i,
|
||||
output ready_o,
|
||||
output [11:0] coeff_out,
|
||||
output valid_o,
|
||||
input ready_i
|
||||
);
|
||||
|
||||
// State definitions
|
||||
localparam S_IDLE = 3'd0;
|
||||
localparam S_LOAD = 3'd1;
|
||||
localparam S_COMP_CALC = 3'd2;
|
||||
localparam S_COMP_C0 = 3'd3;
|
||||
localparam S_COMP_C1 = 3'd4;
|
||||
localparam S_DONE = 3'd5;
|
||||
|
||||
reg [2:0] state, next_state;
|
||||
|
||||
// Coefficient storage (register arrays)
|
||||
reg [11:0] mem_A [0:255];
|
||||
reg [11:0] mem_B [0:255];
|
||||
|
||||
// Counters
|
||||
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
|
||||
reg [6:0] comp_k; // 0..127, current base-case index
|
||||
|
||||
// Registered basecase_mul results
|
||||
reg [11:0] c0_reg, c1_reg;
|
||||
|
||||
// Combinational read signals for COMP_CALC
|
||||
wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits)
|
||||
wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1
|
||||
wire [11:0] mem_a0 = mem_A[addr_even];
|
||||
wire [11:0] mem_a1 = mem_A[addr_odd];
|
||||
wire [11:0] mem_b0 = mem_B[addr_even];
|
||||
wire [11:0] mem_b1 = mem_B[addr_odd];
|
||||
|
||||
// Zeta ROM
|
||||
wire [11:0] zeta;
|
||||
poly_mul_zeta_rom u_zeta (
|
||||
.addr (comp_k),
|
||||
.zeta (zeta)
|
||||
);
|
||||
|
||||
// Basecase multiply
|
||||
wire [11:0] bc_c0, bc_c1;
|
||||
basecase_mul u_bc (
|
||||
.a0 (mem_a0),
|
||||
.a1 (mem_a1),
|
||||
.b0 (mem_b0),
|
||||
.b1 (mem_b1),
|
||||
.zeta(zeta),
|
||||
.c0 (bc_c0),
|
||||
.c1 (bc_c1)
|
||||
);
|
||||
|
||||
// Output interface
|
||||
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
|
||||
assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1);
|
||||
assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg;
|
||||
|
||||
// State transition logic (combinational)
|
||||
always @* begin
|
||||
next_state = state;
|
||||
case (state)
|
||||
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
|
||||
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o)
|
||||
next_state = S_COMP_CALC;
|
||||
S_COMP_CALC: next_state = S_COMP_C0;
|
||||
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
|
||||
S_COMP_C1: if (valid_o && ready_i) begin
|
||||
if (comp_k >= 127) next_state = S_DONE;
|
||||
else next_state = S_COMP_CALC;
|
||||
end
|
||||
S_DONE: next_state = S_IDLE;
|
||||
default: next_state = S_IDLE;
|
||||
endcase
|
||||
end
|
||||
|
||||
// Sequential logic
|
||||
integer i;
|
||||
always @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
state <= S_IDLE;
|
||||
load_cnt <= 8'd0;
|
||||
comp_k <= 7'd0;
|
||||
c0_reg <= 12'd0;
|
||||
c1_reg <= 12'd0;
|
||||
for (i = 0; i < 256; i = i + 1) begin
|
||||
mem_A[i] <= 12'd0;
|
||||
mem_B[i] <= 12'd0;
|
||||
end
|
||||
end else begin
|
||||
state <= next_state;
|
||||
|
||||
// ---- LOAD phase ----
|
||||
// First coefficient captured on IDLE → LOAD transition
|
||||
if (state == S_IDLE && valid_i && ready_o) begin
|
||||
mem_A[0] <= coeff_a_in;
|
||||
mem_B[0] <= coeff_b_in;
|
||||
load_cnt <= 8'd1;
|
||||
end
|
||||
|
||||
// Subsequent coefficients in LOAD state
|
||||
if (state == S_LOAD && valid_i && ready_o) begin
|
||||
mem_A[load_cnt] <= coeff_a_in;
|
||||
mem_B[load_cnt] <= coeff_b_in;
|
||||
load_cnt <= load_cnt + 8'd1;
|
||||
end
|
||||
|
||||
// ---- COMPUTE phase ----
|
||||
// COMP_CALC: capture basecase_mul results
|
||||
if (state == S_COMP_CALC) begin
|
||||
c0_reg <= bc_c0;
|
||||
c1_reg <= bc_c1;
|
||||
end
|
||||
|
||||
// COMP_C0 → COMP_C1: c0 was consumed, increment comp_k
|
||||
if (state == S_COMP_C0 && valid_o && ready_i) begin
|
||||
// comp_k stays same, c1 still to output
|
||||
end
|
||||
|
||||
// COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair
|
||||
if (state == S_COMP_C1 && valid_o && ready_i) begin
|
||||
comp_k <= comp_k + 7'd1;
|
||||
end
|
||||
|
||||
// ---- DONE → IDLE: reset counters ----
|
||||
if (state == S_DONE) begin
|
||||
load_cnt <= 8'd0;
|
||||
comp_k <= 7'd0;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
58
rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v
Normal file
58
rtl/ip/mlkem/poly_mul/poly_mul_zeta_rom.v
Normal file
@@ -0,0 +1,58 @@
|
||||
// poly_mul_zeta_rom.v - Read-only memory for PolyMul zeta values
|
||||
//
|
||||
// Stores precomputed zeta values for base-case NTT-domain polynomial
|
||||
// multiplication. Each entry zeta[k] = (zeta_bitRev[k]^2 * 17) mod Q.
|
||||
//
|
||||
// 128 entries × 12 bits. Pure combinational read.
|
||||
//
|
||||
// zeta_bitRev values are the same bit-reversed twiddle factors used
|
||||
// in NTT. The PolyMul formula squares them and multiplies by 17 mod Q.
|
||||
|
||||
module poly_mul_zeta_rom (
|
||||
input [6:0] addr, // 0..127, selects which zeta to read
|
||||
output [11:0] zeta // zeta = (zeta_bitRev[addr]^2 * 17) mod Q
|
||||
);
|
||||
|
||||
reg [11:0] rom [0:127];
|
||||
|
||||
// Initialize ROM with precomputed zeta values
|
||||
// Generated from Python: zeta = (zeta_bitRev[i]**2 * 17) % 3329
|
||||
initial begin
|
||||
rom[ 0] = 12'd17; rom[ 1] = 12'd3312; rom[ 2] = 12'd2761; rom[ 3] = 12'd568;
|
||||
rom[ 4] = 12'd583; rom[ 5] = 12'd2746; rom[ 6] = 12'd2649; rom[ 7] = 12'd680;
|
||||
rom[ 8] = 12'd1637; rom[ 9] = 12'd1692; rom[ 10] = 12'd723; rom[ 11] = 12'd2606;
|
||||
rom[ 12] = 12'd2288; rom[ 13] = 12'd1041; rom[ 14] = 12'd1100; rom[ 15] = 12'd2229;
|
||||
rom[ 16] = 12'd1409; rom[ 17] = 12'd1920; rom[ 18] = 12'd2662; rom[ 19] = 12'd667;
|
||||
rom[ 20] = 12'd3281; rom[ 21] = 12'd48; rom[ 22] = 12'd233; rom[ 23] = 12'd3096;
|
||||
rom[ 24] = 12'd756; rom[ 25] = 12'd2573; rom[ 26] = 12'd2156; rom[ 27] = 12'd1173;
|
||||
rom[ 28] = 12'd3015; rom[ 29] = 12'd314; rom[ 30] = 12'd3050; rom[ 31] = 12'd279;
|
||||
rom[ 32] = 12'd1703; rom[ 33] = 12'd1626; rom[ 34] = 12'd1651; rom[ 35] = 12'd1678;
|
||||
rom[ 36] = 12'd2789; rom[ 37] = 12'd540; rom[ 38] = 12'd1789; rom[ 39] = 12'd1540;
|
||||
rom[ 40] = 12'd1847; rom[ 41] = 12'd1482; rom[ 42] = 12'd952; rom[ 43] = 12'd2377;
|
||||
rom[ 44] = 12'd1461; rom[ 45] = 12'd1868; rom[ 46] = 12'd2687; rom[ 47] = 12'd642;
|
||||
rom[ 48] = 12'd939; rom[ 49] = 12'd2390; rom[ 50] = 12'd2308; rom[ 51] = 12'd1021;
|
||||
rom[ 52] = 12'd2437; rom[ 53] = 12'd892; rom[ 54] = 12'd2388; rom[ 55] = 12'd941;
|
||||
rom[ 56] = 12'd733; rom[ 57] = 12'd2596; rom[ 58] = 12'd2337; rom[ 59] = 12'd992;
|
||||
rom[ 60] = 12'd268; rom[ 61] = 12'd3061; rom[ 62] = 12'd641; rom[ 63] = 12'd2688;
|
||||
rom[ 64] = 12'd1584; rom[ 65] = 12'd1745; rom[ 66] = 12'd2298; rom[ 67] = 12'd1031;
|
||||
rom[ 68] = 12'd2037; rom[ 69] = 12'd1292; rom[ 70] = 12'd3220; rom[ 71] = 12'd109;
|
||||
rom[ 72] = 12'd375; rom[ 73] = 12'd2954; rom[ 74] = 12'd2549; rom[ 75] = 12'd780;
|
||||
rom[ 76] = 12'd2090; rom[ 77] = 12'd1239; rom[ 78] = 12'd1645; rom[ 79] = 12'd1684;
|
||||
rom[ 80] = 12'd1063; rom[ 81] = 12'd2266; rom[ 82] = 12'd319; rom[ 83] = 12'd3010;
|
||||
rom[ 84] = 12'd2773; rom[ 85] = 12'd556; rom[ 86] = 12'd757; rom[ 87] = 12'd2572;
|
||||
rom[ 88] = 12'd2099; rom[ 89] = 12'd1230; rom[ 90] = 12'd561; rom[ 91] = 12'd2768;
|
||||
rom[ 92] = 12'd2466; rom[ 93] = 12'd863; rom[ 94] = 12'd2594; rom[ 95] = 12'd735;
|
||||
rom[ 96] = 12'd2804; rom[ 97] = 12'd525; rom[ 98] = 12'd1092; rom[ 99] = 12'd2237;
|
||||
rom[100] = 12'd403; rom[101] = 12'd2926; rom[102] = 12'd1026; rom[103] = 12'd2303;
|
||||
rom[104] = 12'd1143; rom[105] = 12'd2186; rom[106] = 12'd2150; rom[107] = 12'd1179;
|
||||
rom[108] = 12'd2775; rom[109] = 12'd554; rom[110] = 12'd886; rom[111] = 12'd2443;
|
||||
rom[112] = 12'd1722; rom[113] = 12'd1607; rom[114] = 12'd1212; rom[115] = 12'd2117;
|
||||
rom[116] = 12'd1874; rom[117] = 12'd1455; rom[118] = 12'd1029; rom[119] = 12'd2300;
|
||||
rom[120] = 12'd2110; rom[121] = 12'd1219; rom[122] = 12'd2935; rom[123] = 12'd394;
|
||||
rom[124] = 12'd885; rom[125] = 12'd2444; rom[126] = 12'd2154; rom[127] = 12'd1175;
|
||||
end
|
||||
|
||||
// Combinational read
|
||||
assign zeta = rom[addr];
|
||||
|
||||
endmodule
|
||||
Reference in New Issue
Block a user