fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pipeline NTT layer execution

Browse Source
This commit is contained in:
FallenSigh
2026-07-08 01:42:13 +08:00
parent 7a7ccacb47
commit bfbfc2ef72
3 changed files with 102 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sync_rtl/ntt/TB/tb_ntt_core_xsim.v
View File

@@ -172,7 +172,7 @@ module tb_ntt_core_xsim;
integer got_done;
// Extract mode and coefficients from vector memory
vec_mode = vector_mem[idx][3075];
vec_mode = |vector_mem[idx][3075:3072];
// Extract 256 input coefficients
// coeff[0] at bits [3074:3063], coeff[255] at bits [11:0]

26
sync_rtl/ntt/butterfly_unit_pipe.v
View File

@@ -33,17 +33,17 @@ module butterfly_unit_pipe (
.valid_o(mul_valid)
);
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5;
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6;
reg [11:0] a_out_r, b_out_r;
reg valid_r;
wire [12:0] a_sum = {1'b0, a_d5} + {1'b0, (mode_d5 ? b_d5 : mul_result)};
wire [12:0] a_sum = {1'b0, a_d6} + {1'b0, (mode_d6 ? b_d6 : mul_result)};
wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0];
wire [11:0] b_sub = (a_d5 >= mul_result) ? (a_d5 - mul_result) :
(a_d5 - mul_result + Q12);
wire [11:0] b_sub = (a_d6 >= mul_result) ? (a_d6 - mul_result) :
(a_d6 - mul_result + Q12);
assign a_out = a_out_r;
assign b_out = b_out_r;
@@ -51,22 +51,22 @@ module butterfly_unit_pipe (
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0;
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0;
a_out_r <= 12'd0;
b_out_r <= 12'd0;
valid_r <= 1'b0;
end else begin
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4;
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5;
valid_r <= mul_valid;
if (mul_valid) begin
a_out_r <= a_mod;
b_out_r <= mode_d5 ? mul_result : b_sub;
b_out_r <= mode_d6 ? mul_result : b_sub;
end
end
end

146
sync_rtl/ntt/ntt_core.v
View File

@@ -3,6 +3,8 @@
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
// The arithmetic hot path is split into:
// address/operand/zeta register -> pipelined Barrett butterfly -> writeback
// Each NTT layer is issued continuously into the butterfly pipeline, then the
// core drains pending writebacks before starting the next dependent layer.
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
@@ -24,11 +26,8 @@ module ntt_core (
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_A = 4'd2;
localparam S_CMP_B = 4'd3;
localparam S_CMP_ISSUE = 4'd4;
localparam S_CMP_WAIT = 4'd5;
localparam S_CMP_WB = 4'd6;
localparam S_CMP_RUN = 4'd2;
localparam S_CMP_DRAIN = 4'd3;
localparam S_OUT_PREP = 4'd7;
localparam S_OUTPUT = 4'd8;
localparam S_OUT_SCALE = 4'd9;
@@ -45,11 +44,14 @@ module ntt_core (
reg [2:0] layer;
reg bf_done;
reg mode_r;
reg layer_issue_done;
reg [4:0] bf_inflight;
reg [DW-1:0] r_a, r_b, r_zeta;
reg [7:0] r_wa, r_wb;
reg [DW-1:0] wr_a_data, wr_b_data;
reg [7:0] wr_wa, wr_wb;
reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7;
reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7;
reg wb_valid_r;
reg [DW-1:0] wb_a_data_r, wb_b_data_r;
reg [7:0] wb_wa_r, wb_wb_r;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
@@ -59,15 +61,24 @@ module ntt_core (
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end);
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
wire [DW-1:0] issue_a = cr[j];
wire [DW-1:0] issue_b = cr[j + layer_len];
wire [DW-1:0] bf_a_out, bf_b_out;
wire bf_valid;
butterfly_unit_pipe u_bf (
.clk(clk),
.rst_n(rst_n),
.valid_i(state == S_CMP_ISSUE),
.a(r_a),
.b(r_b),
.zeta(r_zeta),
.valid_i(issue_fire),
.a(issue_a),
.b(issue_b),
.zeta(zeta),
.mode(mode_r),
.a_out(bf_a_out),
.b_out(bf_b_out),
@@ -95,15 +106,13 @@ module ntt_core (
next_state = state;
case (state)
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_A;
S_CMP_A: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE;
S_CMP_B: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE;
S_CMP_ISSUE: next_state = S_CMP_WAIT;
S_CMP_WAIT: if (bf_valid) next_state = S_CMP_WB;
S_CMP_WB: next_state = S_CMP_A;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN;
S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN;
S_CMP_DRAIN: if (bf_inflight == 5'd0)
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUT_SCALE: if (scale_valid_o && scale_emit_cnt >= 9'd255) next_state = S_DONE;
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
@@ -123,15 +132,17 @@ module ntt_core (
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
r_a <= 12'd0;
r_b <= 12'd0;
r_zeta <= 12'd0;
r_wa <= 8'd0;
r_wb <= 8'd0;
wr_a_data <= 12'd0;
wr_b_data <= 12'd0;
wr_wa <= 8'd0;
wr_wb <= 8'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0;
wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0;
wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0;
wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0;
wb_valid_r <= 1'b0;
wb_a_data_r <= 12'd0;
wb_b_data_r <= 12'd0;
wb_wa_r <= 8'd0;
wb_wb_r <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
@@ -141,6 +152,16 @@ module ntt_core (
state <= next_state;
scale_valid_i <= 1'b0;
wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2;
wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6;
wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2;
wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6;
if (issue_fire && !wb_valid_r)
bf_inflight <= bf_inflight + 5'd1;
else if (!issue_fire && wb_valid_r)
bf_inflight <= bf_inflight - 5'd1;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
@@ -154,6 +175,8 @@ module ntt_core (
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
@@ -169,40 +192,44 @@ module ntt_core (
load_cnt <= load_cnt + 8'd1;
end
if (state == S_CMP_A && !bf_done) begin
r_wa <= j;
r_wb <= j + layer_len;
r_a <= cr[j];
r_b <= cr[j + layer_len];
r_zeta <= zeta;
end
if (state == S_CMP_WAIT && bf_valid) begin
wr_a_data <= bf_a_out;
wr_b_data <= bf_b_out;
wr_wa <= r_wa;
wr_wb <= r_wb;
end
if (state == S_CMP_WB) begin
cr[wr_wa] <= wr_a_data;
cr[wr_wb] <= wr_b_data;
j <= j + 8'd1;
if (j + 8'd1 >= start + layer_len) begin
if (state == S_CMP_RUN && !layer_issue_done) begin
if (issue_group_last) begin
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 9'd256) begin
layer <= layer + 3'd1;
layer_len <= mode_r ? (layer_len << 1) : (layer_len >> 1);
start <= 8'd0;
j <= 8'd0;
if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1;
if (issue_layer_last) begin
layer_issue_done <= 1'b1;
end else begin
start <= start + layer_len + layer_len;
j <= start + layer_len + layer_len;
start <= next_group_start[7:0];
j <= next_group_start[7:0];
end
end else begin
j <= j + 8'd1;
end
end
if (wb_valid_r) begin
cr[wb_wa_r] <= wb_a_data_r;
cr[wb_wb_r] <= wb_b_data_r;
end
wb_valid_r <= bf_valid;
if (bf_valid) begin
wb_a_data_r <= bf_a_out;
wb_b_data_r <= bf_b_out;
wb_wa_r <= wa_d7;
wb_wb_r <= wb_d7;
end
if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin
if (layer + 3'd1 >= LAYERS) begin
bf_done <= 1'b1;
end else begin
layer <= layer + 3'd1;
layer_len <= next_layer_len;
start <= 8'd0;
j <= 8'd0;
layer_issue_done <= 1'b0;
end
end
@@ -247,6 +274,9 @@ module ntt_core (
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wb_valid_r <= 1'b0;
valid_o_r <= 1'b0;
end
end