From bfbfc2ef72d0b16079f3902bdc85125d758d06db Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Wed, 8 Jul 2026 01:42:13 +0800 Subject: [PATCH] Pipeline NTT layer execution --- sync_rtl/ntt/TB/tb_ntt_core_xsim.v | 2 +- sync_rtl/ntt/butterfly_unit_pipe.v | 26 ++--- sync_rtl/ntt/ntt_core.v | 146 +++++++++++++++++------------ 3 files changed, 102 insertions(+), 72 deletions(-) diff --git a/sync_rtl/ntt/TB/tb_ntt_core_xsim.v b/sync_rtl/ntt/TB/tb_ntt_core_xsim.v index 6a3625f..0576314 100644 --- a/sync_rtl/ntt/TB/tb_ntt_core_xsim.v +++ b/sync_rtl/ntt/TB/tb_ntt_core_xsim.v @@ -172,7 +172,7 @@ module tb_ntt_core_xsim; integer got_done; // Extract mode and coefficients from vector memory - vec_mode = vector_mem[idx][3075]; + vec_mode = |vector_mem[idx][3075:3072]; // Extract 256 input coefficients // coeff[0] at bits [3074:3063], coeff[255] at bits [11:0] diff --git a/sync_rtl/ntt/butterfly_unit_pipe.v b/sync_rtl/ntt/butterfly_unit_pipe.v index 1c8736b..c672953 100644 --- a/sync_rtl/ntt/butterfly_unit_pipe.v +++ b/sync_rtl/ntt/butterfly_unit_pipe.v @@ -33,17 +33,17 @@ module butterfly_unit_pipe ( .valid_o(mul_valid) ); - reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5; - reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5; - reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5; + reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6; + reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6; + reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6; reg [11:0] a_out_r, b_out_r; reg valid_r; - wire [12:0] a_sum = {1'b0, a_d5} + {1'b0, (mode_d5 ? b_d5 : mul_result)}; + wire [12:0] a_sum = {1'b0, a_d6} + {1'b0, (mode_d6 ? b_d6 : mul_result)}; wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0]; - wire [11:0] b_sub = (a_d5 >= mul_result) ? (a_d5 - mul_result) : - (a_d5 - mul_result + Q12); + wire [11:0] b_sub = (a_d6 >= mul_result) ? (a_d6 - mul_result) : + (a_d6 - mul_result + Q12); assign a_out = a_out_r; assign b_out = b_out_r; @@ -51,22 +51,22 @@ module butterfly_unit_pipe ( always @(posedge clk or negedge rst_n) begin if (!rst_n) begin - a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; - b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; - mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; + a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0; + b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0; + mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0; a_out_r <= 12'd0; b_out_r <= 12'd0; valid_r <= 1'b0; end else begin - a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; - b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; + a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5; + b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5; mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1; - mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; + mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5; valid_r <= mul_valid; if (mul_valid) begin a_out_r <= a_mod; - b_out_r <= mode_d5 ? mul_result : b_sub; + b_out_r <= mode_d6 ? mul_result : b_sub; end end end diff --git a/sync_rtl/ntt/ntt_core.v b/sync_rtl/ntt/ntt_core.v index 0b77159..a326990 100644 --- a/sync_rtl/ntt/ntt_core.v +++ b/sync_rtl/ntt/ntt_core.v @@ -3,6 +3,8 @@ // Uses 256 individual 12-bit registers and a deeply pipelined butterfly path. // The arithmetic hot path is split into: // address/operand/zeta register -> pipelined Barrett butterfly -> writeback +// Each NTT layer is issued continuously into the butterfly pipeline, then the +// core drains pending writebacks before starting the next dependent layer. // In inverse mode, final x3303 output scaling also uses a pipelined Barrett // multiplier so the output path does not reintroduce a combinational reducer. @@ -24,11 +26,8 @@ module ntt_core ( localparam S_IDLE = 4'd0; localparam S_LOAD = 4'd1; - localparam S_CMP_A = 4'd2; - localparam S_CMP_B = 4'd3; - localparam S_CMP_ISSUE = 4'd4; - localparam S_CMP_WAIT = 4'd5; - localparam S_CMP_WB = 4'd6; + localparam S_CMP_RUN = 4'd2; + localparam S_CMP_DRAIN = 4'd3; localparam S_OUT_PREP = 4'd7; localparam S_OUTPUT = 4'd8; localparam S_OUT_SCALE = 4'd9; @@ -45,11 +44,14 @@ module ntt_core ( reg [2:0] layer; reg bf_done; reg mode_r; + reg layer_issue_done; + reg [4:0] bf_inflight; - reg [DW-1:0] r_a, r_b, r_zeta; - reg [7:0] r_wa, r_wb; - reg [DW-1:0] wr_a_data, wr_b_data; - reg [7:0] wr_wa, wr_wb; + reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7; + reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7; + reg wb_valid_r; + reg [DW-1:0] wb_a_data_r, wb_b_data_r; + reg [7:0] wb_wa_r, wb_wb_r; reg [DW-1:0] coeff_out_r; reg valid_o_r; @@ -59,15 +61,24 @@ module ntt_core ( wire [DW-1:0] zeta; zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); + wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len}; + wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len}; + wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end); + wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256); + wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1); + wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done; + wire [DW-1:0] issue_a = cr[j]; + wire [DW-1:0] issue_b = cr[j + layer_len]; + wire [DW-1:0] bf_a_out, bf_b_out; wire bf_valid; butterfly_unit_pipe u_bf ( .clk(clk), .rst_n(rst_n), - .valid_i(state == S_CMP_ISSUE), - .a(r_a), - .b(r_b), - .zeta(r_zeta), + .valid_i(issue_fire), + .a(issue_a), + .b(issue_b), + .zeta(zeta), .mode(mode_r), .a_out(bf_a_out), .b_out(bf_b_out), @@ -95,15 +106,13 @@ module ntt_core ( next_state = state; case (state) S_IDLE: if (valid_i) next_state = S_LOAD; - S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_A; - S_CMP_A: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE; - S_CMP_B: next_state = bf_done ? S_OUT_PREP : S_CMP_ISSUE; - S_CMP_ISSUE: next_state = S_CMP_WAIT; - S_CMP_WAIT: if (bf_valid) next_state = S_CMP_WB; - S_CMP_WB: next_state = S_CMP_A; + S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN; + S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN; + S_CMP_DRAIN: if (bf_inflight == 5'd0) + next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN; S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; - S_OUT_SCALE: if (scale_valid_o && scale_emit_cnt >= 9'd255) next_state = S_DONE; + S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE; S_DONE: next_state = S_IDLE; default: next_state = S_IDLE; endcase @@ -123,15 +132,17 @@ module ntt_core ( layer <= 3'd0; bf_done <= 1'b0; mode_r <= 1'b0; - r_a <= 12'd0; - r_b <= 12'd0; - r_zeta <= 12'd0; - r_wa <= 8'd0; - r_wb <= 8'd0; - wr_a_data <= 12'd0; - wr_b_data <= 12'd0; - wr_wa <= 8'd0; - wr_wb <= 8'd0; + layer_issue_done <= 1'b0; + bf_inflight <= 5'd0; + wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0; + wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0; + wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0; + wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0; + wb_valid_r <= 1'b0; + wb_a_data_r <= 12'd0; + wb_b_data_r <= 12'd0; + wb_wa_r <= 8'd0; + wb_wb_r <= 8'd0; coeff_out_r <= 12'd0; valid_o_r <= 1'b0; scale_valid_i <= 1'b0; @@ -141,6 +152,16 @@ module ntt_core ( state <= next_state; scale_valid_i <= 1'b0; + wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2; + wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6; + wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2; + wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6; + + if (issue_fire && !wb_valid_r) + bf_inflight <= bf_inflight + 5'd1; + else if (!issue_fire && wb_valid_r) + bf_inflight <= bf_inflight - 5'd1; + if (state != S_OUTPUT && state != S_OUT_SCALE) valid_o_r <= 1'b0; @@ -154,6 +175,8 @@ module ntt_core ( start <= 8'd0; layer <= 3'd0; bf_done <= 1'b0; + layer_issue_done <= 1'b0; + bf_inflight <= 5'd0; mode_r <= mode; if (!mode) begin layer_len <= 8'd128; @@ -169,40 +192,44 @@ module ntt_core ( load_cnt <= load_cnt + 8'd1; end - if (state == S_CMP_A && !bf_done) begin - r_wa <= j; - r_wb <= j + layer_len; - r_a <= cr[j]; - r_b <= cr[j + layer_len]; - r_zeta <= zeta; - end - - if (state == S_CMP_WAIT && bf_valid) begin - wr_a_data <= bf_a_out; - wr_b_data <= bf_b_out; - wr_wa <= r_wa; - wr_wb <= r_wb; - end - - if (state == S_CMP_WB) begin - cr[wr_wa] <= wr_a_data; - cr[wr_wb] <= wr_b_data; - - j <= j + 8'd1; - if (j + 8'd1 >= start + layer_len) begin + if (state == S_CMP_RUN && !layer_issue_done) begin + if (issue_group_last) begin if (!mode_r) zeta_idx <= zeta_idx + 7'd1; else zeta_idx <= zeta_idx - 7'd1; - if ({1'b0,start} + {1'b0,layer_len} + {1'b0,layer_len} >= 9'd256) begin - layer <= layer + 3'd1; - layer_len <= mode_r ? (layer_len << 1) : (layer_len >> 1); - start <= 8'd0; - j <= 8'd0; - if (layer + 3'd1 >= LAYERS) bf_done <= 1'b1; + if (issue_layer_last) begin + layer_issue_done <= 1'b1; end else begin - start <= start + layer_len + layer_len; - j <= start + layer_len + layer_len; + start <= next_group_start[7:0]; + j <= next_group_start[7:0]; end + end else begin + j <= j + 8'd1; + end + end + + if (wb_valid_r) begin + cr[wb_wa_r] <= wb_a_data_r; + cr[wb_wb_r] <= wb_b_data_r; + end + + wb_valid_r <= bf_valid; + if (bf_valid) begin + wb_a_data_r <= bf_a_out; + wb_b_data_r <= bf_b_out; + wb_wa_r <= wa_d7; + wb_wb_r <= wb_d7; + end + + if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin + if (layer + 3'd1 >= LAYERS) begin + bf_done <= 1'b1; + end else begin + layer <= layer + 3'd1; + layer_len <= next_layer_len; + start <= 8'd0; + j <= 8'd0; + layer_issue_done <= 1'b0; end end @@ -247,6 +274,9 @@ module ntt_core ( out_cnt <= 8'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + layer_issue_done <= 1'b0; + bf_inflight <= 5'd0; + wb_valid_r <= 1'b0; valid_o_r <= 1'b0; end end