Pipeline ML-KEM datapath bottlenecks

sync_rtl/ntt/butterfly_unit_pipe.v

@@ -0,0 +1,74 @@
+// butterfly_unit_pipe.v - pipelined Cooley-Tukey / Gentleman-Sande butterfly
+//
+// Fixed-latency wrapper around barrett_mul_pipe. All arithmetic that used to
+// sit in one NTT cycle is now split by registers.
+
+module butterfly_unit_pipe (
+    input         clk,
+    input         rst_n,
+    input         valid_i,
+    input  [11:0] a,
+    input  [11:0] b,
+    input  [11:0] zeta,
+    input         mode,       // 0 = forward NTT, 1 = inverse NTT
+    output [11:0] a_out,
+    output [11:0] b_out,
+    output        valid_o
+);
+    localparam [11:0] Q12 = 12'd3329;
+    localparam [12:0] Q13 = 13'd3329;
+
+    wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12);
+    wire [11:0] mul_data = mode ? diff : b;
+
+    wire [11:0] mul_result;
+    wire        mul_valid;
+    barrett_mul_pipe u_barrett (
+        .clk(clk),
+        .rst_n(rst_n),
+        .valid_i(valid_i),
+        .a(zeta),
+        .b(mul_data),
+        .product(mul_result),
+        .valid_o(mul_valid)
+    );
+
+    reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5;
+    reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5;
+    reg        mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5;
+
+    reg [11:0] a_out_r, b_out_r;
+    reg        valid_r;
+
+    wire [12:0] a_sum = {1'b0, a_d5} + {1'b0, (mode_d5 ? b_d5 : mul_result)};
+    wire [11:0] a_mod = (a_sum >= Q13) ? (a_sum[11:0] - Q12) : a_sum[11:0];
+    wire [11:0] b_sub = (a_d5 >= mul_result) ? (a_d5 - mul_result) :
+                                                 (a_d5 - mul_result + Q12);
+
+    assign a_out = a_out_r;
+    assign b_out = b_out_r;
+    assign valid_o = valid_r;
+
+    always @(posedge clk or negedge rst_n) begin
+        if (!rst_n) begin
+            a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0;
+            b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0;
+            mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0;
+            a_out_r <= 12'd0;
+            b_out_r <= 12'd0;
+            valid_r <= 1'b0;
+        end else begin
+            a_d0 <= a;    a_d1 <= a_d0;    a_d2 <= a_d1;    a_d3 <= a_d2;    a_d4 <= a_d3;    a_d5 <= a_d4;
+            b_d0 <= b;    b_d1 <= b_d0;    b_d2 <= b_d1;    b_d3 <= b_d2;    b_d4 <= b_d3;    b_d5 <= b_d4;
+            mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
+            mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4;
+
+            valid_r <= mul_valid;
+            if (mul_valid) begin
+                a_out_r <= a_mod;
+                b_out_r <= mode_d5 ? mul_result : b_sub;
+            end
+        end
+    end
+
+endmodule