Pipeline ML-KEM datapath bottlenecks
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
// comp_decomp_sync.v - ML-KEM coefficient compression/decompression
|
||||
//
|
||||
// Streaming valid/ready interface. Compression is intentionally iterative to
|
||||
// avoid inferring a long combinational divider for /Q.
|
||||
// Streaming valid/ready interface. The arithmetic is fixed-latency pipelined
|
||||
// so compression does not infer a long combinational divider/reducer.
|
||||
// mode=0: compress — round((2^d * x) / Q) mod 2^d
|
||||
// mode=1: decompress — round((Q * x) / 2^d)
|
||||
// Uses round-half-up (round(2.5)=3, round(3.5)=4).
|
||||
@@ -11,7 +11,7 @@
|
||||
|
||||
`include "sync_rtl/common/defines.vh"
|
||||
|
||||
(* use_dsp = "no" *)
|
||||
(* use_dsp = "yes" *)
|
||||
module comp_decomp_sync (
|
||||
input clk,
|
||||
input rst_n,
|
||||
@@ -25,79 +25,169 @@ module comp_decomp_sync (
|
||||
input ready_i
|
||||
);
|
||||
|
||||
localparam [11:0] Q_VAL = 12'(`Q);
|
||||
localparam [4:0] COMP_MSB = 5'd22; // max ((Q-1)<<11)+Q/2 is 23 bits
|
||||
localparam [24:0] Q25 = 25'd`Q;
|
||||
localparam [12:0] K13 = 13'd5039; // floor(2^24 / 3329)
|
||||
|
||||
reg busy_r;
|
||||
reg [4:0] bit_idx_r;
|
||||
reg [4:0] d_r;
|
||||
reg [23:0] dividend_r;
|
||||
reg [23:0] quotient_r;
|
||||
reg [12:0] rem_r;
|
||||
reg [11:0] data_r;
|
||||
reg valid_r;
|
||||
|
||||
reg mode_s0;
|
||||
reg [4:0] d_s0;
|
||||
reg [11:0] coeff_s0;
|
||||
reg valid_s0;
|
||||
|
||||
reg mode_s1;
|
||||
reg [4:0] d_s1;
|
||||
reg [23:0] comp_dividend_s1;
|
||||
reg [23:0] decomp_rounded_s1;
|
||||
reg valid_s1;
|
||||
|
||||
reg mode_s2;
|
||||
reg [4:0] d_s2;
|
||||
reg [23:0] comp_dividend_s2;
|
||||
reg [36:0] comp_qprod_est_s2;
|
||||
reg [23:0] decomp_rounded_s2;
|
||||
reg valid_s2;
|
||||
|
||||
reg mode_s3;
|
||||
reg [4:0] d_s3;
|
||||
reg [23:0] comp_dividend_s3;
|
||||
reg [12:0] comp_q_est_s3;
|
||||
reg [11:0] decomp_result_s3;
|
||||
reg valid_s3;
|
||||
|
||||
reg mode_s4;
|
||||
reg [4:0] d_s4;
|
||||
reg [23:0] comp_dividend_s4;
|
||||
reg [12:0] comp_q_est_s4;
|
||||
reg [24:0] comp_q_mul_q_s4;
|
||||
reg [11:0] decomp_result_s4;
|
||||
reg valid_s4;
|
||||
|
||||
reg mode_s5;
|
||||
reg [4:0] d_s5;
|
||||
reg [12:0] comp_q_est_s5;
|
||||
reg [24:0] comp_rem_s5;
|
||||
reg [11:0] decomp_result_s5;
|
||||
reg valid_s5;
|
||||
|
||||
wire fire_i = valid_i && ready_o;
|
||||
|
||||
// Accept a new command only when the iterative divider and output slot are
|
||||
// both free. Current users tie ready_i high and issue one coefficient at a
|
||||
// time, but this keeps the module from accepting work it cannot retain.
|
||||
assign ready_o = !busy_r && (!valid_r || ready_i);
|
||||
wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5;
|
||||
|
||||
// Accept a new command only when the pipeline and output slot are both free.
|
||||
// Current users issue one coefficient at a time and wait for valid_o.
|
||||
assign ready_o = !pipe_busy && (!valid_r || ready_i);
|
||||
assign coeff_out = data_r;
|
||||
assign valid_o = valid_r;
|
||||
|
||||
wire [23:0] comp_dividend = ({12'd0, coeff_in} << d) + 24'd1664;
|
||||
wire [23:0] decomp_product =
|
||||
{12'd0, coeff_in} +
|
||||
({12'd0, coeff_in} << 8) +
|
||||
({12'd0, coeff_in} << 10) +
|
||||
({12'd0, coeff_in} << 11);
|
||||
wire [23:0] decomp_rounded = decomp_product + (24'd1 << (d - 1'b1));
|
||||
wire [11:0] decomp_result = decomp_rounded >> d;
|
||||
wire [23:0] coeff_ext_s0 = {12'd0, coeff_s0};
|
||||
wire [23:0] comp_dividend_next = (coeff_ext_s0 << d_s0) + 24'd1664;
|
||||
wire [23:0] decomp_product_next = coeff_s0 * 12'd`Q;
|
||||
wire [23:0] decomp_round_next =
|
||||
decomp_product_next + ((d_s0 == 5'd0) ? 24'd0 : (24'd1 << (d_s0 - 5'd1)));
|
||||
|
||||
wire [12:0] rem_shift = {rem_r[11:0], dividend_r[bit_idx_r]};
|
||||
wire rem_ge_q = rem_shift >= {1'b0, Q_VAL};
|
||||
wire [12:0] rem_sub_q = rem_shift - {1'b0, Q_VAL};
|
||||
wire [12:0] rem_next = rem_ge_q ? rem_sub_q : rem_shift;
|
||||
wire [23:0] quot_next = quotient_r | (rem_ge_q ? (24'd1 << bit_idx_r) : 24'd0);
|
||||
wire [11:0] comp_mask = (12'd1 << d_r) - 12'd1;
|
||||
wire [12:0] comp_q_corr1_s5 =
|
||||
comp_q_est_s5 + ((comp_rem_s5 >= Q25) ? 13'd1 : 13'd0);
|
||||
wire [24:0] comp_rem_corr1_s5 =
|
||||
(comp_rem_s5 >= Q25) ? (comp_rem_s5 - Q25) : comp_rem_s5;
|
||||
wire [12:0] comp_q_corr2_s5 =
|
||||
comp_q_corr1_s5 + ((comp_rem_corr1_s5 >= Q25) ? 13'd1 : 13'd0);
|
||||
wire [11:0] comp_mask_s5 =
|
||||
(d_s5 == 5'd0) ? 12'd0 : (12'd1 << d_s5) - 12'd1;
|
||||
wire [11:0] comp_result_s5 = comp_q_corr2_s5[11:0] & comp_mask_s5;
|
||||
|
||||
always @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
busy_r <= 1'b0;
|
||||
bit_idx_r <= 5'd0;
|
||||
d_r <= 5'd0;
|
||||
dividend_r <= 24'd0;
|
||||
quotient_r <= 24'd0;
|
||||
rem_r <= 13'd0;
|
||||
data_r <= 12'd0;
|
||||
valid_r <= 1'b0;
|
||||
data_r <= 12'd0;
|
||||
valid_r <= 1'b0;
|
||||
|
||||
mode_s0 <= 1'b0;
|
||||
d_s0 <= 5'd0;
|
||||
coeff_s0 <= 12'd0;
|
||||
valid_s0 <= 1'b0;
|
||||
|
||||
mode_s1 <= 1'b0;
|
||||
d_s1 <= 5'd0;
|
||||
comp_dividend_s1 <= 24'd0;
|
||||
decomp_rounded_s1 <= 24'd0;
|
||||
valid_s1 <= 1'b0;
|
||||
|
||||
mode_s2 <= 1'b0;
|
||||
d_s2 <= 5'd0;
|
||||
comp_dividend_s2 <= 24'd0;
|
||||
comp_qprod_est_s2 <= 37'd0;
|
||||
decomp_rounded_s2 <= 24'd0;
|
||||
valid_s2 <= 1'b0;
|
||||
|
||||
mode_s3 <= 1'b0;
|
||||
d_s3 <= 5'd0;
|
||||
comp_dividend_s3 <= 24'd0;
|
||||
comp_q_est_s3 <= 13'd0;
|
||||
decomp_result_s3 <= 12'd0;
|
||||
valid_s3 <= 1'b0;
|
||||
|
||||
mode_s4 <= 1'b0;
|
||||
d_s4 <= 5'd0;
|
||||
comp_dividend_s4 <= 24'd0;
|
||||
comp_q_est_s4 <= 13'd0;
|
||||
comp_q_mul_q_s4 <= 25'd0;
|
||||
decomp_result_s4 <= 12'd0;
|
||||
valid_s4 <= 1'b0;
|
||||
|
||||
mode_s5 <= 1'b0;
|
||||
d_s5 <= 5'd0;
|
||||
comp_q_est_s5 <= 13'd0;
|
||||
comp_rem_s5 <= 25'd0;
|
||||
decomp_result_s5 <= 12'd0;
|
||||
valid_s5 <= 1'b0;
|
||||
end else begin
|
||||
if (valid_r && ready_i)
|
||||
valid_r <= 1'b0;
|
||||
|
||||
if (busy_r) begin
|
||||
rem_r <= rem_next;
|
||||
quotient_r <= quot_next;
|
||||
if (bit_idx_r == 5'd0) begin
|
||||
busy_r <= 1'b0;
|
||||
data_r <= quot_next[11:0] & comp_mask;
|
||||
valid_r <= 1'b1;
|
||||
end else begin
|
||||
bit_idx_r <= bit_idx_r - 5'd1;
|
||||
end
|
||||
end else if (fire_i) begin
|
||||
if (mode) begin
|
||||
data_r <= decomp_result;
|
||||
valid_r <= 1'b1;
|
||||
end else begin
|
||||
busy_r <= 1'b1;
|
||||
bit_idx_r <= COMP_MSB;
|
||||
d_r <= d;
|
||||
dividend_r <= comp_dividend;
|
||||
quotient_r <= 24'd0;
|
||||
rem_r <= 13'd0;
|
||||
end
|
||||
mode_s0 <= mode;
|
||||
d_s0 <= d;
|
||||
coeff_s0 <= coeff_in;
|
||||
valid_s0 <= fire_i;
|
||||
|
||||
mode_s1 <= mode_s0;
|
||||
d_s1 <= d_s0;
|
||||
comp_dividend_s1 <= comp_dividend_next;
|
||||
decomp_rounded_s1 <= decomp_round_next;
|
||||
valid_s1 <= valid_s0;
|
||||
|
||||
mode_s2 <= mode_s1;
|
||||
d_s2 <= d_s1;
|
||||
comp_dividend_s2 <= comp_dividend_s1;
|
||||
comp_qprod_est_s2 <= {13'd0, comp_dividend_s1} * K13;
|
||||
decomp_rounded_s2 <= decomp_rounded_s1;
|
||||
valid_s2 <= valid_s1;
|
||||
|
||||
mode_s3 <= mode_s2;
|
||||
d_s3 <= d_s2;
|
||||
comp_dividend_s3 <= comp_dividend_s2;
|
||||
comp_q_est_s3 <= comp_qprod_est_s2[36:24];
|
||||
decomp_result_s3 <= decomp_rounded_s2 >> d_s2;
|
||||
valid_s3 <= valid_s2;
|
||||
|
||||
mode_s4 <= mode_s3;
|
||||
d_s4 <= d_s3;
|
||||
comp_dividend_s4 <= comp_dividend_s3;
|
||||
comp_q_est_s4 <= comp_q_est_s3;
|
||||
comp_q_mul_q_s4 <= comp_q_est_s3 * Q25;
|
||||
decomp_result_s4 <= decomp_result_s3;
|
||||
valid_s4 <= valid_s3;
|
||||
|
||||
mode_s5 <= mode_s4;
|
||||
d_s5 <= d_s4;
|
||||
comp_q_est_s5 <= comp_q_est_s4;
|
||||
comp_rem_s5 <= {1'b0, comp_dividend_s4} - comp_q_mul_q_s4;
|
||||
decomp_result_s5 <= decomp_result_s4;
|
||||
valid_s5 <= valid_s4;
|
||||
|
||||
if (valid_s5) begin
|
||||
data_r <= mode_s5 ? decomp_result_s5 : comp_result_s5;
|
||||
valid_r <= 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user