fallensigh/mlkem-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pipeline poly mul basecase issue

Browse Source
This commit is contained in:
FallenSigh
2026-07-08 01:04:01 +08:00
parent a0bd07bdf8
commit 7a7ccacb47
1 changed files with 89 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
sync_rtl/poly_mul/poly_mul_sync.v
View File

@@ -4,8 +4,10 @@
// 256-coefficient NTT-domain polynomials. // 256-coefficient NTT-domain polynomials.
// //
// Operation flow: // Operation flow:
// IDLE → LOAD (256× A+B pairs) → COMP_ISSUE → COMP_WAIT // IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
// → COMP_C0 (output c0) → COMP_C1 (output c1) → DONE → IDLE //
// RUN issues one base-case multiply every two cycles, matching the two-cycle
// c0/c1 output bandwidth after the basecase pipeline fills.
// //
// The LOAD phase accepts both A and B coefficients simultaneously // The LOAD phase accepts both A and B coefficients simultaneously
// (one pair per cycle) on coeff_a_in/coeff_b_in. // (one pair per cycle) on coeff_a_in/coeff_b_in.
@@ -22,7 +24,7 @@
// valid_i - Input valid // valid_i - Input valid
// ready_o - Ready to accept input (high in IDLE/LOAD) // ready_o - Ready to accept input (high in IDLE/LOAD)
// coeff_out[11:0] - Result coefficient output // coeff_out[11:0] - Result coefficient output
// valid_o - Output valid (high in COMP_C0/COMP_C1) // valid_o - Output valid during RUN output phase
// ready_i - Output consumer ready // ready_i - Output consumer ready
module poly_mul_sync ( module poly_mul_sync (
@@ -37,13 +39,9 @@ module poly_mul_sync (
); );
// State definitions // State definitions
localparam S_IDLE = 3'd0; localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1; localparam S_LOAD = 3'd1;
localparam S_COMP_ISSUE = 3'd2; localparam S_RUN = 3'd2;
localparam S_COMP_WAIT = 3'd3;
localparam S_COMP_C0 = 3'd4;
localparam S_COMP_C1 = 3'd5;
localparam S_DONE = 3'd6;
reg [2:0] state, next_state; reg [2:0] state, next_state;
@@ -53,7 +51,11 @@ module poly_mul_sync (
// Counters // Counters
reg [7:0] load_cnt; // 0..256 for loading 256 pairs reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [6:0] comp_k; // 0..127, current base-case index reg [7:0] issue_cnt; // 0..128, issued base-cases
reg [7:0] out_cnt; // 0..255, consumed output coefficients
reg issue_gap; // spaces base-case requests to match c0/c1 output
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Registered basecase_mul inputs/results // Registered basecase_mul inputs/results
reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_a0_reg, bc_a1_reg;
@@ -62,9 +64,10 @@ module poly_mul_sync (
reg bc_valid_reg; reg bc_valid_reg;
reg [11:0] c0_reg, c1_reg; reg [11:0] c0_reg, c1_reg;
// Combinational read signals for COMP_CALC // Combinational read signals for the next base-case request.
wire [7:0] addr_even = {comp_k, 1'b0}; // comp_k * 2 (7+1 = 8 bits) wire [6:0] issue_k = issue_cnt[6:0];
wire [7:0] addr_odd = {comp_k, 1'b1}; // comp_k * 2 + 1 wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1
wire [11:0] mem_a0 = mem_A[addr_even]; wire [11:0] mem_a0 = mem_A[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd]; wire [11:0] mem_a1 = mem_A[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even]; wire [11:0] mem_b0 = mem_B[addr_even];
@@ -73,12 +76,12 @@ module poly_mul_sync (
// Zeta ROM // Zeta ROM
wire [11:0] zeta; wire [11:0] zeta;
poly_mul_zeta_rom u_zeta ( poly_mul_zeta_rom u_zeta (
.addr (comp_k), .addr (issue_k),
.zeta (zeta) .zeta (zeta)
); );
// Pipelined basecase multiply. One request is issued at a time; inputs are // Pipelined basecase multiply. Requests are issued every two cycles; inputs
// registered locally so comp_k does not directly drive the DSP input muxes. // are registered locally so issue_cnt does not directly drive DSP input muxes.
wire [11:0] bc_c0, bc_c1; wire [11:0] bc_c0, bc_c1;
wire bc_vo; wire bc_vo;
basecase_mul_pipe u_bc ( basecase_mul_pipe u_bc (
@@ -97,24 +100,18 @@ module poly_mul_sync (
// Output interface // Output interface
assign ready_o = (state == S_IDLE) || (state == S_LOAD); assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = (state == S_COMP_C0) || (state == S_COMP_C1); assign valid_o = out_valid_r;
assign coeff_out = (state == S_COMP_C0) ? c0_reg : c1_reg; assign coeff_out = out_phase ? c1_reg : c0_reg;
// State transition logic (combinational) // State transition logic (combinational)
always @* begin always @* begin
next_state = state; next_state = state;
case (state) case (state)
S_IDLE: if (valid_i && ready_o) next_state = S_LOAD; S_IDLE: if (valid_i && ready_o) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 255 && valid_i && ready_o) S_LOAD: if (load_cnt >= 8'd255 && valid_i && ready_o)
next_state = S_COMP_ISSUE; next_state = S_RUN;
S_COMP_ISSUE: next_state = S_COMP_WAIT; S_RUN: if (valid_o && ready_i && out_cnt == 8'd255)
S_COMP_WAIT: if (bc_vo) next_state = S_COMP_C0; next_state = S_IDLE;
S_COMP_C0: if (valid_o && ready_i) next_state = S_COMP_C1;
S_COMP_C1: if (valid_o && ready_i) begin
if (comp_k >= 127) next_state = S_DONE;
else next_state = S_COMP_ISSUE;
end
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE; default: next_state = S_IDLE;
endcase endcase
end end
@@ -125,7 +122,11 @@ module poly_mul_sync (
if (!rst_n) begin if (!rst_n) begin
state <= S_IDLE; state <= S_IDLE;
load_cnt <= 8'd0; load_cnt <= 8'd0;
comp_k <= 7'd0; issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
bc_a0_reg <= 12'd0; bc_a0_reg <= 12'd0;
bc_a1_reg <= 12'd0; bc_a1_reg <= 12'd0;
bc_b0_reg <= 12'd0; bc_b0_reg <= 12'd0;
@@ -148,6 +149,11 @@ module poly_mul_sync (
mem_A[0] <= coeff_a_in; mem_A[0] <= coeff_a_in;
mem_B[0] <= coeff_b_in; mem_B[0] <= coeff_b_in;
load_cnt <= 8'd1; load_cnt <= 8'd1;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end end
// Subsequent coefficients in LOAD state // Subsequent coefficients in LOAD state
@@ -155,41 +161,65 @@ module poly_mul_sync (
mem_A[load_cnt] <= coeff_a_in; mem_A[load_cnt] <= coeff_a_in;
mem_B[load_cnt] <= coeff_b_in; mem_B[load_cnt] <= coeff_b_in;
load_cnt <= load_cnt + 8'd1; load_cnt <= load_cnt + 8'd1;
if (load_cnt >= 8'd255) begin
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end end
// ---- COMPUTE phase ---- // ---- COMPUTE phase ----
// COMP_ISSUE: cut the comp_k -> memory mux -> basecase DSP path. // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
// bc_valid_reg pulses on the following cycle, while these regs hold // regs, then launch requests every other cycle. The two-cycle issue
// stable inputs through the basecase pipeline launch. // interval matches the c0/c1 output bandwidth, so no deep result FIFO
if (state == S_COMP_ISSUE) begin // is needed for this always-ready system.
bc_a0_reg <= mem_a0; if (state == S_RUN) begin
bc_a1_reg <= mem_a1; if (issue_cnt < 8'd128) begin
bc_b0_reg <= mem_b0; if (!issue_gap) begin
bc_b1_reg <= mem_b1; bc_a0_reg <= mem_a0;
bc_zeta_reg <= zeta; bc_a1_reg <= mem_a1;
bc_valid_reg <= 1'b1; bc_b0_reg <= mem_b0;
bc_b1_reg <= mem_b1;
bc_zeta_reg <= zeta;
bc_valid_reg <= 1'b1;
issue_cnt <= issue_cnt + 8'd1;
issue_gap <= 1'b1;
end else begin
issue_gap <= 1'b0;
end
end
if (out_valid_r && ready_i) begin
out_cnt <= out_cnt + 8'd1;
if (out_cnt == 8'd255) begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end else if (!out_phase) begin
out_phase <= 1'b1;
end else begin
out_valid_r <= 1'b0;
out_phase <= 1'b0;
end
end
if (bc_vo) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
out_valid_r <= 1'b1;
out_phase <= 1'b0;
end
end end
// COMP_WAIT: capture pipelined basecase_mul results when ready. // ---- final output consumed: reset counters before accepting a new op ----
if (state == S_COMP_WAIT && bc_vo) begin if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
c0_reg <= bc_c0;
c1_reg <= bc_c1;
end
// COMP_C0 → COMP_C1: c0 was consumed, increment comp_k
if (state == S_COMP_C0 && valid_o && ready_i) begin
// comp_k stays same, c1 still to output
end
// COMP_C1 → COMP_CALC: c1 was consumed, advance to next pair
if (state == S_COMP_C1 && valid_o && ready_i) begin
comp_k <= comp_k + 7'd1;
end
// ---- DONE → IDLE: reset counters ----
if (state == S_DONE) begin
load_cnt <= 8'd0; load_cnt <= 8'd0;
comp_k <= 7'd0; issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end end
end end
end end