fallensigh/mlkem-sync
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize

Browse Source
This commit is contained in:
FallenSigh
2026-07-11 17:21:21 +08:00
parent 47fbcd31b6
commit d8d57db82a
9 changed files with 504 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
sync_rtl/comp_decomp/comp_decomp_sync.v
View File

@@ -75,9 +75,10 @@ module comp_decomp_sync (
wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5;
// Accept a new command only when the pipeline and output slot are both free.
// Current users issue one coefficient at a time and wait for valid_o.
assign ready_o = !pipe_busy && (!valid_r || ready_i);
// Fully pipelined: accept one coefficient per cycle (B1). Every stage
// advances unconditionally and the sole consumer ties ready_i high, so the
// pipeline never needs to backpressure; streaming feeders issue 1 coeff/cyc.
assign ready_o = 1'b1;
assign coeff_out = data_r;
assign valid_o = valid_r;

2
sync_rtl/ntt/TB/xsim_run.tcl
View File

@@ -40,6 +40,8 @@ xvlog -sv ${SRC_DIR}/butterfly_unit.v
xvlog -sv ${SRC_DIR}/butterfly_unit_pipe.v
# NTT core (FSM-based, instantiates butterfly_unit + zeta_rom + barrett_mul)
xvlog -sv ${SRC_DIR}/../storage/sd_bram.v
xvlog -sv ${SRC_DIR}/coeff_bram_4way.v
xvlog -sv ${SRC_DIR}/ntt_core.v
# ================================================================

29
sync_rtl/ntt/butterfly_unit_pipe.v
View File

@@ -18,21 +18,32 @@ module butterfly_unit_pipe (
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] Q13 = 13'd3329;
wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12);
wire [11:0] mul_data = mode ? diff : b;
// Plan A: register the inputs. Operands arrive combinationally from the
// coeff_bram read; without this stage the path coeff_bram-read -> diff ->
// first multiply stage sits in one cycle (the mlkem_clk critical path at
// 100 MHz). Registering here splits it into (BRAM read -> reg) and
// (reg -> diff -> multiply). Adds one pipeline cycle (absorbed by the NTT
// writeback delay line).
reg [11:0] a_r, b_r, zeta_r;
reg mode_ri, vld_r;
wire [11:0] diff = (b_r >= a_r) ? (b_r - a_r) : (b_r - a_r + Q12);
wire [11:0] mul_data = mode_ri ? diff : b_r;
wire [11:0] mul_result;
wire mul_valid;
barrett_mul_pipe u_barrett (
.clk(clk),
.rst_n(rst_n),
.valid_i(valid_i),
.a(zeta),
.valid_i(vld_r),
.a(zeta_r),
.b(mul_data),
.product(mul_result),
.valid_o(mul_valid)
);
// a/b/mode delay lines (fed from the REGISTERED inputs a_r/b_r/mode_ri so
// a_d6 aligns with mul_valid = valid_i delayed 8).
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6;
@@ -51,6 +62,7 @@ module butterfly_unit_pipe (
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
a_r <= 12'd0; b_r <= 12'd0; zeta_r <= 12'd0; mode_ri <= 1'b0; vld_r <= 1'b0;
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0;
@@ -58,9 +70,12 @@ module butterfly_unit_pipe (
b_out_r <= 12'd0;
valid_r <= 1'b0;
end else begin
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
// input register stage
a_r <= a; b_r <= b; zeta_r <= zeta; mode_ri <= mode; vld_r <= valid_i;
a_d0 <= a_r; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b_r; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode_ri; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5;
valid_r <= mul_valid;

113
sync_rtl/ntt/coeff_bram_4way.v Normal file
View File

@@ -0,0 +1,113 @@
// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients
//
// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM
// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree.
//
// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits)
// local(a) = a[7:1] (7-bit index into the 128-deep bank)
//
// This function makes a dual-butterfly's four simultaneous coefficient accesses
// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT
// layer_len in {2,4,...,128}: j is even and len is even, so
// j -> {0, P }
// j+1 -> {1, P }
// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity)
// j+1+len -> {1, P'}
// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four
// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank
// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong.
//
// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this
// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle.
//
// NOTE: the caller must guarantee the four read addresses map to four distinct
// banks (true for the NTT butterfly pattern) and the four write addresses too.
// Sequential access (load/output) uses a single port and is always safe.
module coeff_bram_4way (
input clk, rst_n,
// Four read addresses (global 8-bit); data returns 1 cycle later.
input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3,
output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3,
// Four write ports.
input wr_en_0, wr_en_1, wr_en_2, wr_en_3,
input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3,
input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3
);
// Bank id and local index for an address.
function [1:0] bank_id;
input [7:0] a;
bank_id = {a[0], ^a[7:1]};
endfunction
wire [1:0] rb0 = bank_id(rd_addr_0);
wire [1:0] rb1 = bank_id(rd_addr_1);
wire [1:0] rb2 = bank_id(rd_addr_2);
wire [1:0] rb3 = bank_id(rd_addr_3);
wire [1:0] wb0 = bank_id(wr_addr_0);
wire [1:0] wb1 = bank_id(wr_addr_1);
wire [1:0] wb2 = bank_id(wr_addr_2);
wire [1:0] wb3 = bank_id(wr_addr_3);
// Per-bank read-address crossbar: bank b reads the local index of whichever
// read port targets it (exactly one, by construction).
wire [6:0] bank_rd_local [0:3];
wire [6:0] bank_wr_local [0:3];
wire bank_wr_en [0:3];
wire [11:0] bank_wr_data [0:3];
wire [11:0] bank_rd_data [0:3];
genvar b;
generate
for (b = 0; b < 4; b = b + 1) begin : gen_xbar
// read address route (priority: 3>2>1>0; ties impossible for NTT)
assign bank_rd_local[b] =
(rb3 == b[1:0]) ? rd_addr_3[7:1] :
(rb2 == b[1:0]) ? rd_addr_2[7:1] :
(rb1 == b[1:0]) ? rd_addr_1[7:1] :
rd_addr_0[7:1];
// write route
wire m0 = wr_en_0 && (wb0 == b[1:0]);
wire m1 = wr_en_1 && (wb1 == b[1:0]);
wire m2 = wr_en_2 && (wb2 == b[1:0]);
wire m3 = wr_en_3 && (wb3 == b[1:0]);
assign bank_wr_en[b] = m0 | m1 | m2 | m3;
assign bank_wr_local[b] =
m3 ? wr_addr_3[7:1] :
m2 ? wr_addr_2[7:1] :
m1 ? wr_addr_1[7:1] : wr_addr_0[7:1];
assign bank_wr_data[b] =
m3 ? wr_data_3 :
m2 ? wr_data_2 :
m1 ? wr_data_1 : wr_data_0;
sd_bram #(.W(12), .D(128), .A(7)) u_bank (
.clk(clk),
.rd_addr(bank_rd_local[b]),
.rd_data(bank_rd_data[b]),
.wr_en(bank_wr_en[b]),
.wr_addr(bank_wr_local[b]),
.wr_data(bank_wr_data[b])
);
end
endgenerate
// Register the read bank-ids to align with sd_bram's 1-cycle read latency,
// then route each bank's output back to the requesting read port.
reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0;
end else begin
rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3;
end
end
assign rd_data_0 = bank_rd_data[rb0_r];
assign rd_data_1 = bank_rd_data[rb1_r];
assign rd_data_2 = bank_rd_data[rb2_r];
assign rd_data_3 = bank_rd_data[rb3_r];
endmodule

311
sync_rtl/ntt/ntt_core.v
View File

@@ -1,12 +1,18 @@
// ntt_core.v - NTT core with individual coefficient registers
// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage
//
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
// The arithmetic hot path is split into:
// address/operand/zeta register -> pipelined Barrett butterfly -> writeback
// Each NTT layer is issued continuously into the butterfly pipeline, then the
// core drains pending writebacks before starting the next dependent layer.
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way
// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port
// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle.
//
// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline
// gains one stage:
// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid)
// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer.
// Read and writeback hit disjoint banks (butterfly index sets in a layer are
// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W).
//
// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read
// latency (ready_i is tied high by all NTT consumers, so no output backpressure).
module ntt_core (
input clk, rst_n,
@@ -21,9 +27,6 @@ module ntt_core (
);
localparam N = 256, LAYERS = 7, DW = 12;
reg [DW-1:0] cr [0:N-1];
integer ci;
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_RUN = 4'd2;
@@ -36,69 +39,122 @@ module ntt_core (
reg [3:0] state, next_state;
reg [7:0] load_cnt;
reg [7:0] out_cnt;
reg [8:0] scale_issue_cnt;
reg [8:0] scale_emit_cnt;
reg [8:0] out_rptr; // output read address pointer (0..256)
reg [8:0] out_cnt; // output coeffs emitted (0..256)
reg [8:0] scale_rptr; // scale read address pointer
reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett
reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
reg mode_r;
reg layer_issue_done;
reg [4:0] bf_inflight;
reg [5:0] bf_inflight;
reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7;
reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7;
reg wb_valid_r;
reg [DW-1:0] wb_a_data_r, wb_b_data_r;
reg [7:0] wb_wa_r, wb_wb_r;
reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed
// 9-deep writeback address delay lines (BRAM read + 8-stage butterfly).
reg [7:0] a0d[0:9]; reg [7:0] b0d[0:9];
reg [7:0] a1d[0:9]; reg [7:0] b1d[0:9];
integer k;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
reg scale_valid_i;
reg [DW-1:0] scale_a_i;
reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present)
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end);
wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end);
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
wire [DW-1:0] issue_a = cr[j];
wire [DW-1:0] issue_b = cr[j + layer_len];
wire [DW-1:0] bf_a_out, bf_b_out;
wire bf_valid;
butterfly_unit_pipe u_bf (
.clk(clk),
.rst_n(rst_n),
.valid_i(issue_fire),
.a(issue_a),
.b(issue_b),
.zeta(zeta),
.mode(mode_r),
.a_out(bf_a_out),
.b_out(bf_b_out),
.valid_o(bf_valid)
// ---- coeff_bram_4way port muxing ----
// Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr).
wire [7:0] cmp_a0 = j;
wire [7:0] cmp_b0 = j + layer_len;
wire [7:0] cmp_a1 = j + 8'd1;
wire [7:0] cmp_b1 = j + 8'd1 + layer_len;
wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] :
(state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0;
// Ports 1..3 carry the other three butterfly operands ONLY during compute;
// otherwise they must mirror port 0 so the read crossbar's per-bank priority
// arbitration doesn't route a stale compute address into port 0's bank.
wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN);
wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0;
wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0;
wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0;
wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3;
// Butterflies fed from BRAM read data one cycle after the address (rd_fire_r).
wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out;
wire bf0_valid, bf1_valid;
butterfly_unit_pipe u_bf0 (
.clk(clk), .rst_n(rst_n),
.valid_i(rd_fire_r),
.a(rd_data_0), .b(rd_data_1),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid)
);
butterfly_unit_pipe u_bf1 (
.clk(clk), .rst_n(rst_n),
.valid_i(rd_fire_r),
.a(rd_data_2), .b(rd_data_3),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid)
);
// zeta must align with the BRAM read data (1 cycle after presentation).
// Scale multiplier (inverse NTT final x3303).
wire [DW-1:0] scale_product;
wire scale_valid_o;
barrett_mul_pipe u_scl (
.clk(clk),
.rst_n(rst_n),
.clk(clk), .rst_n(rst_n),
.valid_i(scale_valid_i),
.a(scale_a_i),
.b(12'd3303),
.product(scale_product),
.valid_o(scale_valid_o)
.a(scale_a_i), .b(12'd3303),
.product(scale_product), .valid_o(scale_valid_o)
);
// ---- write ports ----
wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i);
wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt;
wire wr_en_0 = loading_we | bf0_valid;
wire wr_en_1 = bf0_valid & ~loading_we;
wire wr_en_2 = bf0_valid & ~loading_we;
wire wr_en_3 = bf0_valid & ~loading_we;
wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[9];
wire [7:0] wr_addr_1 = b0d[9];
wire [7:0] wr_addr_2 = a1d[9];
wire [7:0] wr_addr_3 = b1d[9];
wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out;
wire [11:0] wr_data_1 = bf0_b_out;
wire [11:0] wr_data_2 = bf1_a_out;
wire [11:0] wr_data_3 = bf1_b_out;
coeff_bram_4way u_cr (
.clk(clk), .rst_n(rst_n),
.rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1),
.rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3),
.rd_data_0(rd_data_0), .rd_data_1(rd_data_1),
.rd_data_2(rd_data_2), .rd_data_3(rd_data_3),
.wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3),
.wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1),
.wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3),
.wr_data_0(wr_data_0), .wr_data_1(wr_data_1),
.wr_data_2(wr_data_2), .wr_data_3(wr_data_3)
);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign coeff_out = coeff_out_r;
// During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle
// latency, matching the old combinational-read + 1 output reg). Scale path
// uses the registered barrett result.
assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r;
assign valid_o = valid_o_r;
assign done_o = (state == S_DONE);
@@ -108,10 +164,10 @@ module ntt_core (
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN;
S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN;
S_CMP_DRAIN: if (bf_inflight == 5'd0)
S_CMP_DRAIN: if (bf_inflight == 6'd0)
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE;
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
@@ -122,145 +178,104 @@ module ntt_core (
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer_len <= 8'd0;
zeta_idx <= 7'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0;
wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0;
wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0;
wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0;
wb_valid_r <= 1'b0;
wb_a_data_r <= 12'd0;
wb_b_data_r <= 12'd0;
wb_wa_r <= 8'd0;
wb_wb_r <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
scale_a_i <= 12'd0;
for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer_len <= 8'd0;
zeta_idx <= 7'd0; zeta_fed <= 12'd0;
layer <= 3'd0; mode_r <= 1'b0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
rd_fire_r <= 1'b0;
for (k = 0; k <= 9; k = k + 1) begin
a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0;
end
coeff_out_r <= 12'd0; valid_o_r <= 1'b0;
scale_valid_i <= 1'b0; scale_a_i <= 12'd0;
end else begin
state <= next_state;
scale_valid_i <= 1'b0;
wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2;
wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6;
wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2;
wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6;
// read->feed pipeline register (BRAM latency) + zeta alignment
rd_fire_r <= issue_fire;
zeta_fed <= zeta;
if (issue_fire && !wb_valid_r)
bf_inflight <= bf_inflight + 5'd1;
else if (!issue_fire && wb_valid_r)
bf_inflight <= bf_inflight - 5'd1;
// writeback address delay lines: sample j-derived addrs at read cycle
a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1;
for (k = 1; k <= 9; k = k + 1) begin
a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1];
a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1];
end
// in-flight pair accounting: +1 on issue, -1 on writeback
if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1;
else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt <= 8'd1;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer <= 3'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
zeta_idx <= 7'd1;
end else begin
layer_len <= 8'd2;
zeta_idx <= 7'd127;
end
if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end
else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end
end
if (state == S_LOAD && valid_i) begin
cr[load_cnt] <= coeff_in;
load_cnt <= load_cnt + 8'd1;
end
if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1;
// ---- issue advance (read pointer j, step 2) ----
if (state == S_CMP_RUN && !layer_issue_done) begin
if (issue_group_last) begin
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if (issue_layer_last) begin
layer_issue_done <= 1'b1;
if (issue_layer_last) layer_issue_done <= 1'b1;
else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end
end else begin
start <= next_group_start[7:0];
j <= next_group_start[7:0];
end
end else begin
j <= j + 8'd1;
j <= j + 8'd2;
end
end
if (wb_valid_r) begin
cr[wb_wa_r] <= wb_a_data_r;
cr[wb_wb_r] <= wb_b_data_r;
end
wb_valid_r <= bf_valid;
if (bf_valid) begin
wb_a_data_r <= bf_a_out;
wb_b_data_r <= bf_b_out;
wb_wa_r <= wa_d7;
wb_wb_r <= wb_d7;
end
if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin
if (layer + 3'd1 >= LAYERS) begin
bf_done <= 1'b1;
end else begin
// ---- layer advance after drain ----
if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin
if (layer + 3'd1 < LAYERS) begin
layer <= layer + 3'd1;
layer_len <= next_layer_len;
start <= 8'd0;
j <= 8'd0;
start <= 8'd0; j <= 8'd0;
layer_issue_done <= 1'b0;
end
end
// ---- OUT_PREP: reset output pointers ----
if (state == S_OUT_PREP) begin
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
if (!mode_r) begin
coeff_out_r <= cr[0];
valid_o_r <= 1'b1;
end
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
// (scale first addr 0 is presented via out_rptr in this PREP cycle)
end
if (state == S_OUTPUT && valid_o_r && ready_i) begin
if (out_cnt < 8'd255) begin
out_cnt <= out_cnt + 8'd1;
coeff_out_r <= cr[out_cnt + 8'd1];
valid_o_r <= 1'b1;
// ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0;
// valid_o is a registered "presented a valid addr last cycle". Present
// addresses out_rptr = 0..255 (256), giving 256 valid outputs. ----
if (state == S_OUTPUT) begin
if (out_rptr < 9'd256) begin
out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr
valid_o_r <= 1'b1; // data valid next cycle
end else begin
out_cnt <= 8'd0;
valid_o_r <= 1'b0;
end
if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted
end
// ---- OUT_SCALE: read coeff, x3303 via barrett, emit ----
if (state == S_OUT_SCALE) begin
if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1;
// feed barrett with the coeff that arrived this cycle
if (scale_issue_cnt < 9'd256) begin
scale_valid_i <= 1'b1;
scale_a_i <= cr[scale_issue_cnt[7:0]];
scale_a_i <= rd_data_0;
scale_issue_cnt <= scale_issue_cnt + 9'd1;
end
valid_o_r <= 1'b0;
if (scale_valid_o) begin
coeff_out_r <= scale_product;
@@ -271,15 +286,11 @@ module ntt_core (
if (state == S_DONE) begin
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wb_valid_r <= 1'b0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
valid_o_r <= 1'b0;
end
end
end
endmodule

1
sync_rtl/poly_mul/TB/xsim_run.tcl
View File

@@ -46,6 +46,7 @@ xvlog -sv ${PM_DIR}/basecase_mul_pipe.v
xvlog -sv ${PM_DIR}/poly_mul_zeta_rom.v
# PolyMul sync top (instantiates basecase_mul + poly_mul_zeta_rom)
xvlog -sv ${PM_DIR}/../storage/sd_bram.v
xvlog -sv ${PM_DIR}/poly_mul_sync.v
# ================================================================

124
sync_rtl/poly_mul/poly_mul_sync.v
View File

@@ -1,4 +1,4 @@
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier
// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (BRAM storage)
//
// Computes pointwise (Karatsuba-like base-case) multiplication of two
// 256-coefficient NTT-domain polynomials.
@@ -6,18 +6,18 @@
// Operation flow:
// IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE
//
// RUN issues one base-case multiply every two cycles, matching the two-cycle
// c0/c1 output bandwidth after the basecase pipeline fills.
// Coefficient storage moved from 256x12 LUT register arrays to four sd_bram
// banks (A_even/A_odd/B_even/B_odd, each 128x12): the base-case reads
// mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single
// read port. This removes the large multi-port LUT mux (~8k LUT) and the long
// BRAM->register-file routing that was the mlkem_clk critical path.
//
// The LOAD phase accepts both A and B coefficients simultaneously
// (one pair per cycle) on coeff_a_in/coeff_b_in.
// RUN issues one base-case every two cycles. sd_bram reads are synchronous
// (1-cycle latency); the 2-cycle cadence absorbs it (address presented on the
// gap cycle, data consumed on the issue cycle). A one-cycle priming gap is
// inserted at LOAD->RUN so the first base-case's operands are ready.
//
// The COMPUTE phase outputs the 256 result coefficients one per cycle
// via valid/ready handshake on coeff_out/valid_o/ready_i.
//
// Memory: 256×12-bit register arrays for A and B coefficients.
//
// Interface:
// Interface (unchanged):
// clk, rst_n - Clock, active-low reset
// coeff_a_in[11:0]- Polynomial A coefficient input
// coeff_b_in[11:0]- Polynomial B coefficient input
@@ -38,18 +38,12 @@ module poly_mul_sync (
input ready_i
);
// State definitions
localparam S_IDLE = 3'd0;
localparam S_LOAD = 3'd1;
localparam S_RUN = 3'd2;
reg [2:0] state, next_state;
// Coefficient storage (register arrays)
reg [11:0] mem_A [0:255];
reg [11:0] mem_B [0:255];
// Counters
reg [7:0] load_cnt; // 0..256 for loading 256 pairs
reg [7:0] issue_cnt; // 0..128, issued base-cases
reg [7:0] out_cnt; // 0..255, consumed output coefficients
@@ -57,53 +51,61 @@ module poly_mul_sync (
reg out_phase; // 0=c0, 1=c1
reg out_valid_r;
// Registered basecase_mul inputs/results
reg [11:0] bc_a0_reg, bc_a1_reg;
reg [11:0] bc_b0_reg, bc_b1_reg;
reg [11:0] bc_zeta_reg;
reg bc_valid_reg;
reg [11:0] c0_reg, c1_reg;
// Combinational read signals for the next base-case request.
wire [6:0] issue_k = issue_cnt[6:0];
wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2
wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1
wire [11:0] mem_a0 = mem_A[addr_even];
wire [11:0] mem_a1 = mem_A[addr_odd];
wire [11:0] mem_b0 = mem_B[addr_even];
wire [11:0] mem_b1 = mem_B[addr_odd];
// ---- Coefficient storage: even/odd banked sd_bram ----
// A_even[k]=mem_A[2k], A_odd[k]=mem_A[2k+1]; same for B. Read index during
// RUN is issue_k; write index during LOAD is load_cnt>>1 routed by load_cnt[0].
wire [6:0] issue_k = issue_cnt[6:0]; // 0..127 read index
// Zeta ROM
wire [11:0] zeta;
poly_mul_zeta_rom u_zeta (
.addr (issue_k),
.zeta (zeta)
wire load_fire = ((state == S_IDLE) || (state == S_LOAD)) && valid_i && ready_o;
wire [6:0] wr_local = load_cnt[7:1]; // load_cnt/2
wire wr_odd = load_cnt[0];
wire we_even = load_fire && !wr_odd;
wire we_odd = load_fire && wr_odd;
wire [11:0] mem_a0, mem_a1, mem_b0, mem_b1; // sd_bram registered read outputs
sd_bram #(.W(12), .D(128), .A(7)) u_a_even (
.clk(clk), .rd_addr(issue_k), .rd_data(mem_a0),
.wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_a_in)
);
sd_bram #(.W(12), .D(128), .A(7)) u_a_odd (
.clk(clk), .rd_addr(issue_k), .rd_data(mem_a1),
.wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_a_in)
);
sd_bram #(.W(12), .D(128), .A(7)) u_b_even (
.clk(clk), .rd_addr(issue_k), .rd_data(mem_b0),
.wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_b_in)
);
sd_bram #(.W(12), .D(128), .A(7)) u_b_odd (
.clk(clk), .rd_addr(issue_k), .rd_data(mem_b1),
.wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_b_in)
);
// Pipelined basecase multiply. Requests are issued every two cycles; inputs
// are registered locally so issue_cnt does not directly drive DSP input muxes.
wire [11:0] zeta;
poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta));
wire [11:0] bc_c0, bc_c1;
wire bc_vo;
basecase_mul_pipe u_bc (
.clk (clk),
.rst_n(rst_n),
.clk(clk), .rst_n(rst_n),
.valid_i(bc_valid_reg),
.a0 (bc_a0_reg),
.a1 (bc_a1_reg),
.b0 (bc_b0_reg),
.b1 (bc_b1_reg),
.a0(bc_a0_reg), .a1(bc_a1_reg),
.b0(bc_b0_reg), .b1(bc_b1_reg),
.zeta(bc_zeta_reg),
.c0 (bc_c0),
.c1 (bc_c1),
.c0(bc_c0), .c1(bc_c1),
.valid_o(bc_vo)
);
// Output interface
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign valid_o = out_valid_r;
assign coeff_out = out_phase ? c1_reg : c0_reg;
// State transition logic (combinational)
always @* begin
next_state = state;
case (state)
@@ -116,8 +118,6 @@ module poly_mul_sync (
endcase
end
// Sequential logic
integer i;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state <= S_IDLE;
@@ -127,27 +127,16 @@ module poly_mul_sync (
issue_gap <= 1'b0;
out_phase <= 1'b0;
out_valid_r <= 1'b0;
bc_a0_reg <= 12'd0;
bc_a1_reg <= 12'd0;
bc_b0_reg <= 12'd0;
bc_b1_reg <= 12'd0;
bc_zeta_reg <= 12'd0;
bc_valid_reg <= 1'b0;
c0_reg <= 12'd0;
c1_reg <= 12'd0;
for (i = 0; i < 256; i = i + 1) begin
mem_A[i] <= 12'd0;
mem_B[i] <= 12'd0;
end
bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0;
bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0;
bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0;
c0_reg <= 12'd0; c1_reg <= 12'd0;
end else begin
state <= next_state;
bc_valid_reg <= 1'b0;
// ---- LOAD phase ----
// First coefficient captured on IDLE → LOAD transition
// ---- LOAD phase (writes routed to even/odd banks combinationally) ----
if (state == S_IDLE && valid_i && ready_o) begin
mem_A[0] <= coeff_a_in;
mem_B[0] <= coeff_b_in;
load_cnt <= 8'd1;
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
@@ -156,25 +145,21 @@ module poly_mul_sync (
out_valid_r <= 1'b0;
end
// Subsequent coefficients in LOAD state
if (state == S_LOAD && valid_i && ready_o) begin
mem_A[load_cnt] <= coeff_a_in;
mem_B[load_cnt] <= coeff_b_in;
load_cnt <= load_cnt + 8'd1;
if (load_cnt >= 8'd255) begin
issue_cnt <= 8'd0;
out_cnt <= 8'd0;
issue_gap <= 1'b0;
issue_gap <= 1'b1; // priming gap: present issue_k=0 addr first
out_phase <= 1'b0;
out_valid_r <= 1'b0;
end
end
// ---- COMPUTE phase ----
// RUN: cut issue_cnt -> memory mux -> basecase DSP with local input
// regs, then launch requests every other cycle. The two-cycle issue
// interval matches the c0/c1 output bandwidth, so no deep result FIFO
// is needed for this always-ready system.
// Issue one base-case every two cycles. On the gap cycle the sd_bram
// address (issue_k) settles; on the issue cycle its registered output
// (mem_a0/a1/b0/b1) is captured into the base-case input registers.
if (state == S_RUN) begin
if (issue_cnt < 8'd128) begin
if (!issue_gap) begin
@@ -212,7 +197,6 @@ module poly_mul_sync (
end
end
// ---- final output consumed: reset counters before accepting a new op ----
if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin
load_cnt <= 8'd0;
issue_cnt <= 8'd0;

2
sync_rtl/top/TB/xsim_run.tcl
View File

@@ -17,6 +17,8 @@ xvlog -sv --relax -i . sync_rtl/ntt/barrett_mul_pipe.v
xvlog -sv --relax -i . sync_rtl/ntt/zeta_rom.v
xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit.v
xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit_pipe.v
xvlog -sv --relax -i . sync_rtl/storage/sd_bram.v
xvlog -sv --relax -i . sync_rtl/ntt/coeff_bram_4way.v
xvlog -sv --relax -i . sync_rtl/ntt/ntt_core.v
xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul.v
xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul_pipe.v

197
sync_rtl/top/mlkem_top.v
View File

@@ -22,7 +22,7 @@
`include "sync_rtl/common/defines.vh"
(* use_dsp = "yes" *)
module mlkem_top #(
parameter KMAX = 4 // storage sizing (worst case = ML-KEM-1024)
) (
@@ -90,6 +90,9 @@ module mlkem_top #(
);
localparam Q = `Q; // 3329
localparam [11:0] Q12 = `Q;
localparam [12:0] Q13 = `Q;
localparam [13:0] Q14 = `Q;
// Runtime ML-KEM parameter, captured at start_i.
reg [2:0] k_r;
@@ -309,15 +312,15 @@ module mlkem_top #(
// D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT].
wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo;
wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx;
assign bt_we = ((st == ST_M) && pm_vo) || e2_we ||
assign bt_we = ((st == ST_M) && pm_vo) || cbdw_bt_en ||
u_psum_we || u_intt_we || u_v_we || u_w_we;
assign bt_wa = e2_we ? (c_widx & ((1<<PT_AW)-1)) :
assign bt_wa = cbdw_bt_en ? ({6'd0, cbdw_widx} & ((1<<PT_AW)-1)) :
u_psum_we ? ((UPSUM*256 + u_oidx) & ((1<<PT_AW)-1)) :
u_intt_we ? (u_intt_wr[PT_AW-1:0]) :
u_v_we ? (u_vadd_wr[PT_AW-1:0]) :
u_w_we ? (u_wsub_wr[PT_AW-1:0]) :
((m_i*256 + m_oidx) & ((1<<PT_AW)-1));
assign bt_wd = e2_we ? cbd_modq :
assign bt_wd = cbdw_bt_en ? cbdw_data :
u_psum_we ? u_accq :
u_intt_we ? ntt_coeff :
u_v_we ? u_vq :
@@ -378,16 +381,16 @@ module mlkem_top #(
// bank_se rel slot dec_poly (0..K-1), at coeff index dec_widx.
wire dec_u_we = (st == ST_DEC_DECOMP) && !dec_in_c2 && cd_vo;
wire [13:0] dec_u_wr = {2'b0, dec_poly}*256 + dec_widx;
assign bse_we = ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly < {1'b0, k_r, 1'b0})) ||
assign bse_we = cbdw_se_en ||
(((st == ST_N) || (st == ST_ENC_N) || (st == ST_DEC_NTT)) && ntt_vo) ||
u_add_we || dec_u_we;
assign bse_wa = u_add_we ? u_add_uwr[PSE_AW-1:0] :
assign bse_wa = cbdw_se_en ? (({1'b0, cbdw_poly}*256 + cbdw_widx) & ((1<<PSE_AW)-1)) :
u_add_we ? u_add_uwr[PSE_AW-1:0] :
dec_u_we ? dec_u_wr[PSE_AW-1:0] :
(st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ((n_slot*256 + n_widx) & ((1<<PSE_AW)-1))
: ((c_poly*256 + c_widx) & ((1<<PSE_AW)-1));
assign bse_wd = u_add_we ? u_uq :
assign bse_wd = cbdw_se_en ? cbdw_data :
u_add_we ? u_uq :
dec_u_we ? cd_out :
(st == ST_N || st == ST_ENC_N || st == ST_DEC_NTT) ? ntt_coeff : cbd_modq;
always @(posedge clk) begin
@@ -560,9 +563,9 @@ module mlkem_top #(
// per-consumer drives toward the core
wire [1599:0] sha3_kc_state_i, snt_kc_state_i, cbd_kc_state_i;
wire sha3_kc_valid_i, snt_kc_valid_i, cbd_kc_valid_i;
/* verilator lint_off UNUSEDSIGNAL */
/* verilator lint_off UNUSED */
wire sha3_kc_ready_i, snt_kc_ready_i, cbd_kc_ready_i; // all 1'b1
/* verilator lint_on UNUSEDSIGNAL */
/* verilator lint_on UNUSED */
// phase selects (mutually exclusive). Encaps adds ST_ENC_H/G (sha3),
// ST_ENC_A (snt), ST_ENC_C (cbd).
@@ -792,7 +795,35 @@ module mlkem_top #(
);
// signed (two's complement) -> [0,Q): add Q when negative
wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'(Q)) : cbd_coeff;
wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff;
// Pipeline the CBD -> bank write by one cycle. The squeeze_buf[buf_ptr]
// extraction -> CBD add/sub -> mod-Q -> BRAM-write route is long and
// routing-dominated (the mlkem_clk critical path at 100 MHz). Register the
// write data/addr/enable here; the bank write commits one cycle later. No
// consumer reads these coeffs until the following stage, which begins more
// than one cycle after the last CBD coeff, so there is no read-after-write
// hazard. The registered write takes priority in the bank_se/bank_t mux so
// the 1-cycle spillover into the next state uses the correct addr/data.
reg cbdw_se_en, cbdw_bt_en;
reg [4:0] cbdw_poly;
reg [7:0] cbdw_widx;
reg [11:0] cbdw_data;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
cbdw_se_en <= 1'b0; cbdw_bt_en <= 1'b0;
cbdw_poly <= 5'd0; cbdw_widx <= 8'd0; cbdw_data <= 12'd0;
end else begin
cbdw_se_en <= ((st == ST_C) && c_busy && cbd_vo && cbd_ack) ||
((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly < {1'b0, k_r, 1'b0}));
cbdw_bt_en <= (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack &&
(c_poly == {1'b0, k_r, 1'b0});
cbdw_poly <= c_poly;
cbdw_widx <= c_widx;
cbdw_data <= cbd_modq;
end
end
// ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ----
// N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot).
@@ -917,7 +948,7 @@ module mlkem_top #(
wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC)
wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0
wire [12:0] u_sum = {1'b0, u_acc_src} + {1'b0, pm_coeff};
wire [11:0] u_accq = (u_sum >= 13'(Q)) ? (u_sum - 13'(Q)) : u_sum[11:0];
wire [11:0] u_accq = (u_sum >= Q13) ? (u_sum - Q13) : u_sum[11:0];
// INTT load addr (psum read) + in-place writeback to bank_t[UPSUM]
wire [13:0] u_intt_rd = UPSUM*256 + u_ridx[7:0];
wire [13:0] u_intt_wr = UPSUM*256 + u_widx;
@@ -926,7 +957,7 @@ module mlkem_top #(
wire [13:0] u_add_e1rd = ({2'b0,k_r}+{2'b0,u_row})*256 + u_aidx[7:0]; // bank_se e1 (lead)
wire [13:0] u_add_uwr = ({2'b0,k_r}+{2'b0,u_row})*256 + u_awidx; // bank_se u (write)
wire [12:0] u_usum = {1'b0, bt_rd_data} + {1'b0, bse_rd_data}; // psum + e1
wire [11:0] u_uq = (u_usum >= 13'(Q)) ? (u_usum - 13'(Q)) : u_usum[11:0];
wire [11:0] u_uq = (u_usum >= Q13) ? (u_usum - Q13) : u_usum[11:0];
// ================================================================
// E6: v = INTT(sum_j t_hat[j] o y_hat[j]) + e2 + mu (Encaps ST_ENC_V).
@@ -947,8 +978,8 @@ module mlkem_top #(
wire [13:0] u_vadd_wr = UPSUM*256 + u_awidx; // bank_t v (write)
wire [11:0] u_mu = m_r[u_awidx] ? 12'd1665 : 12'd0; // mu by msg bit
wire [13:0] u_vsum = {2'b0, bt_rd_data} + {2'b0, ba_rd_data} + {2'b0, u_mu}; // psum+e2+mu
wire [13:0] u_vsub1 = (u_vsum >= 14'(Q)) ? (u_vsum - 14'(Q)) : u_vsum;
wire [11:0] u_vq = (u_vsub1 >= 14'(Q)) ? (u_vsub1 - 14'(Q)) : u_vsub1[11:0];
wire [13:0] u_vsub1 = (u_vsum >= Q14) ? (u_vsum - Q14) : u_vsum;
wire [11:0] u_vq = (u_vsub1 >= Q14) ? (u_vsub1 - Q14) : u_vsub1[11:0];
wire u_v_we = (st == ST_ENC_V) && (u_sub == 2'd2) && u_avalid;
// ================================================================
@@ -964,7 +995,7 @@ module mlkem_top #(
wire [13:0] u_w_vrd = DEC_VASLOT*256 + u_aidx[7:0]; // bank_a v' (lead)
wire [13:0] u_wsub_wr = UPSUM*256 + u_awidx; // bank_t w (write)
wire [12:0] u_wdiff = {1'b0, ba_rd_data} - {1'b0, bt_rd_data}; // v' - psum
wire [11:0] u_wq = u_wdiff[12] ? (u_wdiff[11:0] + 12'(Q)) : u_wdiff[11:0];
wire [11:0] u_wq = u_wdiff[12] ? (u_wdiff[11:0] + Q12) : u_wdiff[11:0];
wire u_w_we = (st == ST_DEC_W) && (u_sub == 2'd2) && u_avalid;
// ================================================================
@@ -1038,13 +1069,18 @@ module mlkem_top #(
.ready_i(1'b1)
);
// bit-packer / ct walk bookkeeping
reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v
reg [7:0] cp_idx; // coeff 0..255 within poly
reg [2:0] cp_ph; // micro-phase
reg [24:0] cp_buf; // bit accumulator (LSB-first)
reg [5:0] cp_nbits; // valid bits in cp_buf
reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v (feed side)
reg [7:0] cp_idx; // coeff 0..255 within poly (feed side, drives bank addr)
reg [2:0] cp_ph; // feed micro-phase toggle (0=addr settle, 1=feed)
reg [63:0] cp_buf; // bit accumulator (LSB-first); widened for streaming
reg [6:0] cp_nbits; // valid bits in cp_buf
reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2)
reg cp_done; // serialization complete (this region)
reg cp_fed_done; // all coeffs for this region have been fed in
reg [10:0] cp_fcnt; // coeffs fed into comp_decomp this region
reg [10:0] cp_ccnt; // compressed coeffs collected this region
// total coeffs this region: c1 = K*256 (u rows), c2 = 256 (single v)
wire [10:0] cp_ntot = (st == ST_ENC_C2) ? 11'd256 : ({8'd0, k_r} << 8);
// coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly);
// c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0).
wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly]
@@ -1128,7 +1164,7 @@ module mlkem_top #(
wire [11:0] m_acc_src = m_jq ? bse_rd_data : bt_rd_data;
// (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract
wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff};
wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0];
wire [11:0] m_accq = (m_sum >= Q13) ? (m_sum - Q13) : m_sum[11:0];
always @(*) begin
st_next = st;
@@ -1257,10 +1293,13 @@ module mlkem_top #(
cp_poly <= 3'd0;
cp_idx <= 8'd0;
cp_ph <= 3'd0;
cp_buf <= 25'd0;
cp_nbits <= 6'd0;
cp_buf <= 64'd0;
cp_nbits <= 7'd0;
cp_wa <= 12'd0;
cp_done <= 1'b0;
cp_fed_done<= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
ct_we <= 1'b0;
ct_wa <= 11'd0;
ct_wd <= 8'd0;
@@ -1695,66 +1734,83 @@ module mlkem_top #(
cp_poly <= 3'd0;
cp_idx <= 8'd0;
cp_ph <= 3'd0;
cp_buf <= 25'd0;
cp_nbits <= 6'd0;
cp_buf <= 64'd0;
cp_nbits <= 7'd0;
cp_wa <= 12'd0;
cp_done <= 1'b0;
cp_fed_done <= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
cd_valid <= 1'b0;
end
// ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ----
// C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes).
// C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes).
// Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram plus
// variable-latency comp_decomp), then a drain sub-phase emitting whole bytes:
// ph0: present coeff addr (cp_se_full / cp_bt_full by state).
// ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid.
// ph2: drop cd_valid (1-cyc pulse); comp_decomp captures.
// ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first.
// ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance.
// Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at
// each poly boundary (no carry across polys / regions).
//
// Streaming (B1): the comp_decomp pipeline is fully pipelined and its
// sole output consumer never backpressures (ready_i=1'b1), so instead
// of feeding one coeff and draining the whole pipe per coeff (~12 cyc),
// we run a decoupled producer + collector:
// PRODUCER (2-cycle cadence): ph0 lets the bank read address settle
// (1-cyc sd_bram latency), ph1 latches the coeff and pulses
// cd_valid, then advances the address. One coeff enters the pipe
// every 2 cycles = <=5.5 bits/cyc (du<=11), strictly under the
// single-port ct drain rate (8 bits/cyc), so cp_buf stays bounded.
// COLLECTOR (every cycle): when cd_vo, append the compressed cp_d
// bits LSB-first; independently emit one ct byte/cycle whenever
// >=8 bits are buffered. Coeffs come out in-order, one per pipe
// output. Each poly is 256*d bits (multiple of 8), so the buffer
// empties to 0 at region end (cp_done when all collected+drained).
if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin
case (cp_ph)
3'd0: cp_ph <= 3'd1; // addr presented; wait read
3'd1: begin
cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t)
cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse)
cp_ph <= 3'd2;
end
3'd2: begin
cd_valid <= 1'b0; // comp_decomp captured this cyc
cp_ph <= 3'd3;
end
3'd3: begin
if (cd_vo) begin
// Append compressed low cp_d bits LSB-first at bit cp_nbits.
cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits);
cp_nbits <= cp_nbits + {1'b0, cp_d};
cp_ph <= 3'd4;
end
end
default: begin // 3'd4: drain whole bytes
if (cp_nbits >= 6'd8) begin
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= cp_buf[7:0];
cp_wa <= cp_wa + 12'd1;
cp_buf <= cp_buf >> 8;
cp_nbits <= cp_nbits - 6'd8;
// ---- PRODUCER: feed one coeff every 2 cycles ----
cd_valid <= 1'b0;
if (!cp_fed_done) begin
if (cp_ph == 3'd0) begin
cp_ph <= 3'd1; // address settling
end else begin
// coeff fully packed; advance coeff / poly
cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t)
cd_valid <= 1'b1; // 1-cyc feed pulse
cp_fcnt <= cp_fcnt + 11'd1;
if (cp_fcnt + 11'd1 >= cp_ntot) cp_fed_done <= 1'b1;
if (cp_idx == 8'd255) begin
cp_idx <= 8'd0;
if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1;
else cp_done <= 1'b1; // region complete
cp_poly <= cp_poly + 3'd1; // next u row (c1); c2 unused
end else begin
cp_idx <= cp_idx + 8'd1;
end
cp_ph <= 3'd0;
end
end
endcase
// ---- COLLECTOR: append compressed bits + drain ct bytes ----
if (cd_vo) begin
// append cp_d bits at cp_nbits, then drain one byte if >=8.
if ((cp_nbits + {1'b0, cp_d}) >= 7'd8) begin
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) & 64'hFF;
cp_wa <= cp_wa + 12'd1;
cp_buf <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) >> 8;
cp_nbits <= (cp_nbits + {1'b0, cp_d}) - 7'd8;
end else begin
cp_buf <= cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits);
cp_nbits <= cp_nbits + {1'b0, cp_d};
end
cp_ccnt <= cp_ccnt + 11'd1; // one compressed coeff collected
end else if (cp_nbits >= 7'd8) begin
// no new coeff this cycle: keep draining buffered whole bytes.
ct_we <= 1'b1;
ct_wa <= cp_wa;
ct_wd <= cp_buf[7:0];
cp_wa <= cp_wa + 12'd1;
cp_buf <= cp_buf >> 8;
cp_nbits <= cp_nbits - 7'd8;
end
// region complete once all coeffs collected and buffer drained.
if (cp_ccnt >= cp_ntot && cp_nbits < 7'd8)
cp_done <= 1'b1;
end
// Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)).
@@ -1765,9 +1821,12 @@ module mlkem_top #(
cp_poly <= 3'd0;
cp_idx <= 8'd0;
cp_ph <= 3'd0;
cp_buf <= 25'd0;
cp_nbits <= 6'd0;
cp_buf <= 64'd0;
cp_nbits <= 7'd0;
cp_done <= 1'b0;
cp_fed_done <= 1'b0;
cp_fcnt <= 11'd0;
cp_ccnt <= 11'd0;
cd_valid <= 1'b0;
// cp_wa intentionally preserved (= c1_bytes_rt from C1).
end