fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(mlkem): move NTT coeff storage to 4-way banked BRAM

Browse Source 
Replace ntt_core's 256x12 LUT register file (multi-port async read, ~18.5k
LUT) with coeff_bram_4way: 4 sd_bram banks, conflict-free banking function
bank(a)={a[0], ^a[7:1]}, local(a)=a[7:1]. The dual-butterfly's four accesses
{j, j+1, j+len, j+1+len} map to four distinct banks for every layer_len in
{2..128}, so each bank needs only 1 read + 1 write/cycle (single buffer, no
ping-pong). Trades ~13k LUT for +4 BRAM (device BRAM at ~6%).

Pipeline gains one stage for the synchronous BRAM read; writeback address
delay deepened to 9. coeff_out is driven combinationally from the BRAM read
during OUTPUT; unused read ports mirror port 0 outside compute so the read
crossbar arbitration is not polluted by stale addresses.

Verified byte-exact vs NIST KAT: NTT standalone TB 19/19 (fwd+inv), Encaps
9/9, Decaps 9/9 (incl. reject + DECOMP), KeyGen, hello_world e2e — all PASS.
Cycle counts unchanged (ENC_N +4 for BRAM latency).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
FallenSigh
2026-07-11 05:16:58 +08:00
parent be2e54c8ab
commit 78ccc34578
2 changed files with 257 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
rtl/ip/mlkem/ntt/coeff_bram_4way.v Normal file
View File

@@ -0,0 +1,113 @@
// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients
//
// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM
// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree.
//
// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits)
// local(a) = a[7:1] (7-bit index into the 128-deep bank)
//
// This function makes a dual-butterfly's four simultaneous coefficient accesses
// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT
// layer_len in {2,4,...,128}: j is even and len is even, so
// j -> {0, P }
// j+1 -> {1, P }
// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity)
// j+1+len -> {1, P'}
// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four
// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank
// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong.
//
// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this
// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle.
//
// NOTE: the caller must guarantee the four read addresses map to four distinct
// banks (true for the NTT butterfly pattern) and the four write addresses too.
// Sequential access (load/output) uses a single port and is always safe.
module coeff_bram_4way (
input clk, rst_n,
// Four read addresses (global 8-bit); data returns 1 cycle later.
input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3,
output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3,
// Four write ports.
input wr_en_0, wr_en_1, wr_en_2, wr_en_3,
input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3,
input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3
);
// Bank id and local index for an address.
function [1:0] bank_id;
input [7:0] a;
bank_id = {a[0], ^a[7:1]};
endfunction
wire [1:0] rb0 = bank_id(rd_addr_0);
wire [1:0] rb1 = bank_id(rd_addr_1);
wire [1:0] rb2 = bank_id(rd_addr_2);
wire [1:0] rb3 = bank_id(rd_addr_3);
wire [1:0] wb0 = bank_id(wr_addr_0);
wire [1:0] wb1 = bank_id(wr_addr_1);
wire [1:0] wb2 = bank_id(wr_addr_2);
wire [1:0] wb3 = bank_id(wr_addr_3);
// Per-bank read-address crossbar: bank b reads the local index of whichever
// read port targets it (exactly one, by construction).
wire [6:0] bank_rd_local [0:3];
wire [6:0] bank_wr_local [0:3];
wire bank_wr_en [0:3];
wire [11:0] bank_wr_data [0:3];
wire [11:0] bank_rd_data [0:3];
genvar b;
generate
for (b = 0; b < 4; b = b + 1) begin : gen_xbar
// read address route (priority: 3>2>1>0; ties impossible for NTT)
assign bank_rd_local[b] =
(rb3 == b[1:0]) ? rd_addr_3[7:1] :
(rb2 == b[1:0]) ? rd_addr_2[7:1] :
(rb1 == b[1:0]) ? rd_addr_1[7:1] :
rd_addr_0[7:1];
// write route
wire m0 = wr_en_0 && (wb0 == b[1:0]);
wire m1 = wr_en_1 && (wb1 == b[1:0]);
wire m2 = wr_en_2 && (wb2 == b[1:0]);
wire m3 = wr_en_3 && (wb3 == b[1:0]);
assign bank_wr_en[b] = m0 | m1 | m2 | m3;
assign bank_wr_local[b] =
m3 ? wr_addr_3[7:1] :
m2 ? wr_addr_2[7:1] :
m1 ? wr_addr_1[7:1] : wr_addr_0[7:1];
assign bank_wr_data[b] =
m3 ? wr_data_3 :
m2 ? wr_data_2 :
m1 ? wr_data_1 : wr_data_0;
sd_bram #(.W(12), .D(128), .A(7)) u_bank (
.clk(clk),
.rd_addr(bank_rd_local[b]),
.rd_data(bank_rd_data[b]),
.wr_en(bank_wr_en[b]),
.wr_addr(bank_wr_local[b]),
.wr_data(bank_wr_data[b])
);
end
endgenerate
// Register the read bank-ids to align with sd_bram's 1-cycle read latency,
// then route each bank's output back to the requesting read port.
reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0;
end else begin
rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3;
end
end
assign rd_data_0 = bank_rd_data[rb0_r];
assign rd_data_1 = bank_rd_data[rb1_r];
assign rd_data_2 = bank_rd_data[rb2_r];
assign rd_data_3 = bank_rd_data[rb3_r];
endmodule

290
rtl/ip/mlkem/ntt/ntt_core.v
View File

@@ -1,16 +1,18 @@
// ntt_core.v - NTT core with individual coefficient registers (2 butterflies)
// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage
//
// Uses 256 individual 12-bit registers and TWO deeply pipelined butterfly paths
// running in lock-step (R1: dual-butterfly parallelism). Each issue cycle emits
// two butterflies from the SAME group at positions (p, p+1); since ML-KEM's NTT
// has 7 layers with layer_len in {128..2} (always even, never 1), the pair always
// shares one zeta and touches four distinct coefficient indices, so a single
// zeta ROM suffices and there is no intra-group read/write conflict.
// address/operand/zeta register -> 2x pipelined Barrett butterfly -> writeback
// Each NTT layer issues 2 butterflies/cycle (64 cycles/layer vs 128), then the
// core drains pending writebacks before starting the next dependent layer.
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way
// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port
// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle.
//
// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline
// gains one stage:
// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid)
// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer.
// Read and writeback hit disjoint banks (butterfly index sets in a layer are
// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W).
//
// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read
// latency (ready_i is tied high by all NTT consumers, so no output backpressure).
module ntt_core (
input clk, rst_n,
@@ -25,9 +27,6 @@ module ntt_core (
);
localparam N = 256, LAYERS = 7, DW = 12;
reg [DW-1:0] cr [0:N-1];
integer ci;
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_RUN = 4'd2;
@@ -40,65 +39,78 @@ module ntt_core (
reg [3:0] state, next_state;
reg [7:0] load_cnt;
reg [7:0] out_cnt;
reg [8:0] scale_issue_cnt;
reg [8:0] scale_emit_cnt;
reg [8:0] out_rptr; // output read address pointer (0..256)
reg [8:0] out_cnt; // output coeffs emitted (0..256)
reg [8:0] scale_rptr; // scale read address pointer
reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett
reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
reg mode_r;
reg layer_issue_done;
reg [5:0] bf_inflight; // pending issue cycles (pairs) in flight
reg [5:0] bf_inflight;
// Two writeback address delay lines (lane0: j, lane1: j+1), each +len.
reg [7:0] a0_d0,a0_d1,a0_d2,a0_d3,a0_d4,a0_d5,a0_d6,a0_d7;
reg [7:0] b0_d0,b0_d1,b0_d2,b0_d3,b0_d4,b0_d5,b0_d6,b0_d7;
reg [7:0] a1_d0,a1_d1,a1_d2,a1_d3,a1_d4,a1_d5,a1_d6,a1_d7;
reg [7:0] b1_d0,b1_d1,b1_d2,b1_d3,b1_d4,b1_d5,b1_d6,b1_d7;
reg wb_valid_r;
reg [DW-1:0] wb_a0_r, wb_b0_r, wb_a1_r, wb_b1_r;
reg [7:0] wb_a0a_r, wb_b0a_r, wb_a1a_r, wb_b1a_r;
reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed
// 9-deep writeback address delay lines (BRAM read + 8-stage butterfly).
reg [7:0] a0d[0:8]; reg [7:0] b0d[0:8];
reg [7:0] a1d[0:8]; reg [7:0] b1d[0:8];
integer k;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
reg scale_valid_i;
reg [DW-1:0] scale_a_i;
reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present)
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
// pair (j, j+1) is the last in this group when j+2 reaches group_end
wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end);
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
// lane0 = position j, lane1 = position j+1 (same group => same zeta)
wire [DW-1:0] issue_a0 = cr[j];
wire [DW-1:0] issue_b0 = cr[j + layer_len];
wire [DW-1:0] issue_a1 = cr[j + 8'd1];
wire [DW-1:0] issue_b1 = cr[j + 8'd1 + layer_len];
// ---- coeff_bram_4way port muxing ----
// Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr).
wire [7:0] cmp_a0 = j;
wire [7:0] cmp_b0 = j + layer_len;
wire [7:0] cmp_a1 = j + 8'd1;
wire [7:0] cmp_b1 = j + 8'd1 + layer_len;
wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] :
(state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0;
// Ports 1..3 carry the other three butterfly operands ONLY during compute;
// otherwise they must mirror port 0 so the read crossbar's per-bank priority
// arbitration doesn't route a stale compute address into port 0's bank.
wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN);
wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0;
wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0;
wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0;
wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3;
// Butterflies fed from BRAM read data one cycle after the address (rd_fire_r).
wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out;
wire bf0_valid, bf1_valid;
butterfly_unit_pipe u_bf0 (
.clk(clk), .rst_n(rst_n),
.valid_i(issue_fire),
.a(issue_a0), .b(issue_b0),
.zeta(zeta), .mode(mode_r),
.valid_i(rd_fire_r),
.a(rd_data_0), .b(rd_data_1),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid)
);
butterfly_unit_pipe u_bf1 (
.clk(clk), .rst_n(rst_n),
.valid_i(issue_fire),
.a(issue_a1), .b(issue_b1),
.zeta(zeta), .mode(mode_r),
.valid_i(rd_fire_r),
.a(rd_data_2), .b(rd_data_3),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid)
);
// zeta must align with the BRAM read data (1 cycle after presentation).
// Scale multiplier (inverse NTT final x3303).
wire [DW-1:0] scale_product;
wire scale_valid_o;
barrett_mul_pipe u_scl (
@@ -108,8 +120,41 @@ module ntt_core (
.product(scale_product), .valid_o(scale_valid_o)
);
// ---- write ports ----
wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i);
wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt;
wire wr_en_0 = loading_we | bf0_valid;
wire wr_en_1 = bf0_valid & ~loading_we;
wire wr_en_2 = bf0_valid & ~loading_we;
wire wr_en_3 = bf0_valid & ~loading_we;
wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[8];
wire [7:0] wr_addr_1 = b0d[8];
wire [7:0] wr_addr_2 = a1d[8];
wire [7:0] wr_addr_3 = b1d[8];
wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out;
wire [11:0] wr_data_1 = bf0_b_out;
wire [11:0] wr_data_2 = bf1_a_out;
wire [11:0] wr_data_3 = bf1_b_out;
coeff_bram_4way u_cr (
.clk(clk), .rst_n(rst_n),
.rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1),
.rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3),
.rd_data_0(rd_data_0), .rd_data_1(rd_data_1),
.rd_data_2(rd_data_2), .rd_data_3(rd_data_3),
.wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3),
.wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1),
.wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3),
.wr_data_0(wr_data_0), .wr_data_1(wr_data_1),
.wr_data_2(wr_data_2), .wr_data_3(wr_data_3)
);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign coeff_out = coeff_out_r;
// During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle
// latency, matching the old combinational-read + 1 output reg). Scale path
// uses the registered barrett result.
assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r;
assign valid_o = valid_o_r;
assign done_o = (state == S_DONE);
@@ -122,157 +167,113 @@ module ntt_core (
S_CMP_DRAIN: if (bf_inflight == 6'd0)
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE;
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
endcase
end
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer_len <= 8'd0;
zeta_idx <= 7'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 6'd0;
a0_d0<=8'd0;a0_d1<=8'd0;a0_d2<=8'd0;a0_d3<=8'd0;a0_d4<=8'd0;a0_d5<=8'd0;a0_d6<=8'd0;a0_d7<=8'd0;
b0_d0<=8'd0;b0_d1<=8'd0;b0_d2<=8'd0;b0_d3<=8'd0;b0_d4<=8'd0;b0_d5<=8'd0;b0_d6<=8'd0;b0_d7<=8'd0;
a1_d0<=8'd0;a1_d1<=8'd0;a1_d2<=8'd0;a1_d3<=8'd0;a1_d4<=8'd0;a1_d5<=8'd0;a1_d6<=8'd0;a1_d7<=8'd0;
b1_d0<=8'd0;b1_d1<=8'd0;b1_d2<=8'd0;b1_d3<=8'd0;b1_d4<=8'd0;b1_d5<=8'd0;b1_d6<=8'd0;b1_d7<=8'd0;
wb_valid_r <= 1'b0;
wb_a0_r <= 12'd0; wb_b0_r <= 12'd0; wb_a1_r <= 12'd0; wb_b1_r <= 12'd0;
wb_a0a_r <= 8'd0; wb_b0a_r <= 8'd0; wb_a1a_r <= 8'd0; wb_b1a_r <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
scale_a_i <= 12'd0;
for (ci = 0; ci < N; ci = ci + 1) cr[ci] = 12'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer_len <= 8'd0;
zeta_idx <= 7'd0; zeta_fed <= 12'd0;
layer <= 3'd0; mode_r <= 1'b0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
rd_fire_r <= 1'b0;
for (k = 0; k <= 8; k = k + 1) begin
a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0;
end
coeff_out_r <= 12'd0; valid_o_r <= 1'b0;
scale_valid_i <= 1'b0; scale_a_i <= 12'd0;
end else begin
state <= next_state;
scale_valid_i <= 1'b0;
// Writeback address delay lines (8-deep = butterfly latency).
a0_d0<=j; a0_d1<=a0_d0;a0_d2<=a0_d1;a0_d3<=a0_d2;a0_d4<=a0_d3;a0_d5<=a0_d4;a0_d6<=a0_d5;a0_d7<=a0_d6;
b0_d0<=j+layer_len; b0_d1<=b0_d0;b0_d2<=b0_d1;b0_d3<=b0_d2;b0_d4<=b0_d3;b0_d5<=b0_d4;b0_d6<=b0_d5;b0_d7<=b0_d6;
a1_d0<=j+8'd1; a1_d1<=a1_d0;a1_d2<=a1_d1;a1_d3<=a1_d2;a1_d4<=a1_d3;a1_d5<=a1_d4;a1_d6<=a1_d5;a1_d7<=a1_d6;
b1_d0<=j+8'd1+layer_len; b1_d1<=b1_d0;b1_d2<=b1_d1;b1_d3<=b1_d2;b1_d4<=b1_d3;b1_d5<=b1_d4;b1_d6<=b1_d5;b1_d7<=b1_d6;
// read->feed pipeline register (BRAM latency) + zeta alignment
rd_fire_r <= issue_fire;
zeta_fed <= zeta;
// In-flight pair accounting (mirrors single-butterfly version).
if (issue_fire && !wb_valid_r)
bf_inflight <= bf_inflight + 6'd1;
else if (!issue_fire && wb_valid_r)
bf_inflight <= bf_inflight - 6'd1;
// writeback address delay lines: sample j-derived addrs at read cycle
a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1;
for (k = 1; k <= 8; k = k + 1) begin
a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1];
a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1];
end
// in-flight pair accounting: +1 on issue, -1 on writeback
if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1;
else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt <= 8'd1;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 6'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer <= 3'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
zeta_idx <= 7'd1;
end else begin
layer_len <= 8'd2;
zeta_idx <= 7'd127;
end
if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end
else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end
end
if (state == S_LOAD && valid_i) begin
cr[load_cnt] <= coeff_in;
load_cnt <= load_cnt + 8'd1;
end
if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1;
// ---- issue advance: step j by 2 (two butterflies/cycle) ----
// ---- issue advance (read pointer j, step 2) ----
if (state == S_CMP_RUN && !layer_issue_done) begin
if (issue_group_last) begin
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if (issue_layer_last) begin
layer_issue_done <= 1'b1;
end else begin
start <= next_group_start[7:0];
j <= next_group_start[7:0];
end
if (issue_layer_last) layer_issue_done <= 1'b1;
else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end
end else begin
j <= j + 8'd2;
end
end
// ---- writeback: commit previous cycle's pair results ----
if (wb_valid_r) begin
cr[wb_a0a_r] <= wb_a0_r;
cr[wb_b0a_r] <= wb_b0_r;
cr[wb_a1a_r] <= wb_a1_r;
cr[wb_b1a_r] <= wb_b1_r;
end
wb_valid_r <= bf0_valid; // both lanes share latency/valid
if (bf0_valid) begin
wb_a0_r <= bf0_a_out; wb_b0_r <= bf0_b_out;
wb_a1_r <= bf1_a_out; wb_b1_r <= bf1_b_out;
wb_a0a_r <= a0_d7; wb_b0a_r <= b0_d7;
wb_a1a_r <= a1_d7; wb_b1a_r <= b1_d7;
end
// ---- layer advance after drain ----
if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin
if (layer + 3'd1 >= LAYERS) begin
bf_done <= 1'b1;
end else begin
if (layer + 3'd1 < LAYERS) begin
layer <= layer + 3'd1;
layer_len <= next_layer_len;
start <= 8'd0;
j <= 8'd0;
start <= 8'd0; j <= 8'd0;
layer_issue_done <= 1'b0;
end
end
// ---- OUT_PREP: reset output pointers ----
if (state == S_OUT_PREP) begin
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
if (!mode_r) begin
coeff_out_r <= cr[0];
valid_o_r <= 1'b1;
end
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
// (scale first addr 0 is presented via out_rptr in this PREP cycle)
end
if (state == S_OUTPUT && valid_o_r && ready_i) begin
if (out_cnt < 8'd255) begin
out_cnt <= out_cnt + 8'd1;
coeff_out_r <= cr[out_cnt + 8'd1];
valid_o_r <= 1'b1;
// ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0;
// valid_o is a registered "presented a valid addr last cycle". Present
// addresses out_rptr = 0..255 (256), giving 256 valid outputs. ----
if (state == S_OUTPUT) begin
if (out_rptr < 9'd256) begin
out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr
valid_o_r <= 1'b1; // data valid next cycle
end else begin
out_cnt <= 8'd0;
valid_o_r <= 1'b0;
end
if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted
end
// ---- OUT_SCALE: read coeff, x3303 via barrett, emit ----
if (state == S_OUT_SCALE) begin
if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1;
// feed barrett with the coeff that arrived this cycle
if (scale_issue_cnt < 9'd256) begin
scale_valid_i <= 1'b1;
scale_a_i <= cr[scale_issue_cnt[7:0]];
scale_a_i <= rd_data_0;
scale_issue_cnt <= scale_issue_cnt + 9'd1;
end
valid_o_r <= 1'b0;
@@ -285,12 +286,9 @@ module ntt_core (
if (state == S_DONE) begin
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 6'd0;
wb_valid_r <= 1'b0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
valid_o_r <= 1'b0;
end
end