fallensigh/mlkem-sync
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize

Browse Source
This commit is contained in:
FallenSigh
2026-07-11 17:21:21 +08:00
parent 47fbcd31b6
commit d8d57db82a
9 changed files with 504 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sync_rtl/ntt/TB/xsim_run.tcl
View File

@@ -40,6 +40,8 @@ xvlog -sv ${SRC_DIR}/butterfly_unit.v
xvlog -sv ${SRC_DIR}/butterfly_unit_pipe.v
# NTT core (FSM-based, instantiates butterfly_unit + zeta_rom + barrett_mul)
xvlog -sv ${SRC_DIR}/../storage/sd_bram.v
xvlog -sv ${SRC_DIR}/coeff_bram_4way.v
xvlog -sv ${SRC_DIR}/ntt_core.v
# ================================================================

29
sync_rtl/ntt/butterfly_unit_pipe.v
View File

@@ -18,21 +18,32 @@ module butterfly_unit_pipe (
localparam [11:0] Q12 = 12'd3329;
localparam [12:0] Q13 = 13'd3329;
wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12);
wire [11:0] mul_data = mode ? diff : b;
// Plan A: register the inputs. Operands arrive combinationally from the
// coeff_bram read; without this stage the path coeff_bram-read -> diff ->
// first multiply stage sits in one cycle (the mlkem_clk critical path at
// 100 MHz). Registering here splits it into (BRAM read -> reg) and
// (reg -> diff -> multiply). Adds one pipeline cycle (absorbed by the NTT
// writeback delay line).
reg [11:0] a_r, b_r, zeta_r;
reg mode_ri, vld_r;
wire [11:0] diff = (b_r >= a_r) ? (b_r - a_r) : (b_r - a_r + Q12);
wire [11:0] mul_data = mode_ri ? diff : b_r;
wire [11:0] mul_result;
wire mul_valid;
barrett_mul_pipe u_barrett (
.clk(clk),
.rst_n(rst_n),
.valid_i(valid_i),
.a(zeta),
.valid_i(vld_r),
.a(zeta_r),
.b(mul_data),
.product(mul_result),
.valid_o(mul_valid)
);
// a/b/mode delay lines (fed from the REGISTERED inputs a_r/b_r/mode_ri so
// a_d6 aligns with mul_valid = valid_i delayed 8).
reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6;
reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6;
reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6;
@@ -51,6 +62,7 @@ module butterfly_unit_pipe (
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
a_r <= 12'd0; b_r <= 12'd0; zeta_r <= 12'd0; mode_ri <= 1'b0; vld_r <= 1'b0;
a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0;
b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0;
mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0;
@@ -58,9 +70,12 @@ module butterfly_unit_pipe (
b_out_r <= 12'd0;
valid_r <= 1'b0;
end else begin
a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
// input register stage
a_r <= a; b_r <= b; zeta_r <= zeta; mode_ri <= mode; vld_r <= valid_i;
a_d0 <= a_r; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5;
b_d0 <= b_r; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5;
mode_d0 <= mode_ri; mode_d1 <= mode_d0; mode_d2 <= mode_d1;
mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5;
valid_r <= mul_valid;

113
sync_rtl/ntt/coeff_bram_4way.v Normal file
View File

@@ -0,0 +1,113 @@
// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients
//
// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM
// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree.
//
// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits)
// local(a) = a[7:1] (7-bit index into the 128-deep bank)
//
// This function makes a dual-butterfly's four simultaneous coefficient accesses
// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT
// layer_len in {2,4,...,128}: j is even and len is even, so
// j -> {0, P }
// j+1 -> {1, P }
// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity)
// j+1+len -> {1, P'}
// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four
// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank
// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong.
//
// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this
// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle.
//
// NOTE: the caller must guarantee the four read addresses map to four distinct
// banks (true for the NTT butterfly pattern) and the four write addresses too.
// Sequential access (load/output) uses a single port and is always safe.
module coeff_bram_4way (
input clk, rst_n,
// Four read addresses (global 8-bit); data returns 1 cycle later.
input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3,
output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3,
// Four write ports.
input wr_en_0, wr_en_1, wr_en_2, wr_en_3,
input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3,
input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3
);
// Bank id and local index for an address.
function [1:0] bank_id;
input [7:0] a;
bank_id = {a[0], ^a[7:1]};
endfunction
wire [1:0] rb0 = bank_id(rd_addr_0);
wire [1:0] rb1 = bank_id(rd_addr_1);
wire [1:0] rb2 = bank_id(rd_addr_2);
wire [1:0] rb3 = bank_id(rd_addr_3);
wire [1:0] wb0 = bank_id(wr_addr_0);
wire [1:0] wb1 = bank_id(wr_addr_1);
wire [1:0] wb2 = bank_id(wr_addr_2);
wire [1:0] wb3 = bank_id(wr_addr_3);
// Per-bank read-address crossbar: bank b reads the local index of whichever
// read port targets it (exactly one, by construction).
wire [6:0] bank_rd_local [0:3];
wire [6:0] bank_wr_local [0:3];
wire bank_wr_en [0:3];
wire [11:0] bank_wr_data [0:3];
wire [11:0] bank_rd_data [0:3];
genvar b;
generate
for (b = 0; b < 4; b = b + 1) begin : gen_xbar
// read address route (priority: 3>2>1>0; ties impossible for NTT)
assign bank_rd_local[b] =
(rb3 == b[1:0]) ? rd_addr_3[7:1] :
(rb2 == b[1:0]) ? rd_addr_2[7:1] :
(rb1 == b[1:0]) ? rd_addr_1[7:1] :
rd_addr_0[7:1];
// write route
wire m0 = wr_en_0 && (wb0 == b[1:0]);
wire m1 = wr_en_1 && (wb1 == b[1:0]);
wire m2 = wr_en_2 && (wb2 == b[1:0]);
wire m3 = wr_en_3 && (wb3 == b[1:0]);
assign bank_wr_en[b] = m0 | m1 | m2 | m3;
assign bank_wr_local[b] =
m3 ? wr_addr_3[7:1] :
m2 ? wr_addr_2[7:1] :
m1 ? wr_addr_1[7:1] : wr_addr_0[7:1];
assign bank_wr_data[b] =
m3 ? wr_data_3 :
m2 ? wr_data_2 :
m1 ? wr_data_1 : wr_data_0;
sd_bram #(.W(12), .D(128), .A(7)) u_bank (
.clk(clk),
.rd_addr(bank_rd_local[b]),
.rd_data(bank_rd_data[b]),
.wr_en(bank_wr_en[b]),
.wr_addr(bank_wr_local[b]),
.wr_data(bank_wr_data[b])
);
end
endgenerate
// Register the read bank-ids to align with sd_bram's 1-cycle read latency,
// then route each bank's output back to the requesting read port.
reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r;
always @(posedge clk or negedge rst_n) begin
if (!rst_n) begin
rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0;
end else begin
rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3;
end
end
assign rd_data_0 = bank_rd_data[rb0_r];
assign rd_data_1 = bank_rd_data[rb1_r];
assign rd_data_2 = bank_rd_data[rb2_r];
assign rd_data_3 = bank_rd_data[rb3_r];
endmodule

311
sync_rtl/ntt/ntt_core.v
View File

@@ -1,12 +1,18 @@
// ntt_core.v - NTT core with individual coefficient registers
// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage
//
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
// The arithmetic hot path is split into:
// address/operand/zeta register -> pipelined Barrett butterfly -> writeback
// Each NTT layer is issued continuously into the butterfly pipeline, then the
// core drains pending writebacks before starting the next dependent layer.
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
// multiplier so the output path does not reintroduce a combinational reducer.
// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way
// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port
// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle.
//
// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline
// gains one stage:
// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid)
// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer.
// Read and writeback hit disjoint banks (butterfly index sets in a layer are
// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W).
//
// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read
// latency (ready_i is tied high by all NTT consumers, so no output backpressure).
module ntt_core (
input clk, rst_n,
@@ -21,9 +27,6 @@ module ntt_core (
);
localparam N = 256, LAYERS = 7, DW = 12;
reg [DW-1:0] cr [0:N-1];
integer ci;
localparam S_IDLE = 4'd0;
localparam S_LOAD = 4'd1;
localparam S_CMP_RUN = 4'd2;
@@ -36,69 +39,122 @@ module ntt_core (
reg [3:0] state, next_state;
reg [7:0] load_cnt;
reg [7:0] out_cnt;
reg [8:0] scale_issue_cnt;
reg [8:0] scale_emit_cnt;
reg [8:0] out_rptr; // output read address pointer (0..256)
reg [8:0] out_cnt; // output coeffs emitted (0..256)
reg [8:0] scale_rptr; // scale read address pointer
reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett
reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett
reg [7:0] j, start, layer_len;
reg [6:0] zeta_idx;
reg [2:0] layer;
reg bf_done;
reg mode_r;
reg layer_issue_done;
reg [4:0] bf_inflight;
reg [5:0] bf_inflight;
reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7;
reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7;
reg wb_valid_r;
reg [DW-1:0] wb_a_data_r, wb_b_data_r;
reg [7:0] wb_wa_r, wb_wb_r;
reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed
// 9-deep writeback address delay lines (BRAM read + 8-stage butterfly).
reg [7:0] a0d[0:9]; reg [7:0] b0d[0:9];
reg [7:0] a1d[0:9]; reg [7:0] b1d[0:9];
integer k;
reg [DW-1:0] coeff_out_r;
reg valid_o_r;
reg scale_valid_i;
reg [DW-1:0] scale_a_i;
reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present)
wire [DW-1:0] zeta;
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end);
wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end);
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
wire [DW-1:0] issue_a = cr[j];
wire [DW-1:0] issue_b = cr[j + layer_len];
wire [DW-1:0] bf_a_out, bf_b_out;
wire bf_valid;
butterfly_unit_pipe u_bf (
.clk(clk),
.rst_n(rst_n),
.valid_i(issue_fire),
.a(issue_a),
.b(issue_b),
.zeta(zeta),
.mode(mode_r),
.a_out(bf_a_out),
.b_out(bf_b_out),
.valid_o(bf_valid)
// ---- coeff_bram_4way port muxing ----
// Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr).
wire [7:0] cmp_a0 = j;
wire [7:0] cmp_b0 = j + layer_len;
wire [7:0] cmp_a1 = j + 8'd1;
wire [7:0] cmp_b1 = j + 8'd1 + layer_len;
wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] :
(state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0;
// Ports 1..3 carry the other three butterfly operands ONLY during compute;
// otherwise they must mirror port 0 so the read crossbar's per-bank priority
// arbitration doesn't route a stale compute address into port 0's bank.
wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN);
wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0;
wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0;
wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0;
wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3;
// Butterflies fed from BRAM read data one cycle after the address (rd_fire_r).
wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out;
wire bf0_valid, bf1_valid;
butterfly_unit_pipe u_bf0 (
.clk(clk), .rst_n(rst_n),
.valid_i(rd_fire_r),
.a(rd_data_0), .b(rd_data_1),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid)
);
butterfly_unit_pipe u_bf1 (
.clk(clk), .rst_n(rst_n),
.valid_i(rd_fire_r),
.a(rd_data_2), .b(rd_data_3),
.zeta(zeta_fed), .mode(mode_r),
.a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid)
);
// zeta must align with the BRAM read data (1 cycle after presentation).
// Scale multiplier (inverse NTT final x3303).
wire [DW-1:0] scale_product;
wire scale_valid_o;
barrett_mul_pipe u_scl (
.clk(clk),
.rst_n(rst_n),
.clk(clk), .rst_n(rst_n),
.valid_i(scale_valid_i),
.a(scale_a_i),
.b(12'd3303),
.product(scale_product),
.valid_o(scale_valid_o)
.a(scale_a_i), .b(12'd3303),
.product(scale_product), .valid_o(scale_valid_o)
);
// ---- write ports ----
wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i);
wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt;
wire wr_en_0 = loading_we | bf0_valid;
wire wr_en_1 = bf0_valid & ~loading_we;
wire wr_en_2 = bf0_valid & ~loading_we;
wire wr_en_3 = bf0_valid & ~loading_we;
wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[9];
wire [7:0] wr_addr_1 = b0d[9];
wire [7:0] wr_addr_2 = a1d[9];
wire [7:0] wr_addr_3 = b1d[9];
wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out;
wire [11:0] wr_data_1 = bf0_b_out;
wire [11:0] wr_data_2 = bf1_a_out;
wire [11:0] wr_data_3 = bf1_b_out;
coeff_bram_4way u_cr (
.clk(clk), .rst_n(rst_n),
.rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1),
.rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3),
.rd_data_0(rd_data_0), .rd_data_1(rd_data_1),
.rd_data_2(rd_data_2), .rd_data_3(rd_data_3),
.wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3),
.wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1),
.wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3),
.wr_data_0(wr_data_0), .wr_data_1(wr_data_1),
.wr_data_2(wr_data_2), .wr_data_3(wr_data_3)
);
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
assign coeff_out = coeff_out_r;
// During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle
// latency, matching the old combinational-read + 1 output reg). Scale path
// uses the registered barrett result.
assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r;
assign valid_o = valid_o_r;
assign done_o = (state == S_DONE);
@@ -108,10 +164,10 @@ module ntt_core (
S_IDLE: if (valid_i) next_state = S_LOAD;
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN;
S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN;
S_CMP_DRAIN: if (bf_inflight == 5'd0)
S_CMP_DRAIN: if (bf_inflight == 6'd0)
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE;
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
S_DONE: next_state = S_IDLE;
default: next_state = S_IDLE;
@@ -122,145 +178,104 @@ module ntt_core (
if (!rst_n) begin
state <= S_IDLE;
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer_len <= 8'd0;
zeta_idx <= 7'd0;
layer <= 3'd0;
bf_done <= 1'b0;
mode_r <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0;
wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0;
wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0;
wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0;
wb_valid_r <= 1'b0;
wb_a_data_r <= 12'd0;
wb_b_data_r <= 12'd0;
wb_wa_r <= 8'd0;
wb_wb_r <= 8'd0;
coeff_out_r <= 12'd0;
valid_o_r <= 1'b0;
scale_valid_i <= 1'b0;
scale_a_i <= 12'd0;
for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer_len <= 8'd0;
zeta_idx <= 7'd0; zeta_fed <= 12'd0;
layer <= 3'd0; mode_r <= 1'b0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
rd_fire_r <= 1'b0;
for (k = 0; k <= 9; k = k + 1) begin
a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0;
end
coeff_out_r <= 12'd0; valid_o_r <= 1'b0;
scale_valid_i <= 1'b0; scale_a_i <= 12'd0;
end else begin
state <= next_state;
scale_valid_i <= 1'b0;
wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2;
wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6;
wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2;
wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6;
// read->feed pipeline register (BRAM latency) + zeta alignment
rd_fire_r <= issue_fire;
zeta_fed <= zeta;
if (issue_fire && !wb_valid_r)
bf_inflight <= bf_inflight + 5'd1;
else if (!issue_fire && wb_valid_r)
bf_inflight <= bf_inflight - 5'd1;
// writeback address delay lines: sample j-derived addrs at read cycle
a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1;
for (k = 1; k <= 9; k = k + 1) begin
a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1];
a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1];
end
// in-flight pair accounting: +1 on issue, -1 on writeback
if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1;
else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1;
if (state != S_OUTPUT && state != S_OUT_SCALE)
valid_o_r <= 1'b0;
if (state == S_IDLE && valid_i) begin
cr[0] <= coeff_in;
load_cnt <= 8'd1;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
j <= 8'd0;
start <= 8'd0;
layer <= 3'd0;
bf_done <= 1'b0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
j <= 8'd0; start <= 8'd0; layer <= 3'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
mode_r <= mode;
if (!mode) begin
layer_len <= 8'd128;
zeta_idx <= 7'd1;
end else begin
layer_len <= 8'd2;
zeta_idx <= 7'd127;
end
if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end
else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end
end
if (state == S_LOAD && valid_i) begin
cr[load_cnt] <= coeff_in;
load_cnt <= load_cnt + 8'd1;
end
if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1;
// ---- issue advance (read pointer j, step 2) ----
if (state == S_CMP_RUN && !layer_issue_done) begin
if (issue_group_last) begin
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
else zeta_idx <= zeta_idx - 7'd1;
if (issue_layer_last) begin
layer_issue_done <= 1'b1;
end else begin
start <= next_group_start[7:0];
j <= next_group_start[7:0];
end
if (issue_layer_last) layer_issue_done <= 1'b1;
else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end
end else begin
j <= j + 8'd1;
j <= j + 8'd2;
end
end
if (wb_valid_r) begin
cr[wb_wa_r] <= wb_a_data_r;
cr[wb_wb_r] <= wb_b_data_r;
end
wb_valid_r <= bf_valid;
if (bf_valid) begin
wb_a_data_r <= bf_a_out;
wb_b_data_r <= bf_b_out;
wb_wa_r <= wa_d7;
wb_wb_r <= wb_d7;
end
if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin
if (layer + 3'd1 >= LAYERS) begin
bf_done <= 1'b1;
end else begin
// ---- layer advance after drain ----
if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin
if (layer + 3'd1 < LAYERS) begin
layer <= layer + 3'd1;
layer_len <= next_layer_len;
start <= 8'd0;
j <= 8'd0;
start <= 8'd0; j <= 8'd0;
layer_issue_done <= 1'b0;
end
end
// ---- OUT_PREP: reset output pointers ----
if (state == S_OUT_PREP) begin
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
if (!mode_r) begin
coeff_out_r <= cr[0];
valid_o_r <= 1'b1;
end
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
// (scale first addr 0 is presented via out_rptr in this PREP cycle)
end
if (state == S_OUTPUT && valid_o_r && ready_i) begin
if (out_cnt < 8'd255) begin
out_cnt <= out_cnt + 8'd1;
coeff_out_r <= cr[out_cnt + 8'd1];
valid_o_r <= 1'b1;
// ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0;
// valid_o is a registered "presented a valid addr last cycle". Present
// addresses out_rptr = 0..255 (256), giving 256 valid outputs. ----
if (state == S_OUTPUT) begin
if (out_rptr < 9'd256) begin
out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr
valid_o_r <= 1'b1; // data valid next cycle
end else begin
out_cnt <= 8'd0;
valid_o_r <= 1'b0;
end
if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted
end
// ---- OUT_SCALE: read coeff, x3303 via barrett, emit ----
if (state == S_OUT_SCALE) begin
if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1;
// feed barrett with the coeff that arrived this cycle
if (scale_issue_cnt < 9'd256) begin
scale_valid_i <= 1'b1;
scale_a_i <= cr[scale_issue_cnt[7:0]];
scale_a_i <= rd_data_0;
scale_issue_cnt <= scale_issue_cnt + 9'd1;
end
valid_o_r <= 1'b0;
if (scale_valid_o) begin
coeff_out_r <= scale_product;
@@ -271,15 +286,11 @@ module ntt_core (
if (state == S_DONE) begin
load_cnt <= 8'd0;
out_cnt <= 8'd0;
scale_issue_cnt <= 9'd0;
scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0;
bf_inflight <= 5'd0;
wb_valid_r <= 1'b0;
out_rptr <= 9'd0; out_cnt <= 9'd0;
scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0;
layer_issue_done <= 1'b0; bf_inflight <= 6'd0;
valid_o_r <= 1'b0;
end
end
end
endmodule