From 78ccc345781031f0d1531eee8f4a643fb74592b5 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 05:16:58 +0800 Subject: [PATCH] perf(mlkem): move NTT coeff storage to 4-way banked BRAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ntt_core's 256x12 LUT register file (multi-port async read, ~18.5k LUT) with coeff_bram_4way: 4 sd_bram banks, conflict-free banking function bank(a)={a[0], ^a[7:1]}, local(a)=a[7:1]. The dual-butterfly's four accesses {j, j+1, j+len, j+1+len} map to four distinct banks for every layer_len in {2..128}, so each bank needs only 1 read + 1 write/cycle (single buffer, no ping-pong). Trades ~13k LUT for +4 BRAM (device BRAM at ~6%). Pipeline gains one stage for the synchronous BRAM read; writeback address delay deepened to 9. coeff_out is driven combinationally from the BRAM read during OUTPUT; unused read ports mirror port 0 outside compute so the read crossbar arbitration is not polluted by stale addresses. Verified byte-exact vs NIST KAT: NTT standalone TB 19/19 (fwd+inv), Encaps 9/9, Decaps 9/9 (incl. reject + DECOMP), KeyGen, hello_world e2e — all PASS. Cycle counts unchanged (ENC_N +4 for BRAM latency). Co-Authored-By: Claude Opus 4.8 (1M context) --- rtl/ip/mlkem/ntt/coeff_bram_4way.v | 113 +++++++++++ rtl/ip/mlkem/ntt/ntt_core.v | 290 ++++++++++++++--------------- 2 files changed, 257 insertions(+), 146 deletions(-) create mode 100644 rtl/ip/mlkem/ntt/coeff_bram_4way.v diff --git a/rtl/ip/mlkem/ntt/coeff_bram_4way.v b/rtl/ip/mlkem/ntt/coeff_bram_4way.v new file mode 100644 index 0000000..c68e718 --- /dev/null +++ b/rtl/ip/mlkem/ntt/coeff_bram_4way.v @@ -0,0 +1,113 @@ +// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients +// +// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM +// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree. +// +// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits) +// local(a) = a[7:1] (7-bit index into the 128-deep bank) +// +// This function makes a dual-butterfly's four simultaneous coefficient accesses +// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT +// layer_len in {2,4,...,128}: j is even and len is even, so +// j -> {0, P } +// j+1 -> {1, P } +// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity) +// j+1+len -> {1, P'} +// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four +// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank +// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong. +// +// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this +// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle. +// +// NOTE: the caller must guarantee the four read addresses map to four distinct +// banks (true for the NTT butterfly pattern) and the four write addresses too. +// Sequential access (load/output) uses a single port and is always safe. + +module coeff_bram_4way ( + input clk, rst_n, + // Four read addresses (global 8-bit); data returns 1 cycle later. + input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3, + output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3, + // Four write ports. + input wr_en_0, wr_en_1, wr_en_2, wr_en_3, + input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3, + input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3 +); + + // Bank id and local index for an address. + function [1:0] bank_id; + input [7:0] a; + bank_id = {a[0], ^a[7:1]}; + endfunction + + wire [1:0] rb0 = bank_id(rd_addr_0); + wire [1:0] rb1 = bank_id(rd_addr_1); + wire [1:0] rb2 = bank_id(rd_addr_2); + wire [1:0] rb3 = bank_id(rd_addr_3); + + wire [1:0] wb0 = bank_id(wr_addr_0); + wire [1:0] wb1 = bank_id(wr_addr_1); + wire [1:0] wb2 = bank_id(wr_addr_2); + wire [1:0] wb3 = bank_id(wr_addr_3); + + // Per-bank read-address crossbar: bank b reads the local index of whichever + // read port targets it (exactly one, by construction). + wire [6:0] bank_rd_local [0:3]; + wire [6:0] bank_wr_local [0:3]; + wire bank_wr_en [0:3]; + wire [11:0] bank_wr_data [0:3]; + wire [11:0] bank_rd_data [0:3]; + + genvar b; + generate + for (b = 0; b < 4; b = b + 1) begin : gen_xbar + // read address route (priority: 3>2>1>0; ties impossible for NTT) + assign bank_rd_local[b] = + (rb3 == b[1:0]) ? rd_addr_3[7:1] : + (rb2 == b[1:0]) ? rd_addr_2[7:1] : + (rb1 == b[1:0]) ? rd_addr_1[7:1] : + rd_addr_0[7:1]; + // write route + wire m0 = wr_en_0 && (wb0 == b[1:0]); + wire m1 = wr_en_1 && (wb1 == b[1:0]); + wire m2 = wr_en_2 && (wb2 == b[1:0]); + wire m3 = wr_en_3 && (wb3 == b[1:0]); + assign bank_wr_en[b] = m0 | m1 | m2 | m3; + assign bank_wr_local[b] = + m3 ? wr_addr_3[7:1] : + m2 ? wr_addr_2[7:1] : + m1 ? wr_addr_1[7:1] : wr_addr_0[7:1]; + assign bank_wr_data[b] = + m3 ? wr_data_3 : + m2 ? wr_data_2 : + m1 ? wr_data_1 : wr_data_0; + + sd_bram #(.W(12), .D(128), .A(7)) u_bank ( + .clk(clk), + .rd_addr(bank_rd_local[b]), + .rd_data(bank_rd_data[b]), + .wr_en(bank_wr_en[b]), + .wr_addr(bank_wr_local[b]), + .wr_data(bank_wr_data[b]) + ); + end + endgenerate + + // Register the read bank-ids to align with sd_bram's 1-cycle read latency, + // then route each bank's output back to the requesting read port. + reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0; + end else begin + rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3; + end + end + + assign rd_data_0 = bank_rd_data[rb0_r]; + assign rd_data_1 = bank_rd_data[rb1_r]; + assign rd_data_2 = bank_rd_data[rb2_r]; + assign rd_data_3 = bank_rd_data[rb3_r]; + +endmodule diff --git a/rtl/ip/mlkem/ntt/ntt_core.v b/rtl/ip/mlkem/ntt/ntt_core.v index 74402f4..681c1b8 100644 --- a/rtl/ip/mlkem/ntt/ntt_core.v +++ b/rtl/ip/mlkem/ntt/ntt_core.v @@ -1,16 +1,18 @@ -// ntt_core.v - NTT core with individual coefficient registers (2 butterflies) +// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage // -// Uses 256 individual 12-bit registers and TWO deeply pipelined butterfly paths -// running in lock-step (R1: dual-butterfly parallelism). Each issue cycle emits -// two butterflies from the SAME group at positions (p, p+1); since ML-KEM's NTT -// has 7 layers with layer_len in {128..2} (always even, never 1), the pair always -// shares one zeta and touches four distinct coefficient indices, so a single -// zeta ROM suffices and there is no intra-group read/write conflict. -// address/operand/zeta register -> 2x pipelined Barrett butterfly -> writeback -// Each NTT layer issues 2 butterflies/cycle (64 cycles/layer vs 128), then the -// core drains pending writebacks before starting the next dependent layer. -// In inverse mode, final x3303 output scaling also uses a pipelined Barrett -// multiplier so the output path does not reintroduce a combinational reducer. +// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way +// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port +// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle. +// +// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline +// gains one stage: +// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid) +// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer. +// Read and writeback hit disjoint banks (butterfly index sets in a layer are +// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W). +// +// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read +// latency (ready_i is tied high by all NTT consumers, so no output backpressure). module ntt_core ( input clk, rst_n, @@ -25,9 +27,6 @@ module ntt_core ( ); localparam N = 256, LAYERS = 7, DW = 12; - reg [DW-1:0] cr [0:N-1]; - integer ci; - localparam S_IDLE = 4'd0; localparam S_LOAD = 4'd1; localparam S_CMP_RUN = 4'd2; @@ -40,65 +39,78 @@ module ntt_core ( reg [3:0] state, next_state; reg [7:0] load_cnt; - reg [7:0] out_cnt; - reg [8:0] scale_issue_cnt; - reg [8:0] scale_emit_cnt; + reg [8:0] out_rptr; // output read address pointer (0..256) + reg [8:0] out_cnt; // output coeffs emitted (0..256) + reg [8:0] scale_rptr; // scale read address pointer + reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett + reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett reg [7:0] j, start, layer_len; reg [6:0] zeta_idx; reg [2:0] layer; - reg bf_done; reg mode_r; reg layer_issue_done; - reg [5:0] bf_inflight; // pending issue cycles (pairs) in flight + reg [5:0] bf_inflight; - // Two writeback address delay lines (lane0: j, lane1: j+1), each +len. - reg [7:0] a0_d0,a0_d1,a0_d2,a0_d3,a0_d4,a0_d5,a0_d6,a0_d7; - reg [7:0] b0_d0,b0_d1,b0_d2,b0_d3,b0_d4,b0_d5,b0_d6,b0_d7; - reg [7:0] a1_d0,a1_d1,a1_d2,a1_d3,a1_d4,a1_d5,a1_d6,a1_d7; - reg [7:0] b1_d0,b1_d1,b1_d2,b1_d3,b1_d4,b1_d5,b1_d6,b1_d7; - reg wb_valid_r; - reg [DW-1:0] wb_a0_r, wb_b0_r, wb_a1_r, wb_b1_r; - reg [7:0] wb_a0a_r, wb_b0a_r, wb_a1a_r, wb_b1a_r; + reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed + // 9-deep writeback address delay lines (BRAM read + 8-stage butterfly). + reg [7:0] a0d[0:8]; reg [7:0] b0d[0:8]; + reg [7:0] a1d[0:8]; reg [7:0] b1d[0:8]; + integer k; reg [DW-1:0] coeff_out_r; reg valid_o_r; reg scale_valid_i; reg [DW-1:0] scale_a_i; + reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present) wire [DW-1:0] zeta; zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len}; wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len}; - // pair (j, j+1) is the last in this group when j+2 reaches group_end wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end); wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256); wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1); wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done; - // lane0 = position j, lane1 = position j+1 (same group => same zeta) - wire [DW-1:0] issue_a0 = cr[j]; - wire [DW-1:0] issue_b0 = cr[j + layer_len]; - wire [DW-1:0] issue_a1 = cr[j + 8'd1]; - wire [DW-1:0] issue_b1 = cr[j + 8'd1 + layer_len]; + // ---- coeff_bram_4way port muxing ---- + // Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr). + wire [7:0] cmp_a0 = j; + wire [7:0] cmp_b0 = j + layer_len; + wire [7:0] cmp_a1 = j + 8'd1; + wire [7:0] cmp_b1 = j + 8'd1 + layer_len; + wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] : + (state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0; + // Ports 1..3 carry the other three butterfly operands ONLY during compute; + // otherwise they must mirror port 0 so the read crossbar's per-bank priority + // arbitration doesn't route a stale compute address into port 0's bank. + wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN); + wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0; + wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0; + wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0; + wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3; + + // Butterflies fed from BRAM read data one cycle after the address (rd_fire_r). wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out; wire bf0_valid, bf1_valid; butterfly_unit_pipe u_bf0 ( .clk(clk), .rst_n(rst_n), - .valid_i(issue_fire), - .a(issue_a0), .b(issue_b0), - .zeta(zeta), .mode(mode_r), + .valid_i(rd_fire_r), + .a(rd_data_0), .b(rd_data_1), + .zeta(zeta_fed), .mode(mode_r), .a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid) ); butterfly_unit_pipe u_bf1 ( .clk(clk), .rst_n(rst_n), - .valid_i(issue_fire), - .a(issue_a1), .b(issue_b1), - .zeta(zeta), .mode(mode_r), + .valid_i(rd_fire_r), + .a(rd_data_2), .b(rd_data_3), + .zeta(zeta_fed), .mode(mode_r), .a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid) ); + // zeta must align with the BRAM read data (1 cycle after presentation). + // Scale multiplier (inverse NTT final x3303). wire [DW-1:0] scale_product; wire scale_valid_o; barrett_mul_pipe u_scl ( @@ -108,8 +120,41 @@ module ntt_core ( .product(scale_product), .valid_o(scale_valid_o) ); + // ---- write ports ---- + wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i); + wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt; + + wire wr_en_0 = loading_we | bf0_valid; + wire wr_en_1 = bf0_valid & ~loading_we; + wire wr_en_2 = bf0_valid & ~loading_we; + wire wr_en_3 = bf0_valid & ~loading_we; + wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[8]; + wire [7:0] wr_addr_1 = b0d[8]; + wire [7:0] wr_addr_2 = a1d[8]; + wire [7:0] wr_addr_3 = b1d[8]; + wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out; + wire [11:0] wr_data_1 = bf0_b_out; + wire [11:0] wr_data_2 = bf1_a_out; + wire [11:0] wr_data_3 = bf1_b_out; + + coeff_bram_4way u_cr ( + .clk(clk), .rst_n(rst_n), + .rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1), + .rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3), + .rd_data_0(rd_data_0), .rd_data_1(rd_data_1), + .rd_data_2(rd_data_2), .rd_data_3(rd_data_3), + .wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3), + .wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1), + .wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3), + .wr_data_0(wr_data_0), .wr_data_1(wr_data_1), + .wr_data_2(wr_data_2), .wr_data_3(wr_data_3) + ); + assign ready_o = (state == S_IDLE) || (state == S_LOAD); - assign coeff_out = coeff_out_r; + // During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle + // latency, matching the old combinational-read + 1 output reg). Scale path + // uses the registered barrett result. + assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r; assign valid_o = valid_o_r; assign done_o = (state == S_DONE); @@ -122,157 +167,113 @@ module ntt_core ( S_CMP_DRAIN: if (bf_inflight == 6'd0) next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN; S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; - S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; + S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE; S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE; S_DONE: next_state = S_IDLE; default: next_state = S_IDLE; endcase end + always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer_len <= 8'd0; - zeta_idx <= 7'd0; - layer <= 3'd0; - bf_done <= 1'b0; - mode_r <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; - a0_d0<=8'd0;a0_d1<=8'd0;a0_d2<=8'd0;a0_d3<=8'd0;a0_d4<=8'd0;a0_d5<=8'd0;a0_d6<=8'd0;a0_d7<=8'd0; - b0_d0<=8'd0;b0_d1<=8'd0;b0_d2<=8'd0;b0_d3<=8'd0;b0_d4<=8'd0;b0_d5<=8'd0;b0_d6<=8'd0;b0_d7<=8'd0; - a1_d0<=8'd0;a1_d1<=8'd0;a1_d2<=8'd0;a1_d3<=8'd0;a1_d4<=8'd0;a1_d5<=8'd0;a1_d6<=8'd0;a1_d7<=8'd0; - b1_d0<=8'd0;b1_d1<=8'd0;b1_d2<=8'd0;b1_d3<=8'd0;b1_d4<=8'd0;b1_d5<=8'd0;b1_d6<=8'd0;b1_d7<=8'd0; - wb_valid_r <= 1'b0; - wb_a0_r <= 12'd0; wb_b0_r <= 12'd0; wb_a1_r <= 12'd0; wb_b1_r <= 12'd0; - wb_a0a_r <= 8'd0; wb_b0a_r <= 8'd0; wb_a1a_r <= 8'd0; wb_b1a_r <= 8'd0; - coeff_out_r <= 12'd0; - valid_o_r <= 1'b0; - scale_valid_i <= 1'b0; - scale_a_i <= 12'd0; - for (ci = 0; ci < N; ci = ci + 1) cr[ci] = 12'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer_len <= 8'd0; + zeta_idx <= 7'd0; zeta_fed <= 12'd0; + layer <= 3'd0; mode_r <= 1'b0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; + rd_fire_r <= 1'b0; + for (k = 0; k <= 8; k = k + 1) begin + a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0; + end + coeff_out_r <= 12'd0; valid_o_r <= 1'b0; + scale_valid_i <= 1'b0; scale_a_i <= 12'd0; end else begin state <= next_state; scale_valid_i <= 1'b0; - // Writeback address delay lines (8-deep = butterfly latency). - a0_d0<=j; a0_d1<=a0_d0;a0_d2<=a0_d1;a0_d3<=a0_d2;a0_d4<=a0_d3;a0_d5<=a0_d4;a0_d6<=a0_d5;a0_d7<=a0_d6; - b0_d0<=j+layer_len; b0_d1<=b0_d0;b0_d2<=b0_d1;b0_d3<=b0_d2;b0_d4<=b0_d3;b0_d5<=b0_d4;b0_d6<=b0_d5;b0_d7<=b0_d6; - a1_d0<=j+8'd1; a1_d1<=a1_d0;a1_d2<=a1_d1;a1_d3<=a1_d2;a1_d4<=a1_d3;a1_d5<=a1_d4;a1_d6<=a1_d5;a1_d7<=a1_d6; - b1_d0<=j+8'd1+layer_len; b1_d1<=b1_d0;b1_d2<=b1_d1;b1_d3<=b1_d2;b1_d4<=b1_d3;b1_d5<=b1_d4;b1_d6<=b1_d5;b1_d7<=b1_d6; + // read->feed pipeline register (BRAM latency) + zeta alignment + rd_fire_r <= issue_fire; + zeta_fed <= zeta; - // In-flight pair accounting (mirrors single-butterfly version). - if (issue_fire && !wb_valid_r) - bf_inflight <= bf_inflight + 6'd1; - else if (!issue_fire && wb_valid_r) - bf_inflight <= bf_inflight - 6'd1; + // writeback address delay lines: sample j-derived addrs at read cycle + a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1; + for (k = 1; k <= 8; k = k + 1) begin + a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1]; + a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1]; + end + + // in-flight pair accounting: +1 on issue, -1 on writeback + if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1; + else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1; if (state != S_OUTPUT && state != S_OUT_SCALE) valid_o_r <= 1'b0; if (state == S_IDLE && valid_i) begin - cr[0] <= coeff_in; load_cnt <= 8'd1; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer <= 3'd0; - bf_done <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer <= 3'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; mode_r <= mode; - if (!mode) begin - layer_len <= 8'd128; - zeta_idx <= 7'd1; - end else begin - layer_len <= 8'd2; - zeta_idx <= 7'd127; - end + if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end + else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end end - if (state == S_LOAD && valid_i) begin - cr[load_cnt] <= coeff_in; - load_cnt <= load_cnt + 8'd1; - end + if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1; - // ---- issue advance: step j by 2 (two butterflies/cycle) ---- + // ---- issue advance (read pointer j, step 2) ---- if (state == S_CMP_RUN && !layer_issue_done) begin if (issue_group_last) begin if (!mode_r) zeta_idx <= zeta_idx + 7'd1; else zeta_idx <= zeta_idx - 7'd1; - - if (issue_layer_last) begin - layer_issue_done <= 1'b1; - end else begin - start <= next_group_start[7:0]; - j <= next_group_start[7:0]; - end + if (issue_layer_last) layer_issue_done <= 1'b1; + else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end end else begin j <= j + 8'd2; end end - // ---- writeback: commit previous cycle's pair results ---- - if (wb_valid_r) begin - cr[wb_a0a_r] <= wb_a0_r; - cr[wb_b0a_r] <= wb_b0_r; - cr[wb_a1a_r] <= wb_a1_r; - cr[wb_b1a_r] <= wb_b1_r; - end - - wb_valid_r <= bf0_valid; // both lanes share latency/valid - if (bf0_valid) begin - wb_a0_r <= bf0_a_out; wb_b0_r <= bf0_b_out; - wb_a1_r <= bf1_a_out; wb_b1_r <= bf1_b_out; - wb_a0a_r <= a0_d7; wb_b0a_r <= b0_d7; - wb_a1a_r <= a1_d7; wb_b1a_r <= b1_d7; - end - // ---- layer advance after drain ---- if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin - if (layer + 3'd1 >= LAYERS) begin - bf_done <= 1'b1; - end else begin + if (layer + 3'd1 < LAYERS) begin layer <= layer + 3'd1; layer_len <= next_layer_len; - start <= 8'd0; - j <= 8'd0; + start <= 8'd0; j <= 8'd0; layer_issue_done <= 1'b0; end end + // ---- OUT_PREP: reset output pointers ---- if (state == S_OUT_PREP) begin - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - if (!mode_r) begin - coeff_out_r <= cr[0]; - valid_o_r <= 1'b1; - end + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + // (scale first addr 0 is presented via out_rptr in this PREP cycle) end - if (state == S_OUTPUT && valid_o_r && ready_i) begin - if (out_cnt < 8'd255) begin - out_cnt <= out_cnt + 8'd1; - coeff_out_r <= cr[out_cnt + 8'd1]; - valid_o_r <= 1'b1; + // ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0; + // valid_o is a registered "presented a valid addr last cycle". Present + // addresses out_rptr = 0..255 (256), giving 256 valid outputs. ---- + if (state == S_OUTPUT) begin + if (out_rptr < 9'd256) begin + out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr + valid_o_r <= 1'b1; // data valid next cycle end else begin - out_cnt <= 8'd0; valid_o_r <= 1'b0; end + if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted end + // ---- OUT_SCALE: read coeff, x3303 via barrett, emit ---- if (state == S_OUT_SCALE) begin + if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1; + // feed barrett with the coeff that arrived this cycle if (scale_issue_cnt < 9'd256) begin scale_valid_i <= 1'b1; - scale_a_i <= cr[scale_issue_cnt[7:0]]; + scale_a_i <= rd_data_0; scale_issue_cnt <= scale_issue_cnt + 9'd1; end valid_o_r <= 1'b0; @@ -285,12 +286,9 @@ module ntt_core ( if (state == S_DONE) begin load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - layer_issue_done <= 1'b0; - bf_inflight <= 6'd0; - wb_valid_r <= 1'b0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; valid_o_r <= 1'b0; end end