From d8d57db82aa723b66d611bad887b737b37f97163 Mon Sep 17 00:00:00 2001 From: FallenSigh Date: Sat, 11 Jul 2026 17:21:21 +0800 Subject: [PATCH] optimize --- sync_rtl/comp_decomp/comp_decomp_sync.v | 7 +- sync_rtl/ntt/TB/xsim_run.tcl | 2 + sync_rtl/ntt/butterfly_unit_pipe.v | 29 ++- sync_rtl/ntt/coeff_bram_4way.v | 113 +++++++++ sync_rtl/ntt/ntt_core.v | 311 ++++++++++++------------ sync_rtl/poly_mul/TB/xsim_run.tcl | 1 + sync_rtl/poly_mul/poly_mul_sync.v | 124 ++++------ sync_rtl/top/TB/xsim_run.tcl | 2 + sync_rtl/top/mlkem_top.v | 231 +++++++++++------- 9 files changed, 504 insertions(+), 316 deletions(-) create mode 100644 sync_rtl/ntt/coeff_bram_4way.v diff --git a/sync_rtl/comp_decomp/comp_decomp_sync.v b/sync_rtl/comp_decomp/comp_decomp_sync.v index 3ba4cbd..128c50a 100644 --- a/sync_rtl/comp_decomp/comp_decomp_sync.v +++ b/sync_rtl/comp_decomp/comp_decomp_sync.v @@ -75,9 +75,10 @@ module comp_decomp_sync ( wire pipe_busy = valid_s0 | valid_s1 | valid_s2 | valid_s3 | valid_s4 | valid_s5; - // Accept a new command only when the pipeline and output slot are both free. - // Current users issue one coefficient at a time and wait for valid_o. - assign ready_o = !pipe_busy && (!valid_r || ready_i); + // Fully pipelined: accept one coefficient per cycle (B1). Every stage + // advances unconditionally and the sole consumer ties ready_i high, so the + // pipeline never needs to backpressure; streaming feeders issue 1 coeff/cyc. + assign ready_o = 1'b1; assign coeff_out = data_r; assign valid_o = valid_r; diff --git a/sync_rtl/ntt/TB/xsim_run.tcl b/sync_rtl/ntt/TB/xsim_run.tcl index b099a71..5a9edd1 100644 --- a/sync_rtl/ntt/TB/xsim_run.tcl +++ b/sync_rtl/ntt/TB/xsim_run.tcl @@ -40,6 +40,8 @@ xvlog -sv ${SRC_DIR}/butterfly_unit.v xvlog -sv ${SRC_DIR}/butterfly_unit_pipe.v # NTT core (FSM-based, instantiates butterfly_unit + zeta_rom + barrett_mul) +xvlog -sv ${SRC_DIR}/../storage/sd_bram.v +xvlog -sv ${SRC_DIR}/coeff_bram_4way.v xvlog -sv ${SRC_DIR}/ntt_core.v # ================================================================ diff --git a/sync_rtl/ntt/butterfly_unit_pipe.v b/sync_rtl/ntt/butterfly_unit_pipe.v index c672953..c365a24 100644 --- a/sync_rtl/ntt/butterfly_unit_pipe.v +++ b/sync_rtl/ntt/butterfly_unit_pipe.v @@ -18,21 +18,32 @@ module butterfly_unit_pipe ( localparam [11:0] Q12 = 12'd3329; localparam [12:0] Q13 = 13'd3329; - wire [11:0] diff = (b >= a) ? (b - a) : (b - a + Q12); - wire [11:0] mul_data = mode ? diff : b; + // Plan A: register the inputs. Operands arrive combinationally from the + // coeff_bram read; without this stage the path coeff_bram-read -> diff -> + // first multiply stage sits in one cycle (the mlkem_clk critical path at + // 100 MHz). Registering here splits it into (BRAM read -> reg) and + // (reg -> diff -> multiply). Adds one pipeline cycle (absorbed by the NTT + // writeback delay line). + reg [11:0] a_r, b_r, zeta_r; + reg mode_ri, vld_r; + + wire [11:0] diff = (b_r >= a_r) ? (b_r - a_r) : (b_r - a_r + Q12); + wire [11:0] mul_data = mode_ri ? diff : b_r; wire [11:0] mul_result; wire mul_valid; barrett_mul_pipe u_barrett ( .clk(clk), .rst_n(rst_n), - .valid_i(valid_i), - .a(zeta), + .valid_i(vld_r), + .a(zeta_r), .b(mul_data), .product(mul_result), .valid_o(mul_valid) ); + // a/b/mode delay lines (fed from the REGISTERED inputs a_r/b_r/mode_ri so + // a_d6 aligns with mul_valid = valid_i delayed 8). reg [11:0] a_d0, a_d1, a_d2, a_d3, a_d4, a_d5, a_d6; reg [11:0] b_d0, b_d1, b_d2, b_d3, b_d4, b_d5, b_d6; reg mode_d0, mode_d1, mode_d2, mode_d3, mode_d4, mode_d5, mode_d6; @@ -51,6 +62,7 @@ module butterfly_unit_pipe ( always @(posedge clk or negedge rst_n) begin if (!rst_n) begin + a_r <= 12'd0; b_r <= 12'd0; zeta_r <= 12'd0; mode_ri <= 1'b0; vld_r <= 1'b0; a_d0 <= 12'd0; a_d1 <= 12'd0; a_d2 <= 12'd0; a_d3 <= 12'd0; a_d4 <= 12'd0; a_d5 <= 12'd0; a_d6 <= 12'd0; b_d0 <= 12'd0; b_d1 <= 12'd0; b_d2 <= 12'd0; b_d3 <= 12'd0; b_d4 <= 12'd0; b_d5 <= 12'd0; b_d6 <= 12'd0; mode_d0 <= 1'b0; mode_d1 <= 1'b0; mode_d2 <= 1'b0; mode_d3 <= 1'b0; mode_d4 <= 1'b0; mode_d5 <= 1'b0; mode_d6 <= 1'b0; @@ -58,9 +70,12 @@ module butterfly_unit_pipe ( b_out_r <= 12'd0; valid_r <= 1'b0; end else begin - a_d0 <= a; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5; - b_d0 <= b; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5; - mode_d0 <= mode; mode_d1 <= mode_d0; mode_d2 <= mode_d1; + // input register stage + a_r <= a; b_r <= b; zeta_r <= zeta; mode_ri <= mode; vld_r <= valid_i; + + a_d0 <= a_r; a_d1 <= a_d0; a_d2 <= a_d1; a_d3 <= a_d2; a_d4 <= a_d3; a_d5 <= a_d4; a_d6 <= a_d5; + b_d0 <= b_r; b_d1 <= b_d0; b_d2 <= b_d1; b_d3 <= b_d2; b_d4 <= b_d3; b_d5 <= b_d4; b_d6 <= b_d5; + mode_d0 <= mode_ri; mode_d1 <= mode_d0; mode_d2 <= mode_d1; mode_d3 <= mode_d2; mode_d4 <= mode_d3; mode_d5 <= mode_d4; mode_d6 <= mode_d5; valid_r <= mul_valid; diff --git a/sync_rtl/ntt/coeff_bram_4way.v b/sync_rtl/ntt/coeff_bram_4way.v new file mode 100644 index 0000000..c68e718 --- /dev/null +++ b/sync_rtl/ntt/coeff_bram_4way.v @@ -0,0 +1,113 @@ +// coeff_bram_4way.v - 4-way conflict-free banked storage for NTT coefficients +// +// Replaces the 256x12 LUT register file (reg [11:0] cr [0:255]) with 4 BRAM +// banks (each 128x12 via sd_bram), removing the large multi-port LUT mux tree. +// +// Bank function: bank(a) = { a[0], ^a[7:1] } (bit0 XOR parity of upper 7 bits) +// local(a) = a[7:1] (7-bit index into the 128-deep bank) +// +// This function makes a dual-butterfly's four simultaneous coefficient accesses +// {j, j+1, j+len, j+1+len} land in four DISTINCT banks for every ML-KEM NTT +// layer_len in {2,4,...,128}: j is even and len is even, so +// j -> {0, P } +// j+1 -> {1, P } +// j+len -> {0, P'} (flipping bit m>=1 flips the upper-bits parity) +// j+1+len -> {1, P'} +// with P != P'. Four distinct banks => 4 conflict-free reads/cycle from four +// single-read-port banks, and likewise 4 conflict-free writes/cycle. Each bank +// therefore needs only 1 read + 1 write per cycle (an sd_bram), no ping-pong. +// +// Read latency is 1 cycle (sd_bram registered read): present rd_addr_k this +// cycle, rd_data_k valid next cycle. Writes commit on the presenting cycle. +// +// NOTE: the caller must guarantee the four read addresses map to four distinct +// banks (true for the NTT butterfly pattern) and the four write addresses too. +// Sequential access (load/output) uses a single port and is always safe. + +module coeff_bram_4way ( + input clk, rst_n, + // Four read addresses (global 8-bit); data returns 1 cycle later. + input [7:0] rd_addr_0, rd_addr_1, rd_addr_2, rd_addr_3, + output [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3, + // Four write ports. + input wr_en_0, wr_en_1, wr_en_2, wr_en_3, + input [7:0] wr_addr_0, wr_addr_1, wr_addr_2, wr_addr_3, + input [11:0] wr_data_0, wr_data_1, wr_data_2, wr_data_3 +); + + // Bank id and local index for an address. + function [1:0] bank_id; + input [7:0] a; + bank_id = {a[0], ^a[7:1]}; + endfunction + + wire [1:0] rb0 = bank_id(rd_addr_0); + wire [1:0] rb1 = bank_id(rd_addr_1); + wire [1:0] rb2 = bank_id(rd_addr_2); + wire [1:0] rb3 = bank_id(rd_addr_3); + + wire [1:0] wb0 = bank_id(wr_addr_0); + wire [1:0] wb1 = bank_id(wr_addr_1); + wire [1:0] wb2 = bank_id(wr_addr_2); + wire [1:0] wb3 = bank_id(wr_addr_3); + + // Per-bank read-address crossbar: bank b reads the local index of whichever + // read port targets it (exactly one, by construction). + wire [6:0] bank_rd_local [0:3]; + wire [6:0] bank_wr_local [0:3]; + wire bank_wr_en [0:3]; + wire [11:0] bank_wr_data [0:3]; + wire [11:0] bank_rd_data [0:3]; + + genvar b; + generate + for (b = 0; b < 4; b = b + 1) begin : gen_xbar + // read address route (priority: 3>2>1>0; ties impossible for NTT) + assign bank_rd_local[b] = + (rb3 == b[1:0]) ? rd_addr_3[7:1] : + (rb2 == b[1:0]) ? rd_addr_2[7:1] : + (rb1 == b[1:0]) ? rd_addr_1[7:1] : + rd_addr_0[7:1]; + // write route + wire m0 = wr_en_0 && (wb0 == b[1:0]); + wire m1 = wr_en_1 && (wb1 == b[1:0]); + wire m2 = wr_en_2 && (wb2 == b[1:0]); + wire m3 = wr_en_3 && (wb3 == b[1:0]); + assign bank_wr_en[b] = m0 | m1 | m2 | m3; + assign bank_wr_local[b] = + m3 ? wr_addr_3[7:1] : + m2 ? wr_addr_2[7:1] : + m1 ? wr_addr_1[7:1] : wr_addr_0[7:1]; + assign bank_wr_data[b] = + m3 ? wr_data_3 : + m2 ? wr_data_2 : + m1 ? wr_data_1 : wr_data_0; + + sd_bram #(.W(12), .D(128), .A(7)) u_bank ( + .clk(clk), + .rd_addr(bank_rd_local[b]), + .rd_data(bank_rd_data[b]), + .wr_en(bank_wr_en[b]), + .wr_addr(bank_wr_local[b]), + .wr_data(bank_wr_data[b]) + ); + end + endgenerate + + // Register the read bank-ids to align with sd_bram's 1-cycle read latency, + // then route each bank's output back to the requesting read port. + reg [1:0] rb0_r, rb1_r, rb2_r, rb3_r; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + rb0_r <= 2'd0; rb1_r <= 2'd0; rb2_r <= 2'd0; rb3_r <= 2'd0; + end else begin + rb0_r <= rb0; rb1_r <= rb1; rb2_r <= rb2; rb3_r <= rb3; + end + end + + assign rd_data_0 = bank_rd_data[rb0_r]; + assign rd_data_1 = bank_rd_data[rb1_r]; + assign rd_data_2 = bank_rd_data[rb2_r]; + assign rd_data_3 = bank_rd_data[rb3_r]; + +endmodule diff --git a/sync_rtl/ntt/ntt_core.v b/sync_rtl/ntt/ntt_core.v index a326990..8e46be2 100644 --- a/sync_rtl/ntt/ntt_core.v +++ b/sync_rtl/ntt/ntt_core.v @@ -1,12 +1,18 @@ -// ntt_core.v - NTT core with individual coefficient registers +// ntt_core.v - NTT core with 4-way banked BRAM coefficient storage // -// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path. -// The arithmetic hot path is split into: -// address/operand/zeta register -> pipelined Barrett butterfly -> writeback -// Each NTT layer is issued continuously into the butterfly pipeline, then the -// core drains pending writebacks before starting the next dependent layer. -// In inverse mode, final x3303 output scaling also uses a pipelined Barrett -// multiplier so the output path does not reintroduce a combinational reducer. +// Coefficient storage moved from a 256x12 LUT register file to coeff_bram_4way +// (4 x 128x12 sd_bram, conflict-free banked). This removes the large multi-port +// LUT mux tree. Two pipelined butterflies (R1) still issue one pair/cycle. +// +// Because BRAM reads are synchronous (1-cycle latency), the compute pipeline +// gains one stage: +// READ (present rd addrs from j) -> FEED butterflies (t+1, rd_data valid) +// -> 8-stage Barrett butterfly -> WRITEBACK (t+9) into the write buffer. +// Read and writeback hit disjoint banks (butterfly index sets in a layer are +// disjoint), so each bank does 1 read + 1 write per cycle (sd_bram = 1R1W). +// +// LOAD/OUTPUT/OUT_SCALE stream through a single bank port with 1-cycle read +// latency (ready_i is tied high by all NTT consumers, so no output backpressure). module ntt_core ( input clk, rst_n, @@ -21,9 +27,6 @@ module ntt_core ( ); localparam N = 256, LAYERS = 7, DW = 12; - reg [DW-1:0] cr [0:N-1]; - integer ci; - localparam S_IDLE = 4'd0; localparam S_LOAD = 4'd1; localparam S_CMP_RUN = 4'd2; @@ -36,69 +39,122 @@ module ntt_core ( reg [3:0] state, next_state; reg [7:0] load_cnt; - reg [7:0] out_cnt; - reg [8:0] scale_issue_cnt; - reg [8:0] scale_emit_cnt; + reg [8:0] out_rptr; // output read address pointer (0..256) + reg [8:0] out_cnt; // output coeffs emitted (0..256) + reg [8:0] scale_rptr; // scale read address pointer + reg [8:0] scale_issue_cnt; // coeffs fed to scale barrett + reg [8:0] scale_emit_cnt; // coeffs emitted from scale barrett reg [7:0] j, start, layer_len; reg [6:0] zeta_idx; reg [2:0] layer; - reg bf_done; reg mode_r; reg layer_issue_done; - reg [4:0] bf_inflight; + reg [5:0] bf_inflight; - reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7; - reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7; - reg wb_valid_r; - reg [DW-1:0] wb_a_data_r, wb_b_data_r; - reg [7:0] wb_wa_r, wb_wb_r; + reg rd_fire_r; // issue_fire delayed 1 (BRAM read latency) -> feed + // 9-deep writeback address delay lines (BRAM read + 8-stage butterfly). + reg [7:0] a0d[0:9]; reg [7:0] b0d[0:9]; + reg [7:0] a1d[0:9]; reg [7:0] b1d[0:9]; + integer k; reg [DW-1:0] coeff_out_r; reg valid_o_r; reg scale_valid_i; reg [DW-1:0] scale_a_i; + reg [DW-1:0] zeta_fed; // zeta aligned to BRAM read data (1 cyc after present) wire [DW-1:0] zeta; zeta_rom u_z (.addr(zeta_idx), .zeta(zeta)); wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len}; wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len}; - wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end); + wire issue_group_last = ({1'b0, j} + 9'd2 >= group_end); wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256); wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1); wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done; - wire [DW-1:0] issue_a = cr[j]; - wire [DW-1:0] issue_b = cr[j + layer_len]; - wire [DW-1:0] bf_a_out, bf_b_out; - wire bf_valid; - butterfly_unit_pipe u_bf ( - .clk(clk), - .rst_n(rst_n), - .valid_i(issue_fire), - .a(issue_a), - .b(issue_b), - .zeta(zeta), - .mode(mode_r), - .a_out(bf_a_out), - .b_out(bf_b_out), - .valid_o(bf_valid) + // ---- coeff_bram_4way port muxing ---- + // Read addr 0 is shared by compute (j), output (out_rptr), scale (scale_rptr). + wire [7:0] cmp_a0 = j; + wire [7:0] cmp_b0 = j + layer_len; + wire [7:0] cmp_a1 = j + 8'd1; + wire [7:0] cmp_b1 = j + 8'd1 + layer_len; + + wire [7:0] rd_addr_0 = (state == S_OUT_PREP || state == S_OUTPUT) ? out_rptr[7:0] : + (state == S_OUT_SCALE) ? scale_rptr[7:0] : cmp_a0; + // Ports 1..3 carry the other three butterfly operands ONLY during compute; + // otherwise they must mirror port 0 so the read crossbar's per-bank priority + // arbitration doesn't route a stale compute address into port 0's bank. + wire cmp_rd = (state == S_CMP_RUN) || (state == S_CMP_DRAIN); + wire [7:0] rd_addr_1 = cmp_rd ? cmp_b0 : rd_addr_0; + wire [7:0] rd_addr_2 = cmp_rd ? cmp_a1 : rd_addr_0; + wire [7:0] rd_addr_3 = cmp_rd ? cmp_b1 : rd_addr_0; + wire [11:0] rd_data_0, rd_data_1, rd_data_2, rd_data_3; + + // Butterflies fed from BRAM read data one cycle after the address (rd_fire_r). + wire [DW-1:0] bf0_a_out, bf0_b_out, bf1_a_out, bf1_b_out; + wire bf0_valid, bf1_valid; + butterfly_unit_pipe u_bf0 ( + .clk(clk), .rst_n(rst_n), + .valid_i(rd_fire_r), + .a(rd_data_0), .b(rd_data_1), + .zeta(zeta_fed), .mode(mode_r), + .a_out(bf0_a_out), .b_out(bf0_b_out), .valid_o(bf0_valid) ); + butterfly_unit_pipe u_bf1 ( + .clk(clk), .rst_n(rst_n), + .valid_i(rd_fire_r), + .a(rd_data_2), .b(rd_data_3), + .zeta(zeta_fed), .mode(mode_r), + .a_out(bf1_a_out), .b_out(bf1_b_out), .valid_o(bf1_valid) + ); + // zeta must align with the BRAM read data (1 cycle after presentation). + // Scale multiplier (inverse NTT final x3303). wire [DW-1:0] scale_product; wire scale_valid_o; barrett_mul_pipe u_scl ( - .clk(clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(scale_valid_i), - .a(scale_a_i), - .b(12'd3303), - .product(scale_product), - .valid_o(scale_valid_o) + .a(scale_a_i), .b(12'd3303), + .product(scale_product), .valid_o(scale_valid_o) + ); + + // ---- write ports ---- + wire loading_we = (state == S_IDLE && valid_i) || (state == S_LOAD && valid_i); + wire [7:0] load_addr = (state == S_IDLE) ? 8'd0 : load_cnt; + + wire wr_en_0 = loading_we | bf0_valid; + wire wr_en_1 = bf0_valid & ~loading_we; + wire wr_en_2 = bf0_valid & ~loading_we; + wire wr_en_3 = bf0_valid & ~loading_we; + wire [7:0] wr_addr_0 = loading_we ? load_addr : a0d[9]; + wire [7:0] wr_addr_1 = b0d[9]; + wire [7:0] wr_addr_2 = a1d[9]; + wire [7:0] wr_addr_3 = b1d[9]; + wire [11:0] wr_data_0 = loading_we ? coeff_in : bf0_a_out; + wire [11:0] wr_data_1 = bf0_b_out; + wire [11:0] wr_data_2 = bf1_a_out; + wire [11:0] wr_data_3 = bf1_b_out; + + coeff_bram_4way u_cr ( + .clk(clk), .rst_n(rst_n), + .rd_addr_0(rd_addr_0), .rd_addr_1(rd_addr_1), + .rd_addr_2(rd_addr_2), .rd_addr_3(rd_addr_3), + .rd_data_0(rd_data_0), .rd_data_1(rd_data_1), + .rd_data_2(rd_data_2), .rd_data_3(rd_data_3), + .wr_en_0(wr_en_0), .wr_en_1(wr_en_1), .wr_en_2(wr_en_2), .wr_en_3(wr_en_3), + .wr_addr_0(wr_addr_0), .wr_addr_1(wr_addr_1), + .wr_addr_2(wr_addr_2), .wr_addr_3(wr_addr_3), + .wr_data_0(wr_data_0), .wr_data_1(wr_data_1), + .wr_data_2(wr_data_2), .wr_data_3(wr_data_3) ); assign ready_o = (state == S_IDLE) || (state == S_LOAD); - assign coeff_out = coeff_out_r; + // During forward OUTPUT, drive coeff_out straight from the BRAM read (1-cycle + // latency, matching the old combinational-read + 1 output reg). Scale path + // uses the registered barrett result. + assign coeff_out = (state == S_OUTPUT) ? rd_data_0 : coeff_out_r; assign valid_o = valid_o_r; assign done_o = (state == S_DONE); @@ -108,10 +164,10 @@ module ntt_core ( S_IDLE: if (valid_i) next_state = S_LOAD; S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN; S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN; - S_CMP_DRAIN: if (bf_inflight == 5'd0) + S_CMP_DRAIN: if (bf_inflight == 6'd0) next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN; S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT; - S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE; + S_OUTPUT: if (out_cnt >= 9'd256) next_state = S_DONE; S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE; S_DONE: next_state = S_IDLE; default: next_state = S_IDLE; @@ -122,145 +178,104 @@ module ntt_core ( if (!rst_n) begin state <= S_IDLE; load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer_len <= 8'd0; - zeta_idx <= 7'd0; - layer <= 3'd0; - bf_done <= 1'b0; - mode_r <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; - wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0; - wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0; - wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0; - wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0; - wb_valid_r <= 1'b0; - wb_a_data_r <= 12'd0; - wb_b_data_r <= 12'd0; - wb_wa_r <= 8'd0; - wb_wb_r <= 8'd0; - coeff_out_r <= 12'd0; - valid_o_r <= 1'b0; - scale_valid_i <= 1'b0; - scale_a_i <= 12'd0; - for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer_len <= 8'd0; + zeta_idx <= 7'd0; zeta_fed <= 12'd0; + layer <= 3'd0; mode_r <= 1'b0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; + rd_fire_r <= 1'b0; + for (k = 0; k <= 9; k = k + 1) begin + a0d[k] <= 8'd0; b0d[k] <= 8'd0; a1d[k] <= 8'd0; b1d[k] <= 8'd0; + end + coeff_out_r <= 12'd0; valid_o_r <= 1'b0; + scale_valid_i <= 1'b0; scale_a_i <= 12'd0; end else begin state <= next_state; scale_valid_i <= 1'b0; - wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2; - wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6; - wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2; - wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6; + // read->feed pipeline register (BRAM latency) + zeta alignment + rd_fire_r <= issue_fire; + zeta_fed <= zeta; - if (issue_fire && !wb_valid_r) - bf_inflight <= bf_inflight + 5'd1; - else if (!issue_fire && wb_valid_r) - bf_inflight <= bf_inflight - 5'd1; + // writeback address delay lines: sample j-derived addrs at read cycle + a0d[0] <= cmp_a0; b0d[0] <= cmp_b0; a1d[0] <= cmp_a1; b1d[0] <= cmp_b1; + for (k = 1; k <= 9; k = k + 1) begin + a0d[k] <= a0d[k-1]; b0d[k] <= b0d[k-1]; + a1d[k] <= a1d[k-1]; b1d[k] <= b1d[k-1]; + end + + // in-flight pair accounting: +1 on issue, -1 on writeback + if (issue_fire && !bf0_valid) bf_inflight <= bf_inflight + 6'd1; + else if (!issue_fire && bf0_valid) bf_inflight <= bf_inflight - 6'd1; if (state != S_OUTPUT && state != S_OUT_SCALE) valid_o_r <= 1'b0; if (state == S_IDLE && valid_i) begin - cr[0] <= coeff_in; load_cnt <= 8'd1; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - j <= 8'd0; - start <= 8'd0; - layer <= 3'd0; - bf_done <= 1'b0; - layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + j <= 8'd0; start <= 8'd0; layer <= 3'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; mode_r <= mode; - if (!mode) begin - layer_len <= 8'd128; - zeta_idx <= 7'd1; - end else begin - layer_len <= 8'd2; - zeta_idx <= 7'd127; - end + if (!mode) begin layer_len <= 8'd128; zeta_idx <= 7'd1; end + else begin layer_len <= 8'd2; zeta_idx <= 7'd127; end end - if (state == S_LOAD && valid_i) begin - cr[load_cnt] <= coeff_in; - load_cnt <= load_cnt + 8'd1; - end + if (state == S_LOAD && valid_i) load_cnt <= load_cnt + 8'd1; + // ---- issue advance (read pointer j, step 2) ---- if (state == S_CMP_RUN && !layer_issue_done) begin if (issue_group_last) begin if (!mode_r) zeta_idx <= zeta_idx + 7'd1; else zeta_idx <= zeta_idx - 7'd1; - - if (issue_layer_last) begin - layer_issue_done <= 1'b1; - end else begin - start <= next_group_start[7:0]; - j <= next_group_start[7:0]; - end + if (issue_layer_last) layer_issue_done <= 1'b1; + else begin start <= next_group_start[7:0]; j <= next_group_start[7:0]; end end else begin - j <= j + 8'd1; + j <= j + 8'd2; end end - if (wb_valid_r) begin - cr[wb_wa_r] <= wb_a_data_r; - cr[wb_wb_r] <= wb_b_data_r; - end - - wb_valid_r <= bf_valid; - if (bf_valid) begin - wb_a_data_r <= bf_a_out; - wb_b_data_r <= bf_b_out; - wb_wa_r <= wa_d7; - wb_wb_r <= wb_d7; - end - - if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin - if (layer + 3'd1 >= LAYERS) begin - bf_done <= 1'b1; - end else begin + // ---- layer advance after drain ---- + if (state == S_CMP_DRAIN && bf_inflight == 6'd0) begin + if (layer + 3'd1 < LAYERS) begin layer <= layer + 3'd1; layer_len <= next_layer_len; - start <= 8'd0; - j <= 8'd0; + start <= 8'd0; j <= 8'd0; layer_issue_done <= 1'b0; end end + // ---- OUT_PREP: reset output pointers ---- if (state == S_OUT_PREP) begin - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - if (!mode_r) begin - coeff_out_r <= cr[0]; - valid_o_r <= 1'b1; - end + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd1; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + // (scale first addr 0 is presented via out_rptr in this PREP cycle) end - if (state == S_OUTPUT && valid_o_r && ready_i) begin - if (out_cnt < 8'd255) begin - out_cnt <= out_cnt + 8'd1; - coeff_out_r <= cr[out_cnt + 8'd1]; - valid_o_r <= 1'b1; + // ---- OUTPUT: stream coeffs. coeff_out is combinational rd_data_0; + // valid_o is a registered "presented a valid addr last cycle". Present + // addresses out_rptr = 0..255 (256), giving 256 valid outputs. ---- + if (state == S_OUTPUT) begin + if (out_rptr < 9'd256) begin + out_rptr <= out_rptr + 9'd1; // present rd_addr_0 = out_rptr + valid_o_r <= 1'b1; // data valid next cycle end else begin - out_cnt <= 8'd0; valid_o_r <= 1'b0; end + if (valid_o_r) out_cnt <= out_cnt + 9'd1; // count emitted end + // ---- OUT_SCALE: read coeff, x3303 via barrett, emit ---- if (state == S_OUT_SCALE) begin + if (scale_rptr < 9'd256) scale_rptr <= scale_rptr + 9'd1; + // feed barrett with the coeff that arrived this cycle if (scale_issue_cnt < 9'd256) begin scale_valid_i <= 1'b1; - scale_a_i <= cr[scale_issue_cnt[7:0]]; + scale_a_i <= rd_data_0; scale_issue_cnt <= scale_issue_cnt + 9'd1; end - valid_o_r <= 1'b0; if (scale_valid_o) begin coeff_out_r <= scale_product; @@ -271,15 +286,11 @@ module ntt_core ( if (state == S_DONE) begin load_cnt <= 8'd0; - out_cnt <= 8'd0; - scale_issue_cnt <= 9'd0; - scale_emit_cnt <= 9'd0; - layer_issue_done <= 1'b0; - bf_inflight <= 5'd0; - wb_valid_r <= 1'b0; + out_rptr <= 9'd0; out_cnt <= 9'd0; + scale_rptr <= 9'd0; scale_issue_cnt <= 9'd0; scale_emit_cnt <= 9'd0; + layer_issue_done <= 1'b0; bf_inflight <= 6'd0; valid_o_r <= 1'b0; end end end - endmodule diff --git a/sync_rtl/poly_mul/TB/xsim_run.tcl b/sync_rtl/poly_mul/TB/xsim_run.tcl index 73aecdc..6f55b48 100644 --- a/sync_rtl/poly_mul/TB/xsim_run.tcl +++ b/sync_rtl/poly_mul/TB/xsim_run.tcl @@ -46,6 +46,7 @@ xvlog -sv ${PM_DIR}/basecase_mul_pipe.v xvlog -sv ${PM_DIR}/poly_mul_zeta_rom.v # PolyMul sync top (instantiates basecase_mul + poly_mul_zeta_rom) +xvlog -sv ${PM_DIR}/../storage/sd_bram.v xvlog -sv ${PM_DIR}/poly_mul_sync.v # ================================================================ diff --git a/sync_rtl/poly_mul/poly_mul_sync.v b/sync_rtl/poly_mul/poly_mul_sync.v index 29a28d4..bf7f278 100644 --- a/sync_rtl/poly_mul/poly_mul_sync.v +++ b/sync_rtl/poly_mul/poly_mul_sync.v @@ -1,4 +1,4 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (BRAM storage) // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. @@ -6,18 +6,18 @@ // Operation flow: // IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // -// RUN issues one base-case multiply every two cycles, matching the two-cycle -// c0/c1 output bandwidth after the basecase pipeline fills. +// Coefficient storage moved from 256x12 LUT register arrays to four sd_bram +// banks (A_even/A_odd/B_even/B_odd, each 128x12): the base-case reads +// mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single +// read port. This removes the large multi-port LUT mux (~8k LUT) and the long +// BRAM->register-file routing that was the mlkem_clk critical path. // -// The LOAD phase accepts both A and B coefficients simultaneously -// (one pair per cycle) on coeff_a_in/coeff_b_in. +// RUN issues one base-case every two cycles. sd_bram reads are synchronous +// (1-cycle latency); the 2-cycle cadence absorbs it (address presented on the +// gap cycle, data consumed on the issue cycle). A one-cycle priming gap is +// inserted at LOAD->RUN so the first base-case's operands are ready. // -// The COMPUTE phase outputs the 256 result coefficients one per cycle -// via valid/ready handshake on coeff_out/valid_o/ready_i. -// -// Memory: 256×12-bit register arrays for A and B coefficients. -// -// Interface: +// Interface (unchanged): // clk, rst_n - Clock, active-low reset // coeff_a_in[11:0]- Polynomial A coefficient input // coeff_b_in[11:0]- Polynomial B coefficient input @@ -38,18 +38,12 @@ module poly_mul_sync ( input ready_i ); - // State definitions localparam S_IDLE = 3'd0; localparam S_LOAD = 3'd1; localparam S_RUN = 3'd2; reg [2:0] state, next_state; - // Coefficient storage (register arrays) - reg [11:0] mem_A [0:255]; - reg [11:0] mem_B [0:255]; - - // Counters reg [7:0] load_cnt; // 0..256 for loading 256 pairs reg [7:0] issue_cnt; // 0..128, issued base-cases reg [7:0] out_cnt; // 0..255, consumed output coefficients @@ -57,53 +51,61 @@ module poly_mul_sync ( reg out_phase; // 0=c0, 1=c1 reg out_valid_r; - // Registered basecase_mul inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Combinational read signals for the next base-case request. - wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 - wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 - wire [11:0] mem_a0 = mem_A[addr_even]; - wire [11:0] mem_a1 = mem_A[addr_odd]; - wire [11:0] mem_b0 = mem_B[addr_even]; - wire [11:0] mem_b1 = mem_B[addr_odd]; + // ---- Coefficient storage: even/odd banked sd_bram ---- + // A_even[k]=mem_A[2k], A_odd[k]=mem_A[2k+1]; same for B. Read index during + // RUN is issue_k; write index during LOAD is load_cnt>>1 routed by load_cnt[0]. + wire [6:0] issue_k = issue_cnt[6:0]; // 0..127 read index - // Zeta ROM - wire [11:0] zeta; - poly_mul_zeta_rom u_zeta ( - .addr (issue_k), - .zeta (zeta) + wire load_fire = ((state == S_IDLE) || (state == S_LOAD)) && valid_i && ready_o; + wire [6:0] wr_local = load_cnt[7:1]; // load_cnt/2 + wire wr_odd = load_cnt[0]; + + wire we_even = load_fire && !wr_odd; + wire we_odd = load_fire && wr_odd; + + wire [11:0] mem_a0, mem_a1, mem_b0, mem_b1; // sd_bram registered read outputs + sd_bram #(.W(12), .D(128), .A(7)) u_a_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_a_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_b_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_b_in) ); - // Pipelined basecase multiply. Requests are issued every two cycles; inputs - // are registered locally so issue_cnt does not directly drive DSP input muxes. + wire [11:0] zeta; + poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); + wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk (clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0 (bc_a0_reg), - .a1 (bc_a1_reg), - .b0 (bc_b0_reg), - .b1 (bc_b1_reg), + .a0(bc_a0_reg), .a1(bc_a1_reg), + .b0(bc_b0_reg), .b1(bc_b1_reg), .zeta(bc_zeta_reg), - .c0 (bc_c0), - .c1 (bc_c1), + .c0(bc_c0), .c1(bc_c1), .valid_o(bc_vo) ); - // Output interface assign ready_o = (state == S_IDLE) || (state == S_LOAD); assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - // State transition logic (combinational) always @* begin next_state = state; case (state) @@ -116,8 +118,6 @@ module poly_mul_sync ( endcase end - // Sequential logic - integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; @@ -127,27 +127,16 @@ module poly_mul_sync ( issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; - bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; - bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; - bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; - c1_reg <= 12'd0; - for (i = 0; i < 256; i = i + 1) begin - mem_A[i] <= 12'd0; - mem_B[i] <= 12'd0; - end + bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; c1_reg <= 12'd0; end else begin state <= next_state; bc_valid_reg <= 1'b0; - // ---- LOAD phase ---- - // First coefficient captured on IDLE → LOAD transition + // ---- LOAD phase (writes routed to even/odd banks combinationally) ---- if (state == S_IDLE && valid_i && ready_o) begin - mem_A[0] <= coeff_a_in; - mem_B[0] <= coeff_b_in; load_cnt <= 8'd1; issue_cnt <= 8'd0; out_cnt <= 8'd0; @@ -156,25 +145,21 @@ module poly_mul_sync ( out_valid_r <= 1'b0; end - // Subsequent coefficients in LOAD state if (state == S_LOAD && valid_i && ready_o) begin - mem_A[load_cnt] <= coeff_a_in; - mem_B[load_cnt] <= coeff_b_in; load_cnt <= load_cnt + 8'd1; if (load_cnt >= 8'd255) begin issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; + issue_gap <= 1'b1; // priming gap: present issue_k=0 addr first out_phase <= 1'b0; out_valid_r <= 1'b0; end end // ---- COMPUTE phase ---- - // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input - // regs, then launch requests every other cycle. The two-cycle issue - // interval matches the c0/c1 output bandwidth, so no deep result FIFO - // is needed for this always-ready system. + // Issue one base-case every two cycles. On the gap cycle the sd_bram + // address (issue_k) settles; on the issue cycle its registered output + // (mem_a0/a1/b0/b1) is captured into the base-case input registers. if (state == S_RUN) begin if (issue_cnt < 8'd128) begin if (!issue_gap) begin @@ -212,7 +197,6 @@ module poly_mul_sync ( end end - // ---- final output consumed: reset counters before accepting a new op ---- if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin load_cnt <= 8'd0; issue_cnt <= 8'd0; diff --git a/sync_rtl/top/TB/xsim_run.tcl b/sync_rtl/top/TB/xsim_run.tcl index c071d3b..8791e87 100644 --- a/sync_rtl/top/TB/xsim_run.tcl +++ b/sync_rtl/top/TB/xsim_run.tcl @@ -17,6 +17,8 @@ xvlog -sv --relax -i . sync_rtl/ntt/barrett_mul_pipe.v xvlog -sv --relax -i . sync_rtl/ntt/zeta_rom.v xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit.v xvlog -sv --relax -i . sync_rtl/ntt/butterfly_unit_pipe.v +xvlog -sv --relax -i . sync_rtl/storage/sd_bram.v +xvlog -sv --relax -i . sync_rtl/ntt/coeff_bram_4way.v xvlog -sv --relax -i . sync_rtl/ntt/ntt_core.v xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul.v xvlog -sv --relax -i . sync_rtl/poly_mul/basecase_mul_pipe.v diff --git a/sync_rtl/top/mlkem_top.v b/sync_rtl/top/mlkem_top.v index e0395e9..5e00cfd 100644 --- a/sync_rtl/top/mlkem_top.v +++ b/sync_rtl/top/mlkem_top.v @@ -22,7 +22,7 @@ `include "sync_rtl/common/defines.vh" -(* use_dsp = "yes" *) + module mlkem_top #( parameter KMAX = 4 // storage sizing (worst case = ML-KEM-1024) ) ( @@ -90,6 +90,9 @@ module mlkem_top #( ); localparam Q = `Q; // 3329 + localparam [11:0] Q12 = `Q; + localparam [12:0] Q13 = `Q; + localparam [13:0] Q14 = `Q; // Runtime ML-KEM parameter, captured at start_i. reg [2:0] k_r; @@ -309,15 +312,15 @@ module mlkem_top #( // D1 DECOMP writeback: cd_vo while decoding c2 writes v' to bank_a[DEC_VASLOT]. wire dec_v_we = (st == ST_DEC_DECOMP) && dec_in_c2 && cd_vo; wire [13:0] dec_v_wr = DEC_VASLOT*256 + dec_widx; - assign bt_we = ((st == ST_M) && pm_vo) || e2_we || + assign bt_we = ((st == ST_M) && pm_vo) || cbdw_bt_en || u_psum_we || u_intt_we || u_v_we || u_w_we; - assign bt_wa = e2_we ? (c_widx & ((1< [0,Q): add Q when negative - wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'(Q)) : cbd_coeff; + wire [11:0] cbd_modq = cbd_coeff[11] ? (cbd_coeff + 12'd3329) : cbd_coeff; + + // Pipeline the CBD -> bank write by one cycle. The squeeze_buf[buf_ptr] + // extraction -> CBD add/sub -> mod-Q -> BRAM-write route is long and + // routing-dominated (the mlkem_clk critical path at 100 MHz). Register the + // write data/addr/enable here; the bank write commits one cycle later. No + // consumer reads these coeffs until the following stage, which begins more + // than one cycle after the last CBD coeff, so there is no read-after-write + // hazard. The registered write takes priority in the bank_se/bank_t mux so + // the 1-cycle spillover into the next state uses the correct addr/data. + reg cbdw_se_en, cbdw_bt_en; + reg [4:0] cbdw_poly; + reg [7:0] cbdw_widx; + reg [11:0] cbdw_data; + always @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + cbdw_se_en <= 1'b0; cbdw_bt_en <= 1'b0; + cbdw_poly <= 5'd0; cbdw_widx <= 8'd0; cbdw_data <= 12'd0; + end else begin + cbdw_se_en <= ((st == ST_C) && c_busy && cbd_vo && cbd_ack) || + ((st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack && + (c_poly < {1'b0, k_r, 1'b0})); + cbdw_bt_en <= (st == ST_ENC_C) && c_busy && cbd_vo && cbd_ack && + (c_poly == {1'b0, k_r, 1'b0}); + cbdw_poly <= c_poly; + cbdw_widx <= c_widx; + cbdw_data <= cbd_modq; + end + end // ---- ntt_core: forward NTT (mode=0, no scaling) of s[i],e[i] in place ---- // N-stage bookkeeping: process slots S0,S1,E0,E1 (= SLOT_S0 + n_slot). @@ -917,7 +948,7 @@ module mlkem_top #( wire [13:0] u_psum_full = UPSUM*256 + u_acc_radr; // bank_t psum acc addr (MAC) wire [11:0] u_acc_src = u_j0q ? 12'd0 : bt_rd_data; // first term init 0 wire [12:0] u_sum = {1'b0, u_acc_src} + {1'b0, pm_coeff}; - wire [11:0] u_accq = (u_sum >= 13'(Q)) ? (u_sum - 13'(Q)) : u_sum[11:0]; + wire [11:0] u_accq = (u_sum >= Q13) ? (u_sum - Q13) : u_sum[11:0]; // INTT load addr (psum read) + in-place writeback to bank_t[UPSUM] wire [13:0] u_intt_rd = UPSUM*256 + u_ridx[7:0]; wire [13:0] u_intt_wr = UPSUM*256 + u_widx; @@ -926,7 +957,7 @@ module mlkem_top #( wire [13:0] u_add_e1rd = ({2'b0,k_r}+{2'b0,u_row})*256 + u_aidx[7:0]; // bank_se e1 (lead) wire [13:0] u_add_uwr = ({2'b0,k_r}+{2'b0,u_row})*256 + u_awidx; // bank_se u (write) wire [12:0] u_usum = {1'b0, bt_rd_data} + {1'b0, bse_rd_data}; // psum + e1 - wire [11:0] u_uq = (u_usum >= 13'(Q)) ? (u_usum - 13'(Q)) : u_usum[11:0]; + wire [11:0] u_uq = (u_usum >= Q13) ? (u_usum - Q13) : u_usum[11:0]; // ================================================================ // E6: v = INTT(sum_j t_hat[j] o y_hat[j]) + e2 + mu (Encaps ST_ENC_V). @@ -947,8 +978,8 @@ module mlkem_top #( wire [13:0] u_vadd_wr = UPSUM*256 + u_awidx; // bank_t v (write) wire [11:0] u_mu = m_r[u_awidx] ? 12'd1665 : 12'd0; // mu by msg bit wire [13:0] u_vsum = {2'b0, bt_rd_data} + {2'b0, ba_rd_data} + {2'b0, u_mu}; // psum+e2+mu - wire [13:0] u_vsub1 = (u_vsum >= 14'(Q)) ? (u_vsum - 14'(Q)) : u_vsum; - wire [11:0] u_vq = (u_vsub1 >= 14'(Q)) ? (u_vsub1 - 14'(Q)) : u_vsub1[11:0]; + wire [13:0] u_vsub1 = (u_vsum >= Q14) ? (u_vsum - Q14) : u_vsum; + wire [11:0] u_vq = (u_vsub1 >= Q14) ? (u_vsub1 - Q14) : u_vsub1[11:0]; wire u_v_we = (st == ST_ENC_V) && (u_sub == 2'd2) && u_avalid; // ================================================================ @@ -964,7 +995,7 @@ module mlkem_top #( wire [13:0] u_w_vrd = DEC_VASLOT*256 + u_aidx[7:0]; // bank_a v' (lead) wire [13:0] u_wsub_wr = UPSUM*256 + u_awidx; // bank_t w (write) wire [12:0] u_wdiff = {1'b0, ba_rd_data} - {1'b0, bt_rd_data}; // v' - psum - wire [11:0] u_wq = u_wdiff[12] ? (u_wdiff[11:0] + 12'(Q)) : u_wdiff[11:0]; + wire [11:0] u_wq = u_wdiff[12] ? (u_wdiff[11:0] + Q12) : u_wdiff[11:0]; wire u_w_we = (st == ST_DEC_W) && (u_sub == 2'd2) && u_avalid; // ================================================================ @@ -1038,13 +1069,18 @@ module mlkem_top #( .ready_i(1'b1) ); // bit-packer / ct walk bookkeeping - reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v - reg [7:0] cp_idx; // coeff 0..255 within poly - reg [2:0] cp_ph; // micro-phase - reg [24:0] cp_buf; // bit accumulator (LSB-first) - reg [5:0] cp_nbits; // valid bits in cp_buf + reg [2:0] cp_poly; // c1: 0..K-1 (u rows); c2: single v (feed side) + reg [7:0] cp_idx; // coeff 0..255 within poly (feed side, drives bank addr) + reg [2:0] cp_ph; // feed micro-phase toggle (0=addr settle, 1=feed) + reg [63:0] cp_buf; // bit accumulator (LSB-first); widened for streaming + reg [6:0] cp_nbits; // valid bits in cp_buf reg [11:0] cp_wa; // ct_bram byte write address (runs c1 then c2) reg cp_done; // serialization complete (this region) + reg cp_fed_done; // all coeffs for this region have been fed in + reg [10:0] cp_fcnt; // coeffs fed into comp_decomp this region + reg [10:0] cp_ccnt; // compressed coeffs collected this region + // total coeffs this region: c1 = K*256 (u rows), c2 = 256 (single v) + wire [10:0] cp_ntot = (st == ST_ENC_C2) ? 11'd256 : ({8'd0, k_r} << 8); // coeff source: c1 (ST_ENC_C1) reads u[cp_poly] in bank_se rel (K+cp_poly); // c2 (ST_ENC_C2) reads v in bank_t rel slot UPSUM (single poly, cp_poly=0). wire [13:0] cp_se_full = ({2'b0,k_r}+{2'b0,cp_poly})*256 + cp_idx; // bank_se u[cp_poly] @@ -1128,7 +1164,7 @@ module mlkem_top #( wire [11:0] m_acc_src = m_jq ? bse_rd_data : bt_rd_data; // (a + b) mod Q (both < Q, sum < 2Q): one conditional subtract wire [12:0] m_sum = {1'b0, m_acc_src} + {1'b0, pm_coeff}; - wire [11:0] m_accq = (m_sum >= 13'(Q)) ? (m_sum - 13'(Q)) : m_sum[11:0]; + wire [11:0] m_accq = (m_sum >= Q13) ? (m_sum - Q13) : m_sum[11:0]; always @(*) begin st_next = st; @@ -1257,10 +1293,13 @@ module mlkem_top #( cp_poly <= 3'd0; cp_idx <= 8'd0; cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; cp_wa <= 12'd0; cp_done <= 1'b0; + cp_fed_done<= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; ct_we <= 1'b0; ct_wa <= 11'd0; ct_wd <= 8'd0; @@ -1692,69 +1731,86 @@ module mlkem_top #( // Arm E5 (ST_ENC_C1) when U finishes: c1 = byteEncode_du(Compress_du(u)). // Walk K polys * 256 coeffs; reset bit-packer + ct write pointer. if (st == ST_ENC_U && st_next == ST_ENC_C1) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_wa <= 12'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_wa <= 12'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; end // ---- ST_ENC_C1/C2: Compress_d -> byteEncode_d -> ct region ---- // C1 (E5): Compress_du(u[0..K-1]) from bank_se -> ct[0..c1_bytes). // C2 (E7): Compress_dv(v) from bank_t[UPSUM] -> ct[c1_bytes..ct_bytes). - // Per coeff, 5-phase micro-sequence (read-ahead 1 cyc bram plus - // variable-latency comp_decomp), then a drain sub-phase emitting whole bytes: - // ph0: present coeff addr (cp_se_full / cp_bt_full by state). - // ph1: coeff arrives (cp_coeff_src) -> latch into cd_coeff, pulse cd_valid. - // ph2: drop cd_valid (1-cyc pulse); comp_decomp captures. - // ph3: cd_vo high -> cd_out (low cp_d bits) valid; append LSB-first. - // ph4: drain: while >=8 bits buffered, emit one ct byte/cycle; then advance. - // Each poly = 256 coeffs = 32*d bytes (whole), so cp_buf empties at - // each poly boundary (no carry across polys / regions). + // + // Streaming (B1): the comp_decomp pipeline is fully pipelined and its + // sole output consumer never backpressures (ready_i=1'b1), so instead + // of feeding one coeff and draining the whole pipe per coeff (~12 cyc), + // we run a decoupled producer + collector: + // PRODUCER (2-cycle cadence): ph0 lets the bank read address settle + // (1-cyc sd_bram latency), ph1 latches the coeff and pulses + // cd_valid, then advances the address. One coeff enters the pipe + // every 2 cycles = <=5.5 bits/cyc (du<=11), strictly under the + // single-port ct drain rate (8 bits/cyc), so cp_buf stays bounded. + // COLLECTOR (every cycle): when cd_vo, append the compressed cp_d + // bits LSB-first; independently emit one ct byte/cycle whenever + // >=8 bits are buffered. Coeffs come out in-order, one per pipe + // output. Each poly is 256*d bits (multiple of 8), so the buffer + // empties to 0 at region end (cp_done when all collected+drained). if ((st == ST_ENC_C1 || st == ST_ENC_C2) && !cp_done) begin - case (cp_ph) - 3'd0: cp_ph <= 3'd1; // addr presented; wait read - 3'd1: begin - cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) - cd_valid <= 1'b1; // feed comp_decomp (1-cyc pulse) - cp_ph <= 3'd2; - end - 3'd2: begin - cd_valid <= 1'b0; // comp_decomp captured this cyc - cp_ph <= 3'd3; - end - 3'd3: begin - if (cd_vo) begin - // Append compressed low cp_d bits LSB-first at bit cp_nbits. - cp_buf <= cp_buf | (({13'd0, cd_out} & ((25'd1 << cp_d) - 25'd1)) << cp_nbits); - cp_nbits <= cp_nbits + {1'b0, cp_d}; - cp_ph <= 3'd4; - end - end - default: begin // 3'd4: drain whole bytes - if (cp_nbits >= 6'd8) begin - ct_we <= 1'b1; - ct_wa <= cp_wa; - ct_wd <= cp_buf[7:0]; - cp_wa <= cp_wa + 12'd1; - cp_buf <= cp_buf >> 8; - cp_nbits <= cp_nbits - 6'd8; + // ---- PRODUCER: feed one coeff every 2 cycles ---- + cd_valid <= 1'b0; + if (!cp_fed_done) begin + if (cp_ph == 3'd0) begin + cp_ph <= 3'd1; // address settling + end else begin + cd_coeff <= cp_coeff_src; // u (bank_se) or v (bank_t) + cd_valid <= 1'b1; // 1-cyc feed pulse + cp_fcnt <= cp_fcnt + 11'd1; + if (cp_fcnt + 11'd1 >= cp_ntot) cp_fed_done <= 1'b1; + if (cp_idx == 8'd255) begin + cp_idx <= 8'd0; + cp_poly <= cp_poly + 3'd1; // next u row (c1); c2 unused end else begin - // coeff fully packed; advance coeff / poly - if (cp_idx == 8'd255) begin - cp_idx <= 8'd0; - if (cp_poly + 3'd1 < cp_poly_max) cp_poly <= cp_poly + 3'd1; - else cp_done <= 1'b1; // region complete - end else begin - cp_idx <= cp_idx + 8'd1; - end - cp_ph <= 3'd0; + cp_idx <= cp_idx + 8'd1; end + cp_ph <= 3'd0; end - endcase + end + + // ---- COLLECTOR: append compressed bits + drain ct bytes ---- + if (cd_vo) begin + // append cp_d bits at cp_nbits, then drain one byte if >=8. + if ((cp_nbits + {1'b0, cp_d}) >= 7'd8) begin + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) & 64'hFF; + cp_wa <= cp_wa + 12'd1; + cp_buf <= (cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits)) >> 8; + cp_nbits <= (cp_nbits + {1'b0, cp_d}) - 7'd8; + end else begin + cp_buf <= cp_buf | (({52'd0, cd_out} & ((64'd1 << cp_d) - 64'd1)) << cp_nbits); + cp_nbits <= cp_nbits + {1'b0, cp_d}; + end + cp_ccnt <= cp_ccnt + 11'd1; // one compressed coeff collected + end else if (cp_nbits >= 7'd8) begin + // no new coeff this cycle: keep draining buffered whole bytes. + ct_we <= 1'b1; + ct_wa <= cp_wa; + ct_wd <= cp_buf[7:0]; + cp_wa <= cp_wa + 12'd1; + cp_buf <= cp_buf >> 8; + cp_nbits <= cp_nbits - 7'd8; + end + + // region complete once all coeffs collected and buffer drained. + if (cp_ccnt >= cp_ntot && cp_nbits < 7'd8) + cp_done <= 1'b1; end // Arm E7 (ST_ENC_C2) when V finishes: c2 = byteEncode_dv(Compress_dv(v)). @@ -1762,13 +1818,16 @@ module mlkem_top #( // from c1_bytes (NOT reset) so c2 lands right after c1. cp_buf/nbits // are 0 here (c1 ended on a poly/byte boundary) but reset for safety. if (st == ST_ENC_V && st_next == ST_ENC_C2) begin - cp_poly <= 3'd0; - cp_idx <= 8'd0; - cp_ph <= 3'd0; - cp_buf <= 25'd0; - cp_nbits <= 6'd0; - cp_done <= 1'b0; - cd_valid <= 1'b0; + cp_poly <= 3'd0; + cp_idx <= 8'd0; + cp_ph <= 3'd0; + cp_buf <= 64'd0; + cp_nbits <= 7'd0; + cp_done <= 1'b0; + cp_fed_done <= 1'b0; + cp_fcnt <= 11'd0; + cp_ccnt <= 11'd0; + cd_valid <= 1'b0; // cp_wa intentionally preserved (= c1_bytes_rt from C1). end