diff --git a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v index 61f8f99..bf7f278 100644 --- a/rtl/ip/mlkem/poly_mul/poly_mul_sync.v +++ b/rtl/ip/mlkem/poly_mul/poly_mul_sync.v @@ -1,4 +1,4 @@ -// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier +// poly_mul_sync.v - Synchronous NTT-domain polynomial multiplier (BRAM storage) // // Computes pointwise (Karatsuba-like base-case) multiplication of two // 256-coefficient NTT-domain polynomials. @@ -6,18 +6,18 @@ // Operation flow: // IDLE -> LOAD (256x A+B pairs) -> RUN -> IDLE // -// RUN issues one base-case multiply every two cycles, matching the two-cycle -// c0/c1 output bandwidth after the basecase pipeline fills. +// Coefficient storage moved from 256x12 LUT register arrays to four sd_bram +// banks (A_even/A_odd/B_even/B_odd, each 128x12): the base-case reads +// mem[2k] and mem[2k+1] together, so an even/odd split gives each bank a single +// read port. This removes the large multi-port LUT mux (~8k LUT) and the long +// BRAM->register-file routing that was the mlkem_clk critical path. // -// The LOAD phase accepts both A and B coefficients simultaneously -// (one pair per cycle) on coeff_a_in/coeff_b_in. +// RUN issues one base-case every two cycles. sd_bram reads are synchronous +// (1-cycle latency); the 2-cycle cadence absorbs it (address presented on the +// gap cycle, data consumed on the issue cycle). A one-cycle priming gap is +// inserted at LOAD->RUN so the first base-case's operands are ready. // -// The COMPUTE phase outputs the 256 result coefficients one per cycle -// via valid/ready handshake on coeff_out/valid_o/ready_i. -// -// Memory: 256×12-bit register arrays for A and B coefficients. -// -// Interface: +// Interface (unchanged): // clk, rst_n - Clock, active-low reset // coeff_a_in[11:0]- Polynomial A coefficient input // coeff_b_in[11:0]- Polynomial B coefficient input @@ -38,18 +38,12 @@ module poly_mul_sync ( input ready_i ); - // State definitions localparam S_IDLE = 3'd0; localparam S_LOAD = 3'd1; localparam S_RUN = 3'd2; reg [2:0] state, next_state; - // Coefficient storage (register arrays) - reg [11:0] mem_A [0:255]; - reg [11:0] mem_B [0:255]; - - // Counters reg [7:0] load_cnt; // 0..256 for loading 256 pairs reg [7:0] issue_cnt; // 0..128, issued base-cases reg [7:0] out_cnt; // 0..255, consumed output coefficients @@ -57,53 +51,61 @@ module poly_mul_sync ( reg out_phase; // 0=c0, 1=c1 reg out_valid_r; - // Registered basecase_mul inputs/results reg [11:0] bc_a0_reg, bc_a1_reg; reg [11:0] bc_b0_reg, bc_b1_reg; reg [11:0] bc_zeta_reg; reg bc_valid_reg; reg [11:0] c0_reg, c1_reg; - // Combinational read signals for the next base-case request. - wire [6:0] issue_k = issue_cnt[6:0]; - wire [7:0] addr_even = {issue_k, 1'b0}; // issue_k * 2 - wire [7:0] addr_odd = {issue_k, 1'b1}; // issue_k * 2 + 1 - wire [11:0] mem_a0 = mem_A[addr_even]; - wire [11:0] mem_a1 = mem_A[addr_odd]; - wire [11:0] mem_b0 = mem_B[addr_even]; - wire [11:0] mem_b1 = mem_B[addr_odd]; + // ---- Coefficient storage: even/odd banked sd_bram ---- + // A_even[k]=mem_A[2k], A_odd[k]=mem_A[2k+1]; same for B. Read index during + // RUN is issue_k; write index during LOAD is load_cnt>>1 routed by load_cnt[0]. + wire [6:0] issue_k = issue_cnt[6:0]; // 0..127 read index - // Zeta ROM - wire [11:0] zeta; - poly_mul_zeta_rom u_zeta ( - .addr (issue_k), - .zeta (zeta) + wire load_fire = ((state == S_IDLE) || (state == S_LOAD)) && valid_i && ready_o; + wire [6:0] wr_local = load_cnt[7:1]; // load_cnt/2 + wire wr_odd = load_cnt[0]; + + wire we_even = load_fire && !wr_odd; + wire we_odd = load_fire && wr_odd; + + wire [11:0] mem_a0, mem_a1, mem_b0, mem_b1; // sd_bram registered read outputs + sd_bram #(.W(12), .D(128), .A(7)) u_a_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_a_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_a1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_a_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_even ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b0), + .wr_en(we_even), .wr_addr(wr_local), .wr_data(coeff_b_in) + ); + sd_bram #(.W(12), .D(128), .A(7)) u_b_odd ( + .clk(clk), .rd_addr(issue_k), .rd_data(mem_b1), + .wr_en(we_odd), .wr_addr(wr_local), .wr_data(coeff_b_in) ); - // Pipelined basecase multiply. Requests are issued every two cycles; inputs - // are registered locally so issue_cnt does not directly drive DSP input muxes. + wire [11:0] zeta; + poly_mul_zeta_rom u_zeta (.addr(issue_k), .zeta(zeta)); + wire [11:0] bc_c0, bc_c1; wire bc_vo; basecase_mul_pipe u_bc ( - .clk (clk), - .rst_n(rst_n), + .clk(clk), .rst_n(rst_n), .valid_i(bc_valid_reg), - .a0 (bc_a0_reg), - .a1 (bc_a1_reg), - .b0 (bc_b0_reg), - .b1 (bc_b1_reg), + .a0(bc_a0_reg), .a1(bc_a1_reg), + .b0(bc_b0_reg), .b1(bc_b1_reg), .zeta(bc_zeta_reg), - .c0 (bc_c0), - .c1 (bc_c1), + .c0(bc_c0), .c1(bc_c1), .valid_o(bc_vo) ); - // Output interface assign ready_o = (state == S_IDLE) || (state == S_LOAD); assign valid_o = out_valid_r; assign coeff_out = out_phase ? c1_reg : c0_reg; - // State transition logic (combinational) always @* begin next_state = state; case (state) @@ -116,8 +118,6 @@ module poly_mul_sync ( endcase end - // Sequential logic - integer i; always @(posedge clk or negedge rst_n) begin if (!rst_n) begin state <= S_IDLE; @@ -127,27 +127,16 @@ module poly_mul_sync ( issue_gap <= 1'b0; out_phase <= 1'b0; out_valid_r <= 1'b0; - bc_a0_reg <= 12'd0; - bc_a1_reg <= 12'd0; - bc_b0_reg <= 12'd0; - bc_b1_reg <= 12'd0; - bc_zeta_reg <= 12'd0; - bc_valid_reg <= 1'b0; - c0_reg <= 12'd0; - c1_reg <= 12'd0; - for (i = 0; i < 256; i = i + 1) begin - mem_A[i] = 12'd0; - mem_B[i] = 12'd0; - end + bc_a0_reg <= 12'd0; bc_a1_reg <= 12'd0; + bc_b0_reg <= 12'd0; bc_b1_reg <= 12'd0; + bc_zeta_reg <= 12'd0; bc_valid_reg <= 1'b0; + c0_reg <= 12'd0; c1_reg <= 12'd0; end else begin state <= next_state; bc_valid_reg <= 1'b0; - // ---- LOAD phase ---- - // First coefficient captured on IDLE → LOAD transition + // ---- LOAD phase (writes routed to even/odd banks combinationally) ---- if (state == S_IDLE && valid_i && ready_o) begin - mem_A[0] <= coeff_a_in; - mem_B[0] <= coeff_b_in; load_cnt <= 8'd1; issue_cnt <= 8'd0; out_cnt <= 8'd0; @@ -156,25 +145,21 @@ module poly_mul_sync ( out_valid_r <= 1'b0; end - // Subsequent coefficients in LOAD state if (state == S_LOAD && valid_i && ready_o) begin - mem_A[load_cnt] <= coeff_a_in; - mem_B[load_cnt] <= coeff_b_in; load_cnt <= load_cnt + 8'd1; if (load_cnt >= 8'd255) begin issue_cnt <= 8'd0; out_cnt <= 8'd0; - issue_gap <= 1'b0; + issue_gap <= 1'b1; // priming gap: present issue_k=0 addr first out_phase <= 1'b0; out_valid_r <= 1'b0; end end // ---- COMPUTE phase ---- - // RUN: cut issue_cnt -> memory mux -> basecase DSP with local input - // regs, then launch requests every other cycle. The two-cycle issue - // interval matches the c0/c1 output bandwidth, so no deep result FIFO - // is needed for this always-ready system. + // Issue one base-case every two cycles. On the gap cycle the sd_bram + // address (issue_k) settles; on the issue cycle its registered output + // (mem_a0/a1/b0/b1) is captured into the base-case input registers. if (state == S_RUN) begin if (issue_cnt < 8'd128) begin if (!issue_gap) begin @@ -212,7 +197,6 @@ module poly_mul_sync ( end end - // ---- final output consumed: reset counters before accepting a new op ---- if (state == S_RUN && valid_o && ready_i && out_cnt == 8'd255) begin load_cnt <= 8'd0; issue_cnt <= 8'd0;