286 lines
9.7 KiB
Verilog
286 lines
9.7 KiB
Verilog
// ntt_core.v - NTT core with individual coefficient registers
|
|
//
|
|
// Uses 256 individual 12-bit registers and a deeply pipelined butterfly path.
|
|
// The arithmetic hot path is split into:
|
|
// address/operand/zeta register -> pipelined Barrett butterfly -> writeback
|
|
// Each NTT layer is issued continuously into the butterfly pipeline, then the
|
|
// core drains pending writebacks before starting the next dependent layer.
|
|
// In inverse mode, final x3303 output scaling also uses a pipelined Barrett
|
|
// multiplier so the output path does not reintroduce a combinational reducer.
|
|
|
|
module ntt_core (
|
|
input clk, rst_n,
|
|
input [11:0] coeff_in,
|
|
input valid_i,
|
|
output ready_o,
|
|
input mode,
|
|
output [11:0] coeff_out,
|
|
output valid_o,
|
|
input ready_i,
|
|
output done_o
|
|
);
|
|
localparam N = 256, LAYERS = 7, DW = 12;
|
|
|
|
reg [DW-1:0] cr [0:N-1];
|
|
integer ci;
|
|
|
|
localparam S_IDLE = 4'd0;
|
|
localparam S_LOAD = 4'd1;
|
|
localparam S_CMP_RUN = 4'd2;
|
|
localparam S_CMP_DRAIN = 4'd3;
|
|
localparam S_OUT_PREP = 4'd7;
|
|
localparam S_OUTPUT = 4'd8;
|
|
localparam S_OUT_SCALE = 4'd9;
|
|
localparam S_DONE = 4'd10;
|
|
|
|
reg [3:0] state, next_state;
|
|
|
|
reg [7:0] load_cnt;
|
|
reg [7:0] out_cnt;
|
|
reg [8:0] scale_issue_cnt;
|
|
reg [8:0] scale_emit_cnt;
|
|
reg [7:0] j, start, layer_len;
|
|
reg [6:0] zeta_idx;
|
|
reg [2:0] layer;
|
|
reg bf_done;
|
|
reg mode_r;
|
|
reg layer_issue_done;
|
|
reg [4:0] bf_inflight;
|
|
|
|
reg [7:0] wa_d0, wa_d1, wa_d2, wa_d3, wa_d4, wa_d5, wa_d6, wa_d7;
|
|
reg [7:0] wb_d0, wb_d1, wb_d2, wb_d3, wb_d4, wb_d5, wb_d6, wb_d7;
|
|
reg wb_valid_r;
|
|
reg [DW-1:0] wb_a_data_r, wb_b_data_r;
|
|
reg [7:0] wb_wa_r, wb_wb_r;
|
|
|
|
reg [DW-1:0] coeff_out_r;
|
|
reg valid_o_r;
|
|
reg scale_valid_i;
|
|
reg [DW-1:0] scale_a_i;
|
|
|
|
wire [DW-1:0] zeta;
|
|
zeta_rom u_z (.addr(zeta_idx), .zeta(zeta));
|
|
|
|
wire [8:0] group_end = {1'b0, start} + {1'b0, layer_len};
|
|
wire [8:0] next_group_start = {1'b0, start} + {1'b0, layer_len} + {1'b0, layer_len};
|
|
wire issue_group_last = ({1'b0, j} + 9'd1 >= group_end);
|
|
wire issue_layer_last = issue_group_last && (next_group_start >= 9'd256);
|
|
wire [7:0] next_layer_len = mode_r ? (layer_len << 1) : (layer_len >> 1);
|
|
wire issue_fire = (state == S_CMP_RUN) && !layer_issue_done;
|
|
wire [DW-1:0] issue_a = cr[j];
|
|
wire [DW-1:0] issue_b = cr[j + layer_len];
|
|
|
|
wire [DW-1:0] bf_a_out, bf_b_out;
|
|
wire bf_valid;
|
|
butterfly_unit_pipe u_bf (
|
|
.clk(clk),
|
|
.rst_n(rst_n),
|
|
.valid_i(issue_fire),
|
|
.a(issue_a),
|
|
.b(issue_b),
|
|
.zeta(zeta),
|
|
.mode(mode_r),
|
|
.a_out(bf_a_out),
|
|
.b_out(bf_b_out),
|
|
.valid_o(bf_valid)
|
|
);
|
|
|
|
wire [DW-1:0] scale_product;
|
|
wire scale_valid_o;
|
|
barrett_mul_pipe u_scl (
|
|
.clk(clk),
|
|
.rst_n(rst_n),
|
|
.valid_i(scale_valid_i),
|
|
.a(scale_a_i),
|
|
.b(12'd3303),
|
|
.product(scale_product),
|
|
.valid_o(scale_valid_o)
|
|
);
|
|
|
|
assign ready_o = (state == S_IDLE) || (state == S_LOAD);
|
|
assign coeff_out = coeff_out_r;
|
|
assign valid_o = valid_o_r;
|
|
assign done_o = (state == S_DONE);
|
|
|
|
always @* begin
|
|
next_state = state;
|
|
case (state)
|
|
S_IDLE: if (valid_i) next_state = S_LOAD;
|
|
S_LOAD: if (load_cnt >= 8'd255 && valid_i) next_state = S_CMP_RUN;
|
|
S_CMP_RUN: if (layer_issue_done) next_state = S_CMP_DRAIN;
|
|
S_CMP_DRAIN: if (bf_inflight == 5'd0)
|
|
next_state = (layer + 3'd1 >= LAYERS) ? S_OUT_PREP : S_CMP_RUN;
|
|
S_OUT_PREP: next_state = mode_r ? S_OUT_SCALE : S_OUTPUT;
|
|
S_OUTPUT: if (valid_o_r && ready_i && out_cnt >= 8'd255) next_state = S_DONE;
|
|
S_OUT_SCALE: if (scale_emit_cnt >= 9'd256) next_state = S_DONE;
|
|
S_DONE: next_state = S_IDLE;
|
|
default: next_state = S_IDLE;
|
|
endcase
|
|
end
|
|
|
|
always @(posedge clk or negedge rst_n) begin
|
|
if (!rst_n) begin
|
|
state <= S_IDLE;
|
|
load_cnt <= 8'd0;
|
|
out_cnt <= 8'd0;
|
|
scale_issue_cnt <= 9'd0;
|
|
scale_emit_cnt <= 9'd0;
|
|
j <= 8'd0;
|
|
start <= 8'd0;
|
|
layer_len <= 8'd0;
|
|
zeta_idx <= 7'd0;
|
|
layer <= 3'd0;
|
|
bf_done <= 1'b0;
|
|
mode_r <= 1'b0;
|
|
layer_issue_done <= 1'b0;
|
|
bf_inflight <= 5'd0;
|
|
wa_d0 <= 8'd0; wa_d1 <= 8'd0; wa_d2 <= 8'd0; wa_d3 <= 8'd0;
|
|
wa_d4 <= 8'd0; wa_d5 <= 8'd0; wa_d6 <= 8'd0; wa_d7 <= 8'd0;
|
|
wb_d0 <= 8'd0; wb_d1 <= 8'd0; wb_d2 <= 8'd0; wb_d3 <= 8'd0;
|
|
wb_d4 <= 8'd0; wb_d5 <= 8'd0; wb_d6 <= 8'd0; wb_d7 <= 8'd0;
|
|
wb_valid_r <= 1'b0;
|
|
wb_a_data_r <= 12'd0;
|
|
wb_b_data_r <= 12'd0;
|
|
wb_wa_r <= 8'd0;
|
|
wb_wb_r <= 8'd0;
|
|
coeff_out_r <= 12'd0;
|
|
valid_o_r <= 1'b0;
|
|
scale_valid_i <= 1'b0;
|
|
scale_a_i <= 12'd0;
|
|
for (ci = 0; ci < N; ci = ci + 1) cr[ci] <= 12'd0;
|
|
end else begin
|
|
state <= next_state;
|
|
scale_valid_i <= 1'b0;
|
|
|
|
wa_d0 <= j; wa_d1 <= wa_d0; wa_d2 <= wa_d1; wa_d3 <= wa_d2;
|
|
wa_d4 <= wa_d3; wa_d5 <= wa_d4; wa_d6 <= wa_d5; wa_d7 <= wa_d6;
|
|
wb_d0 <= j + layer_len; wb_d1 <= wb_d0; wb_d2 <= wb_d1; wb_d3 <= wb_d2;
|
|
wb_d4 <= wb_d3; wb_d5 <= wb_d4; wb_d6 <= wb_d5; wb_d7 <= wb_d6;
|
|
|
|
if (issue_fire && !wb_valid_r)
|
|
bf_inflight <= bf_inflight + 5'd1;
|
|
else if (!issue_fire && wb_valid_r)
|
|
bf_inflight <= bf_inflight - 5'd1;
|
|
|
|
if (state != S_OUTPUT && state != S_OUT_SCALE)
|
|
valid_o_r <= 1'b0;
|
|
|
|
if (state == S_IDLE && valid_i) begin
|
|
cr[0] <= coeff_in;
|
|
load_cnt <= 8'd1;
|
|
out_cnt <= 8'd0;
|
|
scale_issue_cnt <= 9'd0;
|
|
scale_emit_cnt <= 9'd0;
|
|
j <= 8'd0;
|
|
start <= 8'd0;
|
|
layer <= 3'd0;
|
|
bf_done <= 1'b0;
|
|
layer_issue_done <= 1'b0;
|
|
bf_inflight <= 5'd0;
|
|
mode_r <= mode;
|
|
if (!mode) begin
|
|
layer_len <= 8'd128;
|
|
zeta_idx <= 7'd1;
|
|
end else begin
|
|
layer_len <= 8'd2;
|
|
zeta_idx <= 7'd127;
|
|
end
|
|
end
|
|
|
|
if (state == S_LOAD && valid_i) begin
|
|
cr[load_cnt] <= coeff_in;
|
|
load_cnt <= load_cnt + 8'd1;
|
|
end
|
|
|
|
if (state == S_CMP_RUN && !layer_issue_done) begin
|
|
if (issue_group_last) begin
|
|
if (!mode_r) zeta_idx <= zeta_idx + 7'd1;
|
|
else zeta_idx <= zeta_idx - 7'd1;
|
|
|
|
if (issue_layer_last) begin
|
|
layer_issue_done <= 1'b1;
|
|
end else begin
|
|
start <= next_group_start[7:0];
|
|
j <= next_group_start[7:0];
|
|
end
|
|
end else begin
|
|
j <= j + 8'd1;
|
|
end
|
|
end
|
|
|
|
if (wb_valid_r) begin
|
|
cr[wb_wa_r] <= wb_a_data_r;
|
|
cr[wb_wb_r] <= wb_b_data_r;
|
|
end
|
|
|
|
wb_valid_r <= bf_valid;
|
|
if (bf_valid) begin
|
|
wb_a_data_r <= bf_a_out;
|
|
wb_b_data_r <= bf_b_out;
|
|
wb_wa_r <= wa_d7;
|
|
wb_wb_r <= wb_d7;
|
|
end
|
|
|
|
if (state == S_CMP_DRAIN && bf_inflight == 5'd0) begin
|
|
if (layer + 3'd1 >= LAYERS) begin
|
|
bf_done <= 1'b1;
|
|
end else begin
|
|
layer <= layer + 3'd1;
|
|
layer_len <= next_layer_len;
|
|
start <= 8'd0;
|
|
j <= 8'd0;
|
|
layer_issue_done <= 1'b0;
|
|
end
|
|
end
|
|
|
|
if (state == S_OUT_PREP) begin
|
|
out_cnt <= 8'd0;
|
|
scale_issue_cnt <= 9'd0;
|
|
scale_emit_cnt <= 9'd0;
|
|
if (!mode_r) begin
|
|
coeff_out_r <= cr[0];
|
|
valid_o_r <= 1'b1;
|
|
end
|
|
end
|
|
|
|
if (state == S_OUTPUT && valid_o_r && ready_i) begin
|
|
if (out_cnt < 8'd255) begin
|
|
out_cnt <= out_cnt + 8'd1;
|
|
coeff_out_r <= cr[out_cnt + 8'd1];
|
|
valid_o_r <= 1'b1;
|
|
end else begin
|
|
out_cnt <= 8'd0;
|
|
valid_o_r <= 1'b0;
|
|
end
|
|
end
|
|
|
|
if (state == S_OUT_SCALE) begin
|
|
if (scale_issue_cnt < 9'd256) begin
|
|
scale_valid_i <= 1'b1;
|
|
scale_a_i <= cr[scale_issue_cnt[7:0]];
|
|
scale_issue_cnt <= scale_issue_cnt + 9'd1;
|
|
end
|
|
|
|
valid_o_r <= 1'b0;
|
|
if (scale_valid_o) begin
|
|
coeff_out_r <= scale_product;
|
|
valid_o_r <= 1'b1;
|
|
scale_emit_cnt <= scale_emit_cnt + 9'd1;
|
|
end
|
|
end
|
|
|
|
if (state == S_DONE) begin
|
|
load_cnt <= 8'd0;
|
|
out_cnt <= 8'd0;
|
|
scale_issue_cnt <= 9'd0;
|
|
scale_emit_cnt <= 9'd0;
|
|
layer_issue_done <= 1'b0;
|
|
bf_inflight <= 5'd0;
|
|
wb_valid_r <= 1'b0;
|
|
valid_o_r <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
endmodule
|