389 lines
17 KiB
Systemverilog
389 lines
17 KiB
Systemverilog
// ============================================================================
|
|
// cdma_matmul_bridge.sv
|
|
// Matmul-specific sequencer between the generic CDMA endpoint requester and
|
|
// the matmul local endpoint/control interface.
|
|
//
|
|
// AXI-Lite register map, relative to this bridge window:
|
|
// 0x00 CTRL bit0 start, bit1 skip_result_writeback
|
|
// 0x04 STATUS bit0 busy, bit1 done (W1C)
|
|
// 0x08 SRC_BASE physical ExtRAM input base
|
|
// 0x0C DST_BASE physical ExtRAM result base
|
|
// 0x10 GROUPS
|
|
// 0x14 MAT_BASE reserved for software compatibility
|
|
// 0x18 SRC_STRIDE bytes, default 128
|
|
// 0x1C DST_STRIDE bytes, default 192
|
|
// ============================================================================
|
|
`timescale 1ns / 1ps
|
|
|
|
module cdma_matmul_bridge #(
|
|
parameter int ADDR_WIDTH = 32,
|
|
parameter int AXIL_ADDR_WIDTH = 32,
|
|
parameter int AXIL_DATA_WIDTH = 32
|
|
) (
|
|
input logic clk,
|
|
input logic rst_n,
|
|
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_awaddr,
|
|
input logic s_axil_awvalid,
|
|
output logic s_axil_awready,
|
|
input logic [AXIL_DATA_WIDTH-1:0] s_axil_wdata,
|
|
input logic [AXIL_DATA_WIDTH/8-1:0] s_axil_wstrb,
|
|
input logic s_axil_wvalid,
|
|
output logic s_axil_wready,
|
|
output logic [1:0] s_axil_bresp,
|
|
output logic s_axil_bvalid,
|
|
input logic s_axil_bready,
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_araddr,
|
|
input logic s_axil_arvalid,
|
|
output logic s_axil_arready,
|
|
output logic [AXIL_DATA_WIDTH-1:0] s_axil_rdata,
|
|
output logic [1:0] s_axil_rresp,
|
|
output logic s_axil_rvalid,
|
|
input logic s_axil_rready,
|
|
|
|
output logic cdma_rd_req_valid,
|
|
input logic cdma_rd_req_ready,
|
|
output logic [ADDR_WIDTH-1:0] cdma_rd_req_src,
|
|
output logic [31:0] cdma_rd_req_bytes,
|
|
output logic [3:0] cdma_rd_req_ep_id,
|
|
output logic [3:0] cdma_rd_req_ep_bank,
|
|
output logic [7:0] cdma_rd_req_ep_word_off,
|
|
input logic cdma_rd_req_done,
|
|
|
|
output logic cdma_wr_req_valid,
|
|
input logic cdma_wr_req_ready,
|
|
output logic [ADDR_WIDTH-1:0] cdma_wr_req_dst,
|
|
output logic [31:0] cdma_wr_req_bytes,
|
|
output logic [3:0] cdma_wr_req_ep_id,
|
|
output logic [3:0] cdma_wr_req_ep_bank,
|
|
output logic [7:0] cdma_wr_req_ep_word_off,
|
|
input logic cdma_wr_req_done,
|
|
|
|
output logic mat_start_valid,
|
|
output logic mat_start_bank,
|
|
output logic mat_done_clear,
|
|
output logic mat_crc_clear,
|
|
input logic mat_done,
|
|
|
|
output logic seq_done_irq
|
|
);
|
|
|
|
localparam logic [7:0] REG_CTRL = 8'h00;
|
|
localparam logic [7:0] REG_STATUS = 8'h04;
|
|
localparam logic [7:0] REG_SRC_BASE = 8'h08;
|
|
localparam logic [7:0] REG_DST_BASE = 8'h0C;
|
|
localparam logic [7:0] REG_GROUPS = 8'h10;
|
|
localparam logic [7:0] REG_MAT_BASE = 8'h14;
|
|
localparam logic [7:0] REG_SRC_STRIDE = 8'h18;
|
|
localparam logic [7:0] REG_DST_STRIDE = 8'h1C;
|
|
|
|
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
|
|
localparam logic [31:0] MAT_C_BYTES = 32'd192;
|
|
|
|
logic [31:0] seq_src_base;
|
|
logic [31:0] seq_dst_base;
|
|
logic [31:0] seq_groups;
|
|
logic [31:0] seq_mat_base;
|
|
logic [31:0] seq_src_stride;
|
|
logic [31:0] seq_dst_stride;
|
|
logic seq_busy;
|
|
logic seq_done_flag;
|
|
logic seq_skip_writeback;
|
|
|
|
logic [31:0] load_next_idx;
|
|
logic [31:0] groups_retired;
|
|
logic [1:0] bank_load_active;
|
|
logic [1:0] bank_ab_valid;
|
|
logic [1:0] bank_compute_active;
|
|
logic [1:0] bank_store_pending;
|
|
logic [1:0] bank_store_active;
|
|
logic [31:0] bank_group_idx [2];
|
|
|
|
logic rd_busy;
|
|
logic wr_busy;
|
|
logic rd_active_bank;
|
|
logic wr_active_bank;
|
|
logic [1:0] compute_q;
|
|
logic [1:0] compute_q_count;
|
|
logic mat_done_seen;
|
|
|
|
wire [7:0] wr_off = s_axil_awaddr[7:0];
|
|
wire [7:0] rd_off = s_axil_araddr[7:0];
|
|
wire do_write;
|
|
wire start_req = do_write && (wr_off == REG_CTRL) && s_axil_wstrb[0] && s_axil_wdata[0];
|
|
wire done_clear_req = do_write && (wr_off == REG_STATUS) && s_axil_wstrb[0] && s_axil_wdata[1];
|
|
wire load_target_bank = load_next_idx[0];
|
|
wire load_target_free = !bank_load_active[load_target_bank] &&
|
|
!bank_ab_valid[load_target_bank] &&
|
|
!bank_compute_active[load_target_bank] &&
|
|
!bank_store_pending[load_target_bank] &&
|
|
!bank_store_active[load_target_bank];
|
|
wire bank0_start_ready = bank_ab_valid[0] && !bank_compute_active[0] &&
|
|
!bank_store_pending[0] && !bank_store_active[0];
|
|
wire bank1_start_ready = bank_ab_valid[1] && !bank_compute_active[1] &&
|
|
!bank_store_pending[1] && !bank_store_active[1];
|
|
wire start_sel_bank = bank1_start_ready &&
|
|
(!bank0_start_ready || (bank_group_idx[1] < bank_group_idx[0]));
|
|
wire start_ready_any = bank0_start_ready || bank1_start_ready;
|
|
wire store_sel_bank = bank_store_pending[1] && !bank_store_active[1] &&
|
|
!(bank_store_pending[0] && !bank_store_active[0]);
|
|
wire store_ready_any = (bank_store_pending[0] && !bank_store_active[0]) ||
|
|
(bank_store_pending[1] && !bank_store_active[1]);
|
|
wire mat_done_event = mat_done && !mat_done_seen;
|
|
wire retire_from_store = seq_busy && cdma_wr_req_done;
|
|
wire retire_from_compute = seq_busy && seq_skip_writeback && mat_done_event && (compute_q_count != 2'd0);
|
|
wire [31:0] retire_inc = {31'd0, retire_from_store} + {31'd0, retire_from_compute};
|
|
|
|
function automatic logic [31:0] apply_wstrb(
|
|
input logic [31:0] old_val,
|
|
input logic [31:0] new_val,
|
|
input logic [3:0] wstrb
|
|
);
|
|
begin
|
|
apply_wstrb[7:0] = wstrb[0] ? new_val[7:0] : old_val[7:0];
|
|
apply_wstrb[15:8] = wstrb[1] ? new_val[15:8] : old_val[15:8];
|
|
apply_wstrb[23:16] = wstrb[2] ? new_val[23:16] : old_val[23:16];
|
|
apply_wstrb[31:24] = wstrb[3] ? new_val[31:24] : old_val[31:24];
|
|
end
|
|
endfunction
|
|
|
|
assign s_axil_bresp = 2'b00;
|
|
assign s_axil_rresp = 2'b00;
|
|
assign seq_done_irq = seq_done_flag;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_awvalid && s_axil_wvalid && !s_axil_awready && (!s_axil_bvalid || s_axil_bready)) begin
|
|
s_axil_awready <= 1'b1;
|
|
s_axil_wready <= 1'b1;
|
|
end else begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
assign do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end else if (do_write) begin
|
|
s_axil_bvalid <= 1'b1;
|
|
end else if (s_axil_bready && s_axil_bvalid) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_arready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_arvalid && !s_axil_arready && (!s_axil_rvalid || s_axil_rready)) begin
|
|
s_axil_arready <= 1'b1;
|
|
end else begin
|
|
s_axil_arready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
wire do_read = s_axil_arready && s_axil_arvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
s_axil_rdata <= '0;
|
|
end else if (do_read) begin
|
|
s_axil_rvalid <= 1'b1;
|
|
case (rd_off)
|
|
REG_CTRL: s_axil_rdata <= {30'd0, seq_skip_writeback, 1'b0};
|
|
REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
|
|
REG_SRC_BASE: s_axil_rdata <= seq_src_base;
|
|
REG_DST_BASE: s_axil_rdata <= seq_dst_base;
|
|
REG_GROUPS: s_axil_rdata <= seq_groups;
|
|
REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
|
|
REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
|
|
REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
|
|
default: s_axil_rdata <= 32'd0;
|
|
endcase
|
|
end else if (s_axil_rready && s_axil_rvalid) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
seq_src_base <= 32'h0;
|
|
seq_dst_base <= 32'h0;
|
|
seq_groups <= 32'h0;
|
|
seq_mat_base <= 32'h1f50_0000;
|
|
seq_src_stride <= 32'd128;
|
|
seq_dst_stride <= 32'd192;
|
|
end else if (do_write) begin
|
|
case (wr_off)
|
|
REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
|
|
REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
|
|
REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
|
|
default: begin
|
|
end
|
|
endcase
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b0;
|
|
seq_skip_writeback <= 1'b0;
|
|
load_next_idx <= 32'd0;
|
|
groups_retired <= 32'd0;
|
|
bank_load_active <= 2'b00;
|
|
bank_ab_valid <= 2'b00;
|
|
bank_compute_active <= 2'b00;
|
|
bank_store_pending <= 2'b00;
|
|
bank_store_active <= 2'b00;
|
|
bank_group_idx[0] <= 32'd0;
|
|
bank_group_idx[1] <= 32'd0;
|
|
rd_busy <= 1'b0;
|
|
wr_busy <= 1'b0;
|
|
rd_active_bank <= 1'b0;
|
|
wr_active_bank <= 1'b0;
|
|
compute_q <= 2'b00;
|
|
compute_q_count <= 2'd0;
|
|
mat_done_seen <= 1'b0;
|
|
cdma_rd_req_valid <= 1'b0;
|
|
cdma_rd_req_src <= '0;
|
|
cdma_rd_req_bytes <= '0;
|
|
cdma_rd_req_ep_id <= 4'd0;
|
|
cdma_rd_req_ep_bank <= 4'd0;
|
|
cdma_rd_req_ep_word_off <= 8'd0;
|
|
cdma_wr_req_valid <= 1'b0;
|
|
cdma_wr_req_dst <= '0;
|
|
cdma_wr_req_bytes <= '0;
|
|
cdma_wr_req_ep_id <= 4'd0;
|
|
cdma_wr_req_ep_bank <= 4'd0;
|
|
cdma_wr_req_ep_word_off <= 8'd0;
|
|
mat_start_valid <= 1'b0;
|
|
mat_start_bank <= 1'b0;
|
|
mat_done_clear <= 1'b0;
|
|
mat_crc_clear <= 1'b0;
|
|
end else begin
|
|
mat_start_valid <= 1'b0;
|
|
mat_done_clear <= 1'b0;
|
|
mat_crc_clear <= 1'b0;
|
|
|
|
mat_done_seen <= mat_done;
|
|
|
|
if (done_clear_req)
|
|
seq_done_flag <= 1'b0;
|
|
|
|
if (cdma_rd_req_valid && cdma_rd_req_ready)
|
|
cdma_rd_req_valid <= 1'b0;
|
|
|
|
if (cdma_wr_req_valid && cdma_wr_req_ready)
|
|
cdma_wr_req_valid <= 1'b0;
|
|
|
|
if (start_req && !seq_busy && (seq_groups != 32'd0)) begin
|
|
seq_busy <= 1'b1;
|
|
seq_done_flag <= 1'b0;
|
|
seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1];
|
|
load_next_idx <= 32'd0;
|
|
groups_retired <= 32'd0;
|
|
bank_load_active <= 2'b00;
|
|
bank_ab_valid <= 2'b00;
|
|
bank_compute_active <= 2'b00;
|
|
bank_store_pending <= 2'b00;
|
|
bank_store_active <= 2'b00;
|
|
bank_group_idx[0] <= 32'd0;
|
|
bank_group_idx[1] <= 32'd0;
|
|
rd_busy <= 1'b0;
|
|
wr_busy <= 1'b0;
|
|
compute_q <= 2'b00;
|
|
compute_q_count <= 2'd0;
|
|
mat_done_seen <= 1'b1;
|
|
mat_crc_clear <= 1'b1;
|
|
mat_done_clear <= 1'b1;
|
|
cdma_rd_req_valid <= 1'b0;
|
|
cdma_wr_req_valid <= 1'b0;
|
|
end else if (seq_busy) begin
|
|
if (retire_inc != 32'd0) begin
|
|
groups_retired <= groups_retired + retire_inc;
|
|
if ((groups_retired + retire_inc) >= seq_groups) begin
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b1;
|
|
end
|
|
end
|
|
|
|
if (cdma_rd_req_done) begin
|
|
rd_busy <= 1'b0;
|
|
bank_load_active[rd_active_bank] <= 1'b0;
|
|
bank_ab_valid[rd_active_bank] <= 1'b1;
|
|
end
|
|
|
|
if (cdma_wr_req_done) begin
|
|
wr_busy <= 1'b0;
|
|
bank_store_active[wr_active_bank] <= 1'b0;
|
|
end
|
|
|
|
if (mat_done_event && (compute_q_count != 2'd0)) begin
|
|
mat_done_clear <= 1'b1;
|
|
mat_done_seen <= 1'b1;
|
|
bank_compute_active[compute_q[0]] <= 1'b0;
|
|
if (!seq_skip_writeback) begin
|
|
bank_store_pending[compute_q[0]] <= 1'b1;
|
|
end
|
|
if (compute_q_count == 2'd2)
|
|
compute_q[0] <= compute_q[1];
|
|
compute_q_count <= compute_q_count - 2'd1;
|
|
end
|
|
|
|
if (!rd_busy && !cdma_rd_req_valid && (load_next_idx < seq_groups) && load_target_free) begin
|
|
cdma_rd_req_src <= seq_src_base + (load_next_idx * seq_src_stride);
|
|
cdma_rd_req_bytes <= MAT_AB_BYTES;
|
|
cdma_rd_req_ep_id <= 4'd0;
|
|
cdma_rd_req_ep_bank <= {3'b000, load_target_bank};
|
|
cdma_rd_req_ep_word_off <= 8'd0;
|
|
cdma_rd_req_valid <= 1'b1;
|
|
rd_busy <= 1'b1;
|
|
rd_active_bank <= load_target_bank;
|
|
bank_load_active[load_target_bank] <= 1'b1;
|
|
bank_group_idx[load_target_bank] <= load_next_idx;
|
|
load_next_idx <= load_next_idx + 32'd1;
|
|
end
|
|
|
|
if (!seq_skip_writeback && !wr_busy && !cdma_wr_req_valid && store_ready_any) begin
|
|
cdma_wr_req_dst <= seq_dst_base + (bank_group_idx[store_sel_bank] * seq_dst_stride);
|
|
cdma_wr_req_bytes <= MAT_C_BYTES;
|
|
cdma_wr_req_ep_id <= 4'd0;
|
|
cdma_wr_req_ep_bank <= {3'b000, store_sel_bank};
|
|
cdma_wr_req_ep_word_off <= 8'd0;
|
|
cdma_wr_req_valid <= 1'b1;
|
|
wr_busy <= 1'b1;
|
|
wr_active_bank <= store_sel_bank;
|
|
bank_store_pending[store_sel_bank] <= 1'b0;
|
|
bank_store_active[store_sel_bank] <= 1'b1;
|
|
end
|
|
|
|
if ((compute_q_count < 2'd2) && start_ready_any && !(mat_done_event && (compute_q_count != 2'd0))) begin
|
|
mat_start_valid <= 1'b1;
|
|
mat_start_bank <= start_sel_bank;
|
|
bank_ab_valid[start_sel_bank] <= 1'b0;
|
|
bank_compute_active[start_sel_bank] <= 1'b1;
|
|
if (compute_q_count == 2'd0)
|
|
compute_q[0] <= start_sel_bank;
|
|
else
|
|
compute_q[1] <= start_sel_bank;
|
|
compute_q_count <= compute_q_count + 2'd1;
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
endmodule
|