400 lines
16 KiB
Systemverilog
400 lines
16 KiB
Systemverilog
// ============================================================================
|
|
// cdma_matmul_bridge.sv
|
|
// Matmul-specific sequencer between the generic CDMA endpoint requester and
|
|
// the matmul local endpoint/control interface.
|
|
//
|
|
// AXI-Lite register map, relative to this bridge window:
|
|
// 0x00 CTRL bit0 start, bit1 skip_result_writeback
|
|
// 0x04 STATUS bit0 busy, bit1 done (W1C)
|
|
// 0x08 SRC_BASE physical ExtRAM input base
|
|
// 0x0C DST_BASE physical ExtRAM result base
|
|
// 0x10 GROUPS
|
|
// 0x14 MAT_BASE reserved for software compatibility
|
|
// 0x18 SRC_STRIDE bytes, default 128
|
|
// 0x1C DST_STRIDE bytes, default 192
|
|
// ============================================================================
|
|
`timescale 1ns / 1ps
|
|
|
|
module cdma_matmul_bridge #(
|
|
parameter int ADDR_WIDTH = 32,
|
|
parameter int AXIL_ADDR_WIDTH = 32,
|
|
parameter int AXIL_DATA_WIDTH = 32
|
|
) (
|
|
input logic clk,
|
|
input logic rst_n,
|
|
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_awaddr,
|
|
input logic s_axil_awvalid,
|
|
output logic s_axil_awready,
|
|
input logic [AXIL_DATA_WIDTH-1:0] s_axil_wdata,
|
|
input logic [AXIL_DATA_WIDTH/8-1:0] s_axil_wstrb,
|
|
input logic s_axil_wvalid,
|
|
output logic s_axil_wready,
|
|
output logic [1:0] s_axil_bresp,
|
|
output logic s_axil_bvalid,
|
|
input logic s_axil_bready,
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_araddr,
|
|
input logic s_axil_arvalid,
|
|
output logic s_axil_arready,
|
|
output logic [AXIL_DATA_WIDTH-1:0] s_axil_rdata,
|
|
output logic [1:0] s_axil_rresp,
|
|
output logic s_axil_rvalid,
|
|
input logic s_axil_rready,
|
|
|
|
output logic cdma_req_valid,
|
|
input logic cdma_req_ready,
|
|
output logic [1:0] cdma_req_mode,
|
|
output logic [ADDR_WIDTH-1:0] cdma_req_src,
|
|
output logic [ADDR_WIDTH-1:0] cdma_req_dst,
|
|
output logic [31:0] cdma_req_bytes,
|
|
output logic [3:0] cdma_req_ep_id,
|
|
output logic [3:0] cdma_req_ep_bank,
|
|
output logic [7:0] cdma_req_ep_word_off,
|
|
input logic cdma_req_done,
|
|
|
|
output logic mat_start_valid,
|
|
output logic mat_start_bank,
|
|
output logic mat_done_clear,
|
|
output logic mat_crc_clear,
|
|
input logic mat_done,
|
|
|
|
output logic seq_done_irq
|
|
);
|
|
|
|
localparam logic [1:0] MODE_MEM2EP = 2'd1;
|
|
localparam logic [1:0] MODE_EP2MEM = 2'd2;
|
|
|
|
localparam logic [7:0] REG_CTRL = 8'h00;
|
|
localparam logic [7:0] REG_STATUS = 8'h04;
|
|
localparam logic [7:0] REG_SRC_BASE = 8'h08;
|
|
localparam logic [7:0] REG_DST_BASE = 8'h0C;
|
|
localparam logic [7:0] REG_GROUPS = 8'h10;
|
|
localparam logic [7:0] REG_MAT_BASE = 8'h14;
|
|
localparam logic [7:0] REG_SRC_STRIDE = 8'h18;
|
|
localparam logic [7:0] REG_DST_STRIDE = 8'h1C;
|
|
|
|
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
|
|
localparam logic [31:0] MAT_C_BYTES = 32'd192;
|
|
|
|
typedef enum logic [3:0] {
|
|
SQ_IDLE,
|
|
SQ_CLEAR,
|
|
SQ_LOAD_START,
|
|
SQ_LOAD_REQ,
|
|
SQ_LOAD_WAIT,
|
|
SQ_WAIT_DONE,
|
|
SQ_STORE_START,
|
|
SQ_STORE_REQ,
|
|
SQ_STORE_WAIT,
|
|
SQ_DONE
|
|
} seq_state_t;
|
|
|
|
seq_state_t seq_state;
|
|
|
|
logic [31:0] seq_src_base;
|
|
logic [31:0] seq_dst_base;
|
|
logic [31:0] seq_groups;
|
|
logic [31:0] seq_mat_base;
|
|
logic [31:0] seq_src_stride;
|
|
logic [31:0] seq_dst_stride;
|
|
logic seq_busy;
|
|
logic seq_done_flag;
|
|
logic [31:0] seq_load_idx;
|
|
logic [31:0] seq_store_idx;
|
|
logic seq_compute_bank;
|
|
logic seq_skip_writeback;
|
|
logic seq_defer_load_start;
|
|
logic seq_deferred_start_valid;
|
|
logic seq_deferred_start_bank;
|
|
|
|
wire [7:0] wr_off = s_axil_awaddr[7:0];
|
|
wire [7:0] rd_off = s_axil_araddr[7:0];
|
|
wire start_req = do_write && (wr_off == REG_CTRL) && s_axil_wstrb[0] && s_axil_wdata[0];
|
|
wire done_clear_req = do_write && (wr_off == REG_STATUS) && s_axil_wstrb[0] && s_axil_wdata[1];
|
|
wire [31:0] src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
|
|
wire [31:0] dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
|
|
wire loaded_bank = seq_load_idx[0];
|
|
|
|
function automatic logic [31:0] apply_wstrb(
|
|
input logic [31:0] old_val,
|
|
input logic [31:0] new_val,
|
|
input logic [3:0] wstrb
|
|
);
|
|
begin
|
|
apply_wstrb[7:0] = wstrb[0] ? new_val[7:0] : old_val[7:0];
|
|
apply_wstrb[15:8] = wstrb[1] ? new_val[15:8] : old_val[15:8];
|
|
apply_wstrb[23:16] = wstrb[2] ? new_val[23:16] : old_val[23:16];
|
|
apply_wstrb[31:24] = wstrb[3] ? new_val[31:24] : old_val[31:24];
|
|
end
|
|
endfunction
|
|
|
|
task issue_load_req;
|
|
input [31:0] load_idx;
|
|
begin
|
|
cdma_req_mode <= MODE_MEM2EP;
|
|
cdma_req_src <= seq_src_base + (load_idx * seq_src_stride);
|
|
cdma_req_dst <= '0;
|
|
cdma_req_bytes <= MAT_AB_BYTES;
|
|
cdma_req_ep_id <= 4'd0;
|
|
cdma_req_ep_bank <= {3'b000, load_idx[0]};
|
|
cdma_req_ep_word_off <= 8'd0;
|
|
cdma_req_valid <= 1'b1;
|
|
seq_state <= SQ_LOAD_REQ;
|
|
end
|
|
endtask
|
|
|
|
task issue_store_req;
|
|
begin
|
|
cdma_req_mode <= MODE_EP2MEM;
|
|
cdma_req_src <= '0;
|
|
cdma_req_dst <= dst_addr_for_store;
|
|
cdma_req_bytes <= MAT_C_BYTES;
|
|
cdma_req_ep_id <= 4'd0;
|
|
cdma_req_ep_bank <= {3'b000, seq_compute_bank};
|
|
cdma_req_ep_word_off <= 8'd0;
|
|
cdma_req_valid <= 1'b1;
|
|
seq_state <= SQ_STORE_REQ;
|
|
end
|
|
endtask
|
|
|
|
assign s_axil_bresp = 2'b00;
|
|
assign s_axil_rresp = 2'b00;
|
|
assign seq_done_irq = seq_done_flag;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_awvalid && s_axil_wvalid && !s_axil_awready && (!s_axil_bvalid || s_axil_bready)) begin
|
|
s_axil_awready <= 1'b1;
|
|
s_axil_wready <= 1'b1;
|
|
end else begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
wire do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end else if (do_write) begin
|
|
s_axil_bvalid <= 1'b1;
|
|
end else if (s_axil_bready && s_axil_bvalid) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_arready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_arvalid && !s_axil_arready && (!s_axil_rvalid || s_axil_rready)) begin
|
|
s_axil_arready <= 1'b1;
|
|
end else begin
|
|
s_axil_arready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
wire do_read = s_axil_arready && s_axil_arvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
s_axil_rdata <= '0;
|
|
end else if (do_read) begin
|
|
s_axil_rvalid <= 1'b1;
|
|
case (rd_off)
|
|
REG_CTRL: s_axil_rdata <= {31'd0, start_req};
|
|
REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
|
|
REG_SRC_BASE: s_axil_rdata <= seq_src_base;
|
|
REG_DST_BASE: s_axil_rdata <= seq_dst_base;
|
|
REG_GROUPS: s_axil_rdata <= seq_groups;
|
|
REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
|
|
REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
|
|
REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
|
|
default: s_axil_rdata <= 32'd0;
|
|
endcase
|
|
end else if (s_axil_rready && s_axil_rvalid) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
seq_src_base <= 32'h0;
|
|
seq_dst_base <= 32'h0;
|
|
seq_groups <= 32'h0;
|
|
seq_mat_base <= 32'h1f50_0000;
|
|
seq_src_stride <= 32'd128;
|
|
seq_dst_stride <= 32'd192;
|
|
end else if (do_write) begin
|
|
case (wr_off)
|
|
REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
|
|
REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
|
|
REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
|
|
REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
|
|
default: begin
|
|
end
|
|
endcase
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
seq_state <= SQ_IDLE;
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b0;
|
|
seq_load_idx <= 32'd0;
|
|
seq_store_idx <= 32'd0;
|
|
seq_compute_bank <= 1'b0;
|
|
seq_skip_writeback <= 1'b0;
|
|
seq_defer_load_start <= 1'b0;
|
|
seq_deferred_start_valid <= 1'b0;
|
|
seq_deferred_start_bank <= 1'b0;
|
|
cdma_req_valid <= 1'b0;
|
|
cdma_req_mode <= MODE_MEM2EP;
|
|
cdma_req_src <= '0;
|
|
cdma_req_dst <= '0;
|
|
cdma_req_bytes <= '0;
|
|
cdma_req_ep_id <= 4'd0;
|
|
cdma_req_ep_bank <= 4'd0;
|
|
cdma_req_ep_word_off <= 8'd0;
|
|
mat_start_valid <= 1'b0;
|
|
mat_start_bank <= 1'b0;
|
|
mat_done_clear <= 1'b0;
|
|
mat_crc_clear <= 1'b0;
|
|
end else begin
|
|
mat_start_valid <= 1'b0;
|
|
mat_done_clear <= 1'b0;
|
|
mat_crc_clear <= 1'b0;
|
|
|
|
if (done_clear_req)
|
|
seq_done_flag <= 1'b0;
|
|
|
|
case (seq_state)
|
|
SQ_IDLE: begin
|
|
seq_busy <= 1'b0;
|
|
cdma_req_valid <= 1'b0;
|
|
if (start_req && (seq_groups != 32'd0)) begin
|
|
seq_busy <= 1'b1;
|
|
seq_done_flag <= 1'b0;
|
|
seq_load_idx <= 32'd0;
|
|
seq_store_idx <= 32'd0;
|
|
seq_compute_bank <= 1'b0;
|
|
seq_skip_writeback <= s_axil_wstrb[0] && s_axil_wdata[1];
|
|
seq_defer_load_start <= 1'b0;
|
|
seq_deferred_start_valid <= 1'b0;
|
|
seq_deferred_start_bank <= 1'b0;
|
|
mat_crc_clear <= 1'b1;
|
|
mat_done_clear <= 1'b1;
|
|
seq_state <= SQ_CLEAR;
|
|
end
|
|
end
|
|
|
|
SQ_CLEAR: begin
|
|
issue_load_req(32'd0);
|
|
end
|
|
|
|
SQ_LOAD_START: begin
|
|
issue_load_req(seq_load_idx);
|
|
end
|
|
|
|
SQ_LOAD_REQ: begin
|
|
if (cdma_req_valid && cdma_req_ready) begin
|
|
cdma_req_valid <= 1'b0;
|
|
seq_state <= SQ_LOAD_WAIT;
|
|
end
|
|
end
|
|
|
|
SQ_LOAD_WAIT: begin
|
|
if (cdma_req_done) begin
|
|
seq_load_idx <= seq_load_idx + 32'd1;
|
|
if (seq_defer_load_start) begin
|
|
seq_defer_load_start <= 1'b0;
|
|
seq_deferred_start_valid <= 1'b1;
|
|
seq_deferred_start_bank <= cdma_req_ep_bank[0];
|
|
issue_store_req();
|
|
end else begin
|
|
mat_start_valid <= 1'b1;
|
|
mat_start_bank <= cdma_req_ep_bank[0];
|
|
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin
|
|
issue_load_req(seq_load_idx + 32'd1);
|
|
end else begin
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_WAIT_DONE: begin
|
|
if (mat_done) begin
|
|
mat_done_clear <= 1'b1;
|
|
if (seq_skip_writeback) begin
|
|
seq_store_idx <= seq_store_idx + 32'd1;
|
|
seq_compute_bank <= ~seq_compute_bank;
|
|
if ((seq_store_idx + 32'd1) >= seq_groups) begin
|
|
seq_state <= SQ_DONE;
|
|
end else if (seq_load_idx < seq_groups) begin
|
|
issue_load_req(seq_load_idx);
|
|
end else begin
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end else if (seq_load_idx < seq_groups) begin
|
|
seq_defer_load_start <= 1'b1;
|
|
issue_load_req(seq_load_idx);
|
|
end else begin
|
|
issue_store_req();
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_STORE_START: begin
|
|
issue_store_req();
|
|
end
|
|
|
|
SQ_STORE_REQ: begin
|
|
if (cdma_req_valid && cdma_req_ready) begin
|
|
cdma_req_valid <= 1'b0;
|
|
seq_state <= SQ_STORE_WAIT;
|
|
end
|
|
end
|
|
|
|
SQ_STORE_WAIT: begin
|
|
if (cdma_req_done) begin
|
|
seq_store_idx <= seq_store_idx + 32'd1;
|
|
seq_compute_bank <= ~seq_compute_bank;
|
|
if (seq_deferred_start_valid) begin
|
|
mat_start_valid <= 1'b1;
|
|
mat_start_bank <= seq_deferred_start_bank;
|
|
seq_deferred_start_valid <= 1'b0;
|
|
end
|
|
if ((seq_store_idx + 32'd1) >= seq_groups) begin
|
|
seq_state <= SQ_DONE;
|
|
end else if (seq_load_idx < seq_groups) begin
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end else begin
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_DONE: begin
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b1;
|
|
seq_state <= SQ_IDLE;
|
|
end
|
|
|
|
default: seq_state <= SQ_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
endmodule
|