1137 lines
49 KiB
Systemverilog
1137 lines
49 KiB
Systemverilog
// ============================================================================
|
|
// snix_axil_cdma_mux.sv
|
|
// Multi-Channel AXI-Lite MUX Wrapper for snix_axi_mm2mm engine
|
|
//
|
|
// Register Map (per channel, offset 0x40):
|
|
// 0x00: READ_ADDR (Source Address)
|
|
// 0x04: WRITE_ADDR (Destination Address)
|
|
// 0x08: LENGTH (Transfer length in bytes)
|
|
// 0x0C: TAG (User tag, purely for software tracking)
|
|
// 0x10: CTRL (Write 1 to bit 0 to trigger)
|
|
// [5:3] = AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B...)
|
|
// [13:6] = AXI AxLEN (0=1 beat, 15=16 beats...)
|
|
// 0x14: STATUS (Bit 0: Busy RO, Bit 1: Done W1C)
|
|
// ============================================================================
|
|
`timescale 1ns / 1ps
|
|
|
|
module snix_axil_cdma_mux #(
|
|
parameter int ADDR_WIDTH = 32,
|
|
parameter int DATA_WIDTH = 32, // Matches engine default
|
|
parameter int AXIL_ADDR_WIDTH = 32,
|
|
parameter int AXIL_DATA_WIDTH = 32,
|
|
parameter int ID_WIDTH = 4,
|
|
parameter int USER_WIDTH = 1,
|
|
parameter int PORTS = 8,
|
|
parameter int FIFO_DEPTH = 16
|
|
) (
|
|
input logic clk,
|
|
input logic rst_n,
|
|
|
|
// ==========================================
|
|
// AXI-Lite Slave Interface (CPU CSR Access)
|
|
// ==========================================
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_awaddr,
|
|
input logic s_axil_awvalid,
|
|
output logic s_axil_awready,
|
|
input logic [AXIL_DATA_WIDTH-1:0] s_axil_wdata,
|
|
input logic [AXIL_DATA_WIDTH/8-1:0] s_axil_wstrb,
|
|
input logic s_axil_wvalid,
|
|
output logic s_axil_wready,
|
|
output logic [1:0] s_axil_bresp,
|
|
output logic s_axil_bvalid,
|
|
input logic s_axil_bready,
|
|
input logic [AXIL_ADDR_WIDTH-1:0] s_axil_araddr,
|
|
input logic s_axil_arvalid,
|
|
output logic s_axil_arready,
|
|
output logic [AXIL_DATA_WIDTH-1:0] s_axil_rdata,
|
|
output logic [1:0] s_axil_rresp,
|
|
output logic s_axil_rvalid,
|
|
input logic s_axil_rready,
|
|
|
|
// ==========================================
|
|
// AXI4 Master Interface (To Crossbar/Memory)
|
|
// ==========================================
|
|
output logic [ID_WIDTH-1:0] mm2mm_awid,
|
|
output logic [ADDR_WIDTH-1:0] mm2mm_awaddr,
|
|
output logic [7:0] mm2mm_awlen,
|
|
output logic [2:0] mm2mm_awsize,
|
|
output logic [1:0] mm2mm_awburst,
|
|
output logic mm2mm_awlock,
|
|
output logic [3:0] mm2mm_awcache,
|
|
output logic [2:0] mm2mm_awprot,
|
|
output logic [3:0] mm2mm_awqos,
|
|
output logic [USER_WIDTH-1:0] mm2mm_awuser,
|
|
output logic mm2mm_awvalid,
|
|
input logic mm2mm_awready,
|
|
output logic [DATA_WIDTH-1:0] mm2mm_wdata,
|
|
output logic [DATA_WIDTH/8-1:0] mm2mm_wstrb,
|
|
output logic mm2mm_wlast,
|
|
output logic [USER_WIDTH-1:0] mm2mm_wuser,
|
|
output logic mm2mm_wvalid,
|
|
input logic mm2mm_wready,
|
|
input logic [ID_WIDTH-1:0] mm2mm_bid,
|
|
input logic [1:0] mm2mm_bresp,
|
|
input logic [USER_WIDTH-1:0] mm2mm_buser,
|
|
input logic mm2mm_bvalid,
|
|
output logic mm2mm_bready,
|
|
output logic [ID_WIDTH-1:0] mm2mm_arid,
|
|
output logic [ADDR_WIDTH-1:0] mm2mm_araddr,
|
|
output logic [7:0] mm2mm_arlen,
|
|
output logic [2:0] mm2mm_arsize,
|
|
output logic [1:0] mm2mm_arburst,
|
|
output logic mm2mm_arlock,
|
|
output logic [3:0] mm2mm_arcache,
|
|
output logic [2:0] mm2mm_arprot,
|
|
output logic [3:0] mm2mm_arqos,
|
|
output logic [USER_WIDTH-1:0] mm2mm_aruser,
|
|
output logic mm2mm_arvalid,
|
|
input logic mm2mm_arready,
|
|
input logic [ID_WIDTH-1:0] mm2mm_rid,
|
|
input logic [DATA_WIDTH-1:0] mm2mm_rdata,
|
|
input logic [1:0] mm2mm_rresp,
|
|
input logic mm2mm_rlast,
|
|
input logic [USER_WIDTH-1:0] mm2mm_ruser,
|
|
input logic mm2mm_rvalid,
|
|
output logic mm2mm_rready,
|
|
|
|
// ==========================================
|
|
// Local endpoint interface
|
|
// ==========================================
|
|
output logic ep_wr_valid,
|
|
output logic [3:0] ep_wr_id,
|
|
output logic [3:0] ep_wr_bank,
|
|
output logic [7:0] ep_wr_word_addr,
|
|
output logic [31:0] ep_wr_data,
|
|
input logic ep_wr_ready,
|
|
|
|
output logic ep_rd_valid,
|
|
output logic [3:0] ep_rd_id,
|
|
output logic [3:0] ep_rd_bank,
|
|
output logic [7:0] ep_rd_word_addr,
|
|
input logic [31:0] ep_rd_data,
|
|
input logic ep_rd_ready,
|
|
|
|
// Global Interrupt (OR'd from all channels)
|
|
output logic dma_finish
|
|
);
|
|
|
|
// ==========================================
|
|
// Local Parameters & Utilities
|
|
// ==========================================
|
|
localparam int CH_BITS = $clog2(PORTS);
|
|
localparam logic [1:0] MODE_MEM2MEM = 2'd0;
|
|
localparam logic [1:0] MODE_MEM2EP = 2'd1;
|
|
localparam logic [1:0] MODE_EP2MEM = 2'd2;
|
|
|
|
localparam logic [7:0] SEQ_REG_CTRL = 8'h00;
|
|
localparam logic [7:0] SEQ_REG_STATUS = 8'h04;
|
|
localparam logic [7:0] SEQ_REG_SRC_BASE = 8'h08;
|
|
localparam logic [7:0] SEQ_REG_DST_BASE = 8'h0C;
|
|
localparam logic [7:0] SEQ_REG_GROUPS = 8'h10;
|
|
localparam logic [7:0] SEQ_REG_MAT_BASE = 8'h14;
|
|
localparam logic [7:0] SEQ_REG_SRC_STRIDE = 8'h18;
|
|
localparam logic [7:0] SEQ_REG_DST_STRIDE = 8'h1C;
|
|
|
|
localparam logic [11:0] MAT_CTRL_OFF = 12'h000;
|
|
localparam logic [11:0] MAT_STATUS_OFF = 12'h004;
|
|
localparam logic [11:0] MAT_CRC_OFF = 12'h010;
|
|
localparam logic [11:0] MAT_B0_AB_OFF = 12'h100;
|
|
localparam logic [11:0] MAT_B0_C_OFF = 12'h180;
|
|
localparam logic [11:0] MAT_B1_AB_OFF = 12'h300;
|
|
localparam logic [11:0] MAT_B1_C_OFF = 12'h380;
|
|
|
|
localparam logic [31:0] MAT_STATUS_DONE_CLR = 32'h0000_0006;
|
|
localparam logic [31:0] MAT_STATUS_DONE_BIT = 32'h0000_0002;
|
|
localparam logic [31:0] MAT_CTRL_START_B0 = 32'h0000_0001;
|
|
localparam logic [31:0] MAT_CTRL_START_B1 = 32'h0000_0002;
|
|
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
|
|
localparam logic [31:0] MAT_C_BYTES = 32'd192;
|
|
|
|
// Function to safely apply WSTRB to 32-bit registers
|
|
function automatic logic [31:0] apply_wstrb(
|
|
input logic [31:0] old_val,
|
|
input logic [31:0] new_val,
|
|
input logic [3:0] wstrb
|
|
);
|
|
logic [31:0] res;
|
|
res[7:0] = wstrb[0] ? new_val[7:0] : old_val[7:0];
|
|
res[15:8] = wstrb[1] ? new_val[15:8] : old_val[15:8];
|
|
res[23:16] = wstrb[2] ? new_val[23:16] : old_val[23:16];
|
|
res[31:24] = wstrb[3] ? new_val[31:24] : old_val[31:24];
|
|
return res;
|
|
endfunction
|
|
|
|
// ==========================================
|
|
// Internal Registers (Per Channel)
|
|
// ==========================================
|
|
logic [ADDR_WIDTH-1:0] ch_src_addr [PORTS];
|
|
logic [ADDR_WIDTH-1:0] ch_dst_addr [PORTS];
|
|
logic [31:0] ch_len [PORTS];
|
|
logic [31:0] ch_tag [PORTS];
|
|
logic [31:0] ch_ctrl [PORTS];
|
|
logic [31:0] ch_mode [PORTS];
|
|
logic [31:0] ch_ep_cfg [PORTS];
|
|
|
|
logic [PORTS-1:0] ch_req; // Pending requests (Busy)
|
|
logic [PORTS-1:0] ch_done; // Completion flags
|
|
|
|
logic [PORTS-1:0] arb_set_done; // From Arbiter to CSR
|
|
|
|
logic [31:0] seq_src_base;
|
|
logic [31:0] seq_dst_base;
|
|
logic [31:0] seq_groups;
|
|
logic [31:0] seq_mat_base;
|
|
logic [31:0] seq_src_stride;
|
|
logic [31:0] seq_dst_stride;
|
|
logic seq_busy;
|
|
logic seq_done_flag;
|
|
wire seq_start_req;
|
|
wire seq_done_clear_req;
|
|
|
|
// ==========================================
|
|
// Address Decoding (0x40 offset per channel)
|
|
// ==========================================
|
|
wire [CH_BITS-1:0] wr_ch = s_axil_awaddr[6 +: CH_BITS];
|
|
wire [5:0] wr_reg = s_axil_awaddr[5:0];
|
|
wire [CH_BITS-1:0] rd_ch = s_axil_araddr[6 +: CH_BITS];
|
|
wire [5:0] rd_reg = s_axil_araddr[5:0];
|
|
wire wr_seq_reg = (s_axil_awaddr[11:8] == 4'h2);
|
|
wire rd_seq_reg = (s_axil_araddr[11:8] == 4'h2);
|
|
wire [7:0] wr_seq_off = s_axil_awaddr[7:0];
|
|
wire [7:0] rd_seq_off = s_axil_araddr[7:0];
|
|
assign seq_start_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_CTRL) &&
|
|
s_axil_wstrb[0] && s_axil_wdata[0];
|
|
assign seq_done_clear_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_STATUS) &&
|
|
s_axil_wstrb[0] && s_axil_wdata[1];
|
|
|
|
// ==========================================
|
|
// AXI-Lite Slave Logic (Robust Backpressure)
|
|
// ==========================================
|
|
assign s_axil_bresp = 2'b00;
|
|
assign s_axil_rresp = 2'b00;
|
|
|
|
// Write Path Handshake
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_awvalid && s_axil_wvalid && !s_axil_awready && (!s_axil_bvalid || s_axil_bready)) begin
|
|
s_axil_awready <= 1'b1;
|
|
s_axil_wready <= 1'b1;
|
|
end else begin
|
|
s_axil_awready <= 1'b0;
|
|
s_axil_wready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
wire do_write = s_axil_awready && s_axil_awvalid && s_axil_wready && s_axil_wvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end else begin
|
|
if (do_write) begin
|
|
s_axil_bvalid <= 1'b1;
|
|
end else if (s_axil_bready && s_axil_bvalid) begin
|
|
s_axil_bvalid <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
// Read Path Handshake
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_arready <= 1'b0;
|
|
end else begin
|
|
if (s_axil_arvalid && !s_axil_arready && (!s_axil_rvalid || s_axil_rready)) begin
|
|
s_axil_arready <= 1'b1;
|
|
end else begin
|
|
s_axil_arready <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
wire do_read = s_axil_arready && s_axil_arvalid;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
s_axil_rdata <= '0;
|
|
end else begin
|
|
if (do_read) begin
|
|
s_axil_rvalid <= 1'b1;
|
|
if (rd_seq_reg) begin
|
|
case (rd_seq_off)
|
|
SEQ_REG_CTRL: s_axil_rdata <= {31'd0, seq_start_req};
|
|
SEQ_REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
|
|
SEQ_REG_SRC_BASE: s_axil_rdata <= seq_src_base;
|
|
SEQ_REG_DST_BASE: s_axil_rdata <= seq_dst_base;
|
|
SEQ_REG_GROUPS: s_axil_rdata <= seq_groups;
|
|
SEQ_REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
|
|
SEQ_REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
|
|
SEQ_REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
|
|
default: s_axil_rdata <= 32'd0;
|
|
endcase
|
|
end else if (rd_ch < PORTS) begin
|
|
case (rd_reg)
|
|
6'h00: s_axil_rdata <= ch_src_addr[rd_ch];
|
|
6'h04: s_axil_rdata <= ch_dst_addr[rd_ch];
|
|
6'h08: s_axil_rdata <= ch_len[rd_ch];
|
|
6'h0C: s_axil_rdata <= ch_tag[rd_ch];
|
|
6'h10: s_axil_rdata <= ch_ctrl[rd_ch];
|
|
6'h14: s_axil_rdata <= {30'd0, ch_done[rd_ch], ch_req[rd_ch]};
|
|
6'h18: s_axil_rdata <= ch_mode[rd_ch];
|
|
6'h1C: s_axil_rdata <= ch_ep_cfg[rd_ch];
|
|
default: s_axil_rdata <= 32'd0;
|
|
endcase
|
|
end else begin
|
|
s_axil_rdata <= 32'd0; // Out of bounds
|
|
end
|
|
end else if (s_axil_rready && s_axil_rvalid) begin
|
|
s_axil_rvalid <= 1'b0;
|
|
end
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// Register File Write Logic
|
|
// ==========================================
|
|
assign dma_finish = |ch_done;
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
ch_req <= '0;
|
|
ch_done <= '0;
|
|
seq_src_base <= 32'h0;
|
|
seq_dst_base <= 32'h0;
|
|
seq_groups <= 32'h0;
|
|
seq_mat_base <= 32'h1f50_0000;
|
|
seq_src_stride <= 32'd128;
|
|
seq_dst_stride <= 32'd192;
|
|
for (int i=0; i<PORTS; i++) begin
|
|
ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0;
|
|
ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0;
|
|
ch_mode[i] <= '0; ch_ep_cfg[i] <= '0;
|
|
end
|
|
end else begin
|
|
// 1. Process Hardware Status Updates (Highest Priority for Done flag)
|
|
for (int i=0; i<PORTS; i++) begin
|
|
if (arb_set_done[i]) begin
|
|
ch_req[i] <= 1'b0;
|
|
ch_done[i] <= 1'b1;
|
|
end
|
|
end
|
|
|
|
// 2. Process CPU Writes
|
|
if (do_write && wr_seq_reg) begin
|
|
case (wr_seq_off)
|
|
SEQ_REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
|
|
SEQ_REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
|
|
SEQ_REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
|
|
SEQ_REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
|
|
SEQ_REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
|
|
SEQ_REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
|
|
endcase
|
|
end else if (do_write && wr_ch < PORTS) begin
|
|
case (wr_reg)
|
|
6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h08: if (!ch_req[wr_ch]) ch_len[wr_ch] <= apply_wstrb(ch_len[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h0C: if (!ch_req[wr_ch]) ch_tag[wr_ch] <= apply_wstrb(ch_tag[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h18: if (!ch_req[wr_ch]) ch_mode[wr_ch] <= apply_wstrb(ch_mode[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h1C: if (!ch_req[wr_ch]) ch_ep_cfg[wr_ch] <= apply_wstrb(ch_ep_cfg[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
6'h10: begin
|
|
ch_ctrl[wr_ch] <= apply_wstrb(ch_ctrl[wr_ch], s_axil_wdata, s_axil_wstrb);
|
|
// Trigger Bit Processing
|
|
if (s_axil_wstrb[0] && s_axil_wdata[0]) begin
|
|
ch_req[wr_ch] <= 1'b1;
|
|
// Clean up done bit automatically upon new start
|
|
if (!arb_set_done[wr_ch]) ch_done[wr_ch] <= 1'b0;
|
|
end
|
|
end
|
|
6'h14: begin
|
|
// Software W1C for Done flag (Bit 1)
|
|
// ONLY clear if hardware is not setting it in the exact same cycle
|
|
if (s_axil_wstrb[0] && s_axil_wdata[1]) begin
|
|
if (!arb_set_done[wr_ch]) ch_done[wr_ch] <= 1'b0;
|
|
end
|
|
end
|
|
endcase
|
|
end
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// Round-Robin Arbiter & Engine Driver
|
|
// ==========================================
|
|
typedef enum logic [1:0] {IDLE, START, RUN} state_t;
|
|
state_t state;
|
|
|
|
logic [CH_BITS-1:0] cur_ch;
|
|
logic [CH_BITS-1:0] rr_ptr;
|
|
|
|
// Interfaces to Engine
|
|
logic engine_start;
|
|
logic [ADDR_WIDTH-1:0] engine_src;
|
|
logic [ADDR_WIDTH-1:0] engine_dst;
|
|
logic [31:0] engine_bytes;
|
|
logic [7:0] engine_len;
|
|
logic [2:0] engine_size;
|
|
logic engine_done;
|
|
logic [1:0] engine_mode;
|
|
logic [3:0] engine_ep_id;
|
|
logic [3:0] engine_ep_bank;
|
|
logic [7:0] engine_ep_word_off;
|
|
|
|
logic mm2mm_start;
|
|
logic mm2mm_done;
|
|
logic ep_start;
|
|
logic ep_done;
|
|
|
|
logic [ID_WIDTH-1:0] core_awid;
|
|
logic [ADDR_WIDTH-1:0] core_awaddr;
|
|
logic [7:0] core_awlen;
|
|
logic [2:0] core_awsize;
|
|
logic [1:0] core_awburst;
|
|
logic core_awlock;
|
|
logic [3:0] core_awcache;
|
|
logic [2:0] core_awprot;
|
|
logic [3:0] core_awqos;
|
|
logic [USER_WIDTH-1:0] core_awuser;
|
|
logic core_awvalid;
|
|
logic core_awready;
|
|
logic [DATA_WIDTH-1:0] core_wdata;
|
|
logic [DATA_WIDTH/8-1:0] core_wstrb;
|
|
logic core_wlast;
|
|
logic [USER_WIDTH-1:0] core_wuser;
|
|
logic core_wvalid;
|
|
logic core_wready;
|
|
logic core_bready;
|
|
logic [ID_WIDTH-1:0] core_arid;
|
|
logic [ADDR_WIDTH-1:0] core_araddr;
|
|
logic [7:0] core_arlen;
|
|
logic [2:0] core_arsize;
|
|
logic [1:0] core_arburst;
|
|
logic core_arlock;
|
|
logic [3:0] core_arcache;
|
|
logic [2:0] core_arprot;
|
|
logic [3:0] core_arqos;
|
|
logic [USER_WIDTH-1:0] core_aruser;
|
|
logic core_arvalid;
|
|
logic core_arready;
|
|
logic core_rready;
|
|
|
|
logic [ID_WIDTH-1:0] ep_awid;
|
|
logic [ADDR_WIDTH-1:0] ep_awaddr;
|
|
logic [7:0] ep_awlen;
|
|
logic [2:0] ep_awsize;
|
|
logic [1:0] ep_awburst;
|
|
logic ep_awlock;
|
|
logic [3:0] ep_awcache;
|
|
logic [2:0] ep_awprot;
|
|
logic [3:0] ep_awqos;
|
|
logic [USER_WIDTH-1:0] ep_awuser;
|
|
logic ep_awvalid;
|
|
logic ep_awready;
|
|
logic [DATA_WIDTH-1:0] ep_wdata;
|
|
logic [DATA_WIDTH/8-1:0] ep_wstrb;
|
|
logic ep_wlast;
|
|
logic [USER_WIDTH-1:0] ep_wuser;
|
|
logic ep_wvalid;
|
|
logic ep_wready;
|
|
logic ep_bready;
|
|
logic [ID_WIDTH-1:0] ep_arid;
|
|
logic [ADDR_WIDTH-1:0] ep_araddr;
|
|
logic [7:0] ep_arlen;
|
|
logic [2:0] ep_arsize;
|
|
logic [1:0] ep_arburst;
|
|
logic ep_arlock;
|
|
logic [3:0] ep_arcache;
|
|
logic [2:0] ep_arprot;
|
|
logic [3:0] ep_arqos;
|
|
logic [USER_WIDTH-1:0] ep_aruser;
|
|
logic ep_arvalid;
|
|
logic ep_arready;
|
|
logic ep_rready;
|
|
|
|
logic [ID_WIDTH-1:0] seq_awid;
|
|
logic [ADDR_WIDTH-1:0] seq_awaddr;
|
|
logic [7:0] seq_awlen;
|
|
logic [2:0] seq_awsize;
|
|
logic [1:0] seq_awburst;
|
|
logic seq_awlock;
|
|
logic [3:0] seq_awcache;
|
|
logic [2:0] seq_awprot;
|
|
logic [3:0] seq_awqos;
|
|
logic [USER_WIDTH-1:0] seq_awuser;
|
|
logic seq_awvalid;
|
|
logic seq_awready;
|
|
logic [DATA_WIDTH-1:0] seq_wdata;
|
|
logic [DATA_WIDTH/8-1:0] seq_wstrb;
|
|
logic seq_wlast;
|
|
logic [USER_WIDTH-1:0] seq_wuser;
|
|
logic seq_wvalid;
|
|
logic seq_wready;
|
|
logic seq_bready;
|
|
logic [ID_WIDTH-1:0] seq_arid;
|
|
logic [ADDR_WIDTH-1:0] seq_araddr;
|
|
logic [7:0] seq_arlen;
|
|
logic [2:0] seq_arsize;
|
|
logic [1:0] seq_arburst;
|
|
logic seq_arlock;
|
|
logic [3:0] seq_arcache;
|
|
logic [2:0] seq_arprot;
|
|
logic [3:0] seq_arqos;
|
|
logic [USER_WIDTH-1:0] seq_aruser;
|
|
logic seq_arvalid;
|
|
logic seq_arready;
|
|
logic seq_rready;
|
|
|
|
// Hardware matmul loop sequencer state. Keep these declarations before the
|
|
// DMA routing logic because Vivado does not allow forward enum references.
|
|
typedef enum logic [4:0] {
|
|
SQ_IDLE,
|
|
SQ_CLEAR_CRC,
|
|
SQ_LOAD0,
|
|
SQ_START0,
|
|
SQ_LOAD1,
|
|
SQ_WAIT_DONE,
|
|
SQ_STORE,
|
|
SQ_CLEAR_DONE,
|
|
SQ_START_NEXT,
|
|
SQ_LOAD_NEXT,
|
|
SQ_DONE
|
|
} seq_state_t;
|
|
|
|
typedef enum logic [3:0] {
|
|
OP_IDLE,
|
|
OP_READ_ADDR,
|
|
OP_READ_DATA,
|
|
OP_WRITE_ADDR,
|
|
OP_WRITE_DATA,
|
|
OP_WRITE_RESP,
|
|
OP_REG_READ_ADDR,
|
|
OP_REG_READ_DATA,
|
|
OP_COPY_START,
|
|
OP_COPY_WAIT
|
|
} op_state_t;
|
|
|
|
seq_state_t seq_state;
|
|
op_state_t op_state;
|
|
|
|
logic op_start_copy;
|
|
logic op_start_write;
|
|
logic op_start_read;
|
|
logic op_done;
|
|
logic [31:0] op_src_addr;
|
|
logic [31:0] op_dst_addr;
|
|
logic [31:0] op_bytes;
|
|
logic [31:0] op_write_addr;
|
|
logic [31:0] op_write_data;
|
|
logic [31:0] op_read_addr;
|
|
logic [31:0] op_read_data;
|
|
logic [31:0] op_src_cur;
|
|
logic [31:0] op_dst_cur;
|
|
logic [31:0] op_bytes_left;
|
|
logic [31:0] op_data_buf;
|
|
logic op_buf_valid;
|
|
logic op_copy_store;
|
|
|
|
logic [31:0] seq_group_idx;
|
|
logic [31:0] seq_load_idx;
|
|
logic [31:0] seq_store_idx;
|
|
logic seq_compute_bank;
|
|
logic [31:0] seq_status_data;
|
|
|
|
wire use_ep_engine = (state == RUN) && (engine_mode != MODE_MEM2MEM);
|
|
wire use_seq_engine = seq_busy && (op_state != OP_IDLE) &&
|
|
(op_state != OP_COPY_START) && (op_state != OP_COPY_WAIT);
|
|
wire seq_dma_active = seq_busy && ((op_state == OP_COPY_START) || (op_state == OP_COPY_WAIT));
|
|
|
|
// Dynamic routing to the engine based on current active channel
|
|
assign engine_src = ch_src_addr[cur_ch];
|
|
assign engine_dst = ch_dst_addr[cur_ch];
|
|
assign engine_bytes = ch_len[cur_ch];
|
|
assign engine_len = ch_ctrl[cur_ch][13:6];
|
|
assign engine_size = ch_ctrl[cur_ch][5:3];
|
|
assign engine_mode = ch_mode[cur_ch][1:0];
|
|
assign engine_ep_bank = ch_ep_cfg[cur_ch][3:0];
|
|
assign engine_ep_word_off = ch_ep_cfg[cur_ch][15:8];
|
|
assign engine_ep_id = ch_ep_cfg[cur_ch][19:16];
|
|
assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
|
|
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
|
|
assign ep_start = engine_start && (engine_mode != MODE_MEM2MEM);
|
|
|
|
assign core_awready = (!use_seq_engine && !use_ep_engine) && mm2mm_awready;
|
|
assign core_wready = (!use_seq_engine && !use_ep_engine) && mm2mm_wready;
|
|
assign core_arready = (!use_seq_engine && !use_ep_engine) && mm2mm_arready;
|
|
assign ep_awready = (!use_seq_engine && use_ep_engine) && mm2mm_awready;
|
|
assign ep_wready = (!use_seq_engine && use_ep_engine) && mm2mm_wready;
|
|
assign ep_arready = (!use_seq_engine && use_ep_engine) && mm2mm_arready;
|
|
assign seq_awready = use_seq_engine && mm2mm_awready;
|
|
assign seq_wready = use_seq_engine && mm2mm_wready;
|
|
assign seq_arready = use_seq_engine && mm2mm_arready;
|
|
|
|
assign mm2mm_awid = use_seq_engine ? seq_awid : (use_ep_engine ? ep_awid : core_awid);
|
|
assign mm2mm_awaddr = use_seq_engine ? seq_awaddr : (use_ep_engine ? ep_awaddr : core_awaddr);
|
|
assign mm2mm_awlen = use_seq_engine ? seq_awlen : (use_ep_engine ? ep_awlen : core_awlen);
|
|
assign mm2mm_awsize = use_seq_engine ? seq_awsize : (use_ep_engine ? ep_awsize : core_awsize);
|
|
assign mm2mm_awburst = use_seq_engine ? seq_awburst : (use_ep_engine ? ep_awburst : core_awburst);
|
|
assign mm2mm_awlock = use_seq_engine ? seq_awlock : (use_ep_engine ? ep_awlock : core_awlock);
|
|
assign mm2mm_awcache = use_seq_engine ? seq_awcache : (use_ep_engine ? ep_awcache : core_awcache);
|
|
assign mm2mm_awprot = use_seq_engine ? seq_awprot : (use_ep_engine ? ep_awprot : core_awprot);
|
|
assign mm2mm_awqos = use_seq_engine ? seq_awqos : (use_ep_engine ? ep_awqos : core_awqos);
|
|
assign mm2mm_awuser = use_seq_engine ? seq_awuser : (use_ep_engine ? ep_awuser : core_awuser);
|
|
assign mm2mm_awvalid = use_seq_engine ? seq_awvalid : (use_ep_engine ? ep_awvalid : core_awvalid);
|
|
|
|
assign mm2mm_wdata = use_seq_engine ? seq_wdata : (use_ep_engine ? ep_wdata : core_wdata);
|
|
assign mm2mm_wstrb = use_seq_engine ? seq_wstrb : (use_ep_engine ? ep_wstrb : core_wstrb);
|
|
assign mm2mm_wlast = use_seq_engine ? seq_wlast : (use_ep_engine ? ep_wlast : core_wlast);
|
|
assign mm2mm_wuser = use_seq_engine ? seq_wuser : (use_ep_engine ? ep_wuser : core_wuser);
|
|
assign mm2mm_wvalid = use_seq_engine ? seq_wvalid : (use_ep_engine ? ep_wvalid : core_wvalid);
|
|
assign mm2mm_bready = use_seq_engine ? seq_bready : (use_ep_engine ? ep_bready : core_bready);
|
|
|
|
assign mm2mm_arid = use_seq_engine ? seq_arid : (use_ep_engine ? ep_arid : core_arid);
|
|
assign mm2mm_araddr = use_seq_engine ? seq_araddr : (use_ep_engine ? ep_araddr : core_araddr);
|
|
assign mm2mm_arlen = use_seq_engine ? seq_arlen : (use_ep_engine ? ep_arlen : core_arlen);
|
|
assign mm2mm_arsize = use_seq_engine ? seq_arsize : (use_ep_engine ? ep_arsize : core_arsize);
|
|
assign mm2mm_arburst = use_seq_engine ? seq_arburst : (use_ep_engine ? ep_arburst : core_arburst);
|
|
assign mm2mm_arlock = use_seq_engine ? seq_arlock : (use_ep_engine ? ep_arlock : core_arlock);
|
|
assign mm2mm_arcache = use_seq_engine ? seq_arcache : (use_ep_engine ? ep_arcache : core_arcache);
|
|
assign mm2mm_arprot = use_seq_engine ? seq_arprot : (use_ep_engine ? ep_arprot : core_arprot);
|
|
assign mm2mm_arqos = use_seq_engine ? seq_arqos : (use_ep_engine ? ep_arqos : core_arqos);
|
|
assign mm2mm_aruser = use_seq_engine ? seq_aruser : (use_ep_engine ? ep_aruser : core_aruser);
|
|
assign mm2mm_arvalid = use_seq_engine ? seq_arvalid : (use_ep_engine ? ep_arvalid : core_arvalid);
|
|
assign mm2mm_rready = use_seq_engine ? seq_rready : (use_ep_engine ? ep_rready : core_rready);
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
state <= IDLE;
|
|
engine_start <= 1'b0;
|
|
rr_ptr <= '0;
|
|
cur_ch <= '0;
|
|
arb_set_done <= '0;
|
|
end else begin
|
|
arb_set_done <= '0;
|
|
engine_start <= 1'b0;
|
|
|
|
case (state)
|
|
IDLE: begin
|
|
if (seq_busy) begin
|
|
state <= IDLE;
|
|
end else for (int i = 0; i < PORTS; i++) begin
|
|
logic [CH_BITS:0] check_ch_ext;
|
|
logic [CH_BITS-1:0] check_ch;
|
|
|
|
// Calculate next channel safely avoiding modulo operators in loop
|
|
check_ch_ext = {1'b0, rr_ptr} + i[CH_BITS:0];
|
|
check_ch = (check_ch_ext >= PORTS) ? (check_ch_ext - PORTS) : check_ch_ext[CH_BITS-1:0];
|
|
|
|
if (ch_req[check_ch] && !arb_set_done[check_ch]) begin
|
|
cur_ch <= check_ch;
|
|
rr_ptr <= (check_ch == (PORTS - 1)) ? '0 : (check_ch + 1);
|
|
state <= START;
|
|
break;
|
|
end
|
|
end
|
|
end
|
|
|
|
START: begin
|
|
engine_start <= 1'b1;
|
|
state <= RUN;
|
|
end
|
|
|
|
RUN: begin
|
|
if (engine_done) begin
|
|
arb_set_done[cur_ch] <= 1'b1;
|
|
state <= IDLE;
|
|
end
|
|
// Optional: Add a watchdog timeout counter here if dealing with untrusted PCIe/AXI endpoints
|
|
end
|
|
endcase
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// Instantiate The Original Core Engine
|
|
// ==========================================
|
|
snix_axi_mm2mm #(
|
|
.ADDR_WIDTH(ADDR_WIDTH),
|
|
.DATA_WIDTH(DATA_WIDTH),
|
|
.ID_WIDTH (ID_WIDTH),
|
|
.USER_WIDTH(USER_WIDTH),
|
|
.FIFO_DEPTH(FIFO_DEPTH)
|
|
) u_core_engine (
|
|
.clk (clk),
|
|
.rst_n (rst_n),
|
|
.ctrl_start ((op_state == OP_COPY_START) ? 1'b1 : mm2mm_start),
|
|
.ctrl_stop (1'b0), // Tied off; can be wired if global abort is needed
|
|
.ctrl_src_addr (seq_dma_active ? op_src_cur : engine_src),
|
|
.ctrl_dst_addr (seq_dma_active ? op_dst_cur : engine_dst),
|
|
.ctrl_len (seq_dma_active ? 8'd63 : engine_len),
|
|
.ctrl_size (seq_dma_active ? 3'd2 : engine_size),
|
|
.ctrl_transfer_len (seq_dma_active ? op_bytes_left : engine_bytes),
|
|
.ctrl_done (mm2mm_done),
|
|
// AXI4 Port Connections
|
|
.mm2mm_awid (core_awid), .mm2mm_awaddr (core_awaddr),
|
|
.mm2mm_awlen (core_awlen), .mm2mm_awsize (core_awsize),
|
|
.mm2mm_awburst(core_awburst),.mm2mm_awlock (core_awlock),
|
|
.mm2mm_awcache(core_awcache),.mm2mm_awprot (core_awprot),
|
|
.mm2mm_awqos (core_awqos), .mm2mm_awuser (core_awuser),
|
|
.mm2mm_awvalid(core_awvalid),.mm2mm_awready(core_awready),
|
|
.mm2mm_wdata (core_wdata), .mm2mm_wstrb (core_wstrb),
|
|
.mm2mm_wlast (core_wlast), .mm2mm_wuser (core_wuser),
|
|
.mm2mm_wvalid (core_wvalid), .mm2mm_wready (core_wready),
|
|
.mm2mm_bid (mm2mm_bid), .mm2mm_bresp (mm2mm_bresp),
|
|
.mm2mm_buser (mm2mm_buser), .mm2mm_bvalid (mm2mm_bvalid),
|
|
.mm2mm_bready (core_bready),
|
|
.mm2mm_arid (core_arid), .mm2mm_araddr (core_araddr),
|
|
.mm2mm_arlen (core_arlen), .mm2mm_arsize (core_arsize),
|
|
.mm2mm_arburst(core_arburst),.mm2mm_arlock (core_arlock),
|
|
.mm2mm_arcache(core_arcache),.mm2mm_arprot (core_arprot),
|
|
.mm2mm_arqos (core_arqos), .mm2mm_aruser (core_aruser),
|
|
.mm2mm_arvalid(core_arvalid),.mm2mm_arready(core_arready),
|
|
.mm2mm_rid (mm2mm_rid), .mm2mm_rdata (mm2mm_rdata),
|
|
.mm2mm_rresp (mm2mm_rresp), .mm2mm_rlast (mm2mm_rlast),
|
|
.mm2mm_ruser (mm2mm_ruser), .mm2mm_rvalid (mm2mm_rvalid),
|
|
.mm2mm_rready (core_rready)
|
|
);
|
|
|
|
// ==========================================
|
|
// Hardware matmul loop sequencer
|
|
// ==========================================
|
|
wire seq_has_more_load = (seq_load_idx < seq_groups);
|
|
wire [31:0] seq_src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
|
|
wire [31:0] seq_dst_addr_for_load = seq_mat_base + (seq_load_idx[0] ? MAT_B1_AB_OFF : MAT_B0_AB_OFF);
|
|
wire [31:0] seq_src_addr_for_store = seq_mat_base + (seq_store_idx[0] ? MAT_B1_C_OFF : MAT_B0_C_OFF);
|
|
wire [31:0] seq_dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
|
|
wire [31:0] seq_start_value_for_bank = seq_compute_bank ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
|
|
|
|
assign seq_awid = '0;
|
|
assign seq_awsize = 3'd2;
|
|
assign seq_awburst = 2'b01;
|
|
assign seq_awlock = 1'b0;
|
|
assign seq_awcache = 4'b0;
|
|
assign seq_awprot = 3'b0;
|
|
assign seq_awqos = 4'b0;
|
|
assign seq_awuser = '0;
|
|
assign seq_wstrb = {{(DATA_WIDTH/8){1'b1}}};
|
|
assign seq_wuser = '0;
|
|
assign seq_arid = '0;
|
|
assign seq_arsize = 3'd2;
|
|
assign seq_arburst = 2'b01;
|
|
assign seq_arlock = 1'b0;
|
|
assign seq_arcache = 4'b0;
|
|
assign seq_arprot = 3'b0;
|
|
assign seq_arqos = 4'b0;
|
|
assign seq_aruser = '0;
|
|
|
|
assign seq_araddr = (op_state == OP_REG_READ_ADDR) ? op_read_addr : op_src_cur;
|
|
assign seq_arlen = (op_state == OP_REG_READ_ADDR) ? 8'd0 : 8'd0;
|
|
assign seq_arvalid = (op_state == OP_READ_ADDR) || (op_state == OP_REG_READ_ADDR);
|
|
assign seq_rready = (op_state == OP_REG_READ_DATA) ||
|
|
((op_state == OP_READ_DATA) && !op_buf_valid);
|
|
|
|
assign seq_awaddr = (op_state == OP_WRITE_ADDR && op_copy_store) ? op_dst_cur : op_write_addr;
|
|
assign seq_awlen = 8'd0;
|
|
assign seq_awvalid = (op_state == OP_WRITE_ADDR);
|
|
assign seq_wdata = op_copy_store ? op_data_buf : op_write_data;
|
|
assign seq_wlast = 1'b1;
|
|
assign seq_wvalid = (op_state == OP_WRITE_DATA) && (!op_copy_store || op_buf_valid);
|
|
assign seq_bready = (op_state == OP_WRITE_RESP);
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
op_state <= OP_IDLE;
|
|
op_done <= 1'b0;
|
|
op_src_cur <= '0;
|
|
op_dst_cur <= '0;
|
|
op_bytes_left <= '0;
|
|
op_data_buf <= '0;
|
|
op_buf_valid <= 1'b0;
|
|
op_copy_store <= 1'b0;
|
|
op_read_data <= '0;
|
|
end else begin
|
|
op_done <= 1'b0;
|
|
|
|
case (op_state)
|
|
OP_IDLE: begin
|
|
op_buf_valid <= 1'b0;
|
|
if (op_start_copy) begin
|
|
op_src_cur <= op_src_addr;
|
|
op_dst_cur <= op_dst_addr;
|
|
op_bytes_left <= op_bytes;
|
|
op_copy_store <= 1'b1;
|
|
op_state <= OP_COPY_START;
|
|
end else if (op_start_write) begin
|
|
op_copy_store <= 1'b0;
|
|
op_state <= OP_WRITE_ADDR;
|
|
end else if (op_start_read) begin
|
|
op_copy_store <= 1'b0;
|
|
op_state <= OP_REG_READ_ADDR;
|
|
end
|
|
end
|
|
|
|
OP_READ_ADDR: begin
|
|
if (seq_arvalid && seq_arready)
|
|
op_state <= OP_READ_DATA;
|
|
end
|
|
|
|
OP_READ_DATA: begin
|
|
if (mm2mm_rvalid && seq_rready) begin
|
|
op_data_buf <= mm2mm_rdata;
|
|
op_buf_valid <= 1'b1;
|
|
op_state <= OP_WRITE_ADDR;
|
|
end
|
|
end
|
|
|
|
OP_WRITE_ADDR: begin
|
|
if (seq_awvalid && seq_awready)
|
|
op_state <= OP_WRITE_DATA;
|
|
end
|
|
|
|
OP_WRITE_DATA: begin
|
|
if (seq_wvalid && seq_wready) begin
|
|
op_buf_valid <= 1'b0;
|
|
op_state <= OP_WRITE_RESP;
|
|
end
|
|
end
|
|
|
|
OP_WRITE_RESP: begin
|
|
if (mm2mm_bvalid && seq_bready) begin
|
|
if (op_copy_store && (op_bytes_left > 32'd4)) begin
|
|
op_src_cur <= op_src_cur + 32'd4;
|
|
op_dst_cur <= op_dst_cur + 32'd4;
|
|
op_bytes_left <= op_bytes_left - 32'd4;
|
|
op_state <= OP_READ_ADDR;
|
|
end else begin
|
|
op_done <= 1'b1;
|
|
op_state <= OP_IDLE;
|
|
end
|
|
end
|
|
end
|
|
|
|
OP_REG_READ_ADDR: begin
|
|
if (seq_arvalid && seq_arready)
|
|
op_state <= OP_REG_READ_DATA;
|
|
end
|
|
|
|
OP_REG_READ_DATA: begin
|
|
if (mm2mm_rvalid && seq_rready) begin
|
|
op_read_data <= mm2mm_rdata;
|
|
op_done <= 1'b1;
|
|
op_state <= OP_IDLE;
|
|
end
|
|
end
|
|
|
|
OP_COPY_START: begin
|
|
op_state <= OP_COPY_WAIT;
|
|
end
|
|
|
|
OP_COPY_WAIT: begin
|
|
if (mm2mm_done) begin
|
|
op_done <= 1'b1;
|
|
op_state <= OP_IDLE;
|
|
end
|
|
end
|
|
|
|
default: op_state <= OP_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
seq_state <= SQ_IDLE;
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b0;
|
|
op_start_copy <= 1'b0;
|
|
op_start_write <= 1'b0;
|
|
op_start_read <= 1'b0;
|
|
op_src_addr <= '0;
|
|
op_dst_addr <= '0;
|
|
op_bytes <= '0;
|
|
op_write_addr <= '0;
|
|
op_write_data <= '0;
|
|
op_read_addr <= '0;
|
|
seq_group_idx <= '0;
|
|
seq_load_idx <= '0;
|
|
seq_store_idx <= '0;
|
|
seq_compute_bank <= 1'b0;
|
|
seq_status_data <= '0;
|
|
end else begin
|
|
op_start_copy <= 1'b0;
|
|
op_start_write <= 1'b0;
|
|
op_start_read <= 1'b0;
|
|
|
|
if (seq_done_clear_req)
|
|
seq_done_flag <= 1'b0;
|
|
|
|
case (seq_state)
|
|
SQ_IDLE: begin
|
|
seq_busy <= 1'b0;
|
|
if (seq_start_req && (seq_groups != 32'd0)) begin
|
|
seq_busy <= 1'b1;
|
|
seq_done_flag <= 1'b0;
|
|
seq_group_idx <= 32'd0;
|
|
seq_load_idx <= 32'd0;
|
|
seq_store_idx <= 32'd0;
|
|
seq_compute_bank <= 1'b0;
|
|
op_write_addr <= seq_mat_base + MAT_CRC_OFF;
|
|
op_write_data <= 32'd0;
|
|
op_start_write <= 1'b1;
|
|
seq_state <= SQ_CLEAR_CRC;
|
|
end
|
|
end
|
|
|
|
SQ_CLEAR_CRC: begin
|
|
if (op_done) begin
|
|
op_src_addr <= seq_src_base;
|
|
op_dst_addr <= seq_mat_base + MAT_B0_AB_OFF;
|
|
op_bytes <= MAT_AB_BYTES;
|
|
op_start_copy <= 1'b1;
|
|
seq_load_idx <= 32'd1;
|
|
seq_state <= SQ_LOAD0;
|
|
end
|
|
end
|
|
|
|
SQ_LOAD0: begin
|
|
if (op_done) begin
|
|
op_write_addr <= seq_mat_base + MAT_CTRL_OFF;
|
|
op_write_data <= MAT_CTRL_START_B0;
|
|
op_start_write <= 1'b1;
|
|
seq_state <= SQ_START0;
|
|
end
|
|
end
|
|
|
|
SQ_START0: begin
|
|
if (op_done) begin
|
|
if (seq_load_idx < seq_groups) begin
|
|
op_src_addr <= seq_src_base + (seq_load_idx * seq_src_stride);
|
|
op_dst_addr <= seq_mat_base + MAT_B1_AB_OFF;
|
|
op_bytes <= MAT_AB_BYTES;
|
|
op_start_copy <= 1'b1;
|
|
seq_load_idx <= seq_load_idx + 32'd1;
|
|
seq_state <= SQ_LOAD1;
|
|
end else begin
|
|
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_start_read <= 1'b1;
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_LOAD1: begin
|
|
if (op_done) begin
|
|
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_start_read <= 1'b1;
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
|
|
SQ_WAIT_DONE: begin
|
|
if (op_done) begin
|
|
seq_status_data <= op_read_data;
|
|
if ((op_read_data & MAT_STATUS_DONE_BIT) != 32'd0) begin
|
|
seq_store_idx <= seq_group_idx;
|
|
if ((seq_group_idx + 32'd1) >= seq_groups) begin
|
|
op_src_addr <= seq_mat_base + (seq_compute_bank ? MAT_B1_C_OFF : MAT_B0_C_OFF);
|
|
op_dst_addr <= seq_dst_base + (seq_group_idx * seq_dst_stride);
|
|
op_bytes <= MAT_C_BYTES;
|
|
op_start_copy <= 1'b1;
|
|
seq_state <= SQ_STORE;
|
|
end else begin
|
|
op_write_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_write_data <= MAT_STATUS_DONE_CLR;
|
|
op_start_write <= 1'b1;
|
|
seq_state <= SQ_CLEAR_DONE;
|
|
end
|
|
end else begin
|
|
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_start_read <= 1'b1;
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_STORE: begin
|
|
if (op_done) begin
|
|
if ((seq_group_idx + 32'd1) >= seq_groups) begin
|
|
seq_state <= SQ_DONE;
|
|
end else if (seq_has_more_load) begin
|
|
op_src_addr <= seq_src_addr_for_load;
|
|
op_dst_addr <= seq_dst_addr_for_load;
|
|
op_bytes <= MAT_AB_BYTES;
|
|
op_start_copy <= 1'b1;
|
|
seq_load_idx <= seq_load_idx + 32'd1;
|
|
seq_state <= SQ_LOAD_NEXT;
|
|
end else begin
|
|
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_start_read <= 1'b1;
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
end
|
|
|
|
SQ_CLEAR_DONE: begin
|
|
if (op_done) begin
|
|
seq_group_idx <= seq_group_idx + 32'd1;
|
|
seq_compute_bank <= ~seq_compute_bank;
|
|
op_write_addr <= seq_mat_base + MAT_CTRL_OFF;
|
|
op_write_data <= (~seq_compute_bank) ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
|
|
op_start_write <= 1'b1;
|
|
seq_state <= SQ_START_NEXT;
|
|
end
|
|
end
|
|
|
|
SQ_START_NEXT: begin
|
|
if (op_done) begin
|
|
op_src_addr <= seq_src_addr_for_store;
|
|
op_dst_addr <= seq_dst_addr_for_store;
|
|
op_bytes <= MAT_C_BYTES;
|
|
op_start_copy <= 1'b1;
|
|
seq_state <= SQ_STORE;
|
|
end
|
|
end
|
|
|
|
SQ_LOAD_NEXT: begin
|
|
if (op_done) begin
|
|
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
|
|
op_start_read <= 1'b1;
|
|
seq_state <= SQ_WAIT_DONE;
|
|
end
|
|
end
|
|
|
|
SQ_DONE: begin
|
|
seq_busy <= 1'b0;
|
|
seq_done_flag <= 1'b1;
|
|
seq_state <= SQ_IDLE;
|
|
end
|
|
|
|
default: seq_state <= SQ_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
// ==========================================
|
|
// Endpoint transfer engine (32-bit only in this SoC integration)
|
|
// ==========================================
|
|
typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
|
|
ep_state_t ep_state;
|
|
logic [31:0] ep_axi_addr;
|
|
logic [31:0] ep_bytes_left;
|
|
logic [7:0] ep_word_addr_r;
|
|
|
|
wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
|
|
wire [7:0] ep_axi_len = ep_beats_total - 8'd1;
|
|
wire ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
|
|
wire ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
|
|
|
|
always_ff @(posedge clk) begin
|
|
if (!rst_n) begin
|
|
ep_state <= EP_IDLE;
|
|
ep_done <= 1'b0;
|
|
ep_axi_addr <= '0;
|
|
ep_bytes_left <= '0;
|
|
ep_word_addr_r <= '0;
|
|
end else begin
|
|
ep_done <= 1'b0;
|
|
|
|
case (ep_state)
|
|
EP_IDLE: begin
|
|
if (ep_start) begin
|
|
ep_axi_addr <= ep_mode_mem2ep ? engine_src : engine_dst;
|
|
ep_bytes_left <= engine_bytes;
|
|
ep_word_addr_r <= engine_ep_word_off;
|
|
ep_state <= ep_mode_mem2ep ? EP_AR : EP_AW;
|
|
end
|
|
end
|
|
|
|
EP_AR: begin
|
|
if (ep_arvalid && ep_arready)
|
|
ep_state <= EP_READ;
|
|
end
|
|
|
|
EP_READ: begin
|
|
if (mm2mm_rvalid && ep_rready) begin
|
|
ep_word_addr_r <= ep_word_addr_r + 8'd1;
|
|
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
|
|
if (mm2mm_rlast)
|
|
ep_state <= EP_DONE;
|
|
end
|
|
end
|
|
|
|
EP_AW: begin
|
|
if (ep_awvalid && ep_awready)
|
|
ep_state <= EP_WRITE;
|
|
end
|
|
|
|
EP_WRITE: begin
|
|
if (ep_wvalid && ep_wready) begin
|
|
ep_word_addr_r <= ep_word_addr_r + 8'd1;
|
|
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
|
|
if (ep_wlast)
|
|
ep_state <= EP_BRESP;
|
|
end
|
|
end
|
|
|
|
EP_BRESP: begin
|
|
if (mm2mm_bvalid && ep_bready)
|
|
ep_state <= EP_DONE;
|
|
end
|
|
|
|
EP_DONE: begin
|
|
ep_done <= 1'b1;
|
|
ep_state <= EP_IDLE;
|
|
end
|
|
|
|
default: ep_state <= EP_IDLE;
|
|
endcase
|
|
end
|
|
end
|
|
|
|
assign ep_arid = '0;
|
|
assign ep_araddr = ep_axi_addr;
|
|
assign ep_arlen = ep_axi_len;
|
|
assign ep_arsize = 3'd2;
|
|
assign ep_arburst = 2'b01;
|
|
assign ep_arlock = 1'b0;
|
|
assign ep_arcache = 4'b0;
|
|
assign ep_arprot = 3'b0;
|
|
assign ep_arqos = 4'b0;
|
|
assign ep_aruser = '0;
|
|
assign ep_arvalid = (ep_state == EP_AR);
|
|
assign ep_rready = (ep_state == EP_READ) && ep_wr_ready;
|
|
|
|
assign ep_wr_valid = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep;
|
|
assign ep_wr_id = engine_ep_id;
|
|
assign ep_wr_bank = engine_ep_bank;
|
|
assign ep_wr_word_addr = ep_word_addr_r;
|
|
assign ep_wr_data = mm2mm_rdata[31:0];
|
|
|
|
assign ep_awid = '0;
|
|
assign ep_awaddr = ep_axi_addr;
|
|
assign ep_awlen = ep_axi_len;
|
|
assign ep_awsize = 3'd2;
|
|
assign ep_awburst = 2'b01;
|
|
assign ep_awlock = 1'b0;
|
|
assign ep_awcache = 4'b0;
|
|
assign ep_awprot = 3'b0;
|
|
assign ep_awqos = 4'b0;
|
|
assign ep_awuser = '0;
|
|
assign ep_awvalid = (ep_state == EP_AW);
|
|
assign ep_wdata = ep_rd_data;
|
|
assign ep_wstrb = {{(DATA_WIDTH/8){1'b1}}};
|
|
assign ep_wlast = (ep_bytes_left <= 32'd4) && ep_wvalid;
|
|
assign ep_wuser = '0;
|
|
assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready;
|
|
assign ep_bready = (ep_state == EP_BRESP);
|
|
|
|
assign ep_rd_valid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready;
|
|
assign ep_rd_id = engine_ep_id;
|
|
assign ep_rd_bank = engine_ep_bank;
|
|
assign ep_rd_word_addr = ep_word_addr_r;
|
|
|
|
endmodule
|