fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add direct CDMA matmul path

Browse Source
This commit is contained in:
FallenSigh
2026-07-04 19:15:34 +08:00
parent 3f7c9b7a37
commit b7553418fa
9 changed files with 966 additions and 10112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

570
rtl/ip/cdma/snix_axil_cdma_mux.sv
View File

@@ -93,6 +93,32 @@ module snix_axil_cdma_mux #(
input logic [USER_WIDTH-1:0] mm2mm_ruser,
input logic mm2mm_rvalid,
output logic mm2mm_rready,
// ==========================================
// Local endpoint interface
// ==========================================
output logic ep_wr_valid,
output logic [3:0] ep_wr_id,
output logic [3:0] ep_wr_bank,
output logic [7:0] ep_wr_word_addr,
output logic [31:0] ep_wr_data,
input logic ep_wr_ready,
output logic ep_rd_valid,
output logic [3:0] ep_rd_id,
output logic [3:0] ep_rd_bank,
output logic [7:0] ep_rd_word_addr,
input logic [31:0] ep_rd_data,
input logic ep_rd_ready,
// ==========================================
// Matmul direct control sideband
// ==========================================
output logic mat_start_valid,
output logic mat_start_bank,
output logic mat_done_clear,
output logic mat_crc_clear,
input logic mat_done,
// Global Interrupt (OR'd from all channels)
output logic dma_finish
@@ -102,6 +128,29 @@ module snix_axil_cdma_mux #(
// Local Parameters & Utilities
// ==========================================
localparam int CH_BITS = $clog2(PORTS);
localparam logic [1:0] MODE_MEM2MEM = 2'd0;
localparam logic [1:0] MODE_MEM2EP = 2'd1;
localparam logic [1:0] MODE_EP2MEM = 2'd2;
localparam logic [7:0] SEQ_REG_CTRL = 8'h00;
localparam logic [7:0] SEQ_REG_STATUS = 8'h04;
localparam logic [7:0] SEQ_REG_SRC_BASE = 8'h08;
localparam logic [7:0] SEQ_REG_DST_BASE = 8'h0C;
localparam logic [7:0] SEQ_REG_GROUPS = 8'h10;
localparam logic [7:0] SEQ_REG_MAT_BASE = 8'h14;
localparam logic [7:0] SEQ_REG_SRC_STRIDE = 8'h18;
localparam logic [7:0] SEQ_REG_DST_STRIDE = 8'h1C;
localparam logic [11:0] MAT_CTRL_OFF = 12'h000;
localparam logic [11:0] MAT_STATUS_OFF = 12'h004;
localparam logic [11:0] MAT_CRC_OFF = 12'h010;
localparam logic [11:0] MAT_B0_AB_OFF = 12'h100;
localparam logic [11:0] MAT_B0_C_OFF = 12'h180;
localparam logic [11:0] MAT_B1_AB_OFF = 12'h300;
localparam logic [11:0] MAT_B1_C_OFF = 12'h380;
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
localparam logic [31:0] MAT_C_BYTES = 32'd192;
// Function to safely apply WSTRB to 32-bit registers
function automatic logic [31:0] apply_wstrb(
@@ -125,12 +174,25 @@ module snix_axil_cdma_mux #(
logic [31:0] ch_len [PORTS];
logic [31:0] ch_tag [PORTS];
logic [31:0] ch_ctrl [PORTS];
logic [31:0] ch_mode [PORTS];
logic [31:0] ch_ep_cfg [PORTS];
logic [PORTS-1:0] ch_req; // Pending requests (Busy)
logic [PORTS-1:0] ch_done; // Completion flags
logic [PORTS-1:0] arb_set_done; // From Arbiter to CSR
logic [31:0] seq_src_base;
logic [31:0] seq_dst_base;
logic [31:0] seq_groups;
logic [31:0] seq_mat_base;
logic [31:0] seq_src_stride;
logic [31:0] seq_dst_stride;
logic seq_busy;
logic seq_done_flag;
wire seq_start_req;
wire seq_done_clear_req;
// ==========================================
// Address Decoding (0x40 offset per channel)
// ==========================================
@@ -138,6 +200,14 @@ module snix_axil_cdma_mux #(
wire [5:0] wr_reg = s_axil_awaddr[5:0];
wire [CH_BITS-1:0] rd_ch = s_axil_araddr[6 +: CH_BITS];
wire [5:0] rd_reg = s_axil_araddr[5:0];
wire wr_seq_reg = (s_axil_awaddr[11:8] == 4'h2);
wire rd_seq_reg = (s_axil_araddr[11:8] == 4'h2);
wire [7:0] wr_seq_off = s_axil_awaddr[7:0];
wire [7:0] rd_seq_off = s_axil_araddr[7:0];
assign seq_start_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_CTRL) &&
s_axil_wstrb[0] && s_axil_wdata[0];
assign seq_done_clear_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_STATUS) &&
s_axil_wstrb[0] && s_axil_wdata[1];
// ==========================================
// AXI-Lite Slave Logic (Robust Backpressure)
@@ -197,7 +267,19 @@ module snix_axil_cdma_mux #(
end else begin
if (do_read) begin
s_axil_rvalid <= 1'b1;
if (rd_ch < PORTS) begin
if (rd_seq_reg) begin
case (rd_seq_off)
SEQ_REG_CTRL: s_axil_rdata <= {31'd0, seq_start_req};
SEQ_REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
SEQ_REG_SRC_BASE: s_axil_rdata <= seq_src_base;
SEQ_REG_DST_BASE: s_axil_rdata <= seq_dst_base;
SEQ_REG_GROUPS: s_axil_rdata <= seq_groups;
SEQ_REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
SEQ_REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
SEQ_REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
default: s_axil_rdata <= 32'd0;
endcase
end else if (rd_ch < PORTS) begin
case (rd_reg)
6'h00: s_axil_rdata <= ch_src_addr[rd_ch];
6'h04: s_axil_rdata <= ch_dst_addr[rd_ch];
@@ -205,6 +287,8 @@ module snix_axil_cdma_mux #(
6'h0C: s_axil_rdata <= ch_tag[rd_ch];
6'h10: s_axil_rdata <= ch_ctrl[rd_ch];
6'h14: s_axil_rdata <= {30'd0, ch_done[rd_ch], ch_req[rd_ch]};
6'h18: s_axil_rdata <= ch_mode[rd_ch];
6'h1C: s_axil_rdata <= ch_ep_cfg[rd_ch];
default: s_axil_rdata <= 32'd0;
endcase
end else begin
@@ -219,15 +303,22 @@ module snix_axil_cdma_mux #(
// ==========================================
// Register File Write Logic
// ==========================================
assign dma_finish = |ch_done;
assign dma_finish = (|ch_done) | seq_done_flag;
always_ff @(posedge clk) begin
if (!rst_n) begin
ch_req <= '0;
ch_done <= '0;
seq_src_base <= 32'h0;
seq_dst_base <= 32'h0;
seq_groups <= 32'h0;
seq_mat_base <= 32'h1f50_0000;
seq_src_stride <= 32'd128;
seq_dst_stride <= 32'd192;
for (int i=0; i<PORTS; i++) begin
ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0;
ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0;
ch_mode[i] <= '0; ch_ep_cfg[i] <= '0;
end
end else begin
// 1. Process Hardware Status Updates (Highest Priority for Done flag)
@@ -239,12 +330,23 @@ module snix_axil_cdma_mux #(
end
// 2. Process CPU Writes
if (do_write && wr_ch < PORTS) begin
if (do_write && wr_seq_reg) begin
case (wr_seq_off)
SEQ_REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
SEQ_REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
SEQ_REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
endcase
end else if (do_write && wr_ch < PORTS) begin
case (wr_reg)
6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h08: if (!ch_req[wr_ch]) ch_len[wr_ch] <= apply_wstrb(ch_len[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h0C: if (!ch_req[wr_ch]) ch_tag[wr_ch] <= apply_wstrb(ch_tag[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h18: if (!ch_req[wr_ch]) ch_mode[wr_ch] <= apply_wstrb(ch_mode[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h1C: if (!ch_req[wr_ch]) ch_ep_cfg[wr_ch] <= apply_wstrb(ch_ep_cfg[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h10: begin
ch_ctrl[wr_ch] <= apply_wstrb(ch_ctrl[wr_ch], s_axil_wdata, s_axil_wstrb);
// Trigger Bit Processing
@@ -269,7 +371,7 @@ module snix_axil_cdma_mux #(
// ==========================================
// Round-Robin Arbiter & Engine Driver
// ==========================================
typedef enum logic [1:0] {IDLE, RUN} state_t;
typedef enum logic [1:0] {IDLE, START, RUN} state_t;
state_t state;
logic [CH_BITS-1:0] cur_ch;
@@ -283,13 +385,165 @@ module snix_axil_cdma_mux #(
logic [7:0] engine_len;
logic [2:0] engine_size;
logic engine_done;
logic [1:0] engine_mode;
logic [3:0] engine_ep_id;
logic [3:0] engine_ep_bank;
logic [7:0] engine_ep_word_off;
logic mm2mm_start;
logic mm2mm_done;
logic ep_start;
logic ep_done;
logic [ID_WIDTH-1:0] core_awid;
logic [ADDR_WIDTH-1:0] core_awaddr;
logic [7:0] core_awlen;
logic [2:0] core_awsize;
logic [1:0] core_awburst;
logic core_awlock;
logic [3:0] core_awcache;
logic [2:0] core_awprot;
logic [3:0] core_awqos;
logic [USER_WIDTH-1:0] core_awuser;
logic core_awvalid;
logic core_awready;
logic [DATA_WIDTH-1:0] core_wdata;
logic [DATA_WIDTH/8-1:0] core_wstrb;
logic core_wlast;
logic [USER_WIDTH-1:0] core_wuser;
logic core_wvalid;
logic core_wready;
logic core_bready;
logic [ID_WIDTH-1:0] core_arid;
logic [ADDR_WIDTH-1:0] core_araddr;
logic [7:0] core_arlen;
logic [2:0] core_arsize;
logic [1:0] core_arburst;
logic core_arlock;
logic [3:0] core_arcache;
logic [2:0] core_arprot;
logic [3:0] core_arqos;
logic [USER_WIDTH-1:0] core_aruser;
logic core_arvalid;
logic core_arready;
logic core_rready;
logic [ID_WIDTH-1:0] ep_awid;
logic [ADDR_WIDTH-1:0] ep_awaddr;
logic [7:0] ep_awlen;
logic [2:0] ep_awsize;
logic [1:0] ep_awburst;
logic ep_awlock;
logic [3:0] ep_awcache;
logic [2:0] ep_awprot;
logic [3:0] ep_awqos;
logic [USER_WIDTH-1:0] ep_awuser;
logic ep_awvalid;
logic ep_awready;
logic [DATA_WIDTH-1:0] ep_wdata;
logic [DATA_WIDTH/8-1:0] ep_wstrb;
logic ep_wlast;
logic [USER_WIDTH-1:0] ep_wuser;
logic ep_wvalid;
logic ep_wready;
logic ep_bready;
logic [ID_WIDTH-1:0] ep_arid;
logic [ADDR_WIDTH-1:0] ep_araddr;
logic [7:0] ep_arlen;
logic [2:0] ep_arsize;
logic [1:0] ep_arburst;
logic ep_arlock;
logic [3:0] ep_arcache;
logic [2:0] ep_arprot;
logic [3:0] ep_arqos;
logic [USER_WIDTH-1:0] ep_aruser;
logic ep_arvalid;
logic ep_arready;
logic ep_rready;
// Hardware matmul loop sequencer state. The sequencer uses the endpoint
// path below, so matmul register/memory traffic does not enter the AXI bus.
typedef enum logic [4:0] {
SQ_IDLE,
SQ_CLEAR,
SQ_LOAD,
SQ_LOAD_WAIT,
SQ_WAIT_DONE,
SQ_STORE_START,
SQ_STORE,
SQ_DONE
} seq_state_t;
seq_state_t seq_state;
logic [31:0] seq_group_idx;
logic [31:0] seq_load_idx;
logic [31:0] seq_store_idx;
logic seq_compute_bank;
logic seq_ep_owner;
logic seq_ep_start;
logic [1:0] seq_ep_mode;
logic [31:0] seq_ep_src;
logic [31:0] seq_ep_dst;
logic [31:0] seq_ep_bytes;
logic seq_ep_bank;
wire use_seq_engine = 1'b0;
wire use_ep_engine = ((state == RUN) && (engine_mode != MODE_MEM2MEM)) || seq_ep_owner;
wire select_seq_ep = seq_ep_owner || seq_ep_start;
// Dynamic routing to the engine based on current active channel
assign engine_src = ch_src_addr[cur_ch];
assign engine_dst = ch_dst_addr[cur_ch];
assign engine_bytes = ch_len[cur_ch];
assign engine_src = select_seq_ep ? seq_ep_src : ch_src_addr[cur_ch];
assign engine_dst = select_seq_ep ? seq_ep_dst : ch_dst_addr[cur_ch];
assign engine_bytes = select_seq_ep ? seq_ep_bytes : ch_len[cur_ch];
assign engine_len = ch_ctrl[cur_ch][13:6];
assign engine_size = ch_ctrl[cur_ch][5:3];
assign engine_mode = select_seq_ep ? seq_ep_mode : ch_mode[cur_ch][1:0];
assign engine_ep_bank = select_seq_ep ? {3'b000, seq_ep_bank} : ch_ep_cfg[cur_ch][3:0];
assign engine_ep_word_off = select_seq_ep ? 8'd0 : ch_ep_cfg[cur_ch][15:8];
assign engine_ep_id = select_seq_ep ? 4'd0 : ch_ep_cfg[cur_ch][19:16];
assign engine_done = (engine_mode == MODE_MEM2MEM) ? mm2mm_done : ep_done;
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
assign ep_start = seq_ep_start || (engine_start && (engine_mode != MODE_MEM2MEM));
assign core_awready = (!use_seq_engine && !use_ep_engine) && mm2mm_awready;
assign core_wready = (!use_seq_engine && !use_ep_engine) && mm2mm_wready;
assign core_arready = (!use_seq_engine && !use_ep_engine) && mm2mm_arready;
assign ep_awready = (!use_seq_engine && use_ep_engine) && mm2mm_awready;
assign ep_wready = (!use_seq_engine && use_ep_engine) && mm2mm_wready;
assign ep_arready = (!use_seq_engine && use_ep_engine) && mm2mm_arready;
assign mm2mm_awid = use_ep_engine ? ep_awid : core_awid;
assign mm2mm_awaddr = use_ep_engine ? ep_awaddr : core_awaddr;
assign mm2mm_awlen = use_ep_engine ? ep_awlen : core_awlen;
assign mm2mm_awsize = use_ep_engine ? ep_awsize : core_awsize;
assign mm2mm_awburst = use_ep_engine ? ep_awburst : core_awburst;
assign mm2mm_awlock = use_ep_engine ? ep_awlock : core_awlock;
assign mm2mm_awcache = use_ep_engine ? ep_awcache : core_awcache;
assign mm2mm_awprot = use_ep_engine ? ep_awprot : core_awprot;
assign mm2mm_awqos = use_ep_engine ? ep_awqos : core_awqos;
assign mm2mm_awuser = use_ep_engine ? ep_awuser : core_awuser;
assign mm2mm_awvalid = use_ep_engine ? ep_awvalid : core_awvalid;
assign mm2mm_wdata = use_ep_engine ? ep_wdata : core_wdata;
assign mm2mm_wstrb = use_ep_engine ? ep_wstrb : core_wstrb;
assign mm2mm_wlast = use_ep_engine ? ep_wlast : core_wlast;
assign mm2mm_wuser = use_ep_engine ? ep_wuser : core_wuser;
assign mm2mm_wvalid = use_ep_engine ? ep_wvalid : core_wvalid;
assign mm2mm_bready = use_ep_engine ? ep_bready : core_bready;
assign mm2mm_arid = use_ep_engine ? ep_arid : core_arid;
assign mm2mm_araddr = use_ep_engine ? ep_araddr : core_araddr;
assign mm2mm_arlen = use_ep_engine ? ep_arlen : core_arlen;
assign mm2mm_arsize = use_ep_engine ? ep_arsize : core_arsize;
assign mm2mm_arburst = use_ep_engine ? ep_arburst : core_arburst;
assign mm2mm_arlock = use_ep_engine ? ep_arlock : core_arlock;
assign mm2mm_arcache = use_ep_engine ? ep_arcache : core_arcache;
assign mm2mm_arprot = use_ep_engine ? ep_arprot : core_arprot;
assign mm2mm_arqos = use_ep_engine ? ep_arqos : core_arqos;
assign mm2mm_aruser = use_ep_engine ? ep_aruser : core_aruser;
assign mm2mm_arvalid = use_ep_engine ? ep_arvalid : core_arvalid;
assign mm2mm_rready = use_ep_engine ? ep_rready : core_rready;
always_ff @(posedge clk) begin
if (!rst_n) begin
@@ -304,7 +558,9 @@ module snix_axil_cdma_mux #(
case (state)
IDLE: begin
for (int i = 0; i < PORTS; i++) begin
if (seq_busy) begin
state <= IDLE;
end else for (int i = 0; i < PORTS; i++) begin
logic [CH_BITS:0] check_ch_ext;
logic [CH_BITS-1:0] check_ch;
@@ -315,13 +571,17 @@ module snix_axil_cdma_mux #(
if (ch_req[check_ch] && !arb_set_done[check_ch]) begin
cur_ch <= check_ch;
rr_ptr <= (check_ch == (PORTS - 1)) ? '0 : (check_ch + 1);
engine_start <= 1'b1;
state <= RUN;
state <= START;
break;
end
end
end
START: begin
engine_start <= 1'b1;
state <= RUN;
end
RUN: begin
if (engine_done) begin
arb_set_done[cur_ch] <= 1'b1;
@@ -345,37 +605,287 @@ module snix_axil_cdma_mux #(
) u_core_engine (
.clk (clk),
.rst_n (rst_n),
.ctrl_start (engine_start),
.ctrl_start (mm2mm_start),
.ctrl_stop (1'b0), // Tied off; can be wired if global abort is needed
.ctrl_src_addr (engine_src),
.ctrl_dst_addr (engine_dst),
.ctrl_len (engine_len),
.ctrl_size (engine_size),
.ctrl_transfer_len (engine_bytes),
.ctrl_done (engine_done),
.ctrl_done (mm2mm_done),
// AXI4 Port Connections
.mm2mm_awid (mm2mm_awid), .mm2mm_awaddr (mm2mm_awaddr),
.mm2mm_awlen (mm2mm_awlen), .mm2mm_awsize (mm2mm_awsize),
.mm2mm_awburst(mm2mm_awburst),.mm2mm_awlock (mm2mm_awlock),
.mm2mm_awcache(mm2mm_awcache),.mm2mm_awprot (mm2mm_awprot),
.mm2mm_awqos (mm2mm_awqos), .mm2mm_awuser (mm2mm_awuser),
.mm2mm_awvalid(mm2mm_awvalid),.mm2mm_awready(mm2mm_awready),
.mm2mm_wdata (mm2mm_wdata), .mm2mm_wstrb (mm2mm_wstrb),
.mm2mm_wlast (mm2mm_wlast), .mm2mm_wuser (mm2mm_wuser),
.mm2mm_wvalid (mm2mm_wvalid), .mm2mm_wready (mm2mm_wready),
.mm2mm_awid (core_awid), .mm2mm_awaddr (core_awaddr),
.mm2mm_awlen (core_awlen), .mm2mm_awsize (core_awsize),
.mm2mm_awburst(core_awburst),.mm2mm_awlock (core_awlock),
.mm2mm_awcache(core_awcache),.mm2mm_awprot (core_awprot),
.mm2mm_awqos (core_awqos), .mm2mm_awuser (core_awuser),
.mm2mm_awvalid(core_awvalid),.mm2mm_awready(core_awready),
.mm2mm_wdata (core_wdata), .mm2mm_wstrb (core_wstrb),
.mm2mm_wlast (core_wlast), .mm2mm_wuser (core_wuser),
.mm2mm_wvalid (core_wvalid), .mm2mm_wready (core_wready),
.mm2mm_bid (mm2mm_bid), .mm2mm_bresp (mm2mm_bresp),
.mm2mm_buser (mm2mm_buser), .mm2mm_bvalid (mm2mm_bvalid),
.mm2mm_bready (mm2mm_bready),
.mm2mm_arid (mm2mm_arid), .mm2mm_araddr (mm2mm_araddr),
.mm2mm_arlen (mm2mm_arlen), .mm2mm_arsize (mm2mm_arsize),
.mm2mm_arburst(mm2mm_arburst),.mm2mm_arlock (mm2mm_arlock),
.mm2mm_arcache(mm2mm_arcache),.mm2mm_arprot (mm2mm_arprot),
.mm2mm_arqos (mm2mm_arqos), .mm2mm_aruser (mm2mm_aruser),
.mm2mm_arvalid(mm2mm_arvalid),.mm2mm_arready(mm2mm_arready),
.mm2mm_bready (core_bready),
.mm2mm_arid (core_arid), .mm2mm_araddr (core_araddr),
.mm2mm_arlen (core_arlen), .mm2mm_arsize (core_arsize),
.mm2mm_arburst(core_arburst),.mm2mm_arlock (core_arlock),
.mm2mm_arcache(core_arcache),.mm2mm_arprot (core_arprot),
.mm2mm_arqos (core_arqos), .mm2mm_aruser (core_aruser),
.mm2mm_arvalid(core_arvalid),.mm2mm_arready(core_arready),
.mm2mm_rid (mm2mm_rid), .mm2mm_rdata (mm2mm_rdata),
.mm2mm_rresp (mm2mm_rresp), .mm2mm_rlast (mm2mm_rlast),
.mm2mm_ruser (mm2mm_ruser), .mm2mm_rvalid (mm2mm_rvalid),
.mm2mm_rready (mm2mm_rready)
.mm2mm_rready (core_rready)
);
endmodule
// ==========================================
// Direct matmul endpoint sequencer
// ==========================================
wire [31:0] seq_src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
wire [31:0] seq_dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
wire seq_loaded_bank = seq_load_idx[0];
always_ff @(posedge clk) begin
if (!rst_n) begin
seq_state <= SQ_IDLE;
seq_busy <= 1'b0;
seq_done_flag <= 1'b0;
seq_group_idx <= '0;
seq_load_idx <= '0;
seq_store_idx <= '0;
seq_compute_bank <= 1'b0;
seq_ep_owner <= 1'b0;
seq_ep_start <= 1'b0;
seq_ep_mode <= MODE_MEM2EP;
seq_ep_src <= '0;
seq_ep_dst <= '0;
seq_ep_bytes <= '0;
seq_ep_bank <= 1'b0;
mat_start_valid <= 1'b0;
mat_start_bank <= 1'b0;
mat_done_clear <= 1'b0;
mat_crc_clear <= 1'b0;
end else begin
seq_ep_start <= 1'b0;
mat_start_valid <= 1'b0;
mat_done_clear <= 1'b0;
mat_crc_clear <= 1'b0;
if (seq_ep_start)
seq_ep_owner <= 1'b1;
else if (seq_ep_owner && ep_done)
seq_ep_owner <= 1'b0;
if (seq_done_clear_req)
seq_done_flag <= 1'b0;
case (seq_state)
SQ_IDLE: begin
seq_busy <= 1'b0;
if (seq_start_req && (seq_groups != 32'd0)) begin
seq_busy <= 1'b1;
seq_done_flag <= 1'b0;
seq_group_idx <= 32'd0;
seq_load_idx <= 32'd0;
seq_store_idx <= 32'd0;
seq_compute_bank <= 1'b0;
mat_crc_clear <= 1'b1;
mat_done_clear <= 1'b1;
seq_state <= SQ_CLEAR;
end
end
SQ_CLEAR: begin
seq_state <= SQ_LOAD;
end
SQ_LOAD: begin
if (!seq_ep_owner && !seq_ep_start) begin
seq_ep_mode <= MODE_MEM2EP;
seq_ep_src <= seq_src_addr_for_load;
seq_ep_dst <= '0;
seq_ep_bytes <= MAT_AB_BYTES;
seq_ep_bank <= seq_loaded_bank;
seq_ep_start <= 1'b1;
seq_state <= SQ_LOAD_WAIT;
end
end
SQ_LOAD_WAIT: begin
if (seq_ep_owner && ep_done) begin
mat_start_valid <= 1'b1;
mat_start_bank <= seq_ep_bank;
seq_load_idx <= seq_load_idx + 32'd1;
if ((seq_load_idx + 32'd1) < seq_groups && (seq_load_idx == 32'd0)) begin
seq_state <= SQ_LOAD;
end else begin
seq_state <= SQ_WAIT_DONE;
end
end
end
SQ_WAIT_DONE: begin
if (mat_done) begin
mat_done_clear <= 1'b1;
seq_state <= SQ_STORE_START;
end
end
SQ_STORE_START: begin
if (!seq_ep_owner && !seq_ep_start) begin
seq_ep_mode <= MODE_EP2MEM;
seq_ep_src <= '0;
seq_ep_dst <= seq_dst_addr_for_store;
seq_ep_bytes <= MAT_C_BYTES;
seq_ep_bank <= seq_compute_bank;
seq_ep_start <= 1'b1;
seq_state <= SQ_STORE;
end
end
SQ_STORE: begin
if (seq_ep_owner && ep_done) begin
seq_store_idx <= seq_store_idx + 32'd1;
seq_group_idx <= seq_group_idx + 32'd1;
seq_compute_bank <= ~seq_compute_bank;
if ((seq_store_idx + 32'd1) >= seq_groups) begin
seq_state <= SQ_DONE;
end else if (seq_load_idx < seq_groups) begin
seq_state <= SQ_LOAD;
end else begin
seq_state <= SQ_WAIT_DONE;
end
end
end
SQ_DONE: begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
seq_state <= SQ_IDLE;
end
default: seq_state <= SQ_IDLE;
endcase
end
end
// ==========================================
// Endpoint transfer engine (32-bit only in this SoC integration)
// ==========================================
typedef enum logic [2:0] {EP_IDLE, EP_AR, EP_READ, EP_AW, EP_WRITE, EP_BRESP, EP_DONE} ep_state_t;
ep_state_t ep_state;
logic [31:0] ep_axi_addr;
logic [31:0] ep_bytes_left;
logic [7:0] ep_word_addr_r;
wire [7:0] ep_beats_total = (engine_bytes[31:2] == 30'd0) ? 8'd1 : engine_bytes[9:2];
wire [7:0] ep_axi_len = ep_beats_total - 8'd1;
wire ep_mode_mem2ep = (engine_mode == MODE_MEM2EP);
wire ep_mode_ep2mem = (engine_mode == MODE_EP2MEM);
always_ff @(posedge clk) begin
if (!rst_n) begin
ep_state <= EP_IDLE;
ep_done <= 1'b0;
ep_axi_addr <= '0;
ep_bytes_left <= '0;
ep_word_addr_r <= '0;
end else begin
ep_done <= 1'b0;
case (ep_state)
EP_IDLE: begin
if (ep_start) begin
ep_axi_addr <= ep_mode_mem2ep ? engine_src : engine_dst;
ep_bytes_left <= engine_bytes;
ep_word_addr_r <= engine_ep_word_off;
ep_state <= ep_mode_mem2ep ? EP_AR : EP_AW;
end
end
EP_AR: begin
if (ep_arvalid && ep_arready)
ep_state <= EP_READ;
end
EP_READ: begin
if (mm2mm_rvalid && ep_rready) begin
ep_word_addr_r <= ep_word_addr_r + 8'd1;
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
if (mm2mm_rlast)
ep_state <= EP_DONE;
end
end
EP_AW: begin
if (ep_awvalid && ep_awready)
ep_state <= EP_WRITE;
end
EP_WRITE: begin
if (ep_wvalid && ep_wready) begin
ep_word_addr_r <= ep_word_addr_r + 8'd1;
ep_bytes_left <= (ep_bytes_left > 32'd4) ? (ep_bytes_left - 32'd4) : 32'd0;
if (ep_wlast)
ep_state <= EP_BRESP;
end
end
EP_BRESP: begin
if (mm2mm_bvalid && ep_bready)
ep_state <= EP_DONE;
end
EP_DONE: begin
ep_done <= 1'b1;
ep_state <= EP_IDLE;
end
default: ep_state <= EP_IDLE;
endcase
end
end
assign ep_arid = '0;
assign ep_araddr = ep_axi_addr;
assign ep_arlen = ep_axi_len;
assign ep_arsize = 3'd2;
assign ep_arburst = 2'b01;
assign ep_arlock = 1'b0;
assign ep_arcache = 4'b0;
assign ep_arprot = 3'b0;
assign ep_arqos = 4'b0;
assign ep_aruser = '0;
assign ep_arvalid = (ep_state == EP_AR);
assign ep_rready = (ep_state == EP_READ) && ep_wr_ready;
assign ep_wr_valid = (ep_state == EP_READ) && mm2mm_rvalid && ep_mode_mem2ep;
assign ep_wr_id = engine_ep_id;
assign ep_wr_bank = engine_ep_bank;
assign ep_wr_word_addr = ep_word_addr_r;
assign ep_wr_data = mm2mm_rdata[31:0];
assign ep_awid = '0;
assign ep_awaddr = ep_axi_addr;
assign ep_awlen = ep_axi_len;
assign ep_awsize = 3'd2;
assign ep_awburst = 2'b01;
assign ep_awlock = 1'b0;
assign ep_awcache = 4'b0;
assign ep_awprot = 3'b0;
assign ep_awqos = 4'b0;
assign ep_awuser = '0;
assign ep_awvalid = (ep_state == EP_AW);
assign ep_wdata = ep_rd_data;
assign ep_wstrb = {{(DATA_WIDTH/8){1'b1}}};
assign ep_wlast = (ep_bytes_left <= 32'd4) && ep_wvalid;
assign ep_wuser = '0;
assign ep_wvalid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_rd_ready;
assign ep_bready = (ep_state == EP_BRESP);
assign ep_rd_valid = (ep_state == EP_WRITE) && ep_mode_ep2mem && ep_wready;
assign ep_rd_id = engine_ep_id;
assign ep_rd_bank = engine_ep_bank;
assign ep_rd_word_addr = ep_word_addr_r;
endmodule