fallensigh/ciciec2026
Archived
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CDMA matmul sequencer

Browse Source
This commit is contained in:
FallenSigh
2026-07-01 21:59:57 +08:00
parent 2080a700e9
commit dd8d9c8f5f
4 changed files with 625 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

568
rtl/ip/cdma/snix_axil_cdma_mux.sv
View File

@@ -122,6 +122,30 @@ module snix_axil_cdma_mux #(
localparam logic [1:0] MODE_MEM2MEM = 2'd0; localparam logic [1:0] MODE_MEM2MEM = 2'd0;
localparam logic [1:0] MODE_MEM2EP = 2'd1; localparam logic [1:0] MODE_MEM2EP = 2'd1;
localparam logic [1:0] MODE_EP2MEM = 2'd2; localparam logic [1:0] MODE_EP2MEM = 2'd2;
localparam logic [7:0] SEQ_REG_CTRL = 8'h00;
localparam logic [7:0] SEQ_REG_STATUS = 8'h04;
localparam logic [7:0] SEQ_REG_SRC_BASE = 8'h08;
localparam logic [7:0] SEQ_REG_DST_BASE = 8'h0C;
localparam logic [7:0] SEQ_REG_GROUPS = 8'h10;
localparam logic [7:0] SEQ_REG_MAT_BASE = 8'h14;
localparam logic [7:0] SEQ_REG_SRC_STRIDE = 8'h18;
localparam logic [7:0] SEQ_REG_DST_STRIDE = 8'h1C;
localparam logic [11:0] MAT_CTRL_OFF = 12'h000;
localparam logic [11:0] MAT_STATUS_OFF = 12'h004;
localparam logic [11:0] MAT_CRC_OFF = 12'h010;
localparam logic [11:0] MAT_B0_AB_OFF = 12'h100;
localparam logic [11:0] MAT_B0_C_OFF = 12'h180;
localparam logic [11:0] MAT_B1_AB_OFF = 12'h300;
localparam logic [11:0] MAT_B1_C_OFF = 12'h380;
localparam logic [31:0] MAT_STATUS_DONE_CLR = 32'h0000_0006;
localparam logic [31:0] MAT_STATUS_DONE_BIT = 32'h0000_0002;
localparam logic [31:0] MAT_CTRL_START_B0 = 32'h0000_0001;
localparam logic [31:0] MAT_CTRL_START_B1 = 32'h0000_0002;
localparam logic [31:0] MAT_AB_BYTES = 32'd128;
localparam logic [31:0] MAT_C_BYTES = 32'd192;
// Function to safely apply WSTRB to 32-bit registers // Function to safely apply WSTRB to 32-bit registers
function automatic logic [31:0] apply_wstrb( function automatic logic [31:0] apply_wstrb(
@@ -153,6 +177,17 @@ module snix_axil_cdma_mux #(
logic [PORTS-1:0] arb_set_done; // From Arbiter to CSR logic [PORTS-1:0] arb_set_done; // From Arbiter to CSR
logic [31:0] seq_src_base;
logic [31:0] seq_dst_base;
logic [31:0] seq_groups;
logic [31:0] seq_mat_base;
logic [31:0] seq_src_stride;
logic [31:0] seq_dst_stride;
logic seq_busy;
logic seq_done_flag;
wire seq_start_req;
wire seq_done_clear_req;
// ========================================== // ==========================================
// Address Decoding (0x40 offset per channel) // Address Decoding (0x40 offset per channel)
// ========================================== // ==========================================
@@ -160,6 +195,14 @@ module snix_axil_cdma_mux #(
wire [5:0] wr_reg = s_axil_awaddr[5:0]; wire [5:0] wr_reg = s_axil_awaddr[5:0];
wire [CH_BITS-1:0] rd_ch = s_axil_araddr[6 +: CH_BITS]; wire [CH_BITS-1:0] rd_ch = s_axil_araddr[6 +: CH_BITS];
wire [5:0] rd_reg = s_axil_araddr[5:0]; wire [5:0] rd_reg = s_axil_araddr[5:0];
wire wr_seq_reg = (s_axil_awaddr[11:8] == 4'h2);
wire rd_seq_reg = (s_axil_araddr[11:8] == 4'h2);
wire [7:0] wr_seq_off = s_axil_awaddr[7:0];
wire [7:0] rd_seq_off = s_axil_araddr[7:0];
assign seq_start_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_CTRL) &&
s_axil_wstrb[0] && s_axil_wdata[0];
assign seq_done_clear_req = do_write && wr_seq_reg && (wr_seq_off == SEQ_REG_STATUS) &&
s_axil_wstrb[0] && s_axil_wdata[1];
// ========================================== // ==========================================
// AXI-Lite Slave Logic (Robust Backpressure) // AXI-Lite Slave Logic (Robust Backpressure)
@@ -219,7 +262,19 @@ module snix_axil_cdma_mux #(
end else begin end else begin
if (do_read) begin if (do_read) begin
s_axil_rvalid <= 1'b1; s_axil_rvalid <= 1'b1;
if (rd_ch < PORTS) begin if (rd_seq_reg) begin
case (rd_seq_off)
SEQ_REG_CTRL: s_axil_rdata <= {31'd0, seq_start_req};
SEQ_REG_STATUS: s_axil_rdata <= {30'd0, seq_done_flag, seq_busy};
SEQ_REG_SRC_BASE: s_axil_rdata <= seq_src_base;
SEQ_REG_DST_BASE: s_axil_rdata <= seq_dst_base;
SEQ_REG_GROUPS: s_axil_rdata <= seq_groups;
SEQ_REG_MAT_BASE: s_axil_rdata <= seq_mat_base;
SEQ_REG_SRC_STRIDE: s_axil_rdata <= seq_src_stride;
SEQ_REG_DST_STRIDE: s_axil_rdata <= seq_dst_stride;
default: s_axil_rdata <= 32'd0;
endcase
end else if (rd_ch < PORTS) begin
case (rd_reg) case (rd_reg)
6'h00: s_axil_rdata <= ch_src_addr[rd_ch]; 6'h00: s_axil_rdata <= ch_src_addr[rd_ch];
6'h04: s_axil_rdata <= ch_dst_addr[rd_ch]; 6'h04: s_axil_rdata <= ch_dst_addr[rd_ch];
@@ -249,6 +304,12 @@ module snix_axil_cdma_mux #(
if (!rst_n) begin if (!rst_n) begin
ch_req <= '0; ch_req <= '0;
ch_done <= '0; ch_done <= '0;
seq_src_base <= 32'h0;
seq_dst_base <= 32'h0;
seq_groups <= 32'h0;
seq_mat_base <= 32'h1f50_0000;
seq_src_stride <= 32'd128;
seq_dst_stride <= 32'd192;
for (int i=0; i<PORTS; i++) begin for (int i=0; i<PORTS; i++) begin
ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0; ch_src_addr[i] <= '0; ch_dst_addr[i] <= '0;
ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0; ch_len[i] <= '0; ch_tag[i] <= '0; ch_ctrl[i] <= '0;
@@ -264,7 +325,16 @@ module snix_axil_cdma_mux #(
end end
// 2. Process CPU Writes // 2. Process CPU Writes
if (do_write && wr_ch < PORTS) begin if (do_write && wr_seq_reg) begin
case (wr_seq_off)
SEQ_REG_SRC_BASE: if (!seq_busy) seq_src_base <= apply_wstrb(seq_src_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_DST_BASE: if (!seq_busy) seq_dst_base <= apply_wstrb(seq_dst_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_GROUPS: if (!seq_busy) seq_groups <= apply_wstrb(seq_groups, s_axil_wdata, s_axil_wstrb);
SEQ_REG_MAT_BASE: if (!seq_busy) seq_mat_base <= apply_wstrb(seq_mat_base, s_axil_wdata, s_axil_wstrb);
SEQ_REG_SRC_STRIDE: if (!seq_busy) seq_src_stride <= apply_wstrb(seq_src_stride, s_axil_wdata, s_axil_wstrb);
SEQ_REG_DST_STRIDE: if (!seq_busy) seq_dst_stride <= apply_wstrb(seq_dst_stride, s_axil_wdata, s_axil_wstrb);
endcase
end else if (do_write && wr_ch < PORTS) begin
case (wr_reg) case (wr_reg)
6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb); 6'h00: if (!ch_req[wr_ch]) ch_src_addr[wr_ch] <= apply_wstrb(ch_src_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb); 6'h04: if (!ch_req[wr_ch]) ch_dst_addr[wr_ch] <= apply_wstrb(ch_dst_addr[wr_ch], s_axil_wdata, s_axil_wstrb);
@@ -386,7 +456,43 @@ module snix_axil_cdma_mux #(
logic ep_arready; logic ep_arready;
logic ep_rready; logic ep_rready;
logic [ID_WIDTH-1:0] seq_awid;
logic [ADDR_WIDTH-1:0] seq_awaddr;
logic [7:0] seq_awlen;
logic [2:0] seq_awsize;
logic [1:0] seq_awburst;
logic seq_awlock;
logic [3:0] seq_awcache;
logic [2:0] seq_awprot;
logic [3:0] seq_awqos;
logic [USER_WIDTH-1:0] seq_awuser;
logic seq_awvalid;
logic seq_awready;
logic [DATA_WIDTH-1:0] seq_wdata;
logic [DATA_WIDTH/8-1:0] seq_wstrb;
logic seq_wlast;
logic [USER_WIDTH-1:0] seq_wuser;
logic seq_wvalid;
logic seq_wready;
logic seq_bready;
logic [ID_WIDTH-1:0] seq_arid;
logic [ADDR_WIDTH-1:0] seq_araddr;
logic [7:0] seq_arlen;
logic [2:0] seq_arsize;
logic [1:0] seq_arburst;
logic seq_arlock;
logic [3:0] seq_arcache;
logic [2:0] seq_arprot;
logic [3:0] seq_arqos;
logic [USER_WIDTH-1:0] seq_aruser;
logic seq_arvalid;
logic seq_arready;
logic seq_rready;
wire use_ep_engine = (state == RUN) && (engine_mode != MODE_MEM2MEM); wire use_ep_engine = (state == RUN) && (engine_mode != MODE_MEM2MEM);
wire use_seq_engine = seq_busy && (op_state != OP_IDLE) &&
(op_state != OP_COPY_START) && (op_state != OP_COPY_WAIT);
wire seq_dma_active = seq_busy && ((op_state == OP_COPY_START) || (op_state == OP_COPY_WAIT));
// Dynamic routing to the engine based on current active channel // Dynamic routing to the engine based on current active channel
assign engine_src = ch_src_addr[cur_ch]; assign engine_src = ch_src_addr[cur_ch];
@@ -402,44 +508,47 @@ module snix_axil_cdma_mux #(
assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM); assign mm2mm_start = engine_start && (engine_mode == MODE_MEM2MEM);
assign ep_start = engine_start && (engine_mode != MODE_MEM2MEM); assign ep_start = engine_start && (engine_mode != MODE_MEM2MEM);
assign core_awready = (!use_ep_engine) && mm2mm_awready; assign core_awready = (!use_seq_engine && !use_ep_engine) && mm2mm_awready;
assign core_wready = (!use_ep_engine) && mm2mm_wready; assign core_wready = (!use_seq_engine && !use_ep_engine) && mm2mm_wready;
assign core_arready = (!use_ep_engine) && mm2mm_arready; assign core_arready = (!use_seq_engine && !use_ep_engine) && mm2mm_arready;
assign ep_awready = use_ep_engine && mm2mm_awready; assign ep_awready = (!use_seq_engine && use_ep_engine) && mm2mm_awready;
assign ep_wready = use_ep_engine && mm2mm_wready; assign ep_wready = (!use_seq_engine && use_ep_engine) && mm2mm_wready;
assign ep_arready = use_ep_engine && mm2mm_arready; assign ep_arready = (!use_seq_engine && use_ep_engine) && mm2mm_arready;
assign seq_awready = use_seq_engine && mm2mm_awready;
assign seq_wready = use_seq_engine && mm2mm_wready;
assign seq_arready = use_seq_engine && mm2mm_arready;
assign mm2mm_awid = use_ep_engine ? ep_awid : core_awid; assign mm2mm_awid = use_seq_engine ? seq_awid : (use_ep_engine ? ep_awid : core_awid);
assign mm2mm_awaddr = use_ep_engine ? ep_awaddr : core_awaddr; assign mm2mm_awaddr = use_seq_engine ? seq_awaddr : (use_ep_engine ? ep_awaddr : core_awaddr);
assign mm2mm_awlen = use_ep_engine ? ep_awlen : core_awlen; assign mm2mm_awlen = use_seq_engine ? seq_awlen : (use_ep_engine ? ep_awlen : core_awlen);
assign mm2mm_awsize = use_ep_engine ? ep_awsize : core_awsize; assign mm2mm_awsize = use_seq_engine ? seq_awsize : (use_ep_engine ? ep_awsize : core_awsize);
assign mm2mm_awburst = use_ep_engine ? ep_awburst : core_awburst; assign mm2mm_awburst = use_seq_engine ? seq_awburst : (use_ep_engine ? ep_awburst : core_awburst);
assign mm2mm_awlock = use_ep_engine ? ep_awlock : core_awlock; assign mm2mm_awlock = use_seq_engine ? seq_awlock : (use_ep_engine ? ep_awlock : core_awlock);
assign mm2mm_awcache = use_ep_engine ? ep_awcache : core_awcache; assign mm2mm_awcache = use_seq_engine ? seq_awcache : (use_ep_engine ? ep_awcache : core_awcache);
assign mm2mm_awprot = use_ep_engine ? ep_awprot : core_awprot; assign mm2mm_awprot = use_seq_engine ? seq_awprot : (use_ep_engine ? ep_awprot : core_awprot);
assign mm2mm_awqos = use_ep_engine ? ep_awqos : core_awqos; assign mm2mm_awqos = use_seq_engine ? seq_awqos : (use_ep_engine ? ep_awqos : core_awqos);
assign mm2mm_awuser = use_ep_engine ? ep_awuser : core_awuser; assign mm2mm_awuser = use_seq_engine ? seq_awuser : (use_ep_engine ? ep_awuser : core_awuser);
assign mm2mm_awvalid = use_ep_engine ? ep_awvalid : core_awvalid; assign mm2mm_awvalid = use_seq_engine ? seq_awvalid : (use_ep_engine ? ep_awvalid : core_awvalid);
assign mm2mm_wdata = use_ep_engine ? ep_wdata : core_wdata; assign mm2mm_wdata = use_seq_engine ? seq_wdata : (use_ep_engine ? ep_wdata : core_wdata);
assign mm2mm_wstrb = use_ep_engine ? ep_wstrb : core_wstrb; assign mm2mm_wstrb = use_seq_engine ? seq_wstrb : (use_ep_engine ? ep_wstrb : core_wstrb);
assign mm2mm_wlast = use_ep_engine ? ep_wlast : core_wlast; assign mm2mm_wlast = use_seq_engine ? seq_wlast : (use_ep_engine ? ep_wlast : core_wlast);
assign mm2mm_wuser = use_ep_engine ? ep_wuser : core_wuser; assign mm2mm_wuser = use_seq_engine ? seq_wuser : (use_ep_engine ? ep_wuser : core_wuser);
assign mm2mm_wvalid = use_ep_engine ? ep_wvalid : core_wvalid; assign mm2mm_wvalid = use_seq_engine ? seq_wvalid : (use_ep_engine ? ep_wvalid : core_wvalid);
assign mm2mm_bready = use_ep_engine ? ep_bready : core_bready; assign mm2mm_bready = use_seq_engine ? seq_bready : (use_ep_engine ? ep_bready : core_bready);
assign mm2mm_arid = use_ep_engine ? ep_arid : core_arid; assign mm2mm_arid = use_seq_engine ? seq_arid : (use_ep_engine ? ep_arid : core_arid);
assign mm2mm_araddr = use_ep_engine ? ep_araddr : core_araddr; assign mm2mm_araddr = use_seq_engine ? seq_araddr : (use_ep_engine ? ep_araddr : core_araddr);
assign mm2mm_arlen = use_ep_engine ? ep_arlen : core_arlen; assign mm2mm_arlen = use_seq_engine ? seq_arlen : (use_ep_engine ? ep_arlen : core_arlen);
assign mm2mm_arsize = use_ep_engine ? ep_arsize : core_arsize; assign mm2mm_arsize = use_seq_engine ? seq_arsize : (use_ep_engine ? ep_arsize : core_arsize);
assign mm2mm_arburst = use_ep_engine ? ep_arburst : core_arburst; assign mm2mm_arburst = use_seq_engine ? seq_arburst : (use_ep_engine ? ep_arburst : core_arburst);
assign mm2mm_arlock = use_ep_engine ? ep_arlock : core_arlock; assign mm2mm_arlock = use_seq_engine ? seq_arlock : (use_ep_engine ? ep_arlock : core_arlock);
assign mm2mm_arcache = use_ep_engine ? ep_arcache : core_arcache; assign mm2mm_arcache = use_seq_engine ? seq_arcache : (use_ep_engine ? ep_arcache : core_arcache);
assign mm2mm_arprot = use_ep_engine ? ep_arprot : core_arprot; assign mm2mm_arprot = use_seq_engine ? seq_arprot : (use_ep_engine ? ep_arprot : core_arprot);
assign mm2mm_arqos = use_ep_engine ? ep_arqos : core_arqos; assign mm2mm_arqos = use_seq_engine ? seq_arqos : (use_ep_engine ? ep_arqos : core_arqos);
assign mm2mm_aruser = use_ep_engine ? ep_aruser : core_aruser; assign mm2mm_aruser = use_seq_engine ? seq_aruser : (use_ep_engine ? ep_aruser : core_aruser);
assign mm2mm_arvalid = use_ep_engine ? ep_arvalid : core_arvalid; assign mm2mm_arvalid = use_seq_engine ? seq_arvalid : (use_ep_engine ? ep_arvalid : core_arvalid);
assign mm2mm_rready = use_ep_engine ? ep_rready : core_rready; assign mm2mm_rready = use_seq_engine ? seq_rready : (use_ep_engine ? ep_rready : core_rready);
always_ff @(posedge clk) begin always_ff @(posedge clk) begin
if (!rst_n) begin if (!rst_n) begin
@@ -454,7 +563,9 @@ module snix_axil_cdma_mux #(
case (state) case (state)
IDLE: begin IDLE: begin
for (int i = 0; i < PORTS; i++) begin if (seq_busy) begin
state <= IDLE;
end else for (int i = 0; i < PORTS; i++) begin
logic [CH_BITS:0] check_ch_ext; logic [CH_BITS:0] check_ch_ext;
logic [CH_BITS-1:0] check_ch; logic [CH_BITS-1:0] check_ch;
@@ -499,13 +610,13 @@ module snix_axil_cdma_mux #(
) u_core_engine ( ) u_core_engine (
.clk (clk), .clk (clk),
.rst_n (rst_n), .rst_n (rst_n),
.ctrl_start (mm2mm_start), .ctrl_start ((op_state == OP_COPY_START) ? 1'b1 : mm2mm_start),
.ctrl_stop (1'b0), // Tied off; can be wired if global abort is needed .ctrl_stop (1'b0), // Tied off; can be wired if global abort is needed
.ctrl_src_addr (engine_src), .ctrl_src_addr (seq_dma_active ? op_src_cur : engine_src),
.ctrl_dst_addr (engine_dst), .ctrl_dst_addr (seq_dma_active ? op_dst_cur : engine_dst),
.ctrl_len (engine_len), .ctrl_len (seq_dma_active ? 8'd63 : engine_len),
.ctrl_size (engine_size), .ctrl_size (seq_dma_active ? 3'd2 : engine_size),
.ctrl_transfer_len (engine_bytes), .ctrl_transfer_len (seq_dma_active ? op_bytes_left : engine_bytes),
.ctrl_done (mm2mm_done), .ctrl_done (mm2mm_done),
// AXI4 Port Connections // AXI4 Port Connections
.mm2mm_awid (core_awid), .mm2mm_awaddr (core_awaddr), .mm2mm_awid (core_awid), .mm2mm_awaddr (core_awaddr),
@@ -532,6 +643,375 @@ module snix_axil_cdma_mux #(
.mm2mm_rready (core_rready) .mm2mm_rready (core_rready)
); );
// ==========================================
// Hardware matmul loop sequencer
// ==========================================
typedef enum logic [4:0] {
SQ_IDLE,
SQ_CLEAR_CRC,
SQ_LOAD0,
SQ_START0,
SQ_LOAD1,
SQ_WAIT_DONE,
SQ_STORE,
SQ_CLEAR_DONE,
SQ_START_NEXT,
SQ_LOAD_NEXT,
SQ_DONE
} seq_state_t;
typedef enum logic [3:0] {
OP_IDLE,
OP_READ_ADDR,
OP_READ_DATA,
OP_WRITE_ADDR,
OP_WRITE_DATA,
OP_WRITE_RESP,
OP_REG_READ_ADDR,
OP_REG_READ_DATA,
OP_COPY_START,
OP_COPY_WAIT
} op_state_t;
seq_state_t seq_state;
op_state_t op_state;
logic op_start_copy;
logic op_start_write;
logic op_start_read;
logic op_done;
logic [31:0] op_src_addr;
logic [31:0] op_dst_addr;
logic [31:0] op_bytes;
logic [31:0] op_write_addr;
logic [31:0] op_write_data;
logic [31:0] op_read_addr;
logic [31:0] op_read_data;
logic [31:0] op_src_cur;
logic [31:0] op_dst_cur;
logic [31:0] op_bytes_left;
logic [31:0] op_data_buf;
logic op_buf_valid;
logic op_copy_store;
logic [31:0] seq_group_idx;
logic [31:0] seq_load_idx;
logic [31:0] seq_store_idx;
logic seq_compute_bank;
logic [31:0] seq_status_data;
wire seq_has_more_load = (seq_load_idx < seq_groups);
wire [31:0] seq_src_addr_for_load = seq_src_base + (seq_load_idx * seq_src_stride);
wire [31:0] seq_dst_addr_for_load = seq_mat_base + (seq_load_idx[0] ? MAT_B1_AB_OFF : MAT_B0_AB_OFF);
wire [31:0] seq_src_addr_for_store = seq_mat_base + (seq_store_idx[0] ? MAT_B1_C_OFF : MAT_B0_C_OFF);
wire [31:0] seq_dst_addr_for_store = seq_dst_base + (seq_store_idx * seq_dst_stride);
wire [31:0] seq_start_value_for_bank = seq_compute_bank ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
assign seq_awid = '0;
assign seq_awsize = 3'd2;
assign seq_awburst = 2'b01;
assign seq_awlock = 1'b0;
assign seq_awcache = 4'b0;
assign seq_awprot = 3'b0;
assign seq_awqos = 4'b0;
assign seq_awuser = '0;
assign seq_wstrb = {{(DATA_WIDTH/8){1'b1}}};
assign seq_wuser = '0;
assign seq_arid = '0;
assign seq_arsize = 3'd2;
assign seq_arburst = 2'b01;
assign seq_arlock = 1'b0;
assign seq_arcache = 4'b0;
assign seq_arprot = 3'b0;
assign seq_arqos = 4'b0;
assign seq_aruser = '0;
assign seq_araddr = (op_state == OP_REG_READ_ADDR) ? op_read_addr : op_src_cur;
assign seq_arlen = (op_state == OP_REG_READ_ADDR) ? 8'd0 : 8'd0;
assign seq_arvalid = (op_state == OP_READ_ADDR) || (op_state == OP_REG_READ_ADDR);
assign seq_rready = (op_state == OP_REG_READ_DATA) ||
((op_state == OP_READ_DATA) && !op_buf_valid);
assign seq_awaddr = (op_state == OP_WRITE_ADDR && op_copy_store) ? op_dst_cur : op_write_addr;
assign seq_awlen = 8'd0;
assign seq_awvalid = (op_state == OP_WRITE_ADDR);
assign seq_wdata = op_copy_store ? op_data_buf : op_write_data;
assign seq_wlast = 1'b1;
assign seq_wvalid = (op_state == OP_WRITE_DATA) && (!op_copy_store || op_buf_valid);
assign seq_bready = (op_state == OP_WRITE_RESP);
always_ff @(posedge clk) begin
if (!rst_n) begin
op_state <= OP_IDLE;
op_done <= 1'b0;
op_src_cur <= '0;
op_dst_cur <= '0;
op_bytes_left <= '0;
op_data_buf <= '0;
op_buf_valid <= 1'b0;
op_copy_store <= 1'b0;
op_read_data <= '0;
end else begin
op_done <= 1'b0;
case (op_state)
OP_IDLE: begin
op_buf_valid <= 1'b0;
if (op_start_copy) begin
op_src_cur <= op_src_addr;
op_dst_cur <= op_dst_addr;
op_bytes_left <= op_bytes;
op_copy_store <= 1'b1;
op_state <= OP_COPY_START;
end else if (op_start_write) begin
op_copy_store <= 1'b0;
op_state <= OP_WRITE_ADDR;
end else if (op_start_read) begin
op_copy_store <= 1'b0;
op_state <= OP_REG_READ_ADDR;
end
end
OP_READ_ADDR: begin
if (seq_arvalid && seq_arready)
op_state <= OP_READ_DATA;
end
OP_READ_DATA: begin
if (mm2mm_rvalid && seq_rready) begin
op_data_buf <= mm2mm_rdata;
op_buf_valid <= 1'b1;
op_state <= OP_WRITE_ADDR;
end
end
OP_WRITE_ADDR: begin
if (seq_awvalid && seq_awready)
op_state <= OP_WRITE_DATA;
end
OP_WRITE_DATA: begin
if (seq_wvalid && seq_wready) begin
op_buf_valid <= 1'b0;
op_state <= OP_WRITE_RESP;
end
end
OP_WRITE_RESP: begin
if (mm2mm_bvalid && seq_bready) begin
if (op_copy_store && (op_bytes_left > 32'd4)) begin
op_src_cur <= op_src_cur + 32'd4;
op_dst_cur <= op_dst_cur + 32'd4;
op_bytes_left <= op_bytes_left - 32'd4;
op_state <= OP_READ_ADDR;
end else begin
op_done <= 1'b1;
op_state <= OP_IDLE;
end
end
end
OP_REG_READ_ADDR: begin
if (seq_arvalid && seq_arready)
op_state <= OP_REG_READ_DATA;
end
OP_REG_READ_DATA: begin
if (mm2mm_rvalid && seq_rready) begin
op_read_data <= mm2mm_rdata;
op_done <= 1'b1;
op_state <= OP_IDLE;
end
end
OP_COPY_START: begin
op_state <= OP_COPY_WAIT;
end
OP_COPY_WAIT: begin
if (mm2mm_done) begin
op_done <= 1'b1;
op_state <= OP_IDLE;
end
end
default: op_state <= OP_IDLE;
endcase
end
end
always_ff @(posedge clk) begin
if (!rst_n) begin
seq_state <= SQ_IDLE;
seq_busy <= 1'b0;
seq_done_flag <= 1'b0;
op_start_copy <= 1'b0;
op_start_write <= 1'b0;
op_start_read <= 1'b0;
op_src_addr <= '0;
op_dst_addr <= '0;
op_bytes <= '0;
op_write_addr <= '0;
op_write_data <= '0;
op_read_addr <= '0;
seq_group_idx <= '0;
seq_load_idx <= '0;
seq_store_idx <= '0;
seq_compute_bank <= 1'b0;
seq_status_data <= '0;
end else begin
op_start_copy <= 1'b0;
op_start_write <= 1'b0;
op_start_read <= 1'b0;
if (seq_done_clear_req)
seq_done_flag <= 1'b0;
case (seq_state)
SQ_IDLE: begin
seq_busy <= 1'b0;
if (seq_start_req && (seq_groups != 32'd0)) begin
seq_busy <= 1'b1;
seq_done_flag <= 1'b0;
seq_group_idx <= 32'd0;
seq_load_idx <= 32'd0;
seq_store_idx <= 32'd0;
seq_compute_bank <= 1'b0;
op_write_addr <= seq_mat_base + MAT_CRC_OFF;
op_write_data <= 32'd0;
op_start_write <= 1'b1;
seq_state <= SQ_CLEAR_CRC;
end
end
SQ_CLEAR_CRC: begin
if (op_done) begin
op_src_addr <= seq_src_base;
op_dst_addr <= seq_mat_base + MAT_B0_AB_OFF;
op_bytes <= MAT_AB_BYTES;
op_start_copy <= 1'b1;
seq_load_idx <= 32'd1;
seq_state <= SQ_LOAD0;
end
end
SQ_LOAD0: begin
if (op_done) begin
op_write_addr <= seq_mat_base + MAT_CTRL_OFF;
op_write_data <= MAT_CTRL_START_B0;
op_start_write <= 1'b1;
seq_state <= SQ_START0;
end
end
SQ_START0: begin
if (op_done) begin
if (seq_load_idx < seq_groups) begin
op_src_addr <= seq_src_base + (seq_load_idx * seq_src_stride);
op_dst_addr <= seq_mat_base + MAT_B1_AB_OFF;
op_bytes <= MAT_AB_BYTES;
op_start_copy <= 1'b1;
seq_load_idx <= seq_load_idx + 32'd1;
seq_state <= SQ_LOAD1;
end else begin
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
op_start_read <= 1'b1;
seq_state <= SQ_WAIT_DONE;
end
end
end
SQ_LOAD1: begin
if (op_done) begin
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
op_start_read <= 1'b1;
seq_state <= SQ_WAIT_DONE;
end
end
SQ_WAIT_DONE: begin
if (op_done) begin
seq_status_data <= op_read_data;
if ((op_read_data & MAT_STATUS_DONE_BIT) != 32'd0) begin
seq_store_idx <= seq_group_idx;
if ((seq_group_idx + 32'd1) >= seq_groups) begin
op_src_addr <= seq_mat_base + (seq_compute_bank ? MAT_B1_C_OFF : MAT_B0_C_OFF);
op_dst_addr <= seq_dst_base + (seq_group_idx * seq_dst_stride);
op_bytes <= MAT_C_BYTES;
op_start_copy <= 1'b1;
seq_state <= SQ_STORE;
end else begin
op_write_addr <= seq_mat_base + MAT_STATUS_OFF;
op_write_data <= MAT_STATUS_DONE_CLR;
op_start_write <= 1'b1;
seq_state <= SQ_CLEAR_DONE;
end
end else begin
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
op_start_read <= 1'b1;
end
end
end
SQ_STORE: begin
if (op_done) begin
if ((seq_group_idx + 32'd1) >= seq_groups) begin
seq_state <= SQ_DONE;
end else if (seq_has_more_load) begin
op_src_addr <= seq_src_addr_for_load;
op_dst_addr <= seq_dst_addr_for_load;
op_bytes <= MAT_AB_BYTES;
op_start_copy <= 1'b1;
seq_load_idx <= seq_load_idx + 32'd1;
seq_state <= SQ_LOAD_NEXT;
end else begin
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
op_start_read <= 1'b1;
seq_state <= SQ_WAIT_DONE;
end
end
end
SQ_CLEAR_DONE: begin
if (op_done) begin
seq_group_idx <= seq_group_idx + 32'd1;
seq_compute_bank <= ~seq_compute_bank;
op_write_addr <= seq_mat_base + MAT_CTRL_OFF;
op_write_data <= (~seq_compute_bank) ? MAT_CTRL_START_B1 : MAT_CTRL_START_B0;
op_start_write <= 1'b1;
seq_state <= SQ_START_NEXT;
end
end
SQ_START_NEXT: begin
if (op_done) begin
op_src_addr <= seq_src_addr_for_store;
op_dst_addr <= seq_dst_addr_for_store;
op_bytes <= MAT_C_BYTES;
op_start_copy <= 1'b1;
seq_state <= SQ_STORE;
end
end
SQ_LOAD_NEXT: begin
if (op_done) begin
op_read_addr <= seq_mat_base + MAT_STATUS_OFF;
op_start_read <= 1'b1;
seq_state <= SQ_WAIT_DONE;
end
end
SQ_DONE: begin
seq_busy <= 1'b0;
seq_done_flag <= 1'b1;
seq_state <= SQ_IDLE;
end
default: seq_state <= SQ_IDLE;
endcase
end
end
// ========================================== // ==========================================
// Endpoint transfer engine (32-bit only in this SoC integration) // Endpoint transfer engine (32-bit only in this SoC integration)
// ========================================== // ==========================================

256
sdk/axi_ram.mif
View File

@@ -52,7 +52,7 @@
00000011100000000110000000001101 00000011100000000110000000001101
00000100000000000000000110101100 00000100000000000000000110101100
00011100000000000000000000101100 00011100000000000000000000101100
00000010101110101111000110001100 00000010101100111111000110001100
00011100000000000001000000001101 00011100000000000001000000001101
00000010101111001001000110101101 00000010101111001001000110101101
00011100000000000001000000001110 00011100000000000001000000001110
@@ -94,7 +94,7 @@
00000000000101010000000000001110 00000000000101010000000000001110
00101001000000000001000110101110 00101001000000000001000110101110
00011100000000000000000000001100 00011100000000000000000000001100
00000010100100110001000110001100 00000010100011000001000110001100
00000100000000000011000000101100 00000100000000000011000000101100
00000011100000000000010000001101 00000011100000000000010000001101
00000100000000010001000000101101 00000100000000010001000000101101
@@ -142,17 +142,13 @@
00000000000000000000000000000000 00000000000000000000000000000000
00000000000000000000000000000000 00000000000000000000000000000000
00000000000000000000000000000000 00000000000000000000000000000000
00000010101111110100000001100011 00000010101111111000000001100011
00000010100000000000100000001101
00000010100000000000010000001110
00010101011111100000000000010000 00010101011111100000000000010000
00101001100000000110000001101110 00101001100000000111000001100001
00101001100000000111000001101101 00101001100000000110000001110111
00101001100000001011000001100001
00101001100000001010000001110111
00000010100000010011010000001100 00000010100000010011010000001100
00011100000000000001000000001111 00011100000000000001000000001111
00000010101101100111000111101111 00000010101101101011000111101111
00000011100000000001011000001101 00000011100000000001011000001101
00101001100000000011000001101100 00101001100000000011000001101100
00101010000000000011000001101110 00101010000000000011000001101110
@@ -164,134 +160,36 @@
00101001000000000000001000001110 00101001000000000000001000001110
00101000000000000000000111101100 00101000000000000000000111101100
01011111111111111101110110000000 01011111111111111101110110000000
01010100000001111110100000000000 01010100000001100011100000000000
00010101011111101010000000001100
00000011100000000001000110001110
00000010100000000001100000001111
00101001100000000000000111001111
00000011100000000100000110001101
00101001100000000000000110100000
00010101011111100110000000001100 00010101011111100110000000001100
00010100001110001000000000010010 00000011100010000000000110001100
00101001100000000000000110010010 00000010100000000000100000001110
00101001100000000001000110000000 00101001100000000001000110001110
00010100001110001000000000001101
00101001100000000010000110001101
00000011100101000000000110110011
00000010100000000010100000010010
00101001100000000011000110010011
00101001100000000100000110010010
00010100001111101010000000010001
00000010100000100000000000010000 00000010100000100000000000010000
00101001100000000010000110010000 00101001100000000101000110010001
00000010100000000000010000001111 00101001100000000110000110010000
00101001100000000110000110001111 00000010100000110000000000001111
00000011101111110100010000001110 00000010100000000000010000001110
00101001100000000111000110000000 00101001100000000111000110001111
00000000000101010000000010010111 00000000000101010000000010010111
00010101011111100110000000010001 00000000000101010000000110001101
00101001100000000100000110001110 00101001100000000000000110001110
00101000100000000101001000101100
00000011010000000000100110001100
01011011111111111111100110000000
00000010100000000000100000001100
00101001100000000101001000101100
00101000100000000110000001100101
00010100001110001000000000010100
00010101011111101010000000010010
00101001100000000000001001000101
00000011100000100000001010001110
00101001100000000000001000101110
00101001100000000001001000100000
00000010100000100000000000000100
00101001100000000010001000100100
00000010100000000000010000001100
00000011101111110100010000000111
00010100001110001001001110001001
00010101011111101010000000001010
00010101011111100110000000001101
00010100000000000000000000100110
00101001100000000110001000101100
00101001100000000111001000101100
00101001100000000100001000100111
00000011100100000000000100101001
00000011100001000000001010010100
00000010100000000000010000010000
00000010100000000000010000010011
00000011100000000001000101001111
00000011100011100001110011000110
00000010100000000000100000010010
00000010100000000000010000000101
00000011100000010000000110101110
00000010100000110000000000001000
00000011101111110100010000000100
00000010100000100000000000000111
00000011110000000000011000010001
00101000100000000000000111101100
00000011010000000000100110001100
01011011111111111111100110000000
00101001100000000000000111110010
01011000000000000001101001100101
00000011010000000000000000000000 00000011010000000000000000000000
00101000100000000101000111001100 00101000100000000001000110101100
00000011010000000000100110001100
01011011111111111111100110000000
00101001100000000101000111010010
00101001100000000000000111000000
00101001100000000001000111001001
00101001100000000010000111001000
00101001100000000110000111010010
00101001100000000111000111010001
00101001100000000100000111000100
00101000100000000101000110101100
00000011010000000000100110001100
01011011111111111111100110000000
00000010100000001000000001101100
00000000010000001000101000010000
00101001100000000101000110110010
00000000000100000100000110010000
00101000101111111110001000001100
00101001100000000000000101001100
01011000000000000011001001100110
00101001100000000000000110110100
00101001100000000001000110100000
00101001100000000010000110100111
00101001100000000110000110100101
00101001100000000111000110110001
00000010100000000000011001110011
00000010100000100000001010010100
00000010100000110000000100101001
00000000000101010000001000110000
00101001100000000100000110100100
01010011111111110110111111111111
00010101011111101010000000001110
00000011100000000001000111001110
00101000100000000000000111001100
00000011010000000000100110001100 00000011010000000000100110001100
01011011111111111111100110000000 01011011111111111111100110000000
00000010100000000000100000001100 00000010100000000000100000001100
00010101011111100110000000001101
00101001100000000000000111001100
00000011100000010000000110101101
00101000100000000101000110101100
00000011010000000000100110001100
01011011111111111111100110000000
00000010100000000000100000001111
00101001100000000101000110101111
00010100001110001011000011001100
00101001100000000000000110100000
00000010100000110000000000010010
00000011101001010000000110001100
00101001100000000001000110101100 00101001100000000001000110101100
00101001100000000010000110110010 01010100000001011100110000000000
00000010100000000000010000010001
00000011101111110100010000010000
00010101011111100110000000001110
00101001100000000110000110101111
00101001100000000111000110110001
00000011100000010000000111001110
00101001100000000100000110110000
00101000100000000101000111001100
00000011010000000000100110001100
01011011111111111111100110000000
00000010100000000000100000001100
00101001100000000101000111001100
01010100000001011111010000000000
00011100000000000001000000001110 00011100000000000001000000001110
00000010101011110010000111001110 00000010101101011000000111001110
00101000100000000000000111010010 00101000100000000000000111010010
00010101011111101010000000001101 00010101011111101010000000001101
00010100000000000001111010001100 00010100000000000001111010001100
@@ -299,16 +197,46 @@
00010101011111100000000000010000 00010101011111100000000000010000
00000011100010010000000110001110 00000011100010010000000110001110
00000000000100010101110010000100 00000000000100010101110010000100
00000000001000010011101001010001 00000000001000010011101001010011
01011100000000000000100111000000 01011100000000000000100111000000
00000000001010100000000000000111 00000000001010100000000000000111
00101000100000000000000110110100 00101000100000000000000110110001
00000000001000010100010010010010 00000000001000010100110010010010
01011100000000000000101000100000 01011100000000000000101001100000
00000000001010100000000000000111 00000000001010100000000000000111
00000010100000010011010000001100 00000010100000010011010000001100
00011100000000000001000000001111 00011100000000000001000000001111
00000010101011001111000111101111 00000010101100110101000111101111
00000011100000000001011000001101
00101001100000000011000001101100
00101010000000000011000001101110
00000010100000000000010111101111
00101010000000000000000110101100
00000011010000001000000110001100
01011011111111111111100110000000
00101001000000000000001000001110
00101000000000000000000111101100
01011111111111111110000110000000
00010101011111100000000000010000
00000010100000000111000000001110
00000010100000000010010000010100
00000010101111111111000000010011
00000011100000000001011000001101
00000000000101111011101000101100
00000011010000000011110110001100
00000010100000010101110110001111
01101000000000000000101010001100
00000010100000001100000110001111
00101010000000000000000110101100
00000011010000001000000110001100
01011011111111111111100110000000
00101001000000000000001000001111
00000010101111111111000111001110
01011111111111111101100111010011
00010101011111100000000000010000
00000010100000000011010000001100
00011100000000000001000000001111
00000010101100011011000111101111
00000011100000000001011000001101 00000011100000000001011000001101
00101001100000000011000001101100 00101001100000000011000001101100
00101010000000000011000001101110 00101010000000000011000001101110
@@ -320,57 +248,28 @@
00101001000000000000001000001110 00101001000000000000001000001110
00101000000000000000000111101100 00101000000000000000000111101100
01011111111111111101110110000000 01011111111111111101110110000000
00010101011111100000000000010000 01010100000000101001100000000000
00000010100000000111000000001110
00000010100000000010010000010011
00000010101111111111000000010001
00000011100000000001011000001101
00000000000101111011101010001100
00000011010000000011110110001100
00000010100000010101110110001111
01101000000000000000101001101100
00000010100000001100000110001111
00101010000000000000000110101100
00000011010000001000000110001100
01011011111111111111100110000000
00101001000000000000001000001111
00000010101111111111000111001110
01011111111111111101100111010001
00010101011111100000000000010000 00010101011111100000000000010000
00000010100000000011010000001100 00000010100000000011010000001100
00011100000000000001000000001111 00011100000000000001000000001111
00000010101010110100000111101111 00000010101100010000000111101111
00000011100000000001011000001101 00000011100000000001011000001101
00101001100000000011000001101100 00101001100000000011000001101100
00101010000000000011000001101110 00101010000000000011000001101110
00000010100000000000010111101111 00000010100000000000010111101111
00000011010000000000000000000000
00101010000000000000000110101100 00101010000000000000000110101100
00000011010000001000000110001100 00000011010000001000000110001100
01011011111111111111100110000000 01011011111111111111100110000000
00101001000000000000001000001110 00101001000000000000001000001110
00101000000000000000000111101100 00101000000000000000000111101100
01011111111111111110000110000000 01011111111111111101110110000000
01010100000000101100000000000000
00010101011111100000000000010000
00000010100000000011010000001100
00011100000000000001000000001111
00000010101010101010000111101111
00000011100000000001011000001101
00101001100000000011000001101100
00101010000000000011000001101110
00000010100000000000010111101111
00101010000000000000000110101100
00000011010000001000000110001100
01011011111111111111100110000000
00101001000000000000001000001110
00101000000000000000000111101100
01011111111111111110000110000000
00000000000101010000001001000100 00000000000101010000001001000100
01010100000000101000000000000000 01010100000000100101010000000000
00010101011111100000000000010000 00010101011111100000000000010000
00000010100000000011010000001100 00000010100000000011010000001100
00011100000000000001000000001111 00011100000000000001000000001111
00000010101010011111000111101111 00000010101100000100000111101111
00000011100000000001011000001101 00000011100000000001011000001101
00101001100000000011000001101100 00101001100000000011000001101100
00101010000000000011000001101110 00101010000000000011000001101110
@@ -381,23 +280,12 @@
00101001000000000000001000001110 00101001000000000000001000001110
00101000000000000000000111101100 00101000000000000000000111101100
01011111111111111110000110000000 01011111111111111110000110000000
00101000100000001011000001100001 00101000100000000111000001100001
00101000100000001010000001110111 00101000100000000110000001110111
00000000000101010000000000000100 00000000000101010000000000000100
00000010100000001100000001100011 00000010100000001000000001100011
01001100000000000000000000100000 01001100000000000000000000100000
00000000000000000000000000000000 00000011010000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000000000000000000000000000000
00000010101111100001000001100011 00000010101111100001000001100011
00101001100000011110000001100001 00101001100000011110000001100001
00101001100000011101000001100010 00101001100000011101000001100010

17
sdk/software/examples/asm/dma.h
View File

@@ -18,6 +18,19 @@ typedef struct {
#define CDMA_MUX_BASE 0xbf300000 #define CDMA_MUX_BASE 0xbf300000
#define DMA_CH(i) ((volatile dma_ch_regs_t*)(CDMA_MUX_BASE + (i) * 0x40)) #define DMA_CH(i) ((volatile dma_ch_regs_t*)(CDMA_MUX_BASE + (i) * 0x40))
typedef struct {
volatile u32 CTRL; // 0x200: bit0 start
volatile u32 STATUS; // 0x204: bit0 busy, bit1 done (W1C)
volatile u32 SRC_BASE; // 0x208
volatile u32 DST_BASE; // 0x20C
volatile u32 GROUPS; // 0x210
volatile u32 MAT_BASE; // 0x214
volatile u32 SRC_STRIDE; // 0x218
volatile u32 DST_STRIDE; // 0x21C
} dma_seq_regs_t;
#define DMA_SEQ ((volatile dma_seq_regs_t*)(CDMA_MUX_BASE + 0x200))
// CTRL 寄存器位定义 // CTRL 寄存器位定义
#define CTRL_START_BIT (1 << 0) #define CTRL_START_BIT (1 << 0)
#define CTRL_BURST_SIZE(x) (((x) & 0x7) << 3) // Bits [5:3]: AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B) #define CTRL_BURST_SIZE(x) (((x) & 0x7) << 3) // Bits [5:3]: AXI AxSIZE (0=1B, 1=2B, 2=4B, 3=8B)
@@ -33,5 +46,9 @@ typedef struct {
#define STATUS_BUSY_BIT (1 << 0) #define STATUS_BUSY_BIT (1 << 0)
#define STATUS_DONE_BIT (1 << 1) #define STATUS_DONE_BIT (1 << 1)
#define SEQ_START_BIT (1 << 0)
#define SEQ_BUSY_BIT (1 << 0)
#define SEQ_DONE_BIT (1 << 1)
void dma_start_transfer(int ch_id, u32 src, u32 dst, u32 bytes, u32 tag); void dma_start_transfer(int ch_id, u32 src, u32 dst, u32 bytes, u32 tag);
void dma_wait_polling(int ch_id); void dma_wait_polling(int ch_id);

129
sdk/software/examples/asm/user-sample.c
View File

@@ -82,6 +82,7 @@ unsigned long CORE_CLOCKS_PER_SEC = 33000000L;
#define MATMUL_CTRL (MATMUL_BASE_UNCACHED + 0x000) #define MATMUL_CTRL (MATMUL_BASE_UNCACHED + 0x000)
#define MATMUL_STATUS (MATMUL_BASE_UNCACHED + 0x004) #define MATMUL_STATUS (MATMUL_BASE_UNCACHED + 0x004)
#define MATMUL_BASE_PHYS_FOR_DMA MATMUL_BASE_PHYS
#define CTRL_START_B0 0x1 #define CTRL_START_B0 0x1
#define CTRL_START_B1 0x2 #define CTRL_START_B1 0x2
@@ -100,132 +101,26 @@ int main(void)
if (MATMUL_GROUP_NUM <= 0) return 0; if (MATMUL_GROUP_NUM <= 0) return 0;
/* ── Hot-path local volatile pointers (one-time addr calc) ──────── */ /* ── Hot-path local volatile pointers (one-time addr calc) ──────── */
volatile unsigned int *mat_status = (volatile unsigned int *)MATMUL_STATUS;
volatile unsigned int *mat_ctrl = (volatile unsigned int *)MATMUL_CTRL;
volatile unsigned int *mat_crc = (volatile unsigned int *)MATMUL_CRC; volatile unsigned int *mat_crc = (volatile unsigned int *)MATMUL_CRC;
volatile dma_ch_regs_t *ch0 = DMA_CH(0); volatile dma_seq_regs_t *seq = DMA_SEQ;
volatile dma_ch_regs_t *ch1 = DMA_CH(1);
unsigned int crc; unsigned int crc;
unsigned int cycle_start, cycle_end, cycles, time_us; unsigned int cycle_start, cycle_end, cycles, time_us;
int g, bank;
unsigned int ctrl_start[2] = {CTRL_START_B0, CTRL_START_B1};
/* ── Start marker (must be first output) ──────────────────── */ /* ── Start marker (must be first output) ──────────────────── */
uart_puts("MATMUL_START\r\n"); uart_puts("MATMUL_START\r\n");
cycle_start = get_clock_count(); cycle_start = get_clock_count();
// Reset IP state seq->STATUS = SEQ_DONE_BIT;
*mat_status = STATUS_DONE | STATUS_ERROR; seq->SRC_BASE = EXTRAM_PHYS_AB(0);
*mat_crc = 0; seq->DST_BASE = EXTRAM_PHYS_C(0);
seq->GROUPS = MATMUL_GROUP_NUM;
/* ═══════════════════════════════════════════════════════════════ seq->MAT_BASE = MATMUL_BASE_PHYS_FOR_DMA;
* PROLOGUE: Load group 0 → Bank 0, start IP, pre-fire group 1 seq->SRC_STRIDE = 128;
* ═══════════════════════════════════════════════════════════════ */ seq->DST_STRIDE = 192;
seq->CTRL = SEQ_START_BIT;
// DMA Ch0: Load group 0 A+B (128 bytes) → IP Bank 0 while (!(seq->STATUS & SEQ_DONE_BIT)) {}
ch0->SRC_ADDR = EXTRAM_PHYS_AB(0); seq->STATUS = SEQ_DONE_BIT;
ch0->DST_ADDR = 0;
ch0->LENGTH = 128;
ch0->MODE = DMA_MODE_MEM2EP;
ch0->EP_CFG = DMA_EP_CFG(MATMUL_EP_ID, 0, 0);
ch0->CTRL = DMA_CTRL_VAL;
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
ch0->STATUS = STATUS_DONE_BIT;
// Start IP Bank 0 (group 0)
*mat_ctrl = ctrl_start[0];
// Pre-fire: DMA Ch0 load group 1 → Bank 1 (overlaps with IP compute)
if (MATMUL_GROUP_NUM > 1) {
ch0->SRC_ADDR = EXTRAM_PHYS_AB(1);
ch0->DST_ADDR = 0;
ch0->LENGTH = 128;
ch0->MODE = DMA_MODE_MEM2EP;
ch0->EP_CFG = DMA_EP_CFG(MATMUL_EP_ID, 1, 0);
ch0->CTRL = DMA_CTRL_VAL;
}
/* ═══════════════════════════════════════════════════════════════
* MAIN LOOP: Fully-pipelined — pre-fire DMA load at loop tail
*
* At iteration g:
* IP is computing group (g-1) on prev_bank
* Ch0 loaded group g to cur_bank (pre-fired in iteration g-1)
*
* Pipeline:
* Ch0: | load g+1 | | load g+2 |
* Ch1: | store g-1 | | store g |
* IP : | compute g | | compute g+1 |
* ═══════════════════════════════════════════════════════════════ */
bank = 1; // cur_bank for iteration g=1; prev_bank = bank ^ 1
for (g = 1; g < MATMUL_GROUP_NUM; g++) {
int cur_bank = bank;
int prev_bank = bank ^ 1;
// 1. Wait for previous IP computation (group g-1 on prev_bank)
while (!(*mat_status & STATUS_DONE)) {}
*mat_status = STATUS_DONE;
// 2. Ensure previous DMA Ch1 store is done (group g-2)
if (g > 1) {
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
ch1->STATUS = STATUS_DONE_BIT;
}
// 3. Start DMA Ch1 store: group (g-1) result → ExtRAM
ch1->SRC_ADDR = 0;
ch1->DST_ADDR = EXTRAM_PHYS_C(g - 1);
ch1->LENGTH = 192;
ch1->MODE = DMA_MODE_EP2MEM;
ch1->EP_CFG = DMA_EP_CFG(MATMUL_EP_ID, prev_bank, 0);
ch1->CTRL = DMA_CTRL_VAL;
// 4. Wait for current DMA Ch0 load (group g → cur_bank, pre-fired in iteration g-1)
while (!(ch0->STATUS & STATUS_DONE_BIT)) {}
ch0->STATUS = STATUS_DONE_BIT;
// 5. Start IP computation for current group on cur_bank
*mat_ctrl = ctrl_start[cur_bank];
// ★ 6. PRE-FIRE: DMA Ch0 load next group (g+1) → next_bank
// This runs CONCURRENTLY with IP computation started in step 5!
if (g < MATMUL_GROUP_NUM - 1) {
int next_bank = bank ^ 1;
ch0->SRC_ADDR = EXTRAM_PHYS_AB(g + 1);
ch0->DST_ADDR = 0;
ch0->LENGTH = 128;
ch0->MODE = DMA_MODE_MEM2EP;
ch0->EP_CFG = DMA_EP_CFG(MATMUL_EP_ID, next_bank, 0);
ch0->CTRL = DMA_CTRL_VAL;
}
// Toggle bank for next iteration
bank ^= 1;
}
/* ═══════════════════════════════════════════════════════════════
* EPILOGUE: Flush the final group
* ═══════════════════════════════════════════════════════════════ */
// Wait for last IP computation
while (!(*mat_status & STATUS_DONE)) {}
*mat_status = STATUS_DONE;
// Wait for previous DMA store and start/complete final store
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
ch1->STATUS = STATUS_DONE_BIT;
ch1->SRC_ADDR = 0;
ch1->DST_ADDR = EXTRAM_PHYS_C(MATMUL_GROUP_NUM - 1);
ch1->LENGTH = 192;
ch1->MODE = DMA_MODE_EP2MEM;
ch1->EP_CFG = DMA_EP_CFG(MATMUL_EP_ID, (MATMUL_GROUP_NUM - 1) % 2, 0);
ch1->CTRL = DMA_CTRL_VAL;
while (!(ch1->STATUS & STATUS_DONE_BIT)) {}
ch1->STATUS = STATUS_DONE_BIT;
cycle_end = get_clock_count(); cycle_end = get_clock_count();
cycles = cycle_end - cycle_start; cycles = cycle_end - cycle_start;
time_us = cycles / (CORE_CLOCKS_PER_SEC / 1000000UL); time_us = cycles / (CORE_CLOCKS_PER_SEC / 1000000UL);